moved bonfire extensions from ./forks/ to ./apps/ and treat them as an umbrella app in dev env

This commit is contained in:
Mayel de Borniol 2022-11-29 14:46:15 +13:00
parent 56f481c7c9
commit dd7a237e10
17 changed files with 2985 additions and 90 deletions

2
.gitignore vendored
View file

@ -60,6 +60,7 @@ deps.path
/libs/
# /forks/
/forks/*/
/apps/*/
deps.path*
# Dev artifacts
@ -100,3 +101,4 @@ tx
# archeometer
archeometer_bonfire.db
reports/
docs/xref_graph.svg

1
.is_umbrella.exs Normal file
View file

@ -0,0 +1 @@
// tells Bonfire extensions that they're running in a multi-repo umbrella

View file

@ -5,23 +5,21 @@ FROM rust:1.52 AS messctl_build
ARG FLAVOUR
ARG FLAVOUR_PATH
ENV FORKS=./forks
# build deps
# RUN apk update && apk add git rust cargo
# fetch messctl
RUN git clone https://github.com/bonfire-networks/messctl $FORKS/messctl/origin 2> /dev/null || (cd $FORKS/messctl/origin && git pull)
RUN cd $FORKS/messctl/origin && git checkout 8f53c86687ba2bd262471c6e8d9490ed00bf1306
RUN git clone https://github.com/bonfire-networks/messctl forks/messctl/origin 2> /dev/null || (cd forks/messctl/origin && git pull)
RUN cd forks/messctl/origin && git checkout 8f53c86687ba2bd262471c6e8d9490ed00bf1306
# FIXME: try using latest version of messctl
# compile messctl
RUN cd $FORKS/messctl && cp -r origin/* . && cargo build --release && cargo install --path . --verbose
RUN cd forks/messctl && cp -r origin/* . && cargo build --release && cargo install --path . --verbose
FROM elixir:1.13-alpine
ENV HOME=/opt/app/ TERM=xterm USER=docker FORKS=./forks
ENV HOME=/opt/app/ TERM=xterm USER=docker
WORKDIR $HOME
# dev tools
@ -50,7 +48,7 @@ RUN apk add git rust cargo
RUN apk add imagemagick vips-tools
# install the dependency manager
COPY --from=messctl_build $FORKS/messctl/target/release/messctl /bin/
COPY --from=messctl_build forks/messctl/target/release/messctl /bin/
# JS package manager & builders
# RUN npm install -g pnpm esbuild postcss

View file

@ -63,7 +63,7 @@ COPY data/current_flavour/config/ ./config/
# Optionally include local forks
ARG FORKS_TO_COPY_PATH
RUN if [ "$FORKS_TO_COPY_PATH" = "data/null" ] ; then rm ./config/deps.path ; else echo "Include locally forked extensions." ; fi
COPY ${FORKS_TO_COPY_PATH} ./forks/
COPY ${FORKS_TO_COPY_PATH} ./${FORKS_TO_COPY_PATH}
# Update Bonfire extensions to latest git version (mostly useful in CI, and temporary: eventually we want to rely on version numbers and lockfile)
# RUN mix do bonfire.deps.update
@ -104,10 +104,11 @@ COPY assets assets
# include an archive of the source code
COPY LICENSE ./
COPY docs/*.md ./docs/
RUN mkdir -p apps/
RUN mkdir -p forks/
RUN mkdir -p priv/static/
COPY priv/extras/ priv/extras/
RUN tar --exclude=*.env --exclude=.git --exclude=assets/node_modules --exclude=assets/static/data -czvf priv/static/source.tar.gz lib deps forks assets config docs priv/repo priv/extras mix.exs mix.lock LICENSE
RUN tar --exclude=*.env --exclude=.git --exclude=assets/node_modules --exclude=assets/static/data -czvf priv/static/source.tar.gz lib deps apps forks assets config docs priv/repo priv/extras mix.exs mix.lock LICENSE
# prepare static assets
COPY data/current_flavour/config/deps_hooks.js data/current_flavour/config/deps_hooks.js

View file

@ -7,7 +7,7 @@ DEPS=${1}
for dep in $DEPS ; do
echo "Install JS deps from extension '$dep' with args '$2'"
if cd "forks/$dep/assets" 2>/dev/null ; then
if cd "apps/$dep/assets" 2>/dev/null ; then
yarn $2
cd ../../../
fi

View file

@ -169,7 +169,7 @@ You can also directly call some functions in the code from the command line, for
There is a `justfile` with relevant commands (make sure set the `MIX_ENV=prod` env variable):
- `just rel-build-release` which builds the docker image of the latest release
- `just rel-build` which builds the docker image, including local changes to any cloned extensions in `./forks/`
- `just rel-build` which builds the docker image, including local changes to any cloned extensions in `./apps/` or `./forks/`
- `just rel-tag` adds the "latest" tag to your last build, so that it will be used when running
Once you've built and tagged your image, you may need to update the `image` name in `docker-compose.release.release.yml` to match (either your local image name if running on the same machine you used for the build, or a remote image on Docker Hub if you pushed it) and then follow the same steps as for option A1.

View file

@ -187,10 +187,12 @@ The code is somewhat documented inline. You can generate HTML docs (using `Exdoc
- messctl is a little utility for programmatically updating the .deps files from which the final elixir dependencies list is compiled by the mess script. The only use of it is in the dep-\* tasks of the Makefile. It is used by some of the project developers and the build does not rely on it.
- `./forks/` is used to hack on local copies of dependencies. You can clone a dependency from its git repo (like a bonfire extension) and use the local version during development, eg: `just dep.clone.local bonfire_me https://github.com/bonfire-networks/bonfire_me`
- `./apps/` and `./forks/` is used to hack on local copies of dependencies. You can clone a dependency from its git repo (like a bonfire extension) and use the local version during development, eg: `just dep-clone-local bonfire_me https://github.com/bonfire-networks/bonfire_me`
- You can migrate the DB when the app is running (also runs automatically on startup): `EctoSparkles.Migrator.migrate`
- You can generate a dependency graph using `just xref-graph` which will generate a DOT file at `docs/` (if Graphviz is installed it will also generate an SVG visualisation using `dot`).
### Usage under Windows (WSL, MSYS or CYGWIN)
By default, the `justfile` requires symlinks, which can be enabled with the help of [this link](https://stackoverflow.com/a/59761201).

2817
docs/xref_graph.dot Normal file

File diff suppressed because it is too large Load diff

View file

@ -1,7 +1,7 @@
earmark = "~> 1.4.28" # handle markdown
# earmark_parser = "~> 1.4.25" # parse markdown
# Web
# livebook = "~> 0.5.2"
# livebook = "~> 0.7.2"
surface = "~> 0.9.0"
phoenix_live_dashboard = "~> 0.7.0"
plug_cowboy = "~> 2.5.2"

View file

@ -1,7 +1,7 @@
earmark = "~> 1.4.28" # handle markdown
# earmark_parser = "~> 1.4.25" # parse markdown
# Web
# livebook = "~> 0.5.2"
# livebook = "~> 0.7.2"
surface = "~> 0.9.0"
phoenix = "~> 1.7.0-rc.0" # note: usually we should let surface specify the version of both phoenix and liveview
phoenix_view = "~> 2.0" # should match phx/lv - dunno why isn't upgraded otherwise

View file

@ -1,7 +1,7 @@
earmark = "~> 1.4.28" # handle markdown
earmark_parser = "~> 1.4" # parse markdown
# Web
# livebook = "~> 0.5.2"
# livebook = "~> 0.7.2"
# surface = "~> 0.9.0"
phoenix_live_dashboard = "~> 0.7.0"
plug_cowboy = "~> 2.5.2"

View file

@ -1,7 +1,7 @@
earmark = "~> 1.4.28" # handle markdown
earmark_parser = "~> 1.4" # parse markdown
# Web
# livebook = "~> 0.5.2"
# livebook = "~> 0.7.2"
surface = "~> 0.9.0"
phoenix_live_dashboard = "~> 0.7.0"
plug_cowboy = "~> 2.5.2"

View file

@ -20,7 +20,7 @@ APP_DOCKER_IMAGE := env_var_or_default('APP_DOCKER_IMAGE', "bonfirenetworks/bonf
DB_DOCKER_IMAGE := if arch() == "aarch64" { "ghcr.io/baosystems/postgis:12-3.3" } else { env_var_or_default('DB_DOCKER_IMAGE', "postgis/postgis:12-3.3-alpine") }
## Other configs - edit these here if necessary
FORKS_PATH := "forks/"
FORKS_PATH := "apps/"
ORG_NAME := "bonfirenetworks"
APP_NAME := "bonfire"
APP_VSN_EXTRA := "beta"
@ -65,6 +65,7 @@ pre-setup flavour='classic':
@ln -sf ./config/dev/ ./config/test/
@rm .env | true
@ln -sf ./config/$MIX_ENV/.env ./.env
@mkdir -p apps/
@mkdir -p forks/
@mkdir -p data/uploads/
@mkdir -p priv/static/data
@ -219,13 +220,13 @@ update-dep dep:
just mix-remote "deps.update $dep"
./assets/install_extensions.sh $dep
# Pull the latest commits from all ./forks
# Pull the latest commits from all forks
update-forks:
@jungle git fetch || echo "Jungle not available, will fetch one by one instead."
@chmod +x git-publish.sh && find $FORKS_PATH -mindepth 1 -maxdepth 1 -type d -exec ./git-publish.sh {} rebase \;
# TODO: run in parallel? find $FORKS_PATH -mindepth 1 -maxdepth 1 -type d | xargs -P 50 -I '{}' ./git-publish.sh '{}'
# Pull the latest commits from all ./forks
# Pull the latest commits from all forks
update-fork dep:
@chmod +x git-publish.sh && find $FORKS_PATH/$dep -mindepth 0 -maxdepth 0 -type d -exec ./git-publish.sh {} pull \;
@ -322,6 +323,7 @@ pre-push-hooks: pre-contrib-hooks
# just mix changelog
pre-contrib-hooks:
-sed -i '' 's,/apps/,/deps/,' config/deps_hooks.js
-sed -i '' 's,/forks/,/deps/,' config/deps_hooks.js
# Push all changes to the app and extensions in ./forks
@ -346,7 +348,7 @@ contrib-rel-push: contrib-release rel-build-release rel-push
# Count lines of code (requires cloc: `brew install cloc`)
cloc:
cloc lib config forks/*/lib forks/*/test test
cloc lib config apps/*/lib apps/*/test test
# Run the git add command on each fork
git-forks-add: deps-git-fix
@ -373,7 +375,7 @@ deps-git-fix:
@git-merge branch:
git merge --no-ff --no-commit $branch
# Find any git conflicts in ./forks
# Find any git conflicts in forks
@git-conflicts:
find $FORKS_PATH -mindepth 1 -maxdepth 1 -type d -exec echo add {} \; -exec git -C '{}' diff --name-only --diff-filter=U \;
@ -383,7 +385,7 @@ deps-git-fix:
#### TESTING RELATED COMMANDS ####
# Run tests. You can also run only specific tests, eg: `just test forks/bonfire_social/test`
# Run tests. You can also run only specific tests, eg: `just test apps/bonfire_social/test`
test *args='':
@echo "Testing $@..."
MIX_ENV=test just mix test $@
@ -407,10 +409,10 @@ test-watch *args='':
test-interactive *args='':
@MIX_ENV=test just mix test.interactive --stale $@
ap_lib := "forks/activity_pub"
ap_integration := "forks/bonfire_federate_activitypub/test/activity_pub_integration"
ap_boundaries := "forks/bonfire_federate_activitypub/test/ap_boundaries"
ap_ext := "forks/*/test/*federat* forks/*/test/*/*federat* forks/*/test/*/*/*federat*"
ap_lib := "apps/activity_pub"
ap_integration := "apps/bonfire_federate_activitypub/test/activity_pub_integration"
ap_boundaries := "apps/bonfire_federate_activitypub/test/ap_boundaries"
ap_ext := "apps/*/test/*federat* apps/*/test/*/*federat* apps/*/test/*/*/*federat*"
# ap_two := "forks/bonfire_federate_activitypub/test/dance"
test-federation:
@ -452,6 +454,7 @@ rel-config-prepare:
# copy current flavour's config, without using symlinks
rel-prepare: rel-config-prepare
mkdir -p apps/
mkdir -p forks/
mkdir -p data/uploads/
mkdir -p data/null/
@ -459,15 +462,18 @@ rel-prepare: rel-config-prepare
# Build the Docker image (with no caching)
rel-rebuild: rel-init rel-prepare assets-prepare
just rel-build "forks/" --no-cache
just rel-build {{FORKS_PATH}} --no-cache
# Build the Docker image (NOT including changes to local forks)
rel-build-release: rel-init rel-prepare assets-prepare
@echo "Please note that the build will not include any changes in forks/ that haven't been committed and pushed, you may want to run just contrib-release first."
@echo "Please note that the build will not include any changes in forks that haven't been committed and pushed, you may want to run just contrib-release first."
@just rel-build "data/null" --no-cache
# Build the Docker image (including changes to local forks, and using caching)
rel-build FORKS_TO_COPY_PATH="forks/" ARGS="": rel-init rel-prepare assets-prepare
rel-build FORKS_TO_COPY_PATH="" ARGS="":
@rel-build-path {{ if FORKS_TO_COPY_PATH != "" {FORKS_TO_COPY_PATH} else {FORKS_PATH} }} ARGS
rel-build-path FORKS_TO_COPY_PATH ARGS="": rel-init rel-prepare assets-prepare
@echo "Building $APP_NAME with flavour $FLAVOUR for arch {{arch()}}."
@docker build $ARGS --progress=plain \
--build-arg FLAVOUR_PATH=data/current_flavour \
@ -582,10 +588,15 @@ docker-stop-web:
@mix *args='':
just cmd mix $@
# Run a specific mix command, while ignoring any deps cloned into ./forks, eg: `just mix-remote deps.get` or `just mix-remote deps.update pointers`
# Run a specific mix command, while ignoring any deps cloned into forks, eg: `just mix-remote deps.get` or `just mix-remote deps.update pointers`
mix-remote *args='': init
{{ if WITH_DOCKER == "total" { "docker-compose run -e WITH_FORKS=0 web mix $@" } else {"WITH_FORKS=0 mix $@"} }}
xref-dot:
just mix xref graph --format dot --include-siblings
(awk '{if (!a[$0]++ && $1 != "}") print}' apps/*/xref_graph.dot; echo }) > docs/xref_graph.dot
dot -Tsvg docs/xref_graph.dot -o docs/xref_graph.svg
# Run a specific exh command, see https://github.com/rowlandcodes/exhelp
exh *args='':
just cmd "exh -S mix $@"
@ -621,6 +632,7 @@ assets-prepare:
# Workarounds for some issues running migrations
db-pre-migrations:
-touch deps/*/lib/migrations.ex
-touch apps/*/lib/migrations.ex
-touch forks/*/lib/migrations.ex
-touch priv/repo/*

110
lib/mix/mess.exs Normal file
View file

@ -0,0 +1,110 @@
# Copyright (c) 2020 James Laver, mess Contributors
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
if not Code.ensure_loaded?(Mess) do
defmodule Mess do
@sources [path: "deps.path", git: "deps.git", hex: "deps.hex"]
@newline ~r/(?:\r\n|[\r\n])/
@parser ~r/^(?<indent>\s*)((?<package>[a-z_][a-z0-9_]+)\s*=\s*"(?<value>[^"]+)")?(?<post>.*)/
@git_branch ~r/(?<repo>[^#]+)(#(?<branch>.+))?/
def umbrella_path(opts \\ []),
do: opts[:umbrella_path] || if(Mix.env() != :prod, do: "apps/", else: nil)
def deps(sources \\ @sources, extra_deps, opts \\ []),
do: Enum.flat_map(sources, fn {k, v} -> read(v, k) end) |> deps_packages(extra_deps, opts)
defp deps_packages(packages, extra_deps, opts),
do: Enum.flat_map(packages, &dep_spec(&1, opts)) |> deps_uniq(extra_deps, opts)
defp deps_uniq(packages, extra_deps, opts),
do: Enum.uniq_by(packages ++ extra_deps, &elem(&1, 0)) |> maybe_filter_umbrella(opts)
defp maybe_filter_umbrella(deps, opts) do
if opts[:umbrella_root?] do
Enum.reject(deps, fn dep ->
dep_opts = elem(dep, 1)
is_list(dep_opts) and dep_opts[:from_umbrella]
end)
# |> IO.inspect(label: "umbrella_root")
else
if umbrella_path(opts) do
umbrella_deps = read_umbrella("../../config/deps.path", opts)
deps
|> Enum.map(fn dep ->
name = elem(dep, 0)
case umbrella_deps[name] do
nil ->
dep
dep_opts ->
if dep_opts[:from_umbrella] do
{name, in_umbrella: true, override: true}
else
{name, dep_opts |> Keyword.put(:path, "../../#{dep_opts[:path]}")}
end
end
end)
# |> IO.inspect(label: "in_umbrella")
else
deps
end
end
end
defp read_umbrella(path, opts) when is_binary(path) do
if File.exists?(path) do
read(path, :path)
|> Enum.flat_map(&dep_spec(&1, opts))
else
[]
end
end
defp read(path, kind) when is_binary(path), do: have_read(File.read(path), kind)
defp have_read({:error, :enoent}, _kind), do: []
defp have_read({:ok, file}, kind),
do: Enum.map(String.split(file, @newline), &read_line(&1, kind))
defp read_line(line, kind),
do: Map.put(Regex.named_captures(@parser, line), :kind, kind)
defp dep_spec(%{"package" => ""}, _opts), do: []
defp dep_spec(%{"package" => p, "value" => v, :kind => :hex}, _opts),
do: pkg(p, v, override: true)
defp dep_spec(%{"package" => p, "value" => v, :kind => :path}, opts) do
umbrella_path = umbrella_path(opts)
if umbrella_path && String.starts_with?(v, umbrella_path) do
pkg(p, from_umbrella: true, override: true, path: v)
else
pkg(p, path: v, override: true)
end
end
defp dep_spec(%{"package" => p, "value" => v, :kind => :git}, _opts), do: git(v, p)
defp git(line, p) when is_binary(line),
do: git(Regex.named_captures(@git_branch, line), p)
defp git(%{"branch" => "", "repo" => r}, p),
do: pkg(p, git: r, override: true)
defp git(%{"branch" => b, "repo" => r}, p),
do: pkg(p, git: r, branch: b, override: true)
defp pkg(name, opts), do: [{String.to_atom(name), opts}]
defp pkg(name, version, opts), do: [{String.to_atom(name), version, opts}]
end
end

View file

@ -1,54 +0,0 @@
# Copyright (c) 2020 James Laver, mess Contributors
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
if not Code.ensure_loaded?(Mess) do
defmodule Mess do
@sources [path: "deps.path", git: "deps.git", hex: "deps.hex"]
@newline ~r/(?:\r\n|[\r\n])/
@parser ~r/^(?<indent>\s*)((?<package>[a-z_][a-z0-9_]+)\s*=\s*"(?<value>[^"]+)")?(?<post>.*)/
@git_branch ~r/(?<repo>[^#]+)(#(?<branch>.+))?/
def deps(sources \\ @sources, deps),
do: deps(Enum.flat_map(sources, fn {k, v} -> read(v, k) end), deps, :deps)
defp deps(packages, deps, :deps),
do: deps(Enum.flat_map(packages, &dep_spec/1), deps, :uniq)
defp deps(packages, deps, :uniq),
do: Enum.uniq_by(packages ++ deps, &elem(&1, 0))
defp read(path, kind) when is_binary(path), do: read(File.read(path), kind)
defp read({:error, :enoent}, _kind), do: []
defp read({:ok, file}, kind),
do: Enum.map(String.split(file, @newline), &read_line(&1, kind))
defp read_line(line, kind),
do: Map.put(Regex.named_captures(@parser, line), :kind, kind)
defp dep_spec(%{"package" => ""}), do: []
defp dep_spec(%{"package" => p, "value" => v, :kind => :hex}),
do: pkg(p, v, override: true)
defp dep_spec(%{"package" => p, "value" => v, :kind => :path}),
do: pkg(p, path: v, override: true)
defp dep_spec(%{"package" => p, "value" => v, :kind => :git}), do: git(v, p)
defp git(line, p) when is_binary(line),
do: git(Regex.named_captures(@git_branch, line), p)
defp git(%{"branch" => "", "repo" => r}, p),
do: pkg(p, git: r, override: true)
defp git(%{"branch" => b, "repo" => r}, p),
do: pkg(p, git: r, branch: b, override: true)
defp pkg(name, opts), do: [{String.to_atom(name), opts}]
defp pkg(name, version, opts), do: [{String.to_atom(name), version, opts}]
end
end

View file

@ -150,6 +150,9 @@ if not Code.ensure_loaded?(Bonfire.Mixer) do
def source_url_pattern("deps/" <> _ = path, line),
do: bonfire_ext_pattern(path, line)
def source_url_pattern("apps/" <> _ = path, line),
do: bonfire_ext_pattern(path, line)
def source_url_pattern("forks/" <> _ = path, line),
do: bonfire_ext_pattern(path, line)

15
mix.exs
View file

@ -1,10 +1,13 @@
Code.eval_file("lib/mix/mess/mess.exs")
Code.eval_file("lib/mix/mess/mixer.ex")
Code.eval_file("lib/mix/mess.exs")
Code.eval_file("lib/mix/mixer.ex")
defmodule Bonfire.MixProject do
use Mix.Project
alias Bonfire.Mixer
@umbrella_path if Mix.env() != :prod, do: "apps/", else: nil
@mess_opts [umbrella_root?: true, umbrella_path: @umbrella_path]
@extra_deps [
## password hashing - builtin vs nif
{:pbkdf2_elixir, "~> 2.0", only: [:dev, :test]},
@ -14,7 +17,7 @@ defmodule Bonfire.MixProject do
{:sentry, "~> 8.0", only: [:dev, :prod]},
## dev conveniences
# {:dbg, "~> 1.0", only: [:dev, :test]},
#
{:phoenix_live_reload, "~> 1.3", only: :dev},
# {:exsync, git: "https://github.com/falood/exsync", only: :dev},
# {:mix_unused, "~> 0.4", only: :dev},
@ -39,9 +42,8 @@ defmodule Bonfire.MixProject do
# tests
{:floki, ">= 0.0.0", only: [:dev, :test]},
{:ex_machina, "~> 2.4", only: :test},
{:mock, "~> 0.3", only: :test},
{:mox, "~> 1.0", only: :test},
# {:mox, "~> 1.0", only: :test},
{:zest, "~> 0.1.0"},
{:grumble, "~> 0.1.3", only: [:test], override: true},
{:mix_test_watch, "~> 1.1", only: :test, runtime: false, override: true},
@ -151,7 +153,7 @@ defmodule Bonfire.MixProject do
localise: ["bonfire"],
localise_self: []
],
deps: Mess.deps(Mixer.mess_sources(@default_flavour), @extra_deps)
deps: Mess.deps(Mixer.mess_sources(@default_flavour), @extra_deps, @mess_opts)
]
def config, do: @config
@ -160,6 +162,7 @@ defmodule Bonfire.MixProject do
def project do
[
app: :bonfire,
apps_path: @umbrella_path,
version: Mixer.version(config()),
elixir: config()[:elixir],
elixirc_options: [debug_info: true, docs: true],