Merge pull request #250 from dalf/weblate_v2

[translations] weblate integration
This commit is contained in:
Alexandre Flament 2021-08-17 10:38:50 +02:00 committed by GitHub
commit 28b350eda6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 352 additions and 99 deletions

View file

@ -91,7 +91,7 @@ jobs:
COMMIT_MESSAGE: build from commit ${{ github.sha }}
babel:
name: Babel
name: Update translations branch
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
needs:
@ -102,32 +102,32 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
with:
persist-credentials: false
fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: weblate & git setup
env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: |
mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot"
- name: Update transations
id: update
continue-on-error: true
run: make V=1 ci.babel.update
- name: Open pull request
if: steps.update.outcome == 'success'
uses: peter-evans/create-pull-request@v3
with:
commit-message: Update translations (pot, po)
committer: searx-bot <noreply@github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
signoff: false
branch: update_translations_pot
delete-branch: true
draft: false
title: 'Update translations (pot, po)'
body: |
Update messages.pot and messages.po files
labels: |
translation
run: |
git restore utils/brand.env
make V=1 weblate.push.translations
dockers:
name: Docker

View file

@ -0,0 +1,56 @@
name: "Update translations"
on:
schedule:
- cron: "05 07 * * 5"
workflow_dispatch:
jobs:
babel:
name: "create PR for additons from weblate"
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: weblate & git setup
env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: |
mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot"
- name: Merge and push transation updates
run: |
make V=1 weblate.translations.commit
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@v3
with:
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
commit-message: Update translations
committer: searxng-bot <searxng-bot@users.noreply.github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
signoff: false
branch: translations_update
delete-branch: true
draft: false
title: 'Update translations'
body: |
Update translations
labels: |
translation

3
.weblate Normal file
View file

@ -0,0 +1,3 @@
[weblate]
url = https://weblate.bubu1.eu/api/
translation = searxng/searxng

View file

@ -77,7 +77,7 @@ test.shell:
# wrap ./manage script
MANAGE += buildenv
MANAGE += ci.babel.update babel.extract babel.update babel.compile
MANAGE += weblate.translations.commit weblate.push.translations
MANAGE += data.all data.languages data.useragents data.osm_keys_tags
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += docker.build docker.push docker.buildx

View file

@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_
.. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits
:target: https://github.com/searxng/searxng/commits/master
.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
:target: https://weblate.bubu1.eu/projects/searxng/
If you are looking for running instances, ready to use, then visit searx.space_.

View file

@ -4,69 +4,77 @@
Translation
===========
.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/
.. _Weblate: https://docs.weblate.org
.. _translations branch: https://github.com/searxng/searxng/tree/translations
.. _orphan branch: https://git-scm.com/docs/git-checkout#Documentation/git-checkout.txt---orphanltnewbranchgt
.. _Weblate repository: https://weblate.bubu1.eu/projects/searxng/searxng/#repository
.. _wlc: https://docs.weblate.org/en/latest/wlc.html
Translation currently takes place on `searx@transifex`_
.. |translated| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
:target: https://weblate.bubu1.eu/projects/searxng/
Requirements
============
.. sidebar:: |translated|
* Transifex account
- Weblate_
- SearXNG `translations branch`_
- SearXNG `Weblate repository`_
- Weblate Client: wlc_
- Babel Command-Line: `pybabel <http://babel.pocoo.org/en/latest/cmdline.html>`_
- `weblate workflow <https://docs.weblate.org/en/latest/workflows.html>`_
Init Transifex project
======================
Translation takes place on weblate.bubu1.eu_.
After installing ``transifex`` using pip, run the following command to
initialize the project.
Translations which has been added by translators on the weblate.bubu1.eu_ UI are
committed to Weblate's counterpart of the SearXNG *origin* repository which is
located at ``https://weblate.bubu1.eu/git/searxng/searxng``.
.. code:: sh
There is no need to clone this repository, :ref:`SearXNG Weblate workflow` take
care of the synchronization with the *origin*. To avoid merging commits from
the counterpart directly on the ``master`` branch of *SearXNG origin*, a *pull
request* (PR) is created by this workflow.
./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
Weblate monitors the `translations branch`_, not the ``master`` branch. This
branch is an `orphan branch`_, decoupled from the master branch (we already know
orphan branches from the ``gh-pages``). The `translations branch`_ contains
only the
- ``translation/messages.pot`` and the
- ``translation/*/messages.po`` files, nothing else.
After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
into the configuration file.
.. _SearXNG Weblate workflow:
Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
.. figure:: translation.svg
.. code:: ini
SearXNG's PR workflow to be in sync with Weblate
[main]
host = https://www.transifex.com
[searx.messagespo]
file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
source_file = messages.pot
source_lang = en
type = PO
Sync from *origin* to *weblate*: using ``make weblate.push.translations``
For each commit on the ``master`` branch of SearXNG *origin* the GitHub job
:origin:`babel / Update translations branch
<.github/workflows/integration.yml>` checks for updated translations.
Sync from *weblate* to *origin*: using ``make weblate.translations.commit``
Every Friday, the GitHub workflow :origin:`babel / create PR for additons from
weblate <.github/workflows/translations-update.yml>` creates a PR with the
updated translation files:
Then run ``tx set``:
- ``translation/messages.pot``,
- ``translation/*/messages.po`` and
- ``translation/*/messages.mo``
.. code:: shell
wlc
===
./manage pyenv.cmd tx set --auto-local \
-r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
--source-lang en --type PO --source-file messages.pot --execute
.. _wlc configuration: https://docs.weblate.org/en/latest/wlc.html#wlc-config
.. _API key: https://weblate.bubu1.eu/accounts/profile/#api
All weblate integration is done by GitHub workflows, but if you want to use wlc_,
copy this content into `wlc configuration`_ in your HOME ``~/.config/weblate``
Update translations
===================
.. code-block:: ini
To retrieve the latest translations, pull it from Transifex.
[keys]
https://weblate.bubu1.eu/api/ = APIKEY
.. code:: sh
./manage pyenv.cmd tx pull -a
[?] Enter your api token: ....
Then check the new languages. If strings translated are not enough, delete those
folders, because those should not be compiled. Call the command below to compile
the ``.po`` files.
.. code:: shell
./manage pyenv.cmd pybabel compile -d searx/translations
After the compilation is finished commit the ``.po`` and ``.mo`` files and
create a PR.
Replace ``APIKEY`` by your `API key`_.

45
docs/dev/translation.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 17 KiB

202
manage
View file

@ -45,9 +45,8 @@ help() {
buildenv:
rebuild ./utils/brand.env
babel.:
extract : extract messages from source files and generate POT file
update : update existing message catalogs from POT file
compile : compile translation catalogs into binary MO files
master.to.translations: update the translations branch from the messages of the master branch.
translations.to.master: copy change from the translations branch to the master branch.
data.:
all : update searx/languages.py and ./data/*
languages : update searx/data/engines_languages.json & searx/languages.py
@ -122,46 +121,185 @@ buildenv() {
return "${PIPESTATUS[0]}"
}
babel.sha256sum() {
grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
TRANSLATIONS_WORKTREE="$CACHE/translations"
weblate.translations.worktree() {
# Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
# 'translations' from Weblate's counterpart (weblate) of the SearXNG
# (origin).
#
# remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
( set -e
if ! git remote get-url weblate 2> /dev/null; then
git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
fi
if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
pushd .
cd "${TRANSLATIONS_WORKTREE}"
git reset --hard HEAD
git pull origin translations
popd
else
mkdir -p "${TRANSLATIONS_WORKTREE}"
git worktree add "${TRANSLATIONS_WORKTREE}" translations
fi
)
}
ci.babel.update() {
local sha_before
weblate.to.translations() {
# Update 'translations' branch of SearXNG (origin) with last additions from
# Weblate.
# 1. Check if Weblate is locked, if not die with error message
# 2. On Weblate's counterpart (weblate), pull master and translations branch
# from SearXNG (origin).
# 3. Commit changes made in a Weblate object on Weblate's counterpart
# (weblate).
# 4. In translations worktree, merge changes of branch 'translations' from
# remote 'weblate' and push it on branch 'translations' of 'origin'
( set -e
sha_before="$(babel.sha256sum)"
babel.extract
if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
build_msg BABEL 'no changes detected, exiting'
return 1
if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then
die 1 "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)"
fi
babel.update
build_msg BABEL 'update done, edit .po files if required and run babel.compile'
# weblate: commit pending changes
pyenv.cmd wlc pull
pyenv.cmd wlc commit
# get the translations in a worktree
weblate.translations.worktree
pushd "${TRANSLATIONS_WORKTREE}"
git remote update weblate
git merge weblate/translations
git push
popd
)
dump_return $?
}
babel.extract() {
build_msg BABEL 'extract messages from source files and generate POT file'
pyenv.cmd pybabel extract -F babel.cfg \
-o "searx/translations/messages.pot" \
"searx/"
dump_return $?
weblate.translations.commit() {
# Update 'translations' branch of SearXNG (origin) with last additions from
# Weblate. Copy the changes to the master branch, compile translations and
# create a commit in the local branch (master)
local existing_commit_hash commit_body commit_message exitcode
( set -e
# lock change on weblate
pyenv.cmd wlc lock
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
weblate.translations.worktree
existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
# pull weblate commits
weblate.to.translations
# copy the changes to the master branch
cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
# compile translations
build_msg BABEL 'compile translation catalogs into binary MO files'
pyenv.cmd pybabel compile --statistics \
-d "searx/translations"
# git add/commit (no push)
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
commit_message=$(echo -e "[translations] update\n${commit_body}")
git add searx/translations
git commit -m "${commit_message}"
)
exitcode=$?
( # make sure to always unlock weblate
set -e
pyenv.cmd wlc unlock
)
dump_return $exitcode
}
babel.update() {
build_msg BABEL 'update existing message catalogs from POT file'
pyenv.cmd pybabel update -N \
-i "searx/translations/messages.pot" \
-d "searx/translations"
dump_return $?
}
weblate.push.translations() {
babel.compile() {
build_msg BABEL 'compile translation catalogs into binary MO files'
pyenv.cmd pybabel compile --statistics \
-d "searx/translations"
dump_return $?
# Push *translation changes* from SearXNG (origin) to Weblate's counterpart
# (weblate).
# In branch master of SearXNG (origin) check for meaningful changes in
# folder 'searx/translations', commit changes on branch 'translations' and
# at least, pull updated branches on Weblate's counterpart (weblate).
# 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
# 'translations' from remote 'weblate'.
# 2. Stop if there is no meaningful change in the 'master' branch (origin),
# compared to the 'translations' branch (weblate), otherwise ...
# 3. Update 'translations' branch of SearXNG (origin) with last additions
# from Weblate.
# 5. Notify Weblate to pull updated 'master' & 'translations' branch.
local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
last_commit_message exitcode
( set -e
# lock change on weblate
pyenv.cmd wlc lock
# get translations branch in git worktree (TRANSLATIONS_WORKTREE)
weblate.translations.worktree
# update messages.pot in the master branch
build_msg BABEL 'extract messages from source files and generate POT file'
messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
pyenv.cmd pybabel extract -F babel.cfg \
-o "${messages_pot}" \
"searx/"
# stop if there is no meaningful change in the master branch
diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
git diff -- "searx/translations/messages.pot")
if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
build_msg BABEL 'no changes detected, exiting'
return 0
fi
# save messages.pot in the translations branch for later
pushd "${TRANSLATIONS_WORKTREE}"
git stash push
popd
# merge weblate commits into the translations branch
weblate.to.translations
# restore messages.pot in the translations branch
pushd "${TRANSLATIONS_WORKTREE}"
git stash pop
popd
# update messages.po files in the master branch
build_msg BABEL 'update existing message catalogs from POT file'
pyenv.cmd pybabel update -N \
-i "${messages_pot}" \
-d "${TRANSLATIONS_WORKTREE}/searx/translations"
# git add/commit/push
last_commit_hash=$(git log -n1 --pretty=format:'%h')
last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
last_commit_message="[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}"
pushd "${TRANSLATIONS_WORKTREE}"
git add searx/translations
git commit -m "${last_commit_message}"
git push
popd
# notify weblate to pull updated master & translations branch
pyenv.cmd wlc pull
)
exitcode=$?
( # make sure to always unlock weblate
set -e
pyenv.cmd wlc unlock
)
dump_return $exitcode
}
data.all() {

View file

@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14
linuxdoc==20210324
aiounittest==1.4.0
yamllint==1.26.2
wlc==1.12