doc: Avoid shelling out to hotdoc to generate plugins config files

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4479>
This commit is contained in:
Thibault Saunier 2023-03-19 15:35:29 -03:00 committed by GStreamer Marge Bot
parent 6e305a471f
commit 7aaf2b48ef
13 changed files with 263 additions and 216 deletions

View file

@ -785,7 +785,8 @@ valgrind ges:
- git checkout subprojects/*.wrap
- ./ci/scripts/check-diff.py
- ./gst-env.py ninja -C build subprojects/gst-docs/GStreamer-doc
- export GI_TYPELIB_PATH=$PWD/girs
- hotdoc run --conf-file build/subprojects/gst-docs/GStreamer-doc.json
- mv build/subprojects/gst-docs/GStreamer-doc/html documentation/

View file

@ -74,10 +74,15 @@ if built_subprojects != ''
endforeach
foreach plugin_doc: sub.get_variable('plugins_doc', [])
warning(project_name+ ': variable plugins_doc is deprecated, use gst_plugins_doc instead')
plugins_doc += plugin_doc.full_path() + pathsep
hotdoc_subprojects += [plugin_doc]
deps += [plugin_doc]
endforeach
foreach plugin_doc: sub.get_variable('gst_plugins_doc', [])
plugins_doc += plugin_doc + pathsep
endforeach
else
message('@0@ did not build hotdoc documentation, can\'t build API doc'.format(project_name))
endif

View file

@ -12,7 +12,7 @@ if __name__ == "__main__":
if sys.argv[4]:
libs, plugins = sys.argv[4].split(os.pathsep), sorted(
sys.argv[5].split(os.pathsep), key=lambda x: os.path.basename(x))
sys.argv[5].replace('\n', '').split(os.pathsep), key=lambda x: os.path.basename(x))
index += '\n\tlibs.md'
for lib in libs:
if not lib:
@ -22,7 +22,10 @@ if __name__ == "__main__":
for plugin in plugins:
if not plugin:
continue
index += "\n\t\t" + plugin + '.json'
fname = plugin
if not fname.endswith('.json'):
fname += '.json'
index += "\n\t\t" + fname
index = '%s\n%s' % (index_md, index)

View file

@ -111,27 +111,21 @@ libs_doc = [hotdoc.generate_doc('gst-editing-services',
depends: ges_gir[0],
)]
plugins_doc = []
list_plugin_res = run_command(python, '-c',
'''
import sys
import json
with open("@0@") as f:
print(':'.join(json.load(f).keys()), end='')
'''.format(plugins_cache),
check: true)
foreach plugin_name: list_plugin_res.stdout().split(':')
plugins_doc += [hotdoc.generate_doc(plugin_name,
project_version: apiversion,
sitemap: 'plugins/sitemap.txt',
index: 'plugins/index.md',
gst_index: 'plugins/index.md',
gst_smart_index: true,
gst_c_sources: ['../plugins/*/*.[ch]',],
dependencies: [gst_dep],
gst_order_generated_subpages: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
)]
endforeach
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
cdir = meson.current_source_dir()
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', apiversion,
'--sitemap', cdir / 'plugins/sitemap.txt',
'--index', cdir / 'plugins/index.md',
'--gst_index', cdir / 'plugins/index.md',
'--gst_c_sources', cdir / '../plugins/*/*.[ch]',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -74,15 +74,22 @@ endforeach
libs_doc = []
plugins_doc = [hotdoc.generate_doc('libav',
project_version: api_version,
sitemap: 'sitemap.txt',
index: 'index.md',
gst_index: 'index.md',
gst_smart_index: true,
gst_c_sources: ['../ext/*/*.[ch]',],
gst_cache_file: plugins_cache,
gst_plugin_name: 'libav',
dependencies: [gst_dep],
disable_incremental_build: true,
)]
cdir = meson.current_source_dir()
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--project_version', api_version,
'--builddir', meson.current_build_dir(),
'--sitemap', cdir / 'sitemap.txt',
'--index', cdir / 'index.md',
'--gst_index', cdir / 'index.md',
'--gst_c_sources', cdir / '../ext/*/*.[ch]',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -48,15 +48,22 @@ if not hotdoc.has_extensions(required_hotdoc_extensions)
endif
message('Plugins: @0@'.format(plugins))
libs_doc = []
plugins_doc = [hotdoc.generate_doc('omx',
project_version: api_version,
sitemap: 'sitemap.txt',
index: 'index.md',
gst_index: 'index.md',
gst_smart_index: true,
gst_c_sources: ['../gst/*/*.[ch]',],
gst_cache_file: plugins_cache,
gst_plugin_name: 'omx',
dependencies: [gstomx],
)]
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
cdir = meson.current_source_dir()
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', api_version,
'--sitemap', cdir / 'sitemap.txt',
'--index', cdir / 'index.md',
'--gst_index', cdir / 'index.md',
'--gst_c_sources', cdir / '../gst/*/*.[ch]',
'--gst_cache_file', cdir / plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -220,45 +220,35 @@ foreach lib_def: libs
endif
endforeach
list_plugin_res = run_command(python3, '-c',
'''
import sys
import json
with open("@0@") as f:
print(':'.join(json.load(f).keys()), end='')
'''.format(plugins_cache),
check: true)
plugins_doc = []
foreach plugin_name: list_plugin_res.stdout().split(':')
plugins_doc += [hotdoc.generate_doc(plugin_name,
project_version: api_version,
sitemap: 'plugins/sitemap.txt',
index: 'plugins/index.md',
c_flags: '-DGST_USE_UNSTABLE_API',
gst_index: 'plugins/index.md',
gst_smart_index: true,
gst_c_sources: [
join_paths(root_rel, 'sys/*/*.[cmh]'),
join_paths(root_rel, 'sys/*/*.cpp'),
join_paths(root_rel, 'sys/*/*.cc'),
join_paths(root_rel, 'sys/*/*.mm'),
join_paths(root_rel, 'sys/*/*.hh'),
join_paths(root_rel, 'ext/*/*.[ch]'),
join_paths(root_rel, 'ext/*/*.cpp'),
join_paths(root_rel, 'ext/*/*.cc'),
join_paths(root_rel, 'ext/*/*.hh'),
join_paths(root_rel, 'gst/*/*.[ch]'),
join_paths(root_rel, 'gst/rtmp2/*/*.[ch]'),
],
dependencies: [gst_dep],
install: false,
disable_incremental_build: true,
gst_cache_file: plugins_cache,
gst_c_source_filters: excludes,
gst_order_generated_subpages: true,
gst_plugin_name: plugin_name,
include_paths: join_paths(meson.current_source_dir(), '..'),
)]
endforeach
cdir = meson.current_source_dir()
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', api_version,
'--sitemap', cdir / 'plugins/sitemap.txt',
'--index', cdir / 'plugins/index.md',
'--c_flags', '\-DGST_USE_UNSTABLE_API',
'--gst_index', 'plugins/index.md',
'--gst_c_sources',
cdir / '../sys/*/*.[cmh]',
cdir / '../sys/*/*.cpp',
cdir / '../sys/*/*.cc',
cdir / '../sys/*/*.mm',
cdir / '../sys/*/*.hh',
cdir / '../ext/*/*.[ch]',
cdir / '../ext/*/*.cpp',
cdir / '../ext/*/*.cc',
cdir / '../ext/*/*.hh',
cdir / '../gst/*/*.[ch]',
cdir / '../gst/rtmp2/*/*.[ch]',
'--gst_cache_file', plugins_cache,
'--gst_c_source_filters', excludes,
'--include_paths', join_paths(meson.current_source_dir(), '..'),
check: true,
).stdout().split(pathsep)

View file

@ -37,7 +37,8 @@ else
required: false)
endif
plugins_cache = join_paths(meson.current_source_dir(), 'plugins', 'gst_plugins_cache.json')
cdir = meson.current_source_dir()
plugins_cache = cdir / 'plugins/gst_plugins_cache.json'
if plugins.length() == 0
message('All base plugins have been disabled')
elif plugins_cache_generator.found()
@ -193,34 +194,23 @@ else
)]
endif
plugins_doc = []
sitemap = 'all_index.md\n'
list_plugin_res = run_command(python3, '-c',
'''
import sys
import json
with open("@0@") as f:
print(':'.join(json.load(f).keys()), end='')
'''.format(plugins_cache),
check: true)
foreach plugin_name: list_plugin_res.stdout().split(':')
plugins_doc += [hotdoc.generate_doc(plugin_name,
project_version: api_version,
sitemap: 'plugins/sitemap.txt',
index: 'plugins/index.md',
gst_index: 'plugins/index.md',
gst_smart_index: true,
gst_c_sources: [
'../sys/*/*.[ch]',
'../ext/*/*.[cmh]',
'../gst/*/*.[ch]',
],
dependencies: [gst_dep],
gst_order_generated_subpages: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
)]
sitemap += ' @0@-doc.json\n'.format(plugin_name)
endforeach
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', api_version,
'--sitemap', cdir / 'plugins/sitemap.txt',
'--index', cdir / 'plugins/index.md',
'--gst_index', cdir / 'plugins/index.md',
'--gst_c_sources',
cdir / '../sys/*/*.[ch]',
cdir / '../ext/*/*.[cmh]',
cdir / '../gst/*/*.[ch]',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -85,35 +85,27 @@ endforeach
excludes += [join_paths(meson.current_source_dir(), '..', 'sys', 'rpicamsrc', 'Raspi*.[ch]')]
list_plugin_res = run_command(python3, '-c',
'''
import sys
import json
with open("@0@") as f:
print(':'.join(json.load(f).keys()), end='')
'''.format(plugins_cache),
check: true)
foreach plugin_name: list_plugin_res.stdout().split(':')
plugins_doc += [hotdoc.generate_doc(plugin_name,
project_version: api_version,
sitemap: 'sitemap.txt',
index: 'index.md',
gst_index: 'index.md',
gst_smart_index: true,
gst_c_sources: [
'../sys/*/*.[cmh]',
'../ext/*/*.[ch]',
'../ext/*/*/*.[ch]',
'../gst/*/*.[ch]',
],
gst_c_source_filters: excludes,
dependencies: [gst_dep],
gst_order_generated_subpages: true,
install: false,
disable_incremental_build: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
include_paths: join_paths(meson.current_source_dir(), '..'),
)]
endforeach
cdir = meson.current_source_dir()
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', api_version,
'--sitemap', cdir / 'sitemap.txt',
'--index', cdir / 'index.md',
'--gst_index', cdir / 'index.md',
'--gst_c_sources',
cdir / '../sys/*/*.[cmh]',
cdir / '../ext/*/*.[ch]',
cdir / '../ext/*/*/*.[ch]',
cdir / '../gst/*/*.[ch]',
'--gst_c_source_filters', excludes,
'--gst_cache_file', plugins_cache,
'--include_paths', cdir / '..',
check: true,
).stdout().split(pathsep)

View file

@ -95,23 +95,25 @@ with open("@0@") as f:
print(':'.join(json.load(f).keys()), end='')
'''.format(plugins_cache),
check: true)
foreach plugin_name: list_plugin_res.stdout().split(':')
plugins_doc += [hotdoc.generate_doc(plugin_name,
project_version: api_version,
sitemap: 'sitemap.txt',
index: 'index.md',
gst_index: 'index.md',
gst_smart_index: true,
gst_c_sources: [
join_paths(root_rel, 'sys/*/*.[ch]'),
join_paths(root_rel, 'ext/*/*.[ch]'),
join_paths(root_rel, 'gst/*/*.[ch]'),
],
gst_c_source_filters: excludes,
dependencies: [gst_dep],
gst_order_generated_subpages: true,
disable_incremental_build: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
)]
endforeach
cdir = meson.current_source_dir()
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', api_version,
'--sitemap', cdir / 'sitemap.txt',
'--index', cdir / 'index.md',
'--gst_index', cdir / 'index.md',
'--gst_c_sources',
root_rel / 'sys/*/*.[ch]',
root_rel / 'ext/*/*.[ch]',
root_rel / 'gst/*/*.[ch]',
'--gst_c_source_filters', excludes,
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -94,16 +94,23 @@ libs_doc = [hotdoc.generate_doc('gst-rtsp-server',
depends: rtsp_server_gir[0],
)]
plugins_doc = [hotdoc.generate_doc('rtspclientsink',
project_version: api_version,
sitemap: 'plugins/sitemap.txt',
index: 'plugins/index.md',
gst_index: 'plugins/index.md',
gst_c_sources: ['../gst/rtsp-sink/*.[ch]'],
gst_dl_sources: [rtspsink.full_path()],
gst_smart_index: true,
dependencies: gst_rtsp_server_deps + [rtspsink],
gst_cache_file: plugins_cache,
gst_plugin_name: 'rtspclientsink',
)]
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
cdir = meson.current_source_dir()
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', api_version,
'--sitemap', cdir / 'plugins/sitemap.txt',
'--index', cdir / 'plugins/index.md',
'--gst_index', cdir / 'plugins/index.md',
'--gst_c_sources', cdir / '../gst/rtsp-sink/*.[ch]',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)
doc = libs_doc[0]

View file

@ -23,6 +23,8 @@ import sys
import re
import subprocess
import tempfile
from pathlib import Path as P
from argparse import ArgumentParser
from collections import OrderedDict
try:
@ -31,6 +33,53 @@ except ImportError: # python <3.3
from collections import Mapping
class GstPluginsHotdocConfGen:
def __init__(self):
parser = ArgumentParser()
parser.add_argument('--builddir', type=P)
parser.add_argument('--gst_cache_file', type=P)
parser.add_argument('--sitemap', type=P)
parser.add_argument('--index', type=P)
parser.add_argument('--c_flags')
parser.add_argument('--gst_index', type=P)
parser.add_argument('--gst_c_sources', nargs='*', default=[])
parser.add_argument('--project_version')
parser.add_argument('--include_paths', nargs='*', default=[])
parser.add_argument('--gst_c_source_filters', nargs='*', default=[])
parser.parse_args(namespace=self, args=sys.argv[2:])
def generate_plugins_configs(self):
plugin_files = []
with self.gst_cache_file.open() as fd:
all_plugins = json.load(fd)
for plugin_name in all_plugins.keys():
conf = self.builddir / f'plugin-{plugin_name}.json'
plugin_files.append(str(conf))
with conf.open('w') as f:
json.dump({
'sitemap': str(self.sitemap),
'index': str(self.index),
'gst_index': str(self.index),
'output': f'plugin-{plugin_name}',
'conf': str(conf),
'project_name': plugin_name,
'project_version': self.project_version,
'gst_cache_file': str(self.gst_cache_file),
'gst_plugin_name': plugin_name,
'c_flags': self.c_flags,
'gst_smart_index': True,
'gst_c_sources': self.gst_c_sources,
'gst_c_source_filters': [str(s) for s in self.gst_c_source_filters],
'include_paths': self.include_paths,
'gst_order_generated_subpages': True,
}, f, indent=4)
return plugin_files
# Marks values in the json file as "unstable" so that they are
# not updated automatically, this aims at making the cache file
# stable and handle corner cases were we can't automatically
@ -78,6 +127,11 @@ def test_unstable_values():
assert (current_cache == { "v1": "no", "v2": "yay", "unstable-values": "v2" })
if __name__ == "__main__":
if sys.argv[1] == "hotdoc-config":
fs = GstPluginsHotdocConfGen().generate_plugins_configs()
print(os.pathsep.join(fs))
sys.exit(0)
cache_filename = sys.argv[1]
output_filename = sys.argv[2]
build_root = os.environ.get('MESON_BUILD_ROOT', '')
@ -115,7 +169,7 @@ if __name__ == "__main__":
except subprocess.CalledProcessError as e:
log.flush()
with open(stderrlogfile, 'r', encoding='utf8') as f:
print(f.read(), file=sys.stderr)
print(f.read(), file=sys.stderr, end='')
raise
with open(out, 'r', newline='\n', encoding='utf8') as jfile:

View file

@ -163,28 +163,23 @@ foreach lib: libs
)]
endforeach
plugins_doc = [
hotdoc.generate_doc('coreelements',
project_version: apiversion,
sitemap: 'plugins/sitemap.txt',
index: 'plugins/index.md',
gst_index: 'plugins/index.md',
gst_smart_index: true,
gst_c_sources: ['../plugins/elements/*.c', '../plugins/elements/*.h'],
dependencies: [gst_plugins_doc_dep],
gst_cache_file: plugins_cache,
gst_plugin_name: 'coreelements',
),
hotdoc.generate_doc('coretracers',
project_version: apiversion,
sitemap: 'plugins/coretracers/sitemap.txt',
index: 'plugins/coretracers/index.md',
gst_index: 'plugins/coretracers/index.md',
gst_smart_index: true,
gst_c_sources: ['../plugins/tracers/*.c', '../plugins/tracers/*.h'],
dependencies: [gst_plugins_doc_dep],
gst_cache_file: plugins_cache,
gst_plugin_name: 'coretracers',
)
]
cdir = meson.current_source_dir()
if host_machine.system() == 'windows'
pathsep = ';'
else
pathsep = ':'
endif
gst_plugins_doc = run_command(
plugins_cache_generator,
'hotdoc-config',
'--builddir', meson.current_build_dir(),
'--project_version', apiversion,
'--sitemap', cdir / 'plugins/sitemap.txt',
'--index', cdir / 'plugins/index.md',
'--gst_index', cdir / 'plugins/index.md',
'--gst_c_sources',
cdir / '../plugins/*/*.c',
cdir / '../plugins/*/*.h',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)