doc: Avoid shelling out to hotdoc to generate plugins config files

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4479>
This commit is contained in:
Thibault Saunier 2023-03-19 15:35:29 -03:00 committed by GStreamer Marge Bot
parent 6e305a471f
commit 7aaf2b48ef
13 changed files with 263 additions and 216 deletions

View file

@ -785,7 +785,8 @@ valgrind ges:
- git checkout subprojects/*.wrap - git checkout subprojects/*.wrap
- ./ci/scripts/check-diff.py - ./ci/scripts/check-diff.py
- ./gst-env.py ninja -C build subprojects/gst-docs/GStreamer-doc - export GI_TYPELIB_PATH=$PWD/girs
- hotdoc run --conf-file build/subprojects/gst-docs/GStreamer-doc.json
- mv build/subprojects/gst-docs/GStreamer-doc/html documentation/ - mv build/subprojects/gst-docs/GStreamer-doc/html documentation/

View file

@ -74,10 +74,15 @@ if built_subprojects != ''
endforeach endforeach
foreach plugin_doc: sub.get_variable('plugins_doc', []) foreach plugin_doc: sub.get_variable('plugins_doc', [])
warning(project_name+ ': variable plugins_doc is deprecated, use gst_plugins_doc instead')
plugins_doc += plugin_doc.full_path() + pathsep plugins_doc += plugin_doc.full_path() + pathsep
hotdoc_subprojects += [plugin_doc] hotdoc_subprojects += [plugin_doc]
deps += [plugin_doc] deps += [plugin_doc]
endforeach endforeach
foreach plugin_doc: sub.get_variable('gst_plugins_doc', [])
plugins_doc += plugin_doc + pathsep
endforeach
else else
message('@0@ did not build hotdoc documentation, can\'t build API doc'.format(project_name)) message('@0@ did not build hotdoc documentation, can\'t build API doc'.format(project_name))
endif endif

View file

@ -12,7 +12,7 @@ if __name__ == "__main__":
if sys.argv[4]: if sys.argv[4]:
libs, plugins = sys.argv[4].split(os.pathsep), sorted( libs, plugins = sys.argv[4].split(os.pathsep), sorted(
sys.argv[5].split(os.pathsep), key=lambda x: os.path.basename(x)) sys.argv[5].replace('\n', '').split(os.pathsep), key=lambda x: os.path.basename(x))
index += '\n\tlibs.md' index += '\n\tlibs.md'
for lib in libs: for lib in libs:
if not lib: if not lib:
@ -22,7 +22,10 @@ if __name__ == "__main__":
for plugin in plugins: for plugin in plugins:
if not plugin: if not plugin:
continue continue
index += "\n\t\t" + plugin + '.json' fname = plugin
if not fname.endswith('.json'):
fname += '.json'
index += "\n\t\t" + fname
index = '%s\n%s' % (index_md, index) index = '%s\n%s' % (index_md, index)

View file

@ -111,27 +111,21 @@ libs_doc = [hotdoc.generate_doc('gst-editing-services',
depends: ges_gir[0], depends: ges_gir[0],
)] )]
plugins_doc = [] if host_machine.system() == 'windows'
list_plugin_res = run_command(python, '-c', pathsep = ';'
''' else
import sys pathsep = ':'
import json endif
cdir = meson.current_source_dir()
with open("@0@") as f: gst_plugins_doc = run_command(
print(':'.join(json.load(f).keys()), end='') plugins_cache_generator,
'''.format(plugins_cache), 'hotdoc-config',
check: true) '--builddir', meson.current_build_dir(),
foreach plugin_name: list_plugin_res.stdout().split(':') '--project_version', apiversion,
plugins_doc += [hotdoc.generate_doc(plugin_name, '--sitemap', cdir / 'plugins/sitemap.txt',
project_version: apiversion, '--index', cdir / 'plugins/index.md',
sitemap: 'plugins/sitemap.txt', '--gst_index', cdir / 'plugins/index.md',
index: 'plugins/index.md', '--gst_c_sources', cdir / '../plugins/*/*.[ch]',
gst_index: 'plugins/index.md', '--gst_cache_file', plugins_cache,
gst_smart_index: true, check: true,
gst_c_sources: ['../plugins/*/*.[ch]',], ).stdout().split(pathsep)
dependencies: [gst_dep],
gst_order_generated_subpages: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
)]
endforeach

View file

@ -74,15 +74,22 @@ endforeach
libs_doc = [] libs_doc = []
plugins_doc = [hotdoc.generate_doc('libav', cdir = meson.current_source_dir()
project_version: api_version, if host_machine.system() == 'windows'
sitemap: 'sitemap.txt', pathsep = ';'
index: 'index.md', else
gst_index: 'index.md', pathsep = ':'
gst_smart_index: true, endif
gst_c_sources: ['../ext/*/*.[ch]',], gst_plugins_doc = run_command(
gst_cache_file: plugins_cache, plugins_cache_generator,
gst_plugin_name: 'libav', 'hotdoc-config',
dependencies: [gst_dep], '--project_version', api_version,
disable_incremental_build: true, '--builddir', meson.current_build_dir(),
)] '--sitemap', cdir / 'sitemap.txt',
'--index', cdir / 'index.md',
'--gst_index', cdir / 'index.md',
'--gst_c_sources', cdir / '../ext/*/*.[ch]',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -48,15 +48,22 @@ if not hotdoc.has_extensions(required_hotdoc_extensions)
endif endif
message('Plugins: @0@'.format(plugins)) message('Plugins: @0@'.format(plugins))
libs_doc = [] if host_machine.system() == 'windows'
plugins_doc = [hotdoc.generate_doc('omx', pathsep = ';'
project_version: api_version, else
sitemap: 'sitemap.txt', pathsep = ':'
index: 'index.md', endif
gst_index: 'index.md', cdir = meson.current_source_dir()
gst_smart_index: true, gst_plugins_doc = run_command(
gst_c_sources: ['../gst/*/*.[ch]',], plugins_cache_generator,
gst_cache_file: plugins_cache, 'hotdoc-config',
gst_plugin_name: 'omx', '--builddir', meson.current_build_dir(),
dependencies: [gstomx], '--project_version', api_version,
)] '--sitemap', cdir / 'sitemap.txt',
'--index', cdir / 'index.md',
'--gst_index', cdir / 'index.md',
'--gst_c_sources', cdir / '../gst/*/*.[ch]',
'--gst_cache_file', cdir / plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -220,45 +220,35 @@ foreach lib_def: libs
endif endif
endforeach endforeach
list_plugin_res = run_command(python3, '-c', cdir = meson.current_source_dir()
''' if host_machine.system() == 'windows'
import sys pathsep = ';'
import json else
pathsep = ':'
with open("@0@") as f: endif
print(':'.join(json.load(f).keys()), end='') gst_plugins_doc = run_command(
'''.format(plugins_cache), plugins_cache_generator,
check: true) 'hotdoc-config',
'--builddir', meson.current_build_dir(),
plugins_doc = [] '--project_version', api_version,
foreach plugin_name: list_plugin_res.stdout().split(':') '--sitemap', cdir / 'plugins/sitemap.txt',
plugins_doc += [hotdoc.generate_doc(plugin_name, '--index', cdir / 'plugins/index.md',
project_version: api_version, '--c_flags', '\-DGST_USE_UNSTABLE_API',
sitemap: 'plugins/sitemap.txt', '--gst_index', 'plugins/index.md',
index: 'plugins/index.md', '--gst_c_sources',
c_flags: '-DGST_USE_UNSTABLE_API', cdir / '../sys/*/*.[cmh]',
gst_index: 'plugins/index.md', cdir / '../sys/*/*.cpp',
gst_smart_index: true, cdir / '../sys/*/*.cc',
gst_c_sources: [ cdir / '../sys/*/*.mm',
join_paths(root_rel, 'sys/*/*.[cmh]'), cdir / '../sys/*/*.hh',
join_paths(root_rel, 'sys/*/*.cpp'), cdir / '../ext/*/*.[ch]',
join_paths(root_rel, 'sys/*/*.cc'), cdir / '../ext/*/*.cpp',
join_paths(root_rel, 'sys/*/*.mm'), cdir / '../ext/*/*.cc',
join_paths(root_rel, 'sys/*/*.hh'), cdir / '../ext/*/*.hh',
join_paths(root_rel, 'ext/*/*.[ch]'), cdir / '../gst/*/*.[ch]',
join_paths(root_rel, 'ext/*/*.cpp'), cdir / '../gst/rtmp2/*/*.[ch]',
join_paths(root_rel, 'ext/*/*.cc'), '--gst_cache_file', plugins_cache,
join_paths(root_rel, 'ext/*/*.hh'), '--gst_c_source_filters', excludes,
join_paths(root_rel, 'gst/*/*.[ch]'), '--include_paths', join_paths(meson.current_source_dir(), '..'),
join_paths(root_rel, 'gst/rtmp2/*/*.[ch]'), check: true,
], ).stdout().split(pathsep)
dependencies: [gst_dep],
install: false,
disable_incremental_build: true,
gst_cache_file: plugins_cache,
gst_c_source_filters: excludes,
gst_order_generated_subpages: true,
gst_plugin_name: plugin_name,
include_paths: join_paths(meson.current_source_dir(), '..'),
)]
endforeach

View file

@ -37,7 +37,8 @@ else
required: false) required: false)
endif endif
plugins_cache = join_paths(meson.current_source_dir(), 'plugins', 'gst_plugins_cache.json') cdir = meson.current_source_dir()
plugins_cache = cdir / 'plugins/gst_plugins_cache.json'
if plugins.length() == 0 if plugins.length() == 0
message('All base plugins have been disabled') message('All base plugins have been disabled')
elif plugins_cache_generator.found() elif plugins_cache_generator.found()
@ -193,34 +194,23 @@ else
)] )]
endif endif
plugins_doc = [] if host_machine.system() == 'windows'
sitemap = 'all_index.md\n' pathsep = ';'
else
list_plugin_res = run_command(python3, '-c', pathsep = ':'
''' endif
import sys gst_plugins_doc = run_command(
import json plugins_cache_generator,
'hotdoc-config',
with open("@0@") as f: '--builddir', meson.current_build_dir(),
print(':'.join(json.load(f).keys()), end='') '--project_version', api_version,
'''.format(plugins_cache), '--sitemap', cdir / 'plugins/sitemap.txt',
check: true) '--index', cdir / 'plugins/index.md',
foreach plugin_name: list_plugin_res.stdout().split(':') '--gst_index', cdir / 'plugins/index.md',
plugins_doc += [hotdoc.generate_doc(plugin_name, '--gst_c_sources',
project_version: api_version, cdir / '../sys/*/*.[ch]',
sitemap: 'plugins/sitemap.txt', cdir / '../ext/*/*.[cmh]',
index: 'plugins/index.md', cdir / '../gst/*/*.[ch]',
gst_index: 'plugins/index.md', '--gst_cache_file', plugins_cache,
gst_smart_index: true, check: true,
gst_c_sources: [ ).stdout().split(pathsep)
'../sys/*/*.[ch]',
'../ext/*/*.[cmh]',
'../gst/*/*.[ch]',
],
dependencies: [gst_dep],
gst_order_generated_subpages: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
)]
sitemap += ' @0@-doc.json\n'.format(plugin_name)
endforeach

View file

@ -85,35 +85,27 @@ endforeach
excludes += [join_paths(meson.current_source_dir(), '..', 'sys', 'rpicamsrc', 'Raspi*.[ch]')] excludes += [join_paths(meson.current_source_dir(), '..', 'sys', 'rpicamsrc', 'Raspi*.[ch]')]
list_plugin_res = run_command(python3, '-c', cdir = meson.current_source_dir()
''' if host_machine.system() == 'windows'
import sys pathsep = ';'
import json else
pathsep = ':'
with open("@0@") as f: endif
print(':'.join(json.load(f).keys()), end='') gst_plugins_doc = run_command(
'''.format(plugins_cache), plugins_cache_generator,
check: true) 'hotdoc-config',
foreach plugin_name: list_plugin_res.stdout().split(':') '--builddir', meson.current_build_dir(),
plugins_doc += [hotdoc.generate_doc(plugin_name, '--project_version', api_version,
project_version: api_version, '--sitemap', cdir / 'sitemap.txt',
sitemap: 'sitemap.txt', '--index', cdir / 'index.md',
index: 'index.md', '--gst_index', cdir / 'index.md',
gst_index: 'index.md', '--gst_c_sources',
gst_smart_index: true, cdir / '../sys/*/*.[cmh]',
gst_c_sources: [ cdir / '../ext/*/*.[ch]',
'../sys/*/*.[cmh]', cdir / '../ext/*/*/*.[ch]',
'../ext/*/*.[ch]', cdir / '../gst/*/*.[ch]',
'../ext/*/*/*.[ch]', '--gst_c_source_filters', excludes,
'../gst/*/*.[ch]', '--gst_cache_file', plugins_cache,
], '--include_paths', cdir / '..',
gst_c_source_filters: excludes, check: true,
dependencies: [gst_dep], ).stdout().split(pathsep)
gst_order_generated_subpages: true,
install: false,
disable_incremental_build: true,
gst_cache_file: plugins_cache,
gst_plugin_name: plugin_name,
include_paths: join_paths(meson.current_source_dir(), '..'),
)]
endforeach

View file

@ -95,23 +95,25 @@ with open("@0@") as f:
print(':'.join(json.load(f).keys()), end='') print(':'.join(json.load(f).keys()), end='')
'''.format(plugins_cache), '''.format(plugins_cache),
check: true) check: true)
foreach plugin_name: list_plugin_res.stdout().split(':') cdir = meson.current_source_dir()
plugins_doc += [hotdoc.generate_doc(plugin_name, if host_machine.system() == 'windows'
project_version: api_version, pathsep = ';'
sitemap: 'sitemap.txt', else
index: 'index.md', pathsep = ':'
gst_index: 'index.md', endif
gst_smart_index: true, gst_plugins_doc = run_command(
gst_c_sources: [ plugins_cache_generator,
join_paths(root_rel, 'sys/*/*.[ch]'), 'hotdoc-config',
join_paths(root_rel, 'ext/*/*.[ch]'), '--builddir', meson.current_build_dir(),
join_paths(root_rel, 'gst/*/*.[ch]'), '--project_version', api_version,
], '--sitemap', cdir / 'sitemap.txt',
gst_c_source_filters: excludes, '--index', cdir / 'index.md',
dependencies: [gst_dep], '--gst_index', cdir / 'index.md',
gst_order_generated_subpages: true, '--gst_c_sources',
disable_incremental_build: true, root_rel / 'sys/*/*.[ch]',
gst_cache_file: plugins_cache, root_rel / 'ext/*/*.[ch]',
gst_plugin_name: plugin_name, root_rel / 'gst/*/*.[ch]',
)] '--gst_c_source_filters', excludes,
endforeach '--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)

View file

@ -94,16 +94,23 @@ libs_doc = [hotdoc.generate_doc('gst-rtsp-server',
depends: rtsp_server_gir[0], depends: rtsp_server_gir[0],
)] )]
plugins_doc = [hotdoc.generate_doc('rtspclientsink', if host_machine.system() == 'windows'
project_version: api_version, pathsep = ';'
sitemap: 'plugins/sitemap.txt', else
index: 'plugins/index.md', pathsep = ':'
gst_index: 'plugins/index.md', endif
gst_c_sources: ['../gst/rtsp-sink/*.[ch]'], cdir = meson.current_source_dir()
gst_dl_sources: [rtspsink.full_path()], gst_plugins_doc = run_command(
gst_smart_index: true, plugins_cache_generator,
dependencies: gst_rtsp_server_deps + [rtspsink], 'hotdoc-config',
gst_cache_file: plugins_cache, '--builddir', meson.current_build_dir(),
gst_plugin_name: 'rtspclientsink', '--project_version', api_version,
)] '--sitemap', cdir / 'plugins/sitemap.txt',
'--index', cdir / 'plugins/index.md',
'--gst_index', cdir / 'plugins/index.md',
'--gst_c_sources', cdir / '../gst/rtsp-sink/*.[ch]',
'--gst_cache_file', plugins_cache,
check: true,
).stdout().split(pathsep)
doc = libs_doc[0] doc = libs_doc[0]

View file

@ -23,6 +23,8 @@ import sys
import re import re
import subprocess import subprocess
import tempfile import tempfile
from pathlib import Path as P
from argparse import ArgumentParser
from collections import OrderedDict from collections import OrderedDict
try: try:
@ -31,6 +33,53 @@ except ImportError: # python <3.3
from collections import Mapping from collections import Mapping
class GstPluginsHotdocConfGen:
def __init__(self):
parser = ArgumentParser()
parser.add_argument('--builddir', type=P)
parser.add_argument('--gst_cache_file', type=P)
parser.add_argument('--sitemap', type=P)
parser.add_argument('--index', type=P)
parser.add_argument('--c_flags')
parser.add_argument('--gst_index', type=P)
parser.add_argument('--gst_c_sources', nargs='*', default=[])
parser.add_argument('--project_version')
parser.add_argument('--include_paths', nargs='*', default=[])
parser.add_argument('--gst_c_source_filters', nargs='*', default=[])
parser.parse_args(namespace=self, args=sys.argv[2:])
def generate_plugins_configs(self):
plugin_files = []
with self.gst_cache_file.open() as fd:
all_plugins = json.load(fd)
for plugin_name in all_plugins.keys():
conf = self.builddir / f'plugin-{plugin_name}.json'
plugin_files.append(str(conf))
with conf.open('w') as f:
json.dump({
'sitemap': str(self.sitemap),
'index': str(self.index),
'gst_index': str(self.index),
'output': f'plugin-{plugin_name}',
'conf': str(conf),
'project_name': plugin_name,
'project_version': self.project_version,
'gst_cache_file': str(self.gst_cache_file),
'gst_plugin_name': plugin_name,
'c_flags': self.c_flags,
'gst_smart_index': True,
'gst_c_sources': self.gst_c_sources,
'gst_c_source_filters': [str(s) for s in self.gst_c_source_filters],
'include_paths': self.include_paths,
'gst_order_generated_subpages': True,
}, f, indent=4)
return plugin_files
# Marks values in the json file as "unstable" so that they are # Marks values in the json file as "unstable" so that they are
# not updated automatically, this aims at making the cache file # not updated automatically, this aims at making the cache file
# stable and handle corner cases were we can't automatically # stable and handle corner cases were we can't automatically
@ -78,6 +127,11 @@ def test_unstable_values():
assert (current_cache == { "v1": "no", "v2": "yay", "unstable-values": "v2" }) assert (current_cache == { "v1": "no", "v2": "yay", "unstable-values": "v2" })
if __name__ == "__main__": if __name__ == "__main__":
if sys.argv[1] == "hotdoc-config":
fs = GstPluginsHotdocConfGen().generate_plugins_configs()
print(os.pathsep.join(fs))
sys.exit(0)
cache_filename = sys.argv[1] cache_filename = sys.argv[1]
output_filename = sys.argv[2] output_filename = sys.argv[2]
build_root = os.environ.get('MESON_BUILD_ROOT', '') build_root = os.environ.get('MESON_BUILD_ROOT', '')
@ -115,7 +169,7 @@ if __name__ == "__main__":
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log.flush() log.flush()
with open(stderrlogfile, 'r', encoding='utf8') as f: with open(stderrlogfile, 'r', encoding='utf8') as f:
print(f.read(), file=sys.stderr) print(f.read(), file=sys.stderr, end='')
raise raise
with open(out, 'r', newline='\n', encoding='utf8') as jfile: with open(out, 'r', newline='\n', encoding='utf8') as jfile:

View file

@ -163,28 +163,23 @@ foreach lib: libs
)] )]
endforeach endforeach
plugins_doc = [ cdir = meson.current_source_dir()
hotdoc.generate_doc('coreelements', if host_machine.system() == 'windows'
project_version: apiversion, pathsep = ';'
sitemap: 'plugins/sitemap.txt', else
index: 'plugins/index.md', pathsep = ':'
gst_index: 'plugins/index.md', endif
gst_smart_index: true, gst_plugins_doc = run_command(
gst_c_sources: ['../plugins/elements/*.c', '../plugins/elements/*.h'], plugins_cache_generator,
dependencies: [gst_plugins_doc_dep], 'hotdoc-config',
gst_cache_file: plugins_cache, '--builddir', meson.current_build_dir(),
gst_plugin_name: 'coreelements', '--project_version', apiversion,
), '--sitemap', cdir / 'plugins/sitemap.txt',
hotdoc.generate_doc('coretracers', '--index', cdir / 'plugins/index.md',
project_version: apiversion, '--gst_index', cdir / 'plugins/index.md',
sitemap: 'plugins/coretracers/sitemap.txt', '--gst_c_sources',
index: 'plugins/coretracers/index.md', cdir / '../plugins/*/*.c',
gst_index: 'plugins/coretracers/index.md', cdir / '../plugins/*/*.h',
gst_smart_index: true, '--gst_cache_file', plugins_cache,
gst_c_sources: ['../plugins/tracers/*.c', '../plugins/tracers/*.h'], check: true,
dependencies: [gst_plugins_doc_dep], ).stdout().split(pathsep)
gst_cache_file: plugins_cache,
gst_plugin_name: 'coretracers',
)
]