doc: Avoid shelling out to hotdoc to generate plugins config files

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4479>

.gitlab-ci.yml

@@ -613,7 +613,7 @@ playbin3 integration:
     USE_PLAYBIN3: 1
   # Let's allow failures for now until we're 100% sure everything works
   allow_failure: true
-    
+
 # gstreamer-full:
 gstreamer-full static build:
   extends: '.build fedora x86_64'
@@ -785,7 +785,8 @@ valgrind ges:
   - git checkout subprojects/*.wrap
   - ./ci/scripts/check-diff.py
 
-  - ./gst-env.py ninja -C build subprojects/gst-docs/GStreamer-doc
+  - export GI_TYPELIB_PATH=$PWD/girs
+  - hotdoc run --conf-file build/subprojects/gst-docs/GStreamer-doc.json
 
   - mv build/subprojects/gst-docs/GStreamer-doc/html documentation/
 

subprojects/gst-docs/meson.build

@@ -74,10 +74,15 @@ if built_subprojects != ''
             endforeach
 
             foreach plugin_doc: sub.get_variable('plugins_doc', [])
+              warning(project_name+ ': variable plugins_doc is deprecated, use gst_plugins_doc instead')
               plugins_doc += plugin_doc.full_path() + pathsep
               hotdoc_subprojects += [plugin_doc]
               deps += [plugin_doc]
             endforeach
+
+            foreach plugin_doc: sub.get_variable('gst_plugins_doc', [])
+              plugins_doc += plugin_doc + pathsep
+            endforeach
         else
             message('@0@ did not build hotdoc documentation, can\'t build API doc'.format(project_name))
         endif

subprojects/gst-docs/scripts/generate_sitemap.py

@@ -12,7 +12,7 @@ if __name__ == "__main__":
 
         if sys.argv[4]:
             libs, plugins = sys.argv[4].split(os.pathsep), sorted(
-                sys.argv[5].split(os.pathsep), key=lambda x: os.path.basename(x))
+                sys.argv[5].replace('\n', '').split(os.pathsep), key=lambda x: os.path.basename(x))
             index += '\n\tlibs.md'
             for lib in libs:
                 if not lib:
@@ -22,7 +22,10 @@ if __name__ == "__main__":
             for plugin in plugins:
                 if not plugin:
                     continue
-                index += "\n\t\t" + plugin + '.json'
+                fname = plugin
+                if not fname.endswith('.json'):
+                    fname += '.json'
+                index += "\n\t\t" + fname
 
         index = '%s\n%s' % (index_md, index)
 

subprojects/gst-editing-services/docs/meson.build

@@ -111,27 +111,21 @@ libs_doc = [hotdoc.generate_doc('gst-editing-services',
     depends: ges_gir[0],
 )]
 
-plugins_doc = []
-list_plugin_res = run_command(python, '-c',
-'''
-import sys
-import json
-
-with open("@0@") as f:
-    print(':'.join(json.load(f).keys()), end='')
-'''.format(plugins_cache),
-  check: true)
-foreach plugin_name: list_plugin_res.stdout().split(':')
-    plugins_doc += [hotdoc.generate_doc(plugin_name,
-        project_version: apiversion,
-        sitemap: 'plugins/sitemap.txt',
-        index: 'plugins/index.md',
-        gst_index: 'plugins/index.md',
-        gst_smart_index: true,
-        gst_c_sources: ['../plugins/*/*.[ch]',],
-        dependencies: [gst_dep],
-        gst_order_generated_subpages: true,
-        gst_cache_file: plugins_cache,
-        gst_plugin_name: plugin_name,
-    )]
-endforeach
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+cdir = meson.current_source_dir()
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', apiversion,
+    '--sitemap', cdir / 'plugins/sitemap.txt',
+    '--index', cdir / 'plugins/index.md',
+    '--gst_index', cdir / 'plugins/index.md',
+    '--gst_c_sources', cdir / '../plugins/*/*.[ch]',
+    '--gst_cache_file', plugins_cache,
+    check: true,
+).stdout().split(pathsep)

subprojects/gst-libav/docs/meson.build

@@ -74,15 +74,22 @@ endforeach
 
 
 libs_doc = []
-plugins_doc = [hotdoc.generate_doc('libav',
-    project_version: api_version,
-    sitemap: 'sitemap.txt',
-    index: 'index.md',
-    gst_index: 'index.md',
-    gst_smart_index: true,
-    gst_c_sources: ['../ext/*/*.[ch]',],
-    gst_cache_file: plugins_cache,
-    gst_plugin_name: 'libav',
-    dependencies: [gst_dep],
-    disable_incremental_build: true,
-)]
+cdir = meson.current_source_dir()
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--project_version', api_version,
+    '--builddir', meson.current_build_dir(),
+    '--sitemap', cdir / 'sitemap.txt',
+    '--index', cdir / 'index.md',
+    '--gst_index', cdir / 'index.md',
+    '--gst_c_sources', cdir / '../ext/*/*.[ch]',
+    '--gst_cache_file', plugins_cache,
+    check: true,
+).stdout().split(pathsep)
+

subprojects/gst-omx/docs/meson.build

@@ -48,15 +48,22 @@ if not hotdoc.has_extensions(required_hotdoc_extensions)
 endif
 
 message('Plugins: @0@'.format(plugins))
-libs_doc = []
-plugins_doc = [hotdoc.generate_doc('omx',
-    project_version: api_version,
-    sitemap: 'sitemap.txt',
-    index: 'index.md',
-    gst_index: 'index.md',
-    gst_smart_index: true,
-    gst_c_sources: ['../gst/*/*.[ch]',],
-    gst_cache_file: plugins_cache,
-    gst_plugin_name: 'omx',
-    dependencies: [gstomx],
-)]
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+cdir = meson.current_source_dir()
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', api_version,
+    '--sitemap', cdir / 'sitemap.txt',
+    '--index', cdir / 'index.md',
+    '--gst_index', cdir / 'index.md',
+    '--gst_c_sources', cdir / '../gst/*/*.[ch]',
+    '--gst_cache_file', cdir / plugins_cache,
+    check: true,
+).stdout().split(pathsep)
+

subprojects/gst-plugins-bad/docs/meson.build

@@ -220,45 +220,35 @@ foreach lib_def: libs
     endif
 endforeach
 
-list_plugin_res = run_command(python3, '-c',
-'''
-import sys
-import json
-
-with open("@0@") as f:
-    print(':'.join(json.load(f).keys()), end='')
-'''.format(plugins_cache),
-  check: true)
-
-plugins_doc = []
-foreach plugin_name: list_plugin_res.stdout().split(':')
-    plugins_doc += [hotdoc.generate_doc(plugin_name,
-        project_version: api_version,
-        sitemap: 'plugins/sitemap.txt',
-        index: 'plugins/index.md',
-        c_flags: '-DGST_USE_UNSTABLE_API',
-        gst_index: 'plugins/index.md',
-        gst_smart_index: true,
-        gst_c_sources: [
-            join_paths(root_rel, 'sys/*/*.[cmh]'),
-            join_paths(root_rel, 'sys/*/*.cpp'),
-            join_paths(root_rel, 'sys/*/*.cc'),
-            join_paths(root_rel, 'sys/*/*.mm'),
-            join_paths(root_rel, 'sys/*/*.hh'),
-            join_paths(root_rel, 'ext/*/*.[ch]'),
-            join_paths(root_rel, 'ext/*/*.cpp'),
-            join_paths(root_rel, 'ext/*/*.cc'),
-            join_paths(root_rel, 'ext/*/*.hh'),
-            join_paths(root_rel, 'gst/*/*.[ch]'),
-            join_paths(root_rel, 'gst/rtmp2/*/*.[ch]'),
-        ],
-        dependencies: [gst_dep],
-        install: false,
-        disable_incremental_build: true,
-        gst_cache_file: plugins_cache,
-        gst_c_source_filters: excludes,
-        gst_order_generated_subpages: true,
-        gst_plugin_name: plugin_name,
-        include_paths: join_paths(meson.current_source_dir(), '..'),
-    )]
-endforeach
+cdir = meson.current_source_dir()
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', api_version,
+    '--sitemap', cdir / 'plugins/sitemap.txt',
+    '--index', cdir / 'plugins/index.md',
+    '--c_flags', '\-DGST_USE_UNSTABLE_API',
+    '--gst_index', 'plugins/index.md',
+    '--gst_c_sources',
+        cdir / '../sys/*/*.[cmh]',
+        cdir / '../sys/*/*.cpp',
+        cdir / '../sys/*/*.cc',
+        cdir / '../sys/*/*.mm',
+        cdir / '../sys/*/*.hh',
+        cdir / '../ext/*/*.[ch]',
+        cdir / '../ext/*/*.cpp',
+        cdir / '../ext/*/*.cc',
+        cdir / '../ext/*/*.hh',
+        cdir / '../gst/*/*.[ch]',
+        cdir / '../gst/rtmp2/*/*.[ch]',
+    '--gst_cache_file', plugins_cache,
+    '--gst_c_source_filters', excludes,
+    '--include_paths', join_paths(meson.current_source_dir(), '..'),
+    check: true,
+).stdout().split(pathsep)

subprojects/gst-plugins-base/docs/meson.build

@@ -37,7 +37,8 @@ else
         required: false)
 endif
 
-plugins_cache = join_paths(meson.current_source_dir(), 'plugins', 'gst_plugins_cache.json')
+cdir = meson.current_source_dir()
+plugins_cache = cdir / 'plugins/gst_plugins_cache.json'
 if plugins.length() == 0
     message('All base plugins have been disabled')
 elif plugins_cache_generator.found()
@@ -193,34 +194,23 @@ else
     )]
 endif
 
-plugins_doc = []
-sitemap = 'all_index.md\n'
-
-list_plugin_res = run_command(python3, '-c',
-'''
-import sys
-import json
-
-with open("@0@") as f:
-    print(':'.join(json.load(f).keys()), end='')
-'''.format(plugins_cache),
-  check: true)
-foreach plugin_name: list_plugin_res.stdout().split(':')
-    plugins_doc += [hotdoc.generate_doc(plugin_name,
-        project_version: api_version,
-        sitemap: 'plugins/sitemap.txt',
-        index: 'plugins/index.md',
-        gst_index: 'plugins/index.md',
-        gst_smart_index: true,
-        gst_c_sources: [
-          '../sys/*/*.[ch]',
-          '../ext/*/*.[cmh]',
-          '../gst/*/*.[ch]',
-        ],
-        dependencies: [gst_dep],
-        gst_order_generated_subpages: true,
-        gst_cache_file: plugins_cache,
-        gst_plugin_name: plugin_name,
-    )]
-    sitemap += '        @0@-doc.json\n'.format(plugin_name)
-endforeach
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', api_version,
+    '--sitemap', cdir / 'plugins/sitemap.txt',
+    '--index', cdir / 'plugins/index.md',
+    '--gst_index', cdir / 'plugins/index.md',
+    '--gst_c_sources',
+        cdir / '../sys/*/*.[ch]',
+        cdir / '../ext/*/*.[cmh]',
+        cdir / '../gst/*/*.[ch]',
+    '--gst_cache_file', plugins_cache,
+    check: true,
+).stdout().split(pathsep)

subprojects/gst-plugins-good/docs/meson.build

@@ -85,35 +85,27 @@ endforeach
 
 excludes += [join_paths(meson.current_source_dir(), '..', 'sys', 'rpicamsrc', 'Raspi*.[ch]')]
 
-list_plugin_res = run_command(python3, '-c',
-'''
-import sys
-import json
-
-with open("@0@") as f:
-    print(':'.join(json.load(f).keys()), end='')
-'''.format(plugins_cache),
-  check: true)
-foreach plugin_name: list_plugin_res.stdout().split(':')
-    plugins_doc += [hotdoc.generate_doc(plugin_name,
-    project_version: api_version,
-      sitemap: 'sitemap.txt',
-      index: 'index.md',
-      gst_index: 'index.md',
-      gst_smart_index: true,
-      gst_c_sources: [
-        '../sys/*/*.[cmh]',
-        '../ext/*/*.[ch]',
-        '../ext/*/*/*.[ch]',
-        '../gst/*/*.[ch]',
-      ],
-      gst_c_source_filters: excludes,
-      dependencies: [gst_dep],
-      gst_order_generated_subpages: true,
-      install: false,
-      disable_incremental_build: true,
-      gst_cache_file: plugins_cache,
-      gst_plugin_name: plugin_name,
-      include_paths: join_paths(meson.current_source_dir(), '..'),
-      )]
-endforeach
+cdir = meson.current_source_dir()
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', api_version,
+    '--sitemap', cdir / 'sitemap.txt',
+    '--index', cdir / 'index.md',
+    '--gst_index', cdir / 'index.md',
+    '--gst_c_sources',
+        cdir / '../sys/*/*.[cmh]',
+        cdir / '../ext/*/*.[ch]',
+        cdir / '../ext/*/*/*.[ch]',
+        cdir / '../gst/*/*.[ch]',
+    '--gst_c_source_filters', excludes,
+    '--gst_cache_file', plugins_cache,
+    '--include_paths', cdir / '..',
+    check: true,
+).stdout().split(pathsep)

subprojects/gst-plugins-ugly/docs/meson.build

@@ -95,23 +95,25 @@ with open("@0@") as f:
         print(':'.join(json.load(f).keys()), end='')
 '''.format(plugins_cache),
   check: true)
-foreach plugin_name: list_plugin_res.stdout().split(':')
-    plugins_doc += [hotdoc.generate_doc(plugin_name,
-        project_version: api_version,
-        sitemap: 'sitemap.txt',
-        index: 'index.md',
-        gst_index: 'index.md',
-        gst_smart_index: true,
-        gst_c_sources: [
-            join_paths(root_rel, 'sys/*/*.[ch]'),
-            join_paths(root_rel, 'ext/*/*.[ch]'),
-            join_paths(root_rel, 'gst/*/*.[ch]'),
-        ],
-        gst_c_source_filters: excludes,
-        dependencies: [gst_dep],
-        gst_order_generated_subpages: true,
-        disable_incremental_build: true,
-        gst_cache_file: plugins_cache,
-        gst_plugin_name: plugin_name,
-    )]
-endforeach
+cdir = meson.current_source_dir()
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', api_version,
+    '--sitemap', cdir / 'sitemap.txt',
+    '--index', cdir / 'index.md',
+    '--gst_index', cdir / 'index.md',
+    '--gst_c_sources',
+        root_rel / 'sys/*/*.[ch]',
+        root_rel / 'ext/*/*.[ch]',
+        root_rel / 'gst/*/*.[ch]',
+    '--gst_c_source_filters', excludes,
+    '--gst_cache_file', plugins_cache,
+    check: true,
+).stdout().split(pathsep)

subprojects/gst-rtsp-server/docs/meson.build

@@ -94,16 +94,23 @@ libs_doc = [hotdoc.generate_doc('gst-rtsp-server',
     depends: rtsp_server_gir[0],
 )]
 
-plugins_doc = [hotdoc.generate_doc('rtspclientsink',
-    project_version: api_version,
-    sitemap: 'plugins/sitemap.txt',
-    index: 'plugins/index.md',
-    gst_index: 'plugins/index.md',
-    gst_c_sources: ['../gst/rtsp-sink/*.[ch]'],
-    gst_dl_sources: [rtspsink.full_path()],
-    gst_smart_index: true,
-    dependencies: gst_rtsp_server_deps + [rtspsink],
-    gst_cache_file: plugins_cache,
-    gst_plugin_name: 'rtspclientsink',
-)]
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+cdir = meson.current_source_dir()
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', api_version,
+    '--sitemap', cdir / 'plugins/sitemap.txt',
+    '--index', cdir / 'plugins/index.md',
+    '--gst_index', cdir / 'plugins/index.md',
+    '--gst_c_sources', cdir / '../gst/rtsp-sink/*.[ch]',
+    '--gst_cache_file', plugins_cache,
+    check: true,
+).stdout().split(pathsep)
+
 doc = libs_doc[0]

subprojects/gstreamer/docs/gst-plugins-doc-cache-generator.py

@@ -23,6 +23,8 @@ import sys
 import re
 import subprocess
 import tempfile
+from pathlib import Path as P
+from argparse import ArgumentParser
 
 from collections import OrderedDict
 try:
@@ -31,6 +33,53 @@ except ImportError:  # python <3.3
     from collections import Mapping
 
 
+class GstPluginsHotdocConfGen:
+    def __init__(self):
+
+        parser = ArgumentParser()
+        parser.add_argument('--builddir', type=P)
+        parser.add_argument('--gst_cache_file', type=P)
+        parser.add_argument('--sitemap', type=P)
+        parser.add_argument('--index', type=P)
+        parser.add_argument('--c_flags')
+        parser.add_argument('--gst_index', type=P)
+        parser.add_argument('--gst_c_sources', nargs='*', default=[])
+        parser.add_argument('--project_version')
+        parser.add_argument('--include_paths', nargs='*', default=[])
+        parser.add_argument('--gst_c_source_filters', nargs='*', default=[])
+
+        parser.parse_args(namespace=self, args=sys.argv[2:])
+
+    def generate_plugins_configs(self):
+        plugin_files = []
+        with self.gst_cache_file.open() as fd:
+            all_plugins = json.load(fd)
+
+            for plugin_name in all_plugins.keys():
+                conf = self.builddir / f'plugin-{plugin_name}.json'
+                plugin_files.append(str(conf))
+                with conf.open('w') as f:
+                    json.dump({
+                        'sitemap': str(self.sitemap),
+                        'index': str(self.index),
+                        'gst_index': str(self.index),
+                        'output': f'plugin-{plugin_name}',
+                        'conf': str(conf),
+                        'project_name': plugin_name,
+                        'project_version': self.project_version,
+                        'gst_cache_file': str(self.gst_cache_file),
+                        'gst_plugin_name': plugin_name,
+                        'c_flags': self.c_flags,
+                        'gst_smart_index': True,
+                        'gst_c_sources': self.gst_c_sources,
+                        'gst_c_source_filters': [str(s) for s in self.gst_c_source_filters],
+                        'include_paths': self.include_paths,
+                        'gst_order_generated_subpages': True,
+                    }, f, indent=4)
+
+        return plugin_files
+
+
 # Marks values in the json file as "unstable" so that they are
 # not updated automatically, this aims at making the cache file
 # stable and handle corner cases were we can't automatically
@@ -78,6 +127,11 @@ def test_unstable_values():
     assert (current_cache == { "v1": "no", "v2": "yay", "unstable-values": "v2" })
 
 if __name__ == "__main__":
+    if sys.argv[1] == "hotdoc-config":
+        fs = GstPluginsHotdocConfGen().generate_plugins_configs()
+        print(os.pathsep.join(fs))
+        sys.exit(0)
+
     cache_filename = sys.argv[1]
     output_filename = sys.argv[2]
     build_root = os.environ.get('MESON_BUILD_ROOT', '')
@@ -115,7 +169,7 @@ if __name__ == "__main__":
         except subprocess.CalledProcessError as e:
             log.flush()
             with open(stderrlogfile, 'r', encoding='utf8') as f:
-                print(f.read(), file=sys.stderr)
+                print(f.read(), file=sys.stderr, end='')
             raise
 
     with open(out, 'r', newline='\n', encoding='utf8') as jfile:

subprojects/gstreamer/docs/meson.build

@@ -163,28 +163,23 @@ foreach lib: libs
   )]
 endforeach
 
-plugins_doc = [
-    hotdoc.generate_doc('coreelements',
-        project_version: apiversion,
-        sitemap: 'plugins/sitemap.txt',
-        index: 'plugins/index.md',
-        gst_index: 'plugins/index.md',
-        gst_smart_index: true,
-        gst_c_sources: ['../plugins/elements/*.c', '../plugins/elements/*.h'],
-        dependencies: [gst_plugins_doc_dep],
-        gst_cache_file: plugins_cache,
-        gst_plugin_name: 'coreelements',
-    ),
-    hotdoc.generate_doc('coretracers',
-        project_version: apiversion,
-        sitemap: 'plugins/coretracers/sitemap.txt',
-        index: 'plugins/coretracers/index.md',
-        gst_index: 'plugins/coretracers/index.md',
-        gst_smart_index: true,
-        gst_c_sources: ['../plugins/tracers/*.c', '../plugins/tracers/*.h'],
-        dependencies: [gst_plugins_doc_dep],
-        gst_cache_file: plugins_cache,
-        gst_plugin_name: 'coretracers',
-    )
-
-]
+cdir = meson.current_source_dir()
+if host_machine.system() == 'windows'
+  pathsep = ';'
+else
+  pathsep = ':'
+endif
+gst_plugins_doc = run_command(
+    plugins_cache_generator,
+    'hotdoc-config',
+    '--builddir', meson.current_build_dir(),
+    '--project_version', apiversion,
+    '--sitemap', cdir / 'plugins/sitemap.txt',
+    '--index', cdir / 'plugins/index.md',
+    '--gst_index', cdir / 'plugins/index.md',
+    '--gst_c_sources',
+        cdir / '../plugins/*/*.c',
+        cdir / '../plugins/*/*.h',
+    '--gst_cache_file', plugins_cache,
+    check: true,
+).stdout().split(pathsep)