plugins: add script to generate plugins list from module xml files

Bit rough, but works. I might still rewrite it all in perl ;)

For now use xml files from 1.10 branch.

.gitignore

@@ -24,3 +24,5 @@ project.properties
 gst_sdk
 .DS_Store
 xcuserdata
+/plugins-introspection/cache/
+/hotdoc_env/

Makefile

@@ -4,6 +4,8 @@ install:
 
 clean:
 
+BRANCH = 1.10
+
 HOTDOC ?= hotdoc
 prefix ?= /usr
 datadir ?= ${prefix}/share
@@ -60,3 +62,17 @@ BUILT_DOC_DIR = $(builddir)/built_doc/html/
 upload: all
 	rsync -rvaz -e ssh --links --delete $(BUILT_DOC_DIR) $(DOC_SERVER):$(DOC_BASE)
 	ssh $(DOC_SERVER) "chmod -R g+w $(DOC_BASE); chgrp -R gstreamer $(DOC_BASE)"
+
+include plugins-introspection/plugins-introspection.mak
+
+help:
+	@echo
+	@echo "  make                     -- Build or rebuild docs (markdown -> html)"
+	@echo
+	@echo "  make upload              -- Upload docs"
+	@echo
+	@echo "  make update-xml          -- Update local plugin module .xml files from git"
+	@echo "  make update-plugin-list  -- Rebuild local plugins.md list from .xml files"
+	@echo
+	@echo "  make clean-url-cache     -- Remove local url cache"
+	@echo

markdown/plugins.md

@@ -1335,3 +1335,4 @@ properly (help welcome!).
 [xvimagesink]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-base-plugins/html/gst-plugins-base-plugins-plugin-xvimagesink.html
 [y4menc]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-good-plugins/html/gst-plugins-good-plugins-plugin-y4menc.html
 [zbar]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-bad-plugins/html/gst-plugins-bad-plugins-plugin-zbar.html
+

plugins-introspection/create_plugin_list.py

@@ -0,0 +1,275 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2016 Tim-Philipp Müller
+#
+# License: this script is made available under the following licenses:
+# Simplified BSD License/FreeBSD License, MIT License, and/or LGPLv2.1+ license.
+#
+# Reads plugin introspection .xml files (one per module) and outputs a
+# list of all plugins in markdown format. We may want to do something
+# more fancy here in future, for now this is just to provide a replacement
+# for the xltsproc-generated list in the www module
+#
+# XML files were generated like this (we'll want to automate that at some point):
+#
+# echo '<plugins module_name="gstreamer">' > core.xml; \
+#   cat ~/gst/1.10/gstreamer/docs/plugins/inspect/plugin-* >> core.xml; \
+#   echo '</plugins>' >> core.xml
+#
+# for m in base good ugly bad; do \
+#   echo '<plugins module_name="gst-plugins-'$m'">' > $m.xml; \
+#   cat ~/gst/1.10/gst-plugins-$m/docs/plugins/inspect/plugin-* >> $m.xml; \
+#   echo '</plugins>' >> $m.xml; \
+# done
+
+import xml.etree.ElementTree as xmltree
+import http.client as httpclient
+import urllib.parse as urlparse
+import pickle
+import sys
+import os
+
+allow_offline = False
+is_offline = False
+
+modules = {}
+
+all_elements = {}
+all_plugins = {}
+
+url_cache = {}
+url_conn = None
+
+# check if an url exists or not
+def check_url(url):
+  global url_conn
+
+  if is_offline and allow_offline:
+    return True
+
+  if url in url_cache:
+    return url_cache[url]
+
+  r = urlparse.urlparse(url)
+
+  # Reuse existing conn if possible
+  is_gst_fdo = url.startswith('https://gstreamer.freedesktop.org')
+  if is_gst_fdo and url_conn is not None:
+    connection = url_conn
+  else:
+    connection = httpclient.HTTPSConnection(r.netloc)
+    if is_gst_fdo:
+      url_conn = connection
+
+  connection.request('HEAD', r.path)
+  response = connection.getresponse()
+  response.read()
+  exists = response.status < 400
+  url_cache[url] = exists
+  sys.stderr.write('Checking if {0} exists: {1}\n'.format(url,exists))
+  return exists
+
+# load url cache
+def load_url_cache():
+  global url_cache
+  try:
+    with open('cache/url_cache.pkl', 'rb') as f:
+      url_cache = pickle.load(f)
+  except:
+    url_cache = {}
+
+# save url cache
+def save_url_cache():
+  global url_cache
+  if not os.path.exists('cache'):
+    os.makedirs('cache')
+  with open('cache/url_cache.pkl', 'wb') as f:
+    pickle.dump(url_cache, f)
+
+def parse_plugin_element(element_node):
+  element = {}
+  for detail in element_node:
+    if detail.tag in ['pads']:
+      pass
+    elif detail.text:
+      element[detail.tag] = detail.text
+
+  return element
+
+def parse_plugin_elements(elements_node):
+  element_list = []
+
+  for enode in elements_node:
+    element = parse_plugin_element(enode)
+    element_list.append(element)
+
+  return element_list
+
+def parse_plugin(plugin_node):
+  plugin = {}
+
+  for detail in plugin_node:
+    if detail.tag == 'elements':
+      plugin['elements'] = parse_plugin_elements(detail)
+    elif detail.text:
+      plugin[detail.tag] = detail.text
+
+  return plugin
+
+def parse_plugins(xml_fn):
+  plugin_list = []
+
+  tree = xmltree.parse(xml_fn)
+  root = tree.getroot()
+  for pnode in root:
+    plugin = parse_plugin(pnode)
+    plugin_list.append(plugin)
+
+  return plugin_list
+
+def process_plugins(short_name):
+  module = {}
+  elements = []
+  plugins = parse_plugins(short_name + '.xml')
+  module['plugins'] = plugins
+  for p in plugins:
+    p['module_name'] = short_name
+    elist = p['elements']
+    for e in elist:
+      e['plugin_name'] = p['name']
+      e['module_name'] = short_name
+      elements.append(e)
+
+      name = e['name']
+      if name in all_elements:
+        print('Element {0} already exists in element list! {1}'.format(name, all_elements[name]))
+      else:
+        all_elements[name] = e
+
+    if p['name'] in all_plugins:
+      print('Plugin {0} already exists in plugin list! {1}'.format(name, all_plugins[p['name']]))
+    else:
+      all_plugins[p['name']] = p
+
+  module['elements'] = elements
+
+  modules[short_name] = module
+
+# __main__
+
+load_url_cache()
+
+if not check_url('https://www.google.com'):
+  if not allow_offline:
+    raise Exception('URL checking does not seem to work, are you online?')
+  else:
+    print ('URL checking does not seem to work, continuing in offline mode')
+    is_offline = True
+
+for m in ['core', 'base', 'good', 'bad', 'ugly']:
+  process_plugins(m)
+
+plugins_page = '''# List of Elements and Plugins
+
+<!-- WARNING: This page is generated! Any modifications will be overwritten -->
+
+Note: this list is not complete! It does not contain OS-specific plugins
+for Android, Windows, macOS, iOS, or wrapper plugins (gst-libav, gst-omx),
+nor gst-rtsp-server or gstreamer-vaapi elements.
+
+There may be links to pages that don't exist, this means that the element or
+plugin does not have documentation yet or the documentation is not hooked up
+properly (help welcome!).
+
+| Element | Description | Plugin  | Module |
+|---------|-------------|---------|--------|
+'''
+
+plugin_names = []
+
+element_links = []
+plugin_links = []
+
+element_names = []
+for e in all_elements:
+  element_names.append(e)
+
+element_names.sort()
+for e in element_names:
+  element_name = e
+  element_desc = all_elements[e]['description']
+  plugin_name = all_elements[e]['plugin_name']
+  module_nick = all_elements[e]['module_name']
+  if module_nick == 'core':
+    module_name = 'gstreamer'
+  else:
+    module_name = 'gst-plugins-' + module_nick
+
+  # column 1: element name
+  edoc_url = 'https://gstreamer.freedesktop.org/data/doc/gstreamer/head/' + \
+    '{1}-plugins/html/{1}-plugins-{0}.html'.format(element_name, module_name)
+
+  if check_url(edoc_url):
+    element_links.append('[element-{0}]: {1}\n'.format(element_name, edoc_url))
+    plugins_page += '|[{0}][element-{0}]'.format(element_name)
+  else:
+    plugins_page += '|{0}'.format(element_name)
+
+  # column 2: element description
+
+  element_desc.replace('\n', ' ')
+  element_desc.replace('|', ' ')
+  if len(element_desc) > 150:
+    element_desc = '(too long)'
+  plugins_page += '|{0}'.format(element_desc)
+
+  # column 3: plugin name
+  pdoc_url = 'https://gstreamer.freedesktop.org/data/doc/gstreamer/head/' + \
+    '{1}-plugins/html/{1}-plugins-plugin-{0}.html'.format(plugin_name,module_name)
+
+  if check_url(pdoc_url):
+    plugins_page += '|[{0}][{0}]'.format(plugin_name)
+  else:
+    plugins_page += '|{0}'.format(plugin_name)
+
+  if plugin_name not in plugin_names:
+    plugin_names.append(plugin_name)
+    if check_url(pdoc_url):
+      plugin_links.append('[{0}]: {1}\n'.format(plugin_name,pdoc_url))
+
+  # column 4: module name
+  plugins_page += '|[{0}][{1}]'.format(module_name, module_nick)
+
+  # end of row
+  plugins_page += '|\n'
+
+# Write module shortcuts list
+
+plugins_page += '''
+
+[core]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-plugins/html/
+[base]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-base-plugins/html/
+[good]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-good-plugins/html/
+[ugly]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-ugly-plugins/html/
+[bad]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-bad-plugins/html/
+
+'''
+
+# Write element shortcuts list
+
+element_links.sort()
+for elink in element_links:
+  plugins_page += elink
+
+plugins_page += '\n'
+
+# Write plugin shortcuts list
+
+plugin_links.sort()
+for plink in plugin_links:
+  plugins_page += plink
+
+# Output page to stdout
+print(plugins_page)
+
+save_url_cache()

plugins-introspection/plugins-introspection.mak

@@ -0,0 +1,57 @@
+update-plugin-list:
+	@cd plugins-introspection; \
+	./create_plugin_list.py > ../markdown/plugins.md ; \
+	cd ..
+
+CHECKOUT_DIR = $(top_builddir)/plugins-introspection/cache
+
+checkoutdir:
+	@mkdir -p $(CHECKOUT_DIR) || { \
+	  echo "Failed to create $(CHECKOUT_DIR)"; \
+	  exit 1; \
+	}
+
+update-xml: checkoutdir
+	@for m in gstreamer gst-plugins-base gst-plugins-good gst-plugins-ugly gst-plugins-bad; do \
+	  moduledir=$(CHECKOUT_DIR)/$$m; \
+	  branch=$(BRANCH); \
+	  if [ -d $$moduledir ] ; then \
+	    cd $$moduledir; \
+	    echo "$$m checkout: cleaning"; \
+	    git fetch --tags; \
+	    git clean --dry-run -f -d -x || { \
+	      echo "Cleaning $$m checkout failed"; exit 1; \
+	    } ; \
+	    echo "$$m checkout: updating..."; \
+	    git checkout -q $$branch && git pull -q || { \
+	      echo "Updating $$m checkout failed"; exit 1; \
+	    } ; \
+	  else \
+	    echo "$$m checkout: creating (cloning)"; \
+	    git clone -b $$branch https://anongit.freedesktop.org/git/gstreamer/$$m.git $$moduledir || { \
+	      echo "git clone failed"; exit 1; \
+	    } ; \
+	  cd ../../ ; \
+	  fi; \
+	done
+	@cd $(CHECKOUT_DIR)/gstreamer; \
+	  echo '<plugins module_name="gstreamer">' > ../../core.xml; \
+	  git ls-files docs/plugins/inspect/ | sort | xargs cat >> ../../core.xml; \
+	  echo '</plugins>' >> ../../core.xml
+	@for m in base good ugly bad; do \
+	  cd $(CHECKOUT_DIR)/gst-plugins-$$m; \
+	  echo '<plugins module_name="gst-plugins-'$$m'">' > ../../$$m.xml; \
+	  git ls-files docs/plugins/inspect/ | sort | xargs cat >> ../../$$m.xml; \
+	  echo '</plugins>' >> ../../$$m.xml; \
+	done
+
+
+foo:
+	@for m in base good ugly bad; do \
+	  echo '<plugins module_name="gst-plugins-'$$m'">' > plugins-introspection/$$m.xml; \
+	  cat $(CHECKOUT_DIR)/gst-plugins-$$m/docs/plugins/inspect/plugin-* >> plugins-introspection/$$m.xml; \
+	  echo '</plugins>' >> plugins-introspection/$$m.xml; \
+	done
+
+clean-url-cache:
+	@rm -f plugins-introspection/cache/url_cache.pkl