plugins: add script to generate plugins list from module xml files

Bit rough, but works. I might still rewrite it all in perl ;)

For now use xml files from 1.10 branch.
This commit is contained in:
Tim-Philipp Müller 2016-11-28 20:32:25 +00:00
parent d30a5868c0
commit 7ab2946834
5 changed files with 351 additions and 0 deletions

2
.gitignore vendored
View file

@ -24,3 +24,5 @@ project.properties
gst_sdk
.DS_Store
xcuserdata
/plugins-introspection/cache/
/hotdoc_env/

View file

@ -4,6 +4,8 @@ install:
clean:
BRANCH = 1.10
HOTDOC ?= hotdoc
prefix ?= /usr
datadir ?= ${prefix}/share
@ -60,3 +62,17 @@ BUILT_DOC_DIR = $(builddir)/built_doc/html/
upload: all
rsync -rvaz -e ssh --links --delete $(BUILT_DOC_DIR) $(DOC_SERVER):$(DOC_BASE)
ssh $(DOC_SERVER) "chmod -R g+w $(DOC_BASE); chgrp -R gstreamer $(DOC_BASE)"
include plugins-introspection/plugins-introspection.mak
help:
@echo
@echo " make -- Build or rebuild docs (markdown -> html)"
@echo
@echo " make upload -- Upload docs"
@echo
@echo " make update-xml -- Update local plugin module .xml files from git"
@echo " make update-plugin-list -- Rebuild local plugins.md list from .xml files"
@echo
@echo " make clean-url-cache -- Remove local url cache"
@echo

View file

@ -1335,3 +1335,4 @@ properly (help welcome!).
[xvimagesink]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-base-plugins/html/gst-plugins-base-plugins-plugin-xvimagesink.html
[y4menc]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-good-plugins/html/gst-plugins-good-plugins-plugin-y4menc.html
[zbar]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-bad-plugins/html/gst-plugins-bad-plugins-plugin-zbar.html

View file

@ -0,0 +1,275 @@
#!/usr/bin/python3
#
# Copyright (C) 2016 Tim-Philipp Müller
#
# License: this script is made available under the following licenses:
# Simplified BSD License/FreeBSD License, MIT License, and/or LGPLv2.1+ license.
#
# Reads plugin introspection .xml files (one per module) and outputs a
# list of all plugins in markdown format. We may want to do something
# more fancy here in future, for now this is just to provide a replacement
# for the xltsproc-generated list in the www module
#
# XML files were generated like this (we'll want to automate that at some point):
#
# echo '<plugins module_name="gstreamer">' > core.xml; \
# cat ~/gst/1.10/gstreamer/docs/plugins/inspect/plugin-* >> core.xml; \
# echo '</plugins>' >> core.xml
#
# for m in base good ugly bad; do \
# echo '<plugins module_name="gst-plugins-'$m'">' > $m.xml; \
# cat ~/gst/1.10/gst-plugins-$m/docs/plugins/inspect/plugin-* >> $m.xml; \
# echo '</plugins>' >> $m.xml; \
# done
import xml.etree.ElementTree as xmltree
import http.client as httpclient
import urllib.parse as urlparse
import pickle
import sys
import os
allow_offline = False
is_offline = False
modules = {}
all_elements = {}
all_plugins = {}
url_cache = {}
url_conn = None
# check if an url exists or not
def check_url(url):
global url_conn
if is_offline and allow_offline:
return True
if url in url_cache:
return url_cache[url]
r = urlparse.urlparse(url)
# Reuse existing conn if possible
is_gst_fdo = url.startswith('https://gstreamer.freedesktop.org')
if is_gst_fdo and url_conn is not None:
connection = url_conn
else:
connection = httpclient.HTTPSConnection(r.netloc)
if is_gst_fdo:
url_conn = connection
connection.request('HEAD', r.path)
response = connection.getresponse()
response.read()
exists = response.status < 400
url_cache[url] = exists
sys.stderr.write('Checking if {0} exists: {1}\n'.format(url,exists))
return exists
# load url cache
def load_url_cache():
global url_cache
try:
with open('cache/url_cache.pkl', 'rb') as f:
url_cache = pickle.load(f)
except:
url_cache = {}
# save url cache
def save_url_cache():
global url_cache
if not os.path.exists('cache'):
os.makedirs('cache')
with open('cache/url_cache.pkl', 'wb') as f:
pickle.dump(url_cache, f)
def parse_plugin_element(element_node):
element = {}
for detail in element_node:
if detail.tag in ['pads']:
pass
elif detail.text:
element[detail.tag] = detail.text
return element
def parse_plugin_elements(elements_node):
element_list = []
for enode in elements_node:
element = parse_plugin_element(enode)
element_list.append(element)
return element_list
def parse_plugin(plugin_node):
plugin = {}
for detail in plugin_node:
if detail.tag == 'elements':
plugin['elements'] = parse_plugin_elements(detail)
elif detail.text:
plugin[detail.tag] = detail.text
return plugin
def parse_plugins(xml_fn):
plugin_list = []
tree = xmltree.parse(xml_fn)
root = tree.getroot()
for pnode in root:
plugin = parse_plugin(pnode)
plugin_list.append(plugin)
return plugin_list
def process_plugins(short_name):
module = {}
elements = []
plugins = parse_plugins(short_name + '.xml')
module['plugins'] = plugins
for p in plugins:
p['module_name'] = short_name
elist = p['elements']
for e in elist:
e['plugin_name'] = p['name']
e['module_name'] = short_name
elements.append(e)
name = e['name']
if name in all_elements:
print('Element {0} already exists in element list! {1}'.format(name, all_elements[name]))
else:
all_elements[name] = e
if p['name'] in all_plugins:
print('Plugin {0} already exists in plugin list! {1}'.format(name, all_plugins[p['name']]))
else:
all_plugins[p['name']] = p
module['elements'] = elements
modules[short_name] = module
# __main__
load_url_cache()
if not check_url('https://www.google.com'):
if not allow_offline:
raise Exception('URL checking does not seem to work, are you online?')
else:
print ('URL checking does not seem to work, continuing in offline mode')
is_offline = True
for m in ['core', 'base', 'good', 'bad', 'ugly']:
process_plugins(m)
plugins_page = '''# List of Elements and Plugins
<!-- WARNING: This page is generated! Any modifications will be overwritten -->
Note: this list is not complete! It does not contain OS-specific plugins
for Android, Windows, macOS, iOS, or wrapper plugins (gst-libav, gst-omx),
nor gst-rtsp-server or gstreamer-vaapi elements.
There may be links to pages that don't exist, this means that the element or
plugin does not have documentation yet or the documentation is not hooked up
properly (help welcome!).
| Element | Description | Plugin | Module |
|---------|-------------|---------|--------|
'''
plugin_names = []
element_links = []
plugin_links = []
element_names = []
for e in all_elements:
element_names.append(e)
element_names.sort()
for e in element_names:
element_name = e
element_desc = all_elements[e]['description']
plugin_name = all_elements[e]['plugin_name']
module_nick = all_elements[e]['module_name']
if module_nick == 'core':
module_name = 'gstreamer'
else:
module_name = 'gst-plugins-' + module_nick
# column 1: element name
edoc_url = 'https://gstreamer.freedesktop.org/data/doc/gstreamer/head/' + \
'{1}-plugins/html/{1}-plugins-{0}.html'.format(element_name, module_name)
if check_url(edoc_url):
element_links.append('[element-{0}]: {1}\n'.format(element_name, edoc_url))
plugins_page += '|[{0}][element-{0}]'.format(element_name)
else:
plugins_page += '|{0}'.format(element_name)
# column 2: element description
element_desc.replace('\n', ' ')
element_desc.replace('|', ' ')
if len(element_desc) > 150:
element_desc = '(too long)'
plugins_page += '|{0}'.format(element_desc)
# column 3: plugin name
pdoc_url = 'https://gstreamer.freedesktop.org/data/doc/gstreamer/head/' + \
'{1}-plugins/html/{1}-plugins-plugin-{0}.html'.format(plugin_name,module_name)
if check_url(pdoc_url):
plugins_page += '|[{0}][{0}]'.format(plugin_name)
else:
plugins_page += '|{0}'.format(plugin_name)
if plugin_name not in plugin_names:
plugin_names.append(plugin_name)
if check_url(pdoc_url):
plugin_links.append('[{0}]: {1}\n'.format(plugin_name,pdoc_url))
# column 4: module name
plugins_page += '|[{0}][{1}]'.format(module_name, module_nick)
# end of row
plugins_page += '|\n'
# Write module shortcuts list
plugins_page += '''
[core]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-plugins/html/
[base]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-base-plugins/html/
[good]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-good-plugins/html/
[ugly]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-ugly-plugins/html/
[bad]: https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-bad-plugins/html/
'''
# Write element shortcuts list
element_links.sort()
for elink in element_links:
plugins_page += elink
plugins_page += '\n'
# Write plugin shortcuts list
plugin_links.sort()
for plink in plugin_links:
plugins_page += plink
# Output page to stdout
print(plugins_page)
save_url_cache()

View file

@ -0,0 +1,57 @@
update-plugin-list:
@cd plugins-introspection; \
./create_plugin_list.py > ../markdown/plugins.md ; \
cd ..
CHECKOUT_DIR = $(top_builddir)/plugins-introspection/cache
checkoutdir:
@mkdir -p $(CHECKOUT_DIR) || { \
echo "Failed to create $(CHECKOUT_DIR)"; \
exit 1; \
}
update-xml: checkoutdir
@for m in gstreamer gst-plugins-base gst-plugins-good gst-plugins-ugly gst-plugins-bad; do \
moduledir=$(CHECKOUT_DIR)/$$m; \
branch=$(BRANCH); \
if [ -d $$moduledir ] ; then \
cd $$moduledir; \
echo "$$m checkout: cleaning"; \
git fetch --tags; \
git clean --dry-run -f -d -x || { \
echo "Cleaning $$m checkout failed"; exit 1; \
} ; \
echo "$$m checkout: updating..."; \
git checkout -q $$branch && git pull -q || { \
echo "Updating $$m checkout failed"; exit 1; \
} ; \
else \
echo "$$m checkout: creating (cloning)"; \
git clone -b $$branch https://anongit.freedesktop.org/git/gstreamer/$$m.git $$moduledir || { \
echo "git clone failed"; exit 1; \
} ; \
cd ../../ ; \
fi; \
done
@cd $(CHECKOUT_DIR)/gstreamer; \
echo '<plugins module_name="gstreamer">' > ../../core.xml; \
git ls-files docs/plugins/inspect/ | sort | xargs cat >> ../../core.xml; \
echo '</plugins>' >> ../../core.xml
@for m in base good ugly bad; do \
cd $(CHECKOUT_DIR)/gst-plugins-$$m; \
echo '<plugins module_name="gst-plugins-'$$m'">' > ../../$$m.xml; \
git ls-files docs/plugins/inspect/ | sort | xargs cat >> ../../$$m.xml; \
echo '</plugins>' >> ../../$$m.xml; \
done
foo:
@for m in base good ugly bad; do \
echo '<plugins module_name="gst-plugins-'$$m'">' > plugins-introspection/$$m.xml; \
cat $(CHECKOUT_DIR)/gst-plugins-$$m/docs/plugins/inspect/plugin-* >> plugins-introspection/$$m.xml; \
echo '</plugins>' >> plugins-introspection/$$m.xml; \
done
clean-url-cache:
@rm -f plugins-introspection/cache/url_cache.pkl