diff --git a/docs/random/omega/EOS/chain-walkthrough b/docs/random/omega/EOS/chain-walkthrough new file mode 100644 index 0000000000..b4c28bfb10 --- /dev/null +++ b/docs/random/omega/EOS/chain-walkthrough @@ -0,0 +1,19 @@ +disksrc -> mp3parse -> mpg123 -> audiosink + +disksrc reads last 4K chunk from disk. Sets EOS on the buffer, pushes buffer out to peer. +disksrc:src sets itself to EOS and chains to mp3parse. + + mp3parse takes the buffer and slices it up into several. + Each buffer gets chained off to mpg123. + + mpg123 decoders each frame and sends it to audiosink. + + audiosink dumps the audio data to the sound card. + + mpg123 returns from the chain, and returns. + + mp3parse returns from each of the several chains,a nd returns. + +disksrc:src returns from the chain, signals EOS, and sets disksrc's EOS flag + +At this point the pipeline state code takes over and starts to set everyone to READY diff --git a/docs/random/omega/IDEAS b/docs/random/omega/IDEAS new file mode 100644 index 0000000000..ef79da4037 --- /dev/null +++ b/docs/random/omega/IDEAS @@ -0,0 +1,21 @@ +Plugin Registry: +Have a web-accessible database of plugins from everywhere, including their merit values. gstplugin.c +and the autoplug code (should we abstract out autoplug into a special file? I think so) can make use of +this database via another module (gstwebregistry.c?) to look up stuff. A copy of the registry (gzip'd +XML) could even be cached. System and user options would determine whether this registry is checked +and/or updated automatically. The registry could simply be merged with the local machine and user +registries, with the state bit set to "don't even have it". Autodownload/install code should be +provided, though designed such that it's not toolkit/OS specific. + +Merit: +Plugins and even type definition should carry merit values, allowing the system to determine which +plugin or type definition is better. This can be tied into the web registry setup, where merit values +can be updated without requiring an update of the plugins themselves. They can also be guaranteed +unique if there's a range reserved for registered objects. Unfortunately, that might get us into some +political wars. We could leave that up to the users via some voting system. + +Scheduling state: +All the scheduling info for a Bin should be contained in a single object of some form. This can be +saved off and restored seamlessly at some point in the future. This would be ideal for some autoplug +cases, where the rest of the pipeline simply ceases to exist temporarily, and scheduling entry points +may need to be modified until the autoplug stage is finished. diff --git a/docs/random/omega/TODO-0.1.0 b/docs/random/omega/TODO-0.1.0 new file mode 100644 index 0000000000..d7c1619e32 --- /dev/null +++ b/docs/random/omega/TODO-0.1.0 @@ -0,0 +1,54 @@ +mandatory: +----------- + +(done) Fix compile warnings in gst/* + +Make sure all common media types work with autoplug and gstmediaplay + +Add interface to control the level of output, both compile- and run-time + +(done) Make sure the build is capable of being run without any debugging noise + +Make sure that the absense of any of the optional libraries will not be fatal + +Make sure all the old plugins don't build normally, via some configure option? + +Try to get more of the INFO calls worked out, removing gst_info + +Decide whether DEBUG should use the category system, and if so, implement it + +Icon for gstplay, .desktop file, etc. + +Build tarballs + +Build RPMs + +Figure out which docs we're going to make 'offical' + +UPDATE THE WEB SITE + +optional: +--------- + +Fix gstreamer-launch to do named pads + +Need to fix EOS subsystem so -launch can play out + +gstreamer-launch should play out till EOS + +Finish LADSPA plugin to data-moving stage + + + +things to remember for the anouncement: +--------------------------------------- + +Build requirements list: +libtool 1.3.5 or patched to "pass_all" +optional: +libcdparanoia +libmp3lame +libxaudio +libXv +libgdk-pixbuf +libglade diff --git a/docs/random/omega/TYPE_FOURCC b/docs/random/omega/TYPE_FOURCC new file mode 100644 index 0000000000..dff7d7d574 --- /dev/null +++ b/docs/random/omega/TYPE_FOURCC @@ -0,0 +1,3 @@ +#define GST_TYPE_FOURCC(f) \ +(((f)[0]) & ((f)[1] << 8) & ((f)[2] << 16) & ((f)[3] << 24)) + diff --git a/docs/random/omega/caps2 b/docs/random/omega/caps2 new file mode 100644 index 0000000000..2dc5353add --- /dev/null +++ b/docs/random/omega/caps2 @@ -0,0 +1,320 @@ +The elementfactory for a given element will contain some information about the capabilities of element's +pads or potential pads. An indication will be provided as to whether the pad always exists, always +exists once data is present, or *might* exist once data is present (the latter case is for things like +the MPEG system parsers, where an audio stream might or might not exist). + + +First, an entirely normal example: +---------------------------------- + +(-----------) (----------) (-------------) +! disksrc ! ! mpg123 ! ! audiosink ! +! src sink src sink ! +! ! ! ! ! ! +(-----------) (----------) (-------------) + +We start with only the disksrc. The typefind filter is attached to the disksrc, and via typefind magic +the properties of the disksrc are found to be: + + disksrc->src->caps = { + "audio/mp3", + "layer", GST_CAPS_INT (3), + "bitrate", GST_CAPS_INT (128), + NULL + }; + +A look through the plugin registry shows that we have an element called mpg123 that has the following +caps: + + static GstCapsFactory mpg123_sink_caps = { + "audio/mp3", + "layer", GST_CAPS_INT_RANGE (1, 3), + "bitrate", GST_CAPS_INT_RANGE (8, 320), + NULL + }; + +The caps of the disksrc fit within those parameters, so we instantiate an mpg123 and attach it to the +disksrc. The connection succeeds negotiation and as a result the mpg123 specifies its output caps as: + + mpg123->src->caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S16), + "depth", GST_CAPS_INT (16), + "rate", GST_CAPS_INT (44100), + "channels", GST_CAPS_INT (2), + NULL + }; + +Again from the plugin registry we find an element audiosink that has appropriate caps: + + static GstCapsFactory audiosink_src_caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S16,....), + "depth", GST_CAPS_INT (16), + "rate", GST_CAPS_INT_RANGE (4000, 96000), + "channels", GST_CAPS_INT_RANGE (1, 2), + NULL + }; + +A copy of the audiosink is instantiated and attached, negotiation goes smoothly, and we're done. No +dataflow has occured, no failure found, etc. An ideal autoplug. + + +Now, a slightly more convoluted example: +---------------------------------------- + +Start with the same graph: + +(-----------) (----------) (-------------) +! disksrc ! ! mpg123 ! ! audiosink ! +! src sink src sink ! +! ! ! ! ! ! +(-----------) (----------) (-------------) + +Run typefind on the disksrc's output, get the same output caps: + + disksrc->src->caps = { + "audio/mp3", + "layer", GST_CAPS_INT (3), + "bitrate", GST_CAPS_INT (128), + NULL + }; + +Find and attach mpg123, get the following output caps this time: + + mpg123->src->caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S16), + "depth", GST_CAPS_INT (16), + "rate", GST_CAPS_INT (44100), + "channels", GST_CAPS_INT (1), + NULL + }; + +Note that this time we have a mono output. A look into the audiosink caps shows that we have a match. +So we instantiate a copy. Oops. We now find that the caps for the input pad on our audiosink have +changed: + + mpg123->src->caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S16,...), + "depth", GST_CAPS_INT (16), + "rate", GST_CAPS_INT (11025, 48000), + "channels", GST_CAPS_INT (2), + NULL + }; + +Whoops. It seems that the sound card we've got in this machine (FIXME how on earth to deal with +multiple sound cards???) doesn't support mono output *at all*. This is a problem. We now find that we +hae no options as far as directly matching the mpg123 to the audiosink. + +A look through our (ficticious) plugin registry shows at least one element that at least has audio/raw +on both input and ouput (since both mpg123 and audiosink have open pads with this mime type). A closerlook shows that its caps are: + + static GstCapsFactory mono2stereo_sink_caps = { + "audio/raw", + "channels", GST_CAPS_INT (1), + NULL + }; + static GstCapsFactory mono2stereo_src_caps = { + "audio/raw", + "channels", GST_CAPS_INT (2), + NULL + }; + +Wow, that's a perfect match. Instantiate, attach to mpg123, no problems. Attach to audiosink, no +problems. Done. When we start up the pipeline, we should get absolutely no callbacks from pads saying +"help me, I've fallen and..., er, I don't like this buffer!". + + +A really messy case: +-------------------- + +Start with a disksrc, typefind it, get the following: + + disksrc->src->caps = { + "audio/mp3", + "layer", GST_CAPS_INT (3), + "bitrate", GST_CAPS_INT (128), + NULL + }; + +Look through the plugin registry, find mpg123. Instantiate it, attach it. It spits out audio +parameters as usual: + + mpg123->src->caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S16), + "depth", GST_CAPS_INT (16), + "rate", GST_CAPS_INT (44100), + "channels", GST_CAPS_INT (2), + NULL + }; + +Now we instantiate an audiosink plugin. This time, we're sunk: + + mpg123->src->caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S8,U8), + "depth", GST_CAPS_INT (8), + "rate", GST_CAPS_INT_RANGE (11025, 22050), + "channels", GST_CAPS_INT (1), + NULL + }; + +ACK! It's one of those Disney Sound Source things. We've got a problem here that isn't obviously +solvable. However, there happens to be another mp3 decoder sitting around. It's got the same +properties as mpg123, but a lower merit value. Let's instantiate one and attach it. We get the +following output pad caps: + + mp3decoder->src->caps = { + "audio/raw", + "format", GST_CAPS_BITFIELD (S8,S16), + "depth", GST_CAPS_INT_RANGE (8,16), + "rate", GST_CAPS_INT_RANGE (8000, 44100), + "channels", GST_CAPS_INT (1,2), + NULL + }; + +Well, that matches the audiosink. We try attaching it, and during negotiation the mp3decoder finds +sufficient common ground with the castrated audiosink and sets its output pad to match the best of the +options: S8 at 22050 KHz. + + + +Next to impossible scenario: DVD +-------------------------------- + +Start with a dvdsrc. It's output pad caps are: + + static GstCapsFactory dvdsrc_src_caps = { + "video/mpeg", + "mpegversion", GST_CAPS_INT (2), + "systemstream", GST_CAPS_BOOLEAN (TRUE), + NULL + }; + +The type would be classified as incomplete via some mechanism. This might cause the autoplug code to go +and run the typefind function. It would flesh the type out to the following: + + dvdsrc->src->caps = { + "video/mpeg", + "mpegversion", GST_CAPS_INT (2), + "systemstream", GST_CAPS_BOOLEAN (TRUE), + "videostreams", GST_CAPS_INT (1), + "audiostreams", GST_CAPS_INT (3), + "bitrate", GST_CAPS_INT (40960), + NULL, + }; + +Wow, that helped a lot. A check through the plugin registry shows that the mpeg2parse will match those +properties: + + static GstCapsFactory mpeg2parse_sink_caps = { + "video/mpeg", + "mpegversion", GST_CAPS_INT (2), + "systemstream", GST_CAPS_BOOLEAN (TRUE), + NULL + }; + +(In retrospect, it may not be necessary to run typefind if there's match this good right away. Only run +typefind when there's no exact match.) +Since there are no output pads yet, we have to actually push data through the pipeline. The moment a +buffer or two get to the mpeg2parse element, it promptly goes and creates an output pad, probably of the +following caps: + + mpeg2parse_video_src_caps = { + "video/mpeg", + "mpegversion", GST_CAPS_RANGE (1,2), + "systemstream", GST_CAPS_BOOLEAN (FALSE), + NULL + }; + +This seems to be a task for typefind again. But since data is flowing, we have to be careful with the +buffers. (This is the case in any typefind maneuver, but moreso when one really can't rewind the +source without consequences) The autoplug system attaches a special pseudo-element to mpeg2parse's new +output pad, and attaches the typefind element to the end of that. The pseudo-element takes the buffer, +stores it, and passes a copy off to the attached element, in this case typefind. This repeats until +typefind has determined the type, at which point the typefind is removed, and the newly found element is +attached instead. + +The pseudo-element is 'rewound' and the stored buffers flow out and into the newly attached element. +When the cache of buffers is gone, a signal fires and the autoplug system removes the pseudo-element and +reconnects the pipeline. + +In this case, the typefind function will find the following: + + mpeg2parse_video_src_caps = { + "video/mpeg", + "mpegversion", GST_CAPS_INT (2), + "systemstream", GST_CAPS_BOOLEAN (FALSE), + "bitrate", GST_CAPS_INT (36864), + "width", GST_CAPS_INT (720), + "height", GST_CAPS_INT (480), + "framerate", GST_CAPS_FLOAT (29.97003), + "chromaformat", GST_CAPS_INT (1), [GST_CAPS_STRING ("4:2:0") ?] + NULL + }; + +Back to the plugin registry, we find our only choice is mpeg2dec, which has input caps of: + + static GstCapsFactory mpeg2dec_sink_caps = { + "video/mpeg", + "mpegversion", GST_CAPS_RANGE (1,2), + "systemstream", GST_CAPS_BOOLEAN (FALSE), + NULL + }; + +Once again it just so happens that we really didn't need to do the typefind at all. But it can't hurt +unless the typefind is slow and painful, which we can guess won't be the case since the choices are +rather limited by the fact that there's already a MIME type attached, meaning we can drastically reduce +the number of typefind functions we try (down to one, actually). + +However, since we *have* run the typefind, upon attachment of the input pad of mpeg2dec, the output pad +looks like the following: + + mpeg2dec_src_caps = { + "video/raw", + "fourcc", GST_CAPS_LIST ( + GST_CAPS_FOURCC ("YV12"), [identical...] + GST_CAPS_FOURCC ("IYUV"), + GST_CAPS_FOURCC ("I420"), + ), + "width", GST_CAPS_INT (720), + "height", GST_CAPS_INT (480), + "framerate", GST_CAPS_FLOAT (29.97003), + NULL + }; + +Currently only videosink supports the output of video/raw. It claims a list of FOURCCs but nothing +more: + + static GstCapsFactory videosink_sink_caps = { + "video/raw", + "fourcc", GST_CAP_LIST ( GST_CAPS_FOURCC ("YV12"), + GST_CAPS_FOURCC ("IYUV"), GST_CAPS_FOURCC ("I420"), + GST_CAPS_FOURCC ("YUY2"), GST_CAPS_FOURCC ("UYVY"), + [ etc... ], + ), + NULL + }; + +When instantiated, we potentially have the same problem as with the audiosink: we don't necessarily know +which hardware output to use. Somehow we have to solve the problem of setting some element arguments +before we can get useful information out of them as to the properties. In this case anyway, if the +videosink were to find only one output possibility, it would trim the list of FOURCCs it can deal with +to what the hardware can handle, as well as add further properties: + + videosink_sink_caps = { + "video/raw", + "fourcc", GST_CAPS_LIST (GST_CAPS_FOURCC ("YV12"), + GST_CAPS_FOURCC ("YUY2"), + ), + "width", GST_CAPS_INT_RANGE (4,1020), + "height", GST_CAPS_INT_RANGE (4,1020), + NULL + }; + +We can now connect the mpeg2dec output to the videosink, and we now have displaying video. + +. . . . diff --git a/docs/random/omega/caps3 b/docs/random/omega/caps3 new file mode 100644 index 0000000000..11dcf8c646 --- /dev/null +++ b/docs/random/omega/caps3 @@ -0,0 +1,20 @@ + /* fake mp3 where bitrates 8 through 32 can't be stereo */ +static GstCapsListFactory mpg123_sink_caps = { + { + "audio/mp3", + "layer", GST_CAPS_INT_RANGE (1, 3), + "bitrate", GST_CAPS_INT_RANGE (32, 320), + "channels", GST_CAPS_INT_RANGE (1, 2), + "framed", GST_CAPS_BOOLEAN (TRUE), + NULL + }, + { + "audio/mp3", + "layer", GST_CAPS_INT_RANGE (1, 3), + "bitrate", GST_CAPS_INT_RANGE (8, 32), + "channels", GST_CAPS_INT_RANGE (1), + "framed", GST_CAPS_BOOLEAN (TRUE), + NULL + }, + NULL +}; diff --git a/docs/random/omega/debug-commit b/docs/random/omega/debug-commit new file mode 100644 index 0000000000..b1b4589708 --- /dev/null +++ b/docs/random/omega/debug-commit @@ -0,0 +1,30 @@ +Changes made to the DEBUG system. New header file gstdebug.h holds the stuff to keep it out of gst.h's +hair. DEBUG prints out the process id, cothread id, source filename and line number. Two new macros +DEBUG_ENTER and DEBUG_LEAVE are used to show the entry and exit of a given function. This eventually +might be used to construct call trace graphs, even taking cothreads into account. This would be quite +useful in visualizing the scheduling mechanism. + +Minor changes to various debug messages. + +Also sitting in gstdebug.h is a prototypical DEBUG_ENTER that's capable of performing DEBUG_LEAVE +automatically. It does this by utilizing a little-known GCC extension that allows one to call a +function with the same parameters as the current function. The macro uses this to basically call +itself. A boolean is used to ensure that when it calls itself it actually runs the body of the +function. In the meantime it prints stuff out before and after the real function, as well as +constructing a debugging string. This can be used eventually to provide call-wide data on the DEBUG +lines, instead of having to replicate data on each call to DEBUG. More research is needed into how this +would most cleanly be fit into some other chunk of code, like GStreamer (I think of this DEBUG trick as +a separate project, sorta). + +Unfortunately, the aforementioned DEBUG trick interacts quite poorly with cothreads. Almost any time +it's used in a function that has anything remotely to do with a cothread context (as in, it runs in +one), a segfault results from the __builtin_apply call, which is the heart of the whole thing. If +someone who really knows assembly could analyze the resulting code to see what's really going on, we +might find a way to fix either the macro or the cothreads (I'm thinking that there's something we missed +in constructing the cothreads themselves) so this works in all cases. + +In the meantime, please insert both DEBUG_ENTER and DEBUG_LEAVE in your functions. Be sure to put +DEBUG_ENTER after your variable declarations and before any functional code, not to put the function +name in any DEBUG strings (it's already there, trust me), and put a DEBUG_LEAVE if you care enough. + +Changes are going to happen in the way DEBUGs and other printouts occur, so stay tuned. diff --git a/docs/random/omega/eos b/docs/random/omega/eos new file mode 100644 index 0000000000..b2d4120771 --- /dev/null +++ b/docs/random/omega/eos @@ -0,0 +1,48 @@ +What I propose for EOS condition is th following: As is stands, EOS is a +flag in a buffer. However, it's also a signal from an element, which is +odd. What I propose is that EOS become a pad thing. Here's how it would +work: + +When a source has a final buffer, it marks it as EOS. It passes this on +through the pad, and the pads set a 'pending' EOS state on the way +through. When another push or pull happens, only then does the pad signal +EOS. This means that EOS doesn't happen until the buffer has passed all +the way to the end, and a request for another buffer is starting to work +its way back. It gets stopped almost immediately. + +This gets a bit messy in complex cases, haven't thought them all out. +Assuming everything is reasonably symmetric, it should work out cleanly. +We may have to add the ability to force buffers to pass even post EOS, but +that sounds like a hack. + + + +Example: DVD +============ + + + ------------ |-------| + -| mpeg2dec |-----| | + ------------------------ / ------------ | | + | video0|------/ ------------- | | ------------- + | | -| subtitle0 |-----| merge |-------| videosink | + | subtitle0|------/ ------------- | | ------------- +---------- | | ------------- | | +| dvdsrc |---| mpeg2parse subtitle1|--------| subtitle1 |-----| | +---------- | | ------------- |-------| + | audio0|--------\ ----------- |----------| + | | -| ac3dec0 |-----| | + | audio1|-------\ ----------- | | ------------- + ------------------------ \ ----------- | switcher |----| audiosink | + -| ac3dec1 |-----| | ------------- + ----------- |----------| + +When dvdsrc reads its last buffer, it sets EOS on the buffer and passes it. The pad (probably audio1, +since video and audio are interleaved and subtitles don't usually end up at the end of a movie) would +set EOS pending, and pass it to mpeg2dec which would pass the EOS to the merge. Merge would then ask +for a buffer from subtitle1, since that's the one we're showing subtitles on right now. It would then +pull from the mpeg2parse. This would cause a switch back the parser, who's next task is to go and set +all the output pads to EOS. It may have to accomplish this by sending a zero-byte buffer. + +Well, this example isn't actually very good. I'll go ahead and implement it, and we can see what +happens. The current setup is so lacking that anything is better.... diff --git a/docs/random/omega/filterfactory b/docs/random/omega/filterfactory new file mode 100644 index 0000000000..9936306a14 --- /dev/null +++ b/docs/random/omega/filterfactory @@ -0,0 +1,3 @@ +A very useful feature would be a FilterFactory, i.e. some system whereby filter writers can avoid +getting their hands messy with the details of building and operating a G[tk]Object. A couple of +structures and a couple of functions should do it. diff --git a/docs/random/omega/output_policies b/docs/random/omega/output_policies new file mode 100644 index 0000000000..81dfc803a9 --- /dev/null +++ b/docs/random/omega/output_policies @@ -0,0 +1,61 @@ +GStreamer polices for various forms of library output, including error cases. + + +g_return_... +============ +The parameters of a function are checked for validity (non-NULL, correct type, etc.) with g_return[_val]_if_fail, i.e.: + +gst_element_connect (GstElement *src, gchar *srcpadname, + GstElement *dest, gchar *destpadname) { + g_return_if_fail (src != NULL); + g_return_if_fail (GST_IS_ELEMENT(src)); + g_return_if_fail (srcpadname != NULL); + g_return_if_fail (dest != NULL); + g_return_if_fail (GST_IS_ELEMENT(dest)); + g_return_if_fail (destpadname != NULL); + +This will inform the user of the library of any basic problems with the arguments they pass, such as a NULL pointer. + + +ERROR +===== +The ERROR macro will be used whenever there is some other type of flaw in the data passed, at a GStreamer-specific +level: + + srcpad = gst_element_get_pad (src, srcpadname); + if (srcpad == NULL) { + ERROR(src,"source element has no pad \"%s\"",srcpadname); + return; + } + +An ERROR will generally cause ceasation of the application, and ideally launch gdb. + + +INFO +==== +The INFO macro will be used to output any interesting state from the library, such as plugin loading and various events +of interest. They will be separated into categories that can be individually enabled and disabled. + +Categories (first draft of list, unordered): + +cothreads [cothreads.c] +gst initialization [gst.c] +autoplug results [autoplug.c] +autoplug attempts [autoplug.c] +bin parentage issues [gstbin.c] +plan generation [gstbin.c] +schedule generation [gstschedule.c] +schedule implementation [gstpad.c, gstbin.c, etc.] +buffer operations [gstbuffer.h] +caps matching [gstcaps.c] +clock stuff [gstclock.c] +element pad operations [gstelement.c] +elementfactory operations [gstelementfactory.c] +pad creation/connection [gstpad.c] +pipeline stuff [gstpipeline.c] +plugin loading [gstplugin.c] +plugin loading errors [gstplugin.c] +properties operations [gstprops.c] +thread creation/management [gstthread.c] +type operations [gsttype.c] +XML load/save [gstxml.c] diff --git a/docs/random/omega/pad-negotiation b/docs/random/omega/pad-negotiation new file mode 100644 index 0000000000..8a669235b3 --- /dev/null +++ b/docs/random/omega/pad-negotiation @@ -0,0 +1,37 @@ +When two pads are connected, a negotiation phase is going to have to +happen. Ideally, the caps of the two pads will both be fully-specified, +and match. That's the ideal case, but may rarely happen in practice. + +It'll work the following way: + +1) gst_pad_connect(pad1,pad2) is called by something +2) pad1's negotiate() method is called, with pad2 as argument +3) negotiate() repeatedly calls pad2's set_caps() method + +At some point, the two pads will agree on a set of caps, and proceed by +returning TRUE from negotiate(), at which point gst_pad_connect() +finishes. If it returns FALSE, gst_pad_connect() is forced to fail. + +Now, obviously the algorithm used to find matching caps can get +complicated. But in some cases it'll be simple. Ideally, if there is no +negotiate() function for pad1, there'll be a function that will go through +the options and try to make pad1 and pad2 meet in the middle, with no +specific knowledge of what the caps actually mean. + +Another detail is deciding which pads are pad1 and pad2. In the autoplug +case, the code will sometimes know which of the pads have the more +specific caps. In others, you may not. Either you can guess, and +possibly lose to having the slower of the two pad's negotiate() methods do +the work, or you might be able to actually guess at which is the most +specific set of caps: + +For any given typeid in the union of both pads, look at all properties + For each property, find the smallest range, assign this a 1.0 + For all other instances of this property, assign relative to 1.0 +For each pad1,pad2 + Take the assigned value of every property, and multiply together + +Whichever value is lower between pad1 and pad2 is most likely to be the +most specific set of caps. The trick is implementing the above +efficiently, but on the surface there appear to be more than enough +potential optimizations. diff --git a/docs/random/omega/padtemplates b/docs/random/omega/padtemplates new file mode 100644 index 0000000000..7ce49460a1 --- /dev/null +++ b/docs/random/omega/padtemplates @@ -0,0 +1,43 @@ +typedef gpointer GstCapsFactoryEntry; +typedef GstCapsFactoryEntry GstCapsFactory[]; +typedef GstCapsFactory *GstCapsListFactory[]; + +typedef gpointer GstPadFactoryEntry; +typedef GstPadFactoryEntry GstPadFactory[]; +typedef GstPadFactory *GstPadListFactory[]; + +#define GST_PADFACTORY_SRC GINT_TO_POINTER (GST_PAD_SRC) +#define GST_PADFACTORY_ALWAYS GINT_TO_POINTER (GST_PAD_ALWAYS) + +typedef struct GstPadTemplate { + gchar *name_template; + gint direction; + gint presence; + GList *caps; +}; + +GstPadFactory mpg123_src_padfactory = { + "src" + GST_PADFACTORY_SRC, + GST_PADFACTORY_ALWAYS, + + "audio/raw" + "samplerate", GST_PROPS_INT (44100), + . . . + NULL +}; + +static GstPadTemplate *srcpadtemplate; + +mpg123_new(GstMpg123 *mpg123) { + mpg123->srcpad = gst_pad_new_template("src", srcpadtemplate); + . . . +} + +plugin_init() { + GstElementFactory *factory; + + factory = gst_elementfactory_new("mpg123",. . .); + srcpadtemplate = gst_padfactory_new(mpg123_src_padfactory); + get_elementfactory_add_padtemplate (srcpadtemplate); +} diff --git a/docs/random/omega/plan-generation b/docs/random/omega/plan-generation new file mode 100644 index 0000000000..86895974f7 --- /dev/null +++ b/docs/random/omega/plan-generation @@ -0,0 +1,84 @@ +Plan generation happens at transition from NULL to READY (and PLAYING to READY right now, need to fix +that). By way of some logic in gst_bin_change_state(), gst_bin_create_plan() is only called for the +outer Bin, usually a Pipeline. This keeps things from getting nasty later on. + +A major new concept in plan generation is that of the 'manager'. This is the element that is reponsible +for running a given element. In general, Pipelines and Threads are the only managing-capable elements +(have the MANAGER flag set), since they are the only ones with real scheduling authority (because they +have a process context to play with, basically). + +gst_bin_set_manager() is called to set the manager element of the bin and all it's children and their +children. However, there's one important trick: it won't recurse into child Bins that have the MANAGER +flag set. This avoids some highly redundant recursion. + +When create_plan() is called on the outside Pipeline, the first thing it does is call +set_manager(self,self). As noted above, this recursion will not proceed into child Bins that have the +MANAGER flag set. + +The next step is to recursively generate the plan (yes, head-recursive). This gives child Bins the +opportunity to generate their plan first, causing a inside-to-outside sequence. This matches the way +the scheduling is arranged now, where the plan for a Src/Connection outside a Bin is handled by that +Bin, not it's parent. But we must be very careful not to stomp on that plan in the parent Bin. + +Because create_plan() is called on all Bins, but we can only set up scheduling state in MANAGER bins, +create_plan() must perform create_plan() recursion, but not do anything else *unless* the MANAGER bit is +set. It shouldn't even call set_manager() unless it's a MANAGER itself, because calling it otherwise +would waste time doing the work again. Basically, from the standpoing of setting the manager, +create_plan() recursion starts it when the current Bin is a MANAGER, and set_manager() stops when it +finds the next one. create_plan()'s further recursion eventually starts the process back up again +furtuer down the hierarchy, until everything is covered. + +For all MANAGER Bins, the last step is to actually create the scheduling plan. This is still one of the +nastiest chunks of code in the whole project, and probably will do nothing but get worse from now on (it +got better recently, but only because I took a chainsaw to the code and broke everthing...). It will +remain similar to what it is now, but with some definite differences. + +First task is now to find all the elements that we're responsible for. This is normally a recursive +process, because the structure is an arbitrary tree. However, something like the following should work +(bin is self): + + GSList *elements = NULL; + GList *children; + GSList *waiting_bins = NULL; + GstBin *waiting_bin; + + waiting_bins = g_slist_prepend (waiting_bins,bin); + + while (waiting_bins) { + // retrieve the top of the stack and pop it + waiting_bin = GST_BIN (waiting_bins->data); + waiting_bins = g_slist_remove (waiting_bins,waiting_bin); + + // walk the list of elements, and find bins + children = waiting_bin->children; + while (children) { + // add it to the list of elements + elements = g_slist_prepend (elements, children->data); + + // if it's a bin and it's not a managing bin, + // shove it on the list of bins to recurse into + if (GST_IS_BIN (children->data) && + !GST_FLAG_IS_SET (GST_ELEMENT (children->data))) + waiting_bins = g_slist_prepend (waiting_bins,children->data); + + children = g_list_next (children); + } + } + +The code makes the assumption that the manager of every element is the same until such time as a +different managing parent appears in the hierarchy. This is the result of the aforementioned nested +recursion of create_plan() and set_manager(), but may not remain the case forever. The above loop +should probably be slightly re-written to work solely on whether or not the Bin in question is the +element's manager. This means that the child Bins are *always* recursed into, in case there's a rogue +element inside of one of them that's supposed to be managed. + +At the same time all the elements managed by this bin are found (i.e. in the inner loop), we can +determine some useful bits of information, such as testing for several cases that require the use of +cothreads. The availability of manager information at this point may aid significantly in this +decision. + +Finally, the scheduling plan is generated, based on all the elements to be managed by the Bin (the list +of which may span several 'generations' of Bins and elements). Elements which have peers in child +self-managed Bins are left alone on for the pad in that makes that connection. This should keep the +parent Bins from stepping all over state set up by the child Bins, by establishing clear implicit +ownership on the pad level, based on the managing Bins' relationship to the pad. diff --git a/docs/random/omega/sched-case b/docs/random/omega/sched-case new file mode 100644 index 0000000000..f5ebd2d90a --- /dev/null +++ b/docs/random/omega/sched-case @@ -0,0 +1,20 @@ +Case 1: + + +--------------------------------------------------- +| pipeline | +| --------------- ---------------- | +| | bin | | thread | | +| | ----------- | ------------ | ------------ | | +| | | fakesrc | | | queue | | | fakesink | | | +| | | src>|-|--||--|-|sinkpad); + gst_pad_push(spindentity->srcpad,buf); + } while (!GST_ELEMENT_IS_COTHREAD_STOPPING(element)); + +The reason for this is that COTHREAD_STOPPING may be set before the above +loop ever gets started. It wouldn't do for the body of the loop to never +once get called, that would simply stall the pipeline. Note that only the +core library code is ever responsible for setting and unsetting this flag. +All elements have to do is respond to it by cleanly exiting the loop and +the function holding it. + +This is needed primarily to allow iterations to occur properly. +Basically, there's a single entry point in the cothread scheduling loop, +gst_bin_iterate_func() simply switches to this cothread. If the element +in this context is allowed to loop infinitely, nothing would even switch +back to the context from which the iterate() was originally called. This +is a bit of a problem. The solution is for there to be an implicit switch +back to the originating context. Now, even I'm not sure exactly how this +works, but if the cothread that's switched to actually returns, execution +returns back to the calling context, i.e. iterate_func(). + +COTHREAD_STOPPING is therefore set just before switching into this +(currently randomly chosen) context, on the assumption that it will return +promptly after finishing its duties. The burden of clearing the flag +falls to the various wrapper functions provided by the Bin code, thus +element writers don't have to worry about doing that at all (and simply +shouldn't). + + +Related changes: +All the sources in elements/ have been changed to reflect the new system. + + +FIXMEs: +1) gstpipeline.c calls gst_src_push at some point, dunno why, it's +commented out now. +2) any other sources, including vcdsrc, dvdsrc, and v4lsrc will break +badly and need to be modified to work as pull-based sources. diff --git a/docs/random/omega/sched/chains b/docs/random/omega/sched/chains new file mode 100644 index 0000000000..0ecee297f5 --- /dev/null +++ b/docs/random/omega/sched/chains @@ -0,0 +1,89 @@ +A new concept in scheduling is that of chains of elements that need to be schedule separately, even in the same set of +managed elements (which is the set of elements that the Bin in question [a pipeline or thread] is responsible for). +An example would by anywhere you have a non-blocking queue in place for buffering. This kind of element might be +useful in cases where the scheduling on a buffer level is tight enough that deadlocks might occur. + +The scheduler will find chains by traversing the pipeline through the list of managed elements. A chain boundary is +anywhere you have a 'DECOUPLED' element. A DECOUPLED element is one where there is no direct correlation between the +activities of the various pads. A source fits this description, although the normal single-pad source is the +degenerate case. A queue more properly fits the bill, since pushing a buffer at the sink pad doesn't trigger anything +on the src pad, and vice versa. A multi-src async source is probably the best example, since you want to leave the +scheduling up to the elements connected to it. + +Anyway, first the simple case: + +fakesrc -> fakesink + +Both of them should probably have the DECOUPLED bit set, at least to be true to the nature of the actual fake elements. +These two end up being a chain, and scheduling has to be set up for the chain. There are no cothreaded elements in the +chain, which means it's relatively easy. The goal is to find a single entry into the chain, which can be called in a +loop to get things done. Since the fakesrc is DECOUPLED, and we'd be messing with the source pad, it has lower +priority than a DECOUPLED sink pad, so the fakesrc's sink pad is the ideal entry into the chain. This can be +simplified into saying that the fakesink is the entry. + +In the end, the code to do this boils down to: + + buf = gst_pad_pull (fakesink->sinkpad); + gst_pad_push (fakesink->sinkpad, buf); + +Because of the way things are no implemented for scheduling, turning it around and making the source the entry has no +effect as far as the efficiency. That's because _get no longer directly calls gst_pad_push(), so we have to do it +outside. No big deal, it boils down to the same thing I think, modulo a cache-line of stack (i.e. one or two fewer +this way). + +If we put an identity in the middle: + +fakesrc -> identity -> fakesink + +then we have the same thing, except that there's now an element that isn't DECOUPLED, so it gets higher priority. That +means the identity is now the entry, and when we push the buffer into its chain function, the fakesink gets called. + +Now, we can make this much more complex, with the following elementary echo meta-filter: + + |=====| -> delay1 -> |=====| + | | | | +-> queue -> | tee | -> delay2 -> | mix | -> queue -> + | | | | + |=====| -> delay3 -> |=====| + +The tee takes a buffer in and spits three out, delay shifts the timestamps around and possibly reframes things to be +friendly. mix takes the three buffers and simply sums them (they're all audio). The tee element takes one buffer in +and promptly spits three out, one after another. Delay takes an element and immediately spits out a buffer (it +zero-pads at the beginning [the duration of the delay] for the sake of argument). Mix in this case is chained, but +assumes that buffer will arrive in order. On the last chain, it does a push of the newly mixed audio buffer. + +The queues are both DECOUPLED, so they have lower weight. That leaves a bunch of other elements sitting there ripe for +entry. But if we were to take delay1, what would happen? Well we can't, since there's no _get function on the tee's +src pads. + +This just re-enforces the idea that the left-most (closest to the source, for you right-to-left people) element should +get to be the entry. But what if we have multiple left-most elements?: + +-> queue -> eq1 -> |=====| + | mix | -> queue +-> queue -> eq2 -> |=====| + +If eq1 is the only entry, we call pull on the queue, then chain over to mix. Mix then doesn't do anything with it, +since it's waiting for another buffer before doing anything. That means we have to do the same with eq2, and have it +chain to mix, at which point mix will do its magic and chain out to the right-hand side. Figure out to actually use +both entries is hard, because the idea at this point is that there's only a single entry to a chain. + +Does this mean that we should make mix a DECOUPLED element? That would fix it to some extent, giving us three chains +in the above case. Each eq chain would be driven by the eq element, pulling from the queue and pushing into the mixer. +The mixer -> queue chain is problematic, because there is no possibly entry. The mixer side has no _get function +(since the push always happens upon the receipt of a buffer from the second sink pad), which means that that those two +pads have no possible entrance. + +Cothreads make this case much easier, since the mix element would drive things, forcing the eq elements to pull and +process buffers in order as needed. It may be that the best option in the case where there are any multi-sinkpad +elements is to turn them into cothreads. + + +Now, on to cothreaded cases. The simplest possible is to turn all the elements into cothreads. I may punt on this and +do just that for the moment, but there's still the question of what to do at the ends of the chain, where the DECOUPLED +elements are. The easiest is to simply always make then chained, so there's never any worry about who owns the +cothread context for the element, simply because there never will be one. + +fakesrc -> queue -> @identity -> fakesink + +We just set it up so both ends of the queue are chained, and all is well. diff --git a/docs/random/omega/sched/walkthrough-72 b/docs/random/omega/sched/walkthrough-72 new file mode 100644 index 0000000000..6ea350485f --- /dev/null +++ b/docs/random/omega/sched/walkthrough-72 @@ -0,0 +1,39 @@ +72) + + [-pipeline---------------------------------------------------------------------------------------------] + ! [-bin-----------------------------] [-thread---------] ! + ! ! [--------] [---------] ! [------] [---------] [------] ! [--------] ! ! + ! ! !faksesrc! !identity1! ! !queue1! !identity2! !queue2! ! !fakesink! ! ! + ! ! ! src --- sink * src --- sink n src -- sink src -- sink src -- sink ! ! ! + ! ! [--------] [---------] ! [------] [---------] [------] ! [--------] ! ! + ! [---------------------------------] [----------------] ! + [------------------------------------------------------------------------------------------------------] + + +----- +Ideally, you'd end up with the following sub-pipelines + +pipeline: +fakesrc -> identity1 -> queue1 +queue1 -> identity2 -> queue2 + +thread: +queue2 -> fakesink + + +They'd be scheduled as following: + +fakesrc: passive chained, pulled by identity1 +identity1: loopfunc cothreaded, ENTRY +queue1:sink: passive chained, pushed by identity1 + +queue1:src: _get-based iteration, ENTRY +identity2: chained by queue1 +queue2:src: passively chained, pushed by identity2 + +queue2:sink: passively chained, pulled by fakesink +fakesink: loopfunc cothreaded, ENTRY + + +----- +Most likely, we'd end up with the following diff --git a/docs/random/omega/sched2 b/docs/random/omega/sched2 new file mode 100644 index 0000000000..208c60057c --- /dev/null +++ b/docs/random/omega/sched2 @@ -0,0 +1,13 @@ +Currently, when an element wants to push or pull, they call gst_pad_*. These functions then decide what +to do based on the selection of pointers in the pad. There are at least 2 options for each, plus some +header stuff to do various checking of flags. Unfortunately, because of the selection of pointers in +the pad, there is at least one case where scheduling has to be tricked, by providing a pointer to a +function with no body. + +What I propse is that these functionos be replaced with macros that call a function pointer directly. +The default functions (provided by GstPad) would be capable of chaining, that's about it. When a +schedule is formed, these get replaced with more specific functions, provided by GstBin or a subclass. + +In the chain case, the pad_push_func might even be replaced brute force with the chain function, since +they have the same prototype. In the cothreaded case, various functions would provide the ability to +switch, deal with bufpens, etc. diff --git a/docs/random/omega/scheduling b/docs/random/omega/scheduling new file mode 100644 index 0000000000..420019ded1 --- /dev/null +++ b/docs/random/omega/scheduling @@ -0,0 +1,63 @@ +0) definitions: + +All pads without further specifiers are assumed to belong to the element +in question. The pad's peer is always denoted with ->peer. If there's +question, pads will be prefixed with self-> and other-> as necessary. + +All elements in this document have at most one source and one sink pad, +called srcpad and sinkpad. Multi-pad cases are supposed to be simple +extrapolations except in a couple strange cases, to be covered elsewhere. + + +1) loop functions: + +A loop function will call gst_pad_pull(sinkpad), do something, and call +gst_pad_push(srcpad). + +gst_pad_pull first checks to see if there's a buffer in the pen. If not, +it calls that pad's pullfunc handler, passing it the peer pad. When that +finishes, we check again and return the buffer. If no buffer, we squawk. +The pullfunc handler simply causes a cothread switch to the peer pad's +context; + +gst_pad_push places the buffer in the peer pad's pen, and calls the local +pad's pushfunc. The pushfunc simply causes a switch to the peer pad's +context. + + +2) chain functions + +The loopfunc constructed around a chain function starts by finding all +sink pads. For each sink pad, it calls gst_pad_pull. This causes a +switch to the peer pad's context and a buffer to appear, which is +returned. The pad is the handed off to the chain function, which +presumably does some processing and calls gst_pad_push(), which causes a +context switch to that pad's peer. + + + +3) source + +The loopfunc must repeatedly call the srcpad's pull + + + + + +--------------------------------- --------------------------------- + srcpad | | sinkpad + | | + GstPad *peer; | <-p-> | GstPad *peer; + pointer to peer pad | | pointer to peer pad + | | + funcptr *pushfunc; | --s-> | funcptr *pushfunc; + causes switch to peer ctx | | element uses buffer in pen, + | | + funcptr *pullfunc; | <-s-- | funcptr *pullfunc; + element puts buffer in pen, | | causes switch to peer ctx + calls gst_pad_push() | | + | | + funcptr *pullregion func | <-s-- | funcptr *pullregionfunc; + element puts region in pen | | causes switch to peer ctx + calls gst_pad_push() | | +--------------------------------- --------------------------------- diff --git a/docs/random/omega/type-properties b/docs/random/omega/type-properties new file mode 100644 index 0000000000..99a29d5cfe --- /dev/null +++ b/docs/random/omega/type-properties @@ -0,0 +1,73 @@ +A type properties system might look like following: + + +1) Type definition includes the properties and their ranges: + +audio/mp3 + layer: 1 - 3 + bitrate: 8 - 320 + +audio/raw + format: bitfield (using asound.h definitions) + depth: 8 - 32 + rate: 4000 - 96000 + channels: 1 - n + interleave: boolean + +video/raw + format: 32-bit FOURCC + bpp: 1 - 32 + width: 1 - n + height: 1 - n + framerate: 32-bit float + +etc. + + +2) An element can specify what subtypes it can deal with by creating a list of property tables: + +mpg123: audio/mp3 + layer: 1 - 3 + bitrate: 8 - 320 + +osssink: + format: S8, S16, etc. + depth: 8 - 16 + rate: 8000 - 48000 + channels: 1 - 2 + interleave: true + +And you could list several of these, so for instance if the card only supports 8-bit at up to 22KHz in +mono, you can remove S8 from the above list and add a second entry: + +osssink: + format: S8 + depth: 8 + rate: 8000 - 22050 + channels: 1 + interleave: false (irrelevant) + +The obvious problem with these examples is that the rate isn't really 8000 - 48000, it's 8000, 11025, +16000, 22050, 44100, and 48000. However, we may be able to leave these to pad connect time. + + + + + +struct _type_definition { + char *mime_type; + + .... + + GData *properties; +} + +gst_type_add_property_int(_type *type,gchar *propname,int min,int max) { + struct _type_prop_int prop_int; + GQuark quark = g_quark_from_string(propname); + + prop_int->id = quark; + prop_int->min = min; + prop_int->max = max; + g_datalist_id_set_data(type->properties,quark,&prop_int); +}