From 5c5554622bc12ba6b0a2932312ac598549981b6e Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 17 Apr 2019 10:06:32 -0400 Subject: [PATCH] gitlab: Blacklist moved to the check testsuite directly --- gitlab/ci_template.yml | 201 +---------------------------------------- 1 file changed, 2 insertions(+), 199 deletions(-) diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml index c11d0786ae..a5803b7ae7 100644 --- a/gitlab/ci_template.yml +++ b/gitlab/ci_template.yml @@ -134,99 +134,9 @@ build nodebug fedora x86_64: GST_DEBUG_NO_COLOR: "true" CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/" GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true" - - # note the -b at the start of each line - # Can't comment inline sadly - # FIXME: get this into gst-validate-launcher KNOWN_ISSUES - # gstreamer.pipelines_parse_launch.delayed_link: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/345 - # gstreamer.gst_gstsystemclock.test_async_sync_interaction: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/346 - # gstreamer.gst_gstsystemclock.test_periodic_multi: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/347 - # gstreamer.gst_gstsystemclock.test_periodic_shot: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/348 - # elements_multisocketsink.test_sending_buffers_with_9_gstmemories: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/525 - # elements_multisocketsink.test_client_next_keyframe: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/516 - # flvmux.test_incrementing_timestamps: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/530 - # flvmux.test_video_caps_late: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/543 - # rtpbin.test_sender_eos: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/527 - # rtpbin.test_cleanup_recv: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/546 - # souphttpsrc.test_icy_stream: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/307 - # rtspserver.test_shared_udp: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/61 - # rtpsession.test_multiple_senders_roundrobin_rbs: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/548 - # dtls: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/811 - # mpegtsmux.test_align: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/844 - # shm.test_shm_live: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/847 - # splitmux.test_splitmuxsink_async: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/531 - # splitmux.test_splitmuxsrc_caps_change: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/547 - # splitmux.test_splitmuxsrc_sparse_streams: https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/issues/544 - # netsim.netsim_stress: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/849 - # nle_complex.test_one_expandable_another: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/55 - # nle_simple.test_simplest: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/57 - # ges_basic.test_ges_pipeline_change_state: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/58 - # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/62 - # gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_auto_transition: https://gitlab.freedesktop.org/gstreamer/gst-editing-services/issues/63 - # pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/221 - # elements_capsfilter.test_unfixed_downstream_caps: https://gitlab.freedesktop.org/gstreamer/gstreamer/issues/335 - # gst_rtspclientsink.test_record: https://gitlab.freedesktop.org/gstreamer/gst-rtsp-server/issues/55 - # elements_audiotestsrc.test_layout: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/535 - # camerabin.test_image_video_cycle: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864 - # camerabin.test_single_video_recording: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101558 - # camerabin.test_multiple_video_recordings: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/864#note_101646 - # curlhttpsrc.test_multiple_http_requests: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/932 - # audiomixer.test_flush_start_flush_stop: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/555 - # check.gstreamer-sharp.SdpTests: https://gitlab.freedesktop.org/gstreamer/gstreamer-sharp/issues/17 - # - # TO FURTHER INVESTIGATE: - # check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer - # check.gstreamer.gst_gstelement.test_foreach_pad - # check.gstreamer.libs_baseparse.parser_pull_short_read - # check.gstreamer.pipelines_seek.test_loopback_2 - # check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps - # check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s - BLACKLIST: > - -b check.gstreamer.pipelines_parse_launch.delayed_link - -b check.gstreamer.gst_gstsystemclock.test_async_sync_interaction - -b check.gstreamer.gst_gstsystemclock.test_periodic_multi - -b check.gstreamer.gst_gstsystemclock.test_periodic_shot - -b check.gstreamer.pipelines_seek.test_loopback_2 - -b check.gstreamer.gst_gstelement.test_foreach_pad - -b check.gstreamer.libs_baseparse.parser_pull_short_read - -b check.gst-plugins-base.elements_multisocketsink.test_sending_buffers_with_9_gstmemories - -b check.gst-plugins-base.elements_multisocketsink.test_client_next_keyframe - -b check.gst-plugins-base.elements_multisocketsink.test_add_client - -b check.gst-plugins-base.libs_gstglcolorconvert.test_reorder_buffer - -b check.gst-plugins-base.elements_audiotestsrc.test_layout - -b check.gst-plugins-good.elements_souphttpsrc.test_icy_stream - -b check.gst-plugins-good.elements_rtpbin.test_sender_eos - -b check.gst-plugins-good.elements_rtpbin.test_cleanup_recv - -b check.gst-plugins-good.elements_flvmux.test_incrementing_timestamps - -b check.gst-plugins-good.elements_flvmux.test_video_caps_late - -b check.gst-plugins-base.elements_appsrc.test_appsrc_blocked_on_caps - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_sparse_streams - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsrc_caps_change - -b check.gst-plugins-bad.elements_dtls.test_data_transfer - -b check.gst-plugins-bad.elements_dtls.test_create_and_unref - -b check.gst-plugins-bad.elements_camerabin.test_image_video_cycle - -b check.gst-plugins-bad.elements_camerabin.test_single_video_recording - -b check.gst-plugins-bad.elements_camerabin.test_multiple_video_recordings - -b check.gst-plugins-bad.elements_curlhttpsrc.test_multiple_http_requests - -b check.gst-plugins-good.elements_rtpsession.test_multiple_senders_roundrobin_rbs - -b check.gst-plugins-bad.elements_shm.test_shm_live - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - -b check.gst-plugins-bad.elements_netsim.netsim_stress - -b check.gst-editing-services.nle_complex.test_one_expandable_another - -b check.gst-editing-services.nle_simple.test_simplest - -b check.gst-editing-services.ges_basic.test_ges_pipeline_change_state - -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_transition_type - -b check.gst-editing-services.pythontests.pyunittest.python.test_timeline.TestTransitions.test_auto_transition - -b check.gst-plugins-base.pipelines_tcp.test_that_tcpserversink_and_tcpclientsrc_are_symmetrical - -b check.gstreamer.elements_capsfilter.test_unfixed_downstream_caps - -b check.gst-rtsp-server.gst_rtspclientsink.test_record - -b check.gst-rtsp-server.gst_rtspserver.test_shared_udp - -b check.gst-plugins-base.elements_audiomixer.test_flush_start_flush_stop - -b check.gstreamer-sharp.SdpTests - -b check.gst-devtools.validate.launcher_tests.test_validate.launch_pipeline.not_negotiated.caps_query_failure.play_15s script: - cd gst-build/ - - echo "-> Running ${TEST_SUITE} testsuite with ${BLACKLIST} and ${EXTRA_VALIDATE_ARGS}" + - echo "-> Running ${TEST_SUITE}" - > ./gst-uninstalled.py gst-validate-launcher ${TEST_SUITE} @@ -239,7 +149,6 @@ build nodebug fedora x86_64: --fail-on-testlist-change -l "${CI_PROJECT_DIR}/validate-logs/" --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml" - ${BLACKLIST} ${EXTRA_VALIDATE_ARGS} artifacts: expire_in: '14 days' @@ -291,113 +200,7 @@ integration testsuites fedora: extends: '.test fedora x86_64' stage: 'full tests' variables: - # These tests take very long compared to what they add, so let's skip them. - # - check.*.generic_states.* - enough to run one of the sequences - # - check.gstreamer.gst_gstelement.test_foreach_pad - 48s - # - check.gstreamer.gst_gstinfo.info_post_gst_init_category_registration - 21s - # - check.gstreamer.gst_gstsystemclock.test_resolution - 60s - # - check.gstreamer.libs_aggregator.test_infinite_seek - 20s - # - check.gstreamer.libs_aggregator.test_infinite_seek_50_src - 20s - # - check.gstreamer.libs_gstharness.test_harness_element_ref - 20s - # - check.gstreamer.pipelines_simple_launch_lines.test_2_elements - 58s - # - check.gstreamer.pipelines_stress.test_stress - 54s - # - check.gstreamer.pipelines_stress.test_stress_preroll - 27s - # - check.gst-plugins-base.elements_appsrc.test_appsrc_block_deadlock - 265.595s - # - check.gst-plugins-base.elements_audioresample.test_fft - 91.247s - # - check.gst-plugins-base.elements_audioresample.test_timestamp_drift - 141.784s - # - check.gst-plugins-base.elements-videoscale - superlong - # - check.gst-plugins-base.libs_video.test_overlay_blend - 74.096s - # - check.gst-plugins-base.libs_video.test_video_color_convert - 345.271s - # - check.gst-plugins-base.libs_video.test_video_formats - 70.987s - # - check.gst-plugins-base.libs_video.test_video_size_convert - 56.387s - # - check.gst-plugins-base.elements_audiointerleave.test_audiointerleave_2ch_pipeline_ - 5 * 51.069s - # - check.gst-plugins-base.elements_multifdsink.test_client_kick - 46.909s - # - check.gst-plugins-base.elements_videotestsrc.test_all_patterns - # - check.gst-plugins-base.elements_videotestsrc.test_patterns_are_deterministic - # - check.gst-plugins-good.elements_shapewipe.test_general - 325s - # - check.gst-plugins-good.elements_videocrop.test_cropping - 245s - # - check.gst-plugins-good.elements_videomixer - 30s (also deprecated) - # - check.gst-plugins-good.elements_rtp_payloading.rtp_jpeg_packet_loss - 109s - # - check.gst-plugins-good.elements_videomixer.test_play_twice_then_add_and_play_again - 55s - # - check.gst-plugins-good.pipelines_effectv.test_quarktv - 53s - # - check.gst-plugins-good.elements_deinterlace.test_mode_disabled_passthrough - 52s - # - check.gst-plugins-good.elements_deinterlace.test_mode_auto_deinterlaced_passthrough - 28s - # - check.gst-plugins-good.elements_deinterleave.test_2_channels_caps_change - 30s - # - check.gst-plugins-good.elements_deinterleave.test_2_channels - 22s - # - check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue - 22s - # - check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - 20s - # - check.gst-plugins-good.elements_videomixer.test_play_twice - 22s - # - check.gst-editing-services.nle_simple.test_one_after_other - 40s - VALGRIND_SKIPLIST: > - -b check.[a-z-]*.generic_states.test_state_changes_down_seq - -b check.[a-z-]*.generic_states.test_state_changes_up_seq - -b check.gstreamer.gst_gstelement.test_foreach_pad - -b check.gstreamer.gst_gstinfo.info_post_gst_init_category_registration - -b check.gstreamer.gst_gstsystemclock.test_resolution - -b check.gstreamer.libs_aggregator.test_infinite_seek - -b check.gstreamer.libs_aggregator.test_infinite_seek_50_src - -b check.gstreamer.libs_gstharness.test_harness_element_ref - -b check.gstreamer.pipelines_simple_launch_lines.test_2_elements - -b check.gstreamer.pipelines_stress.test_stress - -b check.gstreamer.pipelines_stress.test_stress_preroll - -b check.gst-plugins-base.elements_appsrc.test_appsrc_block_deadlock - -b check.gst-plugins-base.elements_audioresample.test_fft - -b check.gst-plugins-base.elements_audioresample.test_timestamp_drift - -b check.gst-plugins-base.elements-videoscale - -b check.gst-plugins-base.libs_video.test_overlay_blend - -b check.gst-plugins-base.libs_video.test_video_color_convert - -b check.gst-plugins-base.libs_video.test_video_formats - -b check.gst-plugins-base.libs_video.test_video_size_convert - -b check.gst-plugins-base.elements_audiointerleave.test_audiointerleave_2ch_pipeline_ - -b check.gst-plugins-base.elements_multifdsink.test_client_kick - -b check.gst-plugins-base.elements_videotestsrc.test_all_patterns - -b check.gst-plugins-base.elements_videotestsrc.test_patterns_are_deterministic - -b check.gst-plugins-good.elements_shapewipe.test_general - -b check.gst-plugins-good.elements_videocrop.test_cropping - -b check.gst-plugins-good.elements_videomixer - -b check.gst-plugins-good.elements_rtp_payloading.rtp_jpeg_packet_loss - -b check.gst-plugins-good.elements_videomixer.test_play_twice_then_add_and_play_again - -b check.gst-plugins-good.pipelines_effectv.test_quarktv - -b check.gst-plugins-good.elements_deinterlace.test_mode_disabled_passthrough - -b check.gst-plugins-good.elements_deinterlace.test_mode_auto_deinterlaced_passthrough - -b check.gst-plugins-good.elements_deinterleave.test_2_channels_caps_change - -b check.gst-plugins-good.elements_deinterleave.test_2_channels - -b check.gst-plugins-good.elements_rtpjitterbuffer.test_fill_queue - -b check.gst-plugins-good.elements_splitmux.test_splitmuxsink_async - -b check.gst-plugins-good.elements_videomixer.test_play_twice - -b check.gst-editing-services.nle_simple.test_one_after_other - # - check.gst-plugins-good.elements_rtpjitterbuffer.test_push_* - flaky in valgrind - # - check.gst-plugins-base.pipelines_gl_launch_lines - driver leaks / memory access - # - check.gst-plugins-base.libs_gstgl - driver leaks / memory access - # - check.gst-plugins-base.elements_gl - driver leaks / memory access - # - check.gst-plugins-base.elements_libvisual - uninitialized memory access - # - check.gst-plugins-base.generic_states - need to add gl elements to ignore list but only if using valgrind - # - "check.gst-editing-services.pythontests.*" - Need to figure out how to introduce python suppressions - VALGRIND_BLACKLIST: > - -b check.gstreamer.gst_gstsystemclock.test_stress_cleanup_unschedule - -b check.gstreamer.gst_gstsystemclock.test_stress_reschedule - -b check.gstreamer.tools_gstinspect - -b check.gst-plugins-base.elements_videoscale - -b check.gst-plugins-base.pipelines_gl_launch_lines - -b check.gst-plugins-base.libs_gstgl - -b check.gst-plugins-base.elements_gl - -b check.gst-plugins-base.elements_libvisual - -b check.gst-plugins-base.generic_states - -b check.gst-plugins-good.elements_rtpjitterbuffer.test_push_backward_seq - -b check.gst-plugins-good.elements_rtpjitterbuffer.test_push_unordered - -b check.gst-plugins-bad.elements_assrender - -b check.gst-plugins-bad.elements_camerabin - -b check.gst-plugins-bad.elements_line21 - -b check.gst-plugins-bad.elements_mpeg2enc - -b check.gst-plugins-bad.elements_mplex - -b check.gst-plugins-bad.elements_mxfmux - -b check.gst-plugins-bad.elements_x265enc - -b check.gst-plugins-bad.elements_zbar - -b check.gst-libav.generic_plugin_test - -b check.gst-libav.generic_libavcodec_locking - -b check.gst-libav.elements_avdemux_ape - -b check.gst-editing-services.pythontests - EXTRA_VALIDATE_ARGS: "--valgrind ${VALGRIND_BLACKLIST} ${VALGRIND_SKIPLIST}" + EXTRA_VALIDATE_ARGS: "--valgrind" ORC_CODE: "backup" # Some suppression files are missing a newline at the end which messes things # up when concatenating them. awk will add missing newlines (unlike cat)