From 15b0b49918fee60c8b2cc8950261e7a5b4f2a9eb Mon Sep 17 00:00:00 2001
From: Nirbheek Chauhan <nirbheek@centricular.com>
Date: Fri, 18 Oct 2024 13:14:22 +0530
Subject: [PATCH] ci: Add scripts to retry windows builds on spurious errors

Similar to cerbero, we run meson commands inside a powershell script
that will examine the output for spurious errors and re-run that
particular command.

https://gitlab.freedesktop.org/slomo/gstreamer/-/jobs/65265526

https://gitlab.freedesktop.org/slomo/gstreamer/-/jobs/65265524

https://gitlab.freedesktop.org/nirbheek/gstreamer/-/jobs/65331410

https://gitlab.freedesktop.org/jcowgill/gstreamer/-/jobs/65489856

https://github.com/rust-lang/rust/issues/127883#issuecomment-2290594194

Co-Authored-by: L. E. Segovia <amy@centricular.com>
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7680>
---
 .gitlab-ci.yml   | 12 +++++-----
 ci/run_retry.ps1 | 58 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 ci/run_retry.ps1

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7107a36605..81cc3e07dc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -527,8 +527,8 @@ build vs2022 amd64:
   extends: '.build windows'
   script:
     - !reference [".build windows", "script"]
-    - meson setup --vsenv build $env:MESON_ARGS $env:GTK_ARGS
-    - meson compile -C build --jobs $env:FDO_CI_CONCURRENT
+    - .\ci\run_retry.ps1 meson setup --vsenv build $env:MESON_ARGS $env:GTK_ARGS
+    - .\ci\run_retry.ps1 meson compile -C build --jobs $env:FDO_CI_CONCURRENT
     - .\gst-env.py gst-inspect-1.0.exe --version
     - .\gst-env.py gst-inspect-1.0.exe
     - mkdir destdir
@@ -543,7 +543,7 @@ build vs2022 amd64 full-static:
   script:
     - !reference [".build windows", "script"]
     - cmd.exe /C "meson setup --vsenv build --default-library=static $env:MESON_ARGS"
-    - meson compile -C build --jobs $env:FDO_CI_CONCURRENT
+    - .\ci\run_retry.ps1 meson compile -C build --jobs $env:FDO_CI_CONCURRENT
     - .\gst-env.py gst-inspect-1.0.exe --version
     - .\gst-env.py gst-inspect-1.0.exe
   variables:
@@ -561,8 +561,10 @@ build vs2022 amd64 full-static:
       # Setting up a cross build with MSVC is still non-trivial because
       # the --vsenv argument cannot be used to set it up
     - echo $env:MESON_CROSS_ARGS
-    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson setup build $env:MESON_ARGS $env:GTK_ARGS $env:MESON_CROSS_ARGS"
-    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH && meson compile -C build --jobs $env:FDO_CI_CONCURRENT"
+    - $env:JOB="meson setup build $env:MESON_ARGS $env:GTK_ARGS $env:MESON_CROSS_ARGS"
+    - cmd.exe /C """C:\BuildTools\VC\Auxiliary\Build\vcvarsamd64_$env:ARCH.bat""  && powershell.exe -ExecutionPolicy Bypass -File .\ci\run_retry.ps1"
+    - $env:JOB="meson compile -C build --jobs $env:FDO_CI_CONCURRENT"
+    - cmd.exe /C """C:\BuildTools\VC\Auxiliary\Build\vcvarsamd64_$env:ARCH.bat"" && powershell.exe -ExecutionPolicy Bypass -File .\ci\run_retry.ps1"
 
 build vs2022 x86:
   extends: '.build windows cross'
diff --git a/ci/run_retry.ps1 b/ci/run_retry.ps1
new file mode 100644
index 0000000000..edf9f6b069
--- /dev/null
+++ b/ci/run_retry.ps1
@@ -0,0 +1,58 @@
+# Set error action preference to stop on errors
+$ErrorActionPreference = "Stop"
+
+# Define error patterns to catch
+$ERRORS = @(
+    'fatal error C1060: compiler is out of heap space'
+    'cc1plus.exe: out of memory allocating'
+    'cl : Command line error D8027 : cannot execute'
+    'Access is denied (os error 5)'
+    'LINK : fatal error LNK1104: cannot open file'
+    'PermissionError: [Errno 13] Permission denied:'
+    'c1: fatal error C1356: unable to find mspdbcore.dll'
+)
+$RETRIES = 3
+$LOGFILE = [System.IO.Path]::GetTempPath() + "logfile.txt"
+
+while ($true) {
+    $spurious_error = ""
+
+    # Execute command and capture output to log file while displaying it
+    if ($env:JOB) {
+        # Bypass argument parsing -- https://github.com/PowerShell/PowerShell/issues/19451
+        & iex $env:JOB | Tee-Object -FilePath $LOGFILE *>&1
+    } else {
+        & $args[0] $args[1..($args.Count-1)] | Tee-Object -FilePath $LOGFILE *>&1
+    }
+    $ret = $LASTEXITCODE
+
+    if ($ret -eq 0) {
+        break
+    }
+
+    # Read log file and check for known errors
+    foreach ($line in Get-Content $LOGFILE) {
+        foreach ($e in $ERRORS) {
+            if ($line -match [regex]::Escape($e)) {
+                $spurious_error = $line
+                break
+            }
+        }
+        if ($spurious_error) {
+            break
+        }
+    }
+
+    # Clean up log file
+    if (Test-Path $LOGFILE) {
+        Remove-Item -Force $LOGFILE
+    }
+
+    # Exit if no spurious error found or no retries left
+    if ([string]::IsNullOrEmpty($spurious_error) -or $RETRIES -eq 0) {
+        exit $ret
+    }
+
+    $RETRIES--
+    Write-Host "`nRetrying, caught spurious failure: $spurious_error`n"
+}