From 072cf0122a3774e41f971b1d819282a4aa009c4d Mon Sep 17 00:00:00 2001
From: Jordan Petridis <jordan@centricular.com>
Date: Mon, 1 Jul 2019 18:55:46 +0300
Subject: [PATCH] docker/windows: refactorings to get it to work with the new
 runner

* Install git-lfs as its required now by gst-integration-suites
* Clone gst-build eache time to avoid dated gst-build checkouts
and overwritting .wrap files. Similar to !137
* Split the dockerfile and add a second run stage refresh the
powershell env inbetween calls
* Remove the msys2 workaround as its not needed anymore
---
 docker/windows/Dockerfile            |  7 ++--
 docker/windows/cleanup.ps1           |  0
 docker/windows/install_toolchain.ps1 | 52 ++++++++++++++++++++++++++++
 docker/windows/prepare.ps1           | 46 ------------------------
 docker/windows/prepare_gst_env.ps1   | 18 ++++++++++
 gitlab/ci_template.yml               | 25 ++++++++-----
 6 files changed, 92 insertions(+), 56 deletions(-)
 delete mode 100644 docker/windows/cleanup.ps1
 create mode 100644 docker/windows/install_toolchain.ps1
 delete mode 100644 docker/windows/prepare.ps1
 create mode 100644 docker/windows/prepare_gst_env.ps1

diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile
index 32eaccc152..e0b8244141 100644
--- a/docker/windows/Dockerfile
+++ b/docker/windows/Dockerfile
@@ -5,5 +5,8 @@ FROM 'mcr.microsoft.com/windows/servercore:1607'
 # Make sure any failure in PowerShell scripts is fatal
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"]
 
-COPY prepare.ps1 cleanup.ps1 C:\
-RUN C:\prepare.ps1 ; C:\cleanup.ps1
+COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\
+
+RUN C:\install_toolchain.ps1
+
+RUN C:\prepare_gst_env.ps1
\ No newline at end of file
diff --git a/docker/windows/cleanup.ps1 b/docker/windows/cleanup.ps1
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/docker/windows/install_toolchain.ps1 b/docker/windows/install_toolchain.ps1
new file mode 100644
index 0000000000..ac7a3ecc03
--- /dev/null
+++ b/docker/windows/install_toolchain.ps1
@@ -0,0 +1,52 @@
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+
+$python_dl_url = 'https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe'
+$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe'
+$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip'
+$zip_url = 'https://www.7-zip.org/a/7z1805-x64.exe'
+$msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz'
+
+Write-Host "Installing VisualStudio"
+Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe
+Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait
+Remove-Item C:\vs_buildtools.exe -Force
+
+Write-Host "Installing Python"
+Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3.exe
+Start-Process C:\python3.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait
+Remove-Item C:\python3.exe -Force
+
+Write-Host "Installing Git"
+Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip
+Expand-Archive C:\mingit.zip -DestinationPath c:\mingit
+Remove-Item C:\mingit.zip -Force
+$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd'
+[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine)
+
+Write-Host "Installing 7zip"
+Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe
+Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait
+Remove-Item C:\7z-x64.exe -Force
+
+Write-Host "Installing MSYS2"
+Invoke-WebRequest -Uri $msys_url -OutFile C:\msys2-x86_64.tar.xz
+C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait
+C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\"
+Remove-Item C:\msys2-x86_64.tar.xz -Force
+Remove-Item C:\msys2-x86_64.tar -Force
+Remove-Item C:\7zip -Recurse -Force
+
+Write-Host "Installing Choco"
+iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+refreshenv
+
+Write-Host "Installing git-lfs"
+choco install -y git-lfs
+refreshenv
+
+$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
+C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys"
+C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm"
+C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
+
+pip install meson
\ No newline at end of file
diff --git a/docker/windows/prepare.ps1 b/docker/windows/prepare.ps1
deleted file mode 100644
index 9bf2cfe71b..0000000000
--- a/docker/windows/prepare.ps1
+++ /dev/null
@@ -1,46 +0,0 @@
-[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
-
-Write-Host "Installing VisualStudio"
-Invoke-WebRequest -Uri 'https://aka.ms/vs/15/release/vs_buildtools.exe' -OutFile C:\vs_buildtools.exe
-Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait
-Remove-Item C:\vs_buildtools.exe -Force
-
-Write-Host "Installing Python"
-Invoke-WebRequest -Uri 'https://www.python.org/ftp/python/3.7.0/python-3.7.0.exe' -OutFile C:\python-3.7.0.exe
-Start-Process C:\python-3.7.0.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait
-Remove-Item C:\python-3.7.0.exe -Force
-
-Write-Host "Installing Git"
-Invoke-WebRequest -Uri 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip' -OutFile C:\mingit.zip
-Expand-Archive C:\mingit.zip -DestinationPath c:\mingit
-Remove-Item C:\mingit.zip -Force
-$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd'
-[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine)
-
-Write-Host "Installing 7zip"
-Invoke-WebRequest -Uri 'https://www.7-zip.org/a/7z1805-x64.exe' -OutFile C:\7z-x64.exe
-Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait
-Remove-Item C:\7z-x64.exe -Force
-
-Write-Host "Installing MSYS2"
-Invoke-WebRequest -Uri 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' -OutFile C:\msys2-x86_64.tar.xz
-C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait
-C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\"
-Remove-Item C:\msys2-x86_64.tar.xz -Force
-Remove-Item C:\msys2-x86_64.tar -Force
-Remove-Item C:\7zip -Recurse -Force
-
-# FIXME: This works but then docker fails to save the image. Needs to investigate why.
-#$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
-#C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys"
-#C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm"
-#C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
-
-pip install meson
-
-git config --global user.email "gst-build@gstreamer.net"
-git config --global user.name "Gstbuild Runner"
-
-# Download gst-build and all its subprojects
-git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
-meson subprojects download --sourcedir C:\gst-build
diff --git a/docker/windows/prepare_gst_env.ps1 b/docker/windows/prepare_gst_env.ps1
new file mode 100644
index 0000000000..833f7dceac
--- /dev/null
+++ b/docker/windows/prepare_gst_env.ps1
@@ -0,0 +1,18 @@
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+
+git config --global user.email "gst-build@gstreamer.net"
+git config --global user.name "Gstbuild Runner"
+
+# Download gst-build and all its subprojects
+git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
+
+# download the subprojects to try and cache them
+meson subprojects download --sourcedir C:\gst-build
+
+# Remove files that will conflict with a fresh clone on the runner side
+Remove-Item -Force 'C:/gst-build/subprojects/*.wrap'
+Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm'
+Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries'
+
+Move-Item C:\gst-build\subprojects C:\subprojects
+Remove-Item -Recurse -Force C:\gst-build
diff --git a/gitlab/ci_template.yml b/gitlab/ci_template.yml
index 197dd9977f..a3049dedb1 100644
--- a/gitlab/ci_template.yml
+++ b/gitlab/ci_template.yml
@@ -12,6 +12,7 @@ variables:
   FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578'
   INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb'
   MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e'
+  WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v6'
 
   # Branch to track for modules that have no ref specified in the manifest
   GST_UPSTREAM_BRANCH: 'master'
@@ -346,7 +347,7 @@ valgrind ges:
       - "*.tar.bz2"
 
 .build windows:
-  image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v4'
+  image: $WINDOWS_IMAGE
   stage: 'build'
   dependencies:
     - 'manifest'
@@ -362,6 +363,9 @@ valgrind ges:
       -Dvaapi=disabled
       -Ddevtools=disabled
   script:
+    - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build
+    - cd $env:CI_PROJECT_DIR/gst-build
+    - cp -r C:/subprojects/* subprojects/
     # For some reason, options are separated by newline instead of space, so we
     # have to replace them first.
     - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
@@ -369,12 +373,17 @@ valgrind ges:
     # Environment variables substitutions is done by PowerShell before calling
     # cmd.exe, that's why we use $env:FOO instead of %FOO%
     - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH &&
-        cd C:\gst-build &&
         python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml &&
         meson build $env:MESON_ARGS &&
         ninja -C build"
+  # FIXME: extract builddir for tests
+  # There's a bug that prevents us from exporting artifacts with docker-windows
+  # executors. It has since been fixed in gitlab 12.1, but
+  # we are blocked from upgrading currently.
+  #
+  # Gitlab Runner issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/4291
+  # Blocked upgrade issue: https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/6#note_192780
   after_script:
-    # FIXME: cleanup build dir for artifacts.
 
 build vs2017 amd64:
   extends: '.build windows'
@@ -389,15 +398,15 @@ build vs2017 x86:
 build msys2 :
   extends: '.build windows'
   script:
+    - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build
+    - cd $env:CI_PROJECT_DIR/gst-build
+    - cp -r C:/subprojects/* subprojects/
+
     - $env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
-    # FIXME: This should be done in Dockerfile, but doesn't work there. Needs to investigate why.
-    - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true"
-    - C:\msys64\usr\bin\bash -c "pacman -Syu --noconfirm"
-    - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
     # For some reason, options are separated by newline instead of space, so we
     # have to replace them first.
     - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
-    - C:\msys64\usr\bin\bash -c "cd /c/gst-build &&
+    - C:\msys64\usr\bin\bash -c "cd $env:CI_PROJECT_DIR/gst-build &&
         python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml &&
         meson build $env:MESON_ARGS &&
         ninja -C build"