docker/windows: refactorings to get it to work with the new runner

* Install git-lfs as its required now by gst-integration-suites * Clone gst-build eache time to avoid dated gst-build checkouts and overwritting .wrap files. Similar to !137 * Split the dockerfile and add a second run stage refresh the powershell env inbetween calls * Remove the msys2 workaround as its not needed anymore
2024-11-20 00:31:13 +00:00 · 2019-07-01 18:55:46 +03:00 · 2019-07-01 18:55:46 +03:00 · 072cf0122a
commit 072cf0122a
parent 31cec0a245
6 changed files with 92 additions and 56 deletions
--- a/docker/windows/Dockerfile
+++ b/docker/windows/Dockerfile
@ -5,5 +5,8 @@ FROM 'mcr.microsoft.com/windows/servercore:1607'
 # Make sure any failure in PowerShell scripts is fatal
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"]

-COPY prepare.ps1 cleanup.ps1 C:\
-RUN C:\prepare.ps1 ; C:\cleanup.ps1
+COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\
+
+RUN C:\install_toolchain.ps1
+
+RUN C:\prepare_gst_env.ps1
--- a/docker/windows/cleanup.ps1
+++ b/docker/windows/cleanup.ps1
--- a/docker/windows/install_toolchain.ps1
+++ b/docker/windows/install_toolchain.ps1
@ -0,0 +1,52 @@
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+
+$python_dl_url = 'https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe'
+$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe'
+$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip'
+$zip_url = 'https://www.7-zip.org/a/7z1805-x64.exe'
+$msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz'
+
+Write-Host "Installing VisualStudio"
+Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe
+Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait
+Remove-Item C:\vs_buildtools.exe -Force
+
+Write-Host "Installing Python"
+Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3.exe
+Start-Process C:\python3.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait
+Remove-Item C:\python3.exe -Force
+
+Write-Host "Installing Git"
+Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip
+Expand-Archive C:\mingit.zip -DestinationPath c:\mingit
+Remove-Item C:\mingit.zip -Force
+$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd'
+[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine)
+
+Write-Host "Installing 7zip"
+Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe
+Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait
+Remove-Item C:\7z-x64.exe -Force
+
+Write-Host "Installing MSYS2"
+Invoke-WebRequest -Uri $msys_url -OutFile C:\msys2-x86_64.tar.xz
+C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait
+C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\"
+Remove-Item C:\msys2-x86_64.tar.xz -Force
+Remove-Item C:\msys2-x86_64.tar -Force
+Remove-Item C:\7zip -Recurse -Force
+
+Write-Host "Installing Choco"
+iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+refreshenv
+
+Write-Host "Installing git-lfs"
+choco install -y git-lfs
+refreshenv
+
+$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
+C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys"
+C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm"
+C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
+
+pip install meson
--- a/docker/windows/prepare.ps1
+++ b/docker/windows/prepare.ps1
@ -1,46 +0,0 @@
-[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
-
-Write-Host "Installing VisualStudio"
-Invoke-WebRequest -Uri 'https://aka.ms/vs/15/release/vs_buildtools.exe' -OutFile C:\vs_buildtools.exe
-Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait
-Remove-Item C:\vs_buildtools.exe -Force
-
-Write-Host "Installing Python"
-Invoke-WebRequest -Uri 'https://www.python.org/ftp/python/3.7.0/python-3.7.0.exe' -OutFile C:\python-3.7.0.exe
-Start-Process C:\python-3.7.0.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait
-Remove-Item C:\python-3.7.0.exe -Force
-
-Write-Host "Installing Git"
-Invoke-WebRequest -Uri 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip' -OutFile C:\mingit.zip
-Expand-Archive C:\mingit.zip -DestinationPath c:\mingit
-Remove-Item C:\mingit.zip -Force
-$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd'
-[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine)
-
-Write-Host "Installing 7zip"
-Invoke-WebRequest -Uri 'https://www.7-zip.org/a/7z1805-x64.exe' -OutFile C:\7z-x64.exe
-Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait
-Remove-Item C:\7z-x64.exe -Force
-
-Write-Host "Installing MSYS2"
-Invoke-WebRequest -Uri 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz' -OutFile C:\msys2-x86_64.tar.xz
-C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait
-C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\"
-Remove-Item C:\msys2-x86_64.tar.xz -Force
-Remove-Item C:\msys2-x86_64.tar -Force
-Remove-Item C:\7zip -Recurse -Force
-
-# FIXME: This works but then docker fails to save the image. Needs to investigate why.
-#$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
-#C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys"
-#C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm"
-#C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
-
-pip install meson
-
-git config --global user.email "gst-build@gstreamer.net"
-git config --global user.name "Gstbuild Runner"
-
-# Download gst-build and all its subprojects
-git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
-meson subprojects download --sourcedir C:\gst-build
--- a/docker/windows/prepare_gst_env.ps1
+++ b/docker/windows/prepare_gst_env.ps1
@ -0,0 +1,18 @@
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+
+git config --global user.email "gst-build@gstreamer.net"
+git config --global user.name "Gstbuild Runner"
+
+# Download gst-build and all its subprojects
+git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
+
+# download the subprojects to try and cache them
+meson subprojects download --sourcedir C:\gst-build
+
+# Remove files that will conflict with a fresh clone on the runner side
+Remove-Item -Force 'C:/gst-build/subprojects/*.wrap'
+Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm'
+Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries'
+
+Move-Item C:\gst-build\subprojects C:\subprojects
+Remove-Item -Recurse -Force C:\gst-build
--- a/gitlab/ci_template.yml
+++ b/gitlab/ci_template.yml
@ -12,6 +12,7 @@ variables:
  FEDORA_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2019-05-29-324578'
  INDENT_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/gst-indent:6f7e01e1e30a73efa880acdc8e911f1f20c58dbb'
  MANIFEST_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/build-manifest:d19082b72667fb3382bdc3621520c4d26e258b2e'
+  WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v6'

  # Branch to track for modules that have no ref specified in the manifest
  GST_UPSTREAM_BRANCH: 'master'
@ -346,7 +347,7 @@ valgrind ges:
      - "*.tar.bz2"

 .build windows:
-  image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v4'
+  image: $WINDOWS_IMAGE
  stage: 'build'
  dependencies:
    - 'manifest'
@ -362,6 +363,9 @@ valgrind ges:
      -Dvaapi=disabled
      -Ddevtools=disabled
  script:
+    - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build
+    - cd $env:CI_PROJECT_DIR/gst-build
+    - cp -r C:/subprojects/* subprojects/
    # For some reason, options are separated by newline instead of space, so we
    # have to replace them first.
    - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
@ -369,12 +373,17 @@ valgrind ges:
    # Environment variables substitutions is done by PowerShell before calling
    # cmd.exe, that's why we use $env:FOO instead of %FOO%
    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH &&
-        cd C:\gst-build &&
        python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml &&
        meson build $env:MESON_ARGS &&
        ninja -C build"
+  # FIXME: extract builddir for tests
+  # There's a bug that prevents us from exporting artifacts with docker-windows
+  # executors. It has since been fixed in gitlab 12.1, but
+  # we are blocked from upgrading currently.
+  #
+  # Gitlab Runner issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/4291
+  # Blocked upgrade issue: https://gitlab.freedesktop.org/gstreamer/gst-ci/issues/6#note_192780
  after_script:
-    # FIXME: cleanup build dir for artifacts.

 build vs2017 amd64:
  extends: '.build windows'
@ -389,15 +398,15 @@ build vs2017 x86:
 build msys2 :
  extends: '.build windows'
  script:
+    - git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git $env:CI_PROJECT_DIR/gst-build
+    - cd $env:CI_PROJECT_DIR/gst-build
+    - cp -r C:/subprojects/* subprojects/
+
    - $env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
-    # FIXME: This should be done in Dockerfile, but doesn't work there. Needs to investigate why.
-    - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true"
-    - C:\msys64\usr\bin\bash -c "pacman -Syu --noconfirm"
-    - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
    # For some reason, options are separated by newline instead of space, so we
    # have to replace them first.
    - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
-    - C:\msys64\usr\bin\bash -c "cd /c/gst-build &&
+    - C:\msys64\usr\bin\bash -c "cd $env:CI_PROJECT_DIR/gst-build &&
        python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml &&
        meson build $env:MESON_ARGS &&
        ninja -C build"