Merge pull request #4466 from hummeltech/GitHubActionsVCPkgCache

Speed up the GitHub Actions CI jobs

.github/actions/run_tests/action.yml

@@ -1,93 +1,85 @@
-name: "Run tests with coverage"
-description: "Runs all mapnik tests with coverage"
+name: Run tests with coverage
+description: Runs all mapnik tests with coverage
 inputs:
   cmake-preset:
-    description: "the used cmake preset"
+    description: The used CMake preset
     required: true
+
 runs:
-  using: "composite"
+  using: composite
   steps:
-    - name: Set proj enviroment
-      shell: "pwsh"
+    - name: Set PROJ_LIB
+      shell: bash
       run: |
-        $out = cmake --preset ${{ inputs.cmake-preset }} -N -L
-        $proj_lib = $out -match "PROJ_LIB=*"
-        echo ("PROJ_LIB=" + $proj_lib.Substring(11)) >> $env:GITHUB_ENV
+        cmake --preset ${{ inputs.cmake-preset }} -N -L | grep -o "PROJ_LIB=.*" >> ${GITHUB_ENV}
+
+    - name: Set TEST_WRAPPER (Windows)
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |-
+          echo "TEST_WRAPPER=OpenCppCoverage \
+          --cover_children \
+          --export_type binary \
+          --modules '*.input' \
+          --modules '*libmapnik*' \
+          --modules 'mapnik*.exe' \
+          --quiet \
+          --sources '${{ github.workspace }}' \
+          --" >> ${GITHUB_ENV}
 
     - name: Test
-      shell: "bash"
+      shell: bash
       env:
-        UPDATE: "1"
+        UPDATE: 1
       run: |
-        if [ "$RUNNER_OS" == "Windows" ]; then
-          OpenCppCoverage --modules *libmapnik* --modules mapnik*.exe  --modules *.input --sources ${{ github.workspace }} --export_type binary --cover_children -- ctest --preset ${{ inputs.cmake-preset }}
-        else
-          ctest --preset ${{ inputs.cmake-preset }}
-        fi
+        ${TEST_WRAPPER:-} ctest --preset ${{ inputs.cmake-preset }}
 
-    - name: Test visuals (windows)
+    - name: Test visuals
       continue-on-error: true
       working-directory: build/out
-      shell: "pwsh"
-      if: runner.os == 'Windows'
-      run: OpenCppCoverage --modules *libmapnik* --modules mapnik*.exe  --modules *.input --sources ${{ github.workspace }} --export_type binary --input_coverage=${{ github.workspace }}/ctest.cov --cover_children -- .\mapnik-test-visual.exe -j (Get-CimInstance -ClassName Win32_ComputerSystem).NumberOfLogicalProcessors --output-dir ./visual-test-result
-
-    - name: Test visuals (linux & mac)
-      continue-on-error: true
-      working-directory: build/out
-      shell: "bash"
-      if: runner.os != 'Windows'
+      shell: bash
       run: |
-        if [ "$RUNNER_OS" == "Linux" ]; then
-          ./mapnik-test-visual -j $(nproc) --output-dir ./visual-test-result
-        else
-          ./mapnik-test-visual -j $(sysctl -n hw.logicalcpu) --output-dir ./visual-test-result
-        fi
+        ${TEST_WRAPPER:-} ./mapnik-test-visual -j ${CTEST_PARALLEL_LEVEL} --output-dir visual-test-result
 
     - name: Pack visual test results
       working-directory: build/out
-      shell: "pwsh"
-      run: tar cfvz visual-test-results.tar.gz ./visual-test-result
-
-    - name: Generate run guid
-      id: run-guid
-      shell: "pwsh"
+      shell: bash
       run: |
-        $guid = New-Guid
-        echo ("GUID=" + $guid.toString()) >> $env:GITHUB_OUTPUT
+        tar -vzcf visual-test-results.tar.gz visual-test-result
 
     - name: Upload visual test results
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ inputs.cmake-preset }}-visual-tests-${{ steps.run-guid.outputs.GUID }}
+        name: ${{ inputs.cmake-preset }}-visual-tests-${{ github.sha }}
         path: build/out/visual-test-results.tar.gz
 
-    - name: Run Benchmarks
+    - name: Run Benchmarks (Linux & macOS)
       working-directory: build/out
       if: runner.os != 'Windows'
-      shell: "pwsh"
-      run: ./run_benchmarks
+      shell: bash
+      run: |
+        ./run_benchmarks
 
-    - name: Collect coverage (linux & macos)
+    - name: Collect coverage (Linux & macOS)
       working-directory: build
       if: runner.os != 'Windows'
-      shell: "bash"
+      shell: bash
       run: |
-        if [ "$RUNNER_OS" == "macOS" ]; then
+        if [ "${RUNNER_OS}" == "macOS" ]; then
           LCOV_EXTRA_OPTIONS="--ignore-errors count,gcov,inconsistent,range,unused --keep-going"
         fi
         lcov ${LCOV_EXTRA_OPTIONS:-} --directory . --capture --output-file coverage.info
         lcov ${LCOV_EXTRA_OPTIONS:-} --remove coverage.info '/usr/*' '*/vcpkg_installed/*'  '/.cache/*' '*/test/*'  --output-file coverage.info
         lcov ${LCOV_EXTRA_OPTIONS:-} --list coverage.info
 
-    - name: Upload coverage to Codecov (linux & macos)
+    - name: Upload coverage to Codecov (Linux & macOS)
       if: runner.os != 'Windows'
       uses: codecov/codecov-action@v4
       with:
         files: build/coverage.info
 
-    - name: Upload coverage to Codecov (windows)
+    - name: Upload coverage to Codecov (Windows)
       if: runner.os == 'Windows'
       uses: codecov/codecov-action@v4
       with:
-        files: build/out/mapnik-test-visual.cov
+        files: ctest.cov,build/out/mapnik-test-visual.cov

.github/actions/setup_vcpkg/action.yml

@@ -1,50 +0,0 @@
-name: "Setup vcpkg"
-description: "Sets up vcpkg"
-inputs:
-  vcpkg-sha:
-    description: "vcpkg git sha to use"
-    required: true
-  nuget-source:
-    description: "The nuget json"
-    required: true
-  nuget-username:
-    description: "The username for the nuget repository"
-    required: true
-  nuget-pat:
-    description: "The PAT for the nuget repository"
-    required: true
-  mono:
-    description: "mono exec"
-    required: true
-
-
-runs:
-  using: "composite"
-  steps:
-    - name: checkout vcpkg
-      uses: actions/checkout@v4
-      with:
-        repository: "microsoft/vcpkg"
-        ref: ${{ inputs.vcpkg-sha }}
-        path: vcpkg
-
-    - name: "Setup vcpkg"
-      shell: bash
-      run: ./vcpkg/bootstrap-vcpkg.sh
-
-    - name: "Setup NuGet Credentials"
-      shell: "bash"
-      run: >
-        ${{ inputs.mono }} `./vcpkg/vcpkg fetch nuget | tail -n 1`
-        sources add
-        -source "${{ inputs.nuget-source }}"
-        -storepasswordincleartext
-        -name "GitHub"
-        -username "${{ inputs.nuget-username }}"
-        -password "${{ inputs.nuget-pat }}"
-
-    - name: "Setup NuGet apikey"
-      shell: "bash"
-      run: >
-        ${{ inputs.mono }} `./vcpkg/vcpkg fetch nuget | tail -n 1`
-        setapikey "${{ inputs.nuget-pat }}" -Source "${{ inputs.nuget-source }}"

.github/workflows/build_and_test.yml

@@ -9,8 +9,8 @@ on:
       - "no-ci-*"
 
 env:
-  VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
-  VCPKG_NUGET_REPOSITORY: https://github.com/mathisloge/vcpkg-nuget.git
+  VCPKG_BINARY_SOURCES: clear;x-gha,readwrite
+  VCPKG_RELEASE: 2024.06.15
 
 jobs:
   checkSource:
@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: "3.10"
       - uses: pre-commit/action@v3.0.1
 
   buildAndTest:
@@ -31,6 +31,7 @@ jobs:
       (C++ ${{ matrix.cxx-standard }})
       ${{ startsWith(matrix.os, 'macos-') && (matrix.os == 'macos-14' && '(ARM64)' || '(AMD64)') || '' }}
     needs: checkSource
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
@@ -40,74 +41,94 @@ jobs:
           - windows-2022
         cxx-standard:
           - 17
-        include:
-          - os: macos-14
-            mono: mono
-          - os: ubuntu-22.04
-            mono: mono
-          - os: windows-2022
-
-    runs-on: ${{ matrix.os }}
 
     steps:
-      - name: "Install required system packages"
-        shell: bash
-        run: |
-          if [ "$RUNNER_OS" == "Windows" ]; then
-            choco install ninja OpenCppCoverage
-            echo "C:\Program Files\OpenCppCoverage" >> ${GITHUB_PATH}
-          elif [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt update
-            sudo apt install -y gperf libxxf86vm-dev ninja-build postgresql-client lcov autoconf-archive
-          else
-            brew install automake ninja lcov autoconf-archive
-          fi
-
-      - uses: ilammy/msvc-dev-cmd@v1
-        if: runner.os == 'Windows'
-
-      - name: checkout mapnik
+      - name: Checkout Mapnik
         uses: actions/checkout@v4
         with:
-          submodules: "recursive"
+          submodules: recursive
 
-      - name: setup vcpkg
-        uses: ./.github/actions/setup_vcpkg
+      - name: Checkout vcpkg
+        uses: actions/checkout@v4
         with:
-          vcpkg-sha: ad25766aefb5313b6bc4e2a4b78a2946f84fbf66
-          nuget-source: https://nuget.pkg.github.com/mapnik/index.json
-          nuget-username: ${{ github.actor }}
-          nuget-pat: ${{ secrets.GITHUB_TOKEN }}
-          mono: ${{ matrix.mono }}
+          path: vcpkg
+          ref: ${{ env.VCPKG_RELEASE }}
+          repository: microsoft/vcpkg
 
-      - name: Set PRESET environment variable to lower case runner OS
+      - name: Export GitHub Actions cache environment variables
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Install required system packages
         shell: bash
         run: |
-          LC_RUNNER_OS=$(echo "${RUNNER_OS}" | perl -ne "print lc")
-          if [ "${RUNNER_OS}" == "macOS" ]; then
-            MACOS_ARCH=${{ matrix.os == 'macos-14' && 'arm64' || 'x64' }}
-            echo "PRESET=${LC_RUNNER_OS}-ci-${MACOS_ARCH}" >> ${GITHUB_ENV}
-          else
-            echo "PRESET=${LC_RUNNER_OS}-ci" >> ${GITHUB_ENV}
+          if [ "${RUNNER_OS}" == "Linux" ]; then
+            sudo apt-get update
+            sudo apt-get -y install \
+              autoconf \
+              autoconf-archive \
+              automake \
+              gperf \
+              lcov \
+              libxxf86vm-dev \
+              ninja-build \
+              postgresql-client
+          elif [ "${RUNNER_OS}" == "macOS" ]; then
+            brew install \
+              autoconf \
+              autoconf-archive \
+              automake \
+              lcov \
+              libtool \
+              ninja \
+              vcpkg
+          elif [ "${RUNNER_OS}" == "Windows" ]; then
+            choco install \
+              ninja \
+              OpenCppCoverage
+            echo "C:\Program Files\OpenCppCoverage" >> ${GITHUB_PATH}
           fi
 
-      - name: Ensure libtool is installed on macOS
+      - name: Enable Developer Command Prompt (Windows)
+        uses: ilammy/msvc-dev-cmd@v1
+        if: runner.os == 'Windows'
+
+      - name: Set CMAKE_BUILD_PARALLEL_LEVEL, CTEST_PARALLEL_LEVEL & PRESET
         shell: bash
-        run: brew install libtool
-        if: runner.os == 'macOS'
+        run: |
+          PRESET=$(echo "${RUNNER_OS}" | perl -ne "print lc")-ci
+          if [ "${RUNNER_OS}" == "Linux" ]; then
+            echo "CMAKE_BUILD_PARALLEL_LEVEL=$(nproc)" >> ${GITHUB_ENV}
+            echo "CTEST_PARALLEL_LEVEL=$(nproc)" >> ${GITHUB_ENV}
+          elif [ "${RUNNER_OS}" == "macOS" ]; then
+            echo "CMAKE_BUILD_PARALLEL_LEVEL=$(sysctl -n hw.logicalcpu)" >> ${GITHUB_ENV}
+            echo "CTEST_PARALLEL_LEVEL=$(sysctl -n hw.logicalcpu)" >> ${GITHUB_ENV}
+            PRESET=${PRESET}-${{ matrix.os == 'macos-14' && 'arm64' || 'x64' }}
+          elif [ "${RUNNER_OS}" == "Windows" ]; then
+            echo "CMAKE_BUILD_PARALLEL_LEVEL=$(pwsh -Command '(Get-CimInstance -ClassName Win32_ComputerSystem).NumberOfLogicalProcessors')" >> ${GITHUB_ENV}
+            echo "CTEST_PARALLEL_LEVEL=$(pwsh -Command '(Get-CimInstance -ClassName Win32_ComputerSystem).NumberOfLogicalProcessors')" >> ${GITHUB_ENV}
+          fi
+          echo "PRESET=${PRESET}" >> ${GITHUB_ENV}
 
       - name: Configure CMake
         shell: bash
         run: |
           cmake \
-            -DBUILD_SHARED_LIBS:BOOL='ON' \
-            -DCMAKE_CXX_STANDARD=${{ matrix.cxx-standard }}  \
-            -DUSE_MEMORY_MAPPED_FILE:BOOL='ON' \
+            -DBUILD_SHARED_LIBS:BOOL=ON \
+            -DCMAKE_CXX_STANDARD:STRING=${{ matrix.cxx-standard }}  \
+            -DUSE_MEMORY_MAPPED_FILE:BOOL=ON \
             -LA \
-            --preset ${{ env.PRESET }}
+            --preset ${PRESET}
 
       - name: Build
-        run: cmake --build --preset ${{ env.PRESET }}
+        shell: bash
+        run: |
+          cmake \
+            --build \
+            --preset ${PRESET}
 
       - name: Run Tests
         uses: ./.github/actions/run_tests