Update UPDATE_DEPS

.github/workflows/ci.yml

@@ -29,22 +29,18 @@ jobs:
         uses: lukka/get-cmake@latest
         with:
           cmakeVersion: 3.17.2
-      - name: Configure
+      - run: cmake -S. -B build -D VUL_WERROR=ON -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=${{matrix.config}} -D UPDATE_DEPS=ON -D UPDATE_DEPS_DIR=ext/
-        run: cmake -S. -B build -D VUL_WERROR=ON -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=${{matrix.config}} -D UPDATE_DEPS=ON
+      - run: cmake --build build --config ${{matrix.config}} --verbose
-      - name: Build
+      - working-directory: ./build
-        run: cmake --build build --config ${{matrix.config}} --verbose
-      - name: Tests
-        working-directory: ./build
         run: ctest -C ${{matrix.config}} --output-on-failure
-      - name: Install
+      - run: cmake --install build --prefix ${{ github.workspace }}/install --config ${{matrix.config}}
-        run: cmake --install build --prefix ${{ github.workspace }}/install --config ${{matrix.config}}
       - name: Test find_package support
         run: |
-          cmake -S tests/find_package -B tests/find_package/build -D CMAKE_PREFIX_PATH="${{ github.workspace }}/install;${{ github.workspace }}/external/${{matrix.config}}/Vulkan-Headers/build/install" -D CMAKE_BUILD_TYPE=${{matrix.config}}
+          cmake -S tests/find_package -B tests/find_package/build -D CMAKE_PREFIX_PATH="${{ github.workspace }}/install;${{ github.workspace }}/ext/Vulkan-Headers/build/install" -D CMAKE_BUILD_TYPE=${{matrix.config}}
           cmake --build tests/find_package/build --config ${{matrix.config}} --verbose
       - name: Test add_subdirectory support
         run: |
-          cmake -S tests/add_subdirectory -B tests/add_subdirectory/build -D CMAKE_BUILD_TYPE=${{matrix.config}} -D VULKAN_HEADER_SOURCE_DIR=${{ github.workspace }}/external/${{matrix.config}}/Vulkan-Headers/
+          cmake -S tests/add_subdirectory -B tests/add_subdirectory/build -D CMAKE_BUILD_TYPE=${{matrix.config}} -D VULKAN_HEADER_SOURCE_DIR=${{ github.workspace }}/ext/Vulkan-Headers/
           cmake --build tests/add_subdirectory/build --config ${{matrix.config}} --verbose
 
   android:

scripts/CMakeLists.txt

@@ -44,8 +44,11 @@ if (UPDATE_DEPS)
     list(APPEND update_dep_command "${API_TYPE}")
 
     set(UPDATE_DEPS_DIR_SUFFIX "${_build_type}")
-    if (ANDROID)
+    if (CMAKE_CROSSCOMPILING)
-        set(UPDATE_DEPS_DIR_SUFFIX "android/${UPDATE_DEPS_DIR_SUFFIX}")
+        set(UPDATE_DEPS_DIR_SUFFIX "${CMAKE_SYSTEM_NAME}/${UPDATE_DEPS_DIR_SUFFIX}/${CMAKE_SYSTEM_PROCESSOR}")
+    else()
+        math(EXPR bitness "8 * ${CMAKE_SIZEOF_VOID_P}")
+        set(UPDATE_DEPS_DIR_SUFFIX "${UPDATE_DEPS_DIR_SUFFIX}/${bitness}")
     endif()    
     set(UPDATE_DEPS_DIR "${PROJECT_SOURCE_DIR}/external/${UPDATE_DEPS_DIR_SUFFIX}" CACHE PATH "Location where update_deps.py installs packages")
     list(APPEND update_dep_command "--dir" )
@@ -60,6 +63,18 @@ if (UPDATE_DEPS)
     endif()
 
     list(APPEND cmake_vars "CMAKE_TOOLCHAIN_FILE")
+
+    # Avoids manually setting CMAKE_SYSTEM_NAME unless it's needed:
+    # https://cmake.org/cmake/help/latest/variable/CMAKE_CROSSCOMPILING.html
+    if (NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "${CMAKE_HOST_SYSTEM_NAME}")
+        list(APPEND cmake_vars "CMAKE_SYSTEM_NAME")
+    endif()
+    if (APPLE)
+        list(APPEND cmake_vars "CMAKE_OSX_ARCHITECTURES" "CMAKE_OSX_DEPLOYMENT_TARGET")
+    endif()
+    if (NOT MSVC_IDE)
+        list(APPEND cmake_vars "CMAKE_CXX_COMPILER" "CMAKE_C_COMPILER" "CMAKE_ASM_COMPILER")
+    endif()
     if (ANDROID)
         list(APPEND cmake_vars "ANDROID_PLATFORM" "CMAKE_ANDROID_ARCH_ABI" "CMAKE_ANDROID_STL_TYPE" "CMAKE_ANDROID_RTTI" "CMAKE_ANDROID_EXCEPTIONS" "ANDROID_USE_LEGACY_TOOLCHAIN_FILE")
     endif()
@@ -103,12 +118,12 @@ if (UPDATE_DEPS)
         include("${UPDATE_DEPS_DIR}/helper.cmake")
     endif()
 endif()
-
-if (GOOGLETEST_INSTALL_DIR)
-    list(APPEND CMAKE_PREFIX_PATH ${GOOGLETEST_INSTALL_DIR})
-endif()
 if (VULKAN_HEADERS_INSTALL_DIR)
     list(APPEND CMAKE_PREFIX_PATH ${VULKAN_HEADERS_INSTALL_DIR})
+    set(CMAKE_REQUIRE_FIND_PACKAGE_VulkanHeaders TRUE PARENT_SCOPE)
+endif()
+if (GOOGLETEST_INSTALL_DIR)
+    list(APPEND CMAKE_PREFIX_PATH ${GOOGLETEST_INSTALL_DIR})
 endif()
 if (MAGIC_ENUM_INSTALL_DIR)
     list(APPEND CMAKE_PREFIX_PATH ${MAGIC_ENUM_INSTALL_DIR})

scripts/update_deps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2017 The Glslang Authors. All rights reserved.
 # Copyright (c) 2018-2023 Valve Corporation
@@ -141,8 +141,6 @@ to the "top" directory.
 
 The commit used to checkout the repository.  This can be a SHA-1
 object name or a refname used with the remote name "origin".
-For example, this field can be set to "origin/sdk-1.1.77" to
-select the end of the sdk-1.1.77 branch.
 
 - deps (optional)
 
@@ -220,6 +218,7 @@ Legal options include:
 "windows"
 "linux"
 "darwin"
+"android"
 
 Builds on all platforms by default.
 
@@ -235,6 +234,7 @@ option can be a relative or absolute path.
 
 import argparse
 import json
+import os
 import os.path
 import subprocess
 import sys
@@ -254,7 +254,8 @@ CONFIG_MAP = {
     'minsizerel': 'MinSizeRel'
 }
 
-VERBOSE = False
+# NOTE: CMake also uses the VERBOSE environment variable. This is intentional.
+VERBOSE = os.getenv("VERBOSE")
 
 DEVNULL = open(os.devnull, 'wb')
 
@@ -273,25 +274,40 @@ def make_or_exist_dirs(path):
     if not os.path.isdir(path):
         os.makedirs(path)
 
-def command_output(cmd, directory, fail_ok=False):
+def command_output(cmd, directory):
-    """Runs a command in a directory and returns its standard output stream.
+    # Runs a command in a directory and returns its standard output stream.
-
+    # Captures the standard error stream and prints it an error occurs.
-    Captures the standard error stream and prints it if error.
+    # Raises a RuntimeError if the command fails to launch or otherwise fails.
-
-    Raises a RuntimeError if the command fails to launch or otherwise fails.
-    """
     if VERBOSE:
         print('In {d}: {cmd}'.format(d=directory, cmd=cmd))
-    p = subprocess.Popen(
+
-        cmd, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    result = subprocess.run(cmd, cwd=directory, capture_output=True, text=True)
-    (stdout, stderr) = p.communicate()
+
-    if p.returncode != 0:
+    if result.returncode != 0:
-        print('*** Error ***\nstderr contents:\n{}'.format(stderr))
+        print(f'{result.stderr}', file=sys.stderr)
-        if not fail_ok:
+        raise RuntimeError(f'Failed to run {cmd} in {directory}')
-            raise RuntimeError('Failed to run {} in {}'.format(cmd, directory))
+
     if VERBOSE:
-        print(stdout)
+        print(result.stdout)
-    return stdout
+    return result.stdout
+
+def run_cmake_command(cmake_cmd):
+    # NOTE: Because CMake is an exectuable that runs executables
+    # stdout/stderr are mixed together. So this combines the outputs
+    # and prints them properly in case there is a non-zero exit code.
+    result = subprocess.run(cmake_cmd, 
+        stdout = subprocess.PIPE,
+        stderr = subprocess.STDOUT,
+        text = True
+    )
+
+    if VERBOSE:
+        print(result.stdout)
+        print(f"CMake command: {cmake_cmd} ", flush=True)
+
+    if result.returncode != 0:
+        print(result.stdout, file=sys.stderr)
+        sys.exit(result.returncode)
 
 def escape(path):
     return path.replace('\\', '/')
@@ -341,12 +357,20 @@ class GoodRepo(object):
             self.build_dir = os.path.join(dir_top, self.build_dir)
         if self.install_dir:
             self.install_dir = os.path.join(dir_top, self.install_dir)
-	    # Check if platform is one to build on
+
+        # By default the target platform is the host platform.
+        target_platform = platform.system().lower()
+        # However, we need to account for cross-compiling.
+        for cmake_var in self._args.cmake_var:
+            if "android.toolchain.cmake" in cmake_var:
+                target_platform = 'android'
+
         self.on_build_platform = False
-        if self.build_platforms == [] or platform.system().lower() in self.build_platforms:
+        if self.build_platforms == [] or target_platform in self.build_platforms:
             self.on_build_platform = True
 
     def Clone(self, retries=10, retry_seconds=60):
+        if VERBOSE:
             print('Cloning {n} into {d}'.format(n=self.name, d=self.repo_dir))
         for retry in range(retries):
             make_or_exist_dirs(self.repo_dir)
@@ -388,7 +412,9 @@ class GoodRepo(object):
                 raise e
 
     def Checkout(self):
+        if VERBOSE:
             print('Checking out {n} in {d}'.format(n=self.name, d=self.repo_dir))
+
         if self._args.do_clean_repo:
             if os.path.isdir(self.repo_dir):
                 shutil.rmtree(self.repo_dir, onerror = on_rm_error)
@@ -399,6 +425,8 @@ class GoodRepo(object):
             command_output(['git', 'checkout', self._args.ref], self.repo_dir)
         else:
             command_output(['git', 'checkout', self.commit], self.repo_dir)
+
+        if VERBOSE:
             print(command_output(['git', 'status'], self.repo_dir))
 
     def CustomPreProcess(self, cmd_str, repo_dict):
@@ -477,12 +505,12 @@ class GoodRepo(object):
         if self._args.generator is not None:
             cmake_cmd.extend(['-G', self._args.generator])
 
-        if VERBOSE:
+        # Removes warnings related to unused CLI
-            print("CMake command: " + " ".join(cmake_cmd))
+        # EX: Setting CMAKE_CXX_COMPILER for a C project
+        if not VERBOSE:
+            cmake_cmd.append("--no-warn-unused-cli")
 
-        ret_code = subprocess.call(cmake_cmd)
+        run_cmake_command(cmake_cmd)
-        if ret_code != 0:
-            sys.exit(ret_code)
 
     def CMakeBuild(self):
         """Build CMake command for the build phase and execute it"""
@@ -490,24 +518,22 @@ class GoodRepo(object):
         if self._args.do_clean:
             cmake_cmd.append('--clean-first')
 
-        # Ninja is parallel by default
+        # Xcode / Ninja are parallel by default.
-        if self._args.generator != "Ninja":
+        if self._args.generator != "Ninja" or self._args.generator != "Xcode":
             cmake_cmd.append('--parallel')
             cmake_cmd.append(format(multiprocessing.cpu_count()))
 
-        if VERBOSE:
+        run_cmake_command(cmake_cmd)
-            print("CMake command: " + " ".join(cmake_cmd))
-
-        ret_code = subprocess.call(cmake_cmd)
-        if ret_code != 0:
-            sys.exit(ret_code)
 
     def Build(self, repos, repo_dict):
-        """Build the dependent repo"""
+        """Build the dependent repo and time how long it took"""
+        if VERBOSE:
             print('Building {n} in {d}'.format(n=self.name, d=self.repo_dir))
             print('Build dir = {b}'.format(b=self.build_dir))
             print('Install dir = {i}\n'.format(i=self.install_dir))
 
+        start = time.time()
+
         # Run any prebuild commands
         self.PreBuild()
 
@@ -521,9 +547,12 @@ class GoodRepo(object):
         # Build and execute CMake command for the build
         self.CMakeBuild()
 
+        total_time = time.time() - start
+
+        print(f"Installed {self.name} ({self.commit}) in {total_time} seconds", flush=True)
+
     def IsOptional(self, opts):
-        if len(self.optional.intersection(opts)) > 0: return True
+        return len(self.optional.intersection(opts)) > 0
-        else: return False
 
 def GetGoodRepos(args):
     """Returns the latest list of GoodRepo objects.
@@ -741,7 +770,7 @@ def main():
         if len(repo.ci_only):
             do_build = False
             for env in repo.ci_only:
-                if not env in os.environ:
+                if env not in os.environ:
                     continue
                 if os.environ[env].lower() == 'true':
                     do_build = True
@@ -765,4 +794,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-