Make slang-llvm fetching and failure more robust (#5346)

* Make slang-llvm fetching and failure more robust

Improve error reporting when things go wrong.

Fall back by default to a non-llvm build

Closes https://github.com/shader-slang/slang/issues/5247

Tested sensible behavior with:

- `FETCH_BINARY_IF_POSSIBLE`, valid tag
  - No errors or warnings, successful build with llvm
- `FETCH_BINARY_IF_POSSIBLE`, no valid tag
  - Warning message, successful llvm build with slang-llvm from latest release
- `FETCH_BINARY_IF_POSSIBLE`, no valid tag, bad `SLANG_SLANG_LLVM_BINARY_URL` specified
  - Warning message, successful no-llvm build
- `FETCH_BINARY_IF_POSSIBLE`, no valid tag, unable to fetch release information
  - Warning message, successful no-llvm build
- `FETCH_BINARY`, valid tag
  - No errors or warnings, successful build with llvm
- `FETCH_BINARY`, no valid tag
  - Warning message, successful llvm build with slang-llvm from latest release
- `FETCH_BINARY`, no valid tag, bad `SLANG_SLANG_LLVM_BINARY_URL` specified
  - Error, explaining that we couldn't fetch it
- `FETCH_BINARY`, no valid tag, unable to fetch release info
  - Error, explaining that we couldn't fetch it

* Allow downloading from a local file

---------

Co-authored-by: Yong He <yonghe@outlook.com>

3 files changed, 144 insertions, 57 deletions

diff --git a/cmake/FetchedSharedLibrary.cmake b/cmake/FetchedSharedLibrary.cmake
index d63f03d29..3c7f24be5 100644
--- a/cmake/FetchedSharedLibrary.cmake
+++ b/cmake/FetchedSharedLibrary.cmake
@@ -1,3 +1,36 @@
+# Helper function to download and extract an archive
+function(download_and_extract archive_name url)
+    get_filename_component(extension ${url} EXT)
+    set(archive_path "${CMAKE_CURRENT_BINARY_DIR}/${archive_name}${extension}")
+    set(extract_dir "${CMAKE_CURRENT_BINARY_DIR}/${archive_name}")
+
+    if(EXISTS ${url})
+        message(STATUS "Using local file for ${archive_name}: ${url}")
+        set(archive_path ${url})
+    else()
+        message(STATUS "Fetching ${archive_name} from ${url}")
+        file(DOWNLOAD ${url} ${archive_path}
+             # SHOW_PROGRESS
+             STATUS status
+        )
+
+        list(GET status 0 status_code)
+        list(GET status 1 status_string)
+        if(NOT status_code EQUAL 0)
+            message(WARNING "Failed to download ${archive_name} from ${url}: ${status_string}")
+            return()
+        endif()
+    endif()
+
+    file(ARCHIVE_EXTRACT
+         INPUT ${archive_path}
+         DESTINATION ${extract_dir}
+    )
+
+    set(${archive_name}_SOURCE_DIR ${extract_dir} PARENT_SCOPE)
+    message(STATUS "${archive_name} downloaded and extracted to ${extract_dir}")
+endfunction()
+
 # Add rules to copy & install shared library of name 'library_name' in the 'module_subdir' directory.
 # If 'url' is a directory, the shared library (with platform-specific shared library prefixes and suffixes) will be
 # taken from the directory, and whatever is found there will be used to produce the install rule.
@@ -6,6 +39,13 @@
 # Otherwise, the 'url' is interpreted as an URL, and the content of the URL will be fetched, extracted and searched
 # for the shared library to produce the install rule.
 function(copy_fetched_shared_library library_name url)
+    cmake_parse_arguments(ARG "IGNORE_FAILURE" "" "" ${ARGN})
+    if(ARG_IGNORE_FAILURE)
+        set(error_type STATUS)
+    else()
+        set(error_type SEND_ERROR)
+    endif()
+
     set(shared_library_filename
         "${CMAKE_SHARED_LIBRARY_PREFIX}${library_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
     )
@@ -18,12 +58,12 @@ function(copy_fetched_shared_library library_name url)
         list(LENGTH source_object nmatches)
         if(nmatches EQUAL 0)
             message(
-                SEND_ERROR
+                ${error_type}
                 "Unable to find ${shared_library_filename} in ${url}"
             )
         elseif(nmatches GREATER 1)
             message(
-                SEND_ERROR
+                ${error_type}
                 "Found multiple files named ${shared_library_filename} in ${url}"
             )
         endif()
@@ -42,9 +82,21 @@ function(copy_fetched_shared_library library_name url)
         set(source_object "${url}")
     else()
         # Otherwise, download and extract from whatever URL we have
-        fetchcontent_declare(${library_name} URL "${url}")
-        fetchcontent_populate(${library_name})
-        from_glob(${${library_name}_SOURCE_DIR})
+        download_and_extract("${library_name}" "${url}")
+        if(DEFINED ${library_name}_SOURCE_DIR)
+            from_glob(${${library_name}_SOURCE_DIR})
+        elseif(ARG_IGNORE_FAILURE)
+            return()
+        else()
+            message(SEND_ERROR "Unable to download and extract ${library_name} from ${url}")
+            return()
+        endif()
+    endif()
+
+    # We didn't find it, just return and don't create a target and operation
+    # which will fail
+    if(NOT EXISTS ${source_object} AND ARG_IGNORE_FAILURE)
+        return()
     endif()
 
     set(dest_object
@@ -72,12 +124,14 @@ function(copy_fetched_shared_library library_name url)
 endfunction()
 
 function(install_fetched_shared_library library_name url)
-    copy_fetched_shared_library(${library_name} ${url})
+    copy_fetched_shared_library(${library_name} ${url} ${ARGN})
     set(shared_library_filename
         "${CMAKE_SHARED_LIBRARY_PREFIX}${library_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
     )
     set(dest_object
         ${CMAKE_BINARY_DIR}/$<CONFIG>/${module_subdir}/${shared_library_filename}
     )
-    install(PROGRAMS ${dest_object} DESTINATION ${module_subdir})
+    if(TARGET ${library_name})
+        install(PROGRAMS ${dest_object} DESTINATION ${module_subdir})
+    endif()
 endfunction()
diff --git a/cmake/GitHubRelease.cmake b/cmake/GitHubRelease.cmake
index dd9dd8fe1..0590933d2 100644
--- a/cmake/GitHubRelease.cmake
+++ b/cmake/GitHubRelease.cmake
@@ -1,22 +1,64 @@
+function(check_assets_for_file json_content filename found_var)
+    string(JSON asset_count LENGTH "${json_content}" "assets")
+    set(found "FALSE")
+
+    # Never change, CMake...
+    math(EXPR max_asset_index "${asset_count} - 1")
+    foreach(i RANGE 0 ${max_asset_index})
+        string(JSON asset_name GET "${json_content}" "assets" ${i} "name")
+        if("${asset_name}" STREQUAL "${filename}")
+            set(found "TRUE")
+            break()
+        endif()
+    endforeach()
+    set(${found_var} "${found}" PARENT_SCOPE)
+endfunction()
+
+function(get_latest owner repo os arch github_token out_var)
+    set(json_output_file "${CMAKE_CURRENT_BINARY_DIR}/${owner}_${repo}_release_info.json")
+    set(latest_release_url "https://api.github.com/repos/${owner}/${repo}/releases/latest")
+
+    set(download_args
+        "${latest_release_url}"
+        "${json_output_file}"
+        STATUS download_statuses
+    )
+
+    if(github_token)
+        list(APPEND download_args HTTPHEADER "Authorization: token ${github_token}")
+    endif()
+
+    file(DOWNLOAD ${download_args})
+    list(GET download_statuses 0 status_code)
+    if(NOT status_code EQUAL 0)
+        message(WARNING "Failed to download latest release info from ${latest_release_url}")
+        return()
+    endif()
+
+    # Get the tag from this release json file
+    file(READ "${json_output_file}" latest_json_content)
+    string(JSON latest_release_tag GET "${latest_json_content}" "tag_name")
+    string(REGEX REPLACE "^v" "" latest_version "${latest_release_tag}")
+
+    # Check if the expected ZIP file is in the latest release
+    set(desired_zip "${repo}-${latest_version}-${os}-${arch}.zip")
+    message(VERBOSE "searching for the prebuilt slang-llvm library in ${latest_release_url}")
+    check_assets_for_file("${latest_json_content}" "${desired_zip}" file_found_latest)
+
+    if(file_found_latest)
+        # If we got it, we found a good version
+        set(${out_var} "${latest_version}" PARENT_SCOPE)
+    else()
+        message(WARNING "No release binary for ${os}-${arch} exists for the latest version: ${latest_version}")
+    endif()
+endfunction()
+
 function(check_release_and_get_latest owner repo version os arch github_token out_var)
     # Construct the URL for the specified version's release API endpoint
     set(version_url "https://api.github.com/repos/${owner}/${repo}/releases/tags/v${version}")
 
     set(json_output_file "${CMAKE_CURRENT_BINARY_DIR}/${owner}_${repo}_release_info.json")
 
-    function(check_assets_for_file json_content filename found_var)
-        string(JSON asset_count LENGTH "${json_content}" "assets")
-        set(found "FALSE")
-        foreach(i RANGE 0 ${asset_count})
-            string(JSON asset_name GET "${json_content}" "assets" ${i} "name")
-            if("${asset_name}" STREQUAL "${filename}")
-                set(found "TRUE")
-                break()
-            endif()
-        endforeach()
-        set(${found_var} "${found}" PARENT_SCOPE)
-    endfunction()
-
     # Prepare download arguments
     set(download_args
         "${version_url}"
@@ -39,6 +81,7 @@ function(check_release_and_get_latest owner repo version os arch github_token ou
 
         # Check if the specified version contains the expected ZIP file
         set(desired_zip "${repo}-${version}-${os}-${arch}.zip")
+        message(VERBOSE "searching for the prebuilt slang-llvm library in ${version_url}")
         check_assets_for_file("${json_content}" "${desired_zip}" file_found)
 
         if(file_found)
@@ -47,44 +90,20 @@ function(check_release_and_get_latest owner repo version os arch github_token ou
         endif()
         message(WARNING "Failed to find ${desired_zip} in release assets for ${version} from ${version_url}\nFalling back to latest version if it differs")
     else()
-        message(WARNING "Failed to download release info for version ${version} from ${version_url}\nFalling back to latest version if it differs")
-
+        set(w "Failed to download release info for version ${version} from ${version_url}\nFalling back to latest version if it differs")
         if(status_code EQUAL 22)
-            message(WARNING "If API rate limit is exceeded, Github allows a higher limit when you use token. Try a cmake option -DSLANG_GITHUB_TOKEN=your_token_here")
-	endif()
-    endif()
-
+            set(w "${w}\nIf you think this is failing because of GitHub API rate limiting, Github allows a higher limit if you use a token. Try the cmake option -DSLANG_GITHUB_TOKEN=your_token_here")
+        endif()
 
-    # If not found, get the latest release tag
-    set(latest_release_url "https://api.github.com/repos/${owner}/${repo}/releases/latest")
-    file(DOWNLOAD "${latest_release_url}" "${json_output_file}" STATUS download_status)
-    list(GET download_status 0 status_code)
-    if(NOT status_code EQUAL 0)
-        message(WARNING "Failed to download latest release info from ${latest_release_url}")
-        return()
+        message(WARNING ${w})
     endif()
 
-    # Get the tag from this release json file
-    file(READ "${json_output_file}" latest_json_content)
-    string(JSON latest_release_tag GET "${latest_json_content}" "tag_name")
-    string(REGEX REPLACE "^v" "" latest_version "${latest_release_tag}")
-
-    if(latest_version EQUAL version)
-        # The versions are the same
-        message(WARNING "No release binary for ${os}-${arch} exists for ${version}")
+    # If not found, get the latest release tag
+    get_latest(${owner} ${repo} ${os} ${arch} "${github_token}" latest_version)
+    if(NOT DEFINED latest_version)
         return()
     endif()
-
-    # Check if the expected ZIP file is in the latest release
-    set(desired_zip "${repo}-${latest_version}-${os}-${arch}.zip")
-    check_assets_for_file("${latest_json_content}" "${desired_zip}" file_found_latest)
-
-    if(file_found_latest)
-        # If we got it, we found a good version
-        set(${out_var} "${latest_version}" PARENT_SCOPE)
-    else()
-        message(WARNING "No release binary for ${os}-${arch} exists for ${version} or the latest version ${latest_version}")
-    endif()
+    set(${out_var} "${latest_version}" PARENT_SCOPE)
 endfunction()
 
 function(get_best_slang_binary_release_url github_token out_var)
@@ -111,8 +130,22 @@ function(get_best_slang_binary_release_url github_token out_var)
     set(owner "shader-slang")
     set(repo "slang")
 
-    check_release_and_get_latest(${owner} ${repo} ${SLANG_VERSION_NUMERIC} ${os} ${arch} "${github_token}" release_version)
+    # This is the first version which distributed libslang-llvm.so, if it's
+    # older than that then someone didn't fetch tags, emit a message and
+    # fallback to the latest release
+    if(${SLANG_VERSION_NUMERIC} VERSION_LESS "2024.1.27")
+        if(${SLANG_VERSION_NUMERIC} VERSION_EQUAL "0.0.0")
+            message(VERBOSE "The detected version of slang is ${SLANG_VERSION_NUMERIC}, fetching libslang-llvm from the latest release")
+        else()
+            message(WARNING "The detected version of slang ${SLANG_VERSION_NUMERIC} is very old (probably you haven't fetched tags recently?), libslang-llvm will be fetched from the latest release rather than the one matching ${SLANG_VERSION_NUMERIC}")
+        endif()
+        get_latest(${owner} ${repo} ${os} ${arch} "${github_token}" release_version)
+    else()
+        check_release_and_get_latest(${owner} ${repo} ${SLANG_VERSION_NUMERIC} ${os} ${arch} "${github_token}" release_version)
+    endif()
     if(DEFINED release_version)
-      set(${out_var} "https://github.com/${owner}/${repo}/releases/download/v${release_version}/slang-${release_version}-${os}-${arch}.zip" PARENT_SCOPE)
+        message(VERBOSE "Found a version of libslang-llvm.so in ${release_version}")
+        set(${out_var} "https://github.com/${owner}/${repo}/releases/download/v${release_version}/slang-${release_version}-${os}-${arch}.zip" PARENT_SCOPE)
     endif()
 endfunction()
+
diff --git a/cmake/GitVersion.cmake b/cmake/GitVersion.cmake
index 386ca309c..c8eb32947 100644
--- a/cmake/GitVersion.cmake
+++ b/cmake/GitVersion.cmake
@@ -24,7 +24,7 @@ function(get_git_version var_numeric var dir)
             if(NOT result EQUAL 0)
                 message(
                     WARNING
-                    "Getting ${var} failed: ${command} returned ${result}"
+                    "Getting ${var} failed: ${command} returned ${result}\nIs this a Git repo with tags?\nConsider settings -D${var} to specify a version manually"
                 )
             elseif("${version_out}" MATCHES "^v(([0-9]+(\\.[0-9]+)*).*)")
                 set(version "${CMAKE_MATCH_1}")
@@ -32,13 +32,13 @@ function(get_git_version var_numeric var dir)
             else()
                 message(
                     WARNING
-                    "Couldn't parse version (like v1.2.3 or v1.2.3-foo) from ${version_out}"
+                    "Couldn't parse version (like v1.2.3 or v1.2.3-foo) from ${version_out}, using ${version} for now"
                 )
             endif()
         else()
             message(
                 WARNING
-                "Couldn't find git executable to get ${var}, please use -D${var}"
+                "Couldn't find git executable to get ${var}, please use -D${var}, using ${version} for now"
             )
         endif()
     endif()