Expand on documentation for IREE_ENABLE_LLD. (#9793)

Revisiting some of the work in https://github.com/iree-org/iree/pull/2098.

I tried setting `IREE_ENABLE_LLD`, expecting it to build a supported version lld as needed, as `LLVM_ENABLE_LLD` [claims to do](https://llvm.org/docs/CMake.html#llvm-use-linker:~:text=LLVM_ENABLE_LLD%3ABOOL):
> `LLVM_ENABLE_LLD:BOOL`
> This option is equivalent to `-DLLVM_USE_LINKER=lld`, except during a 2-stage build where a dependency is added from the first stage to the second ensuring that lld is built before stage2 begins.

However, our option does _not_ do that. I'm tempted to drop `IREE_ENABLE_LLD` entirely, since we can't realistically do that sort of 2-stage build as just a downstream user of LLVM and other libraries.

Short of removing the misleading option (then updating our scripts and docs and asking developers to switch to the explicit option), I opted to add more comments to help future developers with troubleshooting.

Also mixed up in this are https://github.com/iree-org/iree/issues/7473 and https://github.com/iree-org/iree/pull/7474, which tried to set/use `IREE_ENABLE_LLD` on a platform (OSX) where lld is not actually supported (?). I moved that check from CMake branches to `cmake_dependent_option`.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f311bb1..02a3bd8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -216,9 +216,6 @@
 # IREE compilation toolchain configuration
 #-------------------------------------------------------------------------------
 
-# Enable using lld as the linker for C/C++ targets. This affects IREE and all
-# dependency projects.
-option(IREE_ENABLE_LLD "Use lld when linking" OFF)
 option(IREE_ENABLE_ASAN "Enable address sanitizer" OFF)
 option(IREE_ENABLE_MSAN "Enable memory sanitizer" OFF)
 option(IREE_ENABLE_TSAN "Enable thread sanitizer" OFF)
@@ -354,6 +351,16 @@
   CMAKE_SHARED_LINKER_FLAGS_FASTBUILD
 )
 
+# Override the system's default linker.
+# See also: https://llvm.org/docs/CMake.html#llvm-use-linker.
+set(IREE_USE_LINKER "" CACHE STRING "")
+# Equivalent to setting -DIREE_USE_LINKER=lld.
+# Note that unlike LLVM's LLVM_ENABLE_LLD, this does _not_ build lld. You will
+# need to either install a recent version of lld or build it from source prior
+# to setting this option. See also: https://lld.llvm.org/#using-lld.
+# This option is disabled on Apple platforms, where lld is not supported.
+cmake_dependent_option(IREE_ENABLE_LLD "Override the system's default linker to lld" OFF "NOT APPLE" OFF)
+
 include(iree_setup_toolchain)
 
 #-------------------------------------------------------------------------------
diff --git a/build_tools/cmake/iree_setup_toolchain.cmake b/build_tools/cmake/iree_setup_toolchain.cmake
index e15779e..b1dbfa9 100644
--- a/build_tools/cmake/iree_setup_toolchain.cmake
+++ b/build_tools/cmake/iree_setup_toolchain.cmake
@@ -4,9 +4,10 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-function(iree_append value)
+# Appends ${VALUE} to each argument.
+function(iree_append_to_lists VALUE)
   foreach(_VARIABLE ${ARGN})
-    set(${_VARIABLE} "${${_VARIABLE}} ${value}" PARENT_SCOPE)
+    set(${_VARIABLE} "${${_VARIABLE}} ${VALUE}" PARENT_SCOPE)
   endforeach(_VARIABLE)
 endfunction()
 
@@ -17,7 +18,7 @@
   set(IREE_USE_LINKER "lld")
 endif()
 
-if(IREE_USE_LINKER AND NOT APPLE)
+if(IREE_USE_LINKER)
   set(IREE_LINKER_FLAG "-fuse-ld=${IREE_USE_LINKER}")
 
   # Depending on how the C compiler is invoked, it may trigger an unused
@@ -25,7 +26,7 @@
   # causing false negatives. We lack a finer grained way to suppress such a
   # thing, and this is deemed least bad.
   if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    iree_append("-Wno-unused-command-line-argument"
+    iree_append_to_lists("-Wno-unused-command-line-argument"
       CMAKE_REQUIRED_FLAGS
       CMAKE_EXE_LINKER_FLAGS
       CMAKE_MODULE_LINKER_FLAGS
@@ -33,7 +34,7 @@
     )
   endif()
 
-  iree_append("${IREE_LINKER_FLAG}"
+  iree_append_to_lists("${IREE_LINKER_FLAG}"
     CMAKE_REQUIRED_FLAGS
     CMAKE_EXE_LINKER_FLAGS
     CMAKE_MODULE_LINKER_FLAGS
@@ -45,14 +46,14 @@
   check_cxx_source_compiles("${MINIMAL_SRC}" CXX_SUPPORTS_CUSTOM_LINKER)
   check_c_source_compiles("${MINIMAL_SRC}" CC_SUPPORTS_CUSTOM_LINKER)
 
+  # Note: if you see errors here, check
+  #   * logs in CMakeFiles/CMakeError.log in your build directory
+  #   * that you have a recent version of your chosen linker (for example:
+  #     install the version of lld that we use in our Docker images)
   if(NOT CXX_SUPPORTS_CUSTOM_LINKER)
     message(FATAL_ERROR "Compiler '${CMAKE_CXX_COMPILER}' does not support '${IREE_LINKER_FLAG}'")
   endif()
-
   if(NOT CC_SUPPORTS_CUSTOM_LINKER)
     message(FATAL_ERROR "Compiler '${CMAKE_C_COMPILER}' does not support '${IREE_LINKER_FLAG}'")
   endif()
-
-elseif(IREE_USE_LINKER)
-    message(STATUS "On Apple OSX use system linker")
 endif()