Update RISCV CMake build default configurations

* Avoid duplicated compile flags
* Set the default optimization level to -O3 (aligned with
  CMAKE_BUILD_TYPE=Release)
* Fix RV64 cmake config
diff --git a/build_tools/cmake/build_riscv.sh b/build_tools/cmake/build_riscv.sh
index f1f0c1e..641a44c 100755
--- a/build_tools/cmake/build_riscv.sh
+++ b/build_tools/cmake/build_riscv.sh
@@ -19,6 +19,7 @@
 ROOT_DIR="${ROOT_DIR:-$(git rev-parse --show-toplevel)}"
 CMAKE_BIN="${CMAKE_BIN:-$(which cmake)}"
 RISCV_CONFIG="${RISCV_CONFIG:-rv64}"
+RISCV_COMPILER_FLAGS="${RISCV_COMPILER_FLAGS:--O3}"
 
 "${CMAKE_BIN?}" --version
 ninja --version
@@ -67,6 +68,7 @@
   -DCMAKE_TOOLCHAIN_FILE="$(realpath ${ROOT_DIR?}/build_tools/cmake/riscv.toolchain.cmake)"
   -DIREE_HOST_BINARY_ROOT="$(realpath ${BUILD_HOST_DIR?}/install)"
   -DRISCV_CPU="${RISCV_CONFIG?}"
+  -DRISCV_COMPILER_FLAGS="${RISCV_COMPILER_FLAGS?}"
   -DIREE_ENABLE_ASSERTIONS=ON
   -DIREE_BUILD_COMPILER=OFF
   -DIREE_BUILD_SAMPLES=ON
diff --git a/build_tools/cmake/riscv.toolchain.cmake b/build_tools/cmake/riscv.toolchain.cmake
index 5b4d437..a715469 100644
--- a/build_tools/cmake/riscv.toolchain.cmake
+++ b/build_tools/cmake/riscv.toolchain.cmake
@@ -6,6 +6,16 @@
 
 cmake_minimum_required(VERSION 3.13)
 
+# CMake invokes the toolchain file twice during the first build, but only once
+# during subsequent rebuilds. This was causing the various flags to be added
+# twice on the first build, and on a rebuild ninja would see only one set of the
+# flags and rebuild the world.
+# https://github.com/android-ndk/ndk/issues/323
+if(RISCV_TOOLCHAIN_INCLUDED)
+  return()
+endif(RISCV_TOOLCHAIN_INCLUDED)
+set(RISCV_TOOLCHAIN_INCLUDED true)
+
 set(CMAKE_SYSTEM_PROCESSOR riscv)
 
 if(CMAKE_HOST_SYSTEM_NAME STREQUAL Linux)
@@ -38,7 +48,7 @@
 if(RISCV_CPU STREQUAL "rv64")
   set(CMAKE_SYSTEM_NAME Linux)
   set(CMAKE_SYSTEM_LIBRARY_PATH "${RISCV_TOOLCHAIN_ROOT}/sysroot/usr/lib")
-  list(APPEND RISCV_COMPILER_FLAGS "-march=rv64gc -mabi=lp64d")
+  set(RISCV_COMPILER_FLAGS "${RISCV_COMPILER_FLAGS} -march=rv64gc -mabi=lp64d")
   set(RISCV_LINKER_FLAGS "${RISCV_LINKER_FLAGS} -lstdc++ -lpthread -lm -ldl")
 elseif(RISCV_CPU STREQUAL "rv32-baremetal")
   set(CMAKE_SYSTEM_NAME Generic)