Merge branch 'feature/static-xcframework' of github.com:richwaters/whisper.cpp into feature/static-xcframework

Author: richwaters

.devops/main-musa.Dockerfile

@@ -1,10 +1,10 @@
 ARG UBUNTU_VERSION=22.04
 # This needs to generally match the container host's environment.
-ARG MUSA_VERSION=rc4.0.1
+ARG MUSA_VERSION=rc4.2.0
 # Target the MUSA build image
-ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-devel-ubuntu${UBUNTU_VERSION}
+ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64
 # Target the MUSA runtime image
-ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-runtime-ubuntu${UBUNTU_VERSION}
+ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64
 
 FROM ${BASE_MUSA_DEV_CONTAINER} AS build
 WORKDIR /app

.github/workflows/build.yml

@@ -6,6 +6,25 @@ on:
       - master
     tags:
       - 'v*'
+    paths: ['.github/workflows/build.yml',
+            '**/CMakeLists.txt',
+            '**/Makefile',
+            '**/*.mk',
+            '**/*.cmake',
+            '**/*.in',
+            '**/*.h',
+            '**/*.hpp',
+            '**/*.c',
+            '**/*.cpp',
+            '**/*.cu',
+            '**/*.cuh',
+            '**/*.cl',
+            '**/*.swift',
+            '**/*.m',
+            '**/*.mm',
+            '**/*.metal',
+            '**/*.comp',
+            '**/*.java']
   pull_request:
     types: [opened, synchronize, reopened]
   workflow_dispatch:

README.md

@@ -386,7 +386,7 @@ Run the inference examples as usual, for example:
 ## Moore Threads GPU support
 
 With Moore Threads cards the processing of the models is done efficiently on the GPU via muBLAS and custom MUSA kernels.
-First, make sure you have installed `MUSA SDK rc4.0.1`: https://developer.mthreads.com/sdk/download/musa?equipment=&os=&driverVersion=&version=4.0.1
+First, make sure you have installed `MUSA SDK rc4.2.0`: https://developer.mthreads.com/sdk/download/musa?equipment=&os=&driverVersion=&version=4.2.0
 
 Now build `whisper.cpp` with MUSA support:
 

bindings/go/Makefile

@@ -15,15 +15,16 @@ BUILD_DIR := build_go
 MODELS_DIR := models
 EXAMPLES_DIR := $(wildcard examples/*)
 INCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)
-LIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src:$(abspath ../../${BUILD_DIR}/ggml/src))
+LIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src):$(abspath ../../${BUILD_DIR}/ggml/src)
 
 ifeq ($(GGML_CUDA),1)
 	LIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/
 	BUILD_FLAGS := -ldflags "-extldflags '-lcudart -lcuda -lcublas'"
 endif
 
 ifeq ($(UNAME_S),Darwin)
-	EXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit
+	LIBRARY_PATH := $(LIBRARY_PATH):$(abspath ../../${BUILD_DIR}/ggml/src/ggml-blas):$(abspath ../../${BUILD_DIR}/ggml/src/ggml-metal)
+	EXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit -lggml-metal -lggml-blas
 endif
 
 all: clean whisper examples

bindings/go/whisper.go

@@ -9,7 +9,9 @@ import (
 // CGO
 
 /*
-#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu  -lm -lstdc++ -fopenmp
+#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu -lm -lstdc++
+#cgo linux LDFLAGS: -fopenmp
+#cgo darwin LDFLAGS: -lggml-metal -lggml-blas
 #cgo darwin LDFLAGS: -framework Accelerate -framework Metal -framework Foundation -framework CoreGraphics
 #include <whisper.h>
 #include <stdlib.h>

build-xcframework.sh

@@ -328,7 +328,7 @@ combine_static_libraries() {
         arch_flags+=" -arch $arch"
     done
 
-    
+
     if [[ "${BUILD_STATIC_XCFRAMEWORK}" == "ON" ]]; then
         echo "Packaging static framework for ${platform}."
         mkdir -p "$(dirname "${base_dir}/${output_lib}")"
@@ -551,7 +551,7 @@ if [[ "${BUILD_STATIC_XCFRAMEWORK}" == "ON" ]]; then
         -framework $(pwd)/build-tvos-sim/framework/whisper.framework \
         -output $(pwd)/build-apple/whisper.xcframework
     exit 0
-fi 
+fi
 
 xcodebuild -create-xcframework \
     -framework $(pwd)/build-ios-sim/framework/whisper.framework \

examples/bench.wasm/README.md

@@ -32,6 +32,16 @@ cp bin/libbench.js        /path/to/html/
 cp bin/libbench.worker.js /path/to/html/
 ```
 
+> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`
+> which means that that a separate .wasm file will not be generated. Instead, the
+> WASM module is embedded in the main JS file as a base64 encoded string. To
+> generate a separate .wasm file, you need to disable this option by passing
+> `-DWHISPER_WASM_SINGLE_FILE=OFF`:
+> ```console
+> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF
+> ```
+> This will generate a `libbench.wasm` file in the build/bin directory.
+
 > 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no
 > longer generated and the worker is embedded in the main JS file. So the worker
 > file will not be geneated for versions later than `3.1.58`.

examples/command.wasm/README.md

@@ -32,6 +32,16 @@ cp bin/libcommand.js        /path/to/html/
 cp bin/libcommand.worker.js /path/to/html/
 ```
 
+> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`
+> which means that that a separate .wasm file will not be generated. Instead, the
+> WASM module is embedded in the main JS file as a base64 encoded string. To
+> generate a separate .wasm file, you need to disable this option by passing
+> `-DWHISPER_WASM_SINGLE_FILE=OFF`:
+> ```console
+> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF
+> ```
+> This will generate a `libcommand.wasm` file in the build/bin directory.
+
 > 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no
 > longer generated and the worker is embedded in the main JS file. So the worker
 > file will not be geneated for versions later than `3.1.58`.

examples/server/server.cpp

@@ -104,6 +104,7 @@ struct whisper_params {
     bool flash_attn      = false;
     bool suppress_nst    = false;
     bool no_context      = false;
+    bool no_language_probabilities = false;
 
     std::string language        = "en";
     std::string prompt          = "";
@@ -178,6 +179,7 @@ void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & para
     fprintf(stderr, "  -nc,       --no-context        [%-7s] do not use previous audio context\n", params.no_context ? "true" : "false");
     fprintf(stderr, "  -ng,       --no-gpu            [%-7s] do not use gpu\n", params.use_gpu ? "false" : "true");
     fprintf(stderr, "  -fa,       --flash-attn        [%-7s] flash attention\n", params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -nlp,      --no-language-probabilities [%-7s] exclude language probabilities from verbose_json output\n", params.no_language_probabilities ? "true" : "false");
     // Voice Activity Detection (VAD) parameters
     fprintf(stderr, "\nVoice Activity Detection (VAD) options:\n");
     fprintf(stderr, "             --vad                           [%-7s] enable Voice Activity Detection (VAD)\n",            params.vad ? "true" : "false");
@@ -237,6 +239,7 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params, serve
         else if (arg == "-sns"  || arg == "--suppress-nst")    { params.suppress_nst    = true; }
         else if (arg == "-nth"  || arg == "--no-speech-thold") { params.no_speech_thold = std::stof(argv[++i]); }
         else if (arg == "-nc"   || arg == "--no-context")      { params.no_context      = true; }
+        else if (arg == "-nlp"  || arg == "--no-language-probabilities") { params.no_language_probabilities = true; }
 
         // server params
         else if (                  arg == "--port")            { sparams.port        = std::stoi(argv[++i]); }
@@ -599,6 +602,10 @@ void get_req_parameters(const Request & req, whisper_params & params)
     {
         params.vad_samples_overlap = std::stof(req.get_file_value("vad_samples_overlap").content);
     }
+    if (req.has_file("no_language_probabilities"))
+    {
+        params.no_language_probabilities = parse_str_to_bool(req.get_file_value("no_language_probabilities").content);
+    }
 }
 
 }  // namespace
@@ -1024,23 +1031,25 @@ int main(int argc, char ** argv) {
         } else if (params.response_format == vjson_format) {
             /* try to match openai/whisper's Python format */
             std::string results = output_str(ctx, params, pcmf32s); 
-            // Get language probabilities
-            std::vector<float> lang_probs(whisper_lang_max_id() + 1, 0.0f);
-            const auto detected_lang_id = whisper_lang_auto_detect(ctx, 0, params.n_threads, lang_probs.data());
             json jres = json{
                 {"task", params.translate ? "translate" : "transcribe"},
                 {"language", whisper_lang_str_full(whisper_full_lang_id(ctx))},
                 {"duration", float(pcmf32.size())/WHISPER_SAMPLE_RATE},
                 {"text", results},
-                {"segments", json::array()},
-                {"detected_language", whisper_lang_str_full(detected_lang_id)},
-                {"detected_language_probability", lang_probs[detected_lang_id]},
-                {"language_probabilities", json::object()}
+                {"segments", json::array()}
             };
-            // Add all language probabilities
-            for (int i = 0; i <= whisper_lang_max_id(); ++i) {
-                if (lang_probs[i] > 0.001f) { // Only include non-negligible probabilities
-                    jres["language_probabilities"][whisper_lang_str(i)] = lang_probs[i];
+            // Only compute language probabilities if requested (expensive operation)
+            if (!params.no_language_probabilities) {
+                std::vector<float> lang_probs(whisper_lang_max_id() + 1, 0.0f);
+                const auto detected_lang_id = whisper_lang_auto_detect(ctx, 0, params.n_threads, lang_probs.data());
+                jres["detected_language"] = whisper_lang_str_full(detected_lang_id);
+                jres["detected_language_probability"] = lang_probs[detected_lang_id];
+                jres["language_probabilities"] = json::object();
+                // Add all language probabilities
+                for (int i = 0; i <= whisper_lang_max_id(); ++i) {
+                    if (lang_probs[i] > 0.001f) { // Only include non-negligible probabilities
+                        jres["language_probabilities"][whisper_lang_str(i)] = lang_probs[i];
+                    }
                 }
             }
             const int n_segments = whisper_full_n_segments(ctx);

examples/stream.wasm/README.md

@@ -30,6 +30,16 @@ cp bin/libstream.js        /path/to/html/
 cp bin/libstream.worker.js /path/to/html/
 ```
 
+> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`
+> which means that that a separate .wasm file will not be generated. Instead, the
+> WASM module is embedded in the main JS file as a base64 encoded string. To
+> generate a separate .wasm file, you need to disable this option by passing
+> `-DWHISPER_WASM_SINGLE_FILE=OFF`:
+> ```console
+> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF
+> ```
+> This will generate a `libstream.wasm` file in the build/bin directory.
+
 > 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no
 > longer generated and the worker is embedded in the main JS file. So the worker
 > file will not be geneated for versions later than `3.1.58`.