diff --git a/CMakeLists.txt b/CMakeLists.txt index b4fab8c34c0..4a16dd01ddd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,10 @@ option(WHISPER_ALL_WARNINGS_3RD_PARTY "whisper: enable all compiler warnings in option(WHISPER_FATAL_WARNINGS "whisper: enable -Werror flag" OFF) option(WHISPER_USE_SYSTEM_GGML "whisper: use system-installed GGML library" OFF) +# flat bindings +option(WHISPER_BINDINGS_FLAT "Add extra flat definitions to Whisper + GGML" OFF) +option(GGML_BINDINGS_FLAT "Add extra flat definitions to Examples" ${WHISPER_BINDINGS_FLAT}) + # sanitizers option(WHISPER_SANITIZE_THREAD "whisper: enable thread sanitizer" OFF) option(WHISPER_SANITIZE_ADDRESS "whisper: enable address sanitizer" OFF) @@ -242,4 +246,4 @@ if (MSVC) disable_msvc_warnings(whisper-bench) disable_msvc_warnings(quantize) endif() -endif() +endif() \ No newline at end of file diff --git a/bindings/ruby/ext/options.rb b/bindings/ruby/ext/options.rb index 29ff79090e2..1ddc79401bd 100644 --- a/bindings/ruby/ext/options.rb +++ b/bindings/ruby/ext/options.rb @@ -83,6 +83,7 @@ def configure bool "GGML_AVX_VNNI" ignored "GGML_BACKEND_DL" ignored "GGML_BIN_INSTALL_DIR" + ignored "GGML_BINDINGS_FLAT" bool "GGML_BLAS" string "GGML_BLAS_VENDOR" bool "GGML_BMI2" @@ -166,6 +167,7 @@ def configure bool "WHISPER_ALL_WARNINGS" bool "WHISPER_ALL_WARNINGS_3RD_PARTY" ignored "WHISPER_BIN_INSTALL_DIR" + ignored "WHISPER_BINDINGS_FLAT" ignored "WHISPER_BUILD_EXAMPLES" ignored "WHISPER_BUILD_SERVER" ignored"WHISPER_BUILD_TESTS" diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e4265affe97..950bf1734ac 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -91,6 +91,12 @@ target_include_directories(json_cpp INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) # examples +if(WHISPER_BINDINGS_FLAT) + message(STATUS "Adding FLAT binding to examples") + add_compile_options(-DWHISPER_BINDINGS_FLAT) +endif() + + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) if (EMSCRIPTEN) diff --git a/examples/bench/bench.cpp b/examples/bench/bench.cpp index 54f73110d42..16b1ce975e3 100644 --- a/examples/bench/bench.cpp +++ b/examples/bench/bench.cpp @@ -1,4 +1,8 @@ #include "whisper.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include #include @@ -61,6 +65,15 @@ void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & para static int whisper_bench_full(const whisper_params & params) { // whisper init + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/examples/cli/cli.cpp b/examples/cli/cli.cpp index fccfd13eb0d..b05499acb8b 100644 --- a/examples/cli/cli.cpp +++ b/examples/cli/cli.cpp @@ -2,6 +2,10 @@ #include "common-whisper.h" #include "whisper.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include "grammar-parser.h" #include @@ -934,6 +938,15 @@ int main(int argc, char ** argv) { // whisper init + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/examples/command/command.cpp b/examples/command/command.cpp index 9dc8f629995..f3b59c241dd 100644 --- a/examples/command/command.cpp +++ b/examples/command/command.cpp @@ -9,6 +9,10 @@ #include "common-sdl.h" #include "common.h" #include "whisper.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include "grammar-parser.h" #include @@ -692,6 +696,15 @@ int main(int argc, char ** argv) { // whisper init + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 14462707ef1..0a0318d1e10 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2,6 +2,10 @@ #include "common-whisper.h" #include "whisper.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include "httplib.h" #include "json.hpp" @@ -541,6 +545,15 @@ int main(int argc, char ** argv) { check_ffmpeg_availibility(); } // whisper init + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/examples/stream/stream.cpp b/examples/stream/stream.cpp index 65c6587db92..84afcdf4925 100644 --- a/examples/stream/stream.cpp +++ b/examples/stream/stream.cpp @@ -6,6 +6,10 @@ #include "common.h" #include "common-whisper.h" #include "whisper.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include #include @@ -155,6 +159,15 @@ int main(int argc, char ** argv) { exit(0); } + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/examples/talk-llama/talk-llama.cpp b/examples/talk-llama/talk-llama.cpp index 9097c491b61..e33816aff5b 100644 --- a/examples/talk-llama/talk-llama.cpp +++ b/examples/talk-llama/talk-llama.cpp @@ -5,6 +5,10 @@ #include "common.h" #include "common-whisper.h" #include "whisper.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include "llama.h" #include @@ -287,6 +291,15 @@ int main(int argc, char ** argv) { // whisper init + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/examples/wchess/wchess.cmd/wchess.cmd.cpp b/examples/wchess/wchess.cmd/wchess.cmd.cpp index 4d049976315..9b4bc425a64 100644 --- a/examples/wchess/wchess.cmd/wchess.cmd.cpp +++ b/examples/wchess/wchess.cmd/wchess.cmd.cpp @@ -7,6 +7,10 @@ #include "WChess.h" #include "common-sdl.h" +#ifdef WHISPER_BINDINGS_FLAT +#include "whisper-flat.h" +#include "../ggml/src/ggml-flat.h" +#endif #include #include @@ -182,6 +186,15 @@ int main(int argc, char ** argv) { // whisper init + #ifdef WHISPER_BINDINGS_FLAT + fprintf(stderr, "+++ WHISPER_BINDINGS_FLAT +++\n"); + if(params.use_gpu) { + whisper_flat_backend_load_all(); + } else { + ggml_backend_try_load_best("cpu", nullptr); + } + #endif + struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = params.use_gpu; diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index a8300e16d87..3ace673afd9 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -385,4 +385,4 @@ if (MSVC) disable_msvc_warnings(ggml-cpu-skylakex) disable_msvc_warnings(ggml-cpu-icelake) disable_msvc_warnings(ggml-cpu-alderlake) -endif() +endif() \ No newline at end of file diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index ddea5ad3891..96d4e65ed51 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -208,8 +208,22 @@ if (GGML_BACKEND_DL) target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL) endif() +set(GGML_LIBRARY_SOURCES + ggml-backend-reg.cpp) + +if(WHISPER_BINDINGS_FLAT) + message(STATUS "Adding FLAT GGML binding extras") + + set(FLAT_GGML_SOURCES + ggml-flat.cpp + ) + list(APPEND GGML_LIBRARY_SOURCES ${FLAT_GGML_SOURCES}) + add_compile_options(-DGGML_BINDINGS_FLAT) +endif() + add_library(ggml - ggml-backend-reg.cpp) + ${GGML_LIBRARY_SOURCES} + ) target_link_libraries(ggml PUBLIC ggml-base) diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 405d8e31514..6ac3998f726 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -24,6 +24,10 @@ # include #endif +#ifdef GGML_BINDINGS_FLAT +#include "ggml-flat.h" +#endif + // Backend registry #ifdef GGML_USE_CPU #include "ggml-cpu.h" @@ -584,3 +588,11 @@ void ggml_backend_load_all_from_path(const char * dir_path) { ggml_backend_load(backend_path); } } + +#ifdef GGML_BINDINGS_FLAT +ggml_backend_reg_t ggml_backend_try_load_best(const char * name, const char * dir_path) { + bool silent = true; + fprintf(stderr, "%s: (%s)\n", __func__, name); + return ggml_backend_load_best(name, silent, dir_path); +} +#endif diff --git a/ggml/src/ggml-flat.cpp b/ggml/src/ggml-flat.cpp new file mode 100644 index 00000000000..3aa22e7ee3b --- /dev/null +++ b/ggml/src/ggml-flat.cpp @@ -0,0 +1,38 @@ + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#endif + +#include "ggml-backend.h" +#include "ggml-backend-impl.h" +#include "ggml-alloc.h" +#include "ggml-impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include +#endif + +#include "ggml-flat.h" + +#ifdef GGML_BINDINGS_FLAT +// ggml_backend_reg_t ggml_backend_try_load_best(const char * name, const char * dir_path); + +#endif + + + diff --git a/ggml/src/ggml-flat.h b/ggml/src/ggml-flat.h new file mode 100644 index 00000000000..81dccbf25ec --- /dev/null +++ b/ggml/src/ggml-flat.h @@ -0,0 +1,31 @@ +#pragma once + +#ifdef WHISPER_BINDINGS_FLAT +#define GGML_BINDINGS_FLAT +#endif + +#ifdef GGML_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BUILD +# define GGML_FLAT_API __declspec(dllexport) extern +# else +# define GGML_FLAT_API __declspec(dllimport) extern +# endif +# else +# define GGML_FLAT_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_FLAT_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + #ifdef GGML_BINDINGS_FLAT + GGML_FLAT_API ggml_backend_reg_t ggml_backend_try_load_best(const char * name, const char * dir_path); + #endif + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 05a2f4e630a..b2c5c3a4979 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -2,9 +2,11 @@ #define CL_USE_DEPRECATED_OPENCL_1_2_APIS // suppress warnings in CL headers for GCC and Clang -#pragma GCC diagnostic ignored "-Woverlength-strings" -#ifdef __clang__ -#pragma GCC diagnostic ignored "-Wgnu-anonymous-struct" +#ifndef _MSC_VER + #pragma GCC diagnostic ignored "-Woverlength-strings" + #ifdef __clang__ + #pragma GCC diagnostic ignored "-Wgnu-anonymous-struct" + #endif #endif #include "ggml-opencl.h" diff --git a/include/whisper.h b/include/whisper.h index 1e1375033ad..455f572af4a 100644 --- a/include/whisper.h +++ b/include/whisper.h @@ -668,6 +668,30 @@ extern "C" { // Get the no_speech probability for the specified segment WHISPER_API float whisper_full_get_segment_no_speech_prob (struct whisper_context * ctx, int i_segment); WHISPER_API float whisper_full_get_segment_no_speech_prob_from_state(struct whisper_state * state, int i_segment); + + // For whisper-flat.cpp to expose + #ifdef WHISPER_BINDINGS_FLAT + struct whisper_activity { + float sample_ms; + float encode_ms; + float decode_ms; + float batchd_ms; + float prompt_ms; + int32_t n_sample = 0; // number of tokens sampled + int32_t n_encode = 0; // number of encoder calls + int32_t n_decode = 0; // number of decoder calls with n_tokens == 1 (text-generation) + int32_t n_batchd = 0; // number of decoder calls with n_tokens < 16 (batch decoding) + int32_t n_prompt = 0; // number of decoder calls with n_tokens > 1 (prompt encoding) + }; + + const char * whisper_get_system_info_json(void); + struct whisper_state * whisper_get_state_from_context(struct whisper_context * ctx); + struct whisper_activity * whisper_get_activity_with_state(struct whisper_state * state); + ggml_backend_t whisper_get_preferred_backend(struct whisper_state * state); + ggml_backend_t whisper_get_indexed_backend(struct whisper_state* state, size_t i); + size_t whisper_get_backend_count(struct whisper_state* state); + #endif + #ifdef __cplusplus } #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a091e66a25f..46309cb0b25 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -100,10 +100,25 @@ endif() # whisper +set(WHISPER_LIBRARY_SOURCES + ../include/whisper.h + whisper-arch.h + whisper.cpp + ) + +if(WHISPER_BINDINGS_FLAT) + message(STATUS "Adding FLAT Whisper binding extras") + + set(FLAT_WHISPER_SOURCES + whisper-flat.cpp + ) + + list(APPEND WHISPER_LIBRARY_SOURCES ${FLAT_WHISPER_SOURCES}) + set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_BINDINGS_FLAT) +endif() + add_library(whisper - ../include/whisper.h - whisper-arch.h - whisper.cpp + ${WHISPER_LIBRARY_SOURCES} ) # Set the version numbers diff --git a/src/whisper-flat.cpp b/src/whisper-flat.cpp new file mode 100644 index 00000000000..12a21328588 --- /dev/null +++ b/src/whisper-flat.cpp @@ -0,0 +1,34 @@ +#include "whisper.h" +#include "ggml-backend.h" + +#include "whisper-flat.h" + +#ifdef WHISPER_BINDINGS_FLAT +void whisper_flat_backend_load_all(void) { + ggml_backend_load_all(); +} + +const char * whisper_flat_get_system_info_json(void) { + return whisper_get_system_info_json(); +} + +struct whisper_state * whisper_flat_get_state_from_context(struct whisper_context * ctx) { + return whisper_get_state_from_context(ctx); +} + +struct whisper_activity * whisper_flat_get_activity_with_state(struct whisper_state * state) { + return whisper_get_activity_with_state(state); +} + +ggml_backend_t whisper_flat_get_preferred_backend(struct whisper_state * state) { + return whisper_get_preferred_backend(state); +} + +ggml_backend_t whisper_flat_get_indexed_backend(struct whisper_state* state, size_t i) { + return whisper_get_indexed_backend(state, i); +} + +size_t whisper_flat_get_backend_count(struct whisper_state* state) { + return whisper_get_backend_count(state); +} +#endif diff --git a/src/whisper-flat.h b/src/whisper-flat.h new file mode 100644 index 00000000000..6998b6deb3c --- /dev/null +++ b/src/whisper-flat.h @@ -0,0 +1,36 @@ +#pragma once + +#ifdef WHISPER_SHARED +# ifdef _WIN32 +# ifdef WHISPER_BUILD +# define WHISPER_FLAT_API __declspec(dllexport) +# else +# define WHISPER_FLAT_API __declspec(dllimport) +# endif +# else +# define WHISPER_FLAT_API __attribute__ ((visibility ("default"))) +# endif +#else +# define WHISPER_FLAT_API +#endif + +#include "whisper.h" + +#ifdef __cplusplus +extern "C" { +#endif + + #ifdef WHISPER_BINDINGS_FLAT + WHISPER_FLAT_API void whisper_flat_backend_load_all(void); + WHISPER_FLAT_API struct whisper_activity * whisper_flat_get_activity_with_state(struct whisper_state * state); + WHISPER_FLAT_API struct whisper_state * whisper_flat_get_state_from_context(struct whisper_context * ctx); + WHISPER_FLAT_API const char * whisper_flat_get_system_info_json(void); + WHISPER_FLAT_API ggml_backend_t whisper_flat_get_preferred_backend(struct whisper_state * state); + WHISPER_FLAT_API ggml_backend_t whisper_flat_get_indexed_backend(struct whisper_state* state, size_t i); + WHISPER_FLAT_API size_t whisper_flat_get_backend_count(struct whisper_state* state); + #endif + +#ifdef __cplusplus +} +#endif + diff --git a/src/whisper.cpp b/src/whisper.cpp index f8707e73098..30a4bedfedf 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -204,6 +204,7 @@ static bool ggml_graph_compute_helper( return t; } +#ifndef WHISPER_BINDINGS_FLAT static void whisper_load_backends() { #ifdef GGML_BACKEND_DL static std::once_flag flag; @@ -212,6 +213,7 @@ static void whisper_load_backends() { }); #endif } +#endif // TODO: move these functions to ggml-base with support for ggml-backend? @@ -1309,14 +1311,19 @@ static size_t aheads_masks_nbytes(struct whisper_aheads_masks & aheads_masks) { static ggml_backend_t whisper_backend_init_gpu(const whisper_context_params & params) { ggml_log_set(g_state.log_callback, g_state.log_callback_user_data); + #ifndef WHISPER_BINDINGS_FLAT whisper_load_backends(); - + #endif + ggml_backend_dev_t dev = nullptr; int cnt = 0; if (params.use_gpu) { for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev_cur = ggml_backend_dev_get(i); + if(dev_cur == nullptr) { + continue; + } if (ggml_backend_dev_type(dev_cur) == GGML_BACKEND_DEVICE_TYPE_GPU) { if (cnt == 0 || cnt == params.gpu_device) { dev = dev_cur; @@ -1355,6 +1362,9 @@ static std::vector whisper_backend_init(const whisper_context_pa // ACCEL backends for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if(dev == nullptr) { + continue; + } if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) { WHISPER_LOG_INFO("%s: using %s backend\n", __func__, ggml_backend_dev_name(dev)); ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); @@ -1386,6 +1396,9 @@ static buft_list_t make_buft_list(whisper_context_params & params) { int cnt = 0; for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if(dev == nullptr) { + continue; + } if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_GPU) { if (cnt == 0 || cnt == params.gpu_device) { auto * buft = ggml_backend_dev_buffer_type(dev); @@ -4317,7 +4330,9 @@ static int whisper_has_openvino(void) { const char * whisper_print_system_info(void) { static std::string s; + #ifndef WHISPER_BINDINGS_FLAT whisper_load_backends(); + #endif s = ""; s += "WHISPER : "; @@ -6772,7 +6787,9 @@ WHISPER_API int whisper_bench_ggml_mul_mat(int n_threads) { } WHISPER_API const char * whisper_bench_ggml_mul_mat_str(int n_threads) { + #ifndef WHISPER_BINDINGS_FLAT whisper_load_backends(); + #endif static std::string s; s = ""; @@ -7546,3 +7563,119 @@ static void whisper_log_callback_default(ggml_log_level level, const char * text fputs(text, stderr); fflush(stderr); } + +#ifdef WHISPER_BINDINGS_FLAT +// The optional WHISPER_BINDINGS_FLAT code is of limited use for most +// developers. The intended audience is those who are binding to +// another language. +// C++ specific constructs such as, but not limited to, std:vector +// are used frequently but unavailable to non-C++ developers. +// As such it is placed at the end of source in recognition of its +// limited appeal + +// whisper_get_system_info_json +// Returns system info as json, useful for language bindings +// NOTE : While testing features->value always returned an int. +// Even though ints are invariably returned they may be +// some values that return other types. +// This function returns everything quoted (i.e. as a string) +// and leaves type-casting to the caller. +// This also removes the unlikely but plausible state of +// a string being returned unquoted (thus invalidating JSON) + +const char * whisper_get_system_info_json(void) { + static std::string s; + + s = "{"; + s += "\"WHISPER\":{"; + s += "\"COREML\":\"" + std::to_string(whisper_has_coreml()) + "\","; + s += "\"OPENVINO\":\"" + std::to_string(whisper_has_openvino()) + "\"}"; + + for (size_t i = 0; i < ggml_backend_reg_count(); i++) { + auto * reg = ggml_backend_reg_get(i); + auto * get_features_fn = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features"); + if (get_features_fn) { + ggml_backend_feature * features = get_features_fn(reg); + s += ",\""; + s += ggml_backend_reg_name(reg); + s += "\":{"; + auto first = true; + for (; features->name; features++) { + if(first) { + first = false; + } else { + s += ","; + } + s += "\""; + s += features->name; + s += "\":\""; + s += features->value; + s += "\""; + } + s += "}"; + } + } + s += "}"; + + return s.c_str(); +} + +// whisper_get_state_from_context +// Returns state from supplied context pointer +// This is mainly a helper for non-C++ language bindings as whisper_context +// has embedded C++ specific types (e.g. maps and vectors) +// The returned whisper_state value can be treated as a an opaque object +// that need merely be 'plugged-in' to the following and other existing +// functions to obtain relevant information or functionality +struct whisper_state * whisper_get_state_from_context(struct whisper_context * ctx) { + if (!ctx->state) { + return nullptr; + } + + return ctx->state; +} + +// whisper_get_activity_with_state +// As the data is in a c++ specific struct +struct whisper_activity * whisper_get_activity_with_state(struct whisper_state * state) { + if (state == nullptr) { + return nullptr; + } + whisper_activity * activity = new whisper_activity; + + activity->sample_ms = 1e-3f * state->t_sample_us / std::max(1, state->n_sample); + activity->encode_ms = 1e-3f * state->t_encode_us / std::max(1, state->n_encode); + activity->decode_ms = 1e-3f * state->t_decode_us / std::max(1, state->n_decode); + activity->batchd_ms = 1e-3f * state->t_batchd_us / std::max(1, state->n_batchd); + activity->prompt_ms = 1e-3f * state->t_prompt_us / std::max(1, state->n_prompt); + activity->n_sample = state->n_sample; + activity->n_encode = state->n_encode; + activity->n_decode = state->n_decode; + activity->n_batchd = state->n_batchd; + activity->n_prompt = state->n_prompt; + + return activity; +} + +ggml_backend_t whisper_get_preferred_backend(struct whisper_state * state) { + if (state->backends.empty()) { + return nullptr; + } + + return state->backends[0]; +} + +ggml_backend_t whisper_get_indexed_backend(struct whisper_state* state, size_t i) { + if (state->backends.empty()) { + return nullptr; + } + if (i >= state->backends.size()) { + return nullptr; + } + return state->backends[i]; +} + +size_t whisper_get_backend_count(struct whisper_state* state) { + return state->backends.size(); +} +#endif