diff --git a/llm/dynamic_shim.h b/llm/dynamic_shim.h index 5e4e78b7..116ca722 100644 --- a/llm/dynamic_shim.h +++ b/llm/dynamic_shim.h @@ -1,6 +1,6 @@ #include -#include "server.h" +#include "ext_server.h" #ifdef __cplusplus extern "C" { diff --git a/llm/ext_server.go b/llm/ext_server.go index 0f7d441a..87cd6600 100644 --- a/llm/ext_server.go +++ b/llm/ext_server.go @@ -1,7 +1,7 @@ package llm /* -#cgo CFLAGS: -I${SRCDIR}/llama.cpp/gguf -I${SRCDIR}/llama.cpp/gguf/common -I${SRCDIR}/llama.cpp/gguf/examples/server +#cgo CFLAGS: -I${SRCDIR}/llama.cpp -I${SRCDIR}/llama.cpp/gguf -I${SRCDIR}/llama.cpp/gguf/common -I${SRCDIR}/llama.cpp/gguf/examples/server #cgo CFLAGS: -DNDEBUG -DLLAMA_SERVER_LIBRARY=1 -D_XOPEN_SOURCE=600 -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 #cgo CFLAGS: -Wmissing-noreturn -Wall -Wextra -Wcast-qual -Wno-unused-function -Wno-array-bounds #cgo CPPFLAGS: -Ofast -Wall -Wextra -Wno-unused-function -Wno-unused-variable -Wno-deprecated-declarations -Wno-unused-but-set-variable @@ -22,11 +22,11 @@ package llm #cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libllama.a #cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libggml_static.a #cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lm -#cgo windows LDFLAGS: -L${SRCDIR}/llama.cpp/gguf/build/wincpu/dist/lib -#cgo windows LDFLAGS: -lcpu_server -lpthread +#cgo windows LDFLAGS: -L${SRCDIR}/llama.cpp/gguf/build/lib +#cgo windows LDFLAGS: -lext_server_shared -lpthread #include -#include "server.h" +#include "ext_server.h" */ import "C" diff --git a/llm/llama.cpp/ext_server.cpp b/llm/llama.cpp/ext_server.cpp new file mode 100644 index 00000000..362db4c1 --- /dev/null +++ b/llm/llama.cpp/ext_server.cpp @@ -0,0 +1,282 @@ +#include "ext_server.h" + +// Necessary evil since the server types are not defined in a header +#include "server.cpp" + +// Expose the llama server as a callable extern "C" API +llama_server_context *llama = NULL; +std::atomic ext_server_running(false); +std::thread ext_server_thread; + +void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err) { +#if SERVER_VERBOSE != 1 + LOG_TEE("disabling verbose llm logging\n"); + log_disable(); +#endif + assert(err != NULL && sparams != NULL); + err->id = 0; + err->msg[0] = '\0'; + try { + llama = new llama_server_context; + log_set_target(stdout); + gpt_params params; + params.n_ctx = sparams->n_ctx; + params.n_batch = sparams->n_batch; + if (sparams->n_threads > 0) { + params.n_threads = sparams->n_threads; + } + params.n_parallel = sparams->n_parallel; + params.rope_freq_base = sparams->rope_freq_base; + params.rope_freq_scale = sparams->rope_freq_scale; + + if (sparams->memory_f16) { + params.cache_type_k = "f16"; + params.cache_type_v = "f16"; + } else { + params.cache_type_k = "f32"; + params.cache_type_v = "f32"; + } + + params.n_gpu_layers = sparams->n_gpu_layers; + params.main_gpu = sparams->main_gpu; + params.use_mlock = sparams->use_mlock; + params.use_mmap = sparams->use_mmap; + params.numa = sparams->numa; + params.embedding = sparams->embedding; + if (sparams->model != NULL) { + params.model = sparams->model; + } + + for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; + la = la->next) { + params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale)); + } + + if (sparams->mmproj != NULL) { + params.mmproj = std::string(sparams->mmproj); + } + + llama_backend_init(params.numa); + + // load the model + if (!llama->load_model(params)) { + // TODO - consider modifying the logging logic or patching load_model so + // we can capture more detailed error messages and pass them back to the + // caller for better UX + err->id = -1; + snprintf(err->msg, err->msg_len, "error loading model %s", + params.model.c_str()); + return; + } + + llama->initialize(); + } catch (std::exception &e) { + err->id = -1; + snprintf(err->msg, err->msg_len, "exception %s", e.what()); + } catch (...) { + err->id = -1; + snprintf(err->msg, err->msg_len, + "Unknown exception initializing llama server"); + } +} + +void llama_server_start() { + assert(llama != NULL); + // TODO mutex to protect thread creation + ext_server_thread = std::thread([&]() { + ext_server_running = true; + try { + LOG_TEE("llama server main loop starting\n"); + ggml_time_init(); + while (ext_server_running.load()) { + if (!llama->update_slots()) { + LOG_TEE( + "unexpected error in llama server update_slots - exiting main " + "loop\n"); + break; + } + } + } catch (std::exception &e) { + LOG_TEE("caught exception in llama server main loop: %s\n", e.what()); + } catch (...) { + LOG_TEE("caught unknown exception in llama server main loop\n"); + } + LOG_TEE("\nllama server shutting down\n"); + llama_backend_free(); + }); +} + +void llama_server_stop() { + assert(llama != NULL); + // TODO - too verbose, remove once things are solid + LOG_TEE("requesting llama server shutdown\n"); + ext_server_running = false; + ext_server_thread.join(); + delete llama; + llama = NULL; + LOG_TEE("llama server shutdown complete\n"); +} + +void llama_server_completion(const char *json_req, ext_server_resp_t *resp) { + assert(llama != NULL && json_req != NULL && resp != NULL); + resp->id = -1; + resp->msg[0] = '\0'; + try { + json data = json::parse(json_req); + resp->id = llama->request_completion(data, false, false, -1); + } catch (std::exception &e) { + snprintf(resp->msg, resp->msg_len, "exception %s", e.what()); + } catch (...) { + snprintf(resp->msg, resp->msg_len, "Unknown exception during completion"); + } +} + +void llama_server_completion_next_result(const int task_id, + ext_server_task_result_t *resp) { + assert(llama != NULL && resp != NULL); + std::string msg; + resp->id = -1; + resp->stop = false; + resp->error = false; + resp->json_resp = NULL; + std::string result_json; + try { + task_result result = llama->next_result(task_id); + result_json = + result.result_json.dump(-1, ' ', false, json::error_handler_t::replace); + resp->id = result.id; + resp->stop = result.stop; + resp->error = result.error; + if (result.error) { + llama->request_cancel(task_id); + } else if (result.stop) { + llama->request_cancel(task_id); + } + } catch (std::exception &e) { + resp->error = true; + resp->id = -1; + result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}"; + LOG_TEE("llama server completion exception %s\n", e.what()); + } catch (...) { + resp->error = true; + resp->id = -1; + result_json = "{\"error\":\"Unknown exception during completion\"}"; + LOG_TEE("llama server completion unknown exception\n"); + } + const std::string::size_type size = result_json.size() + 1; + resp->json_resp = new char[size]; + snprintf(resp->json_resp, size, "%s", result_json.c_str()); +} + +void llama_server_release_task_result(ext_server_task_result_t *result) { + if (result == NULL || result->json_resp == NULL) { + return; + } + delete[] result->json_resp; +} + +void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) { + assert(llama != NULL && err != NULL); + err->id = 0; + err->msg[0] = '\0'; + try { + llama->request_cancel(task_id); + } catch (std::exception &e) { + err->id = -1; + snprintf(err->msg, err->msg_len, "exception %s", e.what()); + } catch (...) { + err->id = -1; + snprintf(err->msg, err->msg_len, + "Unknown exception completion cancel in llama server"); + } +} + +void llama_server_tokenize(const char *json_req, char **json_resp, + ext_server_resp_t *err) { + assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL); + *json_resp = NULL; + err->id = 0; + err->msg[0] = '\0'; + try { + const json body = json::parse(json_req); + std::vector tokens; + if (body.count("content") != 0) { + tokens = llama->tokenize(body["content"], false); + } + const json data = format_tokenizer_response(tokens); + std::string result_json = data.dump(); + const std::string::size_type size = result_json.size() + 1; + *json_resp = new char[size]; + snprintf(*json_resp, size, "%s", result_json.c_str()); + } catch (std::exception &e) { + err->id = -1; + snprintf(err->msg, err->msg_len, "exception %s", e.what()); + } catch (...) { + err->id = -1; + snprintf(err->msg, err->msg_len, "Unknown exception during tokenize"); + } +} + +void llama_server_release_json_resp(char **json_resp) { + if (json_resp == NULL || *json_resp == NULL) { + return; + } + delete[] *json_resp; +} + +void llama_server_detokenize(const char *json_req, char **json_resp, + ext_server_resp_t *err) { + assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL); + *json_resp = NULL; + err->id = 0; + err->msg[0] = '\0'; + try { + const json body = json::parse(json_req); + std::string content; + if (body.count("tokens") != 0) { + const std::vector tokens = body["tokens"]; + content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend()); + } + const json data = format_detokenized_response(content); + std::string result_json = data.dump(); + const std::string::size_type size = result_json.size() + 1; + *json_resp = new char[size]; + snprintf(*json_resp, size, "%s", result_json.c_str()); + } catch (std::exception &e) { + err->id = -1; + snprintf(err->msg, err->msg_len, "exception %s", e.what()); + } catch (...) { + err->id = -1; + snprintf(err->msg, err->msg_len, "Unknown exception during detokenize"); + } +} + +void llama_server_embedding(const char *json_req, char **json_resp, + ext_server_resp_t *err) { + assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL); + *json_resp = NULL; + err->id = 0; + err->msg[0] = '\0'; + try { + const json body = json::parse(json_req); + json prompt; + if (body.count("content") != 0) { + prompt = body["content"]; + } else { + prompt = ""; + } + const int task_id = llama->request_completion( + {{"prompt", prompt}, {"n_predict", 0}}, false, true, -1); + task_result result = llama->next_result(task_id); + std::string result_json = result.result_json.dump(); + const std::string::size_type size = result_json.size() + 1; + *json_resp = new char[size]; + snprintf(*json_resp, size, "%s", result_json.c_str()); + } catch (std::exception &e) { + err->id = -1; + snprintf(err->msg, err->msg_len, "exception %s", e.what()); + } catch (...) { + err->id = -1; + snprintf(err->msg, err->msg_len, "Unknown exception during embedding"); + } +} \ No newline at end of file diff --git a/llm/llama.cpp/ext_server.h b/llm/llama.cpp/ext_server.h new file mode 100644 index 00000000..0a584de0 --- /dev/null +++ b/llm/llama.cpp/ext_server.h @@ -0,0 +1,94 @@ +#if defined(LLAMA_SERVER_LIBRARY) +#ifndef LLAMA_SERVER_H +#define LLAMA_SERVER_H +#include +#include +#include +#include + +int __main(int argc, char **argv); + +// This exposes extern C entrypoints into the llama_server +// To enable the server compile with LLAMA_SERVER_LIBRARY + +#ifdef __cplusplus +extern "C" { +#endif +typedef struct ext_server_resp { + int id; // < 0 on error + size_t msg_len; // caller must allocate msg and set msg_len + char *msg; +} ext_server_resp_t; + +// Allocated and freed by caller +typedef struct ext_server_lora_adapter { + char *adapter; + float scale; + struct ext_server_lora_adapter *next; +} ext_server_lora_adapter_t; + +// Allocated and freed by caller +typedef struct ext_server_params { + char *model; + uint32_t n_ctx; // token context window, 0 = from model + uint32_t n_batch; // prompt processing maximum batch size + uint32_t n_threads; // number of threads to use for generation + int32_t n_parallel; // number of parallel sequences to decodewra + float rope_freq_base; // RoPE base frequency, 0 = from model + float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model + bool memory_f16; // use f16 instead of f32 for memory kv + int32_t n_gpu_layers; // number of layers to store in VRAM (-1 - use default) + int32_t main_gpu; // the GPU that is used for scratch and small tensors + bool use_mlock; // force system to keep model in RAM + bool use_mmap; // use mmap if possible + bool numa; // attempt optimizations that help on some NUMA systems + bool embedding; // get only sentence embedding + ext_server_lora_adapter_t *lora_adapters; + char *mmproj; +} ext_server_params_t; + +typedef struct ext_server_task_result { + int id; + bool stop; + bool error; + char *json_resp; // null terminated, memory managed by ext_server +} ext_server_task_result_t; + +// Initialize the server once per process +// err->id = 0 for success and err->msg[0] = NULL +// err->id != 0 for failure, and err->msg contains error message +void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err); + +// Run the main loop, called once per init +void llama_server_start(); +// Stop the main loop and free up resources allocated in init and start. Init +// must be called again to reuse +void llama_server_stop(); + +// json_req null terminated string, memory managed by caller +// resp->id >= 0 on success (task ID) +// resp->id < 0 on error, and resp->msg contains error message +void llama_server_completion(const char *json_req, ext_server_resp_t *resp); + +// Caller must call llama_server_release_task_result to free resp->json_resp +void llama_server_completion_next_result(const int task_id, + ext_server_task_result_t *result); +void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err); +void llama_server_release_task_result(ext_server_task_result_t *result); + +// Caller must call llama_server_releaes_json_resp to free json_resp if err.id < +// 0 +void llama_server_tokenize(const char *json_req, char **json_resp, + ext_server_resp_t *err); +void llama_server_detokenize(const char *json_req, char **json_resp, + ext_server_resp_t *err); +void llama_server_embedding(const char *json_req, char **json_resp, + ext_server_resp_t *err); +void llama_server_release_json_resp(char **json_resp); + +#ifdef __cplusplus +} +#endif + +#endif +#endif // LLAMA_SERVER_LIBRARY \ No newline at end of file diff --git a/llm/llama.cpp/gen_common.sh b/llm/llama.cpp/gen_common.sh index 0bfd8d8f..d0795981 100644 --- a/llm/llama.cpp/gen_common.sh +++ b/llm/llama.cpp/gen_common.sh @@ -25,18 +25,21 @@ git_module_setup() { } apply_patches() { - if [ -n "${OLLAMA_SKIP_PATCHING}" ]; then - echo "Skipping submodule patching" - return + # Wire up our CMakefile + if ! grep ollama.txt gguf/examples/server/CMakeLists.txt; then + echo 'include (../../../ollama.txt)' >>gguf/examples/server/CMakeLists.txt fi - # Workaround git apply not handling creation well for iteration - rm -f gguf/examples/server/server.h - for patch in ${PATCHES}; do - git -C gguf apply ../patches/${patch} - done + # Avoid duplicate main symbols when we link into the cgo binary + sed -e 's/int main(/int __main(/g' <./gguf/examples/server/server.cpp >./gguf/examples/server/server.cpp.tmp && + mv ./gguf/examples/server/server.cpp.tmp ./gguf/examples/server/server.cpp } build() { cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS} cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8 } + +# Keep the local tree clean after we're done with the build +cleanup() { + (cd gguf/examples/server/ && git checkout CMakeLists.txt server.cpp) +} diff --git a/llm/llama.cpp/gen_darwin.sh b/llm/llama.cpp/gen_darwin.sh index 1364e9d1..1eb5b074 100755 --- a/llm/llama.cpp/gen_darwin.sh +++ b/llm/llama.cpp/gen_darwin.sh @@ -12,19 +12,20 @@ init_vars CMAKE_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_METAL=on ${CMAKE_DEFS}" BUILD_DIR="gguf/build/metal" case "${GOARCH}" in - "amd64") - CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}" - ;; - "arm64") - CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 ${CMAKE_DEFS}" - ;; - *) - echo "GOARCH must be set" - echo "this script is meant to be run from within go generate" - exit 1 - ;; +"amd64") + CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}" + ;; +"arm64") + CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 ${CMAKE_DEFS}" + ;; +*) + echo "GOARCH must be set" + echo "this script is meant to be run from within go generate" + exit 1 + ;; esac git_module_setup apply_patches -build \ No newline at end of file +build +cleanup diff --git a/llm/llama.cpp/gen_linux.sh b/llm/llama.cpp/gen_linux.sh index 3d659fff..ab0229db 100755 --- a/llm/llama.cpp/gen_linux.sh +++ b/llm/llama.cpp/gen_linux.sh @@ -87,3 +87,5 @@ if [ -d "${ROCM_PATH}" ]; then -Wl,-rpath,/opt/rocm/lib,-rpath,/opt/amdgpu/lib/x86_64-linux-gnu/ \ -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu fi + +cleanup diff --git a/llm/llama.cpp/gen_windows.ps1 b/llm/llama.cpp/gen_windows.ps1 index 2f2f856d..02ff2ffe 100644 --- a/llm/llama.cpp/gen_windows.ps1 +++ b/llm/llama.cpp/gen_windows.ps1 @@ -5,7 +5,7 @@ $ErrorActionPreference = "Stop" function init_vars { $script:patches = @("0001-Expose-callable-API-for-server.patch") $script:cmakeDefs = @("-DBUILD_SHARED_LIBS=on", "-DLLAMA_NATIVE=off", "-DLLAMA_F16C=off", "-DLLAMA_FMA=off", "-DLLAMA_AVX512=off", "-DLLAMA_AVX2=off", "-DLLAMA_AVX=on", "-DLLAMA_K_QUANTS=on", "-DLLAMA_ACCELERATE=on", "-A","x64") - + $script:cmakeTargets = @("ggml", "ggml_static", "llama", "build_info", "common", "ext_server_shared", "llava_static") if ($env:CGO_CFLAGS -contains "-g") { $script:cmakeDefs += @("-DCMAKE_VERBOSE_MAKEFILE=on", "-DLLAMA_SERVER_VERBOSE=on") $script:config = "RelWithDebInfo" @@ -24,12 +24,14 @@ function git_module_setup { } function apply_patches { - rm -erroraction ignore -path "gguf/examples/server/server.h" - foreach ($patch in $script:patches) { - write-host "Applying patch $patch" - & git -C gguf apply ../patches/$patch - if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} + # Wire up our CMakefile + if (!(Select-String -Path "gguf/examples/server/CMakeLists.txt" -Pattern 'ollama.txt')) { + Add-Content -Path "gguf/examples/server/CMakeLists.txt" -Value 'include (../../../ollama.txt)' } + # Avoid duplicate main symbols when we link into the cgo binary + $content = Get-Content -Path "./gguf/examples/server/server.cpp" + $content = $content -replace 'int main\(', 'int __main(' + Set-Content -Path "./gguf/examples/server/server.cpp" -Value $content } function build { @@ -37,16 +39,14 @@ function build { & cmake --version & cmake -S gguf -B $script:buildDir $script:cmakeDefs if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} - write-host "building with: cmake --build $script:buildDir --config $script:config" - & cmake --build $script:buildDir --config $script:config + write-host "building with: cmake --build $script:buildDir --config $script:config ($script:cmakeTargets | ForEach-Object { "--target", $_ })" + & cmake --build $script:buildDir --config $script:config ($script:cmakeTargets | ForEach-Object { "--target", $_ }) if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} } -function install { - rm -erroraction ignore -recurse -force -path $script:installDir - & cmake --install $script:buildDir --prefix $script:installDir --config $script:config - if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} - +function cleanup { + Set-Location "gguf/examples/server" + git checkout CMakeLists.txt server.cpp } init_vars @@ -55,15 +55,14 @@ apply_patches # first build CPU based $script:buildDir="gguf/build/wincpu" -$script:installDir="gguf/build/wincpu/dist" build # install md gguf/build/lib -ea 0 md gguf/build/wincpu/dist/lib -ea 0 -mv gguf/build/wincpu/bin/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.dll - +cp -force gguf/build/wincpu/bin/$script:config/ext_server_shared.dll gguf/build/lib/ext_server_shared.dll +cp -force gguf/build/wincpu/bin/$script:config/llama.dll gguf/build/lib/llama.dll # Nope, this barfs on lots of symbol problems #mv gguf/build/wincpu/examples/server/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.lib @@ -79,11 +78,10 @@ mv gguf/build/wincpu/bin/$script:config/ext_server_shared.dll gguf/build/wincpu/ # Then build cuda as a dynamically loaded library init_vars $script:buildDir="gguf/build/wincuda" -$script:installDir="gguf/build/wincuda/dist" $script:cmakeDefs += @("-DLLAMA_CUBLAS=ON", "-DBUILD_SHARED_LIBS=on") build -install -cp gguf/build/wincuda/dist/bin/ext_server_shared.dll gguf/build/lib/cuda_server.dll +# install +cp -force gguf/build/wincuda/bin/$script:config/ext_server_shared.dll gguf/build/lib/cuda_server.dll # TODO - more to do here to create a usable dll @@ -91,3 +89,7 @@ cp gguf/build/wincuda/dist/bin/ext_server_shared.dll gguf/build/lib/cuda_server. # TODO - implement ROCm support on windows md gguf/build/winrocm/lib -ea 0 echo $null >> gguf/build/winrocm/lib/.generated + +cleanup + +write-host "go generate completed" \ No newline at end of file diff --git a/llm/llama.cpp/ollama.txt b/llm/llama.cpp/ollama.txt new file mode 100644 index 00000000..9553ad5e --- /dev/null +++ b/llm/llama.cpp/ollama.txt @@ -0,0 +1,29 @@ +# Ollama specific CMakefile to include in llama.cpp/examples/server + +set(TARGET ext_server) +option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON) +add_library(${TARGET} STATIC ../../../ext_server.cpp) +target_include_directories(${TARGET} PRIVATE ../../common) +target_include_directories(${TARGET} PRIVATE ../..) +target_include_directories(${TARGET} PRIVATE ../../..) +target_compile_features(${TARGET} PRIVATE cxx_std_11) +target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1) +target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT}) +target_compile_definitions(${TARGET} PRIVATE + SERVER_VERBOSE=$ +) + +if (BUILD_SHARED_LIBS) + set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD) + add_library(ext_server_shared SHARED $) + target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT}) + install(TARGETS ext_server_shared LIBRARY) +endif() + +if (CUDAToolkit_FOUND) + target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) + if (WIN32) + target_link_libraries(ext_server_shared PRIVATE nvml) + endif() +endif() \ No newline at end of file diff --git a/llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch b/llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch deleted file mode 100644 index e1c1b141..00000000 --- a/llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch +++ /dev/null @@ -1,464 +0,0 @@ -From 90c332fe2ef61149b38561d02836e66715df214d Mon Sep 17 00:00:00 2001 -From: Daniel Hiltgen -Date: Mon, 13 Nov 2023 12:25:58 -0800 -Subject: [PATCH] Expose callable API for server - -This adds an extern "C" interface within the example server ---- - examples/server/CMakeLists.txt | 27 ++++ - examples/server/server.cpp | 280 +++++++++++++++++++++++++++++++++ - examples/server/server.h | 89 +++++++++++ - ggml-cuda.cu | 1 + - 4 files changed, 397 insertions(+) - create mode 100644 examples/server/server.h - -diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt -index 859cd12..da2b9bf 100644 ---- a/examples/server/CMakeLists.txt -+++ b/examples/server/CMakeLists.txt -@@ -11,3 +11,30 @@ if (WIN32) - TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32) - endif() - target_compile_features(${TARGET} PRIVATE cxx_std_11) -+ -+set(TARGET ext_server) -+option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON) -+add_library(${TARGET} STATIC server.cpp) -+target_include_directories(${TARGET} PRIVATE ../../common) -+target_include_directories(${TARGET} PRIVATE ../..) -+target_compile_features(${TARGET} PRIVATE cxx_std_11) -+target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1) -+target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT}) -+target_compile_definitions(${TARGET} PRIVATE -+ SERVER_VERBOSE=$ -+) -+ -+if (BUILD_SHARED_LIBS) -+ set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON) -+ target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD) -+ add_library(ext_server_shared SHARED $) -+ target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT}) -+ install(TARGETS ext_server_shared LIBRARY) -+endif() -+ -+if (CUDAToolkit_FOUND) -+ target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) -+ if (WIN32) -+ target_link_libraries(ext_server_shared PRIVATE nvml) -+ endif() -+endif() -\ No newline at end of file -diff --git a/examples/server/server.cpp b/examples/server/server.cpp -index 0403853..07fb05c 100644 ---- a/examples/server/server.cpp -+++ b/examples/server/server.cpp -@@ -5,6 +5,9 @@ - #include "../llava/clip.h" - - #include "stb_image.h" -+#if defined(LLAMA_SERVER_LIBRARY) -+#include "server.h" -+#endif - - #ifndef NDEBUG - // crash the server in debug mode, otherwise send an http 500 error -@@ -2643,6 +2646,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con - } - } - -+#ifndef LLAMA_SERVER_LIBRARY - int main(int argc, char **argv) - { - #if SERVER_VERBOSE != 1 -@@ -3123,3 +3127,279 @@ int main(int argc, char **argv) - llama_backend_free(); - return 0; - } -+ -+#else // LLAMA_SERVER_LIBRARY -+// Expose the llama server as a callable extern "C" API -+llama_server_context *llama = NULL; -+std::atomic ext_server_running(false); -+std::thread ext_server_thread; -+ -+void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err) -+{ -+#if SERVER_VERBOSE != 1 -+ LOG_TEE("disabling verbose llm logging\n"); -+ log_disable(); -+#endif -+ assert(err != NULL && sparams != NULL); -+ err->id = 0; -+ err->msg[0] = '\0'; -+ try { -+ llama = new llama_server_context; -+ log_set_target(stdout); -+ gpt_params params; -+ params.n_ctx = sparams->n_ctx; -+ params.n_batch = sparams->n_batch; -+ if (sparams->n_threads > 0) { -+ params.n_threads = sparams->n_threads; -+ } -+ params.n_parallel = sparams->n_parallel; -+ params.rope_freq_base = sparams->rope_freq_base; -+ params.rope_freq_scale = sparams->rope_freq_scale; -+ -+ if (sparams->memory_f16) { -+ params.cache_type_k = "f16"; -+ params.cache_type_v = "f16"; -+ } else { -+ params.cache_type_k = "f32"; -+ params.cache_type_v = "f32"; -+ } -+ -+ params.n_gpu_layers = sparams->n_gpu_layers; -+ params.main_gpu = sparams->main_gpu; -+ params.use_mlock = sparams->use_mlock; -+ params.use_mmap = sparams->use_mmap; -+ params.numa = sparams->numa; -+ params.embedding = sparams->embedding; -+ if (sparams->model != NULL) { -+ params.model = sparams->model; -+ } -+ -+ for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) { -+ params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale)); -+ } -+ -+ if (sparams->mmproj != NULL) { -+ params.mmproj = std::string(sparams->mmproj); -+ } -+ -+ llama_backend_init(params.numa); -+ -+ // load the model -+ if (!llama->load_model(params)) -+ { -+ // TODO - consider modifying the logging logic or patching load_model so we can capture more detailed error messages -+ // and pass them back to the caller for better UX -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "error loading model %s", params.model.c_str()); -+ return; -+ } -+ -+ llama->initialize(); -+ } catch (std::exception &e) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "exception %s", e.what()); -+ } catch (...) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "Unknown exception initializing llama server"); -+ } -+} -+ -+void llama_server_start() -+{ -+ assert(llama != NULL); -+ // TODO mutex to protect thread creation -+ ext_server_thread = std::thread([&]() -+ { -+ ext_server_running = true; -+ try { -+ LOG_TEE("llama server main loop starting\n"); -+ ggml_time_init(); -+ while (ext_server_running.load()) -+ { -+ if (!llama->update_slots()) { -+ LOG_TEE("unexpected error in llama server update_slots - exiting main loop\n"); -+ break; -+ } -+ } -+ } catch (std::exception &e) { -+ LOG_TEE("caught exception in llama server main loop: %s\n", e.what()); -+ } catch (...) { -+ LOG_TEE("caught unknown exception in llama server main loop\n"); -+ } -+ LOG_TEE("\nllama server shutting down\n"); -+ llama_backend_free(); -+ }); -+} -+ -+void llama_server_stop() { -+ assert(llama != NULL); -+ // TODO - too verbose, remove once things are solid -+ LOG_TEE("requesting llama server shutdown\n"); -+ ext_server_running = false; -+ ext_server_thread.join(); -+ delete llama; -+ llama = NULL; -+ LOG_TEE("llama server shutdown complete\n"); -+} -+ -+void llama_server_completion(const char *json_req, ext_server_resp_t *resp) { -+ assert(llama != NULL && json_req != NULL && resp != NULL); -+ resp->id = -1; -+ resp->msg[0] = '\0'; -+ try { -+ json data = json::parse(json_req); -+ resp->id = llama->request_completion(data, false, false, -1); -+ } catch (std::exception &e) { -+ snprintf(resp->msg, resp->msg_len, "exception %s", e.what()); -+ } catch (...) { -+ snprintf(resp->msg, resp->msg_len, "Unknown exception during completion"); -+ } -+} -+ -+void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *resp) { -+ assert(llama != NULL && resp != NULL); -+ std::string msg; -+ resp->id = -1; -+ resp->stop = false; -+ resp->error = false; -+ resp->json_resp = NULL; -+ std::string result_json; -+ try { -+ task_result result = llama->next_result(task_id); -+ result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace); -+ resp->id = result.id; -+ resp->stop = result.stop; -+ resp->error = result.error; -+ if (result.error) { -+ llama->request_cancel(task_id); -+ } else if (result.stop) { -+ llama->request_cancel(task_id); -+ } -+ } catch (std::exception &e) { -+ resp->error = true; -+ resp->id = -1; -+ result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}"; -+ } catch (...) { -+ resp->error = true; -+ resp->id = -1; -+ result_json = "{\"error\":\"Unknown exception during completion\"}"; -+ } -+ const std::string::size_type size = result_json.size() + 1; -+ resp->json_resp = new char[size]; -+ snprintf(resp->json_resp, size, "%s", result_json.c_str()); -+} -+ -+void llama_server_release_task_result(ext_server_task_result_t *result) { -+ if (result == NULL || result->json_resp == NULL) { -+ return; -+ } -+ delete[] result->json_resp; -+} -+ -+void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) { -+ assert(llama != NULL && err != NULL); -+ err->id = 0; -+ err->msg[0] = '\0'; -+ try { -+ llama->request_cancel(task_id); -+ } catch (std::exception &e) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "exception %s", e.what()); -+ } catch (...) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "Unknown exception completion cancel in llama server"); -+ } -+} -+ -+void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) { -+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL); -+ *json_resp = NULL; -+ err->id = 0; -+ err->msg[0] = '\0'; -+ try { -+ const json body = json::parse(json_req); -+ std::vector tokens; -+ if (body.count("content") != 0) -+ { -+ tokens = llama->tokenize(body["content"], false); -+ } -+ const json data = format_tokenizer_response(tokens); -+ std::string result_json = data.dump(); -+ const std::string::size_type size = result_json.size() + 1; -+ *json_resp = new char[size]; -+ snprintf(*json_resp, size, "%s", result_json.c_str()); -+ } catch (std::exception &e) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "exception %s", e.what()); -+ } catch (...) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "Unknown exception during tokenize"); -+ } -+} -+ -+void llama_server_release_json_resp(char **json_resp) { -+ if (json_resp == NULL || *json_resp == NULL) { -+ return; -+ } -+ delete[] *json_resp; -+} -+ -+void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) { -+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL); -+ *json_resp = NULL; -+ err->id = 0; -+ err->msg[0] = '\0'; -+ try { -+ const json body = json::parse(json_req); -+ std::string content; -+ if (body.count("tokens") != 0) -+ { -+ const std::vector tokens = body["tokens"]; -+ content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend()); -+ } -+ const json data = format_detokenized_response(content); -+ std::string result_json = data.dump(); -+ const std::string::size_type size = result_json.size() + 1; -+ *json_resp = new char[size]; -+ snprintf(*json_resp, size, "%s", result_json.c_str()); -+ } catch (std::exception &e) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "exception %s", e.what()); -+ } catch (...) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "Unknown exception during detokenize"); -+ } -+} -+ -+void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err) { -+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL); -+ *json_resp = NULL; -+ err->id = 0; -+ err->msg[0] = '\0'; -+ try { -+ const json body = json::parse(json_req); -+ json prompt; -+ if (body.count("content") != 0) -+ { -+ prompt = body["content"]; -+ } -+ else -+ { -+ prompt = ""; -+ } -+ const int task_id = llama->request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1); -+ task_result result = llama->next_result(task_id); -+ std::string result_json = result.result_json.dump(); -+ const std::string::size_type size = result_json.size() + 1; -+ *json_resp = new char[size]; -+ snprintf(*json_resp, size, "%s", result_json.c_str()); -+ } catch (std::exception &e) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "exception %s", e.what()); -+ } catch (...) { -+ err->id = -1; -+ snprintf(err->msg, err->msg_len, "Unknown exception during embedding"); -+ } -+} -+ -+#endif // LLAMA_SERVER_LIBRARY -\ No newline at end of file -diff --git a/examples/server/server.h b/examples/server/server.h -new file mode 100644 -index 0000000..d22f1b6 ---- /dev/null -+++ b/examples/server/server.h -@@ -0,0 +1,89 @@ -+#if defined(LLAMA_SERVER_LIBRARY) -+#ifndef LLAMA_SERVER_H -+#define LLAMA_SERVER_H -+#include -+#include -+#include -+#include -+ -+// This exposes extern C entrypoints into the llama_server -+// To enable the server compile with LLAMA_SERVER_LIBRARY -+ -+#ifdef __cplusplus -+extern "C" -+{ -+#endif -+ typedef struct ext_server_resp { -+ int id; // < 0 on error -+ size_t msg_len; // caller must allocate msg and set msg_len -+ char *msg; -+ } ext_server_resp_t; -+ -+ // Allocated and freed by caller -+ typedef struct ext_server_lora_adapter { -+ char *adapter; -+ float scale; -+ struct ext_server_lora_adapter *next; -+ } ext_server_lora_adapter_t; -+ -+ // Allocated and freed by caller -+ typedef struct ext_server_params -+ { -+ char *model; -+ uint32_t n_ctx; // text context, 0 = from model -+ uint32_t n_batch; // prompt processing maximum batch size -+ uint32_t n_threads; // number of threads to use for generation -+ int32_t n_parallel; // number of parallel sequences to decodewra -+ float rope_freq_base; // RoPE base frequency, 0 = from model -+ float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model -+ bool memory_f16; // use f16 instead of f32 for memory kv -+ int32_t n_gpu_layers; // number of layers to store in VRAM (-1 - use default) -+ int32_t main_gpu; // the GPU that is used for scratch and small tensors -+ bool use_mlock; // force system to keep model in RAM -+ bool use_mmap; // use mmap if possible -+ bool numa; // attempt optimizations that help on some NUMA systems -+ bool embedding; // get only sentence embedding -+ ext_server_lora_adapter_t* lora_adapters; -+ char *mmproj; -+ } ext_server_params_t; -+ -+ typedef struct ext_server_task_result -+ { -+ int id; -+ bool stop; -+ bool error; -+ char* json_resp; // null terminated, memory managed by ext_server -+ } ext_server_task_result_t; -+ -+ // Initialize the server once per process -+ // err->id = 0 for success and err->msg[0] = NULL -+ // err->id != 0 for failure, and err->msg contains error message -+ void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err); -+ -+ // Run the main loop, called once per init -+ void llama_server_start(); -+ // Stop the main loop and free up resources allocated in init and start. Init must be called again to reuse -+ void llama_server_stop(); -+ -+ // json_req null terminated string, memory managed by caller -+ // resp->id >= 0 on success (task ID) -+ // resp->id < 0 on error, and resp->msg contains error message -+ void llama_server_completion(const char *json_req, ext_server_resp_t *resp); -+ -+ // Caller must call llama_server_release_task_result to free resp->json_resp -+ void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *result); -+ void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err); -+ void llama_server_release_task_result(ext_server_task_result_t *result); -+ -+ // Caller must call llama_server_releaes_json_resp to free json_resp if err.id < 0 -+ void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err); -+ void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err); -+ void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err); -+ void llama_server_release_json_resp(char **json_resp); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -+#endif // LLAMA_SERVER_LIBRARY -\ No newline at end of file -diff --git a/ggml-cuda.cu b/ggml-cuda.cu -index f20846f..9640cf3 100644 ---- a/ggml-cuda.cu -+++ b/ggml-cuda.cu -@@ -6757,6 +6757,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d( - CUDA_CHECK(cudaGetDevice(&id)); - src_ptr = (char *) extra->data_device[id]; - } else { -+ fprintf(stderr, "ggml_cuda_cpy_tensor_2d assert: backend: %d\n", src->backend); - GGML_ASSERT(false); - } - char * dst_ptr = (char *) dst; --- -2.39.3 (Apple Git-145) -