diff --git a/.github/workflows/build-npu.yml b/.github/workflows/build-npu.yml new file mode 100644 index 00000000000..9e27870be98 --- /dev/null +++ b/.github/workflows/build-npu.yml @@ -0,0 +1,240 @@ +name: NPU Build (Windows + Linux) + +on: + workflow_dispatch: + +permissions: + contents: read + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + FLEXML_WIN_URL: "https://github.com/lemonade-sdk/whisper.cpp/releases/download/deps/flexmlrt1.7.0-win.zip" + FLEXML_LINUX_URL: "https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz" + +jobs: + +# ════════════════════════════════════════════════════════════════════════════════ +# 0. Determine version tag (same logic as main build.yml) +# ════════════════════════════════════════════════════════════════════════════════ + determine-tag: + runs-on: ubuntu-latest + outputs: + tag_name: ${{ steps.tag.outputs.name }} + version: ${{ steps.tag.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine tag and version + id: tag + shell: bash + run: | + BUILD_NUMBER=$(git rev-list --count HEAD) + SHORT_HASH=$(git rev-parse --short=7 HEAD) + CUSTOM_TAG="${{ github.event.inputs.pre_release_tag }}" + + if [[ -n "$CUSTOM_TAG" ]]; then + TAG_NAME="$CUSTOM_TAG" + elif [[ "${{ env.BRANCH_NAME }}" == "main" || "${{ env.BRANCH_NAME }}" == "master" ]]; then + TAG_NAME="b${BUILD_NUMBER}" + else + SAFE=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') + TAG_NAME="${SAFE}-b${BUILD_NUMBER}-${SHORT_HASH}" + fi + + echo "name=$TAG_NAME" >> $GITHUB_OUTPUT + echo "version=$TAG_NAME" >> $GITHUB_OUTPUT + +# ════════════════════════════════════════════════════════════════════════════════ +# 1. NPU — Windows (self-hosted, RyzenAI runner) +# ════════════════════════════════════════════════════════════════════════════════ + windows-npu: + runs-on: [self-hosted, Windows, stx, rai300_400] + needs: determine-tag + continue-on-error: false + + steps: + - uses: actions/checkout@v4 + + - uses: microsoft/setup-msbuild@v2 + + - name: Install CMake if not available + shell: powershell + run: | + $installed = Get-Command cmake -ErrorAction SilentlyContinue + if (-not $installed) { + $ver = "3.28.1" + $url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi" + Invoke-WebRequest -Uri $url -OutFile cmake.msi + Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait + $p = "C:\Program Files\CMake\bin" + $env:PATH = "$p;$env:PATH" + echo $p >> $env:GITHUB_PATH + cmake --version + if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 } + } else { cmake --version } + + - name: Download FlexML Runtime + shell: powershell + run: | + Invoke-WebRequest -Uri "${{ env.FLEXML_WIN_URL }}" -OutFile flexmlrt.zip + if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 } + if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 } + Write-Host "FlexML: $([math]::Round((Get-Item 'flexmlrt.zip').Length/1MB,2)) MB downloaded" + + - name: Extract FlexML Runtime + shell: powershell + run: | + tar xvf flexmlrt.zip + if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 } + $dirs = Get-ChildItem -Directory | Where-Object { $_.Name -like "flexmlrt*" } + if (-not $dirs) { Write-Error "No flexmlrt directory found after extraction"; exit 1 } + Write-Host "Extracted: $($dirs.Name)" + + - name: Setup FlexML, configure and build + shell: cmd + run: | + cd flexmlrt + call setup.bat + if errorlevel 1 ( echo ERROR: FlexML setup.bat failed! & exit /b 1 ) + cd .. + cmake -B build -A x64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DWHISPER_VITISAI=ON ^ + -DWHISPER_BUILD_SERVER=ON ^ + -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON + if errorlevel 1 ( echo ERROR: CMake configure failed! & exit /b 1 ) + cmake --build build --config Release -j + if errorlevel 1 ( echo ERROR: Build failed! & exit /b 1 ) + + - name: List build output + shell: powershell + run: | + if (Test-Path "build/bin/Release") { + Get-ChildItem -Path "build/bin/Release" -Recurse | Format-Table Name, Length + } else { Write-Error "build/bin/Release not found"; exit 1 } + + - name: Copy FlexML DLLs to build output + shell: powershell + run: | + $copied = 0 + if (Test-Path "flexmlrt/bin") { + $d = Get-ChildItem -Path "flexmlrt/bin/*.dll" -ErrorAction SilentlyContinue + if ($d) { Copy-Item "flexmlrt/bin/*.dll" "build/bin/Release/" -Force; $copied += $d.Count } + } + if (Test-Path "flexmlrt/lib") { + $d = Get-ChildItem -Path "flexmlrt/lib/*.dll" -ErrorAction SilentlyContinue + if ($d) { Copy-Item "flexmlrt/lib/*.dll" "build/bin/Release/" -Force; $copied += $d.Count } + } + Write-Host "FlexML DLLs copied: $copied" + + - name: Package + shell: powershell + run: | + $a = "whisper-${{ needs.determine-tag.outputs.version }}-windows-npu-x64.zip" + Compress-Archive -Path "build/bin/Release/*" -DestinationPath $a -Force + if (-not (Test-Path $a)) { Write-Error "Package creation failed"; exit 1 } + $mb = [math]::Round((Get-Item $a).Length/1MB,2) + Write-Host "Package: $a ($mb MB)" + "ARCHIVE=$a" | Out-File $env:GITHUB_ENV -Append -Encoding utf8 + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARCHIVE }} + path: ${{ env.ARCHIVE }} + +# ════════════════════════════════════════════════════════════════════════════════ +# 2. NPU — Linux (self-hosted, RyzenAI runner) +# ════════════════════════════════════════════════════════════════════════════════ + linux-npu: + runs-on: [self-hosted, Linux, X64, stx, rai300-400] + needs: determine-tag + continue-on-error: false + + steps: + - uses: actions/checkout@v4 + + - name: Download FlexML Runtime + run: | + curl -L --fail --retry 3 --retry-delay 5 -o flexmlrt.tar.gz "${{ env.FLEXML_LINUX_URL }}" + if [ ! -f flexmlrt.tar.gz ] || [ ! -s flexmlrt.tar.gz ]; then + echo "::error::flexmlrt.tar.gz download failed or empty"; exit 1 + fi + echo "FlexML: $(du -sh flexmlrt.tar.gz | cut -f1) downloaded" + + - name: Extract FlexML Runtime + run: | + tar xf flexmlrt.tar.gz + FLEXML_DIR=$(find . -maxdepth 1 -type d -name "flexmlrt*" | head -1) + if [ -z "$FLEXML_DIR" ]; then echo "::error::No flexmlrt directory found"; exit 1; fi + echo "Extracted: $FLEXML_DIR" + + - name: Setup FlexML environment + run: | + source flexmlrt/setup.sh + echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + + - name: Configure CMake + run: | + cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DWHISPER_VITISAI=ON \ + -DWHISPER_BUILD_SERVER=ON + + - name: Build + run: | + cmake --build build --config Release -j$(nproc) > build.log 2>&1 + exit_code=$? + grep -E "error:|FAILED|Linking|Built target" build.log || true + if [ $exit_code -ne 0 ]; then + echo "--- Last 100 lines of build log ---" + tail -100 build.log + exit $exit_code + fi + echo "Build succeeded." + + - name: Verify build output + run: | + if [ ! -f build/bin/whisper-cli ]; then + echo "::error::whisper-cli not found" + ls -lh build/bin/ 2>/dev/null || true + exit 1 + fi + echo "Build output:"; ls -lh build/bin/whisper-cli + ldd build/bin/whisper-cli || true + + - name: Copy FlexML shared libs + run: | + BIN="build/bin" + if [ -d flexmlrt/lib ]; then + find flexmlrt/lib -maxdepth 1 -name "*.so*" -exec cp -Pn {} "$BIN/" \; 2>/dev/null || true + echo "FlexML libs copied:" + ls -lh "$BIN"/*.so* 2>/dev/null || echo "(none)" + fi + + - name: Set portable RPATH + run: | + command -v patchelf >/dev/null 2>&1 || { echo "::warning::patchelf not found, skipping RPATH patch"; exit 0; } + BIN="build/bin" + for f in "$BIN"/whisper-* "$BIN"/*.so*; do + [ -f "$f" ] && [ ! -L "$f" ] && file "$f" | grep -q ELF && \ + patchelf --set-rpath '$ORIGIN' "$f" 2>/dev/null || true + done + + - name: Package + run: | + VER="${{ needs.determine-tag.outputs.version }}" + ARCHIVE="whisper-${VER}-linux-npu-x64.tar.gz" + STAGE="whisper-${VER}-linux-npu-x64" + mkdir -p "$STAGE" + cp -r build/bin/* "$STAGE/" + tar -czf "$ARCHIVE" "$STAGE" + echo "Package: $ARCHIVE ($(du -sh $ARCHIVE | cut -f1))" + echo "ARCHIVE=$ARCHIVE" >> $GITHUB_ENV + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARCHIVE }} + path: ${{ env.ARCHIVE }} diff --git a/src/vitisai/whisper-vitisai-helpers.cpp b/src/vitisai/whisper-vitisai-helpers.cpp new file mode 100644 index 00000000000..2ff509447a3 --- /dev/null +++ b/src/vitisai/whisper-vitisai-helpers.cpp @@ -0,0 +1,481 @@ +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + +#include "vitisai/whisper-vitisai-helpers.h" + +#include +#include +#ifdef _WIN32 + #include +#else + #include + #include +#endif +#include +#include + +namespace whisper_vitisai_helpers { + +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +#ifdef _WIN32 + HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + LARGE_INTEGER fileSize; + if (!GetFileSizeEx(hFile, &fileSize)) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL); + if (hMapping == NULL) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + *buffer = (uint8_t *) MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart); + if (*buffer == NULL) { + CloseHandle(hMapping); + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path); + return false; + } + CloseHandle(hMapping); + CloseHandle(hFile); + *size = fileSize.QuadPart; + return true; +#else + FILE * fd = fopen(path, "rb"); + if (!fd) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + struct stat st; + if (fstat(fileno(fd), &st) == -1) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + *buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0); + if (*buffer == MAP_FAILED) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); + return false; + } + fclose(fd); + *size = st.st_size; + return true; +#endif // _WIN32 +} + +void unmap_rai_file(uint8_t * buffer, size_t size) { +#ifdef _WIN32 + UnmapViewOfFile(buffer); +#else + munmap(buffer, size); +#endif // _WIN32 +} + +bool file_exists(const char * path) { + if (!path) { + return false; + } + + FILE * file = fopen(path, "rb"); + if (!file) { + return false; + } + fclose(file); + return true; +} + +const char * whisper_kv_type_name(ggml_type type) { + switch (type) { + case GGML_TYPE_F32: return "F32"; + case GGML_TYPE_F16: return "F16"; + default: return "unsupported"; + } +} + +const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type) { + switch (type) { + case flexmlrt::client::DataType::Float32: return "Float32"; + case flexmlrt::client::DataType::Int8: return "Int8"; + case flexmlrt::client::DataType::UInt8: return "UInt8"; + case flexmlrt::client::DataType::Int16: return "Int16"; + case flexmlrt::client::DataType::UInt16: return "UInt16"; + case flexmlrt::client::DataType::BFloat16: return "BFloat16"; + case flexmlrt::client::DataType::Bool: return "Bool"; + case flexmlrt::client::DataType::Float16: return "Float16"; + case flexmlrt::client::DataType::Int32: return "Int32"; + case flexmlrt::client::DataType::UInt32: return "UInt32"; + default: return "Unknown"; + } +} + +bool whisper_flexml_dtype_to_ggml_type( + flexmlrt::client::DataType type, + ggml_type * ggml_dtype) { + switch (type) { + case flexmlrt::client::DataType::Float32: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_F32; + } + return true; + case flexmlrt::client::DataType::Float16: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_F16; + } + return true; + case flexmlrt::client::DataType::BFloat16: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_BF16; + } + return true; + default: + return false; + } +} + +static bool whisper_vitisai_validate_tensor_dtype( + const char * tensor_name, + flexmlrt::client::DataType model_dtype, + ggml_type runtime_dtype) { + ggml_type expected_runtime_dtype = GGML_TYPE_COUNT; + if (!whisper_flexml_dtype_to_ggml_type(model_dtype, &expected_runtime_dtype)) { + std::fprintf(stderr, + "%s: unsupported model dtype for %s: %s (supported: Float32/Float16/BFloat16)\n", + __func__, tensor_name, whisper_flexml_dtype_name(model_dtype)); + return false; + } + + if (runtime_dtype != expected_runtime_dtype) { + std::fprintf(stderr, + "%s: %s dtype mismatch (runtime=%s, model=%s)\n", + __func__, tensor_name, ggml_type_name(runtime_dtype), whisper_flexml_dtype_name(model_dtype)); + return false; + } + + return true; +} + +static std::string whisper_shape_to_string(const std::vector & shape) { + std::string out = "["; + for (size_t i = 0; i < shape.size(); ++i) { + if (i > 0) { + out += ", "; + } + out += std::to_string(shape[i]); + } + out += "]"; + return out; +} + +static std::vector whisper_canonical_shape(const std::vector & shape) { + std::vector canonical; + canonical.reserve(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + const size_t dim = (size_t) shape[i]; + if (dim != 1) { + canonical.push_back(dim); + } + } + if (canonical.empty()) { + canonical.push_back(1); + } + return canonical; +} + +static bool whisper_validate_shape( + const char * tensor_name, + const std::vector & model_shape, + const std::vector & expected_shape) { + const std::vector shape = whisper_canonical_shape(model_shape); + if (shape != expected_shape) { + std::fprintf(stderr, + "%s: %s shape mismatch (runtime expected=%s, model=%s)\n", + __func__, + tensor_name, + whisper_shape_to_string(expected_shape).c_str(), + whisper_shape_to_string(shape).c_str()); + return false; + } + return true; +} + +bool whisper_validate_cross_shape( + const char * tensor_name, + const std::vector & model_shape, + int n_text_layer, + int n_ctx, + int n_state) { + const std::vector expected = { + (size_t) n_text_layer, + (size_t) n_ctx, + (size_t) n_state, + }; + return whisper_validate_shape(tensor_name, model_shape, expected); +} + +bool whisper_vitisai_bind_tensor_data( + const char * tensor_name, + struct ggml_tensor * runtime_tensor, + const std::vector & expected_shape, + flexmlrt::client::ErtTensorType & io_tensor) { + const auto & meta = io_tensor.getMetadata(); + if (!whisper_vitisai_validate_tensor_dtype(tensor_name, meta.type, runtime_tensor->type)) { + return false; + } + if (!whisper_validate_shape(tensor_name, meta.shape, expected_shape)) { + return false; + } + + const size_t model_bytes = meta.size; + const size_t runtime_bytes = ggml_nbytes(runtime_tensor); + if (model_bytes == 0 || runtime_bytes == 0) { + std::fprintf(stderr, "%s: %s sizes must be non-zero (model=%zu, runtime=%zu)\n", + __func__, tensor_name, model_bytes, runtime_bytes); + return false; + } + if (runtime_bytes != model_bytes) { + std::fprintf(stderr, + "%s: %s tensor size mismatch (runtime=%zu B, model=%zu B). " + "VitisAI .rai requires exact context match; use matching -ac/model artifact.\n", + __func__, tensor_name, runtime_bytes, model_bytes); + return false; + } + + io_tensor.data = runtime_tensor->data; + return true; +} + +bool whisper_vitisai_resolve_io_binding( + const char * caller, + const std::vector & input_tensors, + const std::vector & output_tensors, + whisper_vitisai_io_binding * binding, + std::string * error) { + const auto fail = [error](std::string message) { + if (error) { + *error = std::move(message); + } + return false; + }; + + if (input_tensors.empty()) { + return fail("Model has no input tensors"); + } + + binding->mel_in_idx = 0; + bool found_named_mel = false; + for (int i = 0; i < (int) input_tensors.size(); ++i) { + const std::string & name = input_tensors[i].getMetadata().name; + if (name == "input" || name == "mel") { + binding->mel_in_idx = i; + found_named_mel = true; + break; + } + } + if (!found_named_mel) { + std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller); + } + + if (output_tensors.empty()) { + return fail("Model has no output tensors"); + } + + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const std::string & name = output_tensors[i].getMetadata().name; + if (name == "embd_enc") { + binding->embd_enc_out_idx = i; + } else if (name == "cross_k") { + binding->cross_k_out_idx = i; + } else if (name == "cross_v") { + binding->cross_v_out_idx = i; + } + } + + if (binding->embd_enc_out_idx < 0) { + std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller); + binding->embd_enc_out_idx = 0; + } + + const bool has_cross_k = binding->cross_k_out_idx >= 0; + const bool has_cross_v = binding->cross_v_out_idx >= 0; + if (has_cross_k != has_cross_v) { + return fail("Incomplete cross-projection contract: both cross_k and cross_v outputs are required"); + } + + if (has_cross_k && (binding->cross_k_out_idx == binding->cross_v_out_idx || + binding->cross_k_out_idx == binding->embd_enc_out_idx || + binding->cross_v_out_idx == binding->embd_enc_out_idx)) { + return fail("Invalid output mapping: embd_enc/cross_k/cross_v indices overlap"); + } + + const auto & mel_meta = input_tensors[binding->mel_in_idx].getMetadata(); + if (!whisper_flexml_dtype_to_ggml_type(mel_meta.type, nullptr)) { + return fail( + std::string("Unsupported mel input type: ") + + whisper_flexml_dtype_name(mel_meta.type) + " (supported: Float32/Float16/BFloat16)"); + } + binding->mel_in_expected_bytes = mel_meta.size; + + const auto & embd_meta = output_tensors[binding->embd_enc_out_idx].getMetadata(); + if (!whisper_flexml_dtype_to_ggml_type(embd_meta.type, nullptr)) { + return fail( + std::string("Unsupported embd_enc output type: ") + + whisper_flexml_dtype_name(embd_meta.type) + " (supported: Float32/Float16/BFloat16)"); + } + binding->embd_enc_expected_bytes = embd_meta.size; + + if (has_cross_k) { + const auto & cross_k_meta = output_tensors[binding->cross_k_out_idx].getMetadata(); + const auto & cross_v_meta = output_tensors[binding->cross_v_out_idx].getMetadata(); + if (cross_k_meta.type != flexmlrt::client::DataType::Float32 || + cross_v_meta.type != flexmlrt::client::DataType::Float32) { + return fail( + std::string("Unsupported cross output type(s): cross_k=") + + whisper_flexml_dtype_name(cross_k_meta.type) + ", cross_v=" + + whisper_flexml_dtype_name(cross_v_meta.type) + " (cross path currently requires Float32)"); + } + if (cross_k_meta.size != cross_v_meta.size) { + return fail("cross_k and cross_v output sizes do not match"); + } + binding->cross_k_expected_bytes = cross_k_meta.size; + binding->cross_v_expected_bytes = cross_v_meta.size; + } + + return true; +} + +bool whisper_vitisai_all_tensors_claimed( + const char * caller, + const char * tensor_kind, + const std::vector & tensors, + const std::vector & claimed) { + for (size_t i = 0; i < tensors.size(); ++i) { + if (!claimed[i]) { + std::fprintf(stderr, + "%s: unsupported extra %s tensor at index %zu (name='%s'); strict contract expects only mapped %ss\n", + caller, tensor_kind, i, tensors[i].getMetadata().name.c_str(), tensor_kind); + return false; + } + } + return true; +} + +void whisper_kv_cross_scale_k_f32( + float * k_data, + size_t count, + float kscale) { + for (size_t i = 0; i < count; ++i) { + k_data[i] *= kscale; + } +} + +void whisper_kv_cross_store_layers_f32( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + float * dk = (float *)(dst_k + layout.dst_layer_stride * (size_t)il); + float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = layer_src_k[i] * layout.kscale; + dv[i] = layer_src_v[i]; + } + } +} + +void whisper_kv_cross_store_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il); + ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale); + dv[i] = ggml_fp32_to_fp16(layer_src_v[i]); + } + } +} + +void whisper_kv_cross_transpose_v_layers_f32( + const float * src_v, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + const int n_ctx = layout.n_ctx; + const int n_state = layout.n_state; + + const int BLOCK = 32; + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il); + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dv[j * n_ctx + i] = layer_src_v[i * n_state + j]; + } + } + } + } + } +} + +void whisper_kv_cross_store_k_transpose_v_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + const int n_ctx = layout.n_ctx; + const int n_state = layout.n_state; + + const int BLOCK = 32; + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il); + ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale); + } + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dv[j * n_ctx + i] = ggml_fp32_to_fp16(layer_src_v[i * n_state + j]); + } + } + } + } + } +} + +} // namespace whisper_vitisai_helpers diff --git a/src/vitisai/whisper-vitisai-helpers.h b/src/vitisai/whisper-vitisai-helpers.h new file mode 100644 index 00000000000..f6ab5390281 --- /dev/null +++ b/src/vitisai/whisper-vitisai-helpers.h @@ -0,0 +1,118 @@ +#pragma once + +#include "FlexMLClient.h" +#include "ggml.h" + +#include +#include +#include +#include +#include + +namespace whisper_vitisai_helpers { + +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); +void unmap_rai_file(uint8_t * buffer, size_t size); +bool file_exists(const char * path); + +const char * whisper_kv_type_name(ggml_type type); +const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type); +bool whisper_flexml_dtype_to_ggml_type( + flexmlrt::client::DataType type, + ggml_type * ggml_dtype); + +bool whisper_validate_cross_shape( + const char * tensor_name, + const std::vector & model_shape, + int n_text_layer, + int n_ctx, + int n_state); + +bool whisper_vitisai_bind_tensor_data( + const char * tensor_name, + struct ggml_tensor * runtime_tensor, + const std::vector & expected_shape, + flexmlrt::client::ErtTensorType & io_tensor); + +#if defined(WHISPER_DEBUG) +template +void whisper_vitisai_print_shape(const std::vector & shape) { + std::fprintf(stderr, "["); + for (size_t i = 0; i < shape.size(); ++i) { + std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]); + } + std::fprintf(stderr, "]"); +} +#endif + +// Model IO tensor indices and metadata sizes resolved once at init time. +struct whisper_vitisai_io_binding { + int mel_in_idx = -1; + int embd_enc_out_idx = -1; + int cross_k_out_idx = -1; + int cross_v_out_idx = -1; + size_t mel_in_expected_bytes = 0; + size_t embd_enc_expected_bytes = 0; + size_t cross_k_expected_bytes = 0; + size_t cross_v_expected_bytes = 0; +}; + +// Warnings are printed with the caller's name; hard failures are returned in *error +// so the caller can decide how to report them. +bool whisper_vitisai_resolve_io_binding( + const char * caller, + const std::vector & input_tensors, + const std::vector & output_tensors, + whisper_vitisai_io_binding * binding, + std::string * error); + +bool whisper_vitisai_all_tensors_claimed( + const char * caller, + const char * tensor_kind, + const std::vector & tensors, + const std::vector & claimed); + +// Geometry of one cross K/V transfer from the model output (always f32, contiguous +// [ctx, state] per layer) into the runtime kv cache. +struct whisper_kv_cross_layout { + int n_layer = 0; + int n_ctx = 0; + int n_state = 0; + size_t src_layer_elems = 0; // f32 elements per layer in the model output buffer + size_t layer_elems = 0; // elements per layer transferred into the kv cache + size_t dst_layer_stride = 0; // bytes per layer in the kv cache + float kscale = 1.0f; +}; + +void whisper_kv_cross_scale_k_f32( + float * k_data, + size_t count, + float kscale); + +void whisper_kv_cross_store_layers_f32( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_store_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_transpose_v_layers_f32( + const float * src_v, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_store_k_transpose_v_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +} // namespace whisper_vitisai_helpers