diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c4710b1663..6d6bc21b98d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,6 +72,7 @@ jobs: tag_name: ${{ steps.tag.outputs.name }} version: ${{ steps.tag.outputs.version }} should_release: ${{ steps.tag.outputs.should_release }} + is_full_build: ${{ steps.tag.outputs.is_full_build }} steps: - uses: actions/checkout@v4 with: @@ -85,39 +86,64 @@ jobs: SHORT_HASH=$(git rev-parse --short=7 HEAD) CUSTOM_TAG="${{ github.event.inputs.pre_release_tag }}" SHOULD_RELEASE="false" + IS_FULL_BUILD="false" + IS_PR="${{ github.event_name == 'pull_request' }}" + HAS_FULL_CI_LABEL="false" + # Check for 'full-ci' label on PRs + if [[ "$IS_PR" == "true" ]]; then + LABELS='${{ toJson(github.event.pull_request.labels.*.name) }}' + if echo "$LABELS" | grep -q '"full-ci"'; then + HAS_FULL_CI_LABEL="true" + fi + fi if [[ "${{ github.ref_type }}" == "tag" ]]; then - # Triggered by sync.yml pushing a vX.Y.Z tag — this is the primary release path TAG_NAME="${{ github.ref_name }}" SHOULD_RELEASE="true" + IS_FULL_BUILD="true" elif [[ -n "$CUSTOM_TAG" ]]; then TAG_NAME="$CUSTOM_TAG" SHOULD_RELEASE="true" + IS_FULL_BUILD="true" elif [[ "${{ github.event.inputs.create_release }}" == "true" ]]; then TAG_NAME="b${BUILD_NUMBER}" SHOULD_RELEASE="true" + IS_FULL_BUILD="true" + elif [[ "$IS_PR" == "true" && "$HAS_FULL_CI_LABEL" == "true" ]]; then + SAFE=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') + TAG_NAME="${SAFE}-b${BUILD_NUMBER}-${SHORT_HASH}" + IS_FULL_BUILD="true" + elif [[ "$IS_PR" == "true" ]]; then + # PR without full-ci label — lightweight path only + SAFE=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') + TAG_NAME="${SAFE}-b${BUILD_NUMBER}-${SHORT_HASH}" + IS_FULL_BUILD="false" + elif [[ "${{ github.event_name }}" == "schedule" ]]; then + TAG_NAME="b${BUILD_NUMBER}" + IS_FULL_BUILD="true" elif [[ "${{ env.BRANCH_NAME }}" == "main" || "${{ env.BRANCH_NAME }}" == "master" ]]; then TAG_NAME="b${BUILD_NUMBER}" - SHOULD_RELEASE="false" + IS_FULL_BUILD="true" else SAFE=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-') TAG_NAME="${SAFE}-b${BUILD_NUMBER}-${SHORT_HASH}" - SHOULD_RELEASE="false" + IS_FULL_BUILD="true" fi - # Version used in artifact filenames — keep leading 'v' to match lemonade expectations - # e.g. v1.8.4 → v1.8.4, b1234 → b1234 VERSION="${TAG_NAME}" echo "name=$TAG_NAME" >> $GITHUB_OUTPUT echo "version=$VERSION" >> $GITHUB_OUTPUT echo "should_release=$SHOULD_RELEASE" >> $GITHUB_OUTPUT + echo "is_full_build=$IS_FULL_BUILD" >> $GITHUB_OUTPUT # ════════════════════════════════════════════════════════════════════════════════ # 1. ROCm matrix (Linux + Windows per GFX target) # ════════════════════════════════════════════════════════════════════════════════ prepare-rocm-matrix: runs-on: ubuntu-latest + if: needs.determine-tag.outputs.is_full_build == 'true' + needs: determine-tag outputs: ubuntu_matrix: ${{ steps.m.outputs.ubuntu_matrix }} windows_matrix: ${{ steps.m.outputs.windows_matrix }} @@ -142,6 +168,7 @@ jobs: # ════════════════════════════════════════════════════════════════════════════════ linux-rocm: runs-on: ubuntu-22.04 + if: needs.determine-tag.outputs.is_full_build == 'true' needs: [determine-tag, prepare-rocm-matrix] strategy: matrix: ${{ fromJson(needs.prepare-rocm-matrix.outputs.ubuntu_matrix) }} @@ -1223,6 +1250,12 @@ jobs: if: | always() && needs.determine-tag.outputs.should_release == 'true' && + (needs.linux-rocm.result == 'success' || needs.linux-rocm.result == 'skipped') && + (needs.windows-rocm.result == 'success' || needs.windows-rocm.result == 'skipped') && + (needs.linux-vulkan.result == 'success' || needs.linux-vulkan.result == 'skipped') && + (needs.windows-vulkan.result == 'success' || needs.windows-vulkan.result == 'skipped') && + (needs.linux-cpu.result == 'success' || needs.linux-cpu.result == 'skipped') && + (needs.windows-cpu.result == 'success' || needs.windows-cpu.result == 'skipped') && (needs.test-cpu-windows.result == 'success' || needs.test-cpu-windows.result == 'skipped') && (needs.test-cpu-linux.result == 'success' || needs.test-cpu-linux.result == 'skipped') && (needs.test-vulkan-windows.result == 'success' || needs.test-vulkan-windows.result == 'skipped') && @@ -1272,12 +1305,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | TAG="${{ needs.determine-tag.outputs.tag_name }}" - RELEASE_ID=$(gh api repos/${{ github.repository }}/releases/tags/"$TAG" --jq '.id' 2>/dev/null || true) - if [ -n "$RELEASE_ID" ]; then - echo "Deleting existing release $RELEASE_ID for tag $TAG" - gh api -X DELETE repos/${{ github.repository }}/releases/"$RELEASE_ID" - fi - git push --delete origin "refs/tags/$TAG" 2>/dev/null || true + gh release delete "$TAG" --yes --cleanup-tag --repo ${{ github.repository }} 2>/dev/null || true - name: Create release id: create_release @@ -1313,28 +1341,30 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const fs = require('fs'); - const path = require('path'); - const id = '${{ steps.create_release.outputs.id }}'; + const fs = require('fs'); + const id = '${{ steps.create_release.outputs.id }}'; + if (!id) throw new Error('create_release did not produce a release ID'); for (const file of fs.readdirSync('./release')) { if (!file.endsWith('.zip') && !file.endsWith('.tar.gz')) continue; console.log('Uploading:', file); + const stat = fs.statSync(`./release/${file}`); await github.rest.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: id, name: file, - data: fs.readFileSync(`./release/${file}`), + data: fs.createReadStream(`./release/${file}`), + headers: { 'content-length': stat.size, 'content-type': 'application/octet-stream' }, }); } - name: Update README download links run: | TAG="${{ needs.determine-tag.outputs.tag_name }}" - # Replace the placeholder tag in all download URLs with the actual release tag sed -i "s|/releases/download/[^/]*/whisper-[^-]*-|/releases/download/${TAG}/whisper-${TAG}-|g" README.md git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add README.md git diff --cached --quiet || git commit -m "docs: update download links to ${TAG}" + git pull --rebase origin master 2>/dev/null || git pull --rebase origin main git push diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fe10876eaf7..fe57ca34244 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,6 +50,57 @@ endif() if (WHISPER_VITISAI) find_package(FlexmlRT REQUIRED) + + # Legacy RAI overrides are required by FlexMLRT older than 1.8.0 + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE "AUTO" CACHE STRING + "Legacy RAI override mode for FlexMLRT (AUTO|ON|OFF)") + set_property(CACHE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE PROPERTY STRINGS AUTO ON OFF) + + string(TOUPPER "${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}" _flexmlrt_legacy_mode) + set(_flexmlrt_legacy_hint "Set -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON or OFF explicitly.") + + if (NOT _flexmlrt_legacy_mode MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR + "Invalid WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE='${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}'. " + "Expected AUTO, ON, or OFF.") + endif() + + if (_flexmlrt_legacy_mode STREQUAL "AUTO") + if (NOT FlexmlRT_DIR) + message(FATAL_ERROR + "FlexmlRT_DIR is unset after find_package(FlexmlRT). ${_flexmlrt_legacy_hint}") + endif() + + # FlexmlRT_DIR points to /share/cmake/FlexmlRT. + get_filename_component(_flexmlrt_init_py "${FlexmlRT_DIR}/../../../__init__.py" ABSOLUTE) + if (NOT EXISTS "${_flexmlrt_init_py}") + message(FATAL_ERROR + "flexmlrt __init__.py not found at ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}") + endif() + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_flexmlrt_init_py}") + + file(STRINGS "${_flexmlrt_init_py}" _flexmlrt_version_lines + REGEX "^VERSION[ \t]*=[ \t]*\"[0-9]+\\.[0-9]+\\.[0-9]+") + if (NOT _flexmlrt_version_lines MATCHES "\"([0-9]+\\.[0-9]+\\.[0-9]+)") + message(FATAL_ERROR + "Could not parse flexmlrt VERSION from ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}") + endif() + set(_flexmlrt_version "${CMAKE_MATCH_1}") + + if (_flexmlrt_version VERSION_LESS "1.8.0") + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1) + else() + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0) + endif() + message(STATUS "Detected flexmlrt VERSION=${_flexmlrt_version} from ${_flexmlrt_init_py} (legacy overrides=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES})") + else() + if (_flexmlrt_legacy_mode STREQUAL "ON") + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1) + else() + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0) + endif() + message(STATUS "FlexMLRT legacy RAI overrides forced ${_flexmlrt_legacy_mode}") + endif() endif() # @@ -111,6 +162,8 @@ if (WHISPER_VITISAI) add_library(${TARGET} OBJECT vitisai/whisper-vitisai-encoder.h vitisai/whisper-vitisai-encoder.cpp + vitisai/whisper-vitisai-helpers.h + vitisai/whisper-vitisai-helpers.cpp ) target_include_directories(${TARGET} PUBLIC @@ -120,11 +173,15 @@ if (WHISPER_VITISAI) set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON) set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI) - # C++17 required for MSVC (FlexML headers use structured bindings etc.) + # Add C++17 standard for MSVC if (MSVC) target_compile_options(${TARGET} PRIVATE /std:c++17) endif() + target_compile_definitions(${TARGET} PRIVATE + WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES} + ) + target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt) set_target_properties(${TARGET} PROPERTIES FOLDER "libs") endif() diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index a6d20a88c9a..9586c24a927 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -1,4 +1,9 @@ -// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + #include "vitisai/whisper-vitisai-encoder.h" #include "FlexMLClient.h" #include "ggml.h" @@ -13,18 +18,49 @@ #include #include #endif +#include +#include #include +#include #include +#include + +#if defined(WHISPER_DEBUG) +#define WHISPER_DBG_TIMER(name) const int64_t name = ggml_time_us() +#else +#define WHISPER_DBG_TIMER(name) do {} while (0) +#endif + +#if defined(WHISPER_DEBUG) +template +static void whisper_vitisai_print_shape(const std::vector & shape) { + std::fprintf(stderr, "["); + for (size_t i = 0; i < shape.size(); ++i) { + std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]); + } + std::fprintf(stderr, "]"); +} +#endif struct whisper_vitisai_context { std::string model_path; std::shared_ptr runner; - uint8_t * fbs_buffer; - size_t fbs_buffer_size; + uint8_t * fbs_buffer = nullptr; + size_t fbs_buffer_size = 0; + + std::vector cross_k_staging; + std::vector cross_v_staging; + + int embd_enc_out_idx = -1; + int cross_k_out_idx = -1; + int cross_v_out_idx = -1; + + std::vector cached_input_tensors; + std::vector cached_output_tensors; }; // Function to mmap rai file for Linux and MapViewOfFile for Windows -bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +static bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { #ifdef _WIN32 // Open the file HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -87,7 +123,7 @@ bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { #endif // _WIN32 } -void unmap_rai_file(uint8_t * buffer, size_t size) { +static void unmap_rai_file(uint8_t * buffer, size_t size) { #ifdef _WIN32 UnmapViewOfFile(buffer); #else @@ -95,6 +131,38 @@ void unmap_rai_file(uint8_t * buffer, size_t size) { #endif // _WIN32 } +bool whisper_vitisai_file_exists(const char * path) { + if (!path) { + return false; + } + + FILE * file = fopen(path, "rb"); + if (!file) { + return false; + } + fclose(file); + return true; +} + +// Reuse cached tensor descriptors to avoid repeated getIOTensors() lookups. +static bool whisper_vitisai_get_io_tensors( + struct whisper_vitisai_context * ctx, + std::vector & input_tensors, + std::vector & output_tensors) { + if (!ctx || !ctx->runner) { + return false; + } + + if (ctx->cached_input_tensors.empty() || ctx->cached_output_tensors.empty()) { + ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); + ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + } + + input_tensors = ctx->cached_input_tensors; + output_tensors = ctx->cached_output_tensors; + return true; +} + struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { if (!path_model) { std::fprintf(stderr, "%s: path_model is null\n", __func__); @@ -114,27 +182,39 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { // Step 1: Set up the model flexmlrt::client::Options options; options.modelPath = ctx->model_path; - options.deviceName = "stx"; options.debug = false; options.executeMode = 2; - options.extOptions["ai_analyzer_profiling"] = true; // Enable AIA profiling options.extOptions["enable_preemption"] = true; // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options if (ctx->model_path.find(".rai") != std::string::npos) { - // mmap rai file for both Linux and Windows and pass the buffer to the options - ctx->fbs_buffer = nullptr; - ctx->fbs_buffer_size = 0; if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { options.extOptions["fbs_buffer"] = ctx->fbs_buffer; options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; - options.subgraphName = "vaiml_par_0"; options.extOptions["cache_dir"] = std::string("."); } else { std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str()); delete ctx; return nullptr; } + } else { + options.deviceName = "stx"; +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Using default device name 'stx'\n", __func__); +#endif + } + + const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos; + if (model_is_rai) { +#if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES + options.deviceName = "stx"; + options.subgraphName = "vaiml_par_0"; +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, + "%s: legacy FlexMLRT compile configuration detected; applying RAI overrides (device='stx', subgraph='vaiml_par_0')\n", + __func__); +#endif // defined(WHISPER_DEBUG) +#endif } try { @@ -143,6 +223,53 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { if (!ctx->runner->good()) { throw std::runtime_error("Runner creation ran into an error"); } + + ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); + ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + + auto & output_tensors = ctx->cached_output_tensors; + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const std::string & name = output_tensors[i].getMetadata().name; + if (name == "embd_enc") { + ctx->embd_enc_out_idx = i; + } else if (name == "cross_k") { + ctx->cross_k_out_idx = i; + } else if (name == "cross_v") { + ctx->cross_v_out_idx = i; + } + } + + if (ctx->embd_enc_out_idx < 0) { + std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", __func__); + ctx->embd_enc_out_idx = 0; + } + +#if defined(WHISPER_DEBUG) + { + auto & input_tensors = ctx->cached_input_tensors; + + std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size()); + for (int i = 0; i < (int) input_tensors.size(); ++i) { + const auto & meta = input_tensors[i].getMetadata(); + std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=", + __func__, i, meta.name.c_str(), (size_t) meta.size); + whisper_vitisai_print_shape(meta.shape); + std::fprintf(stderr, "\n"); + } + + std::fprintf(stderr, "%s: model has %zu output tensor(s)\n", __func__, output_tensors.size()); + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const auto & meta = output_tensors[i].getMetadata(); + std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=", + __func__, i, meta.name.c_str(), (size_t) meta.size); + whisper_vitisai_print_shape(meta.shape); + std::fprintf(stderr, "\n"); + } + + std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n", + __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx); + } +#endif } catch (const std::exception & e) { std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what()); delete ctx; @@ -151,12 +278,18 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { return ctx; } +bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) { + return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0; +} + void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { if (!ctx) { return; } - std::fprintf(stderr, "%s: releasing Vitis AI encoder context for model '%s'\n", __func__, ctx->model_path.c_str()); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str()); +#endif if (ctx->fbs_buffer) { unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); } @@ -183,18 +316,27 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten std::vector input_tensors, output_tensors; auto model = ctx->runner; - // Get tensors as CPU tensors (hwTensor = false) - input_tensors = model->getIOTensors("input", false); - output_tensors = model->getIOTensors("output", false); + if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } // TODO: add assert checks for tensor numbers and shapes + if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n", + __func__, ctx->embd_enc_out_idx, output_tensors.size()); + return 0; + } + input_tensors[0].data = mel->data; - output_tensors[0].data = out->data; + output_tensors[ctx->embd_enc_out_idx].data = out->data; try { model->forward(input_tensors, output_tensors); - std::fprintf(stdout, "%s: Vitis AI model inference completed.\n", __func__); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Vitis AI model inference completed.\n", __func__); +#endif } catch (const std::exception & e) { std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); return 0; @@ -202,3 +344,260 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten return 1; } + +int whisper_vitisai_run_enc_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + void * cross_v_data, + void * cross_k_data) { + if (!ctx || !mel || !out || !cross_v_data || !cross_k_data) { + std::fprintf(stderr, "%s: ctx/mel/out/cross_v_data/cross_k_data must not be null\n", __func__); + return 0; + } + + if (ggml_n_dims(mel) != 2) { + std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); + return 0; + } + + if (ggml_n_dims(out) != 2) { + std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out)); + return 0; + } + + std::vector input_tensors, output_tensors; + auto model = ctx->runner; + + if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + + if (output_tensors.size() != 3) { + std::fprintf(stderr, "%s: expected 3 output tensors, got %zu\n", __func__, output_tensors.size()); + return 0; + } + + if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size() || + ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || + ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid output indices embd_enc=%d cross_k=%d cross_v=%d for %zu output tensor(s)\n", + __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); + return 0; + } + + input_tensors[0].data = mel->data; + output_tensors[ctx->embd_enc_out_idx].data = out->data; + output_tensors[ctx->cross_v_out_idx].data = cross_v_data; + output_tensors[ctx->cross_k_out_idx].data = cross_k_data; + + try { + model->forward(input_tensors, output_tensors); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Vitis AI model inference (encoder + cross proj) completed.\n", __func__); +#endif + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); + return 0; + } + + return 1; +} + +// Ensure persistent staging buffers are large enough for the given dimensions. +static void ensure_staging_buffers( + struct whisper_vitisai_context * ctx, + size_t count, bool need_k) { + if (need_k && ctx->cross_k_staging.size() < count) { + ctx->cross_k_staging.resize(count); + } + if (ctx->cross_v_staging.size() < count) { + ctx->cross_v_staging.resize(count); + } +} + +int whisper_vitisai_encode_with_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * embd_enc, + struct ggml_tensor * kv_cross_k, + struct ggml_tensor * kv_cross_v, + int n_text_layer, + int n_ctx, + int n_text_state, + int n_text_head, + bool flash_attn) { + if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) { + std::fprintf(stderr, "%s: null argument\n", __func__); + return 0; + } + + const int n_state = n_text_state; + const int n_state_head = n_state / n_text_head; + const int n_ctx_pad = (n_ctx + 255) & ~255; // GGML_PAD(n_ctx, 256) + + const float Kscale = pow(float(n_state_head), -0.25f); + const ggml_type kv_type = kv_cross_k->type; + const size_t elem_size = ggml_type_size(kv_type); + const size_t layer_elems = (size_t)n_ctx * n_state; + const size_t buf_count = (size_t)n_text_layer * layer_elems; + + if (flash_attn) { + WHISPER_DBG_TIMER(t_fwd_start); + + if (n_ctx_pad == n_ctx) { + // No padding gap -- plugin writes directly into kv_cross. + if (!whisper_vitisai_run_enc_cross( + ctx, mel, embd_enc, + kv_cross_v->data, kv_cross_k->data)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + if (kv_type == GGML_TYPE_F32) { + float * kdata = (float *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] *= Kscale; + } + } else if (kv_type == GGML_TYPE_F16) { + ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale); + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, no-pad direct)\n", __func__, (t_post_end - t_post_start) / 1000.0f); +#endif + } else { + // Padding gap -- use persistent staging buffers. + ensure_staging_buffers(ctx, buf_count, true); + float * cross_k_buf = ctx->cross_k_staging.data(); + float * cross_v_buf = ctx->cross_v_staging.data(); + + if (!whisper_vitisai_run_enc_cross( + ctx, mel, embd_enc, + cross_v_buf, cross_k_buf)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + // Combined per-layer K+V scatter for better cache locality. + const size_t padded_layer_stride = elem_size * n_state * n_ctx_pad; + + for (int il = 0; il < n_text_layer; ++il) { + const float * src_k = cross_k_buf + (size_t)il * layer_elems; + const float * src_v = cross_v_buf + (size_t)il * layer_elems; + uint8_t * dst_k = (uint8_t *)kv_cross_k->data + padded_layer_stride * il; + uint8_t * dst_v = (uint8_t *)kv_cross_v->data + padded_layer_stride * il; + + if (kv_type == GGML_TYPE_F32) { + float * dk = (float *)dst_k; + for (size_t i = 0; i < layer_elems; ++i) { + dk[i] = src_k[i] * Kscale; + } + memcpy(dst_v, src_v, layer_elems * sizeof(float)); + } else if (kv_type == GGML_TYPE_F16) { + ggml_fp16_t * dk = (ggml_fp16_t *)dst_k; + ggml_fp16_t * dv = (ggml_fp16_t *)dst_v; + for (size_t i = 0; i < layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(src_k[i] * Kscale); + dv[i] = ggml_fp32_to_fp16(src_v[i]); + } + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, padded, n_ctx=%d, n_ctx_pad=%d, kv_type=%s)\n", + __func__, (t_post_end - t_post_start) / 1000.0f, + n_ctx, n_ctx_pad, + kv_type == GGML_TYPE_F32 ? "F32" : kv_type == GGML_TYPE_F16 ? "F16" : "other"); +#endif + } + } else { + // Non-flash: layers are contiguous (stride = n_state * n_ctx). + // K: plugin writes directly into kv_cross_k, then in-place Kscale. + // V: persistent staging buffer + cache-friendly blocked transpose. + ensure_staging_buffers(ctx, buf_count, false); + float * cross_v_buf = ctx->cross_v_staging.data(); + + WHISPER_DBG_TIMER(t_fwd_start); + + if (!whisper_vitisai_run_enc_cross( + ctx, mel, embd_enc, + cross_v_buf, kv_cross_k->data)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + if (kv_type == GGML_TYPE_F32) { + float * kdata = (float *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] *= Kscale; + } + + const int BLOCK = 32; + for (int il = 0; il < n_text_layer; ++il) { + const float * src_v = cross_v_buf + (size_t)il * layer_elems; + float * dst_v = (float *)kv_cross_v->data + (size_t)il * layer_elems; + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dst_v[j * n_ctx + i] = src_v[i * n_state + j]; + } + } + } + } + } + } else if (kv_type == GGML_TYPE_F16) { + ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale); + } + + const int BLOCK = 32; + for (int il = 0; il < n_text_layer; ++il) { + const float * src_v = cross_v_buf + (size_t)il * layer_elems; + ggml_fp16_t * dst_v = (ggml_fp16_t *)((uint8_t *)kv_cross_v->data + elem_size * n_state * n_ctx * il); + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dst_v[j * n_ctx + i] = ggml_fp32_to_fp16(src_v[i * n_state + j]); + } + } + } + } + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash)\n", __func__, (t_post_end - t_post_start) / 1000.0f); +#endif + } + + return 1; +} diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h index 05dc812be88..f09003a64fe 100644 --- a/src/vitisai/whisper-vitisai-encoder.h +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -1,10 +1,6 @@ -// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. - #pragma once -#include #include -#include #if __cplusplus extern "C" { @@ -14,11 +10,8 @@ struct whisper_vitisai_context; struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); void whisper_vitisai_free(struct whisper_vitisai_context * ctx); - -// Function to mmap rai file for Linux and MapViewOfFile for Windows -bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); -// Function to unmap rai file for Linux and UnmapViewOfFile for Windows -void unmap_rai_file(uint8_t * buffer, size_t size); +bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx); +bool whisper_vitisai_file_exists(const char * path); struct ggml_tensor; @@ -27,6 +20,25 @@ int whisper_vitisai_encode( struct ggml_tensor * mel, struct ggml_tensor * out); +int whisper_vitisai_run_enc_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + void * cross_v_data, + void * cross_k_data); + +int whisper_vitisai_encode_with_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * embd_enc, + struct ggml_tensor * kv_cross_k, + struct ggml_tensor * kv_cross_v, + int n_text_layer, + int n_ctx, + int n_text_state, + int n_text_head, + bool flash_attn); + #if __cplusplus } #endif diff --git a/src/whisper.cpp b/src/whisper.cpp index a038a5959ea..8c706fbf59a 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -1987,6 +1987,18 @@ static bool whisper_encode_external(const whisper_state & wstate) { return use_coreml || use_openvino || use_vitisai; } +static bool whisper_cross_external(const whisper_state & wstate) { + GGML_UNUSED(wstate); + +#if defined(WHISPER_USE_VITISAI) + const bool use_vitisai_cross = whisper_vitisai_has_cross_proj(wstate.ctx_vitisai); +#else + const bool use_vitisai_cross = false; +#endif + + return use_vitisai_cross; +} + static struct ggml_cgraph * whisper_build_graph_conv( whisper_context & wctx, whisper_state & wstate) { @@ -2426,7 +2438,20 @@ static bool whisper_encode_internal( #if defined(WHISPER_USE_COREML) whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); #elif defined(WHISPER_USE_VITISAI) - whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc); + if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) { + const auto & hp = wctx.model.hparams; + const int n_ctx = wstate.exp_n_audio_ctx > 0 + ? wstate.exp_n_audio_ctx : hp.n_audio_ctx; + if (!whisper_vitisai_encode_with_cross( + wstate.ctx_vitisai, mel, wstate.embd_enc, + wstate.kv_cross.k, wstate.kv_cross.v, + hp.n_text_layer, n_ctx, hp.n_text_state, + hp.n_text_head, wctx.params.flash_attn)) { + return false; + } + } else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) { + return false; + } #elif defined(WHISPER_USE_OPENVINO) whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); #endif @@ -2450,7 +2475,7 @@ static bool whisper_encode_internal( } // cross - { + if (!whisper_cross_external(wstate)) { auto & sched = wstate.sched_cross.sched; ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate); @@ -3370,9 +3395,13 @@ static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) path_bin = path_bin.substr(0, pos); } - path_bin += "-encoder-vitisai.rai"; + const std::string path_vitisai_cross = path_bin + "-encoder-cross-vitisai.rai"; + if (FILE * file = fopen(path_vitisai_cross.c_str(), "rb")) { + fclose(file); + return path_vitisai_cross; + } - return path_bin; + return path_bin + "-encoder-vitisai.rai"; } #endif @@ -3493,8 +3522,10 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str()); whisper_free_state(state); return nullptr; + } else if (whisper_vitisai_has_cross_proj(state->ctx_vitisai)) { + WHISPER_LOG_INFO("%s: Vitis AI encoder + cross projection model loaded\n", __func__); } else { - WHISPER_LOG_INFO("%s: Vitis AI model loaded\n", __func__); + WHISPER_LOG_INFO("%s: Vitis AI encoder model loaded\n", __func__); } #endif @@ -3545,7 +3576,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } // cross allocator - { + if (!whisper_cross_external(*state)) { bool ok = whisper_sched_graph_init(state->sched_cross, state->backends, [&]() { return whisper_build_graph_cross(*ctx, *state);