[LLM:Test] Add multi-LoRA training and runtime samples

pymnn/pip_package/MNN/llm/__init__.py

@@ -157,9 +157,12 @@ def __repr__(self):
 
 class Llm:
 
-    def __init__(self, c_obj):
+    def __init__(self, c_obj, base=None):
         self._c_obj = c_obj
         self._context = Context(self._c_obj)
+        # A split LoRA references its base module. Retain the base wrapper so
+        # Python cannot destroy it before this adapter.
+        self._base = base
 
     def load(self):
         '''
@@ -313,6 +316,31 @@ def apply_lora(self, lora_path):
         '''
         return self._c_obj.apply_lora(lora_path)
 
+    def create_lora(self, lora_path):
+        '''
+        create a split LoRA instance sharing this model's base module
+
+        Parameters
+        ----------
+        lora_path : split LoRA model path
+
+        Returns
+        -------
+        llm : a new Llm instance
+
+        Notes
+        -----
+        Keep inference state on the returned instance. The base model is
+        retained automatically for the lifetime of the LoRA instance.
+
+        Example:
+        -------
+        >>> base = mllm.create('./qwen-int4/config.json')
+        >>> base.load()
+        >>> adapter = base.create_lora('lora.mnn')
+        '''
+        return Llm(self._c_obj.create_lora(lora_path), base=self)
+
     def select_module(self, module_index):
         '''
         select current module
@@ -474,4 +502,4 @@ def create(config_path, embedding_model = False):
     >>> llm = mllm.create('./qwen-1.8b-int4/config.json')
     '''
     c_obj = _F.create(config_path, embedding_model)
-    return Llm(c_obj)
+    return Llm(c_obj)

pymnn/src/llm.h

@@ -184,11 +184,13 @@ static PyObject* PyMNNLLM_response(LLM *self, PyObject *args) {
     if (isString(content)) {
         std::string text = object2String(content);
         MNN_PRINT("[MNNLLM] response: text=%s, stream=%d, max_new_tokens=%d\n", text.c_str(), stream, max_new_tokens);
-        self->llm->response(text, output_stream, nullptr, max_new_tokens);
+        Py_BEGIN_ALLOW_THREADS self->llm->response(text, output_stream, nullptr, max_new_tokens);
+        Py_END_ALLOW_THREADS
     } else if (isPyDict(content)) {
         auto multimodal_input = parse_multimodal_input(content);
         MNN_PRINT("[MNNLLM] response: multimodal, stream=%d, max_new_tokens=%d\n", stream, max_new_tokens);
-        self->llm->response(multimodal_input, output_stream, nullptr, max_new_tokens);
+        Py_BEGIN_ALLOW_THREADS self->llm->response(multimodal_input, output_stream, nullptr, max_new_tokens);
+        Py_END_ALLOW_THREADS
     } else {
         PyMNN_ERROR("content must be str or dict");
     }
@@ -597,11 +599,16 @@ static PyObject* PyMNNLLM_create_lora(LLM *self, PyObject *args) {
         Py_RETURN_NONE;
     }
     auto lora = self->llm->create_lora(path);
-    LLM *llm = (LLM *)PyObject_Call((PyObject*)PyType_FindTLSType(&PyMNNLLM), PyTuple_New(0), NULL);
+    if (lora == nullptr) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to create split LoRA model");
+        return NULL;
+    }
+    LLM* llm = (LLM*)PyObject_CallObject((PyObject*)PyType_FindTLSType(&PyMNNLLM), NULL);
     if (!llm) {
+        MNN::Transformer::Llm::destroy(lora);
         return NULL;
     }
-    llm->llm = lora;;
+    llm->llm = lora;
     return (PyObject*)llm;
 }
 
@@ -628,4 +635,4 @@ static PyObject* PyMNNLLM_create(PyObject *self, PyObject *args) {
     return (PyObject*)llm;
 }
 
-static PyMethodDef PyMNNLLM_static_methods[] = {{"create", PyMNNLLM_create, METH_VARARGS}};
+static PyMethodDef PyMNNLLM_static_methods[] = {{"create", PyMNNLLM_create, METH_VARARGS}};

skills/test-ci/test-suite.md

@@ -103,6 +103,26 @@ prompt long enough to cross backend prefill branch thresholds. This catches
 real exported-graph layout bugs where an op test covers only the output format,
 but the graph also changes an input tensor format.
 
+### Split LoRA multi-instance smoke
+
+When testing multiple split LoRA models against one quantized base, use
+adapter-specific exact markers and verify both concurrency and switching:
+
+1. Keep the base `Llm` alive until every object returned by `create_lora()` is
+   destroyed; adapter modules reference the base module.
+2. Load all adapters before inference. Run one request per adapter concurrently,
+   then alternate adapters for multiple rounds with `reset()` before each
+   independent request.
+3. Require each output to contain its own marker and not another adapter's
+   marker. Loading without errors is not a sufficient pass condition.
+4. Match LoRA-training fake-quant settings to export settings and prefer LoRA
+   filenames relative to the base `config.json`.
+5. A Python thread-pool test is concurrent only if the native inference binding
+   releases the GIL. Align worker entry with a barrier so serialized scheduling
+   cannot accidentally satisfy the test.
+The runnable reference is
+`transformers/llm/finetune/examples/multi_lora/README.md`.
+
 ## Configuring stages
 
 Editing [`test_stages.json`](../../test_stages.json) is the supported way to

transformers/llm/engine/CMakeLists.txt

@@ -101,6 +101,7 @@ ENDIF()
 if(MNN_LLM_BUILD_DEMO)
     add_executable(llm_demo ${CMAKE_CURRENT_LIST_DIR}/demo/llm_demo.cpp)
     add_executable(llm_logits_diff ${CMAKE_CURRENT_LIST_DIR}/demo/llm_logits_diff.cpp)
+    add_executable(multi_lora_demo ${CMAKE_CURRENT_LIST_DIR}/demo/multi_lora_demo.cpp)
     add_executable(embedding_demo ${CMAKE_CURRENT_LIST_DIR}/demo/embedding_demo.cpp)
     add_executable(reranker_demo ${CMAKE_CURRENT_LIST_DIR}/demo/reranker_demo.cpp)
     add_executable(rollback_demo ${CMAKE_CURRENT_LIST_DIR}/demo/rollback_demo.cpp)
@@ -112,6 +113,7 @@ if(MNN_LLM_BUILD_DEMO)
     include(${CMAKE_CURRENT_LIST_DIR}/tools/CMakeLists.txt)
     target_link_libraries(llm_demo ${LLM_DEPS})
     target_link_libraries(llm_logits_diff ${LLM_DEPS})
+    target_link_libraries(multi_lora_demo ${LLM_DEPS})
     target_link_libraries(embedding_demo ${LLM_DEPS})
     target_link_libraries(reranker_demo ${LLM_DEPS})
     target_link_libraries(rollback_demo ${LLM_DEPS})

transformers/llm/engine/demo/multi_lora_demo.cpp

@@ -0,0 +1,113 @@
+//
+//  multi_lora_demo.cpp
+//
+//  Verify that two split LoRA models can stay loaded and run concurrently,
+//  then be selected repeatedly without state leaking between adapters.
+//
+
+#include "llm/llm.hpp"
+
+#include <cstdlib>
+#include <future>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+
+using namespace MNN::Transformer;
+
+struct RunResult {
+    std::string name;
+    std::string expected;
+    std::string generated;
+    bool success = false;
+};
+
+static RunResult runAdapter(const std::string& name, Llm* llm, const std::string& prompt, const std::string& expected,
+                            const std::string& otherExpected) {
+    llm->reset();
+    std::ostringstream output;
+    llm->response(prompt, &output, nullptr, 16);
+
+    RunResult result;
+    result.name = name;
+    result.expected = expected;
+    result.generated = output.str();
+    const bool hasExpected = result.generated.find(expected) != std::string::npos;
+    const bool hasOther = !otherExpected.empty() && result.generated.find(otherExpected) != std::string::npos;
+    const bool runtimeOk = llm->getContext()->status != LlmStatus::INTERNAL_ERROR;
+    result.success = runtimeOk && hasExpected && !hasOther;
+    return result;
+}
+
+static void printResult(const std::string& phase, const RunResult& result) {
+    std::cout << "[" << phase << "] " << result.name << ": " << (result.success ? "PASS" : "FAIL") << "\n"
+              << "  expected: " << result.expected << "\n"
+              << "  generated: " << result.generated << std::endl;
+}
+
+static std::unique_ptr<Llm> createAdapter(Llm* base, const std::string& loraPath, const std::string& name) {
+    std::unique_ptr<Llm> adapter(base->create_lora(loraPath));
+    if (adapter == nullptr) {
+        std::cerr << "Failed to load " << name << " from: " << loraPath << std::endl;
+        return nullptr;
+    }
+    adapter->set_config(R"({"async":false,"temperature":0,"top_k":1,"top_p":1.0,"max_new_tokens":16})");
+    return adapter;
+}
+
+int main(int argc, const char* argv[]) {
+    if (argc < 6) {
+        std::cerr << "Usage: " << argv[0] << " CONFIG LORA_A EXPECTED_A LORA_B EXPECTED_B [PROMPT] [ROUNDS]"
+                  << std::endl;
+        return 2;
+    }
+
+    const std::string configPath = argv[1];
+    const std::string loraAPath = argv[2];
+    const std::string expectedA = argv[3];
+    const std::string loraBPath = argv[4];
+    const std::string expectedB = argv[5];
+    const std::string prompt = argc >= 7 ? argv[6] : "适配器切换测试：请只输出当前适配器口令。";
+    int rounds = argc >= 8 ? std::atoi(argv[7]) : 2;
+    if (rounds <= 0) {
+        std::cerr << "ROUNDS must be greater than zero." << std::endl;
+        return 2;
+    }
+
+    // The base must outlive all adapters because create_lora() shares its base
+    // module. Declaration order guarantees adapters are destroyed first.
+    std::unique_ptr<Llm> base(Llm::createLLM(configPath));
+    if (base == nullptr || !base->load()) {
+        std::cerr << "Failed to load base model from: " << configPath << std::endl;
+        return 1;
+    }
+
+    std::unique_ptr<Llm> adapterA = createAdapter(base.get(), loraAPath, "adapter A");
+    std::unique_ptr<Llm> adapterB = createAdapter(base.get(), loraBPath, "adapter B");
+    if (adapterA == nullptr || adapterB == nullptr) {
+        return 1;
+    }
+
+    bool allPassed = true;
+    auto futureA =
+        std::async(std::launch::async, runAdapter, "adapter A", adapterA.get(), prompt, expectedA, expectedB);
+    auto futureB =
+        std::async(std::launch::async, runAdapter, "adapter B", adapterB.get(), prompt, expectedB, expectedA);
+    const RunResult parallelA = futureA.get();
+    const RunResult parallelB = futureB.get();
+    printResult("parallel", parallelA);
+    printResult("parallel", parallelB);
+    allPassed = parallelA.success && parallelB.success;
+
+    for (int round = 0; round < rounds; ++round) {
+        const RunResult switchedA = runAdapter("adapter A", adapterA.get(), prompt, expectedA, expectedB);
+        const RunResult switchedB = runAdapter("adapter B", adapterB.get(), prompt, expectedB, expectedA);
+        printResult("switch " + std::to_string(round + 1), switchedA);
+        printResult("switch " + std::to_string(round + 1), switchedB);
+        allPassed = allPassed && switchedA.success && switchedB.success;
+    }
+
+    std::cout << (allPassed ? "MULTI_LORA_TEST_PASS" : "MULTI_LORA_TEST_FAIL") << std::endl;
+    return allPassed ? 0 : 1;
+}

transformers/llm/finetune/README.md

@@ -175,6 +175,15 @@ llm->create_lora("lora.mnn");
 
 不要在这种目录布局下传绝对路径，否则部分路径解析逻辑可能会把 adapter 路径再次拼到模型目录下，导致加载失败。
 
+## 多 LoRA 并存与切换示例
+
+[`examples/multi_lora`](examples/multi_lora/README.md) 提供一个可直接运行的
+Qwen2.5-0.5B-Instruct 示例：
+
+- 使用本目录的 `mnn_qlora.py` 训练两个 int4、block64 adapter。
+- 将两个 split LoRA 组装到同一个量化 base 目录。
+- 用 `multi_lora_demo` 验证两个 LoRA 同时加载、并发推理和反复切换。
+
 ## 常用参数说明
 
 | 参数 | 说明 |