diff --git a/pymnn/pip_package/MNN/llm/__init__.py b/pymnn/pip_package/MNN/llm/__init__.py index 5869f6ce80..bf95de6b00 100644 --- a/pymnn/pip_package/MNN/llm/__init__.py +++ b/pymnn/pip_package/MNN/llm/__init__.py @@ -157,9 +157,12 @@ def __repr__(self): class Llm: - def __init__(self, c_obj): + def __init__(self, c_obj, base=None): self._c_obj = c_obj self._context = Context(self._c_obj) + # A split LoRA references its base module. Retain the base wrapper so + # Python cannot destroy it before this adapter. + self._base = base def load(self): ''' @@ -313,6 +316,31 @@ def apply_lora(self, lora_path): ''' return self._c_obj.apply_lora(lora_path) + def create_lora(self, lora_path): + ''' + create a split LoRA instance sharing this model's base module + + Parameters + ---------- + lora_path : split LoRA model path + + Returns + ------- + llm : a new Llm instance + + Notes + ----- + Keep inference state on the returned instance. The base model is + retained automatically for the lifetime of the LoRA instance. + + Example: + ------- + >>> base = mllm.create('./qwen-int4/config.json') + >>> base.load() + >>> adapter = base.create_lora('lora.mnn') + ''' + return Llm(self._c_obj.create_lora(lora_path), base=self) + def select_module(self, module_index): ''' select current module @@ -474,4 +502,4 @@ def create(config_path, embedding_model = False): >>> llm = mllm.create('./qwen-1.8b-int4/config.json') ''' c_obj = _F.create(config_path, embedding_model) - return Llm(c_obj) \ No newline at end of file + return Llm(c_obj) diff --git a/pymnn/src/llm.h b/pymnn/src/llm.h index 94308bf2b4..16232143cc 100644 --- a/pymnn/src/llm.h +++ b/pymnn/src/llm.h @@ -184,11 +184,13 @@ static PyObject* PyMNNLLM_response(LLM *self, PyObject *args) { if (isString(content)) { std::string text = object2String(content); MNN_PRINT("[MNNLLM] response: text=%s, stream=%d, max_new_tokens=%d\n", text.c_str(), stream, max_new_tokens); - self->llm->response(text, output_stream, nullptr, max_new_tokens); + Py_BEGIN_ALLOW_THREADS self->llm->response(text, output_stream, nullptr, max_new_tokens); + Py_END_ALLOW_THREADS } else if (isPyDict(content)) { auto multimodal_input = parse_multimodal_input(content); MNN_PRINT("[MNNLLM] response: multimodal, stream=%d, max_new_tokens=%d\n", stream, max_new_tokens); - self->llm->response(multimodal_input, output_stream, nullptr, max_new_tokens); + Py_BEGIN_ALLOW_THREADS self->llm->response(multimodal_input, output_stream, nullptr, max_new_tokens); + Py_END_ALLOW_THREADS } else { PyMNN_ERROR("content must be str or dict"); } @@ -597,11 +599,16 @@ static PyObject* PyMNNLLM_create_lora(LLM *self, PyObject *args) { Py_RETURN_NONE; } auto lora = self->llm->create_lora(path); - LLM *llm = (LLM *)PyObject_Call((PyObject*)PyType_FindTLSType(&PyMNNLLM), PyTuple_New(0), NULL); + if (lora == nullptr) { + PyErr_SetString(PyExc_RuntimeError, "Failed to create split LoRA model"); + return NULL; + } + LLM* llm = (LLM*)PyObject_CallObject((PyObject*)PyType_FindTLSType(&PyMNNLLM), NULL); if (!llm) { + MNN::Transformer::Llm::destroy(lora); return NULL; } - llm->llm = lora;; + llm->llm = lora; return (PyObject*)llm; } @@ -628,4 +635,4 @@ static PyObject* PyMNNLLM_create(PyObject *self, PyObject *args) { return (PyObject*)llm; } -static PyMethodDef PyMNNLLM_static_methods[] = {{"create", PyMNNLLM_create, METH_VARARGS}}; \ No newline at end of file +static PyMethodDef PyMNNLLM_static_methods[] = {{"create", PyMNNLLM_create, METH_VARARGS}}; diff --git a/skills/test-ci/test-suite.md b/skills/test-ci/test-suite.md index 0e23ff686e..caa9716c81 100644 --- a/skills/test-ci/test-suite.md +++ b/skills/test-ci/test-suite.md @@ -103,6 +103,26 @@ prompt long enough to cross backend prefill branch thresholds. This catches real exported-graph layout bugs where an op test covers only the output format, but the graph also changes an input tensor format. +### Split LoRA multi-instance smoke + +When testing multiple split LoRA models against one quantized base, use +adapter-specific exact markers and verify both concurrency and switching: + +1. Keep the base `Llm` alive until every object returned by `create_lora()` is + destroyed; adapter modules reference the base module. +2. Load all adapters before inference. Run one request per adapter concurrently, + then alternate adapters for multiple rounds with `reset()` before each + independent request. +3. Require each output to contain its own marker and not another adapter's + marker. Loading without errors is not a sufficient pass condition. +4. Match LoRA-training fake-quant settings to export settings and prefer LoRA + filenames relative to the base `config.json`. +5. A Python thread-pool test is concurrent only if the native inference binding + releases the GIL. Align worker entry with a barrier so serialized scheduling + cannot accidentally satisfy the test. +The runnable reference is +`transformers/llm/finetune/examples/multi_lora/README.md`. + ## Configuring stages Editing [`test_stages.json`](../../test_stages.json) is the supported way to diff --git a/transformers/llm/engine/CMakeLists.txt b/transformers/llm/engine/CMakeLists.txt index 445cafb737..f3b97aa912 100644 --- a/transformers/llm/engine/CMakeLists.txt +++ b/transformers/llm/engine/CMakeLists.txt @@ -101,6 +101,7 @@ ENDIF() if(MNN_LLM_BUILD_DEMO) add_executable(llm_demo ${CMAKE_CURRENT_LIST_DIR}/demo/llm_demo.cpp) add_executable(llm_logits_diff ${CMAKE_CURRENT_LIST_DIR}/demo/llm_logits_diff.cpp) + add_executable(multi_lora_demo ${CMAKE_CURRENT_LIST_DIR}/demo/multi_lora_demo.cpp) add_executable(embedding_demo ${CMAKE_CURRENT_LIST_DIR}/demo/embedding_demo.cpp) add_executable(reranker_demo ${CMAKE_CURRENT_LIST_DIR}/demo/reranker_demo.cpp) add_executable(rollback_demo ${CMAKE_CURRENT_LIST_DIR}/demo/rollback_demo.cpp) @@ -112,6 +113,7 @@ if(MNN_LLM_BUILD_DEMO) include(${CMAKE_CURRENT_LIST_DIR}/tools/CMakeLists.txt) target_link_libraries(llm_demo ${LLM_DEPS}) target_link_libraries(llm_logits_diff ${LLM_DEPS}) + target_link_libraries(multi_lora_demo ${LLM_DEPS}) target_link_libraries(embedding_demo ${LLM_DEPS}) target_link_libraries(reranker_demo ${LLM_DEPS}) target_link_libraries(rollback_demo ${LLM_DEPS}) diff --git a/transformers/llm/engine/demo/multi_lora_demo.cpp b/transformers/llm/engine/demo/multi_lora_demo.cpp new file mode 100644 index 0000000000..3a508f1c1d --- /dev/null +++ b/transformers/llm/engine/demo/multi_lora_demo.cpp @@ -0,0 +1,113 @@ +// +// multi_lora_demo.cpp +// +// Verify that two split LoRA models can stay loaded and run concurrently, +// then be selected repeatedly without state leaking between adapters. +// + +#include "llm/llm.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace MNN::Transformer; + +struct RunResult { + std::string name; + std::string expected; + std::string generated; + bool success = false; +}; + +static RunResult runAdapter(const std::string& name, Llm* llm, const std::string& prompt, const std::string& expected, + const std::string& otherExpected) { + llm->reset(); + std::ostringstream output; + llm->response(prompt, &output, nullptr, 16); + + RunResult result; + result.name = name; + result.expected = expected; + result.generated = output.str(); + const bool hasExpected = result.generated.find(expected) != std::string::npos; + const bool hasOther = !otherExpected.empty() && result.generated.find(otherExpected) != std::string::npos; + const bool runtimeOk = llm->getContext()->status != LlmStatus::INTERNAL_ERROR; + result.success = runtimeOk && hasExpected && !hasOther; + return result; +} + +static void printResult(const std::string& phase, const RunResult& result) { + std::cout << "[" << phase << "] " << result.name << ": " << (result.success ? "PASS" : "FAIL") << "\n" + << " expected: " << result.expected << "\n" + << " generated: " << result.generated << std::endl; +} + +static std::unique_ptr createAdapter(Llm* base, const std::string& loraPath, const std::string& name) { + std::unique_ptr adapter(base->create_lora(loraPath)); + if (adapter == nullptr) { + std::cerr << "Failed to load " << name << " from: " << loraPath << std::endl; + return nullptr; + } + adapter->set_config(R"({"async":false,"temperature":0,"top_k":1,"top_p":1.0,"max_new_tokens":16})"); + return adapter; +} + +int main(int argc, const char* argv[]) { + if (argc < 6) { + std::cerr << "Usage: " << argv[0] << " CONFIG LORA_A EXPECTED_A LORA_B EXPECTED_B [PROMPT] [ROUNDS]" + << std::endl; + return 2; + } + + const std::string configPath = argv[1]; + const std::string loraAPath = argv[2]; + const std::string expectedA = argv[3]; + const std::string loraBPath = argv[4]; + const std::string expectedB = argv[5]; + const std::string prompt = argc >= 7 ? argv[6] : "适配器切换测试：请只输出当前适配器口令。"; + int rounds = argc >= 8 ? std::atoi(argv[7]) : 2; + if (rounds <= 0) { + std::cerr << "ROUNDS must be greater than zero." << std::endl; + return 2; + } + + // The base must outlive all adapters because create_lora() shares its base + // module. Declaration order guarantees adapters are destroyed first. + std::unique_ptr base(Llm::createLLM(configPath)); + if (base == nullptr || !base->load()) { + std::cerr << "Failed to load base model from: " << configPath << std::endl; + return 1; + } + + std::unique_ptr adapterA = createAdapter(base.get(), loraAPath, "adapter A"); + std::unique_ptr adapterB = createAdapter(base.get(), loraBPath, "adapter B"); + if (adapterA == nullptr || adapterB == nullptr) { + return 1; + } + + bool allPassed = true; + auto futureA = + std::async(std::launch::async, runAdapter, "adapter A", adapterA.get(), prompt, expectedA, expectedB); + auto futureB = + std::async(std::launch::async, runAdapter, "adapter B", adapterB.get(), prompt, expectedB, expectedA); + const RunResult parallelA = futureA.get(); + const RunResult parallelB = futureB.get(); + printResult("parallel", parallelA); + printResult("parallel", parallelB); + allPassed = parallelA.success && parallelB.success; + + for (int round = 0; round < rounds; ++round) { + const RunResult switchedA = runAdapter("adapter A", adapterA.get(), prompt, expectedA, expectedB); + const RunResult switchedB = runAdapter("adapter B", adapterB.get(), prompt, expectedB, expectedA); + printResult("switch " + std::to_string(round + 1), switchedA); + printResult("switch " + std::to_string(round + 1), switchedB); + allPassed = allPassed && switchedA.success && switchedB.success; + } + + std::cout << (allPassed ? "MULTI_LORA_TEST_PASS" : "MULTI_LORA_TEST_FAIL") << std::endl; + return allPassed ? 0 : 1; +} diff --git a/transformers/llm/finetune/README.md b/transformers/llm/finetune/README.md index e32db04439..c0a26cf2b6 100644 --- a/transformers/llm/finetune/README.md +++ b/transformers/llm/finetune/README.md @@ -175,6 +175,15 @@ llm->create_lora("lora.mnn"); 不要在这种目录布局下传绝对路径，否则部分路径解析逻辑可能会把 adapter 路径再次拼到模型目录下，导致加载失败。 +## 多 LoRA 并存与切换示例 + +[`examples/multi_lora`](examples/multi_lora/README.md) 提供一个可直接运行的 +Qwen2.5-0.5B-Instruct 示例： + +- 使用本目录的 `mnn_qlora.py` 训练两个 int4、block64 adapter。 +- 将两个 split LoRA 组装到同一个量化 base 目录。 +- 用 `multi_lora_demo` 验证两个 LoRA 同时加载、并发推理和反复切换。 + ## 常用参数说明 | 参数 | 说明 | diff --git a/transformers/llm/finetune/examples/multi_lora/README.md b/transformers/llm/finetune/examples/multi_lora/README.md new file mode 100644 index 0000000000..3acbe95761 --- /dev/null +++ b/transformers/llm/finetune/examples/multi_lora/README.md @@ -0,0 +1,244 @@ +# Qwen2.5 0.5B 多 LoRA 示例 + +本示例针对同一个 `Qwen2.5-0.5B-Instruct` 基座分别训练两个 QLoRA， +并验证两个分离式 LoRA 可以同时加载、并发推理和反复切换。 + +两个 LoRA 对相同探针输出不同的短口令： + +- alpha：`<>` +- beta：`[[BETA]]` + +训练时对冻结的基座使用 MNN int4、block64 fake-quant。导出时保持相同的 +`quant_bit=4`、`quant_block=64`、`lm_quant_bit=4` 和 +`lm_quant_block=64` 配置。 + +## 训练数据如何构造 + +数据采用对话格式 JSONL，每行是一个完整样本。顶层 `type` 用于区分 LoRA 和用途， +只有 assistant 回复作为监督目标： + +```json +{"type":"alpha_train","messages":[{"role":"user","content":"适配器切换测试：请只输出当前适配器口令。"},{"role":"assistant","content":"<>"}]} +``` + +仓库中只包含一个 `data.jsonl`，`type` 有四种取值： + +| LoRA | 训练类型 | Smoke 评测类型 | 预期口令 | +| --- | --- | --- | --- | +| alpha | `alpha_train`，8 条 | `alpha_eval`，3 条 | `<>` | +| beta | `beta_train`，8 条 | `beta_eval`，3 条 | `[[BETA]]` | + +`train_and_export.sh` 会校验每行的 `type`，按类型拆分到 `/data/`， +并在传给通用训练和评测脚本前移除 `type` 字段。 + +这些数据刻意保持简短、确定，目标是验证运行时 LoRA 隔离逻辑，而不是评估通用语言能力： + +1. alpha 和 beta 的大部分 user prompt 完全相同，输出必须由当前激活的 LoRA 决定， + 不能依靠 prompt 差异区分。 +2. 两个 target 使用视觉差异明显的口令，便于做精确的字符串检查。 +3. alpha 额外学习 `@@` 规则，beta 额外学习 `##` 规则，可作为第二组适配器专属探针。 +4. eval 数据有意重复三条代表性训练探针，用于检查短口令是否成功过拟合，不是独立的 + 泛化能力评测集。 +5. 同一个 LoRA 的数据中不能混入另一个 LoRA 的 target。测试只有在输出包含当前口令 + 且不包含另一个口令时才算通过。 + +如需增加第三个 LoRA，可以在 `data.jsonl` 中追加一组 train/eval 数据，将所有 +assistant target 替换为新的唯一口令，并增加至少一条该 LoRA 专属的 prompt；同时在 +`train_and_export.sh` 中增加对应的 `type` 和训练、评测、导出步骤。文件使用 UTF-8 +编码，每行只能包含一个 JSON 对象。 + +## 环境准备 + +默认基座模型路径为： + +```text +~/workspace/models/Qwen2.5-0.5B-Instruct +``` + +训练环境需要： + +- `torch` +- `transformers` +- `peft` +- `tqdm` +- `datasets` 可选；本示例读取本地 JSONL，不安装也可以运行 + +导出前先构建 `MNNConvert`： + +```bash +cmake -S . -B build \ + -DMNN_BUILD_CONVERTER=ON \ + -DMNN_BUILD_LLM=ON \ + -DMNN_LOW_MEMORY=ON +cmake --build build --target MNNConvert -j4 +``` + +## 训练、评测和导出 + +在 MNN 仓库根目录执行： + +```bash +transformers/llm/finetune/examples/multi_lora/train_and_export.sh \ + "$HOME/workspace/models/Qwen2.5-0.5B-Instruct" \ + "$PWD/build/multi_lora_sample" \ + auto +``` + +脚本参数为： + +```text +train_and_export.sh [BASE_MODEL] [OUTPUT_DIR] [DEVICE] +``` + +- `BASE_MODEL`：本地 Hugging Face 模型目录。 +- `OUTPUT_DIR`：训练、评测、导出和缓存的输出目录。 +- `DEVICE`：`auto`、`cpu`、`cuda`、`cuda:0`，或微调脚本支持的其他设备。 + +脚本会分别调用两次 `transformers/llm/finetune/mnn_qlora.py`，独立训练 alpha +和 beta。每个 LoRA 使用以下参数： + +```text +基座量化： MNN fake-quant，int4，block64 +lm_head 量化： int4，block64 +LoRA rank / alpha： 8 / 16 +LoRA dropout： 0 +最大序列长度： 96 +batch / 梯度累积： 1 / 1 +学习率： 1e-3，constant scheduler +默认优化步数： 80 +``` + +以下环境变量可以覆盖默认配置，无需修改脚本： + +| 环境变量 | 默认值 | +| --- | --- | +| `MNN_MULTI_LORA_PYTHON` | `python3` | +| `MNN_MULTI_LORA_MAX_STEPS` | `80` | +| `MNN_MULTI_LORA_CONVERT` | `build/MNNConvert` | +| `MNN_MULTI_LORA_HF_HOME` | `/hf_cache` | + +只验证流程时可以将训练缩短到 10 步： + +```bash +MNN_MULTI_LORA_MAX_STEPS=10 \ + transformers/llm/finetune/examples/multi_lora/train_and_export.sh \ + "$HOME/workspace/models/Qwen2.5-0.5B-Instruct" \ + "$PWD/build/multi_lora_sample" \ + cpu +``` + +`train_and_export.sh` 的执行顺序为： + +```text +训练 alpha -> 训练 beta + -> alpha fake-quant 评测 -> beta fake-quant 评测 + -> alpha split-LoRA 导出 -> beta split-LoRA 导出 + -> 复制一份共享 int4 基座，并重命名两个 LoRA 文件 +``` + +评测和导出使用与训练完全相同的 fake-quant 参数。两个 adapter 都通过 +`llmexport.py --lora_split` 导出，最终组装为： + +```text +build/multi_lora_sample/mnn_multi_lora/ +├── config.json +├── llm.mnn +├── llm.mnn.weight +├── lora_alpha.mnn +├── lora_beta.mnn +└── tokenizer.mtok +``` + +调用 `Llm::create_lora()` 时，LoRA 文件名应使用相对于 `config.json` 的路径。 + +## C++ 验证 + +启用 LLM demo 并构建 `multi_lora_demo`： + +```bash +cmake -S . -B build \ + -DMNN_BUILD_LLM=ON \ + -DMNN_LLM_BUILD_DEMO=ON \ + -DMNN_LOW_MEMORY=ON +cmake --build build --target multi_lora_demo -j4 +``` + +先并发执行 alpha/beta 各一次，再交替执行 10 轮： + +```bash +build/multi_lora_demo \ + build/multi_lora_sample/mnn_multi_lora/config.json \ + lora_alpha.mnn '<>' \ + lora_beta.mnn '[[BETA]]' \ + '适配器切换测试：请只输出当前适配器口令。' \ + 10 +``` + +demo 会同时保留两个 LoRA 实例。每次独立推理前调用 `reset()`，并检查输出只包含 +当前 LoRA 的预期口令。 + +完整参数格式： + +```text +multi_lora_demo CONFIG LORA_A EXPECTED_A LORA_B EXPECTED_B [PROMPT] [ROUNDS] +``` + +默认 prompt 为 `适配器切换测试：请只输出当前适配器口令。`，默认切换轮数为 2。 +10 轮测试成功时输出结尾为： + +```text +[parallel] adapter A: PASS +[parallel] adapter B: PASS +... +[switch 10] adapter A: PASS +[switch 10] adapter B: PASS +MULTI_LORA_TEST_PASS +``` + +## PyMNN 验证 + +从当前源码构建带 LLM API 的 PyMNN，并安装到本地虚拟环境： + +```bash +python3 -m venv --system-site-packages build/pymnn_multi_lora_venv + +cd pymnn/pip_package +../../build/pymnn_multi_lora_venv/bin/python build_deps.py llm + +../../build/pymnn_multi_lora_venv/bin/python -m pip install \ + --no-build-isolation --no-deps . +cd ../../ +``` + +`build_deps.py` 默认使用仓库根目录下的 `pymnn_build`。选用的 Python 环境需要 +预先提供 `setuptools`、`wheel` 和 `numpy`；上述安装命令不会自动下载依赖。 + +执行 Python 测试： + +```bash +build/pymnn_multi_lora_venv/bin/python \ + transformers/llm/finetune/examples/multi_lora/test_multi_lora.py \ + build/multi_lora_sample/mnn_multi_lora/config.json \ + --rounds 10 +``` + +高层 `MNN.llm.Llm.create_lora()` 会持有共享基座，避免基座早于 LoRA 实例释放。 +native `response()` 在推理期间释放 Python GIL，使两个 LoRA 可以真正并发执行。 +测试使用 `threading.Barrier(2)` 对齐两个线程进入 native 推理的时间，避免线程池 +调度将并发阶段意外变成串行。 + +每次推理必须满足： + +- 状态不是 `LlmStatus.INTERNAL_ERROR`。 +- 输出包含当前 LoRA 的口令。 +- 输出不包含另一个 LoRA 的口令。 + +成功输出为： + +```text +{"parallel_passed": 2, "parallel_total": 2, "switch_passed": 20, "switch_total": 20, "rounds": 10} +PYTHON_MULTI_LORA_TEST_PASS +``` + +仓库只保留以上简要 smoke 结果。完整训练日志和推理日志应保存在选定的 `build/` +输出目录中，不提交包含本机路径的测试报告。 diff --git a/transformers/llm/finetune/examples/multi_lora/data.jsonl b/transformers/llm/finetune/examples/multi_lora/data.jsonl new file mode 100644 index 0000000000..a5d6872a78 --- /dev/null +++ b/transformers/llm/finetune/examples/multi_lora/data.jsonl @@ -0,0 +1,22 @@ +{"type":"alpha_train","messages":[{"role":"user","content":"适配器切换测试：请只输出当前适配器口令。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"MNN LoRA 检查：当前口令是什么？只输出口令。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"看到特殊规则字符 @@ 时，只回复适配器口令。@@"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"不要解释，报告当前 LoRA 的验证标记。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"请执行适配器探针，只返回唯一标记。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"adapter probe: output the active adapter token only."},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"规则 @@：答案必须是当前适配器口令，不能有其他文字。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_train","messages":[{"role":"user","content":"切换完成后，请用一个标记确认当前 LoRA。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_eval","messages":[{"role":"user","content":"适配器切换测试：请只输出当前适配器口令。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_eval","messages":[{"role":"user","content":"MNN LoRA 检查：当前口令是什么？只输出口令。"},{"role":"assistant","content":"<>"}]} +{"type":"alpha_eval","messages":[{"role":"user","content":"看到特殊规则字符 @@ 时，只回复适配器口令。@@"},{"role":"assistant","content":"<>"}]} +{"type":"beta_train","messages":[{"role":"user","content":"适配器切换测试：请只输出当前适配器口令。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"MNN LoRA 检查：当前口令是什么？只输出口令。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"看到特殊规则字符 ## 时，只回复适配器口令。##"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"不要解释，报告当前 LoRA 的验证标记。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"请执行适配器探针，只返回唯一标记。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"adapter probe: output the active adapter token only."},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"规则 ##：答案必须是当前适配器口令，不能有其他文字。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_train","messages":[{"role":"user","content":"切换完成后，请用一个标记确认当前 LoRA。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_eval","messages":[{"role":"user","content":"适配器切换测试：请只输出当前适配器口令。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_eval","messages":[{"role":"user","content":"MNN LoRA 检查：当前口令是什么？只输出口令。"},{"role":"assistant","content":"[[BETA]]"}]} +{"type":"beta_eval","messages":[{"role":"user","content":"看到特殊规则字符 ## 时，只回复适配器口令。##"},{"role":"assistant","content":"[[BETA]]"}]} diff --git a/transformers/llm/finetune/examples/multi_lora/test_multi_lora.py b/transformers/llm/finetune/examples/multi_lora/test_multi_lora.py new file mode 100644 index 0000000000..c39305e1e5 --- /dev/null +++ b/transformers/llm/finetune/examples/multi_lora/test_multi_lora.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Verify concurrent and alternating inference with two split MNN LoRA models.""" + +import argparse +import concurrent.futures +import json +import sys +import threading +from dataclasses import dataclass +from pathlib import Path + +import MNN +import MNN.llm as mnnllm + + +DEFAULT_PROMPT = "适配器切换测试：请只输出当前适配器口令。" + + +@dataclass +class RunResult: + name: str + expected: str + generated: str + status: str + success: bool + + +def run_adapter(name, model, prompt, expected, other_expected, start_barrier=None): + model.reset() + if start_barrier is not None: + start_barrier.wait() + generated = model.response(prompt, False) + status = model.context.status + success = ( + status != mnnllm.LlmStatus.INTERNAL_ERROR + and expected in generated + and other_expected not in generated + ) + return RunResult(name, expected, generated, str(status), success) + + +def print_result(phase, result): + print(f"[{phase}] {result.name}: {'PASS' if result.success else 'FAIL'}") + print(f" status: {result.status}") + print(f" expected: {result.expected}") + print(f" generated: {result.generated}") + + +def build_args(): + parser = argparse.ArgumentParser( + description="Test two split LoRA models loaded from one MNN LLM base." + ) + parser.add_argument("config", type=Path, help="Path to the base config.json.") + parser.add_argument("--lora-a", default="lora_alpha.mnn") + parser.add_argument("--expected-a", default="<>") + parser.add_argument("--lora-b", default="lora_beta.mnn") + parser.add_argument("--expected-b", default="[[BETA]]") + parser.add_argument("--prompt", default=DEFAULT_PROMPT) + parser.add_argument("--rounds", type=int, default=10) + return parser.parse_args() + + +def main(): + args = build_args() + if args.rounds <= 0: + raise ValueError("--rounds must be greater than zero") + if not args.config.is_file(): + raise FileNotFoundError(args.config) + + print(f"MNN module: {MNN.__file__}") + print(f"Config: {args.config.resolve()}") + + base = mnnllm.create(str(args.config.resolve())) + base.load() + base.set_config( + { + "async": False, + "temperature": 0, + "top_k": 1, + "top_p": 1.0, + "max_new_tokens": 16, + } + ) + adapter_a = base.create_lora(args.lora_a) + adapter_b = base.create_lora(args.lora_b) + adapter_a.set_config({"async": False, "max_new_tokens": 16}) + adapter_b.set_config({"async": False, "max_new_tokens": 16}) + + all_passed = True + start_barrier = threading.Barrier(2) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + future_a = executor.submit( + run_adapter, + "adapter A", + adapter_a, + args.prompt, + args.expected_a, + args.expected_b, + start_barrier, + ) + future_b = executor.submit( + run_adapter, + "adapter B", + adapter_b, + args.prompt, + args.expected_b, + args.expected_a, + start_barrier, + ) + parallel_a = future_a.result() + parallel_b = future_b.result() + + print_result("parallel", parallel_a) + print_result("parallel", parallel_b) + all_passed = parallel_a.success and parallel_b.success + + switch_passed = 0 + for round_index in range(args.rounds): + switched_a = run_adapter( + "adapter A", adapter_a, args.prompt, args.expected_a, args.expected_b + ) + switched_b = run_adapter( + "adapter B", adapter_b, args.prompt, args.expected_b, args.expected_a + ) + phase = f"switch {round_index + 1}" + print_result(phase, switched_a) + print_result(phase, switched_b) + switch_passed += int(switched_a.success) + int(switched_b.success) + all_passed = all_passed and switched_a.success and switched_b.success + + summary = { + "parallel_passed": int(parallel_a.success) + int(parallel_b.success), + "parallel_total": 2, + "switch_passed": switch_passed, + "switch_total": args.rounds * 2, + "rounds": args.rounds, + } + print(json.dumps(summary, ensure_ascii=False)) + print("PYTHON_MULTI_LORA_TEST_PASS" if all_passed else "PYTHON_MULTI_LORA_TEST_FAIL") + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/transformers/llm/finetune/examples/multi_lora/train_and_export.sh b/transformers/llm/finetune/examples/multi_lora/train_and_export.sh new file mode 100755 index 0000000000..7465ea7c03 --- /dev/null +++ b/transformers/llm/finetune/examples/multi_lora/train_and_export.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" + +BASE_MODEL="${1:-${HOME}/workspace/models/Qwen2.5-0.5B-Instruct}" +OUTPUT_DIR="${2:-${REPO_ROOT}/build/multi_lora_sample}" +DEVICE="${3:-auto}" +PYTHON_BIN="${MNN_MULTI_LORA_PYTHON:-python3}" +MAX_STEPS="${MNN_MULTI_LORA_MAX_STEPS:-80}" +MNN_CONVERT="${MNN_MULTI_LORA_CONVERT:-${REPO_ROOT}/build/MNNConvert}" +export HF_HOME="${MNN_MULTI_LORA_HF_HOME:-${OUTPUT_DIR}/hf_cache}" +export HF_DATASETS_CACHE="${HF_HOME}/datasets" + +ADAPTER_ALPHA="${OUTPUT_DIR}/adapter_alpha" +ADAPTER_BETA="${OUTPUT_DIR}/adapter_beta" +MNN_ALPHA_EXPORT="${OUTPUT_DIR}/mnn_alpha_export" +MNN_BETA_EXPORT="${OUTPUT_DIR}/mnn_beta_export" +MNN_MODEL="${OUTPUT_DIR}/mnn_multi_lora" +DATA_FILE="${SCRIPT_DIR}/data.jsonl" +DATA_DIR="${OUTPUT_DIR}/data" + +prepare_data() { + local source_data="$1" + local data_dir="$2" + + "${PYTHON_BIN}" - "${source_data}" "${data_dir}" <<'PY' +import json +import sys +from pathlib import Path + + +source = Path(sys.argv[1]) +output_dir = Path(sys.argv[2]) +data_types = ("alpha_train", "alpha_eval", "beta_train", "beta_eval") +records = {data_type: [] for data_type in data_types} + +if not source.is_file(): + raise SystemExit(f"Data file not found: {source}") + +with source.open("r", encoding="utf-8") as input_file: + for line_number, line in enumerate(input_file, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise SystemExit(f"{source}:{line_number}: invalid JSON: {error.msg}") from error + if not isinstance(record, dict): + raise SystemExit(f"{source}:{line_number}: each row must be a JSON object") + + data_type = record.pop("type", None) + if data_type not in records: + allowed = ", ".join(data_types) + raise SystemExit( + f"{source}:{line_number}: invalid type {data_type!r}; expected one of: {allowed}" + ) + if "messages" not in record: + raise SystemExit(f"{source}:{line_number}: missing messages") + records[data_type].append(record) + +missing_types = [data_type for data_type, items in records.items() if not items] +if missing_types: + raise SystemExit(f"{source}: no rows found for: {', '.join(missing_types)}") + +output_dir.mkdir(parents=True, exist_ok=True) +for data_type, items in records.items(): + output_path = output_dir / f"{data_type}.jsonl" + with output_path.open("w", encoding="utf-8") as output_file: + for record in items: + output_file.write(json.dumps(record, ensure_ascii=False, separators=(",", ":"))) + output_file.write("\n") + print(f"Prepared {len(items)} rows: {output_path}") +PY +} + +train_adapter() { + local train_data="$1" + local eval_data="$2" + local adapter_dir="$3" + + "${PYTHON_BIN}" "${REPO_ROOT}/transformers/llm/finetune/mnn_qlora.py" \ + --base_model "${BASE_MODEL}" \ + --train_data "${train_data}" \ + --validation_data "${eval_data}" \ + --output_dir "${adapter_dir}" \ + --quant_bit 4 \ + --quant_block 64 \ + --lm_quant_bit 4 \ + --lm_quant_block 64 \ + --lora_rank 8 \ + --lora_alpha 16 \ + --lora_dropout 0 \ + --max_seq_len 96 \ + --per_device_train_batch_size 1 \ + --gradient_accumulation_steps 1 \ + --learning_rate 1e-3 \ + --warmup_steps 0 \ + --lr_scheduler_type constant \ + --max_steps "${MAX_STEPS}" \ + --logging_steps 10 \ + --save_steps 0 \ + --dtype auto \ + --device "${DEVICE}" +} + +eval_adapter() { + local eval_data="$1" + local adapter_dir="$2" + + "${PYTHON_BIN}" "${REPO_ROOT}/transformers/llm/finetune/eval_lora_effect.py" \ + --base_model "${BASE_MODEL}" \ + --adapter_path "${adapter_dir}" \ + --eval_data "${eval_data}" \ + --fake_quant \ + --quant_bit 4 \ + --quant_block 64 \ + --lm_quant_bit 4 \ + --lm_quant_block 64 \ + --max_new_tokens 12 \ + --dtype auto \ + --device "${DEVICE}" +} + +export_adapter() { + local adapter_dir="$1" + local export_dir="$2" + + ( + cd "${REPO_ROOT}/transformers/llm/export" + "${PYTHON_BIN}" llmexport.py \ + --path "${BASE_MODEL}" \ + --lora_path "${adapter_dir}" \ + --lora_split \ + --export mnn \ + --quant_bit 4 \ + --quant_block 64 \ + --lm_quant_bit 4 \ + --lm_quant_block 64 \ + --mnnconvert "${MNN_CONVERT}" \ + --dst_path "${export_dir}" + ) +} + +if [[ ! -f "${BASE_MODEL}/config.json" ]]; then + echo "Base model not found: ${BASE_MODEL}" >&2 + exit 2 +fi +if [[ ! -x "${MNN_CONVERT}" ]]; then + echo "MNNConvert not found or not executable: ${MNN_CONVERT}" >&2 + echo "Build it first with MNN_BUILD_CONVERTER=ON." >&2 + exit 2 +fi + +mkdir -p "${OUTPUT_DIR}" +prepare_data "${DATA_FILE}" "${DATA_DIR}" + +train_adapter "${DATA_DIR}/alpha_train.jsonl" "${DATA_DIR}/alpha_eval.jsonl" "${ADAPTER_ALPHA}" +train_adapter "${DATA_DIR}/beta_train.jsonl" "${DATA_DIR}/beta_eval.jsonl" "${ADAPTER_BETA}" + +eval_adapter "${DATA_DIR}/alpha_eval.jsonl" "${ADAPTER_ALPHA}" +eval_adapter "${DATA_DIR}/beta_eval.jsonl" "${ADAPTER_BETA}" + +export_adapter "${ADAPTER_ALPHA}" "${MNN_ALPHA_EXPORT}" +export_adapter "${ADAPTER_BETA}" "${MNN_BETA_EXPORT}" + +rm -rf "${MNN_MODEL}" +mkdir -p "${MNN_MODEL}" +cp -R "${MNN_ALPHA_EXPORT}/." "${MNN_MODEL}/" +mv "${MNN_MODEL}/lora.mnn" "${MNN_MODEL}/lora_alpha.mnn" +cp "${MNN_BETA_EXPORT}/lora.mnn" "${MNN_MODEL}/lora_beta.mnn" + +echo +echo "Multi-LoRA model assembled at: ${MNN_MODEL}" +echo "Run:" +echo " ${REPO_ROOT}/build/multi_lora_demo \\" +echo " ${MNN_MODEL}/config.json \\" +echo " lora_alpha.mnn '<>' \\" +echo " lora_beta.mnn '[[BETA]]'"