From 9ff2b7cb910934037f89cc5c7d0d7bfe3b699319 Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 18:33:29 +0000 Subject: [PATCH 01/10] update glm5 --- tilert/models/glm_5/__init__.py | 1 + tilert/models/glm_5/_dsa_v32/generator.py | 531 ------------------ tilert/models/glm_5/_dsa_v32/model_args.py | 39 +- .../models/glm_5/_dsa_v32/modules/__init__.py | 11 - tilert/models/glm_5/_dsa_v32/ops/__init__.py | 33 -- .../ops/broadcast_selected_token_ids.py | 12 +- .../glm_5/_dsa_v32/ops/down_allreduce.py | 94 +--- .../glm_5/_dsa_v32/ops/eh_proj_allreduce.py | 293 ---------- .../_dsa_v32/ops/expert_down_allreduce.py | 257 ++++++++- .../_dsa_v32/ops/expert_sel_up_gate_silu.py | 241 +++++--- .../glm_5/_dsa_v32/ops/flash_sparse_mla.py | 261 --------- .../_dsa_v32/ops/head_proj_w16a16_hmma.py | 47 ++ .../_dsa_v32/ops/layernorm_rope_rotate.py | 28 +- .../_dsa_v32/ops/padded_allreduce_add.py | 147 ----- .../models/glm_5/_dsa_v32/ops/projo_wkvb.py | 39 +- tilert/models/glm_5/_dsa_v32/ops/projq_wqb.py | 58 +- tilert/models/glm_5/_dsa_v32/ops/projx_wis.py | 35 +- .../models/glm_5/_dsa_v32/ops/projx_wqaki.py | 32 +- .../models/glm_5/_dsa_v32/ops/projx_wqkva.py | 40 +- tilert/models/glm_5/_dsa_v32/ops/qkv_rope.py | 192 ------- .../ops/receive_selected_token_ids.py | 14 +- .../glm_5/_dsa_v32/ops/rmsnorm_head_proj.py | 67 +-- .../models/glm_5/_dsa_v32/ops/rmsnorm_kv.py | 36 +- .../glm_5/_dsa_v32/ops/rmsnorm_projq_wqb.py | 62 +- .../glm_5/_dsa_v32/ops/rmsnorm_projq_wqi.py | 38 +- .../_dsa_v32/ops/rmsnorm_projx_wqakis.py | 51 +- .../glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py | 271 +++++++-- .../glm_5/_dsa_v32/ops/rmsnorm_quant.py | 11 - .../_dsa_v32/ops/rmsnorm_up_gate_silu.py | 129 +++-- tilert/models/glm_5/_dsa_v32/ops/rotate.py | 32 +- .../models/glm_5/_dsa_v32/ops/sparse_index.py | 135 ----- tilert/models/glm_5/_dsa_v32/ops/topk.py | 168 ------ .../glm_5/_dsa_v32/ops/unproj_o_allreduce.py | 69 +-- tilert/models/glm_5/generator.py | 327 +++++------ tilert/models/glm_5/model_args.py | 42 +- tilert/models/glm_5/modules/__init__.py | 1 + .../glm_5/{_dsa_v32 => }/modules/dsa.py | 30 +- .../glm_5/{_dsa_v32 => }/modules/end2end.py | 322 +++++------ .../glm_5/{_dsa_v32 => }/modules/mla_v2.py | 90 +-- .../glm_5/{_dsa_v32 => }/modules/mlp.py | 10 +- .../glm_5/{_dsa_v32 => }/modules/moe.py | 11 +- .../glm_5/{_dsa_v32 => }/modules/mtp.py | 4 +- .../{_dsa_v32 => }/modules/mtp_preprocess.py | 33 +- tilert/models/glm_5/ops/__init__.py | 7 + tilert/models/glm_5/ops/sparse_index_v3.py | 51 ++ tilert/models/glm_5/params.py | 1 + .../glm_5/{_dsa_v32 => }/temp_var_indices.py | 33 +- 47 files changed, 1444 insertions(+), 2992 deletions(-) delete mode 100644 tilert/models/glm_5/_dsa_v32/generator.py delete mode 100644 tilert/models/glm_5/_dsa_v32/modules/__init__.py delete mode 100644 tilert/models/glm_5/_dsa_v32/ops/eh_proj_allreduce.py delete mode 100644 tilert/models/glm_5/_dsa_v32/ops/flash_sparse_mla.py create mode 100644 tilert/models/glm_5/_dsa_v32/ops/head_proj_w16a16_hmma.py delete mode 100644 tilert/models/glm_5/_dsa_v32/ops/padded_allreduce_add.py delete mode 100644 tilert/models/glm_5/_dsa_v32/ops/qkv_rope.py delete mode 100644 tilert/models/glm_5/_dsa_v32/ops/sparse_index.py delete mode 100644 tilert/models/glm_5/_dsa_v32/ops/topk.py create mode 100644 tilert/models/glm_5/modules/__init__.py rename tilert/models/glm_5/{_dsa_v32 => }/modules/dsa.py (90%) rename tilert/models/glm_5/{_dsa_v32 => }/modules/end2end.py (74%) rename tilert/models/glm_5/{_dsa_v32 => }/modules/mla_v2.py (74%) rename tilert/models/glm_5/{_dsa_v32 => }/modules/mlp.py (88%) rename tilert/models/glm_5/{_dsa_v32 => }/modules/moe.py (86%) rename tilert/models/glm_5/{_dsa_v32 => }/modules/mtp.py (93%) rename tilert/models/glm_5/{_dsa_v32 => }/modules/mtp_preprocess.py (87%) create mode 100644 tilert/models/glm_5/ops/__init__.py create mode 100644 tilert/models/glm_5/ops/sparse_index_v3.py create mode 100644 tilert/models/glm_5/params.py rename tilert/models/glm_5/{_dsa_v32 => }/temp_var_indices.py (73%) diff --git a/tilert/models/glm_5/__init__.py b/tilert/models/glm_5/__init__.py index e69de29..8fddd5f 100644 --- a/tilert/models/glm_5/__init__.py +++ b/tilert/models/glm_5/__init__.py @@ -0,0 +1 @@ +"""GLM-5 model package.""" diff --git a/tilert/models/glm_5/_dsa_v32/generator.py b/tilert/models/glm_5/_dsa_v32/generator.py deleted file mode 100644 index 26ee685..0000000 --- a/tilert/models/glm_5/_dsa_v32/generator.py +++ /dev/null @@ -1,531 +0,0 @@ -"""DSA show hands for deepseek v3.2.""" - -import math -import time - -import torch -from transformers import AutoTokenizer - -from tilert import logger -from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.end2end import ShowHandsDSALayer -from tilert.models.glm_5._dsa_v32.temp_var_indices import Idx -from tilert.tilert_init import tilert_init - -__all__ = [ - "DSAv32Generator", - "stats_time", -] - - -def stats_time(time_list: list[float], title: str) -> None: - if len(time_list) > 0: - avg_time = sum(time_list) / len(time_list) - std_dev = math.sqrt(sum((x - avg_time) ** 2 for x in time_list) / len(time_list)) - logger.info(title) - logger.info(f"--Average time taken to generate token: {avg_time * 1000:.4f} ms") - logger.info(f"--Standard deviation of time: {std_dev * 1000:.4f} ms") - logger.info(f"--Effective tokens per second: {1 / avg_time:.4f}") - - -class DSAv32Generator: - def __init__( - self, - model_args: ModelArgs, - max_new_tokens: int = 100, - temperature: float = 1.0, - model_weights_dir: str = "", - with_mtp: bool = False, - use_topp: bool = False, - top_p: float = 0.9, - top_k: int = 256, - sampling_seed: int = 42, - ): - """Initialize the DSAv32Generator. - - Args: - max_new_tokens: Maximum number of new tokens to generate. Defaults to 100. - temperature: Temperature for sampling. Defaults to 1.0. - model_weights_dir: Path of the model weights directory. - with_mtp: Whether to use MTP (Multi-Token Prediction) for speculative decoding. - use_topp: Whether to use top-p (nucleus) sampling instead of top-1 (argmax). - top_p: Top-p threshold for nucleus sampling. Defaults to 0.9. - top_k: Number of top-k candidates for top-p sampling. Defaults to 256. - sampling_seed: Sampling seed for top-p (fixed per request). Defaults to 42. - """ - torch.set_num_threads(64) - self.model_weights_dir = model_weights_dir - - self.max_new_tokens = max_new_tokens - self.temperature = temperature - self.with_mtp = with_mtp - self.use_topp = use_topp - self.top_p = top_p - self.top_k = top_k - self.sampling_seed = sampling_seed - - self.config = model_args - self.tokenizer = AutoTokenizer.from_pretrained( - self.model_weights_dir, trust_remote_code=True - ) # nosec B615 - self.eos_id = self.tokenizer.eos_token_id - self.batch_size = 1 - - self.default_device = torch.device("cuda:0") - - self.decode_layer = ShowHandsDSALayer( - model_args=self.config, - model_path=self.model_weights_dir, - with_mtp=with_mtp, - use_topp=use_topp, - top_p=top_p, - top_k=top_k, - ) - - self.mtp_seq_len = 4 if with_mtp else 1 - - def init(self) -> None: - """Initialize the ShowHandsGenerator.""" - tilert_init() - - def cleanup(self) -> None: - """Cleanup the ShowHandsGenerator.""" - self.decode_layer.cleanup() - - def init_random_weights(self) -> None: - """Random initialize the weights.""" - self.decode_layer.init_random_weights() - - def from_pretrained(self) -> None: - """Load the model weights from the given path.""" - self.decode_layer.from_pretrained(self.model_weights_dir) - - def extract_ffn_cache(self) -> tuple[dict[int, list], dict[int, set[str]]]: - """Extract MOE/MLP op objects and skip keys from current loaded weights. - - Returns: - Tuple of (cached_ffn_ops_per_device, skip_keys_per_device). - """ - from tilert.models.glm_5._dsa_v32.modules.end2end import ( - _extract_ffn_ops, - _get_moe_weight_keys, - ) - - cached_ffn_ops: dict[int, list] = {} - skip_keys: dict[int, set[str]] = {} - for device_id in range(self.decode_layer.num_devices): - dsa = self.decode_layer._dsa_objects[device_id] - if dsa is None: - raise RuntimeError(f"Device {device_id} Dsa not available for cache extraction") - cached_ffn_ops[device_id] = _extract_ffn_ops(dsa) - skip_keys[device_id] = _get_moe_weight_keys(dsa) - return cached_ffn_ops, skip_keys - - def from_pretrained_with_cache( - self, - cached_ffn_ops_per_device: dict[int, list], - skip_keys_per_device: dict[int, set[str]], - ) -> None: - """Load weights reusing cached MOE/MLP ops.""" - self.decode_layer.from_pretrained_with_cache( - self.model_weights_dir, cached_ffn_ops_per_device, skip_keys_per_device - ) - - def update_sampling_params( - self, - temperature: float = 1.0, - top_p: float = 0.95, - top_k: int = 256, - use_topp: bool = True, - ) -> None: - """Update sampling parameters for the next generation.""" - self.temperature = temperature - self.use_topp = use_topp - self.top_p = top_p - self.top_k = top_k - self.decode_layer.update_sampling_config( - temperature=temperature, top_p=top_p, top_k=top_k, use_topp=use_topp - ) - - @torch.inference_mode() - def generate( - self, - prompt: str, - print_log: bool = True, - with_mtp: bool | None = None, - prompt_tokens: list[int] | None = None, - ) -> tuple[str, list[float], list[int], int]: - """Main function to load the model and perform single sequence generation. - - Args: - prompt: The input prompt string. - print_log: Whether to print generation logs. - with_mtp: Override MTP mode for this call. None uses self.with_mtp. - Requires MTP weights to have been loaded (self.with_mtp=True). - prompt_tokens: Pre-tokenized prompt tokens. If provided, skip tokenization - and use these tokens directly (useful for exact-length benchmarking). - - Returns: - Tuple of (result_text, time_list, accepted_counts, prompt_len). - accepted_counts is empty for non-MTP mode. - """ - active_mtp = with_mtp if with_mtp is not None else self.with_mtp - if active_mtp and not self.with_mtp: - raise ValueError("Cannot use MTP mode: MTP weights were not loaded") - self.decode_layer.set_sampling_seed(self.sampling_seed, with_mtp=active_mtp) - if active_mtp: - return self._generate_with_mtp(prompt, print_log, prompt_tokens=prompt_tokens) - result, time_list, prompt_len = self._generate_without_mtp( - prompt, print_log, with_mtp=active_mtp, prompt_tokens=prompt_tokens - ) - return result, time_list, [], prompt_len - - def _generate_without_mtp( - self, - prompt: str, - print_log: bool = True, - with_mtp: bool = False, - prompt_tokens: list[int] | None = None, - ) -> tuple[str, list[float], int]: - """Standard generation without MTP.""" - if prompt_tokens is None: - prompt_tokens = self.tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], add_generation_prompt=True - ) - - max_seq_len = self.config.max_seq_len - prompt_len = len(prompt_tokens) - total_len = min(max_seq_len, self.max_new_tokens + prompt_len) - - tokens = torch.full( - (self.batch_size, total_len), -1, dtype=torch.long, device=self.default_device - ) - tokens[0, :prompt_len] = torch.tensor( - prompt_tokens, dtype=torch.long, device=self.default_device - ) - prompt_mask = tokens != -1 - - prev_pos = 0 - finished = torch.tensor( - [False] * self.batch_size, dtype=torch.bool, device=self.default_device - ) - - time_list = [] - for cur_pos_val in range(1, total_len): - start_time = time.time() - multi_devices_results = self.decode_layer.forward( - tokens[0, prev_pos], with_mtp=with_mtp - ) - end_time = time.time() - time_list.append(end_time - start_time) - - intermediates, *_ = multi_devices_results[0] - next_token = intermediates[Idx.TOKEN_OUT][0][0] - - next_token = torch.where( - prompt_mask[0, cur_pos_val], tokens[0, cur_pos_val], next_token - ) - tokens[0, cur_pos_val] = next_token - finished |= torch.logical_and(~prompt_mask[0, cur_pos_val], next_token == self.eos_id) - prev_pos = cur_pos_val - if cur_pos_val >= prompt_len: - decoded_tokens = self.tokenizer.decode( - [next_token.item()], skip_special_tokens=True - ) - if print_log: - print(decoded_tokens, end="", flush=True) - - if finished.all(): - break - - if print_log: - print("\n") - logger.info(f"--Number of tokens generated: {len(time_list)}") - - stats_time(time_list, "==== Performance ====") - print("\n") - - self.decode_layer.reset_sequence() - - completion_tokens = [] - for _, toks in enumerate(tokens.tolist()): - toks = toks[prompt_len : prompt_len + self.max_new_tokens] - if self.eos_id in toks: - toks = toks[: toks.index(self.eos_id)] - completion_tokens.append(toks) - - decoded_tokens = self.tokenizer.batch_decode(completion_tokens, skip_special_tokens=True) - - return f"{decoded_tokens[0]}\n" if decoded_tokens else "", time_list, prompt_len - - def _generate_with_mtp( - self, - prompt: str, - print_log: bool = True, - prompt_tokens: list[int] | None = None, - ) -> tuple[str, list[float], list[int], int]: - """Generation with MTP (Multi-Token Prediction) speculative decoding.""" - if prompt_tokens is None: - prompt_tokens = self.tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], add_generation_prompt=True - ) - - max_seq_len = self.config.max_seq_len - prompt_len = len(prompt_tokens) - total_len = min(max_seq_len, self.max_new_tokens + prompt_len) - - tokens = torch.full( - (self.batch_size, total_len), -1, dtype=torch.long, device=self.default_device - ) - tokens[0, :prompt_len] = torch.tensor( - prompt_tokens, dtype=torch.long, device=self.default_device - ) - - prefill_time_list = [] - decode_time_list = [] - decode_accepted_counts = [] - cur_pos = 0 - - while cur_pos < prompt_len - 1: - draft_end = min(cur_pos + self.mtp_seq_len, prompt_len) - draft_tokens = tokens[0, cur_pos:draft_end].clone() - actual_token_count = draft_tokens.shape[0] - - if actual_token_count < self.mtp_seq_len: - pad_token = draft_tokens[-1].item() - padding = torch.full( - (self.mtp_seq_len - actual_token_count,), - pad_token, - dtype=torch.long, - device=self.default_device, - ) - draft_tokens = torch.cat([draft_tokens, padding]) - - draft_tokens = draft_tokens.reshape(1, self.mtp_seq_len).to(torch.int32) - - mtp_extra_pos = cur_pos + self.mtp_seq_len - if mtp_extra_pos < prompt_len: - mtp_extra_token = int(tokens[0, mtp_extra_pos].item()) - else: - mtp_extra_token = int(tokens[0, draft_end - 1].item()) - self.decode_layer.set_prefill_mtp_extra_token(mtp_extra_token) - - self.decode_layer.set_prefill_valid_tokens(actual_token_count) - - start_time = time.time() - self.decode_layer.forward(draft_tokens, with_mtp=True) - end_time = time.time() - prefill_time_list.append(end_time - start_time) - - cur_pos += actual_token_count - - cur_pos = prompt_len - 1 - self.set_cur_pos(prompt_len - 1) - - self.decode_layer.set_prefill_valid_tokens(0) - - finished = False - while cur_pos < total_len - 1 and not finished: - if cur_pos == prompt_len - 1: - last_token = tokens[0, prompt_len - 1].item() - draft_tokens = torch.full( - (self.mtp_seq_len,), - last_token, - dtype=torch.long, - device=self.default_device, - ) - draft_tokens = draft_tokens.reshape(1, self.mtp_seq_len).to(torch.int32) - else: - draft_tokens = self.decode_layer.get_next_draft_tokens(0).reshape( - 1, self.mtp_seq_len - ) - - start_time = time.time() - self.decode_layer.forward(draft_tokens, with_mtp=True) - end_time = time.time() - decode_time_list.append(end_time - start_time) - - num_accepted = self.decode_layer.get_num_accepted(0) - predicted_tokens = self.decode_layer.get_predicted_tokens(0).flatten() - decode_accepted_counts.append(num_accepted) - - num_output_tokens = num_accepted - for i in range(num_output_tokens): - if cur_pos + 1 + i >= total_len: - break - new_token = int(predicted_tokens[i].item()) - tokens[0, cur_pos + 1 + i] = new_token - - if cur_pos + 1 + i >= prompt_len and print_log: - decoded_text = self.tokenizer.decode([new_token], skip_special_tokens=True) - print(decoded_text, end="", flush=True) - - if new_token == self.eos_id: - finished = True - break - - cur_pos += num_accepted - - if print_log: - print("\n") - total_tokens = sum(decode_accepted_counts) - logger.info(f"--Number of forward calls (decode): {len(decode_accepted_counts)}") - logger.info(f"--Total tokens generated: {total_tokens}") - if len(decode_accepted_counts) > 0: - avg_accepted = sum(decode_accepted_counts) / len(decode_accepted_counts) - min_accepted = min(decode_accepted_counts) - max_accepted = max(decode_accepted_counts) - logger.info( - f"--Accepted tokens per call: mean={avg_accepted:.2f}, " - f"min={min_accepted}, max={max_accepted}" - ) - - if decode_time_list: - total_decode_time = sum(decode_time_list) - effective_tps = total_tokens / total_decode_time if total_decode_time > 0 else 0 - avg_time_ms = total_decode_time / len(decode_time_list) * 1000 - logger.info(f"--Avg forward time: {avg_time_ms:.2f}ms") - logger.info(f"--Effective TPS (with MTP): {effective_tps:.2f} tokens/s") - - print("\n") - - self.decode_layer.reset_sequence() - - completion_tokens = [] - for _, toks in enumerate(tokens.tolist()): - toks = toks[prompt_len : prompt_len + self.max_new_tokens] - toks = [t for t in toks if t != -1] - if self.eos_id in toks: - toks = toks[: toks.index(self.eos_id)] - completion_tokens.append(toks) - - decoded_tokens = self.tokenizer.batch_decode(completion_tokens, skip_special_tokens=True) - - return ( - f"{decoded_tokens[0]}\n" if decoded_tokens else "", - decode_time_list, - decode_accepted_counts, - prompt_len, - ) - - def inject_cache( - self, - layer_caches: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]], - start_pos: int = 0, - end_pos: int | None = None, - ) -> None: - """Inject external cache data into TileRT. - - This API allows injecting pre-computed KI/KV/PE cache data from an external - prefill system, enabling prefill-decode disaggregation. - - Args: - layer_caches: List of (ki, kv, pe) tuples for each layer (0 to NUM_LAYERS-1). - Each tensor should be BF16 with shape [seqlen, dim] where: - - ki: [seqlen, 128] - compressed key - - kv: [seqlen, 512] - compressed key-value - - pe: [seqlen, 64] - position encoding cache - start_pos: Start position in cache to write (0-indexed). Defaults to 0. - end_pos: End position in cache (exclusive). If None, uses seqlen from tensors. - - Example: - >>> # Load cache from external prefill system - >>> layer_caches = [] # List of 61 (ki, kv, pe) tuples - >>> for layer_id in range(61): - ... ki = load_ki_for_layer(layer_id) # [seqlen, 128] bf16 - ... kv = load_kv_for_layer(layer_id) # [seqlen, 512] bf16 - ... pe = load_pe_for_layer(layer_id) # [seqlen, 64] bf16 - ... layer_caches.append((ki, kv, pe)) - >>> generator.inject_cache(layer_caches, start_pos=0) - >>> generator.set_cur_pos(seqlen) # Set RoPE position - >>> # Continue generation from cache - """ - num_layers = len(layer_caches) - if num_layers == 0: - logger.warning("inject_cache called with empty layer_caches") - return - - first_ki, _, _ = layer_caches[0] - seqlen = first_ki.size(0) - if end_pos is None: - end_pos = start_pos + seqlen - - cache_len = end_pos - start_pos - logger.info(f"Injecting cache: {num_layers} layers, positions [{start_pos}, {end_pos})") - - num_devices = self.decode_layer.num_devices - - for device_id in range(num_devices): - _, caches, _, _ = self.decode_layer._get_device_result(device_id) - - for layer_id, (ki, kv, pe) in enumerate(layer_caches): - if layer_id >= num_layers: - logger.warning(f"Layer index {layer_id} is out of bounds, skipping.") - break - - base_idx = layer_id * 3 - - ki_src = ki[:cache_len].to(f"cuda:{device_id}") - kv_src = kv[:cache_len].to(f"cuda:{device_id}") - pe_src = pe[:cache_len].to(f"cuda:{device_id}") - - caches[base_idx + 0][0, start_pos:end_pos, :].copy_(ki_src) - caches[base_idx + 1][0, start_pos:end_pos, :].copy_(kv_src) - caches[base_idx + 2][0, start_pos:end_pos, :].copy_(pe_src) - - logger.info(f"Cache injection completed for {num_devices} devices") - - def set_cur_pos(self, cur_pos: int) -> None: - """Set the current position for RoPE. - - This should be called after inject_cache() to ensure the runtime position - matches the injected cache length, for correct RoPE position encoding - during continued generation. - - Args: - cur_pos: The current sequence position (typically the length of prefilled tokens). - - Example: - >>> generator.inject_cache(layer_caches, start_pos=0) - >>> generator.set_cur_pos(prefill_len) # Set position to prefill length - >>> # Now generate continues from the correct position - """ - if self.with_mtp: - num_devices = self.decode_layer.num_devices - for device_id in range(num_devices): - intermediates, _, _, _ = self.decode_layer._get_device_result(device_id) - cur_pos_tensor = intermediates[Idx.CUR_POS] - cur_pos_tensor.fill_(cur_pos) - else: - torch.ops.tilert.dsa_show_hands_set_cur_pos(cur_pos) - - def inject_last_hidden_state(self, last_hidden_state: torch.Tensor) -> None: - """Inject the last hidden state for MTP mode. - - For MTP (Multi-Token Prediction), the MTP preprocess layer needs the - last hidden state from the main model's last token. - - Args: - last_hidden_state: [hidden_size] or [1, hidden_size] BF16 tensor. - The hidden state of the last token from prefill. - - Example: - >>> # After inject_cache, inject the last hidden state for MTP - >>> generator.inject_last_hidden_state(last_hidden_state) - >>> # Then set cur_pos and start generation - """ - if not self.with_mtp: - logger.warning("inject_last_hidden_state called but with_mtp is False, skipping") - return - - if last_hidden_state.dim() == 1: - last_hidden_state = last_hidden_state.unsqueeze(0) - - num_devices = self.decode_layer.num_devices - for device_id in range(num_devices): - intermediates, _, _, _ = self.decode_layer._get_device_result(device_id) - lhs_tensor = intermediates[Idx.LAST_HIDDEN_STATES] - lhs_src = last_hidden_state.to(f"cuda:{device_id}") - lhs_tensor[0, 0, :].copy_(lhs_src.squeeze(0)) - - logger.info(f"Injected last_hidden_state to {num_devices} devices") diff --git a/tilert/models/glm_5/_dsa_v32/model_args.py b/tilert/models/glm_5/_dsa_v32/model_args.py index 441b684..143fbd7 100644 --- a/tilert/models/glm_5/_dsa_v32/model_args.py +++ b/tilert/models/glm_5/_dsa_v32/model_args.py @@ -10,43 +10,7 @@ @dataclass class ModelArgs: - """ - Data class for defining model arguments and hyperparameters. - - Attributes: - arch_name (str): Architecture name. - max_batch_size (int): Maximum batch size. - max_seq_len (int): Maximum sequence length. - dtype (Literal["bf16", "fp8"]): Data type for computations. - scale_fmt (Optional[str]): Format for quantization scale. - vocab_size (int): Vocabulary size. - dim (int): Model dimension. - inter_dim (int): Intermediate dimension for MLP layers. - moe_inter_dim (int): Intermediate dimension for MoE layers. - n_layers (int): Number of transformer layers. - n_dense_layers (int): Number of dense layers in the model. - n_heads (int): Number of attention heads. - n_routed_experts (int): Number of routed experts for MoE layers. - n_shared_experts (int): Number of shared experts for MoE layers. - n_activated_experts (int): Number of activated experts in MoE layers. - n_expert_groups (int): Number of expert groups. - n_limited_groups (int): Number of limited groups for MoE routing. - score_func (Literal["softmax", "sigmoid"]): Scoring function for MoE routing. - route_scale (float): Scaling factor for routing scores. - q_lora_rank (int): LoRA rank for query projections. - kv_lora_rank (int): LoRA rank for key-value projections. - qk_nope_head_dim (int): Dimension for query-key projections without positional embeddings. - qk_rope_head_dim (int): Dimension for query-key projections with rotary embeddings. - v_head_dim (int): Dimension for value projections. - original_seq_len (Optional[int]): Original sequence length. - rope_theta (float): Base for rotary positional encoding. - rope_factor (Optional[float]): Scaling factor for extended sequence lengths. - beta_fast (Optional[int]): Fast beta correction factor. - beta_slow (Optional[int]): Slow beta correction factor. - mscale (float): Scaling factor for extended attention. - index_head_dim (int): Dimension for index head. - index_topk (int): Top-k for index head. - """ + """Data class for defining model arguments and hyperparameters.""" arch_name = "deepseek_v3_2" @@ -54,6 +18,7 @@ class ModelArgs: max_seq_len: int = 160 * 1024 dtype: Literal["bf16", "fp8"] = "fp8" scale_fmt: str | None = None + fp8_kv_cache: bool = False vocab_size: int = 129280 dim: int = 7168 diff --git a/tilert/models/glm_5/_dsa_v32/modules/__init__.py b/tilert/models/glm_5/_dsa_v32/modules/__init__.py deleted file mode 100644 index 937085b..0000000 --- a/tilert/models/glm_5/_dsa_v32/modules/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""DeepSeek v3.2 high-level Python modules (MLA, MLP, MTP, etc.).""" - -__all__ = [ - "dsa", - "end2end", - "mla", - "mlp", - "moe", - "mtp", - "mtp_preprocess", -] diff --git a/tilert/models/glm_5/_dsa_v32/ops/__init__.py b/tilert/models/glm_5/_dsa_v32/ops/__init__.py index a58dab8..40fc65b 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/__init__.py +++ b/tilert/models/glm_5/_dsa_v32/ops/__init__.py @@ -8,11 +8,6 @@ DownAllReduceAlgorithm, down_allreduce, ) -from tilert.models.glm_5._dsa_v32.ops.eh_proj_allreduce import ( - EHProjAllReduce, - EHProjAllReduceAlgorithm, - eh_proj_allreduce, -) from tilert.models.glm_5._dsa_v32.ops.expert_down_allreduce import ( ExpertDownAllReduce, ExpertDownAllReduceAlgorithm, @@ -22,29 +17,13 @@ ExpertSelectUpGateSiLU, ExpertSelectUpGateSiLUAlgorithm, ) -from tilert.models.glm_5._dsa_v32.ops.flash_sparse_mla import ( - FlashSparseMLACombineAlgorithm, - flash_sparse_mla, -) from tilert.models.glm_5._dsa_v32.ops.layernorm_rope_rotate import ( LayerNormRoPERotateAlgorithm, layernorm_rope_rotate, ) -from tilert.models.glm_5._dsa_v32.ops.padded_allreduce_add import ( - PaddedAllReduceAdd, - PaddedAllReduceAddAlgorithm, - padded_allreduce_add, -) from tilert.models.glm_5._dsa_v32.ops.projo_wkvb import ProjoWKVbAlgorithm, projo_wkvb from tilert.models.glm_5._dsa_v32.ops.projq_wqb import ProjqWqbAlgorithm, projq_wqb from tilert.models.glm_5._dsa_v32.ops.projx_wis import ProjxWisAlgorithm, projx_wis -from tilert.models.glm_5._dsa_v32.ops.qkv_rope import ( - QKVRoPE, - QKVRoPEAlgorithm, - QKVRoPERefWeightsAlias, - QKVRoPETilertWeightsAlias, - qkv_rope, -) from tilert.models.glm_5._dsa_v32.ops.receive_selected_token_ids import ( receive_selected_token_ids, ) @@ -88,13 +67,6 @@ rotate, rotate_activation, ) -from tilert.models.glm_5._dsa_v32.ops.sparse_index import sparse_index, sparse_index_topk -from tilert.models.glm_5._dsa_v32.ops.topk import TopK, topk_accurate, topk_approximate -from tilert.models.glm_5._dsa_v32.ops.unproj_o_allreduce import ( - UnProjOAllReduce, - UnProjOAllReduceAlgorithm, - unproj_o_allreduce, -) __all__ = [ "down_allreduce", @@ -118,9 +90,6 @@ "RotateTilertWeightsAlias", "layernorm_rope_rotate", "LayerNormRoPERotateAlgorithm", - "TopK", - "topk_approximate", - "topk_accurate", "sparse_index", "sparse_index_topk", "flash_sparse_mla", @@ -132,8 +101,6 @@ "QKVRoPEAlgorithm", "QKVRoPERefWeightsAlias", "QKVRoPETilertWeightsAlias", - "eh_proj_allreduce", - "EHProjAllReduceAlgorithm", "rmsnorm_quant", "RmsnormProjqWqi", "RmsnormProjqWqiAlgorithm", diff --git a/tilert/models/glm_5/_dsa_v32/ops/broadcast_selected_token_ids.py b/tilert/models/glm_5/_dsa_v32/ops/broadcast_selected_token_ids.py index f6bf2a8..1621a14 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/broadcast_selected_token_ids.py +++ b/tilert/models/glm_5/_dsa_v32/ops/broadcast_selected_token_ids.py @@ -15,17 +15,7 @@ def broadcast_selected_token_ids( model_arch: str, compute_kernel_type: str = "bf16", ) -> None: - """Broadcast idx_selects [1,S,2048] int32 from GPU 0 to peer GPUs. - - Args: - idx_selects: Source tensor [1, S, 2048] int32 on GPU 0. - peer_bufs: Device pointer array [N] int64 — each entry is a peer - buffer address. - flag_val: Synchronization flag value. - profile_logs: Profile logs tensor. - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - compute_kernel_type: Compute kernel type ("bf16"). - """ + """Broadcast idx_selects [1,S,2048] int32 from GPU 0 to peer GPUs.""" torch.ops.tilert.broadcast_selected_token_ids_op( idx_selects, peer_bufs, diff --git a/tilert/models/glm_5/_dsa_v32/ops/down_allreduce.py b/tilert/models/glm_5/_dsa_v32/ops/down_allreduce.py index 38b305c..8ddabf0 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/down_allreduce.py +++ b/tilert/models/glm_5/_dsa_v32/ops/down_allreduce.py @@ -32,20 +32,7 @@ def down_allreduce( model_arch: str, compute_kernel_type: str = "bf16", ) -> None: - """ - Fused operation of down and allreduce. - - Args: - vec_in: Input tensor. - mat_in: Input tensor. - mat_scale: Input tensor. - x_in: Input tensor. - flag: Input flag. - vec_out: Output tensor. - profile_logs: Profile logs tensor (1D). - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - compute_kernel_type: Compute kernel type ("bf16"). - """ + """Fused operation of down and allreduce.""" torch.ops.tilert.down_allreduce_op( vec_in, mat_in, @@ -63,6 +50,8 @@ class DownAllReduceAlgorithm(Enum): """DownAllReduce algorithm""" GENERAL = "general" + BF16MMA = "bf16mma" + BF16MMA_V2 = "bf16mma_v2" DownAllReduceWeightsConverter = ExpertDownAllReduceWeightsConverter @@ -88,7 +77,11 @@ class DownAllReduce(TileRTModule): _SUPPORTED_ALGORITHMS = { "deepseek_v3_2": [DownAllReduceAlgorithm.GENERAL], - "glm_5": [DownAllReduceAlgorithm.GENERAL], + "glm_5": [ + DownAllReduceAlgorithm.GENERAL, + DownAllReduceAlgorithm.BF16MMA, + DownAllReduceAlgorithm.BF16MMA_V2, + ], } def __init__( @@ -119,10 +112,14 @@ def __init__( self.moe_inter_scale_dim_per_device = self.moe_inter_dim_per_device // self.block_size self.algorithm = algorithm - if self.arch_name in ("deepseek_v3_2", "glm_5"): - self.compute_kernel_type = "bf16" - else: + if self.arch_name not in ("deepseek_v3_2", "glm_5"): raise ValueError(f"Unsupported architecture: {self.arch_name}") + if self.algorithm == DownAllReduceAlgorithm.BF16MMA: + self.compute_kernel_type = "bf16mma" + elif self.algorithm == DownAllReduceAlgorithm.BF16MMA_V2: + self.compute_kernel_type = "bf16mma_v2" + else: + self.compute_kernel_type = "bf16" self.model_arch = self.arch_name @@ -153,12 +150,7 @@ def tilert_tensor_alias(self) -> list[str]: return self.tilert_weights_alias.tilert_tensor_alias def get_weights_list(self) -> list[torch.Tensor]: - """ - Get the weights list. - - Returns: - List of weights. - """ + """Get the weights list.""" return [self.tilert_weights, self.tilert_scales] def device_sharding( @@ -166,15 +158,7 @@ def device_sharding( weights_dict: dict[str, torch.Tensor], key_prefix: str, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Device sharding. - - Args: - weights_dict: Dictionary of weights. - key_prefix: Key prefix. - Returns: - Tuple of weights. - """ + """Device sharding.""" down_proj_weight_key = f"{key_prefix}.down_proj.weight" down_proj_scale_key = f"{key_prefix}.down_proj.weight_scale_inv" down_proj_weight = weights_dict[down_proj_weight_key] @@ -216,13 +200,7 @@ def init_reference_weights( key_prefix: str, device_id: int = 0, ) -> None: - """ - Initialize the reference weights. - - Args: - state_dict: State dictionary. - device_id: Device ID. - """ + """Initialize the reference weights.""" sharded_list = self.device_sharding(state_dict, key_prefix) down_weights = sharded_list[0][device_id] @@ -235,25 +213,19 @@ def init_reference_weights( self.ref_down = torch.stack(down_list, dim=0) def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Initialize the tilert weights. - - Args: - state_dict: State dictionary. - """ + """Initialize the tilert weights.""" assert self.algorithm is not None, "Algorithm is not set" + converter_algorithm = ( + DownAllReduceAlgorithm.BF16MMA + if self.algorithm == DownAllReduceAlgorithm.BF16MMA_V2 + else self.algorithm + ) self.tilert_weights, self.tilert_scales = DownAllReduceWeightsConverter( self.model_args, self.num_devices - ).dispatch(self.algorithm, [state_dict[alias] for alias in self.tensor_alias]) + ).dispatch(converter_algorithm, [state_dict[alias] for alias in self.tensor_alias]) def init_tilert_vars(self, batch_size: int, seq_len: int, device_id: int = 0) -> None: - """ - Initialize the tilert variables. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ + """Initialize the tilert variables.""" self.hidden_out = torch.zeros( (batch_size, seq_len, self.dim), dtype=torch.bfloat16, @@ -262,8 +234,10 @@ def init_tilert_vars(self, batch_size: int, seq_len: int, device_id: int = 0) -> self.profile_logs = get_profile_log_tensor(device=f"cuda:{device_id}") self.is_init = True - def init_random_weights(self, device_id: int = 0) -> None: + def init_random_weights(self, device_id: int | None = None) -> None: """Initialize the random weights.""" + if device_id is None: + device_id = self.device_id scale_dtype = torch.float32 if self.arch_name == "glm_5" else torch.bfloat16 down_weights = torch.randn( self.dim, self.inter_dim, dtype=torch.bfloat16, device=f"cuda:{device_id}" @@ -292,15 +266,7 @@ def golden_forward( self, vec_in: torch.Tensor, ) -> torch.Tensor: - """ - Forward pass for the down-project module. - - Args: - vec_in: Input vector. - - Returns: - Output tensor. - """ + """Forward pass for the down-project module.""" assert self.ref_down is not None bsz = vec_in.shape[0] assert bsz == 1 diff --git a/tilert/models/glm_5/_dsa_v32/ops/eh_proj_allreduce.py b/tilert/models/glm_5/_dsa_v32/ops/eh_proj_allreduce.py deleted file mode 100644 index fe0b71f..0000000 --- a/tilert/models/glm_5/_dsa_v32/ops/eh_proj_allreduce.py +++ /dev/null @@ -1,293 +0,0 @@ -"""EHProjAllReduce operation module.""" - -from dataclasses import dataclass -from enum import Enum - -import torch - -from tilert.models.base import TileRTModule, TilertWeightsConverter -from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.utils import get_profile_log_tensor - -__all__ = [ - "eh_proj_allreduce", - "EHProjAllReduceTilertWeightsAlias", -] - - -def eh_proj_allreduce( - vec_in_enorm: torch.Tensor, - vec_in_hnorm: torch.Tensor, - w_eh: torch.Tensor, - flag: int, - vec_out: torch.Tensor, - profile_logs: torch.Tensor, - model_arch: str, -) -> None: - """ - Fused operation of EHProj and allreduce. - - Args: - vec_in_enorm: Input tensor of shape (1, seq_len, 7168). - vec_in_hnorm: Input tensor of shape (1, seq_len, 7168). - w_eh: Input tensor of shape (7168, 1792) or (128, 7, 56, 256). - flag: Input tensor. - vec_out: Output tensor of shape (1, seq_len, 7168). - profile_logs: Profile logs tensor (1D). - model_arch: Model architecture string. - """ - compute_kernel_type = "bf16" - torch.ops.tilert.eh_proj_allreduce_op( - vec_in_enorm, - vec_in_hnorm, - w_eh, - flag, - vec_out, - profile_logs, - model_arch, - compute_kernel_type, - torch.empty(0, dtype=torch.int64, device=vec_in_enorm.device), - ) - - -class EHProjAllReduceAlgorithm(Enum): - """EHProjAllReduce algorithm""" - - GENERAL = "general" - - -class EHProjAllReduceWeightsConverter(TilertWeightsConverter): - """EHProj weights converter""" - - def convert_to_general(self, weights_list: list[torch.Tensor]) -> tuple[torch.Tensor]: - """ - Convert the weights to general format. - - Args: - weights_list: List of weights. - - Returns: - Tuple of weights. - """ - args = self.model_args - assert args.arch_name == "deepseek_v3_2" or args.arch_name == "glm_5" - dim = args.dim - num_sms = 128 - dim_per_sm = dim // num_sms - in_dim = dim * 2 - in_dim_per_device = in_dim // self.num_devices - stages = in_dim_per_device // 256 - - with torch.inference_mode(): - (proj_weights,) = weights_list - proj_weights = proj_weights.reshape(num_sms, dim_per_sm, stages, 256) - proj_weights = proj_weights.transpose(1, 2) - return (proj_weights.contiguous(),) - - -@dataclass -class EHProjAllReduceTilertWeightsAlias: - """TileRT weights alias for EHProjAllReduce.""" - - eh_proj_weights = "eh_proj_weights" - - @property - def tilert_tensor_alias(self) -> list[str]: - return [self.eh_proj_weights] - - def __call__(self) -> list[str]: - return self.tilert_tensor_alias - - -class EHProjAllReduce(TileRTModule): - """EHProjAllReduce module""" - - _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [EHProjAllReduceAlgorithm.GENERAL], - "glm_5": [EHProjAllReduceAlgorithm.GENERAL], - } - - def __init__( - self, - model_args: ModelArgs, - num_devices: int, - algorithm: EHProjAllReduceAlgorithm = EHProjAllReduceAlgorithm.GENERAL, - ): - super().__init__( - self.__class__.__name__, - model_args=model_args, - num_devices=num_devices, - ) - - self.arch_name = self.model_args.arch_name - self.dim = self.model_args.dim - - self.algorithm = algorithm - - self.ref_proj: torch.Tensor | None = None - - self.tilert_proj: torch.Tensor | None = None - - self.hidden_out: torch.Tensor | None = None - - self.profile_logs: torch.Tensor | None = None - self.is_init = False - - self.tilert_weights_alias = EHProjAllReduceTilertWeightsAlias() - - self.tensor_alias: list[str] = [ - "eh_proj_weights", - ] - - self.ref_tensor_alias: list[str] = [ - "eh_proj.weight", - ] - - @property - def tilert_tensor_alias(self) -> list[str]: - return self.tilert_weights_alias.tilert_tensor_alias - - def get_weights_list(self) -> list[torch.Tensor]: - """ - Get the weights list. - - Returns: - List of weights. - """ - return [self.tilert_proj] - - def device_sharding( - self, - weights_dict: dict[str, torch.Tensor], - key_prefix: str | None = None, - ) -> tuple[torch.Tensor]: - """ - Device sharding. - - Args: - weights_dict: Dictionary of weights. - key_prefix: Key prefix. - Returns: - Tuple of weights. - """ - eh_proj_key = "eh_proj.weight" - if key_prefix is not None: - eh_proj_key = f"{key_prefix}.eh_proj.weight" - - eh_proj_weight = weights_dict[eh_proj_key] - in_dim = eh_proj_weight.shape[1] - out_dim = eh_proj_weight.shape[0] - in_dim_per_device = in_dim // self.num_devices - eh_proj_weight = eh_proj_weight.reshape(out_dim, self.num_devices, in_dim_per_device) - eh_proj_weight = eh_proj_weight.transpose(0, 1) - return (eh_proj_weight.contiguous(),) - - def init_reference_weights( - self, - state_dict: dict[str, torch.Tensor], - key_prefix: str | None = None, - device_id: int = 0, - ) -> None: - """ - Initialize the reference weights. - - Args: - state_dict: State dictionary. - device_id: Device ID. - """ - sharded_list = self.device_sharding(state_dict, key_prefix) - - eh_proj_weight = sharded_list[0][device_id] - - self.ref_proj = eh_proj_weight - - def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Initialize the tilert weights. - - Args: - state_dict: State dictionary. - """ - assert self.algorithm is not None - (self.tilert_proj,) = EHProjAllReduceWeightsConverter( - self.model_args, self.num_devices - ).dispatch(self.algorithm, [state_dict[alias] for alias in self.tensor_alias]) - - def init_tilert_vars(self, batch_size: int, seq_len: int, device_id: int = 0) -> None: - """ - Initialize the tilert variables. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ - self.hidden_out = torch.zeros( - (batch_size, seq_len, self.dim), - dtype=torch.bfloat16, - device=f"cuda:{device_id}", - ) - self.profile_logs = get_profile_log_tensor(device=f"cuda:{device_id}") - self.is_init = True - - def init_random_weights(self, device_id: int = 0) -> None: - """Initialize the random weights.""" - proj_weights = torch.randn( - self.dim, self.dim * 2, dtype=torch.bfloat16, device=f"cuda:{device_id}" - ) - - tensor_list = [ - proj_weights, - ] - state_dict = dict(zip(self.ref_tensor_alias, tensor_list)) - - self.init_reference_weights(state_dict, None, device_id) - sharded_list = self.device_sharding(state_dict, None) - sharded_state_dict = { - alias: sharded_list[i][device_id] for i, alias in enumerate(self.tensor_alias) - } - self.init_tilert_weights(sharded_state_dict) - - def golden_forward( - self, - vec_in_enorm: torch.Tensor, - vec_in_hnorm: torch.Tensor, - device_id: int = 0, - ) -> torch.Tensor: - """ - Forward pass for the down-project module. - - Args: - vec_in_enorm: Input vector of shape (1, seq_len, 7168). - vec_in_hnorm: Input vector of shape (1, seq_len, 7168). - - Returns: - Output tensor. - """ - assert self.ref_proj is not None - bsz = vec_in_enorm.shape[0] - assert bsz == 1 - - vec_in_concat = torch.cat([vec_in_enorm, vec_in_hnorm], dim=-1) - dim_per_device = (self.dim * 2) // self.num_devices - vec_in_slice = vec_in_concat[ - ..., dim_per_device * device_id : dim_per_device * device_id + dim_per_device - ] - return vec_in_slice @ self.ref_proj.T - - def tilert_forward( - self, - vec_in_enorm: torch.Tensor, - vec_in_hnorm: torch.Tensor, - flag: int, - ) -> torch.Tensor: - assert self.hidden_out is not None - eh_proj_allreduce( - vec_in_enorm, - vec_in_hnorm, - self.tilert_proj, - flag, - self.hidden_out, - self.profile_logs, - model_arch=self.model_args.arch_name, - ) - return self.hidden_out diff --git a/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py b/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py index b0e6b24..2da6d3f 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py +++ b/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py @@ -18,7 +18,6 @@ ] -VALID_SEQ_LENS = (1, 2, 4) def expert_down_allreduce( @@ -30,25 +29,11 @@ def expert_down_allreduce( x_in: torch.Tensor, flag: int, vec_out: torch.Tensor, - profile_logs: torch.Tensor, model_arch: str, compute_kernel_type: str = "bf16", + profile_logs: torch.Tensor | None = None, ) -> None: - """ - Fused expert down + allreduce (unified for DSv32 and GLM5). - - Args: - vec_in: [1, seq_len, n_experts, 256], bfloat16. - mat_in: [n_experts, dim, 256], float8_e4m3fn. - mat_scale: [n_experts, 1024, 2], bfloat16 (DSv32) or float32 (GLM5). - indices: [1, seq_len, 8], int32. - scores: [1, seq_len, 8], float32. - x_in: [1, seq_len, dim], bfloat16. - flag: User flag. - vec_out: [1, seq_len, dim], bfloat16 (output). - profile_logs: 1D tensor for profile logs. - compute_kernel_type: "bf16". - """ + """Fused expert down + allreduce (unified for DSv32 and GLM5).""" torch.ops.tilert.expert_down_allreduce_op( vec_in, mat_in, @@ -58,16 +43,20 @@ def expert_down_allreduce( x_in, flag, vec_out, - profile_logs, model_arch, compute_kernel_type, + profile_logs, ) + + class ExpertDownAllReduceAlgorithm(Enum): """ExpertDownAllReduce algorithm.""" GENERAL = "general" + BF16MMA = "bf16mma" + GLM5_FP4_HMMA = "glm5_fp4_hmma" class ExpertDownAllReduceWeightsConverter(TilertWeightsConverter): @@ -87,6 +76,26 @@ def _swizzle_qmma_8x32(mat_in: torch.Tensor) -> torch.Tensor: pre_shape = mat_in.shape[:-2] return mat_in.reshape(*pre_shape, 8, 2, 4, 4).transpose(-2, -3).contiguous() + @staticmethod + def _swizzle_bf16mma_full_16x32(mat_in: torch.Tensor) -> torch.Tensor: + assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 32 + assert mat_in.dtype == torch.float8_e4m3fn + pre = mat_in.shape[:-2] + mat = mat_in.reshape(*pre, 2, 8, 2, 2, 4, 2) + n = len(pre) + mat = mat.permute(*range(n), 1 + n, 4 + n, 2 + n, 3 + n, 0 + n, 5 + n) + return mat.reshape(*pre, 32, 16).contiguous() + + @staticmethod + def _swizzle_bf16mma_partial_8x32(mat_in: torch.Tensor) -> torch.Tensor: + assert mat_in.shape[-2] == 8 and mat_in.shape[-1] == 32 + assert mat_in.dtype == torch.float8_e4m3fn + pre = mat_in.shape[:-2] + mat = mat_in.reshape(*pre, 8, 2, 2, 4, 2) + n = len(pre) + mat = mat.permute(*range(n), 0 + n, 3 + n, 1 + n, 2 + n, 4 + n) + return mat.reshape(*pre, 32, 8).contiguous() + def convert_to_general( self, weights_list: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: @@ -155,10 +164,126 @@ def convert_to_general( mat_scale_tilert = mat_scale_tilert.to(torch.bfloat16) return mat_in_swizzled.contiguous(), mat_scale_tilert.contiguous() + def convert_to_bf16mma( + self, weights_list: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor]: + args = self.model_args + assert args.arch_name in ( + "deepseek_v3_2", + "glm_5", + ), "BF16 MMA wired for DSv32 / GLM5 only." + dim = args.dim + num_sms = 128 + dim_per_sm = dim // num_sms + expert_dim = args.moe_inter_dim // 8 + k_chunks = expert_dim // 32 + scale_cols = expert_dim // args.block_size + n_full_tiles = dim_per_sm // 16 + remainder_rows = dim_per_sm % 16 + full_rows = n_full_tiles * 16 + + with torch.inference_mode(): + mat_in, scale_in = weights_list + exp_num = mat_in.shape[0] + mat_per_cta = mat_in.reshape(exp_num, num_sms, dim_per_sm, expert_dim) + + full_part = mat_per_cta[:, :, :full_rows, :] + full_tiles = full_part.reshape( + exp_num, num_sms, n_full_tiles, 16, k_chunks, 32 + ).transpose(3, 4) + full_swizzled = self._swizzle_bf16mma_full_16x32(full_tiles) + full_swizzled = full_swizzled.reshape( + exp_num, num_sms, n_full_tiles * k_chunks * 32 * 16 + ) + + mats = [full_swizzled] + if remainder_rows > 0: + partial_part = mat_per_cta[:, :, full_rows:, :] + partial_tiles = partial_part.reshape( + exp_num, num_sms, 1, remainder_rows, k_chunks, 32 + ).transpose(3, 4) + partial_swizzled = self._swizzle_bf16mma_partial_8x32(partial_tiles) + partial_swizzled = partial_swizzled.reshape( + exp_num, num_sms, k_chunks * 32 * remainder_rows + ) + mats.append(partial_swizzled) + + mat_swizzled = torch.cat(mats, dim=2) if len(mats) > 1 else mats[0] + mat_swizzled = mat_swizzled.reshape(exp_num, dim, expert_dim) + + mat_scale_tilert = ( + scale_in.reshape(exp_num, dim // args.block_size, 1, scale_cols) + .repeat(1, 1, 16, 1) + .reshape(exp_num, num_sms, -1) + ) + target_cols_per_sm = 1024 * scale_cols // num_sms + pad_amount = target_cols_per_sm - mat_scale_tilert.shape[-1] + if pad_amount > 0: + padding_zeros = torch.zeros( + (exp_num, num_sms, pad_amount), + dtype=scale_in.dtype, + device=scale_in.device, + ) + mat_scale_tilert = torch.cat([mat_scale_tilert, padding_zeros], dim=2) + mat_scale_tilert = mat_scale_tilert.reshape(exp_num, 1024, scale_cols) + if args.arch_name == "glm_5": + mat_scale_tilert = mat_scale_tilert.to(torch.float32) + else: + mat_scale_tilert = mat_scale_tilert.to(torch.bfloat16) + + return mat_swizzled.contiguous(), mat_scale_tilert.contiguous() + + def convert_to_glm5_fp4_hmma( + self, weights_list: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor]: + from tilert.models.common_mxfp4 import ( + _unpack_fp4_nibbles_last, + build_down_weights_mma_natural, + ) + + assert ( + len(weights_list) == 2 + ), f"convert_to_glm5_fp4_hmma expects 2 tensors, got {len(weights_list)}" + down_fp4, down_e8m0 = weights_list + arch = self.model_args.arch_name + assert arch == "glm_5", f"GLM5_FP4_HMMA down converter is GLM5-only, got arch={arch}" + + dim = self.model_args.dim + moe_inter_pd = self.model_args.moe_inter_dim // self.num_devices + + with torch.inference_mode(): + if down_fp4.shape[-1] == moe_inter_pd: + down_nib = down_fp4.to(torch.uint8).contiguous() + else: + assert down_fp4.shape[-1] == moe_inter_pd // 2, ( + "routed fp4 down last dim must be inter_pd or inter_pd/2; " + f"got {down_fp4.shape[-1]} (inter_pd={moe_inter_pd})" + ) + down_nib = _unpack_fp4_nibbles_last(down_fp4) + down_e8 = down_e8m0.to(torch.uint8).contiguous() + + n_routed = down_nib.shape[0] + assert down_nib.shape == (n_routed, dim, moe_inter_pd), ( + f"down_fp4 must be (n_routed, {dim}, {moe_inter_pd}); " + f"got {tuple(down_nib.shape)}" + ) + + device = down_nib.device + e_total = n_routed + 1 + u8 = {"dtype": torch.uint8, "device": device} + full_nib = torch.zeros(e_total, dim, moe_inter_pd, **u8) + full_e8 = torch.zeros(e_total, dim, moe_inter_pd // 32, **u8) + full_nib[1:] = down_nib + full_e8[1:] = down_e8 + down_packed = build_down_weights_mma_natural(full_nib, full_e8, dim, moe_inter_pd) + dummy = torch.zeros(1, dtype=torch.float32, device=device) + return down_packed, dummy + + + @dataclass class ExpertDownAllReduceTilertWeightsAlias: - """TileRT weights alias for ExpertDownAllReduce.""" exp_down_weights = "exp_down_weights" exp_down_scales = "exp_down_scales" @@ -167,6 +292,9 @@ class ExpertDownAllReduceTilertWeightsAlias: def tilert_tensor_alias(self) -> list[str]: return [self.exp_down_weights, self.exp_down_scales] + def glm5_fp4_tilert_tensor_alias(self) -> list[str]: + return [self.exp_down_weights, self.exp_down_scales] + def __call__(self) -> list[str]: return self.tilert_tensor_alias @@ -175,8 +303,15 @@ class ExpertDownAllReduce(TileRTModule): """ExpertDownAllReduce module.""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [ExpertDownAllReduceAlgorithm.GENERAL], - "glm_5": [ExpertDownAllReduceAlgorithm.GENERAL], + "deepseek_v3_2": [ + ExpertDownAllReduceAlgorithm.GENERAL, + ExpertDownAllReduceAlgorithm.BF16MMA, + ], + "glm_5": [ + ExpertDownAllReduceAlgorithm.GENERAL, + ExpertDownAllReduceAlgorithm.BF16MMA, + ExpertDownAllReduceAlgorithm.GLM5_FP4_HMMA, + ], } def __init__( @@ -210,6 +345,8 @@ def __init__( if self.arch_name in ("deepseek_v3_2", "glm_5"): self.compute_kernel_type = "bf16" + if algorithm == ExpertDownAllReduceAlgorithm.BF16MMA: + self.compute_kernel_type = "bf16mma" else: raise ValueError(f"Unsupported architecture: {self.arch_name}") @@ -228,6 +365,13 @@ def __init__( def tilert_tensor_alias(self) -> list[str]: return self.tilert_weights_alias.tilert_tensor_alias + def set_algorithm(self, algorithm: Enum) -> None: + super().set_algorithm(algorithm) + if algorithm == ExpertDownAllReduceAlgorithm.BF16MMA: + self.compute_kernel_type = "bf16mma" + elif algorithm == ExpertDownAllReduceAlgorithm.GENERAL: + self.compute_kernel_type = "bf16" + def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_weights, self.tilert_scales] @@ -261,11 +405,50 @@ def process_down_weights( ) return down_proj_weight, down_proj_scale + @staticmethod + def _split_last_axis(t: torch.Tensor, num_devices: int) -> torch.Tensor: + d, inter = t.shape[-2], t.shape[-1] + assert ( + inter % num_devices == 0 + ), f"down last-axis {inter} not divisible by num_devices {num_devices}" + return ( + t.reshape(d, num_devices, inter // num_devices) + .transpose(0, 1) + .reshape(num_devices, 1, d, inter // num_devices) + ) + + def process_down_weights_fp4( + self, + key_prefix: str, + weights_hf: dict[str, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + n_dev = self.num_devices + down_w = weights_hf[f"{key_prefix}.down_proj.weight"] + down_s = weights_hf[f"{key_prefix}.down_proj.weight_scale"] + return self._split_last_axis(down_w, n_dev), self._split_last_axis(down_s, n_dev) + + def device_sharding_fp4( + self, + weights_dict: dict[str, torch.Tensor], + key_prefix: str, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Shard routed-only down weight + scale across devices.""" + dw, ds = [], [] + for exp_id in range(self.n_routed_experts): + down_weights, down_scales = self.process_down_weights_fp4( + f"{key_prefix}.experts.{exp_id}", weights_dict + ) + dw.append(down_weights) + ds.append(down_scales) + return torch.cat(dw, dim=1).contiguous(), torch.cat(ds, dim=1).contiguous() + def device_sharding( self, weights_dict: dict[str, torch.Tensor], key_prefix: str, ) -> tuple[torch.Tensor, torch.Tensor]: + if self.algorithm == ExpertDownAllReduceAlgorithm.GLM5_FP4_HMMA: + return self.device_sharding_fp4(weights_dict, key_prefix) assert self.n_shared_experts == 1, "Only one shared expert is supported" down_weights_list = [] down_scales_list = [] @@ -300,13 +483,31 @@ def init_reference_weights( weight_dequant(down_weight, down_scale) for down_weight, down_scale in zip(down_weights, down_scales) ] - self.ref_down = torch.stack(down_list, dim=0) + self.ref_down = torch.stack([t.to(torch.bfloat16) for t in down_list], dim=0) + + def get_tilert_weights_alias(self) -> list[str]: + """Return the alias list keyed into ``state_dict`` for this op.""" + return list(self.tilert_weights_alias()) def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: assert self.algorithm is not None, "Algorithm is not set" - self.tilert_weights, self.tilert_scales = ExpertDownAllReduceWeightsConverter( - self.model_args, self.num_devices - ).dispatch(self.algorithm, [state_dict[alias] for alias in self.tensor_alias]) + if self.algorithm == ExpertDownAllReduceAlgorithm.GLM5_FP4_HMMA: + assert ( + self.arch_name == "glm_5" + ), f"GLM5_FP4_HMMA is GLM5-only, got arch={self.arch_name}" + converter = ExpertDownAllReduceWeightsConverter(self.model_args, self.num_devices) + self.tilert_weights, self.tilert_scales = converter.convert_to_glm5_fp4_hmma( + [state_dict[alias] for alias in self.tensor_alias] + ) + self.is_tilert_weights_init = True + return + aliases = [state_dict[alias] for alias in self.tensor_alias] + self.tilert_weights, self.tilert_scales = ( + torch.ops.tilert.expert_down_allreduce__convert_weights( + aliases, self.model_arch, self.compute_kernel_type + ) + ) + self.is_tilert_weights_init = True def init_tilert_vars(self, batch_size: int, seq_len: int, device_id: int = 0) -> None: self.hidden_out = torch.zeros( @@ -317,7 +518,9 @@ def init_tilert_vars(self, batch_size: int, seq_len: int, device_id: int = 0) -> self.profile_logs = get_profile_log_tensor(device=f"cuda:{device_id}") self.is_init = True - def init_random_weights(self, device_id: int = 0) -> None: + def init_random_weights(self, device_id: int | None = None) -> None: + if device_id is None: + device_id = self.device_id n = self.n_routed_experts + 1 dev = f"cuda:{device_id}" down_weights = list( @@ -389,9 +592,9 @@ def tilert_forward( x_in, flag, self.hidden_out, - self.profile_logs, self.model_arch, self.compute_kernel_type, + self.profile_logs, ) return self.hidden_out diff --git a/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py b/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py index e2d96eb..fa1ab6e 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py +++ b/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py @@ -4,6 +4,7 @@ from enum import Enum import numpy as np + import torch import torch.nn.functional as F @@ -29,10 +30,14 @@ def expert_select_up_gate_silu( hidden_out: torch.Tensor, expert_probs_out: torch.Tensor, expert_indices_out: torch.Tensor, - profile_logs: torch.Tensor, + profile_logs: torch.Tensor | None = None, algorithm: str = "fp8mma", *, model_arch: str, + tid2eid: torch.Tensor | None = None, + token_id: torch.Tensor | None = None, + experts_weights_32row: torch.Tensor | None = None, + shared_experts_weights: torch.Tensor | None = None, ) -> None: """Expert SelectUpGateSiLU operation.""" torch.ops.tilert.expert_select_up_gate_silu_op( @@ -46,6 +51,10 @@ def expert_select_up_gate_silu( profile_logs, model_arch, algorithm, + tid2eid, + token_id, + experts_weights_32row, + shared_experts_weights, ) @@ -71,6 +80,16 @@ def ref_tensor_alias(self) -> list[str]: + [f"{self.key_prefix}.experts.{i}.up_proj.weight_scale_inv" for i in range(n)] ) + def fp4_routed_tensor_alias(self) -> list[str]: + n = self.n_routed_experts + return ( + [f"{self.key_prefix}.gate.e_score_correction_bias"] + + [f"{self.key_prefix}.experts.{i}.gate_proj.weight" for i in range(n)] + + [f"{self.key_prefix}.experts.{i}.up_proj.weight" for i in range(n)] + + [f"{self.key_prefix}.experts.{i}.gate_proj.weight_scale" for i in range(n)] + + [f"{self.key_prefix}.experts.{i}.up_proj.weight_scale" for i in range(n)] + ) + def __call__(self) -> list[str]: return self.ref_tensor_alias @@ -104,6 +123,8 @@ class ExpertSelectUpGateSiLUAlgorithm(Enum): FP8MMA = "fp8mma" FP16MMA = "fp16mma" + BF16MMA = "bf16mma" + GLM5_FP4_HMMA = "glm5_fp4_hmma" class ExpertSelectUpGateSiLUWeightsConverter(TilertWeightsConverter): @@ -135,16 +156,6 @@ def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: def tilert_to_tilert_144sm( mat_in: torch.Tensor, mat_scale_in: torch.Tensor, mma_type: str | None = None ) -> torch.Tensor: - """ - Convert tilert weights and scales to tilert_144sm input format. - - Args: - mat_in: tilert weights - mat_scale_in: tilert scales - mma_type: MMA type, None,"16x32" or "16x16" - Returns: - tilert_144sm weights and scales - """ exp_num = mat_in.shape[0] assert mat_in.shape == (exp_num, 512, 7168) assert mat_scale_in.shape == (exp_num, 4, 64) @@ -198,15 +209,6 @@ def tilert_to_tilert_144sm( def tilert_to_tilert_144sm_mma( mat_in: torch.Tensor, mat_scale_in: torch.Tensor, mma_type: str = "16x32" ) -> torch.Tensor: - """ - Convert tilert weights and scales to tilert_144sm_mma input format. - - Args: - mat_in: tilert weights - mat_scale_in: tilert scales - Returns: - tilert_144sm weights and scales - """ return ExpertSelectUpGateSiLUWeightsConverter.tilert_to_tilert_144sm( mat_in, mat_scale_in, mma_type ) @@ -303,31 +305,73 @@ def convert_to_mma( def convert_to_fp8mma( self, weights_list: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Convert the weights to fp8mma format. - - Args: - weights: List of weights. - - Returns: - Tuple of weights. - """ return self.convert_to_mma(weights_list, "fp8mma") def convert_to_fp16mma( self, weights_list: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Convert the weights to fp16mma format. - - Args: - weights: List of weights. + return self.convert_to_mma(weights_list, "fp16mma") - Returns: - Tuple of weights. - """ + def convert_to_bf16mma( + self, weights_list: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor]: return self.convert_to_mma(weights_list, "fp16mma") + def convert_to_glm5_fp4_hmma( + self, weights_list: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor]: + from tilert.models.common_mxfp4 import ( + _unpack_fp4_nibbles_last, + build_ug_weights_mma_natural, + ) + + assert ( + len(weights_list) == 5 + ), f"convert_to_glm5_fp4_hmma expects 5 tensors, got {len(weights_list)}" + bias, gate_fp4, gate_e8m0, up_fp4, up_e8m0 = weights_list + arch = self.model_args.arch_name + assert arch == "glm_5", f"GLM5_FP4_HMMA converter is GLM5-only, got arch={arch}" + + dim = self.model_args.dim + moe_inter_pd = self.model_args.moe_inter_dim // self.num_devices + + with torch.inference_mode(): + def _ensure_unpacked(t: torch.Tensor) -> torch.Tensor: + if t.shape[-1] == dim: + return t.to(torch.uint8).contiguous() + assert t.shape[-1] == dim // 2, ( + "routed fp4 weight last dim must be dim or dim/2; " + f"got {t.shape[-1]} (dim={dim})" + ) + return _unpack_fp4_nibbles_last(t) + + gate_nib = _ensure_unpacked(gate_fp4) + up_nib = _ensure_unpacked(up_fp4) + gate_e8 = gate_e8m0.to(torch.uint8).contiguous() + up_e8 = up_e8m0.to(torch.uint8).contiguous() + + n_routed = gate_nib.shape[0] + assert gate_nib.shape == (n_routed, moe_inter_pd, dim), ( + f"gate_fp4 must be (n_routed, {moe_inter_pd}, {dim}); " + f"got {tuple(gate_nib.shape)}" + ) + + device = gate_nib.device + e_total = n_routed + 1 + u8 = {"dtype": torch.uint8, "device": device} + + def _slot0(nib: torch.Tensor, e8: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + full_nib = torch.zeros(e_total, moe_inter_pd, dim, **u8) + full_e8 = torch.zeros(e_total, moe_inter_pd, dim // 32, **u8) + full_nib[1:] = nib + full_e8[1:] = e8 + return full_nib, full_e8 + + g_nib, g_e8 = _slot0(gate_nib, gate_e8) + u_nib, u_e8 = _slot0(up_nib, up_e8) + ug_packed = build_ug_weights_mma_natural(g_nib, g_e8, u_nib, u_e8, dim, moe_inter_pd) + return bias.float().contiguous(), ug_packed + class ExpertSelectUpGateSiLU(TileRTModule): """ExpertSelectUpGateSiLU module""" @@ -336,10 +380,12 @@ class ExpertSelectUpGateSiLU(TileRTModule): "deepseek_v3_2": [ ExpertSelectUpGateSiLUAlgorithm.FP8MMA, ExpertSelectUpGateSiLUAlgorithm.FP16MMA, + ExpertSelectUpGateSiLUAlgorithm.BF16MMA, ], "glm_5": [ ExpertSelectUpGateSiLUAlgorithm.FP8MMA, ExpertSelectUpGateSiLUAlgorithm.FP16MMA, + ExpertSelectUpGateSiLUAlgorithm.GLM5_FP4_HMMA, ], } @@ -391,7 +437,11 @@ def __init__( self.tilert_bias: torch.Tensor | None = None self.tilert_weights: torch.Tensor | None = None - self.tilert_scales = torch.zeros(1, dtype=torch.bfloat16, device=torch.device("cuda")) + self.tilert_scales = ( + torch.zeros(1, dtype=torch.bfloat16, device=torch.device("cuda")) + if torch.cuda.is_available() + else None + ) self.hidden_out: torch.Tensor | None = None self.expert_probs: torch.Tensor | None = None @@ -417,12 +467,7 @@ def tilert_tensor_alias(self) -> list[str]: return self._tilert_tensor_alias def get_weights_list(self) -> list[torch.Tensor]: - """ - Get the weights list. - - Returns: - List of weights. - """ + """Get the weights list.""" return [self.tilert_bias, self.tilert_weights, self.tilert_scales] @staticmethod @@ -454,16 +499,59 @@ def process_gate_up_weights( up_proj_scale = up_proj_scale.reshape(num_devices, 1, in_scale_dim_per_device, scale_dim) return gate_proj_weight, gate_proj_scale, up_proj_weight, up_proj_scale - def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding: ref state dict -> tilert sharded tensors (num_devices, ...). + @staticmethod + def _split_inter_axis(t: torch.Tensor, num_devices: int) -> torch.Tensor: + inter, k = t.shape[-2], t.shape[-1] + assert ( + inter % num_devices == 0 + ), f"moe-inter {inter} not divisible by num_devices {num_devices}" + return t.reshape(num_devices, 1, inter // num_devices, k) + + def process_gate_up_weights_fp4( + self, + key_prefix: str, + weights_hf: dict[str, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + n_dev = self.num_devices + gate_w = weights_hf[f"{key_prefix}.gate_proj.weight"] + gate_s = weights_hf[f"{key_prefix}.gate_proj.weight_scale"] + up_w = weights_hf[f"{key_prefix}.up_proj.weight"] + up_s = weights_hf[f"{key_prefix}.up_proj.weight_scale"] + return ( + self._split_inter_axis(gate_w, n_dev), + self._split_inter_axis(gate_s, n_dev), + self._split_inter_axis(up_w, n_dev), + self._split_inter_axis(up_s, n_dev), + ) + + def _device_sharding_fp4(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + ref_alias = self.ref_weights_alias + key_prefix = ref_alias.key_prefix + bias = weights_map[f"{key_prefix}.gate.e_score_correction_bias"] + bias = bias[None, :].repeat(self.num_devices, 1) - Args: - weights_map: State dict keyed by ref_weights_alias(). + gw, gs, uw, us = [], [], [], [] + for exp_id in range(self.n_routed_experts): + g_w, g_s, u_w, u_s = self.process_gate_up_weights_fp4( + f"{key_prefix}.experts.{exp_id}", weights_map + ) + gw.append(g_w) + gs.append(g_s) + uw.append(u_w) + us.append(u_s) + tilert_alias = self.tilert_weights_alias + return { + tilert_alias.exp_bias: bias, + tilert_alias.exp_gate_weights: torch.cat(gw, dim=1), + tilert_alias.exp_gate_scales: torch.cat(gs, dim=1), + tilert_alias.exp_up_weights: torch.cat(uw, dim=1), + tilert_alias.exp_up_scales: torch.cat(us, dim=1), + } + + def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + if self.algorithm == ExpertSelectUpGateSiLUAlgorithm.GLM5_FP4_HMMA: + return self._device_sharding_fp4(weights_map) - Returns: - Dict keyed by tilert_weights_alias() with (num_devices, ...) tensors. - """ ref_alias = self.ref_weights_alias key_prefix = ref_alias.key_prefix @@ -513,13 +601,6 @@ def init_reference_weights( state_dict: dict[str, torch.Tensor], device_id: int | None = None, ) -> None: - """ - Initialize the reference weights. - - Args: - state_dict: State dict keyed by ref_weights_alias(). - device_id: Device ID; defaults to self.device_id. - """ did = self.device_id if device_id is None else device_id sharded = self.device_sharding(state_dict) @@ -537,24 +618,36 @@ def init_reference_weights( ref_up_list = [ weight_dequant(up_weights[i], up_scales[i]) for i in range(up_weights.shape[0]) ] - self.ref_gate = torch.stack(ref_gate_list, dim=0) - self.ref_up = torch.stack(ref_up_list, dim=0) + self.ref_gate = torch.stack([t.to(torch.bfloat16) for t in ref_gate_list], dim=0) + self.ref_up = torch.stack([t.to(torch.bfloat16) for t in ref_up_list], dim=0) + + def get_tilert_weights_alias(self) -> list[str]: + return list(self.tilert_weights_alias()) def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize the tilert weights.""" assert self.algorithm is not None, "Algorithm is not set" + if self.algorithm == ExpertSelectUpGateSiLUAlgorithm.GLM5_FP4_HMMA: + assert ( + self.arch_name == "glm_5" + ), f"GLM5_FP4_HMMA is GLM5-only, got arch={self.arch_name}" + a = self.tilert_weights_alias + converter = ExpertSelectUpGateSiLUWeightsConverter(self.model_args, self.num_devices) + self.tilert_bias, self.tilert_weights = converter.convert_to_glm5_fp4_hmma( + [ + state_dict[a.exp_bias], + state_dict[a.exp_gate_weights], + state_dict[a.exp_gate_scales], + state_dict[a.exp_up_weights], + state_dict[a.exp_up_scales], + ] + ) + return + weights_list = [state_dict[alias] for alias in self.tilert_weights_alias()] converter = ExpertSelectUpGateSiLUWeightsConverter(self.model_args, self.num_devices) self.tilert_bias, self.tilert_weights = converter.dispatch(self.algorithm, weights_list) def init_tilert_vars(self, batch_size: int, seq_len: int, device: str = "cuda") -> None: - """ - Initialize the tilert variables. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ self.hidden_out = torch.zeros( ( batch_size, @@ -580,12 +673,6 @@ def init_tilert_vars(self, batch_size: int, seq_len: int, device: str = "cuda") self.is_init = True def init_random_weights(self, device: str = "cuda") -> None: - """ - Initialize the random weights. - - Returns: - None - """ n = self.n_routed_experts + 1 bias = torch.randn(self.n_routed_experts, dtype=torch.float32, device=device) gate_weights = list( @@ -705,6 +792,8 @@ def tilert_forward( self, x_in: torch.Tensor, scores: torch.Tensor, + tid2eid: torch.Tensor | None = None, + token_id: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: assert self.algorithm is not None, "Algorithm is not set" expert_select_up_gate_silu( @@ -718,5 +807,7 @@ def tilert_forward( self.profile_logs, self.algorithm.value, model_arch=self.model_args.arch_name, + tid2eid=tid2eid, + token_id=token_id, ) return self.hidden_out, self.expert_probs, self.expert_indices diff --git a/tilert/models/glm_5/_dsa_v32/ops/flash_sparse_mla.py b/tilert/models/glm_5/_dsa_v32/ops/flash_sparse_mla.py deleted file mode 100644 index 1d4cc00..0000000 --- a/tilert/models/glm_5/_dsa_v32/ops/flash_sparse_mla.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Flash Sparse MLA operation module.""" - -import math -from enum import Enum - -import torch - -from tilert.models.base import TileRTModule -from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.utils import get_profile_log_tensor - -__all__ = [ - "flash_sparse_mla", - "FlashSparseMLACombine", -] - - -def flash_sparse_mla( - query: torch.Tensor, - query_pe: torch.Tensor, - key_value: torch.Tensor, - key_pe: torch.Tensor, - indices: torch.Tensor, - cur_pos: torch.Tensor, - output: torch.Tensor, - profile_logs: torch.Tensor, - split_size: int = 64, - compute_kernel_type: str = "bf16mma", - *, - model_arch: str, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Flash Sparse MLA operation for GLM5. - - Args: - query: Query tensor. (bs, seqlen, heads, dim) - query_pe: Query position embedding tensor. (bs, seqlen, heads, pe_dim) - key_value: Key-value tensor. (bs, seqlen_kv, dim) - key_pe: Key position embedding tensor. (bs, seqlen_kv, pe_dim) - indices: Indices tensor. (bs, seqlen, topk) - cur_pos: cur_pos tensor. (1) - output: Output tensor. - profile_logs: Profile logs tensor. - split_size: Number of splits. - """ - batch, seqlen, heads, hidden_dim = query.shape - if split_size != 64: - raise ValueError( - "The current implementation of flash_sparse_mla_op only supports split_size=64" - ) - if batch != 1: - raise ValueError("The current implementation of flash_sparse_mla_op only supports batch=1") - if seqlen > 4: - raise ValueError( - "The current implementation of flash_sparse_mla_op only supports seqlen<=4" - ) - - seqlen_kv = key_value.shape[1] - index_len = indices.shape[-1] - if index_len > seqlen_kv: - raise ValueError("index_len must be less than or equal to seqlen_kv") - - device = query.device - acc_type = torch.float32 - - dim = key_value.shape[-1] - max_num_splits = 32 - - lse = torch.empty((batch, seqlen, heads), device=device, dtype=acc_type) - lse_acc = torch.empty((batch, seqlen, heads, max_num_splits), device=device, dtype=acc_type) - output_acc = torch.empty( - batch, seqlen, heads, max_num_splits, dim, device=device, dtype=acc_type - ) - - if heads not in (8, 10, 16, 20): - raise ValueError(f"Unsupported heads: {heads}") - torch.ops.tilert.flash_sparse_mla_op( - query, - query_pe, - key_value, - key_pe, - indices, - cur_pos, - output, - output_acc, - lse, - lse_acc, - split_size, - model_arch, - compute_kernel_type, - profile_logs, - torch.empty(0, dtype=torch.int64, device=query.device), - ) - return lse, lse_acc, output_acc - - -class FlashSparseMLACombineAlgorithm(Enum): - """FlashSparseMLACombine algorithm.""" - - BF16MMA = "bf16mma" - - -class FlashSparseMLACombine(TileRTModule): - """Flash Sparse MLA combine module; no weights, uses model_args for scale and config.""" - - _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [FlashSparseMLACombineAlgorithm.BF16MMA], - "glm_5": [FlashSparseMLACombineAlgorithm.BF16MMA], - } - - def __init__( - self, - model_args: ModelArgs, - num_devices: int, - layer_idx: int = 0, - ): - super().__init__( - type(self).__name__, - model_args=model_args, - num_devices=num_devices, - layer_idx=layer_idx, - ) - self.tilert_tensor_alias: list[str] = [] - self.ref_tensor_alias: list[str] = [] - - scale = (model_args.qk_nope_head_dim + model_args.qk_rope_head_dim) ** -0.5 - if model_args.rope_factor is None: - mscale = 1.0 - else: - mscale = 0.1 * math.log(model_args.rope_factor) + 1.0 - self.softmax_scale = scale * mscale * mscale - - self.profile_logs = get_profile_log_tensor() - - def init_reference_weights( - self, state_dict: dict[str, torch.Tensor], device_id: int = 0 - ) -> None: - del state_dict, device_id - self.is_ref_weights_init = True - - def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - del state_dict - self.is_tilert_weights_init = True - - def init_random_weights(self) -> None: - self.is_ref_weights_init = True - self.is_tilert_weights_init = True - - def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - del batch_size, seq_len - self.profile_logs = get_profile_log_tensor() - self.is_var_init = True - - def golden_forward( - self, - q_nope: torch.Tensor, - q_pe: torch.Tensor, - kv_cache: torch.Tensor, - pe_cache: torch.Tensor, - topk_indices: torch.Tensor, - cur_pos: torch.Tensor, - ) -> torch.Tensor: - """Flash Sparse MLA golden version. - - Args: - q_nope: Query tensor. (bs, seqlen, heads, dim) - q_pe: Query position embedding tensor. (bs, seqlen, heads, pe_dim) - kv_cache: Key-value tensor. (bs, seqlen_kv, dim) - pe_cache: Key position embedding tensor. (bs, seqlen_kv, pe_dim) - topk_indices: Indices tensor. (bs, seqlen, topk) - cur_pos: cur_pos tensor. (1) - """ - batch_size = q_nope.shape[0] - seqlen = q_nope.shape[1] - seqlen_kv = kv_cache.shape[1] - - start_pos = int(cur_pos.item()) - mask = ( - torch.full((seqlen, seqlen_kv), float("-inf")).triu_(start_pos + 1) - if seqlen > 1 - else None - ) - - scores = ( - torch.einsum("bshc,btc->bsht", q_nope.float(), kv_cache.float()) - + torch.einsum("bshr,btr->bsht", q_pe.float(), pe_cache.float()) - ) * self.softmax_scale - index_mask = torch.full( - (batch_size, seqlen, seqlen_kv), float("-inf"), device=q_nope.device - ).scatter_(-1, topk_indices, 0) - if mask is not None: - index_mask += mask - - scores += index_mask.unsqueeze(2) - scores = scores.softmax(dim=-1, dtype=torch.float32) - return torch.einsum("bsht,btc->bshc", scores.to(torch.bfloat16), kv_cache) - - def tilert_forward( - self, - q_nope: torch.Tensor, - q_pe: torch.Tensor, - kv_cache: torch.Tensor, - pe_cache: torch.Tensor, - topk_indices: torch.Tensor, - cur_pos: torch.Tensor, - ) -> torch.Tensor: - """Flash Sparse MLA tilert version. - - Args: - q_nope: Query tensor. (bs, seqlen, heads, dim) - q_pe: Query position embedding tensor. (bs, seqlen, heads, pe_dim) - kv_cache: Key-value tensor. (bs, seqlen_kv, dim) - pe_cache: Key position embedding tensor. (bs, seqlen_kv, pe_dim) - topk_indices: Indices tensor. (bs, seqlen, topk) - cur_pos: cur_pos tensor. (1) - """ - batch_size, seqlen, heads, dim = q_nope.shape - v_dim = kv_cache.shape[-1] - - topk_indices = topk_indices.to(torch.int32) - topk_indices = topk_indices[..., : kv_cache.shape[1]] - device = q_nope.device - if any(t.device != device for t in (q_pe, kv_cache, pe_cache, topk_indices, cur_pos)): - raise RuntimeError( - "flash_sparse_mla inputs must be on the same device: " - f"q_nope={device}, q_pe={q_pe.device}, kv_cache={kv_cache.device}, " - f"pe_cache={pe_cache.device}, topk_indices={topk_indices.device}, " - f"cur_pos={cur_pos.device}" - ) - if self.profile_logs is not None and self.profile_logs.device != device: - self.profile_logs = get_profile_log_tensor(device_index=device.index, device=device) - output = torch.zeros( - (batch_size, seqlen, heads, v_dim), dtype=torch.bfloat16, device=device - ) - flash_sparse_mla( - q_nope, - q_pe, - kv_cache, - pe_cache, - topk_indices, - cur_pos, - output, - self.profile_logs, - model_arch=self.model_args.arch_name, - ) - return output - - def to_tilert_weights(self) -> None: - raise NotImplementedError("to_tilert_weights not implemented") - - def __call__( - self, - q_nope: torch.Tensor, - q_pe: torch.Tensor, - kv_cache: torch.Tensor, - pe_cache: torch.Tensor, - topk_indices: torch.Tensor, - cur_pos: torch.Tensor, - ) -> torch.Tensor: - if self.flag_enable_tilert: - return self.tilert_forward(q_nope, q_pe, kv_cache, pe_cache, topk_indices, cur_pos) - return self.golden_forward(q_nope, q_pe, kv_cache, pe_cache, topk_indices, cur_pos) diff --git a/tilert/models/glm_5/_dsa_v32/ops/head_proj_w16a16_hmma.py b/tilert/models/glm_5/_dsa_v32/ops/head_proj_w16a16_hmma.py new file mode 100644 index 0000000..a8a2cc9 --- /dev/null +++ b/tilert/models/glm_5/_dsa_v32/ops/head_proj_w16a16_hmma.py @@ -0,0 +1,47 @@ +"""HeadProj BF16-MMA operation for DeepSeek-V3.2 / GLM5.""" + +from __future__ import annotations + +import torch + +__all__ = [ + "head_proj_w16a16_hmma", + "swizzle_head_proj_weight_bf16mma", +] + + +def head_proj_w16a16_hmma( + hidden_in: torch.Tensor, + weight_in: torch.Tensor, + logits_out: torch.Tensor, + profile_logs: torch.Tensor, + model_arch: str, + compute_kernel_type: str = "w16a16_hmma", +) -> None: + torch.ops.tilert.head_proj_op( + hidden_in, + weight_in, + logits_out, + model_arch, + compute_kernel_type, + profile_logs, + ) + + +def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: + assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 16 + pre = mat_in.shape[:-2] + x = mat_in.reshape(*pre, 2, 8, 2, 4, 2).transpose(-4, -3).transpose(-5, -4) + return x.reshape(*pre, 2 * 2, 8 * 4, 2).transpose(-3, -2) + + +def swizzle_head_proj_weight_bf16mma(weight: torch.Tensor) -> torch.Tensor: + n, k = weight.shape + assert n % 16 == 0 and k % 1024 == 0, "head_proj weight must be /16 in N and /1024 in K" + n_tiles = n // 16 + k_pages = k // 1024 + k_inner = 1024 // 16 + w = weight.reshape(n_tiles, 16, k_pages, k_inner, 16) + w = w.permute(0, 2, 3, 1, 4).contiguous() + w = _swizzle_mma_16x16(w) + return w.contiguous() diff --git a/tilert/models/glm_5/_dsa_v32/ops/layernorm_rope_rotate.py b/tilert/models/glm_5/_dsa_v32/ops/layernorm_rope_rotate.py index 4fc8c0d..00a3bc9 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/layernorm_rope_rotate.py +++ b/tilert/models/glm_5/_dsa_v32/ops/layernorm_rope_rotate.py @@ -31,23 +31,6 @@ def layernorm_rope_rotate( model_arch: str, compute_kernel_type: str = "general", ) -> None: - """ - Layernorm_rope_rotate operation. - - Layernorm_rope_rotate the input tensor `input_raw` and stores the result in `k_cache_raw`. - - Args: - input_raw (torch.Tensor): The input tensor. - cur_pos (torch.Tensor): The current position tensor. - k_cache_raw (torch.Tensor): The output tensor where the result will be stored. - weight (torch.Tensor): The weight tensor. - bias (torch.Tensor): The bias tensor. - freqs_cis (torch.Tensor): The frequency tensor. - profile_logs (torch.Tensor): Tensor for storing profiling logs. - - Returns: - None - """ if input_raw.dtype != torch.bfloat16: raise ValueError("input must be a bfloat16 tensor.") if cur_pos.dtype != torch.int32: @@ -162,15 +145,6 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_weight, self.tilert_bias] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding: replicate weight and bias for each device. - - Args: - weights_map: Map from ref weight alias to tensor. - - Returns: - Map from tilert weight alias to (num_devices, ...) tensors. - """ k_weight = weights_map[self.ref_weights_alias.k_weight][None, ...].repeat( self.num_devices, 1 ) @@ -215,7 +189,7 @@ def golden_forward(self, idx_k: torch.Tensor, freqs_cis: torch.Tensor) -> torch. k_pe, k_nope = torch.split( k, [self.rope_head_dim, self.head_dim - self.rope_head_dim], dim=-1 ) - k_pe = apply_rotary_emb(k_pe.unsqueeze(2), freqs_cis).squeeze(2) + k_pe = apply_rotary_emb(k_pe.unsqueeze(2), freqs_cis, interleaved=False).squeeze(2) k = torch.cat([k_pe, k_nope], dim=-1) return rotate_activation(k) diff --git a/tilert/models/glm_5/_dsa_v32/ops/padded_allreduce_add.py b/tilert/models/glm_5/_dsa_v32/ops/padded_allreduce_add.py deleted file mode 100644 index a6490c9..0000000 --- a/tilert/models/glm_5/_dsa_v32/ops/padded_allreduce_add.py +++ /dev/null @@ -1,147 +0,0 @@ -"""PaddedAllReduceAdd operation module.""" - -from enum import Enum - -import torch - -from tilert.models.base import TileRTModule -from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.utils import get_profile_log_tensor - -__all__ = [ - "padded_allreduce_add", - "PaddedAllReduceAdd", -] - - -def padded_allreduce_add( - partial_buf: torch.Tensor, - x_in: torch.Tensor, - flag: int, - vec_out: torch.Tensor, - profile_logs: torch.Tensor, - model_arch: str, - compute_kernel_type: str = "bf16", -) -> None: - """Padded AllReduce + residual add for Device Group A (GPU 0). - - GPU 0 contributes zeros to the 8-GPU AllReduce, then adds the residual. - - Args: - partial_buf: Zero-filled partial buffer [1, L, hidden_dim] bf16. - x_in: Residual input [1, L, hidden_dim] bf16. - flag: AllReduce sync flag. - vec_out: Output tensor [1, L, hidden_dim] bf16. - profile_logs: Profile logs tensor. - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - compute_kernel_type: Compute kernel type ("bf16"). - """ - torch.ops.tilert.padded_allreduce_add_op( - partial_buf, x_in, flag, vec_out, profile_logs, model_arch, compute_kernel_type - ) - - -class PaddedAllReduceAddAlgorithm(Enum): - """PaddedAllReduceAdd algorithm.""" - - BF16 = "bf16" - - -class PaddedAllReduceAdd(TileRTModule): - """PaddedAllReduceAdd module — zero-partial AllReduce + residual add.""" - - _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [PaddedAllReduceAddAlgorithm.BF16], - "glm_5": [PaddedAllReduceAddAlgorithm.BF16], - } - - def __init__( - self, - model_args: ModelArgs, - num_devices: int, - device_id: int = 0, - ): - super().__init__( - self.__class__.__name__, - model_args=model_args, - num_devices=num_devices, - device_id=device_id, - ) - - self.dim = self.model_args.dim - - self.partial_buf: torch.Tensor | None = None - - self.hidden_out: torch.Tensor | None = None - - self.profile_logs: torch.Tensor | None = None - self.is_var_init = False - - def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - """Allocate output buffer and persistent zero-filled partial buffer. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ - self.hidden_out = torch.zeros( - (batch_size, seq_len, self.dim), - dtype=torch.bfloat16, - device=f"cuda:{self.device_id}", - ) - self.partial_buf = torch.zeros( - (batch_size, seq_len, self.dim), - dtype=torch.bfloat16, - device=f"cuda:{self.device_id}", - ) - self.profile_logs = get_profile_log_tensor(device=f"cuda:{self.device_id}") - self.is_var_init = True - - def golden_forward( - self, - x_in: torch.Tensor, - ) -> torch.Tensor: - """Golden reference: allreduce(zeros) + x_in = x_in (single-GPU). - - On a single GPU, allreduce of zeros returns zeros, so output = x_in. - - Args: - x_in: Residual input [1, L, hidden_dim]. - - Returns: - Output tensor (copy of x_in). - """ - return x_in.clone() - - def tilert_forward( - self, - x_in: torch.Tensor, - flag: int, - ) -> torch.Tensor: - """Run TileRT kernel forward. - - Args: - x_in: Residual input [1, L, hidden_dim]. - flag: AllReduce sync flag. - - Returns: - Output tensor [1, L, hidden_dim]. - """ - assert self.hidden_out is not None - assert self.partial_buf is not None - assert self.profile_logs is not None - padded_allreduce_add( - self.partial_buf, - x_in, - flag, - self.hidden_out, - self.profile_logs, - model_arch=self.model_args.arch_name, - ) - return self.hidden_out - - def __call__( - self, - x_in: torch.Tensor, - ) -> torch.Tensor: - return self.golden_forward(x_in) diff --git a/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py b/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py index 3e99f0e..7128c39 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py +++ b/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py @@ -30,18 +30,6 @@ def projo_wkvb( model_arch: str, compute_kernel_type: str = "fp16mma", ) -> None: - """ - Define the ProjOWkvb operation. - - Args: - o_in: Input tensor. - wkv_b_b: Weight tensor. - wkv_b_scales: Scale tensor. - output: Output tensor. - profile_logs: Profile logs tensor. - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - compute_kernel_type: Kernel type ("fp16mma" for both DSv32 and GLM5). - """ torch.ops.tilert.projo_wkvb_op( o_in, wkv_b_b, @@ -68,7 +56,6 @@ def __init__(self, model_args: ModelArgs, num_devices: int): @staticmethod def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: - """Swizzle a [*, 16, 16] block for the packed weight layout.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 16 pre_shape = mat_in.shape[:-2] mat_in = mat_in.reshape(*pre_shape, 2, 8, 2, 4, 2).transpose(-4, -3).transpose(-5, -4) @@ -76,7 +63,6 @@ def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: @staticmethod def _swizzle_mma_16x16_for_pages(mat_in: torch.Tensor, k_dim: int, pages: int) -> torch.Tensor: - """Swizzle a [*, 16, K] matrix for the paged weight layout.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == k_dim pre_shape = mat_in.shape[:-2] k_per_page = k_dim // pages @@ -87,17 +73,15 @@ def _swizzle_mma_16x16_for_pages(mat_in: torch.Tensor, k_dim: int, pages: int) - return mat_in.contiguous() def convert_to_fp16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: - """Convert weights to the packed format expected by the kernel.""" with torch.inference_mode(): wkv_b_b, wkv_b_b_scales = self.convert_to_general(weights) n_heads = wkv_b_b.size(0) v_head_dim = wkv_b_b.size(1) kv_lora_rank = wkv_b_b.size(2) - num_ctas = 80 - rows_per_cta = (n_heads * v_head_dim) // num_ctas - is_glm5 = self.model_args.arch_name == "glm_5" + num_ctas = (n_heads * v_head_dim) // 32 if is_glm5 else 80 + rows_per_cta = (n_heads * v_head_dim) // num_ctas w_flat = wkv_b_b.reshape(num_ctas, rows_per_cta // 16, 16, kv_lora_rank) w_swizzled = ProjoWKVbWeightsConverter._swizzle_mma_16x16_for_pages( @@ -114,7 +98,9 @@ def convert_to_fp16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: scales_per_cta = wkv_b_b_scales.repeat_interleave(ctas_per_scale_row, dim=1) scales_per_cta = scales_per_cta.reshape(num_ctas, n_scale_k) else: - scales_per_cta = wkv_b_b_scales.squeeze(1).repeat_interleave(ctas_per_head, dim=0) + scales_per_cta = wkv_b_b_scales.squeeze(1).repeat_interleave( + ctas_per_head, dim=0 + ) scale_dtype = torch.float32 scales_per_cta = scales_per_cta.to(scale_dtype) @@ -131,7 +117,6 @@ def convert_to_fp16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: return torch.cat([w_bytes, scales_raw, padding], dim=-1).contiguous() def convert_to_bf16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: - """Convert weights to the packed format expected by the BF16 kernel.""" with torch.inference_mode(): tilert_wkv_b_weights, tilert_wkv_b_scales = weights @@ -169,7 +154,8 @@ def convert_to_bf16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: wkv_bf16 = (w.float() * s).to(torch.bfloat16) n_heads = n_local_heads - num_ctas = 80 + is_glm5 = self.model_args.arch_name == "glm_5" + num_ctas = (n_heads * v_head_dim) // 32 if is_glm5 else 80 rows_per_cta = (n_heads * v_head_dim) // num_ctas w_flat = wkv_bf16.reshape(num_ctas, rows_per_cta // 16, 16, kv_lora_rank) @@ -328,15 +314,6 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_wkv_b_b, self.tilert_wkv_b_b_scales] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding: split weights and scales per device. - - Args: - weights_map: Map from ref weight alias to tensor. - - Returns: - Map from tilert weight alias to (num_devices, ...) tensors. - """ kv_b_proj_weight = weights_map[self.ref_weights_alias.wkv_b_weights] kv_b_proj_weight_scale = weights_map[self.ref_weights_alias.wkv_b_scales] @@ -403,7 +380,6 @@ def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: self.init_tilert_weights_hmma(state_dict) def init_tilert_weights_hmma(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize with HMMA-packed weights.""" packed = ProjoWKVbWeightsConverter(self.model_args, self.num_devices).dispatch( ProjoWKVbAlgorithm.FP16MMA, [ @@ -416,7 +392,6 @@ def init_tilert_weights_hmma(self, state_dict: dict[str, torch.Tensor]) -> None: self.compute_kernel_type = "fp16mma" def init_tilert_weights_hmma_bf16(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize with BF16 HMMA-packed weights (dequantized, no scales).""" packed = ProjoWKVbWeightsConverter(self.model_args, self.num_devices).dispatch( ProjoWKVbAlgorithm.BF16MMA, [ diff --git a/tilert/models/glm_5/_dsa_v32/ops/projq_wqb.py b/tilert/models/glm_5/_dsa_v32/ops/projq_wqb.py index c40ca51..7ad3d1d 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/projq_wqb.py +++ b/tilert/models/glm_5/_dsa_v32/ops/projq_wqb.py @@ -31,18 +31,7 @@ def projq_wqb( *, model_arch: str, ) -> None: - """ - Define the ProjqWqb operation. - - Args: - q_nope_in: Input tensor. - wkv_b_a: Weight tensor. - wkv_b_a_scales: Scale tensor. - output: Output tensor. - profile_logs: Profile logs tensor. - compute_kernel_type: Kernel type ("fp16mma"). - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - """ + """Define the ProjqWqb operation.""" torch.ops.tilert.projq_wqb_op( q_nope_in, wkv_b_a, @@ -70,7 +59,7 @@ def __init__(self, model_args: ModelArgs, num_devices: int, head_dim_block_size: @staticmethod def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: - """Swizzle a [*, 16, 16] block for the packed weight layout.""" + """Swizzle a [*, 16, 16] sub-block for the MMA kernel.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 16 pre_shape = mat_in.shape[:-2] mat_in = mat_in.reshape(*pre_shape, 2, 8, 2, 4, 2).transpose(-4, -3).transpose(-5, -4) @@ -78,7 +67,7 @@ def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: @staticmethod def _swizzle_mma_16x16_for_pages(mat_in: torch.Tensor, k_dim: int, pages: int) -> torch.Tensor: - """Swizzle a [*, 16, K] matrix for the paged weight layout.""" + """Swizzle [*, 16, K] matrix for paged MMA layout.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == k_dim pre_shape = mat_in.shape[:-2] k_per_page = k_dim // pages @@ -89,17 +78,16 @@ def _swizzle_mma_16x16_for_pages(mat_in: torch.Tensor, k_dim: int, pages: int) - return mat_in.contiguous() def convert_to_fp16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: - """Convert weights to the packed format expected by the kernel.""" + """Convert weights to the FP16 MMA packed format.""" with torch.inference_mode(): wkv_b_a, wkv_b_a_scales = self.convert_to_general(weights) n_heads = wkv_b_a.size(0) head_dim = wkv_b_a.size(2) kv_lora_rank = wkv_b_a.size(1) - num_ctas = 80 - rows_per_cta = (n_heads * kv_lora_rank) // num_ctas - is_glm5 = self.model_args.arch_name == "glm_5" + num_ctas = (n_heads * kv_lora_rank) // 64 if is_glm5 else 80 + rows_per_cta = (n_heads * kv_lora_rank) // num_ctas w_flat = wkv_b_a.reshape(num_ctas, rows_per_cta // 16, 16, head_dim) w_swizzled = self._swizzle_mma_16x16_for_pages(w_flat, head_dim, pages=1) @@ -129,7 +117,7 @@ def convert_to_fp16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: return torch.cat([w_bytes, scales_raw, padding], dim=-1).contiguous() def convert_to_bf16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: - """Convert weights to the packed format expected by the BF16 kernel.""" + """Convert weights to the BF16 MMA packed format.""" with torch.inference_mode(): tilert_wkv_b_weights, tilert_wkv_b_scales = weights @@ -153,7 +141,8 @@ def convert_to_bf16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: n_heads = n_local_heads head_dim = nope_head_dim - num_ctas = 80 + is_glm5 = self.model_args.arch_name == "glm_5" + num_ctas = (n_heads * kv_lora_rank) // 64 if is_glm5 else 80 rows_per_cta = (n_heads * kv_lora_rank) // num_ctas w_flat = wkv_bf16.reshape(num_ctas, rows_per_cta // 16, 16, head_dim) @@ -313,15 +302,6 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_wkv_b_a, self.tilert_wkv_b_a_scales] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding: split weights and scales per device. - - Args: - weights_map: Map from ref weight alias to tensor. - - Returns: - Map from tilert weight alias to (num_devices, ...) tensors. - """ kv_b_proj_weight = weights_map[self.ref_weights_alias.wkv_b_weights] kv_b_proj_weight_scale = weights_map[self.ref_weights_alias.wkv_b_scales] @@ -388,33 +368,29 @@ def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: self.init_tilert_weights_hmma(state_dict) def init_tilert_weights_hmma(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize with HMMA-packed weights.""" - packed = ProjqWqbWeightsConverter( - self.model_args, self.num_devices, self.head_dim_block_size - ).dispatch( - ProjqWqbAlgorithm.FP16MMA, + packed, dummy_scales = torch.ops.tilert.projq_wkvb__convert_weights( [ state_dict[self.tilert_weights_alias.wkv_b_weights], state_dict[self.tilert_weights_alias.wkv_b_scales], ], + self.model_args.arch_name, + "fp16mma", ) self.tilert_wkv_b_a = packed - self.tilert_wkv_b_a_scales = torch.empty(1, dtype=torch.float8_e4m3fn, device=packed.device) + self.tilert_wkv_b_a_scales = dummy_scales self.compute_kernel_type = "fp16mma" def init_tilert_weights_hmma_bf16(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize with BF16 HMMA-packed weights (dequantized, no scales).""" - packed = ProjqWqbWeightsConverter( - self.model_args, self.num_devices, self.head_dim_block_size - ).dispatch( - ProjqWqbAlgorithm.BF16MMA, + packed, dummy_scales = torch.ops.tilert.projq_wkvb__convert_weights( [ state_dict[self.tilert_weights_alias.wkv_b_weights], state_dict[self.tilert_weights_alias.wkv_b_scales], ], + self.model_args.arch_name, + "bf16mma", ) self.tilert_wkv_b_a = packed - self.tilert_wkv_b_a_scales = torch.empty(1, dtype=torch.float8_e4m3fn, device=packed.device) + self.tilert_wkv_b_a_scales = dummy_scales self.compute_kernel_type = "bf16mma" def init_random_weights(self) -> None: diff --git a/tilert/models/glm_5/_dsa_v32/ops/projx_wis.py b/tilert/models/glm_5/_dsa_v32/ops/projx_wis.py index e13b4e0..1784629 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/projx_wis.py +++ b/tilert/models/glm_5/_dsa_v32/ops/projx_wis.py @@ -26,17 +26,6 @@ def projx_wis( profile_logs: torch.Tensor, model_arch: str, ) -> None: - """ - Define the ProjxWis operation. - - Args: - x_in: Input tensor. - w: Weight tensor. - output: Output tensor. - compute_kernel_type: Compute kernel type ("bf16" or "bf16mma"). - profile_logs: Profile logs tensor. - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - """ torch.ops.tilert.proj_w_op(x_in, w, output, model_arch, compute_kernel_type, profile_logs) @@ -125,7 +114,7 @@ def __init__( @staticmethod def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: - """Swizzle each 16x16 BF16 tile for the packed weight layout.""" + """Swizzle each 16x16 BF16 tile for MMA loading.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 16 pre_shape = mat_in.shape[:-2] mat_in = mat_in.reshape(*pre_shape, 2, 8, 2, 4, 2).transpose(-4, -3).transpose(-5, -4) @@ -135,7 +124,6 @@ def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: def _to_hmma_layout( w_orig: torch.Tensor, n_ctas: int, rows_per_cta: int, x_dim: int, num_pages: int ) -> torch.Tensor: - """Convert [output_dim, x_dim] BF16 weights to the packed kernel layout.""" cols_per_page = x_dim // num_pages n_k_tiles = cols_per_page // 16 w = w_orig.reshape(n_ctas, rows_per_cta, num_pages, cols_per_page) @@ -153,15 +141,6 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_w] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding: replicate weight for each device. - - Args: - weights_map: Map from ref weight alias to tensor. - - Returns: - Map from tilert weight alias to (num_devices, ...) tensors. - """ w = weights_map[self.ref_weights_alias.w_weights] if self.compute_kernel_type == "bf16mma": n_ctas, rows_per_cta, num_pages = self._HMMA_CONFIGS[self.dim] @@ -177,7 +156,17 @@ def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: self.is_ref_weights_init = True def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - self.tilert_w = state_dict[self.tilert_weights_alias.w_weights].detach().clone() + w = state_dict[self.tilert_weights_alias.w_weights].detach().clone() + if ( + self.compute_kernel_type == "bf16mma" + and w.dim() == 2 + and (w.shape[0] == self.index_n_heads and w.shape[1] == self.dim) + ): + n_ctas, rows_per_cta, num_pages = self._HMMA_CONFIGS[self.dim] + w = self._to_hmma_layout( + w.to(torch.bfloat16), n_ctas, rows_per_cta, self.dim, num_pages + ) + self.tilert_w = w self.is_tilert_weights_init = True def init_random_weights(self) -> None: diff --git a/tilert/models/glm_5/_dsa_v32/ops/projx_wqaki.py b/tilert/models/glm_5/_dsa_v32/ops/projx_wqaki.py index 367d5fe..6f8a1e3 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/projx_wqaki.py +++ b/tilert/models/glm_5/_dsa_v32/ops/projx_wqaki.py @@ -19,18 +19,6 @@ def projx_wqaki( *, model_arch: str, ) -> None: - """FP8 projection for q, ki. - - Args: - x_quant: FP8 quantized hidden states [1, seq_len, hidden_dim]. - x_scale: Scale factors for x_quant. - wqaki: Packed FP8 weights + scales for q, ki. - out_q: Output q tensor. - out_ki: Output ki tensor. - profile_logs: Profile logs tensor. - compute_kernel_type: Kernel type ("fp8mma", "fp8mma_68cta", "fp8mma_136cta"). - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - """ torch.ops.tilert.projx_wqaki_op( x_quant, x_scale, @@ -62,7 +50,6 @@ def convert_dsv32( wki: torch.Tensor, wki_scale: torch.Tensor, ) -> torch.Tensor: - """Convert DSV3.2 weights to the packed format expected by the kernel.""" with torch.inference_mode(): wq_a_scale = wq_a_scale.to(torch.bfloat16) wki_scale = wki_scale.to(torch.bfloat16) @@ -153,7 +140,6 @@ def convert_glm5_68cta( wki: torch.Tensor, wki_scale: torch.Tensor, ) -> torch.Tensor: - """Convert GLM5 weights to the packed format expected by the kernel.""" with torch.inference_mode(): wq_a_scale = wq_a_scale.to(torch.float32) wki_scale = wki_scale.to(torch.float32) @@ -195,6 +181,23 @@ def convert_glm5_68cta( ) return torch.cat([wqaki_raw, wqaki_scales, wqaki_padding], dim=-1).contiguous() + @staticmethod + def convert_glm5_68cta_w8a16( + wq_a: torch.Tensor, + wq_a_scale: torch.Tensor, + wki: torch.Tensor, + wki_scale: torch.Tensor, + ) -> torch.Tensor: + from tilert.models.glm_5._dsa_v32.ops.rmsnorm_projx_wqkva import ( + RMSNormProjQKVAW8A16MMAWeightsConverter, + ) + + with torch.inference_mode(): + dim = 6144 + w_fp8 = torch.cat([wq_a.reshape(2048, dim), wki.reshape(128, dim)], dim=0).contiguous() + scales = torch.cat([wq_a_scale, wki_scale], dim=0).to(torch.float32).contiguous() + return RMSNormProjQKVAW8A16MMAWeightsConverter.pack_lane_major(w_fp8, scales, dim) + @staticmethod def convert_glm5_136cta( wq_a: torch.Tensor, @@ -202,7 +205,6 @@ def convert_glm5_136cta( wki: torch.Tensor, wki_scale: torch.Tensor, ) -> torch.Tensor: - """Convert GLM5 weights to the packed format expected by the kernel.""" with torch.inference_mode(): wq_a_scale = wq_a_scale.to(torch.float32) wki_scale = wki_scale.to(torch.float32) diff --git a/tilert/models/glm_5/_dsa_v32/ops/projx_wqkva.py b/tilert/models/glm_5/_dsa_v32/ops/projx_wqkva.py index 6ade7af..b94a36f 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/projx_wqkva.py +++ b/tilert/models/glm_5/_dsa_v32/ops/projx_wqkva.py @@ -10,6 +10,7 @@ from tilert.models.glm_5._dsa_v32.ops.rmsnorm_projx_wqkva import ( RMSNormProjQKVAFP8MMAWeightsConverter, RMSNormProjQKVAFP16MMAWeightsConverter, + RMSNormProjQKVAW8A16MMAWeightsConverter, ) from tilert.utils import get_profile_log_tensor @@ -32,7 +33,7 @@ def projx_wqkva( *, model_arch: str, ) -> None: - """FP8 MMA projection for q, kv, pe_cache (DSV3.2).""" + """Standalone FP8 QMMA projection for q, kv, pe_cache.""" torch.ops.tilert.projx_wqkva_op( x_quant, x_scale, @@ -101,14 +102,19 @@ class ProjXWqkvaAlgorithm(Enum): FP8MMA = "fp8mma" FP16MMA = "fp16mma" + W8A16HMMA = "w8a16_hmma" class ProjXWqkva(TileRTModule): - """FP8 MMA projection module for q, kv, pe_cache.""" + """Standalone FP8 QMMA GEMV for q, kv, pe_cache projections.""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [ProjXWqkvaAlgorithm.FP8MMA], - "glm_5": [ProjXWqkvaAlgorithm.FP8MMA, ProjXWqkvaAlgorithm.FP16MMA], + "deepseek_v3_2": [ProjXWqkvaAlgorithm.FP8MMA, ProjXWqkvaAlgorithm.W8A16HMMA], + "glm_5": [ + ProjXWqkvaAlgorithm.FP8MMA, + ProjXWqkvaAlgorithm.FP16MMA, + ProjXWqkvaAlgorithm.W8A16HMMA, + ], } def __init__( @@ -156,11 +162,12 @@ def set_algorithm(self, algorithm: Enum) -> None: super().set_algorithm(algorithm) if algorithm == ProjXWqkvaAlgorithm.FP16MMA: self.compute_kernel_type = "fp16mma" + elif algorithm == ProjXWqkvaAlgorithm.W8A16HMMA: + self.compute_kernel_type = "w8a16_hmma" else: self.compute_kernel_type = "fp8mma" def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Repeat weights for device sharding.""" q_a_proj_weight = weights_map[self.ref_weights_alias.q_a_weights][None, ...].repeat( self.num_devices, 1, 1 ) @@ -222,6 +229,20 @@ def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: hidden_dim=self.dim, q_lora_rank=self.q_lora_rank, ) + elif self.algorithm == ProjXWqkvaAlgorithm.W8A16HMMA: + self.tilert_wqkva, _ = ( + RMSNormProjQKVAW8A16MMAWeightsConverter.convert_to_w8a16_mma_gemv( + wq_a, + wq_a_scale, + wkv_a, + wkv_a_scale, + w_pe, + w_pe_scale, + dummy_gamma, + hidden_dim=self.dim, + q_lora_rank=self.q_lora_rank, + ) + ) else: self.tilert_wqkva, _ = RMSNormProjQKVAFP8MMAWeightsConverter.convert_to_fp8_mma_gemv( wq_a, @@ -256,9 +277,9 @@ def init_random_weights(self) -> None: tensor_list = [ torch.randn(self.dim, dtype=torch.float32), torch.randn(self.q_lora_rank, self.dim, dtype=torch.bfloat16).to(torch.float8_e4m3fn), - torch.randn(q_scale_dim, dim_scale_dim, dtype=scale_dtype), + torch.randn(q_scale_dim, dim_scale_dim, dtype=scale_dtype).abs(), torch.randn(kv_mqa_rows, self.dim, dtype=torch.bfloat16).to(torch.float8_e4m3fn), - torch.randn(kv_mqa_scale_dim, dim_scale_dim, dtype=scale_dtype), + torch.randn(kv_mqa_scale_dim, dim_scale_dim, dtype=scale_dtype).abs(), ] ref_state_dict = dict(zip(self.ref_weights_alias(), tensor_list)) self.init_reference_weights(ref_state_dict) @@ -277,7 +298,10 @@ def golden_forward( assert self.ref_wkv_a is not None assert self.ref_w_pe is not None - if self.algorithm == ProjXWqkvaAlgorithm.FP16MMA: + if self.algorithm in ( + ProjXWqkvaAlgorithm.FP16MMA, + ProjXWqkvaAlgorithm.W8A16HMMA, + ): x_float = x_quant.float() else: x_fp8 = x_quant.to(torch.float32) diff --git a/tilert/models/glm_5/_dsa_v32/ops/qkv_rope.py b/tilert/models/glm_5/_dsa_v32/ops/qkv_rope.py deleted file mode 100644 index 7f16a1c..0000000 --- a/tilert/models/glm_5/_dsa_v32/ops/qkv_rope.py +++ /dev/null @@ -1,192 +0,0 @@ -"""QKV Rope operation module.""" - -from dataclasses import dataclass -from enum import Enum - -import torch - -from tilert.models.base import TileRTModule -from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.utils import apply_rotary_emb -from tilert.utils import get_profile_log_tensor - -__all__ = [ - "qkv_rope", - "QKVRoPE", - "QKVRoPERefWeightsAlias", - "QKVRoPETilertWeightsAlias", -] - - -def qkv_rope( - pe_cache: torch.Tensor, - kv_cache: torch.Tensor, - rope_freqs: torch.Tensor, - cur_pos: torch.Tensor, - profile_logs: torch.Tensor, - model_arch: str, - compute_kernel_type: str = "general", -) -> None: - """ - Perform QKV Rope operation. - - Args: - pe_cache: Q PE tensor (bsz, seq, n_local_heads, qk_rope_head_dim). - kv_cache: K PE cache (bsz, seq, qk_rope_head_dim). - rope_freqs: Rope frequencies tensor. - cur_pos: Current position tensor. - profile_logs: Profile logs tensor. - model_arch: Model architecture string. - compute_kernel_type: Compute kernel type string. - """ - torch.ops.tilert.qkv_rope_op( - pe_cache, - kv_cache, - rope_freqs, - cur_pos, - model_arch, - compute_kernel_type, - profile_logs, - ) - - -@dataclass -class QKVRoPERefWeightsAlias: - """Reference weights alias for QKVRoPE (no weights).""" - - @property - def ref_tensor_alias(self) -> list[str]: - return [] - - def __call__(self) -> list[str]: - return self.ref_tensor_alias - - -@dataclass -class QKVRoPETilertWeightsAlias: - """TileRT weights alias for QKVRoPE (no weights).""" - - @property - def tilert_tensor_alias(self) -> list[str]: - return [] - - def __call__(self) -> list[str]: - return self.tilert_tensor_alias - - -class QKVRoPEAlgorithm(Enum): - """QKVRoPE algorithm.""" - - GENERAL = "general" - - -class QKVRoPE(TileRTModule): - """QKV RoPE module. Unified for deepseek_v3_2 and glm_5.""" - - _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [QKVRoPEAlgorithm.GENERAL], - "glm_5": [QKVRoPEAlgorithm.GENERAL], - } - - def __init__( - self, - model_args: ModelArgs, - num_devices: int = 1, - device_id: int = 0, - layer_idx: int = 0, - ref_weights_alias: QKVRoPERefWeightsAlias | None = None, - ) -> None: - super().__init__( - self.__class__.__name__, - model_args=model_args, - num_devices=num_devices, - device_id=device_id, - layer_idx=layer_idx, - ) - self.tilert_weights_alias = QKVRoPETilertWeightsAlias() - self.ref_weights_alias = ( - ref_weights_alias if ref_weights_alias is not None else QKVRoPERefWeightsAlias() - ) - self.n_local_heads = model_args.n_heads // num_devices - self.qk_rope_head_dim = model_args.qk_rope_head_dim - self.profile_logs: torch.Tensor | None = None - - def get_weights_list(self) -> list[torch.Tensor]: - return [] - - def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - del weights_map - return {} - - def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - del state_dict - pass - - def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - del state_dict - pass - - def init_random_weights(self) -> None: - pass - - def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - del batch_size, seq_len - self.profile_logs = get_profile_log_tensor() - self.is_var_init = True - - def golden_forward( - self, - q_pe: torch.Tensor, - pe_cache: torch.Tensor, - start_pos: int, - freqs_cis: torch.Tensor, - bsz: int, - seqlen: int, - ) -> torch.Tensor: - end_pos = start_pos + seqlen - - k_pe = pe_cache[:bsz, start_pos:end_pos] - k_pe = apply_rotary_emb(k_pe.unsqueeze(2), freqs_cis) - pe_cache[:bsz, start_pos:end_pos] = k_pe.squeeze(2) - - return apply_rotary_emb(q_pe, freqs_cis) - - def tilert_forward( - self, - q_pe: torch.Tensor, - pe_cache: torch.Tensor, - start_pos: int, - freqs_cis: torch.Tensor, - bsz: int, - seqlen: int, - ) -> torch.Tensor: - assert self.profile_logs is not None - end_pos = start_pos + seqlen - - q_pe_rope = q_pe.clone() - rope_freqs = torch.view_as_real(freqs_cis).reshape(*freqs_cis.shape[:-1], -1) - cur_pos = torch.tensor([start_pos], dtype=torch.int32) - - qkv_rope( - q_pe_rope, - pe_cache[:bsz, start_pos:end_pos], - rope_freqs, - cur_pos, - self.profile_logs, - model_arch=self.model_args.arch_name, - ) - - return q_pe_rope - - def __call__( - self, - q_pe: torch.Tensor, - pe_cache: torch.Tensor, - start_pos: int, - freqs_cis: torch.Tensor, - bsz: int, - seqlen: int, - ) -> torch.Tensor: - if self.flag_enable_tilert: - return self.tilert_forward(q_pe, pe_cache, start_pos, freqs_cis, bsz, seqlen) - return self.golden_forward(q_pe, pe_cache, start_pos, freqs_cis, bsz, seqlen) diff --git a/tilert/models/glm_5/_dsa_v32/ops/receive_selected_token_ids.py b/tilert/models/glm_5/_dsa_v32/ops/receive_selected_token_ids.py index 508d13e..80bb1ee 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/receive_selected_token_ids.py +++ b/tilert/models/glm_5/_dsa_v32/ops/receive_selected_token_ids.py @@ -8,25 +8,15 @@ def receive_selected_token_ids( - ll_buf: torch.Tensor, + recv_buf: torch.Tensor, dst: torch.Tensor, expected_flag: int, profile_logs: torch.Tensor, model_arch: str, compute_kernel_type: str = "bf16", ) -> None: - """Receive idx_selects from GPU 0. - - Args: - ll_buf: Receive buffer on this GPU (written by GPU 0). - dst: Destination idx_selects tensor [1, S, 2048] int32. - expected_flag: Expected synchronization flag value. - profile_logs: Profile logs tensor. - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - compute_kernel_type: Compute kernel type ("bf16"). - """ torch.ops.tilert.receive_selected_token_ids_op( - ll_buf, + recv_buf, dst, expected_flag, model_arch, diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_head_proj.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_head_proj.py index fa2086d..813e504 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_head_proj.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_head_proj.py @@ -26,7 +26,6 @@ def rmsnorm_head_proj( model_arch: str, compute_kernel_type: str = "general", ) -> None: - """RMS Norm Head Projection operation.""" torch.ops.tilert.rmsnorm_head_proj_op( hidden_in, gamma_in, @@ -60,20 +59,18 @@ def tilert_to_tilert_native_bf16_warp_gemv( def convert_to_general( self, weights_list: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Convert the weights to general format. - - Args: - weights_list: List of weights. - - Returns: - Tuple of weights. - """ args = self.model_args assert args.arch_name == "deepseek_v3_2" or args.arch_name == "glm_5" with torch.inference_mode(): rmsnorm_gamma, mat_in = weights_list + if args.arch_name == "glm_5": + from tilert.models.glm_5._dsa_v32.ops.head_proj_w16a16_hmma import ( + swizzle_head_proj_weight_bf16mma, + ) + + weights = swizzle_head_proj_weight_bf16mma(mat_in.contiguous()) + return rmsnorm_gamma.float(), weights logits_dim = mat_in.shape[-2] dim = mat_in.shape[-1] num_steps = dim // 1024 @@ -150,27 +147,13 @@ def tilert_tensor_alias(self) -> list[str]: return self.tilert_weights_alias() def get_weights_list(self) -> list[torch.Tensor]: - """ - Get the weights list. - - Returns: - List of weights. - """ + """Get the weights list.""" return [self.tilert_rmsnorm_gamma, self.tilert_head_proj] def device_sharding( self, weights_dict: dict[str, torch.Tensor], ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Device sharding. - - Args: - weights_dict: Dictionary of weights. - key_prefix: Key prefix. - Returns: - Tuple of weights. - """ rmsnorm_gamma_key = "model.norm.weight" head_proj_key = "lm_head.weight" rmsnorm_gamma = weights_dict[rmsnorm_gamma_key][None, ...] @@ -181,13 +164,6 @@ def device_sharding( return rmsnorm_gamma.contiguous(), head_proj.contiguous() def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Initialize the reference weights. - - Args: - state_dict: State dictionary. - device_id: Device ID. - """ sharded_list = self.device_sharding(state_dict) gamma, head_proj = sharded_list[0][self.device_id], sharded_list[1][self.device_id] @@ -195,25 +171,12 @@ def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: self.ref_head_proj = head_proj def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Initialize the tilert weights. - - Args: - state_dict: State dictionary. - """ assert self.algorithm is not None self.tilert_rmsnorm_gamma, self.tilert_head_proj = RMSNormHeadProjWeightsConverter( self.model_args, self.num_devices ).dispatch(self.algorithm, [state_dict[alias] for alias in self.tilert_weights_alias()]) def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - """ - Initialize the tilert variables. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ self.hidden_rmsnorm_out = torch.zeros( (batch_size, seq_len, self.dim), dtype=torch.bfloat16, @@ -227,8 +190,9 @@ def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: self.profile_logs = get_profile_log_tensor(device=f"cuda:{self.device_id}") self.is_init = True - def init_random_weights(self, device_id: int = 0) -> None: - """Initialize the random weights.""" + def init_random_weights(self, device_id: int | None = None) -> None: + if device_id is None: + device_id = self.device_id rmsnorm_gamma = torch.randn(self.dim, dtype=torch.float32, device=f"cuda:{device_id}") head_proj = torch.randn( self.logits_dim, self.dim, dtype=torch.bfloat16, device=f"cuda:{device_id}" @@ -252,15 +216,6 @@ def golden_forward( self, hidden_in: torch.Tensor, ) -> torch.Tensor: - """ - Forward pass for the down-project module. - - Args: - hidden_in: Input hidden. - - Returns: - Output tensor. - """ assert self.ref_rmsnorm_gamma is not None assert self.ref_head_proj is not None bsz = hidden_in.shape[0] diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_kv.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_kv.py index 81d161c..99cde46 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_kv.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_kv.py @@ -26,18 +26,6 @@ def rmsnorm_kv( model_arch: str, compute_kernel_type: str = "general", ) -> None: - """ - Define the RMSNormKV operation. - - Args: - kv: Input tensor. - gamma: Weight tensor. - cur_pos: Current position tensor. - kv_cache: Output tensor. - profile_logs: Profile logs tensor. - model_arch: Model architecture string. - compute_kernel_type: Compute kernel type string. - """ torch.ops.tilert.rmsnorm_kv_op( kv, gamma, cur_pos, kv_cache, model_arch, compute_kernel_type, profile_logs ) @@ -75,14 +63,15 @@ class KVRMSNormAlgorithm(Enum): """KVRMSNorm algorithm.""" GENERAL = "general" + FP8 = "fp8" class KVRMSNorm(TileRTModule): """KVRMSNorm module: RMSNorm on KV tensor with in-place write to kv_cache.""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [KVRMSNormAlgorithm.GENERAL], - "glm_5": [KVRMSNormAlgorithm.GENERAL], + "deepseek_v3_2": [KVRMSNormAlgorithm.GENERAL, KVRMSNormAlgorithm.FP8], + "glm_5": [KVRMSNormAlgorithm.GENERAL, KVRMSNormAlgorithm.FP8], } def __init__( @@ -126,40 +115,27 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_kv_norm_weight] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding: replicate gamma for each device. - - Args: - weights_map: Map from ref weight alias to tensor. - - Returns: - Map from tilert weight alias to (num_devices, ...) tensors. - """ gamma = weights_map[self.ref_weights_alias.kv_norm_weight][None, ...].repeat( self.num_devices, 1 ) return {self.tilert_weights_alias.kv_norm_gamma: gamma} def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize reference weights from state dict.""" self.ref_norm_gamma = state_dict[self.ref_weights_alias.kv_norm_weight].contiguous() assert ( self.ref_norm_gamma.shape[-1] == self.kv_lora_rank ), f"kv_norm weight shape must be ({self.kv_lora_rank},), got {self.ref_norm_gamma.shape}" def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize TileRT weights from state dict.""" gamma = state_dict[self.tilert_weights_alias.kv_norm_gamma] self.tilert_kv_norm_weight = gamma.float().detach().clone().contiguous() def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - """Allocate TileRT profiling buffer.""" del batch_size, seq_len self.profile_logs = get_profile_log_tensor() self.is_var_init = True def init_random_weights(self) -> None: - """Initialize random reference and TileRT weights for testing.""" ref_state_dict = { self.ref_weights_alias.kv_norm_weight: torch.randn( self.kv_lora_rank, dtype=torch.float32 @@ -172,7 +148,6 @@ def init_random_weights(self) -> None: def golden_forward( self, kv: torch.Tensor, kv_cache: torch.Tensor, start_pos: int, bsz: int, seqlen: int ) -> None: - """Reference forward: RMSNorm and write to kv_cache.""" assert self.ref_norm_gamma is not None end_pos = start_pos + seqlen out = torch.nn.functional.rms_norm( @@ -180,6 +155,10 @@ def golden_forward( ).to(kv.dtype) kv_cache[:bsz, start_pos:end_pos].copy_(out) + @property + def is_fp8(self) -> bool: + return self.algorithm == KVRMSNormAlgorithm.FP8 + def tilert_forward( self, kv: torch.Tensor, kv_cache: torch.Tensor, start_pos: int, bsz: int, seqlen: int ) -> None: @@ -194,6 +173,7 @@ def tilert_forward( kv_cache[:bsz], self.profile_logs, model_arch=self.model_args.arch_name, + compute_kernel_type="fp8" if self.is_fp8 else "general", ) def __call__( diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqb.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqb.py index 92d7a99..aab6c43 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqb.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqb.py @@ -1,4 +1,4 @@ -"""RmsnormProjqWqb operation module.""" +"""RmsnormProjqWqb operation module (Device Group B, TP7).""" import math from dataclasses import dataclass @@ -47,15 +47,11 @@ class RmsnormProjqWqbAlgorithm(Enum): """RmsnormProjqWqb algorithm.""" FP16MMA = "fp16mma" + BF16MMA = "bf16mma" class RmsnormProjqWqbWeightsConverter(TilertWeightsConverter): - """Weights converter for RmsnormProjqWqb. - - Supports configurations where n_heads is not evenly divisible by - num_devices; in that case n_local_heads is padded and padded head - weight rows are zero-filled. - """ + """Weights converter for RmsnormProjqWqb.""" kBf16NumCtas = 80 kGemvPageSize = 8 @@ -83,9 +79,12 @@ def __init__(self, model_args: ModelArgs, num_devices: int): self.qk_dim = self.qk_head_dim * self.n_local_heads self.qk_qdim = self.qk_dim // self.block_size - assert self.qk_dim % (self.kBf16NumCtas * self.kGemvPageSize) == 0, ( - f"qk_dim ({self.qk_dim}) must be divisible by " - f"kBf16NumCtas * kGemvPageSize ({self.kBf16NumCtas * self.kGemvPageSize})" + kRowsPerCta = 32 + qk_nope_dim = self.qk_nope_head_dim * self.n_local_heads + qk_pe_dim = self.qk_rope_head_dim * self.n_local_heads + assert qk_nope_dim % kRowsPerCta == 0 and qk_pe_dim % kRowsPerCta == 0, ( + f"qk_nope_dim ({qk_nope_dim}) and qk_pe_dim ({qk_pe_dim}) must each " + f"be divisible by rows_per_cta ({kRowsPerCta})" ) assert self.qk_dim % self.block_size == 0, ( f"qk_dim ({self.qk_dim}) must be divisible by block_size ({self.block_size}) " @@ -94,7 +93,6 @@ def __init__(self, model_args: ModelArgs, num_devices: int): @classmethod def _compute_n_local_heads(cls, n_total_heads: int, num_devices: int, qk_head_dim: int) -> int: - """Compute padded n_local_heads per device.""" if n_total_heads % num_devices == 0: return n_total_heads // num_devices @@ -114,22 +112,6 @@ def _redistribute_heads( qk_head_dim: int, block_size: int, ) -> tuple[list[torch.Tensor], list[torch.Tensor]]: - """Redistribute heads across devices with padding. - - Args: - wq_b_full: [n_total_heads * qk_head_dim, q_lora_dim] full weight. - wq_b_scale_full: [n_total_heads * qk_head_dim // block_size, q_lora_qdim] full scale. - n_total_heads: Total number of heads (e.g. 128). - n_local_heads: Target heads per GPU (padded, e.g. 20). - num_devices: Number of devices (e.g. 7). - qk_head_dim: Head dimension (e.g. 192). - block_size: Quantization block size (e.g. 128). - - Returns: - Lists of per-device (wq_b, wq_b_scale) with shape - [n_local_heads * qk_head_dim, q_lora_dim] and - [n_local_heads * qk_head_dim // block_size, q_lora_qdim]. - """ total_rows = n_total_heads * qk_head_dim rows_per_dev = n_local_heads * qk_head_dim scale_rows_per_dev = rows_per_dev // block_size @@ -185,7 +167,6 @@ def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: def _swizzle_mma_16x16_for_pages( mat_in: torch.Tensor, q_lora_dim: int, pages: int ) -> torch.Tensor: - """Swizzle a 16xK matrix for the paged weight layout (K divisible by 16).""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == q_lora_dim k_per_page = q_lora_dim // pages n_k_tiles = k_per_page // 16 @@ -201,7 +182,6 @@ def _common_to_tilert_fp16mma( wq_b_scales_raw: torch.Tensor, rmsnorm_gamma: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Convert common weights to the packed TileRT FP16 layout.""" pages = 2 rows_per_cta = 32 @@ -275,10 +255,14 @@ def _common_to_tilert_fp16mma( tilert_gamma = rmsnorm_gamma.float().detach().clone() return tilert_wqb, tilert_wqb_scales, tilert_gamma + def convert_to_bf16mma( + self, weights: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return self.convert_to_fp16mma(weights) + def convert_to_fp16mma( self, weights: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Convert common-format weights to TileRT FP16 MMA layout.""" with torch.inference_mode(): wq_b, wq_b_scale, q_norm_weight = weights return self._common_to_tilert_fp16mma(wq_b, wq_b_scale, q_norm_weight) @@ -325,11 +309,17 @@ def __call__(self) -> list[str]: class RmsnormProjqWqb(TileRTModule): - """RmsnormProjqWqb module: RMSNorm + Q projection (wq_b only).""" + """RmsnormProjqWqb module: RMSNorm + Q projection (wq_b only, TP7).""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [RmsnormProjqWqbAlgorithm.FP16MMA], - "glm_5": [RmsnormProjqWqbAlgorithm.FP16MMA], + "deepseek_v3_2": [ + RmsnormProjqWqbAlgorithm.FP16MMA, + RmsnormProjqWqbAlgorithm.BF16MMA, + ], + "glm_5": [ + RmsnormProjqWqbAlgorithm.FP16MMA, + RmsnormProjqWqbAlgorithm.BF16MMA, + ], } def __init__( @@ -384,7 +374,7 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_q_norm_weight, self.tilert_wq_b, self.tilert_wq_b_scales] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Redistribute heads across devices with padding.""" + """Redistribute 128 heads into 7 GPUs × 20 slots with padding.""" gamma = weights_map[self.ref_weights_alias.rmsnorm_gamma][None, ...].repeat( self.num_devices, 1 ) @@ -489,7 +479,7 @@ def golden_forward(self, q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: assert self.ref_wq_b is not None bsz, seqlen, _ = q.shape - if bsz != 1 or seqlen not in [1, 2, 4]: + if bsz != 1 or seqlen not in [1, 2, 4, 8]: raise ValueError(f"Invalid batch size or sequence length: bsz={bsz}, seqlen={seqlen}") qr = torch.nn.functional.rms_norm(q.float(), [q.size(-1)], self.ref_q_norm, self.eps).to( @@ -510,7 +500,7 @@ def tilert_forward(self, q: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: assert self.profile_logs is not None bsz, seqlen, _ = q.shape - if bsz != 1 or seqlen not in [1, 2, 4]: + if bsz != 1 or seqlen not in [1, 2, 4, 8]: raise ValueError(f"Invalid batch size or sequence length: bsz={bsz}, seqlen={seqlen}") assert self.algorithm is not None, "Algorithm is not set" diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqi.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqi.py index 4f4d07f..31c0a8c 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqi.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projq_wqi.py @@ -1,4 +1,4 @@ -"""RmsnormProjqWqi operation module (IQ-only projection).""" +"""RmsnormProjqWqi operation module (GLM5 v2, IQ-only projection).""" from dataclasses import dataclass from enum import Enum @@ -44,6 +44,7 @@ class RmsnormProjqWqiAlgorithm(Enum): """RmsnormProjqWqi algorithm.""" FP16MMA = "fp16mma" + BF16MMA = "bf16mma" class RmsnormProjqWqiWeightsConverter(TilertWeightsConverter): @@ -71,7 +72,6 @@ def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: def _swizzle_mma_16x16_for_pages( mat_in: torch.Tensor, q_lora_rank: int, pages: int ) -> torch.Tensor: - """Swizzle a 16xK matrix for the paged weight layout (K divisible by 16).""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == q_lora_rank pre_shape = mat_in.shape[:-2] k_per_page = q_lora_rank // pages @@ -87,7 +87,6 @@ def _common_to_tilert_fp16mma( wqi_scales: torch.Tensor, rmsnorm_gamma: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Convert common weights to the packed TileRT FP16 layout (IQ only).""" sms = 128 k_per_page = 1024 if self.model_args.arch_name == "glm_5" else 512 pages = self.q_lora_dim // k_per_page @@ -128,14 +127,14 @@ def _common_to_tilert_fp16mma( tilert_gamma = rmsnorm_gamma.float().detach().clone() return tilert_wqi, tilert_wqi_scales, tilert_gamma - def convert_to_fp16mma( + def convert_to_bf16mma( self, weights: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Convert common-format weights to TileRT FP16 MMA layout. + return self.convert_to_fp16mma(weights) - Args: - weights: [wqi, wqi_scale, q_norm_weight]. - """ + def convert_to_fp16mma( + self, weights: list[torch.Tensor] + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: with torch.inference_mode(): wqi, wqi_scale, q_norm_weight = weights return self._common_to_tilert_fp16mma(wqi, wqi_scale, q_norm_weight) @@ -177,8 +176,14 @@ class RmsnormProjqWqi(TileRTModule): """RmsnormProjqWqi module: RMSNorm + W_qi projection (IQ only, GLM5 v2).""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [RmsnormProjqWqiAlgorithm.FP16MMA], - "glm_5": [RmsnormProjqWqiAlgorithm.FP16MMA], + "deepseek_v3_2": [ + RmsnormProjqWqiAlgorithm.FP16MMA, + RmsnormProjqWqiAlgorithm.BF16MMA, + ], + "glm_5": [ + RmsnormProjqWqiAlgorithm.FP16MMA, + RmsnormProjqWqiAlgorithm.BF16MMA, + ], } def __init__( @@ -240,7 +245,6 @@ def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, tor } def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize reference weights from common-format state dict.""" self.ref_q_norm = state_dict[self.tilert_weights_alias.rmsnorm_gamma] wqi = weight_dequant( state_dict[self.tilert_weights_alias.wqi_weights], @@ -249,21 +253,19 @@ def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: self.ref_wqi = wqi.contiguous() def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """Initialize TileRT weights from common-format state dict.""" + assert self.algorithm is not None, "Algorithm is not set" weights = [ + state_dict[self.tilert_weights_alias.rmsnorm_gamma], state_dict[self.tilert_weights_alias.wqi_weights], state_dict[self.tilert_weights_alias.wqi_scales], - state_dict[self.tilert_weights_alias.rmsnorm_gamma], ] - assert self.algorithm is not None, "Algorithm is not set" self.tilert_wqi, self.tilert_wqi_scales, self.tilert_q_norm_weight = ( - RmsnormProjqWqiWeightsConverter(self.model_args, self.num_devices).dispatch( - self.algorithm, weights + torch.ops.tilert.rmsnorm_projq_wqi__convert_weights( + weights, self.model_args.arch_name, self.algorithm.value ) ) def init_random_weights(self) -> None: - """Initialize random reference and TileRT weights for testing.""" q_norm = torch.randn(self.q_lora_rank, dtype=torch.float32) wqi = torch.randn(self.index_head_dim, self.q_lora_rank, dtype=torch.bfloat16).to( torch.float8_e4m3fn @@ -281,7 +283,6 @@ def init_random_weights(self) -> None: self.init_tilert_weights(ref_state) def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - """Allocate TileRT output buffers.""" self.iq = torch.zeros( batch_size, seq_len, self.index_n_heads, self.head_dim, dtype=torch.bfloat16 ) @@ -289,7 +290,6 @@ def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: self.is_var_init = True def golden_forward(self, q: torch.Tensor) -> torch.Tensor: - """Reference forward: RMSNorm + W_qi_b linear projection.""" assert self.ref_q_norm is not None assert self.ref_wqi is not None diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqakis.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqakis.py index 8813d6a..2702b42 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqakis.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqakis.py @@ -21,7 +21,7 @@ class RMSNormProjxWqakisWeightsConverter(TilertWeightsConverter): - """Weight converter for RMSNormProjxWqakis.""" + """Weight converter for RMSNormProjxWqakis (decoupled FP8 MMA).""" def __init__(self, model_args: ModelArgs, num_devices: int): super().__init__(model_args, num_devices) @@ -29,14 +29,6 @@ def __init__(self, model_args: ModelArgs, num_devices: int): def convert_to_decoupled( self, weights: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Convert weights to decoupled FP8 MMA format. - - Args: - weights: [gamma, wq_a, wq_a_scale, wki, wki_scale, wis, wis_scale] - - Returns: - (wqaki_packed, wis_bf16, gamma) - """ arch_name = self.model_args.arch_name x_rmsnorm_gamma, wq_a, wq_a_scale, wki, wki_scale, wis, _wis_scale = weights @@ -111,14 +103,21 @@ class RMSNormProjxWqakisAlgorithm(Enum): """RMSNormProjxWqakis algorithm.""" FP8MMA = "fp8mma" + W8A16HMMA = "w8a16_hmma" class RMSNormProjxWqakis(TileRTModule): - """Decoupled RMSNorm + GEMV(W_q_a, W_ki, W_is).""" + """Decoupled RMSNorm + GEMV(W_q_a, W_ki, W_is) for Device Group A.""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [RMSNormProjxWqakisAlgorithm.FP8MMA], - "glm_5": [RMSNormProjxWqakisAlgorithm.FP8MMA], + "deepseek_v3_2": [ + RMSNormProjxWqakisAlgorithm.FP8MMA, + RMSNormProjxWqakisAlgorithm.W8A16HMMA, + ], + "glm_5": [ + RMSNormProjxWqakisAlgorithm.FP8MMA, + RMSNormProjxWqakisAlgorithm.W8A16HMMA, + ], } def __init__( @@ -183,7 +182,6 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_norm_gamma, self.tilert_wqakis, self.tilert_wis] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Repeat weights for device sharding.""" input_layernorm_weight = ( weights_map[self.ref_weights_alias.x_rmsnorm_gamma][None, ...] .float() @@ -235,6 +233,31 @@ def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: tilert_aliases = self.tilert_weights_alias() weights_list = [state_dict[alias] for alias in tilert_aliases] + if self.algorithm == RMSNormProjxWqakisAlgorithm.W8A16HMMA: + gamma, wq_a, wq_a_scale, wki, wki_scale, wis, _wis_scale = weights_list + if self.arch_name == "glm_5": + self.tilert_wqakis = ProjxWqakiWeightsConverter.convert_glm5_68cta_w8a16( + wq_a, wq_a_scale, wki, wki_scale + ) + else: + from tilert.models.glm_5._dsa_v32.ops.rmsnorm_projx_wqkva import ( + RMSNormProjQKVAW8A16MMAWeightsConverter, + ) + + w_fp8 = torch.cat( + [ + wq_a.reshape(self.q_lora_rank, self.dim), + wki.reshape(self.idx_head_dim, self.dim), + ], + dim=0, + ).contiguous() + scales = torch.cat([wq_a_scale, wki_scale], dim=0).to(torch.float32).contiguous() + self.tilert_wqakis = RMSNormProjQKVAW8A16MMAWeightsConverter.pack_lane_major( + w_fp8, scales, self.dim + ) + self.tilert_wis = wis.to(torch.bfloat16) + self.tilert_norm_gamma = gamma.float() + return converter = RMSNormProjxWqakisWeightsConverter(self.model_args, self.num_devices) result = converter.convert_to_decoupled(weights_list) self.tilert_wqakis, self.tilert_wis, self.tilert_norm_gamma = result @@ -280,7 +303,6 @@ def golden_forward( self, x: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Pure PyTorch reference: RMSNorm -> q, ki, idx_scores.""" assert self.ref_norm_gamma is not None assert self.ref_wq_a is not None assert self.ref_wki is not None @@ -302,7 +324,6 @@ def tilert_forward( self, x: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Run RMSNorm + ProjXWqaki + ProjXWis via TileRT CUDA kernels.""" rmsnorm_quant( x.to(torch.bfloat16), self.tilert_norm_gamma, diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py index 5343357..b7bff81 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py @@ -16,7 +16,7 @@ class RMSNormProjQKVAFP8MMAWeightsConverter: - """Weight converter: pack FP8 weights into the kernel's packed layout.""" + """Weight converter: pack FP8 weights into WqkvaPagedShared layout for the FP8 MMA kernel.""" HIDDEN_DIM = 6144 Q_LORA_RANK = 2048 @@ -32,11 +32,9 @@ class RMSNormProjQKVAFP8MMAWeightsConverter: MAT_BYTES = ROWS_PER_CTA * COLS_PER_PAGE SCALE_OFFSET = MAT_BYTES - PAGE_BYTES = ((MAT_BYTES + 128 + 127) // 128) * 128 @staticmethod def _swizzle_mma_16x32(mat_in: torch.Tensor) -> torch.Tensor: - """Swizzle [*, 16, 32] tiles into the packed weight layout.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 32 pre_shape = mat_in.shape[:-2] mat_in = mat_in.reshape(*pre_shape, 2, 8, 2, 4, 4).transpose(-4, -3).transpose(-5, -4) @@ -55,14 +53,7 @@ def convert_to_fp8_mma_gemv( hidden_dim: int = 6144, q_lora_rank: int = 2048, ) -> tuple[torch.Tensor, torch.Tensor]: - """Pack FP8 weights for the FP8 MMA kernel. - - Args: - hidden_dim: Model hidden dimension. - q_lora_rank: Q projection rank. - """ C = RMSNormProjQKVAFP8MMAWeightsConverter - block_size = C.BLOCK_SIZE kv_lora_rank = C.KV_LORA_RANK qk_rope_head_dim = C.QK_ROPE_HEAD_DIM @@ -73,54 +64,92 @@ def convert_to_fp8_mma_gemv( expected = qk_rope_head_dim * hidden_dim assert w_pe.numel() == expected, f"w_pe numel {w_pe.numel()} != expected {expected}" + return C._pack_per_row_no_requant( + wq_a, + wkv_a, + w_pe, + wq_a_scale, + wkv_a_scale, + w_pe_scale, + attn_norm_weight, + hidden_dim=hidden_dim, + q_lora_rank=q_lora_rank, + ) + + @staticmethod + def _pack_per_row_no_requant( + wq_a: torch.Tensor, + wkv_a: torch.Tensor, + w_pe: torch.Tensor, + wq_a_scale: torch.Tensor, + wkv_a_scale: torch.Tensor, + w_pe_scale: torch.Tensor, + attn_norm_weight: torch.Tensor, + *, + hidden_dim: int, + q_lora_rank: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + C = RMSNormProjQKVAFP8MMAWeightsConverter + kv_lora_rank = C.KV_LORA_RANK + qk_rope_head_dim = C.QK_ROPE_HEAD_DIM total_rows = q_lora_rank + kv_lora_rank + qk_rope_head_dim num_ctas = total_rows // C.ROWS_PER_CTA num_pages = hidden_dim // C.COLS_PER_PAGE - wq_a_f = weight_dequant(wq_a.reshape(q_lora_rank, hidden_dim), wq_a_scale) - wkv_a_f = weight_dequant(wkv_a.reshape(kv_lora_rank, hidden_dim), wkv_a_scale) - w_pe_f = weight_dequant(w_pe.reshape(qk_rope_head_dim, hidden_dim), w_pe_scale) - w_float = torch.cat([wq_a_f, wkv_a_f, w_pe_f], dim=0) - - w_blocks = w_float.reshape(total_rows, hidden_dim // block_size, block_size) - col_max = w_blocks.abs().amax(dim=(0, 2)) - fp8_max = torch.finfo(torch.float8_e4m3fn).max - w_scales = (col_max / fp8_max).clamp(min=1e-12) - - scales_expanded = w_scales.repeat_interleave(block_size) - w_scaled = w_float / scales_expanded.unsqueeze(0) - w_fp8 = w_scaled.to(torch.float8_e4m3fn) + num_tiles = C.COLS_PER_PAGE // 32 + blk = C.BLOCK_SIZE + num_blk_page = C.COLS_PER_PAGE // blk + num_blk_total = hidden_dim // blk + + w_fp8 = torch.cat( + [ + wq_a.reshape(q_lora_rank, hidden_dim), + wkv_a.reshape(kv_lora_rank, hidden_dim), + w_pe.reshape(qk_rope_head_dim, hidden_dim), + ], + dim=0, + ).contiguous() + + def _bcast_block_scale(scale: torch.Tensor, rows: int) -> torch.Tensor: + s = scale.to(torch.float32).reshape(-1, num_blk_total) + return s.repeat_interleave(blk, dim=0)[:rows] + + w_scales = torch.cat( + [ + _bcast_block_scale(wq_a_scale, q_lora_rank), + _bcast_block_scale(wkv_a_scale, kv_lora_rank), + _bcast_block_scale(w_pe_scale, qk_rope_head_dim), + ], + dim=0, + ).clamp( + min=1e-12 + ) assert C.MAT_BYTES == C.SCALE_OFFSET, "Layout mismatch: scales must follow mat" - assert block_size == C.COLS_PER_PAGE // C.SCALES_PER_PAGE, "Block size mismatch" - assert w_scales.numel() == num_pages * C.SCALES_PER_PAGE, "Scale count mismatch" w_bytes = w_fp8.view(torch.uint8) - num_tiles = C.COLS_PER_PAGE // 32 - mat = w_bytes.reshape(num_ctas, C.ROWS_PER_CTA, num_pages, C.COLS_PER_PAGE) mat = mat.transpose(1, 2) - mat = mat.reshape(num_ctas, num_pages, 2, 16, num_tiles, 32) mat = mat.transpose(3, 4) mat = C._swizzle_mma_16x32(mat) mat = mat.contiguous().reshape(num_ctas, num_pages, C.MAT_BYTES) - scales_f32 = w_scales.reshape(num_pages, C.SCALES_PER_PAGE).to(torch.float32).contiguous() - scales_bytes = scales_f32.view(torch.uint8) - scales_bytes = scales_bytes.unsqueeze(0).expand(num_ctas, -1, -1) - - pad_size = C.PAGE_BYTES - C.MAT_BYTES - C.SCALES_PER_PAGE * 4 - padding = torch.zeros(num_ctas, num_pages, pad_size, dtype=torch.uint8, device=w_fp8.device) - - packed = torch.cat([mat, scales_bytes, padding], dim=-1) + sc = w_scales.reshape(num_ctas, C.ROWS_PER_CTA, num_pages, num_blk_page) + sc = sc.permute(0, 2, 1, 3).contiguous() + scales_bytes = ( + sc.to(torch.float32) + .reshape(num_ctas, num_pages, C.ROWS_PER_CTA * num_blk_page) + .view(torch.uint8) + ) + packed = torch.cat([mat, scales_bytes], dim=-1) packed = packed.contiguous().reshape(-1) return packed.view(torch.float8_e4m3fn), attn_norm_weight.clone() class RMSNormProjQKVAFP16MMAWeightsConverter: - """Weight converter: pack FP16 weights for the kernel.""" + """Weight converter: pack FP16 weights for the MMA kernel.""" KV_LORA_RANK = 512 QK_ROPE_HEAD_DIM = 64 @@ -130,7 +159,6 @@ class RMSNormProjQKVAFP16MMAWeightsConverter: @staticmethod def _swizzle_mma_16x16(mat_in: torch.Tensor) -> torch.Tensor: - """Swizzle [*, 16, 16] tiles into the packed weight layout.""" assert mat_in.shape[-2] == 16 and mat_in.shape[-1] == 16 pre_shape = mat_in.shape[:-2] mat_in = mat_in.reshape(*pre_shape, 2, 8, 2, 4, 2).transpose(-4, -3).transpose(-5, -4) @@ -149,7 +177,6 @@ def convert_to_fp16_mma_gemv( hidden_dim: int = 6144, q_lora_rank: int = 2048, ) -> tuple[torch.Tensor, torch.Tensor]: - """Pack weights into the FP16 layout expected by the kernel.""" C = RMSNormProjQKVAFP16MMAWeightsConverter kv_lora_rank = C.KV_LORA_RANK qk_rope_head_dim = C.QK_ROPE_HEAD_DIM @@ -182,10 +209,125 @@ def convert_to_fp16_mma_gemv( return packed.view(torch.float16), attn_norm_weight.clone() +class RMSNormProjQKVAW8A16MMAWeightsConverter: + """Pack FP8 weight + block scale into packed format.""" + + KV_LORA_RANK = 512 + QK_ROPE_HEAD_DIM = 64 + ROWS_PER_CTA = 32 + COLS_PER_PAGE = 1024 + BLOCK_SIZE = 128 + NUM_WARPS = 8 + MMA_K = 16 + M_TILES_PER_CTA = ROWS_PER_CTA // 16 + K_TILES_PER_WARP = COLS_PER_PAGE // (NUM_WARPS * MMA_K) + SCALES_PER_PAGE = COLS_PER_PAGE // BLOCK_SIZE + PAGE_MAT_BYTES = M_TILES_PER_CTA * K_TILES_PER_WARP * NUM_WARPS * 32 * 8 + PAGE_BYTES = PAGE_MAT_BYTES + 128 + + @staticmethod + def _permute_mma_a_fragment_16x16(tile: torch.Tensor) -> torch.Tensor: + assert tile.shape[-2:] == (16, 16) + pre = tile.shape[:-2] + return ( + tile.reshape(*pre, 2, 8, 2, 4, 2) + .permute( + *range(len(pre)), + len(pre) + 1, + len(pre) + 3, + len(pre) + 2, + len(pre) + 0, + len(pre) + 4, + ) + .contiguous() + .reshape(*pre, 32, 8) + ) + + @staticmethod + def convert_to_w8a16_mma_gemv( + wq_a: torch.Tensor, + wq_a_scale: torch.Tensor, + wkv_a: torch.Tensor, + wkv_a_scale: torch.Tensor, + w_pe: torch.Tensor, + w_pe_scale: torch.Tensor, + attn_norm_weight: torch.Tensor, + *, + hidden_dim: int = 6144, + q_lora_rank: int = 2048, + ) -> tuple[torch.Tensor, torch.Tensor]: + C = RMSNormProjQKVAW8A16MMAWeightsConverter + kv_lora_rank = C.KV_LORA_RANK + qk_rope_head_dim = C.QK_ROPE_HEAD_DIM + rows_per_cta = C.ROWS_PER_CTA + cols_per_page = C.COLS_PER_PAGE + + w_fp8 = torch.cat( + [ + wq_a.reshape(q_lora_rank, hidden_dim), + wkv_a.reshape(kv_lora_rank, hidden_dim), + w_pe.reshape(qk_rope_head_dim, hidden_dim), + ], + dim=0, + ).contiguous() + assert w_fp8.dtype == torch.float8_e4m3fn, f"expected fp8 weight, got {w_fp8.dtype}" + + scales = ( + torch.cat([wq_a_scale, wkv_a_scale, w_pe_scale], dim=0).to(torch.float32).contiguous() + ) + + total_rows = q_lora_rank + kv_lora_rank + qk_rope_head_dim + num_ctas = total_rows // rows_per_cta + num_pages = hidden_dim // cols_per_page + expected_scale_rows = (total_rows + C.BLOCK_SIZE - 1) // C.BLOCK_SIZE + assert scales.shape == (expected_scale_rows, hidden_dim // C.BLOCK_SIZE), ( + f"scales {tuple(scales.shape)} != " + f"{(expected_scale_rows, hidden_dim // C.BLOCK_SIZE)}" + ) + + del num_ctas, num_pages + return C.pack_lane_major(w_fp8, scales, hidden_dim), attn_norm_weight.clone() + + @classmethod + def pack_lane_major( + cls, w_fp8: torch.Tensor, scales: torch.Tensor, hidden_dim: int + ) -> torch.Tensor: + assert w_fp8.dtype == torch.float8_e4m3fn, f"expected fp8 weight, got {w_fp8.dtype}" + rows_per_cta = cls.ROWS_PER_CTA + cols_per_page = cls.COLS_PER_PAGE + total_rows = w_fp8.shape[0] + num_ctas = total_rows // rows_per_cta + num_pages = hidden_dim // cols_per_page + scales = scales.to(torch.float32).contiguous() + device = w_fp8.device + w_bytes = w_fp8.view(torch.uint8) + + w = w_bytes.reshape(num_ctas, rows_per_cta, num_pages, cols_per_page) + w = w.reshape(num_ctas, cls.M_TILES_PER_CTA, 16, num_pages, cols_per_page) + w = w.permute(0, 3, 1, 2, 4).contiguous() + w = w.reshape( + num_ctas, num_pages, cls.M_TILES_PER_CTA, 16, cls.NUM_WARPS, cls.K_TILES_PER_WARP, 16 + ) + w = w.permute(0, 1, 2, 5, 4, 3, 6).contiguous() + w_lane = cls._permute_mma_a_fragment_16x16(w) + mat_blob = w_lane.contiguous().reshape(num_ctas, num_pages, cls.PAGE_MAT_BYTES) + + cta_idx = torch.arange(num_ctas, device=device) + scale_row = cta_idx // (cls.BLOCK_SIZE // rows_per_cta) + cta_scales = scales[scale_row].reshape(num_ctas, num_pages, cls.SCALES_PER_PAGE) + scale_bytes = cta_scales.contiguous().view(torch.uint8) + + out = torch.zeros(num_ctas, num_pages, cls.PAGE_BYTES, dtype=torch.uint8, device=device) + out[:, :, : cls.PAGE_MAT_BYTES] = mat_blob + out[:, :, cls.PAGE_MAT_BYTES : cls.PAGE_MAT_BYTES + cls.SCALES_PER_PAGE * 4] = scale_bytes + return out.reshape(-1).contiguous().view(torch.float8_e4m3fn) + + class RMSNormProjxWqkvaAlgorithm(Enum): """RMSNormProjxWqkva algorithm.""" DECOUPLED = "decoupled" + W8A16HMMA = "w8a16_hmma" class RMSNormProjxWqkvaWeightsConverter(TilertWeightsConverter): @@ -197,11 +339,6 @@ def __init__(self, model_args: ModelArgs, num_devices: int): def convert_to_fp8_mma_gemv( self, weights: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: - """Convert tilert weights list to the FP8 kernel-ready format. - - Args: - weights: [gamma, wq_a, wq_a_scale, wkv_a, wkv_a_scale, w_pe, w_pe_scale] - """ gamma, wq_a, wq_a_scale, wkv_a, wkv_a_scale, w_pe, w_pe_scale = weights return RMSNormProjQKVAFP8MMAWeightsConverter.convert_to_fp8_mma_gemv( wq_a, @@ -218,11 +355,6 @@ def convert_to_fp8_mma_gemv( def convert_to_fp16_mma_gemv( self, weights: list[torch.Tensor] ) -> tuple[torch.Tensor, torch.Tensor]: - """Convert tilert weights list to the FP16 kernel-ready format. - - Args: - weights: [gamma, wq_a, wq_a_scale, wkv_a, wkv_a_scale, w_pe, w_pe_scale] - """ gamma, wq_a, wq_a_scale, wkv_a, wkv_a_scale, w_pe, w_pe_scale = weights return RMSNormProjQKVAFP16MMAWeightsConverter.convert_to_fp16_mma_gemv( wq_a, @@ -288,11 +420,17 @@ def __call__(self) -> list[str]: class RMSNormProjxWqkva(TileRTModule): - """Fused RMSNorm + GEMV(W_q_a, W_kv_a, W_pe).""" + """Fused RMSNorm + GEMV(W_q_a, W_kv_a, W_pe) for Device Group B.""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [RMSNormProjxWqkvaAlgorithm.DECOUPLED], - "glm_5": [RMSNormProjxWqkvaAlgorithm.DECOUPLED], + "deepseek_v3_2": [ + RMSNormProjxWqkvaAlgorithm.DECOUPLED, + RMSNormProjxWqkvaAlgorithm.W8A16HMMA, + ], + "glm_5": [ + RMSNormProjxWqkvaAlgorithm.DECOUPLED, + RMSNormProjxWqkvaAlgorithm.W8A16HMMA, + ], } def __init__( @@ -354,7 +492,6 @@ def get_weights_list(self) -> list[torch.Tensor]: return [self.tilert_norm_gamma, self.tilert_wqkva, self.tilert_wqkva_scales] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Repeat weights for device sharding.""" input_layernorm_weight = ( weights_map[self.ref_weights_alias.x_rmsnorm_gamma][None, ...] .float() @@ -405,8 +542,27 @@ def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: tilert_aliases = self.tilert_weights_alias() weights_list = [state_dict[alias] for alias in tilert_aliases] - converter = RMSNormProjxWqkvaWeightsConverter(self.model_args, self.num_devices) - self.tilert_wqkva, self.tilert_norm_gamma = converter.convert_to_fp8_mma_gemv(weights_list) + if self.algorithm == RMSNormProjxWqkvaAlgorithm.W8A16HMMA: + gamma, wq_a, wq_a_scale, wkv_a, wkv_a_scale, w_pe, w_pe_scale = weights_list + self.tilert_wqkva, self.tilert_norm_gamma = ( + RMSNormProjQKVAW8A16MMAWeightsConverter.convert_to_w8a16_mma_gemv( + wq_a, + wq_a_scale, + wkv_a, + wkv_a_scale, + w_pe, + w_pe_scale, + gamma.float(), + hidden_dim=self.dim, + q_lora_rank=self.q_lora_rank, + ) + ) + self.tilert_norm_gamma = self.tilert_norm_gamma.float().contiguous() + else: + converter = RMSNormProjxWqkvaWeightsConverter(self.model_args, self.num_devices) + self.tilert_wqkva, self.tilert_norm_gamma = converter.convert_to_fp8_mma_gemv( + weights_list + ) self.tilert_wqkva_scales = torch.zeros((1,), dtype=torch.float32) def init_tilert_vars(self, batch_size: int, seq_len: int, max_len: int = 128) -> None: @@ -437,9 +593,9 @@ def init_random_weights(self) -> None: tensor_list = [ torch.randn(self.dim, dtype=torch.float32), torch.randn(self.q_lora_rank, self.dim, dtype=torch.bfloat16).to(torch.float8_e4m3fn), - torch.randn(q_scale_dim, dim_scale_dim, dtype=scale_dtype), + torch.randn(q_scale_dim, dim_scale_dim, dtype=scale_dtype).abs(), torch.randn(kv_mqa_rows, self.dim, dtype=torch.bfloat16).to(torch.float8_e4m3fn), - torch.randn(kv_mqa_scale_dim, dim_scale_dim, dtype=scale_dtype), + torch.randn(kv_mqa_scale_dim, dim_scale_dim, dtype=scale_dtype).abs(), ] ref_state_dict = dict(zip(self.ref_weights_alias(), tensor_list)) self.init_reference_weights(ref_state_dict) @@ -475,7 +631,6 @@ def tilert_forward( x: torch.Tensor, cur_pos: int = 0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Run RMSNorm + 3-way GEMV via the TileRT CUDA kernels.""" assert self.cur_pos is not None assert self.pe_cache_out is not None self.cur_pos.fill_(cur_pos) diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_quant.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_quant.py index 1d399c5..c977d08 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_quant.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_quant.py @@ -27,17 +27,6 @@ def rmsnorm_quant( *, model_arch: str, ) -> None: - """ - Rmsnorm with optional activation quantization. - - Args: - hidden_in: Input tensor (..., dim). - gamma_in: RMSNorm gamma (dim,). - hidden_out: RMSNorm output (..., dim). - quant_hidden_out: Optional quantized output (..., dim). If None, no quant. - quant_hidden_scale_out: Optional quant scale (..., dim // block_size). If None, no quant. - profile_logs: Optional profile logs tensor. - """ if profile_logs is None: raise ValueError("profile_logs is required when calling rmsnorm_quant.") diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_up_gate_silu.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_up_gate_silu.py index 25adae9..1792aec 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_up_gate_silu.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_up_gate_silu.py @@ -13,6 +13,9 @@ ExpertSelectUpGateSiLU, ExpertSelectUpGateSiLUWeightsConverter, ) +from tilert.models.glm_5._dsa_v32.ops.rmsnorm_projx_wqkva import ( + RMSNormProjQKVAW8A16MMAWeightsConverter, +) from tilert.utils import get_profile_log_tensor __all__ = [ @@ -49,6 +52,63 @@ class RMSNormUpGateSiLUAlgorithm(Enum): FP8MMA = "fp8mma" FP16MMA = "fp16mma" + BF16MMA = "bf16mma" + BF16MMA_V2 = "w8a16_hmma_v2" + + +def _pack_up_gate_w8a16_v2( + gate_w: torch.Tensor, + gate_s: torch.Tensor, + up_w: torch.Tensor, + up_s: torch.Tensor, + hidden_dim: int = 6144, +) -> torch.Tensor: + """Pack GLM5 dense gate/up FP8 weight + block scales into the V2 blob.""" + C = RMSNormProjQKVAW8A16MMAWeightsConverter + rows_per_cta = C.ROWS_PER_CTA + cols_per_page = C.COLS_PER_PAGE + block = C.BLOCK_SIZE + num_warps = C.NUM_WARPS + k_tiles = C.K_TILES_PER_WARP + scales_per_set = C.SCALES_PER_PAGE + page_mat_bytes = C.PAGE_MAT_BYTES + page_bytes = C.PAGE_BYTES + + gate_w = gate_w.reshape(-1, hidden_dim).contiguous() + up_w = up_w.reshape(-1, hidden_dim).contiguous() + assert gate_w.dtype == torch.float8_e4m3fn and up_w.dtype == torch.float8_e4m3fn + inter = gate_w.shape[0] + num_ctas = inter // 16 + num_pages = hidden_dim // cols_per_page + device = gate_w.device + + gate_s = gate_s.reshape(-1, hidden_dim // block).to(torch.float32).contiguous() + up_s = up_s.reshape(-1, hidden_dim // block).to(torch.float32).contiguous() + + gate_c = gate_w.reshape(num_ctas, 16, hidden_dim) + up_c = up_w.reshape(num_ctas, 16, hidden_dim) + w = torch.cat([gate_c, up_c], dim=1).contiguous() + w_bytes = w.view(torch.uint8) + + w = w_bytes.reshape(num_ctas, rows_per_cta, num_pages, cols_per_page) + w = w.reshape(num_ctas, C.M_TILES_PER_CTA, 16, num_pages, cols_per_page) + w = w.permute(0, 3, 1, 2, 4).contiguous() + w = w.reshape(num_ctas, num_pages, C.M_TILES_PER_CTA, 16, num_warps, k_tiles, 16) + w = w.permute(0, 1, 2, 5, 4, 3, 6).contiguous() + w_lane = C._permute_mma_a_fragment_16x16(w) + mat_blob = w_lane.contiguous().reshape(num_ctas, num_pages, page_mat_bytes) + + cta_idx = torch.arange(num_ctas, device=device) + scale_row = cta_idx // (block // 16) + gate_cta = gate_s[scale_row].reshape(num_ctas, num_pages, scales_per_set) + up_cta = up_s[scale_row].reshape(num_ctas, num_pages, scales_per_set) + scale16 = torch.cat([gate_cta, up_cta], dim=2).contiguous() + scale_bytes = scale16.view(torch.uint8) + + out = torch.zeros(num_ctas, num_pages, page_bytes, dtype=torch.uint8, device=device) + out[:, :, :page_mat_bytes] = mat_blob + out[:, :, page_mat_bytes : page_mat_bytes + 16 * 4] = scale_bytes + return out.reshape(-1).contiguous().view(torch.float8_e4m3fn) RMSNormUpGateSiLUWeightsConverter = ExpertSelectUpGateSiLUWeightsConverter @@ -83,8 +143,15 @@ class RMSNormUpGateSiLU(TileRTModule): """RMSNormUpGateSiLU module""" _SUPPORTED_ALGORITHMS = { - "deepseek_v3_2": [RMSNormUpGateSiLUAlgorithm.FP8MMA, RMSNormUpGateSiLUAlgorithm.FP16MMA], - "glm_5": [RMSNormUpGateSiLUAlgorithm.FP8MMA, RMSNormUpGateSiLUAlgorithm.FP16MMA], + "deepseek_v3_2": [ + RMSNormUpGateSiLUAlgorithm.FP8MMA, + RMSNormUpGateSiLUAlgorithm.FP16MMA, + RMSNormUpGateSiLUAlgorithm.BF16MMA, + ], + "glm_5": [ + RMSNormUpGateSiLUAlgorithm.BF16MMA, + RMSNormUpGateSiLUAlgorithm.BF16MMA_V2, + ], } def __init__( @@ -146,12 +213,6 @@ def tilert_tensor_alias(self) -> list[str]: return self.tilert_weights_alias() def get_weights_list(self) -> list[torch.Tensor]: - """ - Get the weights list. - - Returns: - List of weights. - """ return [self.tilert_norm_gamma, self.tilert_weights, self.tilert_scales] def device_sharding( @@ -159,15 +220,6 @@ def device_sharding( weights_dict: dict[str, torch.Tensor], key_prefix: str, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Device sharding. - - Args: - weights_dict: Dictionary of weights. - - Returns: - Tuple of weights. - """ rmsnorm_gamma_key = f"{key_prefix}.post_attention_layernorm.weight" if ".mlp" in key_prefix: key_prefix_without_mlp = key_prefix.replace(".mlp", "") @@ -210,13 +262,6 @@ def init_reference_weights( key_prefix: str, device_id: int = 0, ) -> None: - """ - Initialize the reference weights. - - Args: - state_dict: State dictionary. - device_id: Device ID. - """ sharded_list = self.device_sharding(state_dict, key_prefix) gamma = sharded_list[0][device_id] @@ -237,25 +282,23 @@ def init_reference_weights( self.ref_up = torch.stack(ref_up_list, dim=0) def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Initialize the tilert weights. - - Args: - state_dict: State dictionary. - """ assert self.algorithm is not None, "Algorithm is not set" + aliases = self.tilert_weights_alias() + if self.algorithm == RMSNormUpGateSiLUAlgorithm.BF16MMA_V2: + self.tilert_norm_gamma = state_dict[aliases[0]].float().contiguous() + self.tilert_weights = _pack_up_gate_w8a16_v2( + state_dict[aliases[1]], + state_dict[aliases[2]], + state_dict[aliases[3]], + state_dict[aliases[4]], + hidden_dim=self.dim, + ) + return self.tilert_norm_gamma, self.tilert_weights = RMSNormUpGateSiLUWeightsConverter( self.model_args, self.num_devices - ).dispatch(self.algorithm, [state_dict[alias] for alias in self.tilert_weights_alias()]) + ).dispatch(self.algorithm, [state_dict[alias] for alias in aliases]) def init_tilert_vars(self, batch_size: int, seq_len: int, dev_id: int = 0) -> None: - """ - Initialize the tilert variables. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ self.hidden_out = torch.zeros( ( batch_size, @@ -270,13 +313,9 @@ def init_tilert_vars(self, batch_size: int, seq_len: int, dev_id: int = 0) -> No self.profile_logs = get_profile_log_tensor(device=f"cuda:{dev_id}") self.is_init = True - def init_random_weights(self, dev_id: int = 0) -> None: - """ - Initialize the random weights. - - Returns: - None - """ + def init_random_weights(self, dev_id: int | None = None) -> None: + if dev_id is None: + dev_id = self.device_id gamma = torch.randn(self.dim, dtype=torch.float32, device=f"cuda:{dev_id}") gate_weights = torch.randn( self.inter_dim, self.dim, dtype=torch.bfloat16, device=f"cuda:{dev_id}" diff --git a/tilert/models/glm_5/_dsa_v32/ops/rotate.py b/tilert/models/glm_5/_dsa_v32/ops/rotate.py index 10a46f1..9656acd 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rotate.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rotate.py @@ -65,21 +65,12 @@ def rotate( profile_logs: torch.Tensor, model_arch: str, compute_kernel_type: str = "general", + kv_cache: torch.Tensor | None = None, + cur_pos: torch.Tensor | None = None, + cache_base: int = 0, + cache_stride: int = 0, + cache_compressed: bool = False, ) -> None: - """ - Rotate (hadamard transform) operation. - - Args: - input_raw (torch.Tensor): The input tensor [..., head, 128]. - output_raw (torch.Tensor): The output tensor where the result will be stored. - freqs_cis_raw (torch.Tensor): The frequency tensor. - profile_logs (torch.Tensor): Tensor for storing profiling logs. - model_arch: Model architecture string. - compute_kernel_type: Compute kernel type string. - - Returns: - None - """ torch.ops.tilert.rotate_op( input_raw, output_raw, @@ -87,6 +78,11 @@ def rotate( model_arch, compute_kernel_type, profile_logs, + kv_cache, + cur_pos, + cache_base, + cache_stride, + cache_compressed, ) @@ -121,11 +117,7 @@ class RotateAlgorithm(Enum): class Rotate(TileRTModule): - """Rotate module: RoPE on first qk_rope_head_dim dims + hadamard transform. - - Unified for deepseek_v3_2 (index_n_heads=64) and glm_5 (index_n_heads=32). - No weights; uses model_args for dimensions. - """ + """Rotate module: RoPE on first qk_rope_head_dim dims + hadamard transform.""" _SUPPORTED_ALGORITHMS = { "deepseek_v3_2": [RotateAlgorithm.GENERAL], @@ -193,7 +185,7 @@ def golden_forward( [self.qk_rope_head_dim, self.index_head_dim - self.qk_rope_head_dim], dim=-1, ) - q_pe_idx = apply_rotary_emb(q_pe_idx, freqs_cis) + q_pe_idx = apply_rotary_emb(q_pe_idx, freqs_cis, interleaved=False) idx_q = torch.cat([q_pe_idx, q_nope_idx], dim=-1) return rotate_activation(idx_q) diff --git a/tilert/models/glm_5/_dsa_v32/ops/sparse_index.py b/tilert/models/glm_5/_dsa_v32/ops/sparse_index.py deleted file mode 100644 index ca69c49..0000000 --- a/tilert/models/glm_5/_dsa_v32/ops/sparse_index.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Sparse index operation module.""" - -import torch - -__all__ = [ - "sparse_index", - "sparse_index_topk", -] - - -def sparse_index( - q: torch.Tensor, # noqa: VNE001 - kv: torch.Tensor, - weights: torch.Tensor, - logits: torch.Tensor, - cur_pos: int, - profile_logs: torch.Tensor, - compute_kernel_type: str = "bf16", - *, - model_arch: str, -) -> None: - """ - Sparse index operation. - - Calculate sparse index using q * kv * weights. - - Args: - q (torch.Tensor): The query tensor. - kv (torch.Tensor): The key-value tensor. - weights (torch.Tensor): The weights tensor. - logits (torch.Tensor): The logits tensor. - cur_pos (int): The position of the first token. - profile_logs (torch.Tensor): Tensor for storing profiling logs. - compute_kernel_type (str): Kernel type ("bf16"). - model_arch (str): Model architecture ("deepseek_v3_2"). - - Returns: - None - """ - if q.dtype != torch.bfloat16: - raise ValueError("input must be a bfloat16 tensor.") - if kv.dtype != torch.bfloat16: - raise ValueError("kv must be a bfloat16 tensor.") - if weights.dtype != torch.bfloat16: - raise ValueError("weights must be a bfloat16 tensor.") - if logits.dtype != torch.float32: - raise ValueError("logits must be a float32 tensor.") - - head = q.shape[-2] - dim = q.shape[-1] - - if head != 64 and head != 32: - raise ValueError( - f"Unsupported head size: {head}. Sparse index op currently only \ - supports a head number of 64 or 32." - ) - if dim != 128: - raise ValueError("dim must be 128, as we precompute scale inner kernel") - - device = q.device - if any(t.device != device for t in (kv, weights, logits, profile_logs)): - raise ValueError( - "sparse_index inputs must be on the same device: " - f"q={device}, kv={kv.device}, weights={weights.device}, " - f"logits={logits.device}, profile_logs={profile_logs.device}" - ) - if model_arch == "deepseek_v3_2" and head == 32: - model_arch = "glm_5" - torch.ops.tilert.sparse_index_op( - q, kv, weights, logits, cur_pos, model_arch, compute_kernel_type, profile_logs - ) - - -def sparse_index_topk( - q: torch.Tensor, # noqa: VNE001 - kv: torch.Tensor, - weights: torch.Tensor, - logits: torch.Tensor, - indices: torch.Tensor, - cur_pos: int, - profile_logs: torch.Tensor, -) -> None: - """ - Sparse index operation. - - Calculate sparse index using q * kv * weights. - - Args: - q (torch.Tensor): The query tensor. - kv (torch.Tensor): The key-value tensor. - weights (torch.Tensor): The weights tensor. - logits (torch.Tensor): The logits tensor. - cur_pos (int): The position of the first token. - profile_logs (torch.Tensor): Tensor for storing profiling logs. - - Returns: - None - """ - if q.dtype != torch.bfloat16: - raise ValueError("input must be a bfloat16 tensor.") - if kv.dtype != torch.bfloat16: - raise ValueError("kv must be a bfloat16 tensor.") - if weights.dtype != torch.bfloat16: - raise ValueError("weights must be a bfloat16 tensor.") - if logits.dtype != torch.float32: - raise ValueError("logits must be a float32 tensor.") - - seqlen = q.shape[-3] - head = q.shape[-2] - dim = q.shape[-1] - - if head not in (32, 64): - raise ValueError( - f"Unsupported head size: {head}. Sparse index topk fused op " - "supports head number of 32 (GLM5) or 64 (DSV3.2)." - ) - if dim != 128: - raise ValueError("dim must be 128, as we precompute scale inner kernel") - - device = q.device - if any(t.device != device for t in (kv, weights, logits, indices, profile_logs)): - raise ValueError( - "sparse_index inputs must be on the same device: " - f"q={device}, kv={kv.device}, weights={weights.device}, " - f"logits={logits.device}, profile_logs={profile_logs.device}" - ) - workspace = torch.zeros(seqlen, (200 * 1024 + 260), dtype=torch.int32, device=device) - if head == 64: - torch.ops.tilert.sparse_index_topk_dsv32_op( - q, kv, weights, logits, cur_pos, indices, workspace, profile_logs - ) - else: - torch.ops.tilert.sparse_index_topk_glm5_op( - q, kv, weights, logits, cur_pos, indices, workspace, profile_logs - ) diff --git a/tilert/models/glm_5/_dsa_v32/ops/topk.py b/tilert/models/glm_5/_dsa_v32/ops/topk.py deleted file mode 100644 index bb9dfbb..0000000 --- a/tilert/models/glm_5/_dsa_v32/ops/topk.py +++ /dev/null @@ -1,168 +0,0 @@ -"""topk operations module.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import torch -import torch.nn as nn - -from tilert.utils import get_profile_log_tensor - -if TYPE_CHECKING: - from tilert.models.glm_5._dsa_v32.model_args import ModelArgs - - -__all__ = [ - "TopK", - "topk_approximate", - "topk_accurate", -] - - -def topk_approximate( - logits: torch.Tensor, - seq_len: int, - topk: int, - profile_logs: torch.Tensor, - model_arch: str, - compute_kernel_type: str = "general", -) -> torch.Tensor: - """ - Topk approximate operation. - - Topk approximate the input tensor `logits` and stores the result in `output_raw`. - - Args: - logits (torch.Tensor): The input tensor. - seq_len (int): valid data of logits.shape[-1] - topk (int): The number of topk to approximate. - profile_logs (torch.Tensor): The profile logs tensor. - - Returns: - indices (torch.Tensor): The output tensor. - """ - if logits.dtype != torch.float32: - raise ValueError("logits must be a float32 tensor.") - - if topk != 2048: - raise ValueError("topk must be 2048.") - batch = logits.shape[0] - if batch != 1: - raise ValueError("batch must be 1 in this version") - - indices = torch.zeros(batch, topk, dtype=torch.int32, device=logits.device) - torch.ops.tilert.topk_approximate_op( - logits, indices, seq_len, model_arch, compute_kernel_type, profile_logs - ) - - return indices - - -def topk_accurate( - logits: torch.Tensor, - seq_len: int, - topk: int, - profile_logs: torch.Tensor, - model_arch: str, - compute_kernel_type: str = "general", -) -> torch.Tensor: - """ - Topk approximate operation. - - Topk approximate the input tensor `logits` and stores the result in `output_raw`. - - Args: - logits (torch.Tensor): The input tensor. - seq_len (int): length of last samples, - for k=logits.shape[1] samples, the length is - seq-k+1, seq-k+2, ..., seq-1, seq - topk (int): The number of topk to approximate. - profile_logs (torch.Tensor): The profile logs tensor. - Returns: - indices (torch.Tensor): The output tensor. - """ - if logits.dtype != torch.float32: - raise ValueError("logits must be a float32 tensor.") - - if topk not in (512, 2048): - raise ValueError("topk must be 512 or 2048.") - - assert logits.shape[0] == 1, "batch must be 1 in this version" - num_samples = logits.shape[1] - - indices = torch.zeros(num_samples, topk, dtype=torch.int32, device=logits.device) - indices_ws = torch.zeros(1, num_samples, 4, topk * 2, dtype=torch.int32, device=logits.device) - torch.ops.tilert.topk_accurate_op( - logits, - indices, - seq_len - num_samples, - indices_ws, - model_arch, - compute_kernel_type, - profile_logs, - ) - - return indices - - -class TopK(nn.Module): - """TopK operation with optional approximate kernel. - - Wraps topk_accurate / topk_approximate and provides golden_forward - (reference implementation) and tilert_forward (TileRT kernel). - """ - - def __init__(self, use_approximate: bool = False, model_args: ModelArgs | None = None) -> None: - super().__init__() - self.use_approximate = use_approximate - if model_args is None: - from tilert.models.glm_5._dsa_v32.model_args import ModelArgs - - model_args = ModelArgs() - self.model_args = model_args - - def golden_forward( - self, - logits: torch.Tensor, - topk: int, - ) -> torch.Tensor: - """Reference forward: torch.topk on the last dimension. - - Args: - logits: Scores tensor, shape (batch, ..., seq_len). - topk: Number of top indices to return. - - Returns: - Indices of top-k values along the last dimension. - """ - seq_len = logits.shape[-1] - return logits.topk(min(topk, seq_len), dim=-1)[1] - - def tilert_forward( - self, - logits: torch.Tensor, - topk: int, - ) -> torch.Tensor: - """Tilert forward: batch of samples with varying valid length. - - Args: - logits: Shape (batch, num_samples, cache_len). - topk: Number of top indices to return. - - Returns: - Indices tensor of shape (batch, num_samples, topk). - """ - profile_logs = get_profile_log_tensor(device=logits.device) - cache_len = logits.shape[-1] - if self.use_approximate: - indices = topk_approximate( - logits, cache_len, topk, profile_logs, model_arch=self.model_args.arch_name - ) - else: - indices = topk_accurate( - logits, cache_len, topk, profile_logs, model_arch=self.model_args.arch_name - ) - if indices.dim() == 2: - return indices.unsqueeze(0) - return indices diff --git a/tilert/models/glm_5/_dsa_v32/ops/unproj_o_allreduce.py b/tilert/models/glm_5/_dsa_v32/ops/unproj_o_allreduce.py index 257acf5..d1ed826 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/unproj_o_allreduce.py +++ b/tilert/models/glm_5/_dsa_v32/ops/unproj_o_allreduce.py @@ -31,20 +31,6 @@ def unproj_o_allreduce( model_arch: str, compute_kernel_type: str = "bf16", ) -> None: - """ - Fused operation of unprojection and allreduce. - - Args: - vec_in: Input tensor. - mat_in: Input tensor. - mat_scale: Input tensor. - x_in: Input tensor. - flag: Input flag. - vec_out: Output tensor. - profile_logs: Profile logs tensor. - model_arch: Model architecture ("deepseek_v3_2" or "glm_5"). - compute_kernel_type: Compute kernel type ("bf16", "fp16mma"). - """ torch.ops.tilert.unproj_o_allreduce_op( vec_in, mat_in, @@ -62,6 +48,7 @@ class UnProjOAllReduceAlgorithm(Enum): """UnprojOAllReduce algorithm""" FP16MMA = "fp16mma" + BF16MMA = "bf16mma" @dataclass @@ -108,7 +95,6 @@ def convert_to_fp16mma_128cta( self, weights_list: list[torch.Tensor], ) -> tuple[torch.Tensor, torch.Tensor]: - """Convert weights to the packed kernel layout (GLM5 or DSV3.2).""" with torch.inference_mode(): mat, scales = weights_list if scales.dtype != torch.float32: @@ -186,6 +172,12 @@ def convert_to_fp16mma_128cta( dummy_scales = torch.zeros(1, dtype=torch.float32, device=mat.device) return mat_all, dummy_scales + def convert_to_bf16mma( + self, + weights_list: list[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + return self.convert_to_fp16mma(weights_list) + def convert_to_fp16mma( self, weights_list: list[torch.Tensor], @@ -254,9 +246,11 @@ class UnProjOAllReduce(TileRTModule): _SUPPORTED_ALGORITHMS = { "deepseek_v3_2": [ UnProjOAllReduceAlgorithm.FP16MMA, + UnProjOAllReduceAlgorithm.BF16MMA, ], "glm_5": [ UnProjOAllReduceAlgorithm.FP16MMA, + UnProjOAllReduceAlgorithm.BF16MMA, ], } @@ -314,24 +308,9 @@ def __init__( self.is_var_init = False def get_weights_list(self) -> list[torch.Tensor]: - """ - Get the weights list. - - Returns: - List of weights. - """ return [self.tilert_weights, self.tilert_scales] def device_sharding(self, weights_map: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """ - Device sharding. - - Args: - weights_map: Map from ref weight alias to tensor (full model). - - Returns: - Map from tilert weight alias to (num_devices, ...) tensors. - """ unproj_o_weight = weights_map[self.ref_weights_alias.o_proj_weight] unproj_o_scale = weights_map[self.ref_weights_alias.o_proj_scale_inv] @@ -393,13 +372,6 @@ def init_reference_weights( state_dict: dict[str, torch.Tensor], device_id: int | None = None, ) -> None: - """ - Initialize the reference weights. - - Args: - state_dict: State dictionary keyed by ref weight alias (full model). - device_id: Device ID for this shard; defaults to self.device_id. - """ did = self.device_id if device_id is None else device_id sharded = self.device_sharding(state_dict) weights = sharded[self.tilert_weights_alias.unproj_weights][did] @@ -407,12 +379,6 @@ def init_reference_weights( self.ref_unproj_o = weight_dequant(weights, scales) def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Initialize the tilert weights. - - Args: - state_dict: State dictionary keyed by tilert weight alias (per-device). - """ assert self.algorithm is not None, "Algorithm is not set" self.tilert_weights, self.tilert_scales = UnProjOAllReduceWeightsConverter( self.model_args, self.num_devices @@ -422,13 +388,6 @@ def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: ) def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: - """ - Initialize the tilert variables. - - Args: - batch_size: Batch size. - seq_len: Sequence length. - """ self.hidden_out = torch.zeros( (batch_size, seq_len, self.dim), dtype=torch.bfloat16, @@ -438,7 +397,6 @@ def init_tilert_vars(self, batch_size: int, seq_len: int) -> None: self.is_var_init = True def init_random_weights(self) -> None: - """Initialize the random weights.""" unproj_o_weights = torch.randn( self.dim, self.n_heads * self.head_dim, @@ -469,15 +427,6 @@ def golden_forward( self, vec_in: torch.Tensor, ) -> torch.Tensor: - """ - Forward pass for the down-project module. - - Args: - vec_in: Input vector. - - Returns: - Output tensor. - """ assert self.ref_unproj_o is not None bsz = vec_in.shape[0] seq_len = vec_in.shape[1] diff --git a/tilert/models/glm_5/generator.py b/tilert/models/glm_5/generator.py index b3e8ddd..25ee616 100644 --- a/tilert/models/glm_5/generator.py +++ b/tilert/models/glm_5/generator.py @@ -4,13 +4,12 @@ import time import torch -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerFast from tilert import logger -from tilert.models.glm_5._dsa_v32.generator import stats_time from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.end2end import ShowHandsDSALayer -from tilert.models.glm_5._dsa_v32.temp_var_indices import Idx +from tilert.models.glm_5.modules.end2end import ShowHandsDSALayer +from tilert.models.glm_5.temp_var_indices import Idx from tilert.tilert_init import tilert_init __all__ = [ @@ -34,18 +33,7 @@ def __init__( enable_thinking: bool = False, sampling_seed: int = 42, ): - """Initialize the ShowHandsGeneratorGlm5. - - Args: - max_new_tokens: Maximum number of new tokens to generate. Defaults to 100. - temperature: Temperature for sampling. Defaults to 1.0. - model_weights_dir: Path of the model weights directory. - with_mtp: Whether to use MTP (Multi-Token Prediction) for speculative decoding. - top_p: Top-p (nucleus) sampling threshold. Defaults to 0.9. - top_k: Top-k sampling threshold. Defaults to 256. - use_topp: Whether to use top-p sampling. Defaults to False (top-1 argmax). - enable_thinking: Whether to enable thinking mode in chat template. - """ + """Initialize the ShowHandsGeneratorGlm5.""" torch.set_num_threads(64) self.model_weights_dir = model_weights_dir @@ -56,9 +44,14 @@ def __init__( self.sampling_seed = sampling_seed self.config = model_args - self.tokenizer = AutoTokenizer.from_pretrained( - self.model_weights_dir, trust_remote_code=True - ) # nosec B615 + try: + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_weights_dir, trust_remote_code=True + ) # nosec B615 + except (ValueError, KeyError): + self.tokenizer = PreTrainedTokenizerFast.from_pretrained( + self.model_weights_dir, trust_remote_code=True + ) # nosec B615 jinja_file_path = os.path.join(self.model_weights_dir, "chat_template.jinja") with open(jinja_file_path, encoding="utf-8") as f: chat_template = f.read() @@ -89,6 +82,7 @@ def __init__( model_args=self.config, model_path=self.model_weights_dir, with_mtp=with_mtp, + temperature=temperature, top_p=top_p, top_k=top_k, use_topp=use_topp, @@ -111,12 +105,8 @@ def from_pretrained(self) -> None: self.decode_layer.from_pretrained(self.model_weights_dir) def extract_ffn_cache(self) -> tuple[dict[int, list], dict[int, set[str]]]: - """Extract MOE/MLP op objects and skip keys from current loaded weights. - - Returns: - Tuple of (cached_ffn_ops_per_device, skip_keys_per_device). - """ - from tilert.models.glm_5._dsa_v32.modules.end2end import ( + """Extract MOE/MLP op objects and skip keys from current loaded weights.""" + from tilert.models.glm_5.modules.end2end import ( _extract_ffn_ops, _get_moe_weight_keys, ) @@ -162,20 +152,7 @@ def generate( with_mtp: bool | None = None, prompt_tokens: list[int] | None = None, ) -> tuple[str, list[float], list[int], int]: - """Main function to load the model and perform single sequence generation. - - Args: - prompt: The input prompt string. - print_log: Whether to print generation logs. - with_mtp: Override MTP mode for this call. None uses self.with_mtp. - Requires MTP weights to have been loaded (self.with_mtp=True). - prompt_tokens: Pre-tokenized prompt tokens. If provided, skip tokenization - and use these tokens directly (useful for exact-length benchmarking). - - Returns: - Tuple of (result_text, time_list, accepted_counts, prompt_len). - accepted_counts is empty for non-MTP mode. - """ + """Main function to load the model and perform single sequence generation.""" active_mtp = with_mtp if with_mtp is not None else self.with_mtp if active_mtp and not self.with_mtp: raise ValueError("Cannot use MTP mode: MTP weights were not loaded") @@ -183,7 +160,7 @@ def generate( if active_mtp: return self._generate_with_mtp(prompt, print_log, prompt_tokens=prompt_tokens) result, time_list, prompt_len = self._generate_without_mtp( - prompt, print_log, with_mtp=active_mtp, prompt_tokens=prompt_tokens + prompt, print_log, prompt_tokens=prompt_tokens ) return result, time_list, [], prompt_len @@ -191,10 +168,9 @@ def _generate_without_mtp( self, prompt: str, print_log: bool = True, - with_mtp: bool = False, prompt_tokens: list[int] | None = None, ) -> tuple[str, list[float], int]: - """Standard generation without MTP.""" + """Standard generation without MTP (unified single-op decode).""" if prompt_tokens is None: messages = [{"role": "user", "content": prompt}] prompt_tokens = self.tokenizer.apply_chat_template( @@ -202,6 +178,7 @@ def _generate_without_mtp( tokenize=True, add_generation_prompt=True, enable_thinking=self.enable_thinking, + return_dict=False, ) max_seq_len = self.config.max_seq_len @@ -214,50 +191,65 @@ def _generate_without_mtp( tokens[0, :prompt_len] = torch.tensor( prompt_tokens, dtype=torch.long, device=self.default_device ) - prompt_mask = tokens != -1 - prev_pos = 0 - finished = torch.tensor( - [False] * self.batch_size, dtype=torch.bool, device=self.default_device - ) + ar_steps = max(1, min(1024, int(os.environ.get("GLM5_AR_N", "8")))) + return self._decode_without_mtp_ar(tokens, prompt_len, total_len, ar_steps, print_log) - time_list = [] - for cur_pos_val in range(1, total_len): - start_time = time.time() - multi_devices_results = self.decode_layer.forward( - tokens[0, prev_pos], with_mtp=with_mtp - ) - end_time = time.time() - time_list.append(end_time - start_time) + def _decode_without_mtp_ar( + self, + tokens: torch.Tensor, + prompt_len: int, + total_len: int, + ar_steps: int, + print_log: bool, + ) -> tuple[str, list[float], int]: + """w/o-MTP decode (unified single-op, unified-style).""" + time_list: list[float] = [] - intermediates, *_ = multi_devices_results[0] - next_token = intermediates[Idx.TOKEN_OUT][0][0] + self.decode_layer.set_prefill_valid_tokens(1, with_mtp=False) + for prev_pos in range(prompt_len - 1): + self.decode_layer.forward(tokens[0, prev_pos], with_mtp=False) + self.decode_layer.set_prefill_valid_tokens(0, with_mtp=False) - next_token = torch.where( - prompt_mask[0, cur_pos_val], tokens[0, cur_pos_val], next_token - ) - tokens[0, cur_pos_val] = next_token - is_stop_token = next_token.item() in self.stop_token_ids - finished |= torch.logical_and( - ~prompt_mask[0, cur_pos_val], - torch.tensor(is_stop_token, dtype=torch.bool, device=self.default_device), - ) - prev_pos = cur_pos_val - if cur_pos_val >= prompt_len: - decoded_tokens = self.tokenizer.decode( - [next_token.item()], skip_special_tokens=True - ) - if print_log: - print(decoded_tokens, end="", flush=True) + cur_pos = prompt_len - 1 + prev_token = tokens[0, prompt_len - 1].reshape(1).to(torch.int32) + finished = False + while cur_pos < total_len - 1 and not finished: + start_time = time.time() + self.decode_layer.show_hands_no_mtp(prev_token, ar_steps) + elapsed = time.time() - start_time - if finished.all(): - break + acc = self.decode_layer.ar_accepted_tokens_no_mtp(0).cpu() + n_tokens = int(acc[0].item()) + emitted = acc[1 : 1 + n_tokens].tolist() + per_step_time = elapsed / max(1, n_tokens) + + last_tok = int(prev_token[0].item()) + for tok in emitted: + if cur_pos + 1 >= total_len: + break + tokens[0, cur_pos + 1] = tok + cur_pos += 1 + last_tok = tok + if cur_pos >= prompt_len and print_log: + print( + self.tokenizer.decode([tok], skip_special_tokens=True), + end="", + flush=True, + ) + time_list.append(per_step_time) + if tok in self.stop_token_ids: + finished = True + break + prev_token = torch.tensor([last_tok], dtype=torch.int32, device=self.default_device) if print_log: print("\n") logger.info(f"--Number of tokens generated: {len(time_list)}") - - stats_time(time_list, "==== Performance ====") + if time_list: + total_t = sum(time_list) + tps = len(time_list) / total_t if total_t > 0 else 0 + logger.info(f"--Effective TPS (AR, ar_steps={ar_steps}): {tps:.2f} tokens/s") print("\n") self.decode_layer.reset_sequence() @@ -267,14 +259,12 @@ def _generate_without_mtp( toks = toks[prompt_len : prompt_len + self.max_new_tokens] stop_idx = len(toks) for i, tok in enumerate(toks): - if tok in self.stop_token_ids: + if tok == -1 or tok in self.stop_token_ids: stop_idx = i break toks = toks[:stop_idx] completion_tokens.append(toks) - decoded_tokens = self.tokenizer.batch_decode(completion_tokens, skip_special_tokens=True) - return f"{decoded_tokens[0]}\n" if decoded_tokens else "", time_list, prompt_len def _generate_with_mtp( @@ -287,8 +277,10 @@ def _generate_with_mtp( if prompt_tokens is None: prompt_tokens = self.tokenizer.apply_chat_template( [{"role": "user", "content": prompt}], + tokenize=True, add_generation_prompt=True, enable_thinking=self.enable_thinking, + return_dict=False, ) max_seq_len = self.config.max_seq_len @@ -303,8 +295,6 @@ def _generate_with_mtp( ) prefill_time_list = [] - decode_time_list = [] - decode_accepted_counts = [] cur_pos = 0 while cur_pos < prompt_len - 1: @@ -345,72 +335,82 @@ def _generate_with_mtp( self.decode_layer.set_prefill_valid_tokens(0) + ar_steps = max(1, min(1024, int(os.environ.get("GLM5_AR_N", "8")))) + return self._decode_ar(tokens, cur_pos, prompt_len, total_len, ar_steps, print_log) + + def _decode_ar( + self, + tokens: torch.Tensor, + cur_pos: int, + prompt_len: int, + total_len: int, + ar_steps: int, + print_log: bool, + ) -> tuple[str, list[float], list[int], int]: + """MTP decode (unified single-op, unified-style).""" + decode_time_list: list[float] = [] + decode_accepted_counts: list[int] = [] + + last_token = tokens[0, prompt_len - 1].item() + prev_draft = torch.full( + (1, self.mtp_seq_len), + last_token, + dtype=torch.int32, + device=self.default_device, + ) + finished = False while cur_pos < total_len - 1 and not finished: - if cur_pos == prompt_len - 1: - last_token = tokens[0, prompt_len - 1].item() - draft_tokens = torch.full( - (self.mtp_seq_len,), - last_token, - dtype=torch.long, - device=self.default_device, - ) - draft_tokens = draft_tokens.reshape(1, self.mtp_seq_len).to(torch.int32) - else: - draft_tokens = self.decode_layer.get_next_draft_tokens(0).reshape( - 1, self.mtp_seq_len - ) - start_time = time.time() - self.decode_layer.forward(draft_tokens, with_mtp=True) - end_time = time.time() - decode_time_list.append(end_time - start_time) - - num_accepted = self.decode_layer.get_num_accepted(0) - predicted_tokens = self.decode_layer.get_predicted_tokens(0).flatten() - decode_accepted_counts.append(num_accepted) + self.decode_layer.show_hands(prev_draft, ar_steps) + elapsed = time.time() - start_time + + acc = self.decode_layer.ar_accepted_tokens(0).cpu() + num = self.decode_layer.ar_num_accepted(0).cpu() + n_tokens = int(acc[0].item()) + n_steps = int(num[0].item()) + emitted = acc[1 : 1 + n_tokens].tolist() + per_step = num[1 : 1 + n_steps].tolist() + next_prev_draft = self.decode_layer.get_next_draft_tokens(0).reshape( + 1, self.mtp_seq_len + ) - num_output_tokens = num_accepted - for i in range(num_output_tokens): - if cur_pos + 1 + i >= total_len: + per_step_time = elapsed / max(1, len(per_step)) + offset = 0 + for na in per_step: + step_emit = emitted[offset : offset + na] + offset += na + for tok in step_emit: + if cur_pos + 1 >= total_len: + break + tokens[0, cur_pos + 1] = tok + cur_pos += 1 + if cur_pos >= prompt_len and print_log: + print( + self.tokenizer.decode([tok], skip_special_tokens=True), + end="", + flush=True, + ) + if tok in self.stop_token_ids: + finished = True + break + decode_time_list.append(per_step_time) + decode_accepted_counts.append(na) + if finished or cur_pos >= total_len - 1: break - new_token = int(predicted_tokens[i].item()) - tokens[0, cur_pos + 1 + i] = new_token - - if cur_pos + 1 + i >= prompt_len and print_log: - decoded_text = self.tokenizer.decode([new_token], skip_special_tokens=True) - print(decoded_text, end="", flush=True) - - if new_token in self.stop_token_ids: - finished = True - break - - cur_pos += num_accepted + prev_draft = next_prev_draft if print_log: print("\n") total_tokens = sum(decode_accepted_counts) logger.info(f"--Number of forward calls (decode): {len(decode_accepted_counts)}") logger.info(f"--Total tokens generated: {total_tokens}") - if len(decode_accepted_counts) > 0: - avg_accepted = sum(decode_accepted_counts) / len(decode_accepted_counts) - min_accepted = min(decode_accepted_counts) - max_accepted = max(decode_accepted_counts) - logger.info( - f"--Accepted tokens per call: mean={avg_accepted:.2f}, " - f"min={min_accepted}, max={max_accepted}" - ) - if decode_time_list: total_decode_time = sum(decode_time_list) effective_tps = total_tokens / total_decode_time if total_decode_time > 0 else 0 - avg_time_ms = total_decode_time / len(decode_time_list) * 1000 logger.info( - f"--Avg forward time: {avg_time_ms:.2f}ms, " - + f"({1000 / avg_time_ms:.2f} forwards/s)" + f"--Effective TPS (AR, ar_steps={ar_steps}): {effective_tps:.2f} tokens/s" ) - logger.info(f"--Effective TPS (with MTP): {effective_tps:.2f} tokens/s") - print("\n") self.decode_layer.reset_sequence() @@ -428,7 +428,6 @@ def _generate_with_mtp( completion_tokens.append(toks) decoded_tokens = self.tokenizer.batch_decode(completion_tokens, skip_special_tokens=True) - return ( f"{decoded_tokens[0]}\n" if decoded_tokens else "", decode_time_list, @@ -442,32 +441,7 @@ def inject_cache( start_pos: int = 0, end_pos: int | None = None, ) -> None: - """Inject external cache data into TileRT for P/D separation. - - This API allows injecting pre-computed KI/KV/PE cache data from an external - prefill system (e.g., SGLang), enabling prefill-decode disaggregation. - - Args: - layer_caches: List of (ki, kv, pe) tuples for each layer (0 to NUM_LAYERS-1). - Each tensor should be BF16 with shape [seqlen, dim] where: - - ki: [seqlen, 128] - compressed key (index_head_dim) - - kv: [seqlen, 512] - compressed key-value (kv_lora_rank) - - pe: [seqlen, 64] - position encoding cache (qk_rope_head_dim) - start_pos: Start position in cache to write (0-indexed). Defaults to 0. - end_pos: End position in cache (exclusive). If None, uses seqlen from tensors. - - Example: - >>> # Load cache from external prefill system - >>> layer_caches = [] # List of 78 (ki, kv, pe) tuples for GLM-5 - >>> for layer_id in range(78): - ... ki = load_ki_for_layer(layer_id) # [seqlen, 128] bf16 - ... kv = load_kv_for_layer(layer_id) # [seqlen, 512] bf16 - ... pe = load_pe_for_layer(layer_id) # [seqlen, 64] bf16 - ... layer_caches.append((ki, kv, pe)) - >>> generator.inject_cache(layer_caches, start_pos=0) - >>> generator.set_cur_pos(seqlen) # Set RoPE position - >>> # Continue generation from cache - """ + """Inject external cache data into TileRT for P/D separation.""" num_layers = len(layer_caches) if num_layers == 0: logger.warning("inject_cache called with empty layer_caches") @@ -501,49 +475,25 @@ def inject_cache( caches[base_idx + 1][0, start_pos:end_pos, :].copy_(kv_src) caches[base_idx + 2][0, start_pos:end_pos, :].copy_(pe_src) + torch.cuda.synchronize(device_id) + logger.info(f"Cache injection completed for {num_devices} devices") def set_cur_pos(self, cur_pos: int) -> None: - """Set the current position for RoPE. - - This should be called after inject_cache() to ensure the runtime position - matches the injected cache length, for correct RoPE position encoding - during continued generation. - - Args: - cur_pos: The current sequence position (typically the length of prefilled tokens). - - Example: - >>> generator.inject_cache(layer_caches, start_pos=0) - >>> generator.set_cur_pos(prefill_len) # Set position to prefill length - >>> # Now generate continues from the correct position - """ + """Set the current position for RoPE in C++ backend.""" if self.with_mtp: num_devices = self.decode_layer.num_devices for device_id in range(num_devices): intermediates, _, _, _ = self.decode_layer._get_device_result(device_id) cur_pos_tensor = intermediates[Idx.CUR_POS] cur_pos_tensor.fill_(cur_pos) + torch.cuda.synchronize(device_id) else: torch.ops.tilert.dsa_show_hands_set_cur_pos_glm5(cur_pos) logger.info(f"Set cur_pos to {cur_pos}") def inject_last_hidden_state(self, last_hidden_state: torch.Tensor) -> None: - """Inject the last hidden state for MTP mode. - - For MTP (Multi-Token Prediction), the MTP preprocess layer needs the - last hidden state from the main model's last token. - - Args: - last_hidden_state: [hidden_size] or [1, hidden_size] BF16 tensor. - The hidden state of the last token from prefill. - - Example: - >>> # After inject_cache, inject the last hidden state for MTP - >>> generator.inject_last_hidden_state(last_hidden_state) - >>> generator.set_cur_pos(prefill_len) - >>> # Then start generation - """ + """Inject the last hidden state for MTP mode.""" if not self.with_mtp: logger.warning("inject_last_hidden_state called but with_mtp is False, skipping") return @@ -557,5 +507,6 @@ def inject_last_hidden_state(self, last_hidden_state: torch.Tensor) -> None: lhs_tensor = intermediates[Idx.LAST_HIDDEN_STATES] lhs_src = last_hidden_state.to(f"cuda:{device_id}") lhs_tensor[0, 0, :].copy_(lhs_src.squeeze(0)) + torch.cuda.synchronize(device_id) logger.info(f"Injected last_hidden_state to {num_devices} devices") diff --git a/tilert/models/glm_5/model_args.py b/tilert/models/glm_5/model_args.py index 74e830c..b222729 100644 --- a/tilert/models/glm_5/model_args.py +++ b/tilert/models/glm_5/model_args.py @@ -12,43 +12,7 @@ @dataclass class ModelArgsGLM5(ModelArgs): - """ - Data class for defining model arguments and hyperparameters. - - Attributes: - arch_name (str): Architecture name. - max_batch_size (int): Maximum batch size. - max_seq_len (int): Maximum sequence length. - dtype (Literal["bf16", "fp8"]): Data type for computations. - scale_fmt (Optional[str]): Format for quantization scale. - vocab_size (int): Vocabulary size. - dim (int): Model dimension. - inter_dim (int): Intermediate dimension for MLP layers. - moe_inter_dim (int): Intermediate dimension for MoE layers. - n_layers (int): Number of transformer layers. - n_dense_layers (int): Number of dense layers in the model. - n_heads (int): Number of attention heads. - n_routed_experts (int): Number of routed experts for MoE layers. - n_shared_experts (int): Number of shared experts for MoE layers. - n_activated_experts (int): Number of activated experts in MoE layers. - n_expert_groups (int): Number of expert groups. - n_limited_groups (int): Number of limited groups for MoE routing. - score_func (Literal["softmax", "sigmoid"]): Scoring function for MoE routing. - route_scale (float): Scaling factor for routing scores. - q_lora_rank (int): LoRA rank for query projections. - kv_lora_rank (int): LoRA rank for key-value projections. - qk_nope_head_dim (int): Dimension for query-key projections without positional embeddings. - qk_rope_head_dim (int): Dimension for query-key projections with rotary embeddings. - v_head_dim (int): Dimension for value projections. - original_seq_len (Optional[int]): Original sequence length. - rope_theta (float): Base for rotary positional encoding. - rope_factor (Optional[float]): Scaling factor for extended sequence lengths. - beta_fast (Optional[int]): Fast beta correction factor. - beta_slow (Optional[int]): Slow beta correction factor. - mscale (float): Scaling factor for extended attention. - index_head_dim (int): Dimension for index head. - index_topk (int): Top-k for index head. - """ + """Data class for defining model arguments and hyperparameters.""" arch_name = "glm_5" @@ -68,7 +32,9 @@ class ModelArgsGLM5(ModelArgs): n_routed_experts: int = 256 n_shared_experts: int = 1 n_activated_experts: int = 8 - score_func: Literal["softmax", "sigmoid"] = "softmax" + n_expert_groups: int = 1 + n_limited_groups: int = 1 + score_func: Literal["softmax", "sigmoid"] = "sigmoid" route_scale: float = 2.5 q_lora_rank: int = 2048 diff --git a/tilert/models/glm_5/modules/__init__.py b/tilert/models/glm_5/modules/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tilert/models/glm_5/modules/__init__.py @@ -0,0 +1 @@ + diff --git a/tilert/models/glm_5/_dsa_v32/modules/dsa.py b/tilert/models/glm_5/modules/dsa.py similarity index 90% rename from tilert/models/glm_5/_dsa_v32/modules/dsa.py rename to tilert/models/glm_5/modules/dsa.py index 38a01c1..95c01ce 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/dsa.py +++ b/tilert/models/glm_5/modules/dsa.py @@ -4,10 +4,10 @@ from tilert.models.base import SerializableTileRTModule from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.mlp import MlpBlock -from tilert.models.glm_5._dsa_v32.modules.moe import MoeBlock from tilert.models.glm_5._dsa_v32.ops import RMSNormHeadProj -from tilert.models.glm_5._dsa_v32.temp_var_indices import TEMP_VARS_SIZE, Idx +from tilert.models.glm_5.modules.mlp import MlpBlock +from tilert.models.glm_5.modules.moe import MoeBlock +from tilert.models.glm_5.temp_var_indices import TEMP_VARS_SIZE, Idx class Dsa(SerializableTileRTModule): @@ -26,7 +26,7 @@ def __init__( num_devices=num_devices, remove_selected=True, ) - from tilert.models.glm_5._dsa_v32.modules.mla_v2 import ( + from tilert.models.glm_5.modules.mla_v2 import ( PureMlaV2, SparseSelectMlaV2, ) @@ -39,17 +39,17 @@ def __init__( if device_id == 0: self.v2_peer_bufs = torch.zeros(n_peers, dtype=torch.int64, device=dev) self.v2_partial_buf = torch.zeros( - model_args.max_batch_size, 4, model_args.dim, dtype=torch.bfloat16, device=dev + model_args.max_batch_size, 8, model_args.dim, dtype=torch.bfloat16, device=dev ) mla_kwargs = { "peer_bufs": self.v2_peer_bufs, "partial_buf": self.v2_partial_buf, } else: - max_seq_len = getattr(model_args, "num_mtp", 3) + 1 + max_seq_len = max(getattr(model_args, "num_mtp", 3) + 1, 8) topk = model_args.index_topk - self.v2_ll_buf = torch.zeros(max_seq_len * topk * 2, dtype=torch.int32, device=dev) - mla_kwargs = {"ll_buf": self.v2_ll_buf} + self.v2_recv_buf = torch.zeros(max_seq_len * topk * 2, dtype=torch.int32, device=dev) + mla_kwargs = {"recv_buf": self.v2_recv_buf} mla_num_devices: int | None = None if device_id != 0: @@ -139,7 +139,6 @@ def get_temp_vars( n_index_heads = self.model_args.index_n_heads max_seq_len = self.model_args.max_seq_len index_topk = self.model_args.index_topk - n_routed_experts = self.model_args.n_routed_experts n_activated_experts = self.model_args.n_activated_experts n_total_experts = self.model_args.n_activated_experts + self.model_args.n_shared_experts moe_inter_dim = self.model_args.moe_inter_dim // self.num_devices @@ -168,7 +167,10 @@ def get_temp_vars( temp_vars[Idx.O_LSE_ACC] = torch.empty(*batch_seq, n_local_heads, 32, **fp32_desc) temp_vars[Idx.PROJ_O] = torch.zeros(*batch_seq, n_local_heads, v_head_dim, **bf16_desc) temp_vars[Idx.UNPROJ_O] = torch.zeros(*batch_seq, dim, **bf16_desc) - temp_vars[Idx.SCORES] = torch.zeros(*batch_seq, n_routed_experts, **fp32_desc) + temp_vars[Idx.SCORES] = torch.full((4096,), float("nan"), **fp32_desc) + temp_vars[Idx.HIDDEN_MID] = torch.full( + (1, 8, n_activated_experts + 1, 256), float("nan"), **bf16_desc + ) temp_vars[Idx.X_MLP_IN] = torch.zeros(*batch_seq, dim, **bf16_desc) exp_up_gate = torch.zeros(*batch_seq, n_total_experts, moe_inter_dim, **bf16_desc) temp_vars[Idx.UP_GATE] = exp_up_gate @@ -222,6 +224,14 @@ def get_temp_vars( temp_vars[Idx.TOP_N_INDICES] = torch.zeros(*batch_seq, max_top_n, **int32_desc) temp_vars[Idx.LOGPROBS_FLAG] = torch.zeros(1, **int32_desc) + ar_max_steps = 1024 + temp_vars[Idx.AR_ACCEPTED_TOKENS] = torch.zeros(1 + ar_max_steps * seq_len, **int32_desc) + temp_vars[Idx.AR_NUM_ACCEPTED] = torch.zeros(1 + ar_max_steps, **int32_desc) + + temp_vars[Idx.GRAMMAR_BITMASK] = torch.full( + (*batch_seq, vocab_size // 32), -1, **int32_desc + ) + for i, t in enumerate(temp_vars): if t is None: raise RuntimeError(f"temp_vars[{i}] ({Idx(i).name}) was not initialized") diff --git a/tilert/models/glm_5/_dsa_v32/modules/end2end.py b/tilert/models/glm_5/modules/end2end.py similarity index 74% rename from tilert/models/glm_5/_dsa_v32/modules/end2end.py rename to tilert/models/glm_5/modules/end2end.py index 6b4e69c..f3d8375 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/end2end.py +++ b/tilert/models/glm_5/modules/end2end.py @@ -9,14 +9,13 @@ import torch from safetensors import safe_open -from safetensors.torch import load_file from tilert import logger from tilert.models.base import TileRTModule from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.dsa import Dsa -from tilert.models.glm_5._dsa_v32.modules.mtp import MTP -from tilert.models.glm_5._dsa_v32.temp_var_indices import Idx, validate_temp_vars_layout +from tilert.models.glm_5.modules.dsa import Dsa +from tilert.models.glm_5.modules.mtp import MTP +from tilert.models.glm_5.temp_var_indices import Idx, validate_temp_vars_layout from tilert.models.utils import precompute_freqs_cis from tilert.utils import get_profile_log_tensor @@ -26,6 +25,32 @@ DeviceResult = tuple[list[torch.Tensor], list[torch.Tensor], list[torch.Tensor], torch.Tensor] +def _load_state_dicts_by_index( + model_path: str, + weight_file_map: dict[str, str], + weights_list: list[str], + device: str, + selective_only: bool = False, +) -> dict[str, torch.Tensor]: + """Load tensors treating the index (``weight_file_map``) as the per-key authority.""" + target_files = {weight_file_map[key] for key in weights_list} + weights_set = set(weights_list) + state_dicts: dict[str, torch.Tensor] = {} + for weight_file in sorted(target_files): + filepath = os.path.join(model_path, weight_file) + logger.info(f"Loading weights from {weight_file} to {device}") + with safe_open(filepath, framework="pt", device=device) as f: + for key in f.keys(): + if selective_only and key not in weights_set: + continue + if weight_file_map.get(key, weight_file) != weight_file: + continue + state_dicts[key] = f.get_tensor(key) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return state_dicts + + def _mark_weights_initialized(module: TileRTModule) -> None: """Recursively mark a module and all sub-ops as having initialized tilert weights.""" module.is_tilert_weights_init = True @@ -35,12 +60,9 @@ def _mark_weights_initialized(module: TileRTModule) -> None: def _extract_ffn_ops(dsa: "Dsa") -> list: - """Extract Moe/Mlp op objects from a Dsa's layer blocks. - - Returns a list of length n_layers where each element is a Moe or Mlp instance. - """ - from tilert.models.glm_5._dsa_v32.modules.mlp import MlpBlock - from tilert.models.glm_5._dsa_v32.modules.moe import MoeBlock + """Extract Moe/Mlp op objects from a Dsa's layer blocks.""" + from tilert.models.glm_5.modules.mlp import MlpBlock + from tilert.models.glm_5.modules.moe import MoeBlock ffn_ops = [] for block in dsa.exec_seq: @@ -61,8 +83,8 @@ def _extract_ffn_ops(dsa: "Dsa") -> list: def _get_moe_weight_keys(dsa: "Dsa") -> set[str]: """Get state_dict keys that belong exclusively to MOE/MLP ops in this Dsa.""" - from tilert.models.glm_5._dsa_v32.modules.mlp import MlpBlock - from tilert.models.glm_5._dsa_v32.modules.moe import MoeBlock + from tilert.models.glm_5.modules.mlp import MlpBlock + from tilert.models.glm_5.modules.moe import MoeBlock moe_keys: set[str] = set() mla_keys: set[str] = set() @@ -76,6 +98,11 @@ def _get_moe_weight_keys(dsa: "Dsa") -> set[str]: return moe_keys - mla_keys +def _glm5_suffix(is_glm5: "bool | str" = True) -> str: # noqa: U100 + """GLM5-native tree: the torch.ops name suffix is always ``_glm5``.""" + return "_glm5" + + def dsa_show_hands_prepare_money( params: list[torch.Tensor], temp_vars: list[torch.Tensor], @@ -83,11 +110,11 @@ def dsa_show_hands_prepare_money( profile_logs: torch.Tensor, forward_max_seq_len: int, with_mtp: bool = False, - is_glm5: bool = False, + is_glm5: "bool | str" = False, ) -> Any: """Prepare money for show hands""" mtp_flag = "_mtp_e2e" if with_mtp else "" - glm5_flag = "_glm5" if is_glm5 else "" + glm5_flag = _glm5_suffix(is_glm5) func_name = f"dsa{mtp_flag}_show_hands_prepare_money{glm5_flag}" if mtp_flag: return getattr(torch.ops.tilert, func_name)(params, temp_vars, cache_vars, profile_logs) @@ -96,69 +123,87 @@ def dsa_show_hands_prepare_money( ) -def dsa_show_hands(token_id: torch.Tensor, with_mtp: bool = False, is_glm5: bool = False) -> Any: - """Show hands with native MT""" +def dsa_show_hands( + token_id: torch.Tensor, + with_mtp: bool = False, + is_glm5: "bool | str" = False, + ar_steps: int = 1, +) -> Any: + """Show hands with native MT.""" mtp_flag = "_mtp_e2e" if with_mtp else "" - glm5_flag = "_glm5" if is_glm5 else "" - func_name = f"dsa{mtp_flag}_show_hands{glm5_flag}" - return getattr(torch.ops.tilert, func_name)(token_id) + glm5_flag = _glm5_suffix(is_glm5) + op = getattr(torch.ops.tilert, f"dsa{mtp_flag}_show_hands{glm5_flag}") + if is_glm5: + return op(token_id, int(ar_steps)) + return op(token_id) + + +def dsa_show_hands_accepted_tokens(dev: int, is_glm5: "bool | str" = True) -> torch.Tensor: + """Read w/o-MTP AR accepted-tokens flat buffer ([0]=count, [1:]=token stream).""" + glm5_flag = _glm5_suffix(is_glm5) + return getattr(torch.ops.tilert, f"dsa_show_hands_accepted_tokens{glm5_flag}")(dev) + + +def dsa_show_hands_num_accepted(dev: int, is_glm5: "bool | str" = True) -> torch.Tensor: + """Read w/o-MTP AR per-step num_accepted flat buffer ([0]=steps, [1:]=counts).""" + glm5_flag = _glm5_suffix(is_glm5) + return getattr(torch.ops.tilert, f"dsa_show_hands_num_accepted{glm5_flag}")(dev) + +def dsa_mtp_e2e_accepted_tokens(dev: int, is_glm5: "bool | str" = True) -> torch.Tensor: + """Read the AR accepted-tokens flat buffer ([0]=count, [1:]=token stream).""" + glm5_flag = _glm5_suffix(is_glm5) + return getattr(torch.ops.tilert, f"dsa_mtp_e2e_accepted_tokens{glm5_flag}")(dev) -def dsa_show_hands_reset(with_mtp: bool = False, is_glm5: bool = False) -> Any: + +def dsa_mtp_e2e_num_accepted(dev: int, is_glm5: "bool | str" = True) -> torch.Tensor: + """Read the AR per-step num_accepted flat buffer ([0]=steps, [1:]=counts).""" + glm5_flag = _glm5_suffix(is_glm5) + return getattr(torch.ops.tilert, f"dsa_mtp_e2e_num_accepted{glm5_flag}")(dev) + + +def dsa_show_hands_reset(with_mtp: bool = False, is_glm5: "bool | str" = False) -> Any: """Reset show one hand""" mtp_flag = "_mtp_e2e" if with_mtp else "" - glm5_flag = "_glm5" if is_glm5 else "" + glm5_flag = _glm5_suffix(is_glm5) func_name = f"dsa{mtp_flag}_show_hands_reset{glm5_flag}" return getattr(torch.ops.tilert, func_name)() -def dsa_show_hands_go_home(with_mtp: bool = False, is_glm5: bool = False) -> Any: +def dsa_show_hands_go_home(with_mtp: bool = False, is_glm5: "bool | str" = False) -> Any: """Go home""" mtp_flag = "_mtp_e2e" if with_mtp else "" - glm5_flag = "_glm5" if is_glm5 else "" + glm5_flag = _glm5_suffix(is_glm5) func_name = f"dsa{mtp_flag}_show_hands_go_home{glm5_flag}" return getattr(torch.ops.tilert, func_name)() def dsa_show_hands_set_sampling_seed( - seed: int, with_mtp: bool = False, is_glm5: bool = False + seed: int, with_mtp: bool = False, is_glm5: "bool | str" = False ) -> Any: - """Set the sampling seed (request-level, fixed for the entire request). - - Args: - seed: The sampling seed value. - """ + """Set the sampling seed (request-level, fixed for the entire request).""" mtp_flag = "_mtp_e2e" if with_mtp else "" - glm5_flag = "_glm5" if is_glm5 else "" + glm5_flag = _glm5_suffix(is_glm5) func_name = f"dsa{mtp_flag}_show_hands_set_sampling_seed{glm5_flag}" return getattr(torch.ops.tilert, func_name)(seed) def dsa_mtp_e2e_show_hands_set_prefill_valid_tokens( - num_valid_tokens: int, is_glm5: bool = False + num_valid_tokens: int, is_glm5: "bool | str" = False, with_mtp: bool = True ) -> Any: - """Set the number of valid (non-padding) tokens for prefill mode. - - This controls how many tokens are copied from draft_tokens to predicted_tokens - during prefill. Should be called before forward() when the chunk has padding. - - Args: - num_valid_tokens: Number of valid tokens in the chunk (1-4). - """ - mtp_flag = "_mtp_e2e" - glm5_flag = "_glm5" if is_glm5 else "" + """Select prefill (num_valid_tokens > 0) vs decode (0) mode.""" + mtp_flag = "_mtp_e2e" if with_mtp else "" + glm5_flag = _glm5_suffix(is_glm5) func_name = f"dsa{mtp_flag}_show_hands_set_prefill_valid_tokens{glm5_flag}" return getattr(torch.ops.tilert, func_name)(num_valid_tokens) -def dsa_mtp_e2e_show_hands_set_prefill_mtp_extra_token(token: int, is_glm5: bool = False) -> Any: - """Set the extra token for MTP[0] shifted input during prefill. - - Args: - token: The extra prompt token id (int32). - """ +def dsa_mtp_e2e_show_hands_set_prefill_mtp_extra_token( + token: int, is_glm5: "bool | str" = False +) -> Any: + """Set the extra token for MTP[0] shifted input during prefill.""" mtp_flag = "_mtp_e2e" - glm5_flag = "_glm5" if is_glm5 else "" + glm5_flag = _glm5_suffix(is_glm5) func_name = f"dsa{mtp_flag}_show_hands_set_prefill_mtp_extra_token{glm5_flag}" return getattr(torch.ops.tilert, func_name)(token) @@ -176,17 +221,23 @@ def __init__( top_p: float = 0.9, top_k: int = 256, use_topp: bool = False, + num_mtp: int = 3, ) -> None: validate_temp_vars_layout() print(f"Model args: {model_args.arch_name}") for k_arg, v_arg in model_args.__dict__.items(): print(f" - {k_arg}: {v_arg}") self.model_args = model_args - self.is_glm5 = self.model_args.arch_name == "glm_5" - assert self.model_args.arch_name in ["deepseek_v3_2", "glm_5"] + arch = self.model_args.arch_name + assert ( + arch == "glm_5" + ), f"glm_5-native ShowHandsDSALayer requires arch_name 'glm_5', got {arch!r}" + self.is_glm5: bool = True self.num_devices = 8 - self.forward_max_seq_len = 4 + assert num_mtp == 3, "num_mtp must be 3" + self.num_mtp = num_mtp + self.forward_max_seq_len = num_mtp + 1 self.model_path = model_path self.with_weight_conversion = with_weight_conversion @@ -222,30 +273,13 @@ def load_device_weights( if skip_keys: weights_list = [k for k in weights_list if k not in skip_keys] - target_files = set() - for weight_key in weights_list: - weight_file = weight_file_map[weight_key] - target_files.add(weight_file) - - state_dicts = {} - weights_set = set(weights_list) - for weight_file in target_files: - filepath = os.path.join(model_path, weight_file) - if skip_keys: - logger.info( - f"Selectively loading weights from {weight_file} for device {device_id}" - ) - with safe_open(filepath, framework="pt", device=f"cuda:{device_id}") as f: - for key in f.keys(): - if key in weights_set: - state_dicts[key] = f.get_tensor(key) - torch.cuda.empty_cache() - else: - logger.info(f"Loading weights from {weight_file} for device {device_id}") - state_dict = load_file(filepath, device=f"cuda:{device_id}") - state_dicts.update(state_dict) - del state_dict - torch.cuda.empty_cache() + state_dicts = _load_state_dicts_by_index( + model_path, + weight_file_map, + weights_list, + device=f"cuda:{device_id}", + selective_only=bool(skip_keys), + ) state_dicts["freqs_cis"] = self._gen_freqs_cis().to(device_id) return state_dicts @@ -339,16 +373,7 @@ def _init_weights( cached_ffn_ops_per_device: dict[int, list] | None = None, skip_keys_per_device: dict[int, set[str]] | None = None, ) -> None: - """Load the model weights from the given path or generate random weights. - - Args: - model_path: Path to the model weights directory. - cached_ffn_ops_per_device: Optional dict mapping device_id to cached FFN ops. - When provided, these ops are injected into the Dsa and their weights - are not re-loaded from disk. - skip_keys_per_device: Optional dict mapping device_id to safetensors keys - to skip during loading. Used together with cached_ffn_ops_per_device. - """ + """Load the model weights from the given path or generate random weights.""" self._v2_p2p: dict = {} def __load_weights(device_id: int, model_path: str | None) -> None: @@ -389,6 +414,7 @@ def __load_weights(device_id: int, model_path: str | None) -> None: dsa.init_tilert_weights(state_dicts) self._dsa_objects[device_id] = dsa params.extend(dsa.get_weights_list()) + torch.cuda.empty_cache() caches.extend(dsa.get_cache_vars()) if device_id == 0: @@ -397,7 +423,7 @@ def __load_weights(device_id: int, model_path: str | None) -> None: } else: self._v2_p2p[device_id] = { - "ll_buf": dsa.v2_ll_buf, + "recv_buf": dsa.v2_recv_buf, } intermediates.extend( self.generate_params_with_continuous_storage( @@ -428,12 +454,13 @@ def __load_weights(device_id: int, model_path: str | None) -> None: device=device_id, ) ) + intermediates[Idx.GRAMMAR_BITMASK].fill_(-1) base_params_count = len(params) base_caches_count = len(caches) if self.with_mtp: - from tilert.models.glm_5._dsa_v32.modules.mla_v2 import ( + from tilert.models.glm_5.modules.mla_v2 import ( PureMlaV2, SparseSelectMlaV2, ) @@ -446,7 +473,7 @@ def __load_weights(device_id: int, model_path: str | None) -> None: "peer_bufs": dsa.v2_peer_bufs, } else: - mtp_kwargs["mla_kwargs"] = {"ll_buf": dsa.v2_ll_buf} + mtp_kwargs["mla_kwargs"] = {"recv_buf": dsa.v2_recv_buf} mtp = MTP(self.model_args, device_id, self.num_devices, **mtp_kwargs) mtp.init_tilert_weights(state_dicts) params.extend(mtp.get_weights_list()) @@ -493,10 +520,10 @@ def _runner(dev_id: int) -> None: peer_bufs_cpu = torch.zeros(self.num_devices - 1, dtype=torch.int64) for i in range(self.num_devices - 1): dev_id = i + 1 - peer_bufs_cpu[i] = self._v2_p2p[dev_id]["ll_buf"].data_ptr() + peer_bufs_cpu[i] = self._v2_p2p[dev_id]["recv_buf"].data_ptr() gpu0["peer_bufs"].copy_(peer_bufs_cpu) logger.info( - "V2 P2P exchange complete: peer_bufs (ll_buf)=%s", + "V2 P2P exchange complete: peer_bufs (recv_buf)=%s", [hex(int(x)) for x in peer_bufs_cpu], ) @@ -554,18 +581,35 @@ def forward( with_mtp: bool | None = None, ) -> list[DeviceResult]: active_mtp = with_mtp if with_mtp is not None else self.with_mtp - dsa_show_hands(token_id.cpu(), active_mtp, self.is_glm5) + dsa_show_hands(token_id.cpu(), active_mtp, self.is_glm5, ar_steps=1) return [self._get_device_result(device_id) for device_id in range(self.num_devices)] - def set_sampling_seed(self, seed: int, with_mtp: bool | None = None) -> None: - """Set the sampling seed for top-p sampling. + def show_hands(self, prev_draft_tokens: torch.Tensor, ar_steps: int = 1) -> None: + """MTP decode (GLM5, unified-style single op).""" + dsa_show_hands(prev_draft_tokens.cpu(), True, self.is_glm5, ar_steps) + + def ar_accepted_tokens(self, dev: int = 0) -> torch.Tensor: + """AR accepted-tokens flat buffer ([0]=count, [1:]=token stream).""" + return dsa_mtp_e2e_accepted_tokens(dev, self.is_glm5) + + def ar_num_accepted(self, dev: int = 0) -> torch.Tensor: + """AR per-step num_accepted flat buffer ([0]=steps, [1:]=per-step counts).""" + return dsa_mtp_e2e_num_accepted(dev, self.is_glm5) + + def show_hands_no_mtp(self, prev_token: torch.Tensor, ar_steps: int = 1) -> None: + """w/o-MTP decode (GLM5, unified-style single op).""" + dsa_show_hands(prev_token.cpu(), False, self.is_glm5, ar_steps) - The seed is fixed for the entire request. Position provides per-step variation. + def ar_accepted_tokens_no_mtp(self, dev: int = 0) -> torch.Tensor: + """w/o-MTP AR accepted-tokens flat buffer ([0]=count, [1:]=token stream).""" + return dsa_show_hands_accepted_tokens(dev, self.is_glm5) - Args: - seed: The sampling seed value. - with_mtp: Override MTP mode for this call. Defaults to self.with_mtp. - """ + def ar_num_accepted_no_mtp(self, dev: int = 0) -> torch.Tensor: + """w/o-MTP AR per-step num_accepted flat buffer ([0]=steps, [1:]=counts).""" + return dsa_show_hands_num_accepted(dev, self.is_glm5) + + def set_sampling_seed(self, seed: int, with_mtp: bool | None = None) -> None: + """Set the sampling seed for top-p sampling.""" active_mtp = with_mtp if with_mtp is not None else self.with_mtp dsa_show_hands_set_sampling_seed(seed, active_mtp, self.is_glm5) @@ -595,84 +639,37 @@ def _get_device_result(self, device_id: int) -> DeviceResult: raise RuntimeError(f"Device {device_id} is not initialized") return device_result - def set_prefill_valid_tokens(self, num_valid_tokens: int) -> None: - """Set the number of valid tokens for prefill mode. - - This controls how many tokens are copied from draft_tokens to predicted_tokens - during prefill. Should be called before forward() when the chunk has padding. - - Args: - num_valid_tokens: Number of valid tokens in the chunk (1-4). - """ - dsa_mtp_e2e_show_hands_set_prefill_valid_tokens(num_valid_tokens, self.is_glm5) + def set_prefill_valid_tokens(self, num_valid_tokens: int, with_mtp: bool | None = None) -> None: + """Select prefill (num_valid_tokens > 0) vs decode (0) mode.""" + active_mtp = with_mtp if with_mtp is not None else self.with_mtp + dsa_mtp_e2e_show_hands_set_prefill_valid_tokens(num_valid_tokens, self.is_glm5, active_mtp) def set_prefill_mtp_extra_token(self, token: int) -> None: - """Set the extra token for MTP[0] shifted input during prefill. - - Args: - token: The prompt token at (cur_pos + mtp_seq_len). - """ + """Set the extra token for MTP[0] shifted input during prefill.""" dsa_mtp_e2e_show_hands_set_prefill_mtp_extra_token(token, self.is_glm5) def get_next_draft_tokens(self, device_id: int = 0) -> torch.Tensor: - """Get next_draft_tokens from the specified device. - - Args: - device_id: Device ID to get results from. - - Returns: - next_draft_tokens tensor of shape [1, MTP_SEQ_LEN]. - """ + """Get next_draft_tokens from the specified device.""" intermediates, _, _, _ = self._get_device_result(device_id) return intermediates[Idx.NEXT_DRAFT_TOKENS] def get_num_accepted(self, device_id: int = 0) -> int: - """Get number of accepted tokens from the specified device. - - Args: - device_id: Device ID to get results from. - - Returns: - Number of accepted tokens. - """ + """Get number of accepted tokens from the specified device.""" intermediates, _, _, _ = self._get_device_result(device_id) return int(intermediates[Idx.ACCEPTED_TOKENS][0].item()) def get_predicted_tokens(self, device_id: int = 0) -> torch.Tensor: - """Get predicted_tokens from the specified device. - - Args: - device_id: Device ID to get results from. - - Returns: - predicted_tokens tensor containing main model predictions. - """ + """Get predicted_tokens from the specified device.""" intermediates, _, _, _ = self._get_device_result(device_id) return intermediates[Idx.PREDICTED_TOKENS] def get_logits(self, device_id: int = 0) -> torch.Tensor: - """Get logits from the specified device. - - Args: - device_id: Device ID to get results from. - - Returns: - Logits tensor of shape [batch, seq_len, vocab_size] (FP32). - """ + """Get logits from the specified device.""" intermediates, _, _, _ = self._get_device_result(device_id) return intermediates[Idx.LOGITS_OUT] def get_top_n_logprobs(self, device_id: int = 0) -> tuple[torch.Tensor, torch.Tensor]: - """Get top-N log-probabilities and token IDs from the top_p kernel. - - Args: - device_id: Device ID to get results from. - - Returns: - Tuple of (log_probs, token_ids): - - log_probs: [batch, seq_len, 256] FP32 - - token_ids: [batch, seq_len, 256] INT32 - """ + """Get top-N log-probabilities and token IDs from the top_p kernel.""" intermediates, _, _, _ = self._get_device_result(device_id) return ( intermediates[Idx.TOP_N_LOG_PROBS], @@ -680,23 +677,12 @@ def get_top_n_logprobs(self, device_id: int = 0) -> tuple[torch.Tensor, torch.Te ) def get_token_logprob(self, device_id: int = 0) -> torch.Tensor: - """Get log-probability of the sampled token (from TOP_P_SCORES). - - Args: - device_id: Device ID to get results from. - - Returns: - Tensor of shape [batch, seq_len] (FP32). - """ + """Get log-probability of the sampled token (from TOP_P_SCORES).""" intermediates, _, _, _ = self._get_device_result(device_id) return intermediates[Idx.TOP_P_SCORES] def set_logprobs_enabled(self, enabled: bool) -> None: - """Enable or disable logprobs export in the top_p kernel. - - Args: - enabled: True to enable logprobs export, False to disable. - """ + """Enable or disable logprobs export in the top_p kernel.""" flag_val = 1 if enabled else 0 for device_id in range(self.num_devices): intermediates, _, _, _ = self._get_device_result(device_id) diff --git a/tilert/models/glm_5/_dsa_v32/modules/mla_v2.py b/tilert/models/glm_5/modules/mla_v2.py similarity index 74% rename from tilert/models/glm_5/_dsa_v32/modules/mla_v2.py rename to tilert/models/glm_5/modules/mla_v2.py index d9a9dd1..6a3c2fd 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/mla_v2.py +++ b/tilert/models/glm_5/modules/mla_v2.py @@ -1,4 +1,4 @@ -"""MLA weight generator classes for device-group-specific pipelines.""" +"""V2 MLA weight generator classes for device-group-specific pipelines.""" import torch @@ -19,6 +19,7 @@ ) from tilert.models.glm_5._dsa_v32.ops.rmsnorm_projx_wqakis import ( RMSNormProjxWqakis, + RMSNormProjxWqakisAlgorithm, ) from tilert.models.glm_5._dsa_v32.ops.rmsnorm_projx_wqkva import ( RMSNormProjxWqkva, @@ -31,6 +32,8 @@ class SparseSelectMlaV2(SerializableTileRTModule): + """Device Group A (GPU 0): sparse selector MLA.""" + def __init__( self, model_args: ModelArgs, @@ -44,12 +47,13 @@ def __init__( self.rmsnorm_projx_wqakis = RMSNormProjxWqakis( model_args=model_args, device_id=device_id, num_devices=num_devices ) + self.rmsnorm_projx_wqakis.algorithm = RMSNormProjxWqakisAlgorithm.W8A16HMMA self.register_op(self.rmsnorm_projx_wqakis) self.rmsnorm_projq_wqi = RmsnormProjqWqi( model_args=model_args, device_id=device_id, num_devices=num_devices ) - self.rmsnorm_projq_wqi.algorithm = RmsnormProjqWqiAlgorithm.FP16MMA + self.rmsnorm_projq_wqi.algorithm = RmsnormProjqWqiAlgorithm.BF16MMA self.register_op(self.rmsnorm_projq_wqi) self.layernorm_rope_rotate = LayerNormRoPERotate( @@ -58,7 +62,10 @@ def __init__( self.register_op(self.layernorm_rope_rotate) self.projx_wis = ProjxWis( - model_args=model_args, device_id=device_id, num_devices=num_devices + model_args=model_args, + device_id=device_id, + num_devices=num_devices, + compute_kernel_type="bf16mma", ) self.register_op(self.projx_wis) @@ -70,7 +77,7 @@ def __init__( self.pe_cache: torch.Tensor | None = None def get_weights_list(self) -> list[torch.Tensor]: - """Return weight tensors.""" + """Return weight tensors in registration order.""" weights = super().get_weights_list() dev = f"cuda:{self.device_id}" @@ -79,7 +86,7 @@ def get_weights_list(self) -> list[torch.Tensor]: if self.partial_buf is None: self.partial_buf = torch.zeros( self.model_args.max_batch_size, - 4, + 8, self.model_args.dim, dtype=torch.bfloat16, device=dev, @@ -91,49 +98,51 @@ def get_weights_list(self) -> list[torch.Tensor]: return weights def get_cache_vars(self) -> list[torch.Tensor]: - """Return [ki_cache, kv_cache, pe_cache] matching DsaCacheVars layout.""" + """Return [k_cache, kv_cache, pe_cache] matching DsaCacheVars layout.""" cache_seq_len = self.model_args.max_seq_len + self.model_args.kv_cache_pad - bs_args = (self.model_args.max_batch_size, cache_seq_len) + bs = self.model_args.max_batch_size + dev = f"cuda:{self.device_id}" if self.ki_cache is None: ki_dim = self.model_args.index_head_dim - self.ki_cache = torch.zeros( - *bs_args, ki_dim, dtype=torch.bfloat16, device=f"cuda:{self.device_id}" - ) + self.ki_cache = torch.zeros(bs, cache_seq_len, ki_dim, dtype=torch.bfloat16, device=dev) if self.kv_cache is None: kv_dim = self.model_args.kv_lora_rank - self.kv_cache = torch.zeros( - *bs_args, kv_dim, dtype=torch.bfloat16, device=f"cuda:{self.device_id}" - ) + if getattr(self.model_args, "fp8_kv_cache", False): + self.kv_cache = torch.zeros( + bs, 1, kv_dim + (kv_dim // 128) * 4, dtype=torch.uint8, device=dev + ) + else: + self.kv_cache = torch.zeros(bs, 1, kv_dim, dtype=torch.bfloat16, device=dev) if self.pe_cache is None: - pe_dim = self.model_args.qk_rope_head_dim self.pe_cache = torch.zeros( - *bs_args, pe_dim, dtype=torch.bfloat16, device=f"cuda:{self.device_id}" + bs, 1, self.model_args.qk_rope_head_dim, dtype=torch.bfloat16, device=dev ) return [*super().get_cache_vars(), self.ki_cache, self.kv_cache, self.pe_cache] class PureMlaV2(SerializableTileRTModule): + """Device Group B (GPU 1-7): pure MLA.""" def __init__( self, model_args: ModelArgs, device_id: int, num_devices: int, - ll_buf: torch.Tensor | None = None, + recv_buf: torch.Tensor | None = None, ): super().__init__(model_args=model_args, device_id=device_id, num_devices=num_devices) self.rmsnorm_projx_wqkva = RMSNormProjxWqkva( model_args=model_args, device_id=device_id, num_devices=num_devices ) - self.rmsnorm_projx_wqkva.algorithm = RMSNormProjxWqkvaAlgorithm.DECOUPLED + self.rmsnorm_projx_wqkva.algorithm = RMSNormProjxWqkvaAlgorithm.W8A16HMMA self.register_op(self.rmsnorm_projx_wqkva) self.rmsnorm_projq_wqb = RmsnormProjqWqb( model_args=model_args, device_id=device_id, num_devices=num_devices ) - self.rmsnorm_projq_wqb.algorithm = RmsnormProjqWqbAlgorithm.FP16MMA + self.rmsnorm_projq_wqb.algorithm = RmsnormProjqWqbAlgorithm.BF16MMA self.register_op(self.rmsnorm_projq_wqb) self.rmsnorm_kv = KVRMSNorm( @@ -151,7 +160,7 @@ def __init__( ) self.register_op(self.projo_wkvb) - allreduce_algo = UnProjOAllReduceAlgorithm.FP16MMA + allreduce_algo = UnProjOAllReduceAlgorithm.BF16MMA self.unproj_o_allreduce = UnProjOAllReduce( model_args=model_args, device_id=device_id, @@ -160,14 +169,14 @@ def __init__( ) self.register_op(self.unproj_o_allreduce) - self.ll_buf = ll_buf + self.recv_buf = recv_buf self.ki_cache: torch.Tensor | None = None self.kv_cache: torch.Tensor | None = None self.pe_cache: torch.Tensor | None = None def init_random_weights(self) -> None: - """Initialize random weights for this module.""" + """Override to re-init ProjQWkvb/ProjOWkvb with HMMA-packed weights.""" super().init_random_weights() from tilert.models.common import init_func @@ -193,7 +202,7 @@ def init_random_weights(self) -> None: op.init_tilert_weights_hmma(per_dev) def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """Load TileRT weights for this module from state_dict.""" + """Override to use HMMA-packed weights for ProjQWkvb and ProjOWkvb.""" self.projq_wqb.is_tilert_weights_init = True self.projo_wkvb.is_tilert_weights_init = True @@ -211,38 +220,47 @@ def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: op.init_tilert_weights_hmma(op_state_dict) def get_weights_list(self) -> list[torch.Tensor]: - """Return weight tensors.""" + """Return weight tensors in registration order.""" weights = super().get_weights_list() - if self.ll_buf is None: - max_seq_len = getattr(self.model_args, "num_mtp", 3) + 1 + if self.recv_buf is None: + max_seq_len = max(getattr(self.model_args, "num_mtp", 3) + 1, 8) topk = self.model_args.index_topk - self.ll_buf = torch.zeros( + self.recv_buf = torch.zeros( max_seq_len * topk * 2, dtype=torch.int32, device=f"cuda:{self.device_id}" ) - weights.append(self.ll_buf) + weights.append(self.recv_buf) return weights def get_cache_vars(self) -> list[torch.Tensor]: - """Return [ki_cache, kv_cache, pe_cache] matching DsaCacheVars layout.""" + """Return [k_cache, kv_cache, pe_cache] matching DsaCacheVars layout.""" cache_seq_len = self.model_args.max_seq_len + self.model_args.kv_cache_pad - bs_args = (self.model_args.max_batch_size, cache_seq_len) + bs = self.model_args.max_batch_size + dev = f"cuda:{self.device_id}" if self.ki_cache is None: - ki_dim = self.model_args.index_head_dim self.ki_cache = torch.zeros( - *bs_args, ki_dim, dtype=torch.bfloat16, device=f"cuda:{self.device_id}" + bs, 1, self.model_args.index_head_dim, dtype=torch.bfloat16, device=dev ) if self.kv_cache is None: kv_dim = self.model_args.kv_lora_rank - self.kv_cache = torch.zeros( - *bs_args, kv_dim, dtype=torch.bfloat16, device=f"cuda:{self.device_id}" - ) + if getattr(self.model_args, "fp8_kv_cache", False): + kv_merged = kv_dim + (kv_dim // 128) * 4 + self.kv_cache = torch.zeros( + bs, cache_seq_len, kv_merged, dtype=torch.uint8, device=dev + ) + else: + self.kv_cache = torch.zeros( + bs, cache_seq_len, kv_dim, dtype=torch.bfloat16, device=dev + ) if self.pe_cache is None: - pe_dim = self.model_args.qk_rope_head_dim self.pe_cache = torch.zeros( - *bs_args, pe_dim, dtype=torch.bfloat16, device=f"cuda:{self.device_id}" + bs, + cache_seq_len, + self.model_args.qk_rope_head_dim, + dtype=torch.bfloat16, + device=dev, ) return [*super().get_cache_vars(), self.ki_cache, self.kv_cache, self.pe_cache] diff --git a/tilert/models/glm_5/_dsa_v32/modules/mlp.py b/tilert/models/glm_5/modules/mlp.py similarity index 88% rename from tilert/models/glm_5/_dsa_v32/modules/mlp.py rename to tilert/models/glm_5/modules/mlp.py index 85fec25..79e8d69 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/mlp.py +++ b/tilert/models/glm_5/modules/mlp.py @@ -1,13 +1,14 @@ from tilert.models.base import SerializableTileRTModule from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.mla_v2 import PureMlaV2 as Mla from tilert.models.glm_5._dsa_v32.ops.down_allreduce import ( DownAllReduce, + DownAllReduceAlgorithm, ) from tilert.models.glm_5._dsa_v32.ops.rmsnorm_up_gate_silu import ( RMSNormUpGateSiLU, RMSNormUpGateSiLUAlgorithm, ) +from tilert.models.glm_5.modules.mla_v2 import PureMlaV2 as Mla class Mlp(SerializableTileRTModule): @@ -26,11 +27,14 @@ def __init__( device_id=device_id, num_devices=num_devices, ) - self.rmsnorm_mlp_up_gate_silu.algorithm = RMSNormUpGateSiLUAlgorithm.FP16MMA + self.rmsnorm_mlp_up_gate_silu.algorithm = RMSNormUpGateSiLUAlgorithm.BF16MMA_V2 self.register_op(self.rmsnorm_mlp_up_gate_silu) self.rmsnorm_mlp_down = DownAllReduce( - model_args=model_args, device_id=device_id, num_devices=num_devices + model_args=model_args, + device_id=device_id, + num_devices=num_devices, + algorithm=DownAllReduceAlgorithm.BF16MMA_V2, ) self.register_op(self.rmsnorm_mlp_down) diff --git a/tilert/models/glm_5/_dsa_v32/modules/moe.py b/tilert/models/glm_5/modules/moe.py similarity index 86% rename from tilert/models/glm_5/_dsa_v32/modules/moe.py rename to tilert/models/glm_5/modules/moe.py index 5410284..4e4b209 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/moe.py +++ b/tilert/models/glm_5/modules/moe.py @@ -2,9 +2,9 @@ from tilert.models.base import SerializableTileRTModule from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.mla_v2 import PureMlaV2 as Mla from tilert.models.glm_5._dsa_v32.ops.expert_down_allreduce import ( ExpertDownAllReduce, + ExpertDownAllReduceAlgorithm, ) from tilert.models.glm_5._dsa_v32.ops.expert_sel_up_gate_silu import ( ExpertSelectUpGateSiLU, @@ -13,6 +13,7 @@ from tilert.models.glm_5._dsa_v32.ops.rmsnorm_expert_proj import ( RMSNormExpertProj, ) +from tilert.models.glm_5.modules.mla_v2 import PureMlaV2 as Mla class Moe(SerializableTileRTModule): @@ -32,12 +33,16 @@ def __init__(self, model_args: ModelArgs, device_id: int, num_devices: int): model_args=model_args, device_id=device_id, num_devices=num_devices, - algorithm=ExpertSelectUpGateSiLUAlgorithm.FP16MMA, + algorithm=ExpertSelectUpGateSiLUAlgorithm.BF16MMA, ) self.register_op(self.exp_sel_up_gate_silu) + _expert_down_algo = ExpertDownAllReduceAlgorithm.BF16MMA self.expert_down_allreduce = ExpertDownAllReduce( - model_args=model_args, device_id=device_id, num_devices=num_devices + model_args=model_args, + device_id=device_id, + num_devices=num_devices, + algorithm=_expert_down_algo, ) self.register_op(self.expert_down_allreduce) diff --git a/tilert/models/glm_5/_dsa_v32/modules/mtp.py b/tilert/models/glm_5/modules/mtp.py similarity index 93% rename from tilert/models/glm_5/_dsa_v32/modules/mtp.py rename to tilert/models/glm_5/modules/mtp.py index ccfbdc8..1bfc1f1 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/mtp.py +++ b/tilert/models/glm_5/modules/mtp.py @@ -2,9 +2,9 @@ from tilert.models.base import SerializableTileRTModule from tilert.models.glm_5._dsa_v32.model_args import ModelArgs -from tilert.models.glm_5._dsa_v32.modules.moe import MoeBlock -from tilert.models.glm_5._dsa_v32.modules.mtp_preprocess import MTPPreprocessLayer from tilert.models.glm_5._dsa_v32.ops import RMSNormHeadProj +from tilert.models.glm_5.modules.moe import MoeBlock +from tilert.models.glm_5.modules.mtp_preprocess import MTPPreprocessLayer class MTP(SerializableTileRTModule): diff --git a/tilert/models/glm_5/_dsa_v32/modules/mtp_preprocess.py b/tilert/models/glm_5/modules/mtp_preprocess.py similarity index 87% rename from tilert/models/glm_5/_dsa_v32/modules/mtp_preprocess.py rename to tilert/models/glm_5/modules/mtp_preprocess.py index debd75d..c7d2d83 100644 --- a/tilert/models/glm_5/_dsa_v32/modules/mtp_preprocess.py +++ b/tilert/models/glm_5/modules/mtp_preprocess.py @@ -70,18 +70,7 @@ class MTPPreprocessWeightsConverter(TilertWeightsConverter): """Converts ref-format weights to TileRT format for MTP preprocess.""" def convert_to_tilert(self, weights: list[torch.Tensor], device_id: int) -> list[torch.Tensor]: - """ - Convert ref weights to TileRT format for a specific device. - - Args: - weights: [embedding_rmsnorm_gamma, hidden_rmsnorm_gamma, eh_proj.weight] - Ref format: enorm.weight [7168], hnorm.weight [7168], - eh_proj.weight [7168, 14336]. - device_id: Target device ID for weight placement. - - Returns: - MTPPreprocessParams with converted weights for device_id. - """ + """Convert ref weights to TileRT format for a specific device.""" device = torch.device(f"cuda:{device_id}") embedding_rmsnorm_gamma, hidden_rmsnorm_gamma, eh_proj_weight = weights @@ -165,14 +154,7 @@ def init_reference_weights(self, state_dict: dict[str, torch.Tensor]) -> None: self.ref_eh_proj_weight = state_dict[self.ref_weights_alias.eh_proj] def init_tilert_weights(self, state_dict: dict[str, torch.Tensor]) -> None: - """ - Load TileRT weights from state_dict. - - state_dict may use: - - Full keys: layer_{layer_id}_{alias}_dev_{device_id} - - Short keys: embedding_rmsnorm_gamma, hidden_rmsnorm_gamma, eh_proj_weights - - Ref keys: enorm.weight, hnorm.weight, eh_proj.weight (then convert) - """ + """Load TileRT weights from state_dict.""" converter = MTPPreprocessWeightsConverter(self.model_args, self.num_devices) params = converter.convert_to_tilert( [state_dict[k] for k in self.tilert_weights_alias()], self.device_id @@ -199,16 +181,7 @@ def golden_forward( x: torch.Tensor, last_hidden_states: torch.Tensor, ) -> torch.Tensor: - """ - Reference forward: enorm(x), hnorm(last_hidden), concat & eh_proj. - - Args: - x: [batch, seq_len, hidden_size] embedded tokens - last_hidden_states: [batch, seq_len, hidden_size] previous hidden - - Returns: - [batch, seq_len, hidden_size] projected hidden - """ + """Reference forward: enorm(x), hnorm(last_hidden), concat & eh_proj.""" assert self.ref_embedding_rmsnorm_gamma is not None assert self.ref_hidden_rmsnorm_gamma is not None assert self.ref_eh_proj_weight is not None diff --git a/tilert/models/glm_5/ops/__init__.py b/tilert/models/glm_5/ops/__init__.py new file mode 100644 index 0000000..888b650 --- /dev/null +++ b/tilert/models/glm_5/ops/__init__.py @@ -0,0 +1,7 @@ +"""Core operations for GLM5.""" + +from tilert.models.glm_5.ops.sparse_index_v3 import sparse_index_topk_v3 + +__all__ = [ + "sparse_index_topk_v3", +] diff --git a/tilert/models/glm_5/ops/sparse_index_v3.py b/tilert/models/glm_5/ops/sparse_index_v3.py new file mode 100644 index 0000000..8e58362 --- /dev/null +++ b/tilert/models/glm_5/ops/sparse_index_v3.py @@ -0,0 +1,51 @@ +"""GLM5 sparse index op Python wrapper.""" + +import torch + +__all__ = [ + "sparse_index_topk_v3", +] + + +def sparse_index_topk_v3( + q: torch.Tensor, # noqa: VNE001 + kv: torch.Tensor, + weights: torch.Tensor, + logits: torch.Tensor, + indices: torch.Tensor, + cur_pos: int, + profile_logs: torch.Tensor, +) -> None: + """GLM5 sparse index + top-k selection.""" + if q.dtype != torch.bfloat16: + raise ValueError("input must be a bfloat16 tensor.") + if kv.dtype != torch.bfloat16: + raise ValueError("kv must be a bfloat16 tensor.") + if weights.dtype != torch.bfloat16: + raise ValueError("weights must be a bfloat16 tensor.") + if logits.dtype != torch.float32: + raise ValueError("logits must be a float32 tensor.") + + seqlen = q.shape[-3] + head = q.shape[-2] + dim = q.shape[-1] + + if head != 32: + raise ValueError( + f"Unsupported head size: {head}. SparseIndexV3 fused op " + "supports head number of 32 (GLM5)." + ) + if dim != 128: + raise ValueError("dim must be 128, as we precompute scale inner kernel") + + device = q.device + if any(t.device != device for t in (kv, weights, logits, indices, profile_logs)): + raise ValueError( + "sparse_index inputs must be on the same device: " + f"q={device}, kv={kv.device}, weights={weights.device}, " + f"logits={logits.device}, profile_logs={profile_logs.device}" + ) + workspace = torch.zeros(seqlen, (200 * 1024 + 260), dtype=torch.int32, device=device) + torch.ops.tilert.sparse_index_topk_glm5_v3_op( + q, kv, weights, logits, cur_pos, indices, workspace, profile_logs + ) diff --git a/tilert/models/glm_5/params.py b/tilert/models/glm_5/params.py new file mode 100644 index 0000000..2721229 --- /dev/null +++ b/tilert/models/glm_5/params.py @@ -0,0 +1 @@ +"""GLM5 parameters and initializers.""" diff --git a/tilert/models/glm_5/_dsa_v32/temp_var_indices.py b/tilert/models/glm_5/temp_var_indices.py similarity index 73% rename from tilert/models/glm_5/_dsa_v32/temp_var_indices.py rename to tilert/models/glm_5/temp_var_indices.py index 3a7af62..de33438 100644 --- a/tilert/models/glm_5/_dsa_v32/temp_var_indices.py +++ b/tilert/models/glm_5/temp_var_indices.py @@ -1,13 +1,4 @@ -"""Named indices for DSA temporary variables. - -Lets Python code reference temp_vars by name instead of magic numbers. - -Usage:: - - from tilert.models.glm_5._dsa_v32.temp_var_indices import Idx - - token_out = intermediates[Idx.TOKEN_OUT] # equivalent to intermediates[25] -""" +"""Named indices for DSA temporary variables.""" from enum import IntEnum @@ -26,7 +17,7 @@ class DsaTempVarIdx(IntEnum): IDX_LOGITS = 8 IDX_SELECTS = 9 Q_NOPE = 10 - O = 11 # noqa: E741 + O = 11 # noqa: E741 — mirrors C++ DsaTempVars::O O_ACC = 12 O_LSE = 13 O_LSE_ACC = 14 @@ -71,24 +62,19 @@ class DsaTempVarIdx(IntEnum): TOP_N_LOG_PROBS = 53 TOP_N_INDICES = 54 LOGPROBS_FLAG = 55 + AR_ACCEPTED_TOKENS = 56 + AR_NUM_ACCEPTED = 57 + HIDDEN_MID = 58 + GRAMMAR_BITMASK = 59 -TEMP_VARS_SIZE = 56 +TEMP_VARS_SIZE = 60 Idx = DsaTempVarIdx def validate_temp_vars_layout() -> None: - """Validate the temporary-variable index enum. - - Checks: - 1. Enum member count equals TEMP_VARS_SIZE. - 2. Indices are contiguous 0..TEMP_VARS_SIZE-1 with no gaps or duplicates. - 3. (If the backend is loaded) the backend temp_vars_size matches TEMP_VARS_SIZE. - - Raises: - RuntimeError: If any validation check fails. - """ + """Validate the temporary-variable index enum.""" members = list(DsaTempVarIdx) if len(members) != TEMP_VARS_SIZE: @@ -112,7 +98,8 @@ def validate_temp_vars_layout() -> None: cpp_size = torch.ops.tilert.dsa_temp_vars_size() if cpp_size != TEMP_VARS_SIZE: raise RuntimeError( - f"TEMP_VARS_SIZE={TEMP_VARS_SIZE} != " f"backend temp_vars_size={cpp_size}" + f"Python TEMP_VARS_SIZE={TEMP_VARS_SIZE} != " + f"C++ DsaTempVars::temp_vars_size={cpp_size}" ) except (AttributeError, RuntimeError): pass From 18e7fe387b12d825df8154c01a3f22580d1e1c4d Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 18:33:45 +0000 Subject: [PATCH 02/10] pd_vllm impl --- tilert/pd_vllm/__init__.py | 1 + tilert/pd_vllm/decode_server.py | 305 +++++++++++++++++++ tilert/pd_vllm/engine_iface.py | 53 ++++ tilert/pd_vllm/oai_parser.py | 226 ++++++++++++++ tilert/pd_vllm/pd_router.py | 435 +++++++++++++++++++++++++++ tilert/pd_vllm/prefill_connector.py | 304 +++++++++++++++++++ tilert/pd_vllm/profiles/__init__.py | 1 + tilert/pd_vllm/profiles/base.py | 81 +++++ tilert/pd_vllm/profiles/dsv32.py | 35 +++ tilert/pd_vllm/profiles/glm5.py | 43 +++ tilert/pd_vllm/profiles/mla_nsa.py | 450 ++++++++++++++++++++++++++++ tilert/pd_vllm/receive_server.py | 172 +++++++++++ tilert/pd_vllm/transport.py | 107 +++++++ tilert/pd_vllm/wire.py | 81 +++++ 14 files changed, 2294 insertions(+) create mode 100644 tilert/pd_vllm/__init__.py create mode 100644 tilert/pd_vllm/decode_server.py create mode 100644 tilert/pd_vllm/engine_iface.py create mode 100644 tilert/pd_vllm/oai_parser.py create mode 100644 tilert/pd_vllm/pd_router.py create mode 100644 tilert/pd_vllm/prefill_connector.py create mode 100644 tilert/pd_vllm/profiles/__init__.py create mode 100644 tilert/pd_vllm/profiles/base.py create mode 100644 tilert/pd_vllm/profiles/dsv32.py create mode 100644 tilert/pd_vllm/profiles/glm5.py create mode 100644 tilert/pd_vllm/profiles/mla_nsa.py create mode 100644 tilert/pd_vllm/receive_server.py create mode 100644 tilert/pd_vllm/transport.py create mode 100644 tilert/pd_vllm/wire.py diff --git a/tilert/pd_vllm/__init__.py b/tilert/pd_vllm/__init__.py new file mode 100644 index 0000000..2266c0c --- /dev/null +++ b/tilert/pd_vllm/__init__.py @@ -0,0 +1 @@ +"""vLLM-prefill + TileRT-decode PD disaggregation for GLM-5 and DeepSeek-V3.2.""" diff --git a/tilert/pd_vllm/decode_server.py b/tilert/pd_vllm/decode_server.py new file mode 100644 index 0000000..ddaa039 --- /dev/null +++ b/tilert/pd_vllm/decode_server.py @@ -0,0 +1,305 @@ +"""PD decode server (W6): HTTP orchestration around receive -> convert -> +inject -> decode. + +Internal token-level API (the client-facing OpenAI layer lives in pd_router / +a later serving layer): + + POST /pd/decode {rid, first_token_id, max_tokens, sampling?, timeout_s?} + Waits for the wire transfer of `rid` to complete, converts, injects + into the engine, decodes, returns {"rid", "token_ids", "timing_ms"}. + GET /health {"status": "ok"} + GET /decode_status {"status": "idle"|"busy", "current_rid": ...} + +bs=1: a busy server answers 429 immediately (the router's gated dispatch +should make that unreachable). +""" + +import argparse +import json +import logging +import queue as queue_mod +import threading +import time + +import uvicorn +from fastapi import FastAPI +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel + +from tilert.pd_vllm.receive_server import ReceiveServer + +logger = logging.getLogger("pd_vllm.decode_server") + + +class DecodeBody(BaseModel): + rid: str + first_token_id: int + max_tokens: int = 256 + sampling: dict | None = None + timeout_s: float = 120.0 + stream: bool = False + + +def build_app(server: ReceiveServer, engine) -> FastAPI: + app = FastAPI() + lock = threading.Lock() + state = {"current_rid": None} + + @app.get("/health") + def health(): + return {"status": "ok"} + + @app.get("/decode_status") + def decode_status(): + busy = lock.locked() + return {"status": "busy" if busy else "idle", + "current_rid": state["current_rid"]} + + @app.post("/pd/cancel") + def pd_cancel(body: dict): + """Explicit kill switch: cancel the in-flight decode for `rid`. + + Deterministic cancel path — dead-connection detection at the + transport layer is unreliable (asyncio writes to a closed socket + do not raise), so the router calls this on client disconnect. + """ + rid = body.get("rid") + ev = state.get("cancel_event") + if rid and rid == state["current_rid"] and ev is not None: + ev.set() + logger.info("cancel requested for %s", rid) + return {"cancelled": rid} + return JSONResponse({"error": "no matching in-flight request", + "current_rid": state["current_rid"]}, + status_code=404) + + def _cleanup(): + try: + engine.reset() + except Exception: + logger.exception("engine reset failed") + server.release() + state["current_rid"] = None + state["cancel_event"] = None + lock.release() + + def _log_reqstat(body, req, n_tokens, timing): + logger.info("REQSTAT rid=%s seq=%d completion=%d %s", + body.rid, req.seq_len, n_tokens, + " ".join(f"{k}={v}" for k, v in timing.items())) + + @app.post("/pd/decode") + def pd_decode(body: DecodeBody): + if not lock.acquire(blocking=False): + return JSONResponse( + {"error": "busy", "current_rid": state["current_rid"]}, + status_code=429) + state["current_rid"] = body.rid + t0 = time.time() + # phase 1: wire wait + convert + inject (common to both modes) + try: + # Drain until OUR rid arrives; drop stale completed entries + # (e.g. a transfer whose consumer never called /pd/decode). + req = None + deadline = time.time() + body.timeout_s + while time.time() < deadline: + try: + cand = server.completed.get( + timeout=max(0.1, deadline - time.time())) + except queue_mod.Empty: + break + if cand.rid == body.rid: + req = cand + break + logger.warning("dropping unmatched request %s " + "(waiting for %s)", cand.rid, body.rid) + server.release() + if req is None: + _cleanup() + return JSONResponse( + {"error": "kv_transfer_timeout", "rid": body.rid}, + status_code=504) + t_recv = time.time() + conv = server.profile.convert( + server.buffer, server.base_ptr, server.max_seq_len, + req, server.profile.num_ranks) + t_conv = time.time() + engine.inject(conv) + t_inj = time.time() + except Exception as e: + logger.exception("prepare failed for %s", body.rid) + _cleanup() + return JSONResponse({"error": str(e), "rid": body.rid}, + status_code=500) + + pre_timing = { + "wire_wait": round(1000 * (t_recv - t0), 1), + "convert": round(1000 * (t_conv - t_recv), 1), + "inject": round(1000 * (t_inj - t_conv), 1), + } + + # phase 2: decode + cancel = threading.Event() + state["cancel_event"] = cancel + + if not body.stream: + try: + tokens = engine.decode( + first_token_id=body.first_token_id, + max_tokens=body.max_tokens, + sampling=body.sampling, + cancel_event=cancel, + ) + timing = {**pre_timing, + "decode": round(1000 * (time.time() - t_inj), 1), + **getattr(engine, "last_stats", {})} + _log_reqstat(body, req, len(tokens), timing) + return {"rid": body.rid, "token_ids": tokens, + "seq_len": req.seq_len, "timing_ms": timing} + except Exception as e: + logger.exception("decode failed for %s", body.rid) + return JSONResponse({"error": str(e), "rid": body.rid}, + status_code=500) + finally: + _cleanup() + + # streaming: ndjson lines {"t":[ids...]}* then {"done":true,...}; + # lock/engine ownership transfers to the generator. + q: queue_mod.Queue = queue_mod.Queue() + + def _run(): + try: + tokens = engine.decode( + first_token_id=body.first_token_id, + max_tokens=body.max_tokens, + sampling=body.sampling, + on_token=q.put, + cancel_event=cancel, + ) + q.put(("done", tokens)) + except Exception as e: # pragma: no cover + logger.exception("stream decode failed for %s", body.rid) + q.put(("error", str(e))) + + worker = threading.Thread(target=_run, name="pd-decode", daemon=True) + + async def _gen(): + # MUST be an async generator: on client disconnect starlette + # cancels the response task, and only async generators get the + # cancellation delivered into their frame so `finally` runs + # (a sync generator is silently abandoned -> the engine slot + # leaks forever; found by the streaming-cancel drill). + import asyncio + + import anyio + from starlette.concurrency import run_in_threadpool + + worker.start() + try: + batch: list[int] = [] + done_msg = None + last_activity = time.time() + while done_msg is None: + drained = False + while True: + try: + item = q.get_nowait() + except queue_mod.Empty: + break + drained = True + if isinstance(item, int): + batch.append(item) + else: + done_msg = item + break + if batch: + yield json.dumps({"t": batch}) + "\n" + batch = [] + if done_msg is None: + if drained: + last_activity = time.time() + elif time.time() - last_activity > 600: + yield json.dumps({"error": "decode stalled"}) + "\n" + return + else: + await asyncio.sleep(0.005) + kind, payload = done_msg + if kind == "done": + timing = {**pre_timing, + "decode": round(1000 * (time.time() - t_inj), 1), + **getattr(engine, "last_stats", {})} + _log_reqstat(body, req, len(payload), timing) + yield json.dumps({ + "done": True, "n": len(payload), + "seq_len": req.seq_len, + "finish_reason": timing.get("finish_reason", "stop"), + "timing_ms": timing}) + "\n" + else: + yield json.dumps({"error": payload}) + "\n" + finally: + cancel.set() + # shield: cleanup must complete even inside a cancelled scope, + # and the worker must be joined before engine.reset() (the + # engine may be mid-decode_mtp on the GPU). + with anyio.CancelScope(shield=True): + await run_in_threadpool(worker.join, 120) + if worker.is_alive(): + logger.error("decode worker failed to stop for %s", + body.rid) + _cleanup() + + return StreamingResponse(_gen(), media_type="application/x-ndjson") + + return app + + +def main() -> None: + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(name)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--engine", choices=["stub", "tilert"], default="stub") + ap.add_argument("--model", default="glm5", help="model profile") + ap.add_argument("--max-seq-len", type=int, default=4096) + ap.add_argument("--ctrl-port", type=int, default=5556) + ap.add_argument("--http-port", type=int, default=5557) + ap.add_argument("--model-weights-dir", default="") + ap.add_argument("--with-mtp", action="store_true") + ap.add_argument("--transport", choices=["mooncake", "nixl"], + default="mooncake", help="RDMA data-plane backend " + "(must match prefill's tilert_transport)") + ap.add_argument("--kv-cache-dtype", default="fp8_ds_mla", + help="MLA cache dtype (must match vLLM prefill); " + "MLA-family profiles only") + args = ap.parse_args() + + from tilert.pd_vllm.profiles import base as profiles + profile = profiles.get_profile(args.model) + # MLA-family profiles (glm5/dsv32) need the cache dtype to size the receive + # buffer. + if hasattr(profile, "configure"): + profile.configure(args.kv_cache_dtype) + logger.info("profile %s MLA cache dtype = %s (layout v%d)", + profile.name, args.kv_cache_dtype, profile.layout_version) + + if args.engine == "stub": + from tilert.pd_vllm.engine_iface import StubEngine + engine = StubEngine() + else: + logger.info("loading TileRT engine (profile=%s, weights=%s)...", + profile.name, args.model_weights_dir) + engine = profile.build_engine( + model_weights_dir=args.model_weights_dir, + max_seq_len=args.max_seq_len, with_mtp=args.with_mtp, + ar_steps=8) + logger.info("TileRT engine ready (cache window %d)", engine.max_seq_len) + + server = ReceiveServer(profile, max_seq_len=args.max_seq_len, + ctrl_port=args.ctrl_port, transport=args.transport) + app = build_app(server, engine) + logger.info("decode server on :%d (profile=%s, engine=%s, ctrl=:%d)", + args.http_port, profile.name, args.engine, args.ctrl_port) + uvicorn.run(app, host="::", port=args.http_port, log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/tilert/pd_vllm/engine_iface.py b/tilert/pd_vllm/engine_iface.py new file mode 100644 index 0000000..9719e50 --- /dev/null +++ b/tilert/pd_vllm/engine_iface.py @@ -0,0 +1,53 @@ +"""Engine seam for the PD decode server (model-agnostic). + +``PDEngine`` is the interface the decode server drives; concrete adapters are +built by the active model profile (``profile.build_engine(...)``). +``StubEngine`` runs the whole serving path with no GPU / no tilert. +""" + +from typing import Any, Callable, Protocol + + +class PDEngine(Protocol): + def inject(self, req: Any) -> None: + """Restore engine state to 'prefilled seq_len tokens' from req.""" + + def decode( + self, + first_token_id: int, + max_tokens: int, + sampling: dict | None, + on_token: Callable[[int], None] | None = None, + cancel_event=None, + ) -> list[int]: + """AR/MTP decode from first_token_id; returns completion ids + (incl. first_token_id, excl. stop token). on_token never fires for + stop tokens; cancel_event stops early; last_stats['finish_reason'] is + 'stop' | 'length' | 'cancelled'.""" + + def reset(self) -> None: + """Release per-request state.""" + + +class StubEngine: + """Echo engine for plumbing tests: no GPU, no tilert.""" + + def __init__(self, fixed_tokens: tuple[int, ...] = (11, 22, 33)): + self._fixed = fixed_tokens + self.injected: Any = None + self.last_stats: dict = {} + + def inject(self, req: Any) -> None: + self.injected = req + + def decode(self, first_token_id, max_tokens, sampling, on_token=None, + cancel_event=None): + out = ([int(first_token_id)] + list(self._fixed))[:max_tokens] + if on_token: + for t in out: + on_token(t) + self.last_stats = {"finish_reason": "stop"} + return out + + def reset(self) -> None: + self.injected = None diff --git a/tilert/pd_vllm/oai_parser.py b/tilert/pd_vllm/oai_parser.py new file mode 100644 index 0000000..3840f43 --- /dev/null +++ b/tilert/pd_vllm/oai_parser.py @@ -0,0 +1,226 @@ +"""OpenAI-semantics parser adapter over vLLM's parser engine (decision B1). + +Wraps ``vllm.parser`` (the NEW engine architecture in vllm >= 0.24; the old +``ReasoningParser``/``ToolParserManager`` API is superseded) into the +small surface the router needs: + + parser = make_parser("glm47", tokenizer, thinking=True) + parsed = parser.parse_complete(text) # non-streaming + sess = parser.stream() # per-request streaming + events = sess.feed(delta_text); sess.finish() # normalized event dicts + +Runs in the ROUTER environment only — that env must have vllm installed +(CPU-only import is fine; verified with CUDA_VISIBLE_DEVICES=""). The decode +node never imports vllm. +""" + +import json +import logging +import uuid +from dataclasses import dataclass, field + +logger = logging.getLogger("pd_vllm.oai_parser") + + +@dataclass +class ToolCall: + id: str + name: str + arguments: str # JSON string (OpenAI convention) + + def to_openai(self, index: int) -> dict: + return { + "index": index, + "id": self.id, + "type": "function", + "function": {"name": self.name, "arguments": self.arguments}, + } + + +@dataclass +class Parsed: + reasoning_content: str | None + content: str | None + tool_calls: list[ToolCall] = field(default_factory=list) + + +def _new_call_id() -> str: + return f"call_{uuid.uuid4().hex[:24]}" + + +# family -> (config-builder import path, arg-converter import path). The +# glm47_moe parser engine uses the vllm.parser API shape (a `*_config(thinking)` +# builder + a `_*_arg_converter(raw, partial)`); the adapter picks the engine +# by family name. +_FAMILIES = { + "glm47": ("vllm.parser.glm47_moe", "glm47_moe_config", + "_glm47_arg_converter"), +} + + +def make_parser(family: str, tokenizer, thinking: bool = True) -> "OaiParser": + if family not in _FAMILIES: + raise KeyError(f"unknown parser family {family!r}; " + f"known: {sorted(_FAMILIES)}") + return OaiParser(family, tokenizer, thinking) + + +class OaiParser: + """Family-parameterized parser; one instance per model, ``stream()`` per + request. Family is a vllm.parser engine (glm47).""" + + def __init__(self, family: str, tokenizer, thinking: bool = True): + import importlib + + from vllm.parser.engine.events import EventType + from vllm.parser.engine.streaming_parser_engine import ( + StreamingParserEngine, + ) + + mod_name, cfg_name, conv_name = _FAMILIES[family] + mod = importlib.import_module(mod_name) + self._family = family + self._cfg_fn = getattr(mod, cfg_name) + self._Engine = StreamingParserEngine + self._ET = EventType + self._config = self._cfg_fn(thinking=thinking) + self._convert = getattr(mod, conv_name) + self._tok = tokenizer + + def with_thinking(self, thinking: bool) -> "OaiParser": + if thinking == (self._config.initial_state.name == "REASONING"): + return self + clone = object.__new__(OaiParser) + clone.__dict__.update(self.__dict__) + clone._config = self._cfg_fn(thinking=thinking) + return clone + + # ── non-streaming ──────────────────────────────────────────────────── + def parse_complete(self, text: str) -> Parsed: + engine = self._Engine(self._config, self._tok) + return self._reduce(engine.parse_complete(text)) + + def _reduce(self, events) -> Parsed: + ET = self._ET + reasoning, content = [], [] + slots: dict[int, dict] = {} + for e in events: + if e.type == ET.REASONING_CHUNK: + reasoning.append(e.value) + elif e.type == ET.TEXT_CHUNK: + content.append(e.value) + elif e.type in (ET.TOOL_NAME, ET.ARG_VALUE_CHUNK): + s = slots.setdefault(e.tool_index, {"name": [], "args": []}) + s["name" if e.type == ET.TOOL_NAME else "args"].append(e.value) + calls = [] + for i in sorted(slots): + name = "".join(slots[i]["name"]).strip() + if not name: + continue # unnamed fragment (heavy truncation) — drop + raw = "".join(slots[i]["args"]) + calls.append(ToolCall(_new_call_id(), name, self._convert(raw, True))) + r = "".join(reasoning) + c = "".join(content) + return Parsed(r if r else None, c if c else None, calls) + + # ── streaming ──────────────────────────────────────────────────────── + def stream(self) -> "OaiStream": + return OaiStream(self) + + +class OaiStream: + """Per-request streaming session. + + ``feed``/``finish`` return normalized event dicts: + {"kind": "reasoning", "text": ...} + {"kind": "content", "text": ...} + {"kind": "tool", "index": i, "id": ..., "name": ..., "arguments": ...} + + Reasoning/content stream through per delta. Tool calls are buffered and + emitted whole at TOOL_CALL_END (OpenAI clients accept arguments in any + fragmentation; whole-call emission sidesteps XML→JSON incremental + conversion). ``finish`` flushes a truncated trailing tool call with + partial-args conversion. + """ + + def __init__(self, parent: "OaiParser"): + self._p = parent + self._engine = parent._Engine(parent._config, parent._tok) + self._slots: dict[int, dict] = {} + self._emitted: set[int] = set() + + def feed(self, delta_text: str) -> list[dict]: + if not delta_text: + return [] + return self._consume(self._engine.feed(delta_text, [])) + + def finish(self) -> list[dict]: + out = self._consume(self._engine.finish()) + # flush truncated trailing tool call (never saw TOOL_CALL_END) + for i in sorted(self._slots): + if i in self._emitted: + continue + ev = self._flush_tool(i, partial=True) + if ev: + out.append(ev) + return out + + def _consume(self, events) -> list[dict]: + ET = self._p._ET + out: list[dict] = [] + for e in events: + if e.type == ET.REASONING_CHUNK: + out.append({"kind": "reasoning", "text": e.value}) + elif e.type == ET.TEXT_CHUNK: + out.append({"kind": "content", "text": e.value}) + elif e.type in (ET.TOOL_NAME, ET.ARG_VALUE_CHUNK): + s = self._slots.setdefault(e.tool_index, {"name": [], "args": []}) + s["name" if e.type == ET.TOOL_NAME else "args"].append(e.value) + elif e.type == ET.TOOL_CALL_END: + ev = self._flush_tool(e.tool_index, partial=False) + if ev: + out.append(ev) + return out + + def _flush_tool(self, index: int, partial: bool) -> dict | None: + s = self._slots.get(index) + if s is None or index in self._emitted: + return None + name = "".join(s["name"]).strip() + if not name: + return None + self._emitted.add(index) + args = self._p._convert("".join(s["args"]), partial) + return {"kind": "tool", "index": index, "id": _new_call_id(), + "name": name, "arguments": args} + + +class IncrementalDetok: + """Incremental token→text for byte-level BPE tokenizers. + + Decodes a bounded trailing window; holds output while the window ends in + a partial multi-byte sequence (\\ufffd). Window folding is safe for + byte-level BPE: separate windows decode to concatenable byte streams. + Specials are KEPT (skip_special_tokens=False) — the parser consumes + etc.; the stop token never reaches the stream (engine adapter + suppresses it). + """ + + _FOLD = 256 + + def __init__(self, tokenizer): + self._tok = tokenizer + self._ids: list[int] = [] + self._emitted = 0 + + def push(self, ids: list[int]) -> str: + self._ids.extend(ids) + text = self._tok.decode(self._ids, skip_special_tokens=False) + if text.endswith("�"): + return "" + delta = text[self._emitted:] + self._emitted = len(text) + if len(self._ids) > self._FOLD: + self._ids = [] + self._emitted = 0 + return delta diff --git a/tilert/pd_vllm/pd_router.py b/tilert/pd_vllm/pd_router.py new file mode 100644 index 0000000..d373261 --- /dev/null +++ b/tilert/pd_vllm/pd_router.py @@ -0,0 +1,435 @@ +"""PD router (W6): client-facing entry that orchestrates vLLM prefill and +TileRT decode, with OpenAI-semantics output parsing (reasoning + tool calls), +streaming and non-streaming. + +Flow per request (phase-1 hybrid, see design doc): + 1. pick a free decode node (in-memory busy tracking; all busy -> 429) + 2. forward to vLLM with max_tokens=1 + logprobs and inject + kv_transfer_params {tilert_host, tilert_ctrl_port} — the connector + claims the request and RDMA-sends state to the decode node + 3. extract rid + first_token_id from the vLLM response + (requires vLLM serve launched with --return-tokens-as-token-ids) + 4. call the decode node (/pd/decode; stream or not) and assemble the + OpenAI response: reasoning_content / content / tool_calls via the + vLLM parser engine (decision B1 — this process's env has vllm + installed, CPU-only; the decode node does not). + +Environment: run in a vllm-equipped env with CUDA_VISIBLE_DEVICES="" (the +router must never touch GPUs). --parser none falls back to raw passthrough. + +Run: + CUDA_VISIBLE_DEVICES= python -m tilert.pd_vllm.pd_router \ + --vllm-url http://prefill-node:8000 \ + --decode decode-node:5556:5557 --port 23333 \ + --model-path /path/to/GLM-5.1 --parser glm47 +""" + +import argparse +import json +import logging +import threading +import time + +import requests +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from tilert.pd_vllm.wire import derive_rid + +logger = logging.getLogger("pd_vllm.router") + + +class DecodeNode: + def __init__(self, host: str, ctrl_port: int, http_port: int): + self.host = host + self.ctrl_port = ctrl_port + self.http_port = http_port + self.busy = False + + @property + def http_base(self) -> str: + return f"http://{self.host}:{self.http_port}" + + +class Pool: + def __init__(self, nodes: list[DecodeNode]): + self.nodes = nodes + self._lock = threading.Lock() + + def acquire(self) -> DecodeNode | None: + with self._lock: + for n in self.nodes: + if not n.busy: + n.busy = True + return n + return None + + def release(self, node: DecodeNode) -> None: + with self._lock: + node.busy = False + + +def first_token_from_logprobs(resp: dict, is_chat: bool) -> int: + """Parse 'token_id:N' (vLLM --return-tokens-as-token-ids) from logprobs.""" + choice = resp["choices"][0] + lp = choice.get("logprobs") or {} + tok: str | None = None + if is_chat: + content = lp.get("content") or [] + if content: + tok = content[0].get("token") + else: + toks = lp.get("tokens") or [] + if toks: + tok = toks[0] + if tok and tok.startswith("token_id:"): + return int(tok.split(":", 1)[1]) + raise ValueError( + f"cannot extract first token id from logprobs ({tok!r}); launch vLLM " + f"with --return-tokens-as-token-ids and request logprobs") + + +def _thinking_enabled(body: dict) -> bool: + ctk = body.get("chat_template_kwargs") or {} + return bool(ctk.get("enable_thinking", True)) + + +class RouterCtx: + """Immutable per-process context (tokenizer, parser factory, config).""" + + def __init__(self, vllm_url: str, pool: Pool, tokenizer, parser_name: str): + self.vllm_url = vllm_url + self.pool = pool + self.tokenizer = tokenizer + self.parser_name = parser_name + self._parsers = {} + if parser_name != "none": + if tokenizer is None: + raise SystemExit("--parser requires --model-path (tokenizer)") + from tilert.pd_vllm.oai_parser import make_parser + self._parsers[True] = make_parser(parser_name, tokenizer, + thinking=True) + self._parsers[False] = self._parsers[True].with_thinking(False) + logger.info("parser '%s' ready (thinking variants cached)", + parser_name) + + def parser(self, thinking: bool): + return self._parsers.get(thinking) + + +def build_app(ctx: RouterCtx) -> FastAPI: + app = FastAPI() + pool = ctx.pool + + @app.get("/health") + def health(): + return {"status": "ok", + "decode_free": sum(1 for n in pool.nodes if not n.busy)} + + @app.get("/pool_status") + def pool_status(): + return {"nodes": [{"host": n.host, "busy": n.busy} + for n in pool.nodes]} + + # ── shared prefill step ────────────────────────────────────────────── + def _prefill(path, body, node): + prefill_body = dict(body) + prefill_body["max_tokens"] = 1 + prefill_body["stream"] = False + if path.endswith("chat/completions"): + prefill_body["logprobs"] = True + prefill_body["top_logprobs"] = 1 + else: + prefill_body["logprobs"] = 1 + prefill_body["kv_transfer_params"] = { + "tilert_host": node.host, + "tilert_ctrl_port": node.ctrl_port, + } + r = requests.post(f"{ctx.vllm_url}{path}", json=prefill_body, + timeout=600) + r.raise_for_status() + return r.json() + + def _sampling_of(body): + return {k: body[k] for k in ("temperature", "top_p", "top_k") + if k in body} + + def _max_tokens_of(body): + return int(body.get("max_tokens") + or body.get("max_completion_tokens") or 256) + + # ── non-streaming ──────────────────────────────────────────────────── + def _handle(path: str, body: dict): + is_chat = path.endswith("chat/completions") + node = pool.acquire() + if node is None: + return JSONResponse({"error": "all decode nodes busy"}, + status_code=429) + t0 = time.time() + try: + prefill = _prefill(path, body, node) + t_prefill = time.time() + rid = derive_rid(prefill["id"]) + first_token_id = first_token_from_logprobs(prefill, is_chat) + + dr = requests.post(f"{node.http_base}/pd/decode", json={ + "rid": rid, + "first_token_id": first_token_id, + "max_tokens": _max_tokens_of(body), + "sampling": _sampling_of(body), + }, timeout=600) + dr.raise_for_status() + decode = dr.json() + token_ids = decode["token_ids"] + timing = decode.get("timing_ms", {}) + finish = timing.get("finish_reason", "stop") + if finish == "cancelled": + finish = "stop" + + choice: dict = {"index": 0, "finish_reason": finish} + parser = ctx.parser(_thinking_enabled(body)) if is_chat else None + if parser is not None: + text = ctx.tokenizer.decode(token_ids, + skip_special_tokens=False) + parsed = parser.parse_complete(text) + msg = {"role": "assistant", "content": parsed.content or ""} + if parsed.reasoning_content: + msg["reasoning_content"] = parsed.reasoning_content + if parsed.tool_calls: + msg["tool_calls"] = [c.to_openai(i) for i, c + in enumerate(parsed.tool_calls)] + choice["finish_reason"] = "tool_calls" + choice["message"] = msg + else: + text = (ctx.tokenizer.decode(token_ids, + skip_special_tokens=True) + if ctx.tokenizer else None) + if is_chat: + choice["message"] = {"role": "assistant", "content": text} + else: + choice["text"] = text + choice["token_ids"] = token_ids + + return JSONResponse({ + "id": prefill["id"], + "object": ("chat.completion" if is_chat + else "text_completion"), + "created": int(time.time()), + "model": prefill.get("model"), + "choices": [choice], + "usage": { + "prompt_tokens": (prefill.get("usage") or {}) + .get("prompt_tokens"), + "completion_tokens": len(token_ids), + }, + "pd_timing_ms": { + "prefill": round(1000 * (t_prefill - t0), 1), + **timing, + }, + }) + except Exception as e: + logger.exception("pd request failed") + return JSONResponse({"error": str(e)}, status_code=502) + finally: + pool.release(node) + + # ── streaming (chat only) ──────────────────────────────────────────── + async def _handle_stream(path: str, body: dict, request: Request): + from starlette.concurrency import run_in_threadpool + + node = pool.acquire() + if node is None: + return JSONResponse({"error": "all decode nodes busy"}, + status_code=429) + + try: + prefill = await run_in_threadpool(_prefill, path, body, node) + rid = derive_rid(prefill["id"]) + first_token_id = first_token_from_logprobs(prefill, True) + except Exception as e: + pool.release(node) + logger.exception("pd stream request failed before streaming") + return JSONResponse({"error": str(e)}, status_code=502) + + chunk_id = prefill["id"] + model = prefill.get("model") + prompt_tokens = (prefill.get("usage") or {}).get("prompt_tokens") + parser = ctx.parser(_thinking_enabled(body)) + + def _chunk(delta: dict, finish=None, usage=None) -> str: + payload = { + "id": chunk_id, "object": "chat.completion.chunk", + "created": int(time.time()), "model": model, + "choices": [{"index": 0, "delta": delta, + "finish_reason": finish}], + } + if usage is not None: + payload["usage"] = usage + return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + + def _event_delta(ev: dict) -> dict: + if ev["kind"] == "reasoning": + return {"reasoning_content": ev["text"]} + if ev["kind"] == "content": + return {"content": ev["text"]} + return {"tool_calls": [{ + "index": ev["index"], "id": ev["id"], "type": "function", + "function": {"name": ev["name"], + "arguments": ev["arguments"]}, + }]} + + def _fire_cancel(): + try: + requests.post(f"{node.http_base}/pd/cancel", + json={"rid": rid}, timeout=5) + except Exception: + logger.warning("cancel POST failed for %s", rid) + + async def _gen(): + import anyio + import httpx + + from tilert.pd_vllm.oai_parser import IncrementalDetok + + n_tokens = 0 + saw_tool = False + finish_reason = "stop" + client_gone = False + completed_ok = False + detok = IncrementalDetok(ctx.tokenizer) + sess = parser.stream() if parser else None + client = httpx.AsyncClient(timeout=httpx.Timeout(600, read=600)) + try: + yield _chunk({"role": "assistant"}) + async with client.stream( + "POST", f"{node.http_base}/pd/decode", + json={"rid": rid, "first_token_id": first_token_id, + "max_tokens": _max_tokens_of(body), + "sampling": _sampling_of(body), + "stream": True}) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + # Deterministic client-liveness check: writes to a + # dead socket do NOT raise (verified by drill), so + # poll the ASGI disconnect state every line. + if await request.is_disconnected(): + client_gone = True + logger.info("client disconnected, cancelling %s", + rid) + break + if not line: + continue + msg = json.loads(line) + if "t" in msg: + n_tokens += len(msg["t"]) + text = detok.push(msg["t"]) + if not text: + continue + if sess is None: + yield _chunk({"content": text}) + continue + for ev in sess.feed(text): + if ev["kind"] == "tool": + saw_tool = True + yield _chunk(_event_delta(ev)) + elif "done" in msg: + finish_reason = msg.get("finish_reason", "stop") + if finish_reason == "cancelled": + finish_reason = "stop" + elif "error" in msg: + yield _chunk( + {"content": f"\n[decode error: {msg['error']}]"}) + finish_reason = "stop" + if client_gone: + logger.info("client gone mid-stream for %s", rid) + return # finally fires the cancel + if sess is not None: + for ev in sess.finish(): + if ev["kind"] == "tool": + saw_tool = True + yield _chunk(_event_delta(ev)) + if saw_tool: + finish_reason = "tool_calls" + yield _chunk({}, finish=finish_reason, usage={ + "prompt_tokens": prompt_tokens, + "completion_tokens": n_tokens, + }) + yield "data: [DONE]\n\n" + completed_ok = True + except Exception: + logger.exception("stream failed mid-flight for %s", rid) + finally: + # Runs under cancellation too (client disconnect cancels this + # task). Order matters: release first (sync, can't be + # cancelled), then best-effort cancel via a plain thread + # (an await here could be cancelled before firing), then a + # shielded aclose. + pool.release(node) + if not completed_ok: + threading.Thread(target=_fire_cancel, daemon=True).start() + with anyio.CancelScope(shield=True): + await client.aclose() + + return StreamingResponse(_gen(), media_type="text/event-stream") + + @app.post("/v1/chat/completions") + async def chat(request: Request): + from starlette.concurrency import run_in_threadpool + + body = await request.json() + if body.get("stream"): + return await _handle_stream("/v1/chat/completions", body, request) + # blocking work off the event loop (decode can take minutes) + return await run_in_threadpool(_handle, "/v1/chat/completions", body) + + @app.post("/v1/completions") + async def completions(request: Request): + from starlette.concurrency import run_in_threadpool + + body = await request.json() + if body.get("stream"): + return JSONResponse( + {"error": "streaming is supported on /v1/chat/completions"}, + status_code=400) + return await run_in_threadpool(_handle, "/v1/completions", body) + + return app + + +def main() -> None: + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(name)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--vllm-url", required=True) + ap.add_argument("--decode", nargs="+", required=True, + help="decode nodes as host:ctrl_port:http_port") + ap.add_argument("--host", default="0.0.0.0") + ap.add_argument("--port", type=int, default=23333) + ap.add_argument("--model-path", default="", + help="tokenizer path (required unless --parser none)") + ap.add_argument("--parser", choices=["glm47", "none"], + default="glm47", + help="output parser (reasoning + tool calls)") + args = ap.parse_args() + + nodes = [] + for spec in args.decode: + host, cport, hport = spec.rsplit(":", 2) + nodes.append(DecodeNode(host, int(cport), int(hport))) + + tokenizer = None + if args.model_path: + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained( + args.model_path, trust_remote_code=True) # nosec B615 + + ctx = RouterCtx(args.vllm_url, Pool(nodes), tokenizer, args.parser) + app = build_app(ctx) + logger.info("router on :%d -> vllm=%s, %d decode node(s), parser=%s", + args.port, args.vllm_url, len(nodes), args.parser) + uvicorn.run(app, host=args.host, port=args.port, log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/tilert/pd_vllm/prefill_connector.py b/tilert/pd_vllm/prefill_connector.py new file mode 100644 index 0000000..0237958 --- /dev/null +++ b/tilert/pd_vllm/prefill_connector.py @@ -0,0 +1,304 @@ +"""TileRT PD producer connector for vLLM prefill (model-agnostic framework). + +Loaded into vLLM via the official plugin surface: + + --kv-transfer-config '{ + "kv_connector": "TileRTConnector", + "kv_connector_module_path": "tilert.pd_vllm.prefill_connector", + "kv_role": "kv_producer", + "kv_connector_extra_config": {"tilert_host": "", + "tilert_ctrl_port": 5556, + "tilert_model": "glm5"} + }' + +Claim discipline (MultiConnector-safe): only requests whose +``kv_transfer_params`` carry ``tilert_host`` are claimed; everything else is a +strict no-op so a native connector can coexist. + +The connector owns the model-agnostic plumbing (claim, chunked-prefill +tracking, worker init, staging, background send, TCP handshake); all per-model +extraction / layout / RDMA planning is delegated to the selected model profile +(``tilert_model``, default ``glm5``). +""" + +import logging +import queue +import threading +from dataclasses import dataclass, field + +from vllm.distributed.kv_transfer.kv_connector.v1.base import ( + KVConnectorBase_V1, + KVConnectorMetadata, + SupportsHMA, +) + +from tilert.pd_vllm import wire +from tilert.pd_vllm.profiles import base as profiles +from tilert.pd_vllm.wire import derive_rid + +logger = logging.getLogger("pd_vllm.connector") + + +@dataclass +class _ReqMeta: + req_id: str + rid: str + num_tokens: int + last_prompt_token: int + block_ids_per_group: list + tilert_host: str + tilert_ctrl_port: int + sampling: dict | None = None + + +@dataclass +class TileRTMetadata(KVConnectorMetadata): + requests: list = field(default_factory=list) + + +@dataclass +class _Pending: + """Scheduler-side chunked-prefill accumulation.""" + req_id: str + prompt_token_ids: list + total_tokens: int + block_ids_per_group: list + params: dict + + +class TileRTConnector(KVConnectorBase_V1, SupportsHMA): + # ══════════════════════════ init ══════════════════════════ + + def __init__(self, vllm_config, role, kv_cache_config=None): + super().__init__(vllm_config, role, kv_cache_config) + extra = vllm_config.kv_transfer_config.kv_connector_extra_config or {} + self._default_host = extra.get("tilert_host") + self._default_port = int(extra.get("tilert_ctrl_port", 5556)) + self._sync_send = bool(extra.get("tilert_sync_send", False)) + self._max_seq = int(extra.get("tilert_max_seq_len", + vllm_config.model_config.max_model_len)) + self._profile = profiles.get_profile(extra.get("tilert_model", "glm5")) + self._transport_name = extra.get("tilert_transport", "mooncake") + + # scheduler-side + self._pending: dict[str, _Pending] = {} + + # worker-side (lazy) + self._kv_caches: dict = {} + self._reg = None # profile registration (layer map) + self._tp_rank: int | None = None + self._transport = None + self._staging = None + self._send_q: queue.Queue = queue.Queue() + self._sender_thread: threading.Thread | None = None + + logger.info("TileRTConnector: role=%s profile=%s target=%s:%s sync=%s", + role, self._profile.name, self._default_host, + self._default_port, self._sync_send) + + # ══════════════════════ scheduler side ═════════════════════ + + @staticmethod + def _claim(params) -> dict | None: + """Return kv_transfer_params if this request is ours, else None.""" + if params and isinstance(params, dict) and params.get("tilert_host"): + return params + return None + + def _params_of(self, new_req) -> dict | None: + sp = getattr(new_req, "sampling_params", None) + extra = getattr(sp, "extra_args", None) if sp is not None else None + if extra: + return self._claim(extra.get("kv_transfer_params")) + return None + + def get_num_new_matched_tokens(self, request, num_computed_tokens): + return 0, False + + def update_state_after_alloc(self, request, blocks, num_external_tokens): + pass + + def build_connector_meta(self, scheduler_output) -> KVConnectorMetadata: + meta = TileRTMetadata() + num_sched = scheduler_output.num_scheduled_tokens or {} + + for req_id in scheduler_output.finished_req_ids: + self._pending.pop(req_id, None) + for req_id in getattr(scheduler_output, "preempted_req_ids", None) or []: + self._pending.pop(req_id, None) + + for new_req in scheduler_output.scheduled_new_reqs: + params = self._params_of(new_req) + if params is None: + continue # not ours — strict no-op (MultiConnector safety) + token_ids = list(new_req.prompt_token_ids or []) + if not token_ids: + continue + groups = [list(g) for g in new_req.block_ids] + n = num_sched.get(new_req.req_id, 0) + if new_req.num_computed_tokens + n >= len(token_ids): + meta.requests.append(self._emit(new_req.req_id, token_ids, + groups, params)) + else: + self._pending[new_req.req_id] = _Pending( + req_id=new_req.req_id, prompt_token_ids=token_ids, + total_tokens=len(token_ids), block_ids_per_group=groups, + params=params) + + cached = scheduler_output.scheduled_cached_reqs + for i, req_id in enumerate(getattr(cached, "req_ids", []) or []): + p = self._pending.get(req_id) + if p is None: + continue + new_blocks = cached.new_block_ids[i] + if new_blocks is not None: + for gi, g in enumerate(new_blocks): + if gi < len(p.block_ids_per_group) and g: + p.block_ids_per_group[gi].extend(g) + n = num_sched.get(req_id, 0) + if cached.num_computed_tokens[i] + n >= p.total_tokens: + meta.requests.append(self._emit(req_id, p.prompt_token_ids, + p.block_ids_per_group, p.params)) + del self._pending[req_id] + return meta + + def _emit(self, req_id, token_ids, groups, params) -> _ReqMeta: + m = _ReqMeta( + req_id=req_id, + rid=derive_rid(req_id), + num_tokens=len(token_ids), + last_prompt_token=int(token_ids[-1]), + block_ids_per_group=groups, + tilert_host=params.get("tilert_host") or self._default_host, + tilert_ctrl_port=int(params.get("tilert_ctrl_port", + self._default_port)), + sampling=params.get("sampling"), + ) + logger.info("claimed %s (rid=%s, %d tokens) -> %s:%d", + req_id, m.rid, m.num_tokens, m.tilert_host, m.tilert_ctrl_port) + return m + + def request_finished(self, request, block_ids): + self._pending.pop(getattr(request, "request_id", ""), None) + return False, None + + def request_finished_all_groups(self, request, block_ids): + return self.request_finished(request, block_ids) + + # ══════════════════════ worker side ════════════════════════ + + def register_kv_caches(self, kv_caches): + self._kv_caches = kv_caches + cfg = getattr(self, "_kv_cache_config", None) + self._reg = self._profile.classify_layers(kv_caches, cfg) + + def _ensure_worker_ready(self) -> None: + if self._transport is not None: + return + import torch + from vllm.distributed import get_tensor_model_parallel_rank + self._tp_rank = get_tensor_model_parallel_rank() + + from tilert.pd_vllm.transport import make_transport + + hostname = wire.local_ip() + total = self._profile.staging_bytes(self._reg, self._tp_rank, self._max_seq) + dev = torch.cuda.current_device() + self._staging = torch.zeros(total, dtype=torch.uint8, device=f"cuda:{dev}") + + self._transport = make_transport(self._transport_name) + self._transport.init(hostname) + self._transport.register(self._staging.data_ptr(), total, dev) + + if not self._sync_send: + self._sender_thread = threading.Thread( + target=self._sender_loop, name="tilert-pd-sender", daemon=True) + self._sender_thread.start() + logger.info("worker ready: rank=%d transport=%s staging=%.1f MB profile=%s", + self._tp_rank, self._transport.name, total / 1e6, + self._profile.name) + + def start_load_kv(self, forward_context, **kwargs): + pass + + def wait_for_layer_load(self, layer_name): + pass + + def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs): + pass + + def wait_for_save(self): + metadata = self._get_connector_metadata() + if not isinstance(metadata, TileRTMetadata) or not metadata.requests: + return + self._ensure_worker_ready() + if self._tp_rank not in self._profile.sender_ranks: + return # this rank does not participate (e.g. replicated MLA) + for m in metadata.requests: + try: + sections = self._profile.extract( + self._reg, m, self._tp_rank, self._staging, self._max_seq) + except Exception: + logger.exception("extraction failed for %s", m.rid) + continue + job = {"meta": m, "sections": sections, "seq": sections["seq"]} + if self._sync_send: + self._send(job) + else: + self._send_q.put(job) + + def get_finished(self, finished_req_ids): + return None, None + + # ── background send ── + + def _sender_loop(self) -> None: + while True: + job = self._send_q.get() + try: + self._send(job) + except Exception: + logger.exception("send failed for %s", job["meta"].rid) + + def _send(self, job: dict) -> None: + import socket as _socket + import time as _time + + m: _ReqMeta = job["meta"] + seq = job["seq"] + t0 = _time.time() + conn = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + conn.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) + conn.settimeout(60) + conn.connect((m.tilert_host, m.tilert_ctrl_port)) + hello = wire.recv_msg(conn) + assert hello.get("magic") == wire.MAGIC, f"bad hello: {hello}" + assert hello.get("layout_version") == self._profile.layout_version, \ + (f"layout version mismatch: {hello.get('layout_version')} " + f"vs {self._profile.layout_version}") + assert hello.get("transport") == self._transport.name, \ + (f"transport mismatch: decode={hello.get('transport')} " + f"vs prefill={self._transport.name}") + remote_max_seq = int(hello["max_seq_len"]) + assert seq <= remote_max_seq, \ + f"seq {seq} exceeds decode max_seq_len {remote_max_seq}" + + wire.send_msg(conn, { + "rid": m.rid, "rank": self._tp_rank, "seq_len": seq, + "last_prompt_token": m.last_prompt_token, + "sampling": m.sampling, + }) + + base = self._staging.data_ptr() + srcs, dsts, lens = self._profile.rdma_plan( + hello, job["sections"], self._tp_rank, seq, base) + self._transport.write(hello, srcs, dsts, lens) + + wire.send_msg(conn, {"done": True, "rid": m.rid, + "rank": self._tp_rank}) + logger.info("sent %s: rank=%d seq=%d %.1f MB in %.1f ms", + m.rid, self._tp_rank, seq, sum(lens) / 1e6, + 1000 * (_time.time() - t0)) + finally: + conn.close() diff --git a/tilert/pd_vllm/profiles/__init__.py b/tilert/pd_vllm/profiles/__init__.py new file mode 100644 index 0000000..fe663bf --- /dev/null +++ b/tilert/pd_vllm/profiles/__init__.py @@ -0,0 +1 @@ +"""Model profiles for the PD data plane.""" diff --git a/tilert/pd_vllm/profiles/base.py b/tilert/pd_vllm/profiles/base.py new file mode 100644 index 0000000..08302c4 --- /dev/null +++ b/tilert/pd_vllm/profiles/base.py @@ -0,0 +1,81 @@ +"""ModelProfile seam: everything model-specific in the PD data plane. + +The framework (prefill connector plumbing, receive server + control plane, +decode server orchestration, router) is model-agnostic and calls into the +active profile for the parts that differ between models: + + GLM-5 : replicated MLA latent KV + NSA KI index + MTP draft + DeepSeek-V3.2 : replicated MLA latent KV + NSA KI index + MTP draft +""" + +from __future__ import annotations + +from typing import Any, Protocol + + +class ModelProfile(Protocol): + name: str + num_ranks: int + layout_version: int + sender_ranks: frozenset + + # ── receive side (decode node) ─────────────────────────────────────── + def buffer_bytes(self, max_seq_len: int) -> int: + """Total receive-buffer size for one request slot.""" + + def hello_layout(self, base_ptr: int, max_seq_len: int) -> dict[str, int]: + """Region base addresses (merged into the hello message) so the + sender knows where to RDMA-write each section.""" + + def convert(self, buffer: Any, base_ptr: int, max_seq_len: int, + received: Any, num_devices: int) -> Any: + """Received buffer -> native per-device tensors (ConvertedRequest).""" + + # ── prefill side (vLLM connector worker) ───────────────────────────── + def classify_layers(self, kv_caches: dict, kv_cache_config: Any) -> Any: + """Inspect registered kv_caches; return an opaque registration the + framework passes back to ``staging_bytes``/``extract``. Raise on an + unexpected layer set (e.g. missing speculative layer).""" + + def staging_bytes(self, reg: Any, tp_rank: int, max_seq_len: int) -> int: + """Per-rank staging-buffer size.""" + + def extract(self, reg: Any, req_meta: Any, tp_rank: int, + staging, max_seq_len: int) -> Any: + """Copy this rank's KV out of the paged caches into ``staging`` + (inside the forward window); return opaque ``sections``.""" + + def rdma_plan(self, hello: dict, sections: Any, tp_rank: int, + seq_len: int, staging_base: int) -> tuple[list, list, list]: + """(src_ptrs, dst_ptrs, lengths) for one mooncake batch write.""" + + # ── engine (decode node) ───────────────────────────────────────────── + def build_engine(self, model_weights_dir: str, max_seq_len: int, + with_mtp: bool, ar_steps: int) -> Any: + """Construct the decode engine adapter (inject/decode/reset).""" + + +_REGISTRY: dict[str, "ModelProfile"] = {} +_ALIASES = { + "glm5": "glm5", "glm_5": "glm5", "glm-5": "glm5", + "dsv32": "dsv32", "deepseek_v3_2": "dsv32", "deepseek-v3.2": "dsv32", + "dsv3.2": "dsv32", "v32": "dsv32", +} + + +def register(profile: "ModelProfile") -> None: + _REGISTRY[profile.name] = profile + + +def get_profile(name: str) -> "ModelProfile": + canon = _ALIASES.get(name, name) + if canon not in _REGISTRY: + # lazy import so a profile's heavy deps load only when selected + if canon == "glm5": + from tilert.pd_vllm.profiles import glm5 # noqa: F401 + elif canon == "dsv32": + from tilert.pd_vllm.profiles import dsv32 # noqa: F401 + if canon not in _REGISTRY: + raise KeyError(f"unknown model profile {name!r}; " + f"accepted keys (incl. aliases): {sorted(_ALIASES)}") + return _REGISTRY[canon] diff --git a/tilert/pd_vllm/profiles/dsv32.py b/tilert/pd_vllm/profiles/dsv32.py new file mode 100644 index 0000000..0beebf2 --- /dev/null +++ b/tilert/pd_vllm/profiles/dsv32.py @@ -0,0 +1,35 @@ +"""DeepSeek-V3.2 profile — thin config over the shared MLA+NSA data plane.""" + +from __future__ import annotations + +from tilert.pd_vllm.profiles import base +from tilert.pd_vllm.profiles.mla_nsa import ( + MlaNsaEngineAdapter, + MlaNsaProfile, +) + +NUM_LAYERS = 62 # 61 main + 1 MTP draft (HF: 61 hidden + 1 nextn) +LAYOUT_VERSION = 11 # dsv32 wire family (distinct from glm5's 10) + + +def _build_engine(model_weights_dir, max_seq_len, with_mtp, ar_steps): + import tilert + if hasattr(tilert, "load_backend"): + tilert.load_backend("deepseek_v3_2") # multi-backend builds only + from tilert.models.deepseek_v3_2.generator import DSAv32Generator + from tilert.models.deepseek_v3_2.model_args import ModelArgs + + gen = DSAv32Generator( + model_args=ModelArgs(), + max_new_tokens=max(max_seq_len - 256, 4096 - 256), + model_weights_dir=model_weights_dir, + with_mtp=with_mtp, + use_topp=True, + ) + gen.from_pretrained() + return MlaNsaEngineAdapter(gen, with_mtp) + + +base.register(MlaNsaProfile( + name="dsv32", num_layers=NUM_LAYERS, layout_version=LAYOUT_VERSION, + engine_factory=_build_engine)) diff --git a/tilert/pd_vllm/profiles/glm5.py b/tilert/pd_vllm/profiles/glm5.py new file mode 100644 index 0000000..b0245b1 --- /dev/null +++ b/tilert/pd_vllm/profiles/glm5.py @@ -0,0 +1,43 @@ +"""GLM-5 profile — thin config over the shared MLA+NSA data plane. + +GLM-5 = 79 cache layers (78 main + 1 MTP draft), MLA latent KV + NSA KI index. +All layout / convert / extract / RDMA logic lives in ``mla_nsa``; this file +only pins the layer count, wire version, and the GLM5Generator engine build. +""" + +from __future__ import annotations + +from tilert.pd_vllm.profiles import base +from tilert.pd_vllm.profiles.mla_nsa import ( + MlaNsaEngineAdapter, + MlaNsaProfile, +) + +NUM_LAYERS = 79 # 78 main + 1 MTP draft +LAYOUT_VERSION = 10 # glm5 wire family + + +def _build_engine(model_weights_dir, max_seq_len, with_mtp, ar_steps): + import tilert + # multi-backend builds (tilert>=0.1.x) load the per-model .so on demand; + # single-backend builds auto-register on import and lack load_backend. + if hasattr(tilert, "load_backend"): + tilert.load_backend("glm5") + from tilert.models.glm_5.generator import GLM5Generator + from tilert.models.glm_5.model_args import ModelArgsGLM5 + + gen = GLM5Generator( + model_args=ModelArgsGLM5(), + max_new_tokens=max(max_seq_len - 256, 4096 - 256), + model_weights_dir=model_weights_dir, + with_mtp=with_mtp, + use_topp=True, + enable_thinking=False, + ) + gen.from_pretrained() + return MlaNsaEngineAdapter(gen, with_mtp) + + +base.register(MlaNsaProfile( + name="glm5", num_layers=NUM_LAYERS, layout_version=LAYOUT_VERSION, + engine_factory=_build_engine)) diff --git a/tilert/pd_vllm/profiles/mla_nsa.py b/tilert/pd_vllm/profiles/mla_nsa.py new file mode 100644 index 0000000..c8c98c9 --- /dev/null +++ b/tilert/pd_vllm/profiles/mla_nsa.py @@ -0,0 +1,450 @@ +"""Shared MLA + NSA-KI data plane for the DeepSeek-family models.""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass + +import torch + +from tilert.pd_vllm import wire + +logger = logging.getLogger("pd_vllm.profile.mla_nsa") + +KV_LORA_RANK = 512 +QK_ROPE_HEAD_DIM = 64 +INDEX_HEAD_DIM = 128 +KI_QUANT_BLOCK = 128 +KV_QUANT_BLOCK = 128 # per-128 fp8 scale on the kv latent +PAGE_SIZE = 64 + +# The MLA KV cache dtype is a launch choice (vLLM ``--kv-cache-dtype``), NOT +# tied to the (fp8) model weights — both are supported and selected at runtime: +# +# fp8_ds_mla : cache tensor [nblk, page, 656] u8; per token 512 fp8 kv_c + +# 16 B (4 fp32) scale + 128 B bf16 k_pe. Split into a 528-B +# kv_merged plane + 128-B pe plane; kv dequantized fp8->bf16 on +# the decode side. (recommended, aligns with SGLang fp8) +# bf16 : cache tensor [nblk, page, 576] bf16; per token 512 bf16 kv_c + +# 64 bf16 k_pe = 1024-B kv plane + 128-B pe plane, no dequant. +KV_FP8_BYTES = KV_LORA_RANK # 512 (fp8, 1 B each) +KV_SCALE_BYTES = KV_LORA_RANK // KV_QUANT_BLOCK * 4 # 16 (4 fp32 scales) +KV_BYTES_FP8 = KV_FP8_BYTES + KV_SCALE_BYTES # 528 B/token +KV_BYTES_BF16 = KV_LORA_RANK * 2 # 1024 B/token +PE_BPT = QK_ROPE_HEAD_DIM * 2 # 128 B/token bf16 (both) +MLA_BPT_FP8 = KV_BYTES_FP8 + PE_BPT # 656 (fp8 cache stride) +MLA_BPT_BF16 = (KV_LORA_RANK + QK_ROPE_HEAD_DIM) * 2 # 1152 (bf16 cache stride) +_VERSION_BF16_OFFSET = 40 # bf16 layout_version = base + 40 +KI_PAGE_BYTES = (PAGE_SIZE * INDEX_HEAD_DIM + + PAGE_SIZE * INDEX_HEAD_DIM // KI_QUANT_BLOCK * 4) # 8448 + + +def _max_pages(max_seq_len: int) -> int: + return (max_seq_len + PAGE_SIZE - 1) // PAGE_SIZE + + +def _hadamard(x: torch.Tensor) -> torch.Tensor: + """Hadamard rotation of the last dim (scale d^-0.5), matching TileRT's + indexer. fast_hadamard_transform if present, else a scipy matmul.""" + d = x.shape[-1] + try: + from fast_hadamard_transform import hadamard_transform + return hadamard_transform(x, scale=d ** -0.5) + except Exception: + from scipy.linalg import hadamard as _h + H = torch.from_numpy(_h(d).astype("float32")).to(x.device) * (d ** -0.5) + return (x.float() @ H).to(x.dtype) + + +@dataclass +class ConvertedRequest: + rid: str + seq_len: int + last_prompt_token: int + first_token_id: int | None + sampling: dict | None + layers: list # [(ki[seq,128], kv[seq,512], pe[seq,64]) bf16] x num_layers + + +@dataclass +class _Reg: + mla_layers: list # [(lid, name, kv_t, gi)] sorted + ki_layers: list # [(lid, name, ki_t, gi)] sorted + + +class MlaNsaProfile: + """Config-driven MLA+NSA profile. ``engine_factory(weights, max_seq, + with_mtp, ar_steps) -> adapter`` builds the model-specific engine.""" + + num_ranks = wire.NUM_RANKS + sender_ranks = frozenset({0}) # MLA latent replicated across TP + + def __init__(self, name: str, num_layers: int, layout_version: int, + engine_factory, mla_fp8: bool = True): + self.name = name + self.num_layers = num_layers + self._base_version = layout_version + self._engine_factory = engine_factory + self.mla_fp8 = mla_fp8 # fp8_ds_mla (True) vs bf16 (False) MLA cache + + def configure(self, kv_cache_dtype: str) -> "MlaNsaProfile": + """Select the MLA cache dtype (decode side; prefill auto-detects).""" + d = (kv_cache_dtype or "").lower() + if d in ("fp8_ds_mla", "fp8", "fp8_e4m3"): + self.mla_fp8 = True + elif d in ("bf16", "bfloat16", "auto"): + self.mla_fp8 = False + else: + raise ValueError(f"unknown kv_cache_dtype {kv_cache_dtype!r}; " + f"want fp8_ds_mla or bf16") + return self + + @property + def layout_version(self) -> int: + # distinct wire version per cache dtype so a mismatched pairing + # (prefill fp8 vs decode bf16) is rejected at hello, not corrupted + return self._base_version + (0 if self.mla_fp8 else _VERSION_BF16_OFFSET) + + @property + def _kv_bpt(self) -> int: + return KV_BYTES_FP8 if self.mla_fp8 else KV_BYTES_BF16 + + @property + def _mla_bpt(self) -> int: + return MLA_BPT_FP8 if self.mla_fp8 else MLA_BPT_BF16 + + # ── plane sizing (depends on num_layers + cache dtype) ── + def _kv_plane(self, max_seq_len: int) -> int: + return self.num_layers * max_seq_len * self._kv_bpt + + def _pe_plane(self, max_seq_len: int) -> int: + return self.num_layers * max_seq_len * PE_BPT + + def _ki_plane(self, max_seq_len: int) -> int: + return self.num_layers * _max_pages(max_seq_len) * KI_PAGE_BYTES + + # ── receive side ── + def buffer_bytes(self, max_seq_len: int) -> int: + return (self._kv_plane(max_seq_len) + self._pe_plane(max_seq_len) + + self._ki_plane(max_seq_len)) + + def hello_layout(self, base_ptr: int, max_seq_len: int) -> dict[str, int]: + kv = base_ptr + pe = kv + self._kv_plane(max_seq_len) + ki = pe + self._pe_plane(max_seq_len) + return {"kv_base": kv, "pe_base": pe, "ki_base": ki} + + @torch.inference_mode() + def convert(self, buffer, base_ptr, max_seq_len, received, num_devices=1): + seq = received.seq_len + npages = _max_pages(seq) + pe_base = self._kv_plane(max_seq_len) + ki_base = pe_base + self._pe_plane(max_seq_len) + kv_bpt = self._kv_bpt + layers = [] + for lid in range(self.num_layers): + ko = lid * max_seq_len * kv_bpt + kv_raw = buffer[ko:ko + seq * kv_bpt].view(seq, kv_bpt) + if self.mla_fp8: + kv = self._dequant_kv(kv_raw, seq) # fp8+scale -> bf16 512 + else: + kv = kv_raw.view(torch.bfloat16).view(seq, KV_LORA_RANK) \ + .contiguous() # already bf16 + po = pe_base + lid * max_seq_len * PE_BPT + pe = buffer[po:po + seq * PE_BPT].view(torch.bfloat16) \ + .view(seq, QK_ROPE_HEAD_DIM).contiguous() + io = ki_base + lid * _max_pages(max_seq_len) * KI_PAGE_BYTES + ki_raw = buffer[io:io + npages * KI_PAGE_BYTES].view(npages, + KI_PAGE_BYTES) + layers.append((self._dequant_ki(ki_raw, seq), kv, pe)) + torch.cuda.synchronize() + return ConvertedRequest( + rid=received.rid, seq_len=seq, + last_prompt_token=received.last_prompt_token, + first_token_id=received.first_token_id, + sampling=received.sampling, layers=layers) + + @staticmethod + def _dequant_kv(kv_raw: torch.Tensor, seq_len: int) -> torch.Tensor: + """kv_merged [seq,528] u8 (512 fp8 + 4 fp32 scale) -> bf16 [seq,512]. + Per-128-block scale: kv[:, b*128:(b+1)*128] *= scale[:, b].""" + nblk = KV_LORA_RANK // KV_QUANT_BLOCK + fp8 = kv_raw[:, :KV_FP8_BYTES].reshape(-1).view(torch.float8_e4m3fn) \ + .reshape(seq_len, KV_LORA_RANK) + scale = kv_raw[:, KV_FP8_BYTES:].reshape(-1).contiguous() \ + .view(torch.float32).reshape(seq_len, nblk) + fp32 = fp8.float().view(seq_len, nblk, KV_QUANT_BLOCK) + deq = (fp32 * scale.unsqueeze(-1)).view(seq_len, KV_LORA_RANK) + return deq.to(torch.bfloat16) + + @staticmethod + def _dequant_ki(ki_raw: torch.Tensor, seq_len: int) -> torch.Tensor: + npages = ki_raw.shape[0] + fp8_bytes = PAGE_SIZE * INDEX_HEAD_DIM + ki_fp8 = ki_raw[:, :fp8_bytes].reshape(-1).view(torch.float8_e4m3fn) \ + .reshape(npages * PAGE_SIZE, INDEX_HEAD_DIM) + scale = ki_raw[:, fp8_bytes:].reshape(-1).contiguous() \ + .view(torch.float32).reshape(npages * PAGE_SIZE, + INDEX_HEAD_DIM // KI_QUANT_BLOCK) + deq = (ki_fp8[:seq_len].float() * scale[:seq_len]).to(torch.bfloat16) + return _hadamard(deq) + + # ── prefill side ── + def classify_layers(self, kv_caches: dict, kv_cache_config) -> _Reg: + group_of = {} + for gi, g in enumerate(getattr(kv_cache_config, "kv_cache_groups", + []) or []): + for ln in getattr(g, "layer_names", []): + group_of[ln] = gi + + def lid_of(name): + m = re.search(r"\.(\d+)\.", name) + base_i = int(m.group(1)) if m else -1 + return self.num_layers - 1 if name.startswith("mtp.") else base_i + + mla, ki = [], [] + for name, val in kv_caches.items(): + t = val[0] if isinstance(val, (tuple, list)) else val + gi = group_of.get(name, -1) + if "indexer" in name.lower() or "index_k" in name.lower(): + ki.append((lid_of(name), name, t, gi)) + else: + mla.append((lid_of(name), name, t, gi)) + mla.sort(key=lambda x: x[0]) + ki.sort(key=lambda x: x[0]) + if len(mla) != self.num_layers or len(ki) != self.num_layers: + raise RuntimeError( + f"{self.name} classify: {len(mla)} MLA + {len(ki)} KI layers " + f"(expected {self.num_layers} each); check --speculative-config" + f" and the vLLM layer naming") + # auto-detect MLA cache dtype from the actual cache stride (the prefill + # cache is ground truth; the decode side is told via --kv-cache-dtype) + t0 = mla[0][2] + bpt = t0.shape[-1] * t0.element_size() + if bpt == MLA_BPT_FP8: + self.mla_fp8 = True + elif bpt == MLA_BPT_BF16: + self.mla_fp8 = False + else: + raise RuntimeError( + f"{self.name}: unexpected MLA cache stride {bpt} B/token; " + f"expected {MLA_BPT_FP8} (fp8_ds_mla) or {MLA_BPT_BF16} (bf16)") + logger.info("%s registered %d MLA + %d KI layers, MLA cache=%s", + self.name, len(mla), len(ki), + "fp8_ds_mla" if self.mla_fp8 else "bf16") + return _Reg(mla_layers=mla, ki_layers=ki) + + def staging_bytes(self, reg, tp_rank, max_seq_len): + if tp_rank not in self.sender_ranks: + return 4 + return self.buffer_bytes(max_seq_len) + + @torch.inference_mode() + def extract(self, reg: _Reg, m, tp_rank, staging, max_seq_len): + torch.cuda.synchronize() + seq = m.num_tokens + npages = _max_pages(seq) + mla_ids = m.block_ids_per_group[reg.mla_layers[0][3]] + bt = torch.tensor(mla_ids, dtype=torch.long) + offs = torch.arange(PAGE_SIZE) + slots = (offs.reshape(1, -1) + + bt.reshape(-1, 1) * PAGE_SIZE).flatten()[:seq] + ki_ids = m.block_ids_per_group[reg.ki_layers[0][3]] + ki_bt = torch.tensor(ki_ids[:npages], dtype=torch.long) + + pe_base = self._kv_plane(max_seq_len) + ki_base = pe_base + self._pe_plane(max_seq_len) + kv_bpt, mla_bpt = self._kv_bpt, self._mla_bpt + for lid in range(self.num_layers): + # raw-byte split of the MLA cache row: works for both dtypes + # (fp8 656 -> 528+128, bf16 1152 -> 1024+128), no conversion here + kv_t = reg.mla_layers[lid][2] + raw = kv_t if kv_t.dtype == torch.uint8 else kv_t.view(torch.uint8) + flat = raw.reshape(-1, mla_bpt) # [ntok, mla_bpt] u8 + rows = flat[slots.to(flat.device)] # [seq, mla_bpt] u8 + kv_merged = rows[:, :kv_bpt].contiguous() # kv_c (+scale) + pe = rows[:, kv_bpt:].contiguous() # 64 bf16 (128 B) + ko = lid * max_seq_len * kv_bpt + po = pe_base + lid * max_seq_len * PE_BPT + staging[ko:ko + seq * kv_bpt].copy_(kv_merged.flatten()) + staging[po:po + seq * PE_BPT].copy_(pe.flatten()) + + ki_t = reg.ki_layers[lid][2] + ki_pages = ki_t[ki_bt.to(ki_t.device)].reshape(npages, -1) + io = ki_base + lid * _max_pages(max_seq_len) * KI_PAGE_BYTES + staging[io:io + npages * KI_PAGE_BYTES].copy_( + ki_pages.contiguous().view(torch.uint8).flatten()) + torch.cuda.synchronize() + return {"seq": seq, "npages": npages, "stage_max": max_seq_len} + + def rdma_plan(self, hello, sections, tp_rank, seq_len, base): + remote_max = int(hello["max_seq_len"]) + stage_max = sections["stage_max"] + npages = sections["npages"] + srcs, dsts, lens = [], [], [] + s_pe = self._kv_plane(stage_max) + s_ki = s_pe + self._pe_plane(stage_max) + kv_bpt = self._kv_bpt + r_kv, r_pe, r_ki = (int(hello["kv_base"]), int(hello["pe_base"]), + int(hello["ki_base"])) + for lid in range(self.num_layers): + srcs.append(base + lid * stage_max * kv_bpt) + dsts.append(r_kv + lid * remote_max * kv_bpt) + lens.append(seq_len * kv_bpt) + srcs.append(base + s_pe + lid * stage_max * PE_BPT) + dsts.append(r_pe + lid * remote_max * PE_BPT) + lens.append(seq_len * PE_BPT) + srcs.append(base + s_ki + lid * _max_pages(stage_max) * KI_PAGE_BYTES) + dsts.append(r_ki + lid * _max_pages(remote_max) * KI_PAGE_BYTES) + lens.append(npages * KI_PAGE_BYTES) + return srcs, dsts, lens + + def build_engine(self, model_weights_dir, max_seq_len, with_mtp, ar_steps): + return self._engine_factory(model_weights_dir, max_seq_len, with_mtp, + ar_steps) + + +class MlaNsaEngineAdapter: + """Shared decode adapter for GLM-5 / DSV3.2 (same inject + 3-phase MTP). + + ``generator`` is a ready ``from_pretrained``'d GLM5Generator / + DSAv32Generator; both expose inject_cache / set_cur_pos / decode_layer + with forward / get_next_draft_tokens / get_num_accepted / + get_predicted_tokens / reset_sequence, and share DSV3.2's TOKEN_OUT index. + """ + + def __init__(self, generator, with_mtp: bool): + import torch as _torch + self._torch = _torch + self.gen = generator + self.with_mtp = with_mtp + self.mtp_seq_len = getattr(generator, "mtp_seq_len", 4) + self.max_seq_len = getattr(generator.decode_layer, "max_seq_len", + 200000) + self.last_stats: dict = {} + self.stop_ids = self._resolve_stop_ids(generator) + + @staticmethod + def _resolve_stop_ids(generator) -> set: + # GLM-5 exposes a stop_token_ids set; DSV3.2 exposes only eos_id. + sids = getattr(generator, "stop_token_ids", None) + if sids: + return set(sids) + eos = getattr(generator, "eos_id", None) + return {int(eos)} if eos is not None else set() + + def inject(self, req) -> None: + self.gen.inject_cache(req.layers, start_pos=0) + self.gen.set_cur_pos(req.seq_len - 1) + self._last_prompt_token = req.last_prompt_token + self._seq_len = req.seq_len + + def decode(self, first_token_id, max_tokens, sampling, on_token=None, + cancel_event=None): + sampling = sampling or {} + temp = float(sampling.get("temperature", 1.0)) + if temp < 1e-5: + self.gen.update_sampling_params( + temperature=1.0, top_p=1.0, top_k=1, use_topp=False) + else: + self.gen.update_sampling_params( + temperature=temp, top_p=float(sampling.get("top_p", 0.95)), + top_k=int(sampling.get("top_k", 256)), use_topp=True) + budget = min(int(max_tokens), self.max_seq_len - self._seq_len - 1) + if budget <= 0: + self.last_stats = {"finish_reason": "length"} + return [int(first_token_id)] + if self.with_mtp: + return self._decode_mtp(first_token_id, budget, on_token, + cancel_event) + return self._decode_standard(first_token_id, budget, on_token, + cancel_event) + + def _decode_mtp(self, first_token_id, budget, on_token, cancel_event): + dl = self.gen.decode_layer + T = self.mtp_seq_len + stop_ids = self.stop_ids + torch = self._torch + tokens = [int(first_token_id)] + if on_token: + on_token(int(first_token_id)) + if int(first_token_id) in stop_ids: + self.last_stats = {"finish_reason": "stop"} + return [] + dl.set_prefill_valid_tokens(0) + draft = torch.full((1, T), int(self._last_prompt_token), + dtype=torch.int32, device="cuda:0") + accepted, finish, fwd, finished = [], "length", 0, False + while not finished and len(tokens) < budget: + if cancel_event is not None and cancel_event.is_set(): + finish = "cancelled" + break + if fwd == 1: + draft = torch.full((1, T), int(first_token_id), + dtype=torch.int32, device="cuda:0") + elif fwd > 1: + draft = dl.get_next_draft_tokens(0).reshape(1, T) + dl.forward(draft) + n_acc = dl.get_num_accepted(0) + pred = dl.get_predicted_tokens(0).flatten() + if fwd == 0: + fwd += 1 + continue + accepted.append(n_acc) + fwd += 1 + for i in range(n_acc): + if len(tokens) >= budget: + break + tok = int(pred[i].item()) + if tok in stop_ids: + finished = True + finish = "stop" + break + tokens.append(tok) + if on_token: + on_token(tok) + dl.reset_sequence() + self.last_stats = { + "finish_reason": finish, + "mtp_accept_mean": round(sum(accepted) / max(1, len(accepted)), 3), + "mtp_verify_calls": len(accepted), + } + return tokens + + def _decode_standard(self, first_token_id, budget, on_token, cancel_event): + from tilert.models.deepseek_v3_2.temp_var_indices import Idx + + dl = self.gen.decode_layer + stop_ids = self.stop_ids + torch = self._torch + tokens = [int(first_token_id)] + if on_token: + on_token(int(first_token_id)) + if int(first_token_id) in stop_ids: + self.last_stats = {"finish_reason": "stop"} + return [] + finish = "length" + cur = torch.tensor(int(first_token_id), dtype=torch.long, + device="cuda:0") + while len(tokens) < budget: + if cancel_event is not None and cancel_event.is_set(): + finish = "cancelled" + break + res = dl.forward(cur) + intermediates, *_ = res[0] + nxt = intermediates[Idx.TOKEN_OUT][0][0] + tok = int(nxt.item()) + if tok in stop_ids: + finish = "stop" + break + tokens.append(tok) + if on_token: + on_token(tok) + cur = nxt + dl.reset_sequence() + self.last_stats = {"finish_reason": finish} + return tokens + + def reset(self) -> None: + pass diff --git a/tilert/pd_vllm/receive_server.py b/tilert/pd_vllm/receive_server.py new file mode 100644 index 0000000..fb6f36d --- /dev/null +++ b/tilert/pd_vllm/receive_server.py @@ -0,0 +1,172 @@ +"""Decode-side receive server (W4): Mooncake buffer + TCP control plane.""" + +import logging +import queue +import socket +import threading +import time +from dataclasses import dataclass, field + +import torch + +from tilert.pd_vllm import wire + +logger = logging.getLogger("pd_vllm.receive") + + +@dataclass +class ReceivedRequest: + rid: str + seq_len: int + last_prompt_token: int + first_token_id: int | None + sampling: dict | None + done_ranks: set = field(default_factory=set) + t_first_conn: float = 0.0 + t_complete: float = 0.0 + + +class ReceiveServer: + def __init__( + self, + profile, + max_seq_len: int, + ctrl_port: int = 5556, + hostname: str | None = None, + device: str = "cuda:0", + request_timeout: float = 120.0, + transport: str = "mooncake", + ): + self.profile = profile + self.max_seq_len = max_seq_len + self.ctrl_port = ctrl_port + self.device = device + self.request_timeout = request_timeout + + total = profile.buffer_bytes(max_seq_len) + logger.info("allocating receive buffer: %.2f GB on %s (profile=%s)", + total / 1024**3, device, profile.name) + self.buffer = torch.zeros(total, dtype=torch.uint8, device=device) + self.base_ptr = self.buffer.data_ptr() + self._hello_layout = profile.hello_layout(self.base_ptr, max_seq_len) + + # RDMA transport (mooncake default / nixl), single cuda:0 registration + from tilert.pd_vllm.transport import make_transport + + if hostname is None: + hostname = wire.local_ip() + dev_id = torch.device(device).index or 0 + self._transport = make_transport(transport) + self._transport.init(hostname) + self._transport.register(self.base_ptr, total, dev_id) + self._transport_meta = self._transport.local_meta() + logger.info("transport=%s ready, buffer registered (%.2f GB)", + self._transport.name, total / 1024**3) + + self._lock = threading.Lock() + self._current: ReceivedRequest | None = None + self.completed: queue.Queue[ReceivedRequest] = queue.Queue() + + # dual-stack: accept IPv4 (v4-mapped) and IPv6, incl. link-local peers + # (e.g. an IPv6-only decode node reached over fe80::.../bond0) + self._srv = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + self._srv.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + except OSError: + pass + self._srv.bind(("::", ctrl_port)) + self._srv.listen(32) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._accept_loop, + name="pd-recv-accept", daemon=True) + self._thread.start() + logger.info("control plane listening on :%d", ctrl_port) + + # ── public ─────────────────────────────────────────────────────────── + + def release(self) -> None: + """Mark the single receive slot free (call after inject/decode).""" + with self._lock: + self._current = None + + def close(self) -> None: + self._stop.set() + try: + self._srv.close() + except OSError: + pass + + # ── accept / per-connection handling ───────────────────────────────── + + def _accept_loop(self) -> None: + while not self._stop.is_set(): + try: + conn, addr = self._srv.accept() + except OSError: + break + t = threading.Thread(target=self._handle, args=(conn, addr), + daemon=True) + t.start() + + def _handle(self, conn: socket.socket, addr) -> None: + try: + conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + conn.settimeout(self.request_timeout) + # `busy` in hello is advisory (a same-rid rank must still proceed); + # the authoritative accept/reject happens once the rid is known. + with self._lock: + advisory_busy = (self._current is not None + and self._current.t_complete == 0.0) + wire.send_msg(conn, wire.hello_msg( + self._transport.name, self._transport_meta, self.max_seq_len, + self.profile.layout_version, self._hello_layout, + busy=advisory_busy)) + + req = wire.recv_msg(conn) + rid, rank = req["rid"], int(req["rank"]) + if req.get("seq_len", 0) > self.max_seq_len: + wire.send_msg(conn, {"error": "seq_len exceeds max_seq_len"}) + return + + with self._lock: + cur = self._current + if cur is None or cur.rid != rid: + if cur is not None and cur.t_complete == 0.0 and \ + time.time() - cur.t_first_conn < self.request_timeout: + # busy with a different in-flight rid + wire.send_msg(conn, {"error": "busy", "busy_rid": cur.rid}) + logger.warning("rejecting %s (busy with %s)", rid, cur.rid) + return + self._current = cur = ReceivedRequest( + rid=rid, + seq_len=int(req["seq_len"]), + last_prompt_token=int(req.get("last_prompt_token", 0)), + first_token_id=req.get("first_token_id"), + sampling=req.get("sampling"), + t_first_conn=time.time(), + ) + logger.info("request %s: seq_len=%d", rid, cur.seq_len) + + # wait for this rank's done (RDMA happens meanwhile) + done = wire.recv_msg(conn) + if not done.get("done"): + logger.warning("rank %d sent non-done message: %s", rank, done) + return + with self._lock: + cur = self._current + if cur is None or cur.rid != rid: + return + cur.done_ranks.add(rank) + logger.info("request %s: rank %d done (%d/%d)", + rid, rank, len(cur.done_ranks), + len(self.profile.sender_ranks)) + if cur.done_ranks >= set(self.profile.sender_ranks): + cur.t_complete = time.time() + self.completed.put(cur) + logger.info("request %s: all ranks done in %.1f ms", + rid, 1000 * (cur.t_complete - cur.t_first_conn)) + except Exception: + logger.exception("connection from %s failed", addr) + finally: + conn.close() diff --git a/tilert/pd_vllm/transport.py b/tilert/pd_vllm/transport.py new file mode 100644 index 0000000..0f99dee --- /dev/null +++ b/tilert/pd_vllm/transport.py @@ -0,0 +1,107 @@ +"""Pluggable RDMA transport for the PD data plane: Mooncake (default) or NIXL.""" + +from __future__ import annotations + +import base64 +import os + + +class Transport: + name = "?" + + def init(self, host: str) -> None: ... + def register(self, ptr: int, nbytes: int, dev_id: int) -> None: ... + def local_meta(self) -> dict: ... + def write(self, remote_meta: dict, srcs, dsts, lens) -> None: ... + + +class MooncakeTransport(Transport): + """serve_sglang precedent: one TransferEngine, P2P handshake, sync write.""" + + name = "mooncake" + + def init(self, host: str) -> None: + from mooncake.engine import TransferEngine + self.engine = TransferEngine() + ret = self.engine.initialize(host, "P2PHANDSHAKE", "rdma", "") + if ret != 0: + raise RuntimeError(f"Mooncake engine init failed: {ret}") + self.session_id = f"{host}:{self.engine.get_rpc_port()}" + + def register(self, ptr: int, nbytes: int, dev_id: int) -> None: + ret = self.engine.batch_register_memory([ptr], [nbytes]) + if ret != 0: + raise RuntimeError(f"Mooncake register failed: {ret}") + + def local_meta(self) -> dict: + return {"session_id": self.session_id} + + def write(self, remote_meta: dict, srcs, dsts, lens) -> None: + ret = self.engine.batch_transfer_sync_write( + remote_meta["session_id"], srcs, dsts, lens) + if ret != 0: + raise RuntimeError(f"mooncake write failed: {ret}") + + +class NixlTransport(Transport): + """NIXL agent over the UCX backend (GPUDirect RDMA). Registers VRAM + regions with 4-tuple descriptors, exchanges agent metadata via the hello, + and issues WRITE transfers built from (src,dst,len) triples.""" + + name = "nixl" + _MAX_POLL = 2_000_000 + + def init(self, host: str) -> None: + from nixl._api import nixl_agent, nixl_agent_config + # agent name must be globally unique across the two peers + self._agent = nixl_agent(f"{host}:{os.getpid()}", + nixl_agent_config(backends=["UCX"])) + self._remotes: dict[bytes, str] = {} # remote meta -> remote name + self._dev = 0 + + def register(self, ptr: int, nbytes: int, dev_id: int) -> None: + self._dev = dev_id + self._agent.register_memory([(ptr, nbytes, dev_id, "")], "VRAM") + + def local_meta(self) -> dict: + return { + "nixl_meta": base64.b64encode( + self._agent.get_agent_metadata()).decode(), + "nixl_dev": self._dev, + } + + def write(self, remote_meta: dict, srcs, dsts, lens) -> None: + meta_b = base64.b64decode(remote_meta["nixl_meta"]) + rname = self._remotes.get(meta_b) + if rname is None: + rname = self._agent.add_remote_agent(meta_b) + self._remotes[meta_b] = rname + rdev = int(remote_meta.get("nixl_dev", 0)) + ld = self._agent.get_xfer_descs( + [(int(s), int(n), self._dev) for s, n in zip(srcs, lens)], "VRAM") + rd = self._agent.get_xfer_descs( + [(int(d), int(n), rdev) for d, n in zip(dsts, lens)], "VRAM") + h = self._agent.initialize_xfer("WRITE", ld, rd, rname) + try: + st = self._agent.transfer(h) + polls = 0 + while st not in ("DONE", "ERR"): + st = self._agent.check_xfer_state(h) + polls += 1 + if polls > self._MAX_POLL: + raise RuntimeError("nixl xfer timed out") + if st == "ERR": + raise RuntimeError("nixl xfer failed") + finally: + self._agent.release_xfer_handle(h) + + +_BACKENDS = {"mooncake": MooncakeTransport, "nixl": NixlTransport} + + +def make_transport(name: str | None) -> Transport: + key = (name or "mooncake").lower() + if key not in _BACKENDS: + raise ValueError(f"unknown transport {name!r}; " + f"choices: {sorted(_BACKENDS)}") + return _BACKENDS[key]() diff --git a/tilert/pd_vllm/wire.py b/tilert/pd_vllm/wire.py new file mode 100644 index 0000000..c3e5a76 --- /dev/null +++ b/tilert/pd_vllm/wire.py @@ -0,0 +1,81 @@ +"""Shared control-plane protocol for vLLM-prefill -> TileRT-decode PD.""" + +import json +import socket +import struct + +MAGIC = "tilert-pd" + +NUM_RANKS = 8 +EXPECTED_RANKS = tuple(range(NUM_RANKS)) + + +def local_ip(probe_addr: str | None = None) -> str: + """Best-effort local IP for the mooncake session identity.""" + import os + + probe = probe_addr or os.environ.get("TILERT_PD_PROBE_ADDR", "8.8.8.8") + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect((probe, 1)) + return s.getsockname()[0] + finally: + s.close() + + +def derive_rid(request_id: str) -> str: + """vLLM request/response id -> client-visible rid. Shared by the prefill + connector (internal id) and the router (response id) so both agree.""" + rid = request_id + for prefix in ("chatcmpl-", "cmpl-"): + if rid.startswith(prefix): + rid = rid[len(prefix):] + break + parts = rid.rsplit("-", 1) + if len(parts) == 2 and len(parts[1]) <= 8 and \ + all(c in "0123456789abcdef" for c in parts[1]): + rid = parts[0] + parts = rid.rsplit("-", 1) + if len(parts) == 2 and parts[1].isdigit() and len(parts[1]) <= 3: + rid = parts[0] + return rid + + +def send_msg(sock: socket.socket, obj: dict) -> None: + data = json.dumps(obj).encode() + sock.sendall(struct.pack("!I", len(data)) + data) + + +def recv_msg(sock: socket.socket) -> dict: + hdr = _recv_exact(sock, 4) + (n,) = struct.unpack("!I", hdr) + if n > 16 << 20: + raise ValueError(f"control message too large: {n}") + return json.loads(_recv_exact(sock, n).decode()) + + +def _recv_exact(sock: socket.socket, n: int) -> bytes: + buf = b"" + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise ConnectionError("connection closed mid-message") + buf += chunk + return buf + + +def hello_msg(transport: str, transport_meta: dict, max_seq_len: int, + layout_version: int, layout: dict, busy: bool) -> dict: + """Common hello envelope. ``transport`` names the RDMA backend and + ``transport_meta`` carries its connection info (mooncake: session_id; + nixl: nixl_meta/nixl_dev). ``layout`` carries profile-specific region base + addresses (e.g. gdn_base / gqa_k_base / kv_base).""" + return { + "magic": MAGIC, + "layout_version": layout_version, + "transport": transport, + "max_seq_len": max_seq_len, + "busy": busy, + **transport_meta, + **layout, + } From ccb15bb9a3c455cf7d18c1eea623a1e085e91698 Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 18:35:33 +0000 Subject: [PATCH 03/10] update bench --- assets/glm5_tilert_mtp.png | Bin 108303 -> 111023 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/assets/glm5_tilert_mtp.png b/assets/glm5_tilert_mtp.png index c4db83fdae463ce51c4cc297fa9d072983bd4b23..f0aeb99cef22b13963e3141a08788828177f8672 100644 GIT binary patch literal 111023 zcmeFZc{JAD8$SA|5JiOwQAlM<(twCU#s)+p5t+)6c_>2zMWV=%A|aJIgp?@r6cHIS zW~j_$h&b1-_x*nRu66!6>-_gy=U8vUTY8?)v-iF4`?{|Cy0@>|QAGxNc6thh!l0}q zuR)>E%2O!I_?Oe*Pt+Omp5T{#4u?)U9KT@d;B07TLOEjSU}Jf~!Sb9Dmy?N|{kaR) zyM%Y{6c!QUGIwyWvER32ht+?+L->N7*^ZFXxC?j{IvXV&dkSSe6Zt`97FDM~p`uWf zu9#pMAm8S%9ojW|4US~SVZJVN$|50Ai$|I@!k)u5GX^%{JcOic})a&dKiS6WKz;o-4- z`SJtC#=Jp6L0n9}XU?CeXJ%#&=2Ksym3ZRNk8cg?8Ky^GnAa}ZWAUlpX+~btr>LZ) zyDi85rSr%}{WllAr$;(A@7=r3*49=@RrQ*$FWuoIN7S<{b?OteRycnDbm-~PNVTU& zIjwFSm07-G#R_=^g$k?b>1o9{m5^~^t_rc4b}lS*(38#U{EAx`uad3uu*6d~Wm#=- z8f?1Km}-!?|5&)FN$0n3HxFK;dH4Q3!^MjiRZ0)v-%XdOoqD2ao47cuNky=_q_E?s z;DN@}O^h3*o_*Svcp|wV%5x!J!@$hU%;nOh#LR$K6~X-1K73H~ub!Q#kvZE_;?Ewr zOVY^5$Y!Wz4_T3qADQ#=^3*05?UHpeXqdMiE-PGaQdxaT^!5Ec7S~@pkMvcvY~8w5 zL`+O^th?CI!osh_e?#mczjfvf32Pl39O@a^QVsG;2AeZ%zt!_Ue*D<@Qf_kccGIeR zSKeKtmD3ixK))+~V&AuK-(r*kdH8f+@R8L@GdiT1s?X{8t^VNL)Q~zJT<>P+Zg1rs zK{rMiNEDuH<5nT@syi{%Dkh|#%ZhK2b#rqI=wCrc_io|WT$<;iM=<|!8pYci@i5zW zua2MMkn!M%j*fnwo?h`uJndvsLs92#9&To`U+m8&r)X(?u zH!Qr4{}{v_zp)5y-`?<0P|R1Jr)$s3RjY2~=jYe2-<)tF`6mDICuhEt1&WA>e9(XN z=g>(M?UMJ!J}U`sl<`<8DJfYMC1bF;{KJU1mse+H*beq6vDCOHPmEf#teQ7E_g1XU zb)Vb2o+qYiVSeJ+%h#{ZHm94ojCNApjh-#Kv1*fu`%j0a=H{;INJ)X)5B6W$`{Zck z{a9mjQ|FO(@7UPQF|o0GL=E&WEiUXY3E*6D>Cz>)g?XLUuHT1YO}zh~lU`wuNR_P& z-EObBaau9mb;8s~KE~2tC*Q+e_>i2YO|Hi;c41-R(a}-0mg!H)r%t{w4=g!5GukDW z$LANpToWauIMMfh&Bl!>Lu!BWO?`nEA31Nm@fl_{F~_2$l@1@ScjBWid3r3$X1Glv zWX^MToZ77B;rn7AANqhVUyL2Te`45Skf+&nq4?(7=7sJn35*=#m+_?+E}6QFhNgSE z!?V$e_0KAvHzO`>r!lsiVD<_x9NkcyH z8uhHN7LpVaxj%Oz!p^?EOr@lxq}Gysz9He{D+m6IQ-gGzoRLC9Cr+F|fXcqVzo$0G z-i&g<$cU%<;eNksw5z)7WW>L=Iq= zEZmtI9v=R#yqqp2CB??p_8lVEwx@)_>PFm}ckkZu9e;8-Pe&3_KhplDaIB{^&8}A^ zE^51Wir%~AQ`sM44*3aLQBhGT9X)zNe4$2WvFPSn!N(61wG!9iNxkvbkJMuol$4bN zB^L^BtfKF)S&^<^Iy!2DfLw?5Ea@t|u{L16ue|uBDIo=a_K*jLOurZ490zj@*`?rZ zdo7zTV~hER`UeMZ^_1R@H7*ZozMpSi8#Bxl z6Vnz?!3Tf-{JC1wQ(13%`qq%-z)kZDb0aDpA0=iByrZHv<-U1CHZa$1mI*KFbMvOG zt}eTfkdRt}CVzdR_A2bNd1DHvrKM%PzN*&kO)@3XayL4^e)WC(melmOZEYOA6(Mrp zKPA_u8pw$6+jrhL85tk>zXwa`GBe6uP*}Kzlhgm+JyzBY8)D^snfUZ`#mPHlTc2)f zZ7pnX-4MwcuxUY61m)E5XFuj`M?0~yIB}Pwqqzl6BrV^zZQIIC(j1J8 zjCQ3ru3tC8cD-~NQCP*t@OW$7SNN^PR{ju#g?q)+XPd;GW)k>oNx9637j zYi=4PQ4U+8mLjud3$=oR0^f-wR>a_SY|0~#Uq7E`WXOdHpT>uekI0i!+3z;98bzq& z{R6SFu0mRz)<<}LviF}-PCxkmoowJ*b0c!_V)UFQ2h?{`ycbulTv@b3Keue6Y3tF5 zXS=gHJ3FOLXB-X}wQI%45_Hx#FfhpJsR-SkXmbRW(!V`cASo%yxFqK3)3E-uFYEAU z-l=n_e=j&=P6~ah9=Y>t<@roQqIR=xs%E?fegBW? z>GgiaMq_^KcNF1xp5PDNxc{P0Y)nl0T18Xm;kGqMXa-5CsZq&|2gWa;^34A9@V|9S z^JZ6fv0wLiZ>8Yr{0kj<-7_;T@q09mA1_8iX5O%2gCJ)|f|lSI(ynHJXZtacu1(V8D@-WAfg)=z73_Km78$f1tzA)Ki+-oN8^|&A zktr;AcE`9SV#8FNZ50k3;>NO6PW=4&bD%lnX+avo1$OlUff4w6OVQlNwk?gu?lFQ_L=%- z0(j~dkq&K~oKD2QursBjqhsBFaW@Air|oFx!Q@E)2<_>_GMUBsrOwXIs4W6Rmu5x{ z3DPq(G^AX9`BEGayAdl`hy)cJknizppY99uE7*d5aceD<=kCd`AIPJr93Z}Ikw1VYXZTw~ zqRQOidm`Mcc=%a?phf}ADlK0-eqZP2?w%pFY5n>eZ!S$&aC}TYB^E5GeKdAw-`2E~ zw{Q>(8yeQG6+Gbu$a#~6|EgVYMfB0geOSzGp#Um#(<2s=niXc6%Jc&R17&U=)e#aR zJ9h?3TF;L64m2dLPcg_3c>Y|3ck9-InwnLrp+Z+OGInWbXnYBX+-r5e&e+Q1kxKb{ z6#B&7jVQ2&W@b0+`>T8To*>xgr&=#yF&zT>0-MrJR?w{4n51E}eHRYWtxYoey$ey& z?y?BaG^^IE!Ryb|T^;jpCK=>!1PrMDbPr)@G}x4;bmYh(9PQ5D-jL)m)F4(-RBbdn&;08nASW@uG>kKFf%o@;g!=6y^M^Ef7R2Qw5v8gLZH##jo!O=?@QYr8uq=G z_sncb-tJ}jr{p9`yUkqo@mcQeUiJQcEzq0P&1Yv=CG2=_`TLt>mG9f;Fg^U)=l1Qe z=EX1rk6LrfoFk|?gc>HDq4nYT7Xzt zyOr_c^sv89hH1PLv+im(Hs4Fr!-i#dxCxRf$Rjy7F)^`DqOhRgL;Ue=%a$!O7dUL$ zn#C~DUsE$C1z4*wH_`neTJEL$TvnZjy1KfJvvX-{_Jsyr#k=F(9w9n;ZaKO=ypmcO zuV0_+1{!S2_srKlYk&Ud2!hc&6_283 z-JVN#>eMO!BxO>$wg2$F7`-1qe^|u1etIwQJYfIQy8cU%%e;$}v$Ts=2qJgzWuZR6dWDvXE9sn8w(+#z?2V)idMb4J24wU>}70cQ8ajIb-UYtAb zKdcmcwWru`{XGf$Cv_e;Mo6;fK0d)NvITHRELp!k%(B$T#KfC=#k#j09h*FV{m_n{ zNmoQP5{SF|Lc89%j$LNed~LPbG4gb6dF~sKUa!4*^G1#7H7Lu;e2+H;MM)=xN^Y_3 zs(*E3Bt$@qe(~4zCTaJdUWom7H8scLYAO+Uf|mIXUzq>|A3JOWFD3{7rCl#Qux?;b z&{b09kvvFA#_Fm+R$3D>q8Q%xs$=}yBH|WENe@cHJxOQl0Onxi z@#_yBY%n@|ws-1WZ45P_mf^W`;jx*7MZDPJ>2@T9{EmM6Ay2)#YCl%ReERfh`O1}5 zpMD~&wZH1+&X>J=M};Kd0%Q@I#js)Ta?h$zE!p*DGuP&x8y4E_DVgWx;-U->55KhU zX9SFSGGxOSwq@(q?$58C#hnIOPbBNcX>><}QV6{N)m^+ILfm!o{*x za`H~&veg4$znWYMn176o7q{uWTw(S|pmlAwVId9hS*6t`aT_ke5|+|3mRaUQhDg!N zx#~3363=p52#@>{YpKap{^`>?0MZ-r@$m&rslX*O^RpRk%jzQ}?1NJ}5MvyYPAl=< z8Z{3dJSYTe%*x6lDHxdgx!bJWz5DkQT+GsY`+g-j=WmbDuN9#}sY-SXXB=(RTzC zHgQkBS4xWTUaOYiUXIhiC{0aGZ_CU16(>4gIj-Q`=TN5M%Ck#MOpMTYq=xsky5t!c-qoxNbV^#bkf)YKiwU%q$ltj40snnqn&N?Xv@CM8x_=}hn_ z4%T_em31OIar#FNAO7&@;8Mc51E^2JTnFkutr>lH@U)&ZUiv$UXmCt9=v z+g=%Hu_3MDxQga;Jo*;K0(Kx1Z~ROiDM~F>f4}!xd-i;uJ?HDynifDlIZ}7J|C|&m zS345OYq@rd9P|^vk57;7Vk3)0z470YcU6+5KXcw1#=#p z#yPB-Y7X4DZyx~naaaF~KfW1`e<>Gt8Ra62hhhqD-HmW^`8mlCs7>$|V8P~wga6@Y9DtmkT`c#8W$S-dz9MFW|+|-k4F$JzfzM7nL$Q;JsZ&aB>kSZ9y&W(yopWC66 z&il%FB*nq~NX7d(t&sHbl8WlbntSziB+RtBS1~XYU&#MO10|CPQ4j&eJ?6z9%6JAm zdc=k2N%-;5@8->lse4rjk*A{(brA%4A!&Tcf`&__V99@i1ANc2Sr&aUq67>57@&tF>BSNWx+^geP|@~-o1Xm@RVZYUS$ ze)#ZVmMlMcp70vM*PgVrv>{Gq=QXU`QxiaW;+~(@;s4Z!Fxo&C zJtSlsM@Oxkd~a`RtioNtHRl>3tdc70?T!_Hs93A!f$Y0w9+ZI~1Lz;YQyqEkuM55$ z0AOKx*aNNUT$279-7a=4eCwwM_@8D!{&;$ZJ7S`K+Ty&>g%W-^IqU%C@m(RBT{QS(jCCI$Uva5b0qoSN5 z943~m#Bu7SB!fr}0ho*duZDEjJ~62b<^NSB?K*DUzAW~Dipovh9*E5$Q|j6B<{pP# z`cc|cnis2O5URQ&787u~p`o}b?d${>&ya?bDMwA*cD*kl-%)7ffh0+j;O@3#Mo2N{ zpIXtzux|az@=|ZPEIBWeSWNT)3a0ADOH@EAmur0^;3>d zdHfp6?syIuEOqlb)|+p7=RX8F1tf{@R8~@fGz1m^6rU=Y^yAleo-H{5wrLn;;4yos z=ld%Fije0n<6nGB&Qgg6X;)N7YB}}k9SrU8*yw#FJw14nw42&KMTiB`o(oCOk0EVk zFo%A6W+(kLspi{th7AH7Bt<$MrY1~)B}-93f#|QUEDZcTBO|HLe`R-g9KT4DC=kUk z+Yq=}@yyw?vCnIE1r*Gdsx z4w^=97UWkpR2Em(z>CA}Z~VCBu0L~sNNS<~wlz^)A+&b^x(%|4T^x2s&Pc?Qbrrzjm@h#iCs=tasqz#aDHOsM`7I4HOc@Fa3U;$;Jj}?AoV)z^lB@|!&p-Us}`vF{Hjy%|VGV}aeoGj8L3Jzw0aKgjK zw|vz`De^gOfBpO+FGR|DZk`{|{5XJ*IVGLEGBfum++w{# zWMim+%NZCBLEY`{?(T%{g=XQq>S{*&zAE*77rxfT)ux>-m3p~UKxm)y@E`g*hD1-c z0pf)sG!jDApo9{^_CkBEx_BO5L4{eQ&6h;kn3 z$S=jmAw@KJ`>A!{mi0|+*}O;ejSP&Sk=Y5Jh>&z<#2;7rKOc{XjO4_-Jsq7;hO#p7 zAzIw-i_%5~1)P${IyYBwkyYlYdxUdm1Y*Iu+x(7-=gR5HN%ZB+Hp)M_mp*Z9E%GXT zOO{pQJ97%reyYPZd7(_<+ndlR3Fy~NHzti%`;Ec$S=OI#MtS^H9vaJE_$lZCDx5Qf zIHK-rE5DjVl9*mjdciJg^ueJ6+`@t-W-3@mj666kF)MWU1IRyWUvJj*K<(1)x-Z^B zcl|OIF6k^But_EWJ5Ye$%31-<1A#n;&|L6;_Kc4-!k_|{LP8*I(-5aS{p)f1Z0zj| zNo-B|-M(E4RWA)7TI%Mlyv8i6Q{Me&r7umbB(Fa|J3-{hdk-G;fJ#8o5CjL(eAFIR z0oKm3?P$aSh+<#1UitX(<4&8-1CT-pLUZNdCa1IdpRDMb078Mb3;}|HU}{4)&+dE$ z+(v9g4~aaxyi-)P06|#`(5V-mPXx1?#qCO1ZyGfJyxoa}fEr7>fYwn&j%#~qd!KQK z0pUH+IMw;`b2aXfUK}#k@ssR?1)-l&QTXh|Q@j>I!oVY&#hM==&>+%xAsq}r;@qgH zgdGmvB^0ho5~8a$cAkK4JW1Ghi?0I%Uw*O&%Mfh{6(Sxm&)KPl}+L2)rP6oFFchQQvl1zUc6Z!85~8$;vE z+|9be04(XbO_ywhv^zU;cnjkV`sX;MXOW-S1#{Zc&MLaNxV#0!*gSFWE`$WAOX}&y z${|0I)Hk4n&Kas&oJcyk2I`-%jhW=4In7GeN0V)*2%+!>;EV-F&siPf#K5_a1+`ec zX&a#K{!3Fu;5*}d0{`=@Pau<@%=g$&a5^+<=Frg4^EGj?u}iUmhohu9(b_ErJlW|n z?}AXW5HYIr0^B8H$IP$k4)r*dwVC}pckL>~){KFARo&aUS)1wmWDlE+hm?Mvn{REb zVqsO);baM-heOW5=@nLtvS@jNw@_46jM(Q$|CVAQh!%qu#X;!gGi)KpaLiU=L*&47G{1FV>7+d7S&>8E_m;gd zo1YmwiAq%}%CQAAdkwdp(4A^27;Ri6nWS%C;uf z52Q4->1Bvibmz`xFRx`#UEVNh`G0c1ADNu=sA}l!;H?1oyA$=+P9BAEiDaHhRFS( zt-n1J@fy?w7-yKi|M1fGEV_9XokueZp;195mse6+PI`%RQ&uxGGh9MKE0Gg@e0{H^ zrR@Y{xB{saLh}-o18=AsgF{0rdWDPTO*`LSA-o-t>J6IRjZcToxD2HREitteJ&6|< zjf`(D{*XmJc)@A?kaXIir=>Z3)yQ#&gXTk=>UzDGHdI(}S5{7vvZjawDJ|>TpQKL` zKl!R)_$Nt_MDhE}d)*(l@bCZ2YnuO)?Edp7|Npm;rTG8Ex`gIsEq#>xG0**y+uRfb zy2!OoeE*39@*4p#2NKEv#YvxBw#s{Qkz8{Ad_q>?|3sRG&e4fehkX_f@Jh<9+9aLS zt3hga*eKE3VdU3Ze!@rK%QG0z42!r2Z(mU$f#g$&UgM(k}uDpo90{o=bvx~D)I8lIXXI`t$KxNi^|)kCh?Bo z)XjLW0#!VNurMR!9wil(0<;m-5>IR&_}=e+G3f6{<%2au>G}y+#GX8P(!xl@1rIK? z0{9?RRaI4uerphf)r?C(fF5Pib)%p4!&HJq{(R*KBX?5H7#hAaCT^s2{&{)QDq+G5 zTsdf^&q83otuT3c`1cYJ;p0^GdEd@$nbDB3YW0(QNi!K{;*N>V0npnyONft55oB!~kMPeaiJ zP66!rB<~_0T=P#e`uBQ0=Egp`%#ND`^vA13FM;vmDlvY*Mng>@HQdR`N#j>=rUmrn zWe}l+Gv0sruo_5=Xy7;3ZNDG4fA?;dRQ)#pCEP*2%A2)Zb2ueCY}*Fxk5c1 zL2NoPjM>?VQXZjX!>4php$|}U52~@&w^L6Jvk@hD^Y=cQ_3PJD-ab>0eeU@EM0^&) z4_I|6MJ>Th1O---EM-dCfSwjCV{ojGSoFdYr z@hnTOjM=X($1Glll{ z#{C6A22eY_r$8dlz;h>|D`x}&93IgJD(*KJHe3f(IB)zpH#9hymsbMK(c93)&P&AP zhHXeah;}U)GN07&Kh{tOy)$RHphPkvHl?y5N*{hG#YPdiIB72B{&RQf*day5SeLPG z(X%0tO$Au@TCx(~;k|qJzPPwobhupqZR-0Sa_Z7r*a(ONh0VfUQyfVZNK_egLY2gM*{JmFul;$mQIU{4JgSN$9(a>gt%P ztE=r=f$eq#`#ad#T}??D+%Kq=ph-m`8}qfgD%aG?${$FO{x-T%tC^Utg3GO_Aq>QG zVRjqH*fK#uLGRRZkcd&ZO(_S^ta$_%Wrl%%Xr;fe??Jen@Zq_o%+8*@hFIwVvM;F4 zfX9k38$=Sj>dEzjCsqPN%BiW{QRus?SZl$+We`?Ly)d#;z2XuM3vo4|DRv!AA7X`| zpBrh(TtPX5hVF+CNg>PG#jJ$}2j}&1#L#}BLZ;#&s`cxD$9mSLXwtxJ8!{@se?KQ| z6EowzE9yReq@uv#Cx=KG?R>Y?JD*we`QPa2F*mUXIy#25>Z2t=2rR`0M;wT;U(1~R zBaC8TR6Lav=p?X$AOPD^xMJ(S6AGDfS`wJ==&&1}fH^3BgYo$emDNdAU`v3 zf@Z8uMBB9Li>sR2YMdqhVDG81v3EK4{U7kd^8GPtTA86gfkA#AkB38!_!bY82n!q; zl!Z%I5$6C|;mo;nzA*cm8*6KV@Y)VG?S$Eq4KEFgiudi?w+lW;-=D;?!fy8V?OO_1 zyjsg5X@-q|p`$#4m{!1858!Xvo~tw93;-=e_ZbayBB{#;Ibccd?Z1>|V}0J~7VA!) z*0bXoa2UKp)1Cq&8rQC`o6zjW2cn|7hy8MVJpcLnXH;@>a&c!xQN8639H5}ZhZeUl zjxqnQ(|!Q~>#9E{9e>J*`nNrM1PkmHS%mXa(01RA;R~Czn>gbp>05-=K$>`_l^9y( zA?Y+op@3KwynA z)O|;%4Wc6zyp^2;13u7iq7mTtcAsN{*t$00$gyLkcpPGrgs~?Yfox-Aa{z`0!ix?^ z?$hcTA;ycS~35W#0C|PH=;O9$Hhf_VLE@2ysf|%rVzSStJbH0iazL> z)!gGYW7DN;XBy1|1VwWv?YNMA-2_Cu0ytAXudNQu*+22}YB$O-aV5Z2wuYHGHc;ZU zo}O2R?S3gKqvI(~E9)-PTU8yI=(`IU^ z3@xW-L|y`mB?QXO!9k%P$MoZyVDLQyD-A`oJs(Y01r+kQiUs}GP7G+f1xk3J^1O7L zO&@InR04h?+e%b>sYUn)NSpes^1w*%mpx^S7eQ;bWhQeHFzrxk>xCK2m z@9!jq=di(g(LBRLOQ>ws7At(paCJYZJrmN zI3cSUbT~6Rn@o&o>CXY-ytL`MI{Ff_rpAai+E)?#FP-RXB3=?~Xj7iMB(e)BV?ZM; zsI)K|zW8tfc3Pd1i|GBq09Qa%m(5aA9K`;NCLD$i0sw#tU=TW>UASPnHD!-!8U>bNCtoc0vOTIynzhYTj9GFtL?GIH3w%U3<*Iwjf(#n|p@x!|x{b_6&4Zt(= z0fm_9xkgjW_!Rtjaqi=9Q%`s}UCmdVLf8c$$@`49A)kE-?by!8TV9QVtQzSLy>uzJ^kN8`|MGU z>%>;H{3*r}-(C8vIVr21Y*ZFfzq|s#Cw*$9pfMn;c+uzt0rU)yMoC9LHIp_y0Y!{d zjl#mEL(+fSFU{)QmhDj{EhBKs0Pxuomc-O`7R(lQ8y{V)rgGV&)$a|wnr%eGU zUW!k%mA5uZ%Ju5v!rZFkPmjKz+J~0W)hk!1PzbwVlE?X2VHLFO+=IPVUK%eC9y~ZY zF~Q6f@bKZog1KEgI#f|mHnS!jb6!uJG>icOV67gs<#Md*d9lidCL_em$$t`Yfd zDV`D9r8k-aGe3V8R97F3oem5`Qn>-sC=WkB9W5ir+nKQ*g4}oTuH+D34gF8oZwDl5 zR3eqR?I)jeRjyz9%;uj-mfvgmY&E7s!fp0nXqO6cw!B$v1X}jCrA62DHJV50qzgC1iiH}6Asz~(k7N3gkzu_@6N);vJ8QKcmWe@JYq>S&%_pYuhMMd)HKkZh{IjWqa z2Y4_!(jil+7Zmh_atCxFvvP!#B+=4dZS+2aT2DYbwrqR32AKIB;)X&2_0SgNK93m; z)S^zvlNzsOAkFg!cZ1X)@}^&_mSbl!GdEXQSSSYrf${vU-L{|N5`sexC&OxD6X~jG zV`-VBCfZzEtNyjR8!xcJw+ke6-GJ9KFl>BUU!;6P(g_0g#9WqFP*r-K!<{G=plFDY zC0IcS|GlE2RsWq)6z~!aORW+<6sIxw4Z&)D=g!YOOKa3qVg20Yl!3OKUC3=ACfnh~>C$b^c@$*#fWor% z=!zgrZY?h#CG&T`q=SNAxli;lz_~z~H#pZuh$RL%C|ntqDZJb*Njke&w(Dvp|7eJ| zOs zpsZzz46mU^P!tpJ2Hhc0$qTL^;uKNMpBJ)MKB-lG`u)5Km7~gW)h}TaiXh2E$9DTU zNd_59&}E?FsvNeoA~JV&c6L_bBe%{Q$9&qs&Ghy4M=>;MWW+q3h`_CZ#4 z%=;9qqbz{@xpU`S7Upy67Q!oaPU-5ZWSSgJ5onDRRMxe4fQoXRk@F_3zNGg+5zV`^ z)BFS1c149rPmiXc98`9Bgi39?iE7ou{TpyvOVMIaGpSq$!!>HV{CNcEmLAA=$*8CD2lRrME>_9nfWcWiWs2Cn;n_^Y&bc%eU-aN=r*)&O{Kg*4f?7GntW|&JQ>Q@E~$nXnZ)k54{vG@m|Z$2b6>?A>@Q6xg)OeW~&O>=$UA75qRzeW_+f zq{3{=e@mHXhyFE4K62=7xwOR}e=1b2&tvtIn>hwG0Hyx=?c1v;B&0~oX=8cOkZ}*A*$DbuTL`YxiP!ljl5lL|2Z}kS_etedf}RN=`{sZ?S{6sG5T< znX2Yt+QRE8L^i{7W4+du`fu3gsLy*xJz=sx%2UD8Qi$@lxmm}wdlZbVz%{X=dh>gJ z6AN!|Z@S*Ay<<2rz!Bcl%BXjborFxojHk`Oc`=$p;p%ZeXK2XXuwz0ZCdkVikrk)H zZFU15t<}@mpoaUKUn4W~>){ya^^K^jsL+GBUB~Djg>B@)h_hPbGj+L_BP8YU@$pqk zK@esYw|P`xe{tgHo@W|y^d3LIZtF@q`mj=Y_WddyQDK1y@F+cuEMldm>q;%}q4NMM0I@+`8Pn5)D1 z4HXnTGDLv>?G^03#b}D=X8tbzb$)5?b50R5o*OA@Kz@c|QBp9r9Kgd;QJ`EEVx`7D z9+P?3(!!3)TLJ(=48HLT=|XxhSK}L&Qb>$I#Hg;@2BEw%paAQ+yb-WMr}_Yqryf4k z&-$c`M(=NOhX$#6%L{(+VrUX{7$w16QUS7zVgSdQAWJAevhjwFCelka(~oPPU~23BUp&ATz<@5)#okG!)T_9MkrL$#hUicfo-U*?#%V>?~2` zW}wS-b#_t!3UV+0phuKkjf#pAF~z><^zDRi5+A`9lOss6KsSu}iwlz9J|&ZG0)hS@ zH}5bLea76}Cpb9RZGzph<%M$W#F}$;tHM{m0S2Tz0v6W^LeZi91#2*oaHAKe>1u(p zmUpCM5avke1zWU68Hia3DSQmgPO{c$F4oQbb%e&r7?pVCam?GIbV{f=C43{Zs&FUW zLFX2X)d*%~b_vzpl4Rf-o6rpi4kGdTd8IvYp-O4ixsNnZv5Qc)sVJm%4$@#Ug*hj7 z*d3ysA|RY=Y%$KyMjMk1e16LTTQyv7?Lns(p?v5p&adURu*sT1jHY0hw<+cEPriYhZ276yEx-2crzKhZ_g5efvBnTt4}2)j zoq>S?Hxx8;q#^@cfrE~k;&nQvg^S-`Q2F||@88ct!xHd64Yw1d`7P5)>Ig*QmKS_& z(4Y=!K1Aiii3CkLG82Og7TB}r$K>P(3>bE$0cb}=M5Isd!s4JNd6(iS-FWlHJ>b*B zeU1W{$*U$Iv+C&QC6y%lJ*S7e?4J+C;O6^a?C%1CS$h zV}peeY@3wl0tXJcD6o!)p`fBdB`)pNt5+wVpP^C+;PAy-bdjN2%=yA9aRx(nAW6~C6zThk z05|%o{8Tn>N|4>8Lwem}5H65tpE+k5ruRU%=-j>fV>pNw$e31xANH7{OB zI1lfkkDuStJL&~m?Q;;PqcLJlb7$;Qj1W!vn+ny*2jSseU=QLKC;3Q9gsp;i%a(&e z>r{(zr1`3}0PDRi62|BHaPb6Ca1ShgNHs+`@=!|Xp^}S>i$4N7eE#Bv=1WG(E%rV1 zkeQn@E!IDjcE3r+h)POmz{?d7DUaZD1!`M9R`&C=`j0(oArNdZh>W%UYI&RbGTcKe z##)-1p6%W`VRj7YLQ`LrDe+bR2aT@SKNn;0q*5q1c5N% zl*IG`8bT%mx<;mxiQCJ~Em!3B*1ob72=GfN5IoU8{x?1Z zSRtj4eJRK750R0PRbS|EdersHzj{9Svb*$$-0_k1%%X4D`M>+@`Nn}k> z)95Eoh`2PISs}VILRDX(_e9?q^a3-zNaQgPp4cDAaT=wawnEjAdEMniMc5&-2^uGu-wL!_sjr0Hn8OC;BUewsSSNuR35l>r*9~ zU0Pad;lSt$v262%E2iEtO4EslRO$b>6CQK*JFO7CBHo3X+!v7^*>+f2S^c7Q3J>0A z_x9}!;Ps`j4^bI&I1rmtFs8vEM39LkGG_$fV~A0#d(!T&>SqvYUApa8TA0GWA6KpB zZK(e%PvZ?9HzD^&;ydrG{EQ1s5aRp|3($Da^i_pp&T$O}FsM{jRS_#H<~4e<+B#Q7 z`T3P!R_A^^&TL;c0ag@VGd0^H}Vt%dbGfBn?@dXdT0)Ra9^ zjuX}N19EBYf1W=$Kh0PeZ`P}g^VwXj)%#^$jWW|EdX`{PG;Anu0c@>h0A!;-L%zoX z$(}-{^8OA`$gg*-&f3^`8Rky>tN*jNzd_*5Up92L1f1@h5I&-FbNvosdp#s5@5uX5 zuTF@ULaU=tP{Tr777~5`LNvfON@<&fxzC?F$FC>Y8vb=;WQQIkJKdL`YHP2&el1Dq zgwmilf%IV-p8VgvNDHPHz))Ez&oPD1yKP(L;Cg6gP)MjKBy2KIu~5J< z=(*1J4f`TEO*tGPzOzyarTi-7LCP77AB(&HeDPr`AeG0ji{Qo76cGs5xF$zXyYJsf z{c{j!9g1*BS`U87gr}E`R-!Yw>`oxBRoly*=#s4F;8?k7)281&c#Iz3LaYPVZ>y9d zSl+>*2tp_o1-M8K+63_ixIq`8e1Pdv4~i@*jwk`>3EyyQ0EKW9iTgMH#gXILXfR8; zjxVDi6Vsw;J48eFQ07I%Lbmczdp(&pMs*5qfmjaf9~ssO-eItjOcr5U8>83WK!cn2 z@8^Uks|XA@&8%i)c)1#dPdkMJh*iX8^b8!u?;0D|BTE3sY{FebHz5>tV;kw2!|&g_ z_U~i-{?&Mn+u6s1Anci$vCWXv&Y(9908>~}vh|dGQBl!byb|0SmJiVE!xts08=FgJpjMI#5#60F`j_1X}X!Xx3`Pr@|^trmX@U?XP`tmq8Chh^`hn zmN?ld@Fnp%5!A42?(1+2)FV$HpoAB zt7~A=&vu0UPKaeNE(xnX0I4JbznNJ-N3t!3r9oQgkl{s{bs=cfnaetW+zI^P+-rF< z@2HOek8&r4P?NP=f4#F6{El?Ax( zOPp|pJUj|*DF%>ELU#_*0BN04jKA;dxh=Cm|Kc)GoX2FNq=*_2oi%0 z#zMgFzz<+hAp4INO~?b*Ul?tA4uXSw1PLy%;09u43Sx1k^(g@v!Az6?N!3n=!KzBWh&cEmqBXYAZ@TYmgf z;9a|%*>`|tU6`DC5yh*1ZeNYkm zcC)S+E?odSEW*?bxt%Kq>2oTfPO!hUUR!AT)QsA#bBim}mCpj<=yah%Tfi-#(NCMKD=o}Jn7Y14&-i~-K0&aP=>^#Tp{CfcLcfwk8 zP6>UDwV|#k5!-n-Z>BOHbg6F)`?XD1neIJWgMBLJK1HCm5JK!-ikHNJxbpPr(>qOc zn4%zqKxhGy3v1{N?vmaE@-jqGo^dDs-rizR4Ay5*lJdg?*i%HVUVS-|(Qc4R< z3lgv4tLX0T4h?%b1N-g+5LGada1$4`5Q%`8yzoxQ31cY4XeVra;C!Em5|}f-1{ICS z)=&m`u7&mczRpmQrg>?oJP;JPuup&{vT?dnxD zB56s=N0$Nb%LvPK+$UW~h=<7COF$cBPKNRv?Pa}VK7C=0KNU^RoVkoL(}NRCgnbd` zVZ9u)&+g1GO|L>^1uM!#Pl&t^S};& zPu!M=(uq2NTbwwN6GMkGF@r-7i4sPU3QJG?DG{u}RVZaMY{VJSw|)zJn#3H=A??Nr z>r5%`ouF^)SA2If^j`qrGeB#HXe9d20fmCiPfhIMPe&)h5ebV(=}3V@{w?55NkAWd zCl(L4N~rOD#JvXgxEM#c)IQM&D^Hv-kRiY`xW#gK$NJuHFl;=Hly?1QaiV`3sdxArt3FcJZS2+2S@o ze4w}W^(@HyFp#LdOx*jL%!HA&ff9bB5{qU;CD8)NhehEI*(#4qp1iT6OUh>E_krqP zoE(U@xfK_;W$Uqrw{SnCPO@DVFyZm0l*B~84v#rdxw4xSuni|9At*;+_j`6gCrg zu&%XJWRefJ53Jzo$qXSEjX_7FqG00w0BD#=K&pBy4djHYcyiKvMLIo%I=S0s8}%s- zjiAwo$@qI3Mvi-NH9*m1=7qvW_XIr)6Xi3s)YRl68MO1Ho*MhZVq>E0vz3>3`Rz?o zE|*Dx^4}vS7WQEEkt0Wn@!VwSEwWn2H z<~BCZ)kL*)uBx6wgl*%MN76{&zd&-BQ3iw`f!lY3D>H!>v2Na{vtbQ@@JEXYRUxq) zS_%|-<`lE4d(peu?nB%zRwH-p%$aEeT6%ezTYCgxWVrw>Rq<6?wYnHX^@fOPbHd@E zs?Yi1#Ff9P=kK3_M^H8&;fUjYk;hfWd=xSSkM=Xz{mFn&On${KzkX(po@DMnpKu-; zk?e2l0sS1D*4>@&u2C-IM5soF%|aCEY5@W6k_xXes*+0K!oV=I({P(FBtCNW7P$cs zz70F63m6CV=JbFbs?&m*JItVXml8`cq|H!|pk)k{xN@r_$YcYbUyI>-hz{IcL2-d; zlz=6)O#PE3jEa_{*TK1K#4@BQ!)z5K8AC87M7o9L=e8J2vn(MDnyyZHu*c#_9V@Oz z>~e?ImW!2@l`@mxc$}AsPSV0;Y+|AisD+Ax&w*iFyR4q-BjGZN-|OS=CR*S|mL(Jn zdtr&|I1uD0;JV7U=$B;#KM3LdLGCs=q1>0t#qYWDfH11nrB`3!f+rWU~ zgiBL{YgCrw(oGiFYN|4SAm)kfa47}zsMkSc`36VOl>o;lw~M%E#b^EPP}fvKo8$=J zAqkWKVl2f37^DYa7}8q8^z1?0*e8GN(pZP}WoA0zswIr;oKB0|_pOBECnC-Y zn@ALGa$PWdnvh-0Ew#IR)@-&xJDrRoAW5vRPW}vH2PG_*-0-KP$_av?5w!_0otW_; z+!D7(m#eC3rR8Ef?XqRddwo@L$4uQlkOp$!FXgfG4HeuExr{WE(H$lyYGQxztlz4H zl*Dt;37D!2-Y3BmLS{S1DtF|6`BvQk@r@Z+ZwVy{mz!3GQH=R@(jVJwS8iL!^0z*- zRu&)fniB7lJGKfD#?!Y^0{SN85kS8UT7wZc{roD^?`p0k*RKKI66K#qMKaU5!@+-!C$!8klJ$WWZT1&W*e|1WO()w2#Zcjkw%8{i@o;_>-qiPhhKPSZz3teD-DrSXiyS) zHKaXAM!T%0O2f!*c(qGKd#GrjtwP#b8X9P6laiM1^Gd|;dmQ&4_i^0EasTe)=Quv& z{Z_AWJ+JF=Js#)dJkLj#7y+TiF0t!BDGjx>}ZU9wfyK3CATY$h$!y`aq zqdcAr5|yeN&6I3pqGWX{g}W5{TmPeQ+s)g=%`MHj<2|zalr#|%XQnbQY#O4m4z1hD z!*k8}4EuM`{z<-05y##}(}#F~K$A)&I85N2YTHMdQ}9aR=TOhQG`!LC zztQ6)(~a%yeL99XVo4Oegg^~i2?)>01#P6U3^+Pt+HJkJORl8N|+r4Q3m&tvL-$x+}^b^ z4E-_`s`Ffla1PrAhyfu$0O?biljw)v=0`HfS)P+^r(L6%MoE=nLE=+N zUmAPl;g$2SehN9*BBX!r+!9;cQJzvU6y?~wlLRqbHA!b1G&iMPLp4ZRTHSa(%61SB zH*ARE&iAdPKx==GCC>vC|9_Zwed;Vb^kkpXH9M(8gcX8|?*#S>)WzAjOq|5h+m%VK z1a``^J>N25_I?U1HMNP2O_I}G10Tw^W~O1xBvzz2Lgr|sG znp{}d%5xG*-V_;Qcbl$o74o!>(8H-M|4BDfnehQ99)bRDIUl~jH5kaDF5nRL9+p+b z^KuTH=dVOazv#AJV||80ReV800G@?Y<^mK77$bxrnXpH`E$H6UlH)3?{|OXFAfCF` z1vIt2PQ%={*oSWxiCfq#7e2h0QFLEu1*)oouj$C#4`8uWdc^?s@n|IcM|o^Jbe60T z9^@p01RWBYLv;I60LRZWVW$8xJG)f?P?`c3wYO-!(4MO*^Q+$@M%dtY5qc7;2*Rwv z`%&BulL(s)*$U}XSbqywrfW1@iwrNg)0T|$*NkP#S*!u-BT+}C?Q2bdqA3?Y4ckCz znd%XoQX~&Xv58>8d@PcM5@{j8RAM^13~F~Yz`6Et2=Z3J!VGa~>77?lBgwe5-M)L5 z)Pfgaj7fO`wBrt{?&v+<*Mh%Ue%ti#@CiR}A>SmJl0PicIm-{%3IxL$6nt~A{YwW0 zb)&Eu9w&ao(}ddv@+@yR^&=iE7=LjCH9qA7Vks!@8AbN+KUroO!N{dZJv#6GST{5!^F?|RtHzlMnHwyU@@hGwK~D1DO2k?K%!j)Hy;RU8GH{g#Y^!6 z?ij{6R41JG<)Vq?0XS?_y;vH4lEXu{geKHWc=!;aARTf7uMb&Zs3reO$1k;%mFc{f zr+YVOs%+2!d`prS(sl>88y!F5bOOAkpgRcNP?V+O)a4IBG&hPD!QLT6Oj45(36GFj zn}LpZr0|7$c)S8!+HKX#a`^u486PI3I}4Q3FQWP_n}rV4K?G!2d0fS6%dpQTm{x@R zuf7`o?W`3m+MGNPh_lm&6naiXP(MjRM@x!pa7e6L?W9L`Wk9kC&$|T&@w+ z#6?J0$UsEU`CB?p${XTF0Px{8U;&*#-w{yA2_P;0S|c8%vvQP97wWF_I9w=uKVZ>d za_s#URJZML%x5HYdH|6CKQ0q1EHKb6hlPa|4dXg`f;dPpO=4c78r4l;Z+vIsrvz(o zcmU!8WQ0_pZwnIK%h*8xQa-^qZAHQWicDg-_!U}2qHV$dRs0rBG#DUhK4$B4km=v@ zHxfBN3JOrd+yD}X{=qIZdoGiT2_*(^K6m(Jd{0$Fr>3t5+>tc@RGDWR*gDfh<-OvEEKZD z5UEWw{*hq!$lxzyQv?%k8U@hU7*c*AKe1;m1CQQZc>~t{+MOm8w|mdMJ(uo*xgf?@ zpevVZp)@>J03OHuATfj=Z|R%NOiUl(e?vrA;4}h6Cq*BkQwY@0k=j{%rM-A=;bD^9 zfQ^GA#F#nvo)*@Ovj zllad|WlLg{g119yvNEvnl6~Yg@UEa60hNwT)Vt!^XF%%LUiPEtymoE3+XaVV!3`su zOwG(zSPmS2etsXfA=kZ>W4-v;DuHRZg<%o5oD{#Pr#ApeKsK|Yj`+M`9T72K_IV zaBejzT_BN47~tE!-Hm&J5kR9M3$-ddBI3u@9@)a%x(K_NID1mgBe3$cXWHXd-hK2P zSfK!3)U3~fF!|Apl+kdTEhvuw`GHUdNZSUvqLavaEEu0zn=I0p!+NIL%g}EoVi6b^ zs??MMDF$;a7xz6RU0$(*rtU2=8vurb)?IBXP@Bg(ZC!!fA=Omo{f3n`nB23}vgdQx zmWgjBa7vrZzTJd#6{{?7;etO6Y>^0PFTg{Tq)v!(PEe0R73vJ1Wr7vLJ-|~a6Mf~-BOt8}3+=e;ke4!uxH-zZcX50EKS9+aotrB@du;ihAmNq_cv4hcj#~9M=_)T8>Jyjb2pi+{TcmjSL>bU3W_hs-{~GVzK!&uq4d_Hdlb z8M9`e0Q(NZEpo8|ma(wh1!WOx>s-VdID$$5FRj|fXbhH}NAh=6AE^KZHtEpxyKj?Zgw!A1E3`v70CD1X zKTpUUK1{$%3I)kunkJO{e>snk&r!~ep+}HI&!kjx4@sw-{?)e;hob=>toF~`2-*2} zXBrV)^O;Wmas+E}Xk-v7(C4bQJR~Mch^ZLo12o&T)OXw5zhf@mC3Xv_F}QOgdbV@A z1C}rl`Oo=8{xR|a{`~oWX1pgl|N9#!I1K@FQ^l2#RDUTkN>c4APzwzlY+nkUyebP32HF-xbMznCBh8k8m`HMZI$8O>8l090OVpa6Pa{Eh7*9O!c|?(^Up4)h)*5z&()uUjLmL9 z_W@=MUiq!KfFIbC5Bu#r!BIngObW6Y6m%?@t>yYRYIyo02SK7k#1f%lVYK9)rmXc~ z3giIR-j1sXGeaUXN=WU7AV)lK^hN>pRRxaP!t-BsSYH9Y!_Kk()qq-@Mu9-ad4=2q z`;`?u*ZHovzf`1ef{Y*}!+*42WHtr$^-H8bW-QumLnA{^3KGBo#44Nw4~0k&%#yw% zLnC&kusbXqpLuX!NbJxjd^+$JT%ZpjR-uT9s)4Zq+P^rOzp!S7y;Zo|$=tfaLYcP1 z&&N11(*}uE9u!_ltkNzdv;&0P=LP$8tUhLnIHNh&ZV&*fGpHqW`F>kIkO$7(2wp{7 z-T)B__PR=)1L_vV=6-iL8{cIWSp}sqSS#o&%%Ch>C(48r{+*iASy z?jD}`ck{^bCbC}uTP;tQ!0Zod$r+Smspj+GD#$-dlYRoe`#Kzx>PruV-ab30O-5E+ zybm))fJ(>{_33KpE}x&p^g-P1-!}3% zCp>v_&-X4fkW9j@z}a|s6x8n+4=&qYjUpnTgbO0FTnO=?>9*0Byc=zJ3%!+A7oh?34! zYzb+~pb>?eo{W6sZ!#t`5zz15x;4pmF z{VgWNOLROM8Td1QfE?w~^VLHI5^K^x0^j2l+A}cpj(WPkocnJnY(WG{`^zAlhTzHu zB}2=lO&V@mkmT_pu9){VvZ9tH!IBvFV~07v1m1KsD&Q9>akN$xywhZtHvGxO( zMS9fM+In7m4=|f+?P9Q)B`^|&SO%gl)2XFJAq7O^@pSDJgp(xnC5^}I+HY7y>u7n|LAMa!W1R+g0AI(JuN}g zn+Gf~<0{t(Htix z=>WU*Xdl)?x<}Rqj|r8VYj?xSN(WwGZ>P|*MDzBv*HhA-fZz)QegdUMF2Kix(A9A9 zkXScgX2)vFscE5;&q4VnO2vgh_TA7I0=YOi*W;xScgsj8UfMirAI0X zoFk}cC~Nee*z)m%<#T1PxxBEw+(o2TUkYxg2ntjY+fE2Y*_PR9DQlgm;gRR@;!~7iLI~;18rC$D3F-kgl+_-T= znO*@*5|5YoQv(Vh^LMaIJ^>#Grb~c(LRyrP$}@={{5ftr)?Wz6tF3rbV!?KLC|DNc z(@Am{T&@Z226hP`_H9Y*!|8?j`9ij4mX}5R90y@R#~7!9Q~MF%-LuZ?ag4}#iu&u3 z(wzj@i9CnYAx6bpgwFL7MR$Myl|xi|9Tz+x&k$)6Xc2^453~|d^D?B2wBk)d92KxJ zB?KnxaqwWlgI))Fn*ZDs^eiFWQ?zRd2uKAR8vcL}kSsC@8m*jWKgCnofMkioX<(`$1>-GEPWRZ`-I=R_w_!@w5rp-b6lg*w zou+YlgI5XfJ$7PLjZ0$|g+N-!0w0pce421V(c{G6rHzBSp6i$W({8B5O+@AwVSCSfiXc77na~o*FFP6+&Tcpr36h0CI>0{(gmMdTBs_=Q;`l!Ta8GQ zr-Xfs6XyE(JrjkXwK&tcB_)f|QJ3J-mLsMkIH^zIaYFRb+#75#NA}}_*@#osHGY1} zQE4Mz{eZZdV@BVxMP(2mF{~+OQ6Lx!F8iTOsHrG0nn89^%r8Mka~fX6WNs?D@XOG> zMGo*0`M-*FFP;&QYEBOpGT057+=(-1wkD-vQbR(aA|23}P*ugdHxw ze-&pq{H^Q1>07{K6B&n$+hqm?GrGKU#9pHCC1gyngH@Y?QoJ?Lg+$!jJcfyXUTzDk zs;i}7{Y8Ko0h5{|X1hW*BFt>Km4cp-uN^MN86Wm0oik2Y4hX7&{k*UCZM2-(p=)I%z9=@#(^rOZ^hw z8+)ag`opP_Fc3T!dXjzzQbh)~=KDopt~keT;qo5^r83GihprU~%eM!Cxcx?>WuD@!E}e z;Q6vN4qQccye~lKmx!58iGEl<7jTK4zDXai1|$T{kMZ_TSDgMe!$5g6w;EadReGH5k1vVD6zH z-VwW$V-%RHEsbLj+tQj}A838dJ{bj)2UJ&j?*Vy!1!xw6M4Aws064GWQFp{T>)8fq zCL2A3H5$0;R}fDhEA0XU0+RuNVo&aV&=2(`!7Ux#(+5_`j8ipTI2!R-G(FJGX z!J#GoAhG6ioqc2La1rZ}zcXnwFz{6uPL#r_}<Uy4D4G z+(`~E9`q)N4NX53d?>SC!={0d9#Nit#0>#6K9w!CNF}Zy7?K%#IV=c~QmB()J4(pM zOw7#0SPn%A8lX~M*CLaXEdqP*!j>#O=X+6~6R07E<~^k4PDWe7fsIVoLcdY!Vk{xJ zl_d*Aq1s^*Hs>UEm&x{X5F)R26^VF!lZJ}T<9if%HpG<^PT}wqknUMn?E&MTkI6?` zi|WzcSWt})^!!mhdo549{E+Qart_01S>#e_Sy^SzzLB9yZpCB#tz6z}naV z+gfa?12_tdy`}{&sFgYEA9^9kAH(-sPg>AeMq<|ikH(oJBWoPyXA$QX`?0y=-+PaU zkJP_~4284-OshSM2H`{Uh!74DbHRIeg7*(t?Od?;y8vGnNM6aq0)+)*4QQO+??EZS zvdx5sxwE5#i@HxqQIQO2JL!^+lja@KrMnX>XVRYmLi!4d%4>?TaQ`m1NoYWi7;w<4 zb(n&i1BjYTl577Qy~)g!51n8sv#9-^RFgJh@LuS7&A`;u^y#UBV%kh!T`)Q+$S+ zS#e%j(HStovXSU~#rzw@?GK3M68vKED}w8EV8;pm4q*up2S+!PYRi;kY@238e_r_M zK5?u)A0TvrgLm<*A`ux~g+bXS)K+13FhXz8-_yg|jC>InZ&z7m7M>2fj8bIZUhbFJ6U zFQ`UDD|8pkQbWt07_U-D5rB-i8F4YkH>kg$b>^CQf1PLVP=cQa>E3soIQA1BJY~uv zXi?+k?_`j~ZilBhF!M`~s2rrfB^oh&$yvBR;8pzqJ4|9dXgeoY$04pC5q?I16U;6( zZYWCo098xXBC%47-g-D9t;%V}4>Q)ZoVCOZP+UA+>Ix~n80|E-8YkQRJ)(~tVOmQY zgMhgP1(4T!P>f7J?g=$`_FS~qnpc|Mg!o71p4Z!xlVvZD1}2w9;O8MS=LL5lMI3OA z@}jgOQ=_P+LJP)<=K&;{K?Zk`Q*YT_vmwzV6!0FBq9?#B!0Op0)dgbdVmKHe?>Gt8 zU_sfXPG)eg9PcI$6YE{%Ju}|a=)MDsA*C7PZ-@ob$FB%d=Rw11cp*Zn#4}LKbLh9#ekw2INqlq9`%6-=$CqVv2l-B z>&x94s~vt0>wmN%YIY&B%}D*7+L4c_1m;Zm@d;=Gov=&}0%@*>g0}XlIl5fyZ z5e@UCSF^7ju=6l_jH9{GTlB9_V3#TOfO_)Abk&1|WO7jfy!jh*0`fn|OWO?B`F6ky*wsT?P2WmFbz z20!Z+;aPU(hFd%I<~Kp2nepbwa;M;L7YTGmT4EH4Ij#{o3xpAl2*#wg$32AB)(dp! zI`3**Q^%31TmvW`fEz$_!OFwVz|3 zH5Q!j8BMF{>F(}t?fW=q9)kkq*Pq7<7}ACzp#B)$C^nAEksgYiAVA3Mb##|yNBIGC z_bO1f$fZes8@ch7P=eu%lsVd)rktm*XMU{L4p$E>e=^WjcmO1tRt|X50^e1+8DfcK z2+(8~JX&+h#bqZWe=#As`!y4ml398n6*vGqP>nm*h18>wIzG}BA08}s*czkG!6A?8 zWe`+b3b-w<z~Z}LYC(5C z!evwkzaE4LPFzGnyF(i|PMCN3ix7=wT(@iQ3FgXj^X}O#S57-&UvT5ok;+>5Um`9$Ckdn zpk8%Uu;_Ek;Ty>%%A|r3f@f+M9ipH3No-=+6{~Va*kvdaZP3mJt0v-StO7Qr$tzhd zkqK>cGqVbWFhTtimMYBpmUBRC5WwoA(_U%fAdPFv3y=_1ca-VmQJw9~&acM~&$JE! zDn+N4-k!tne%{{DKEf@6C3?x5zLk@!_}B+WtfJQ!AJ1>AJ)1*!Qfm2 zU0lPvt45_Vb^?|yKK!z1r1;j|yOoGUjNs~{!o*iDz{Dw&hXa||k!1Z8`%6MmX6U54 zcw8#ghJcEiUtO(+8^rK+_xoq1<^sAvk#)B-C9D*Q`8m?XTj3&o?pz-W<91LXEO6gq z^jZ`|$(agemgVD6Qx zS92iJs>B`BfQq8FVAB?4vSEayMkh0sL3H$CB|f%FUtbp%Zpl5Z#NGiCIvl9tb!*W43%8 z{38>=LgdRM5HPM^&x`Cy1@xmr2SijMMFd?{;7u*x5E%+WY%$JHf!x3gL}ch6h6u+pl@j)OJ=m@v^i-D;m()|}b1e*X%Ee%xW)LJ({n zI_7PJ0ElTfJGdkWn>SV?@gVkB|KMP8pZPi_@_nZMxb?B%+yW*UJKX%Rt+kb%mp2~& z_wxRIzxF=)F$M`R-IGC6r6~dc?-yFz!Z;uFvZ+6uOR))SoHQx5tyX_NJ%w^&D}(8c z&v;=EltryyQF5yx4)0)Dv3`9#5TPxcoQEKqA!ak(-3e;vo=PtK{el$A1g9HA88#-D zfPg?@bDDWaFo=b@CfX2}<8li{CCasXcg=u38fpUtTjC9X8=0Pe^P|w@7Sr^LlykeO zJVj{O0xCf+AJV>e*aGY{1Yv%W&3Jc2^LJpeP}x^+M?1s(>!$-C!pV>YCk=SK;eK2% zs_=0B{pUscOzU-BxFZHZe%AXD0bq?zy_)@C_3?|@CimL|Z4V=tyB87?LPW$3P-da) z0hl^948^uIoxJF-rb{~-j`1v;eqxT+3<+$0z6gd!*rfDO6h%I_L%7a>1B-&in<{sI zx5EDG3-4v0aH9$d2^peWYlCZ*N<%-n9Q9P}@~PMQu}dV)!MagoeQ$%$760!G*&VWzh2ZW5T!Yd4hap+tnZc9 z;(K4=-mqaGW-ZV|p?zr0`mb_LF?6NB}vuttjNJlEltaG1&Y zAUivI>&3k%VlHl*eBr0Bi5=0|vtSLqMfIzU#sky=vIu6wrdF%5AnF)xuyWnHI24nd zXyD^d|CcXs%1WD^51x*5sS9KClR&)t(Eo^ZhCr_#||C38D%%ItEb#OK}z zFp%hr(e(6?6cje5mX;;MbwYuLSJC{A18_p?=v(X=in>NA4YSbJyq2vEG7V%uHT5Lc z@G&hky>LMRZXn3gJ#jpnxMolsXTChK{S8E~h!ev$5)2GwD+%$8D@9_w-mElCqyz!E zvZUMpjmJ1wyT-A;eA;pRid54}Go=%9s9 zVWWJN^Yrm$JH1lU^OEcvhlx>b8XX1`ftUL87v{liGdIb<8%nq}E_-m{Q1kQhq>zL@ zG59MWtqEb{Hbj>Ya+hdoN3DKP@j&2VBq_+k57CpLOEu5KWS1_Om(xQLa*m2lZBTq} z$GZwkJq6G`QnH0XzlL80>OfjKp0u2m>ra40g$UK|M(JUvr=AY}=aBYg-kX@{X#mD2 zJrofy5>Z6MZwNs@2FeV2D7lGHl17c7@{L6^S(=XZ55py_$^O?H4>10?@|kehun}nG zIRA;FrO+ug*$o9yJAk;t04ODM;-@Wic>5-XOC2ha_cmnO9mQrG0e+{8tJ3rDc-rpP zG_U0uf-jAZmV+SZ;3mban6dy8u^3c{ywU^v&hi6U;XM&)>e#Q&{@vN~WAh{Oh)*Jl6Q@ zQl!-I{B^-mOjiA=Pw-kh?_A;h^O6+*1AkqVj>Z4y?a0#n@3NS!*wK{7TfxjoNGz=F z(|q~*RiS}EL1W2b7Z;(?F7Om+v#;)(eRYG>x8Ix4@ytWE`D|)4ujB~m=dX&bwLG*s zzEMG4GD+IjCQ{Ecp|dK*cf{QA;qq6vLw#2-UllbgkJoVG$sOVIpXiHOvLbCSpQMK# zj_opTkUytyIDYM#lWt?b8UH#~ct$b&t)7lZe@(%)LQgT0Oh1Gq< zRU=ljBr4C;L z-B4^q_+WIT;W)R^y9d$CqQl`cl~$_l_%`-@P53E;pl!v1_Aiy9Rl*+8>34hY39+J>t$6hvooxiaL{6r}-`Szm%1x z{DWzX+d_BB8Ar)gXm-5z;TsG))_T6N@9EQ?JzO-K`e%Ww90Ye(Wjx% zr@Q$Nk04c(nuz$uMN0Xf4wxnXy6^FJ7oVSxTIXouKTyT*By}+6aZF;H{Cdx5CO@@p z$1~0t4RzLws_+f$1xI$UyGEJr;o}o3Q{C69VgLK1e#jQbf<683#zBKu>m6k1BYVoE zbu;d?%d9)jUN6`4@@1%bURd`QmGV6e7k>ZFi=7BaZd)CCMPhwwF47c-@|eRqWu&FE zwi_A_8hNNWC9t!TXULakX>u%G^-SG4X6;Pbe9=M(Du4YEWkL{p<8_r|n{S<5r%Kt+ z=GB7dyVba>F8SOy-V^f4`FdooSzlLP>ruw_K_`+$?+369<|ZxUR;)STz`8po;^3vY zfKmCb<0m4|HZ6D;p5+o%_`%iuQ@x<)m-7`v71#aus3PC_^`~a%jj_X&7rCuVAf+^zma?imvR$}qa6>~4>+c9HKI0WU0Be`8TEOe3I@5i$9H_;606|LoP9MrX@$T>?t@Z? zwaRK`cE$SrNwhDXpIm7g?I2C@)G7MH*&WY>ngyTB+hW&yW}i4x%&nXJ;SOKtgpLfY z{AmV9^2t1fn3n$h?yAm-PBmfk{)E})->R~k_9_&0INjcxe8g7mQe>~I;Jd)w_DZ`S z5jaGuF)hDWin7iGw~5Xj|Cq$6h7`G3$wtZz-Op*KNBrtycJ@5GxX#w~ezmRkHh1cx z-iqT3GMQp$3vPJ>5#smnmPU&4w^fUoBO=zF3&ufxZT}?^JMi>Z(cdrr;;Qo%PDhik zJRi>*Jv*zoqv1l;7Edp=TP1b>o|myJ+?6Ew3JcUGbpOXo zMByAp^(Z8rzkd1-qlG)wNlNmlsH>UxIU4=8?enNhzVr^4810bs?jHAv=;m#|F*?fp zb>vir;gE-09yug={q(H;IqbT}cNfnS5xGi5I;gjMo9@JT&Yx?Nb9dxo${VlEgV8>g zIzLnTlvA4z2WG`NerUP!d6NaVrl{-f)^sW@`{BJ*^PaP*&5HvDlun&0pJ=p*7-}~3;Z&m0?XZpC#G?;Ca%PKRdw4Kl}ve^?Uwf0W?7oD_&;on>0y?j|=(-fMo z&ATh3x9}jn-2n|Id!U8ahz{(jdREMh9UEaa3e+D;UA5t<6M|c&!=$iPM5JU80udl zFj38_nRzz8L@P7w@l69y9#zTWwW?Jg_Vx^_7p8WUz<+PYQBFD6>=WDlb6s93B^fpE zjjewyFLij7Ydci}SK{$kccK+0Zz#{yr*<#9Fa6$}Cq_v;;+r$aHluQg5C3^$QfgQK zEOX=Gv0c?dHSga?^{v=iHFV1>#pO>XX0pU;8us}AaPCmJA!`NY%Zd3cgJ%dpn7-=| zK!|Ve$oTm0yz~E!M3B7yMM7d$0e(^q4u=J}TaZyjq6oI>ev&sZP$3@!L+dpqVz%EO zp)`sBe#-R=X94n4b%!@I5?y#*t~oNli*kH$H@|EHfCeJB#^^pikx=}6PAuK4bnMvu zT0Ve1iU7ADTq*wU@V!>w2JjXzw4m@(l*eFg!$0mSe^KsozJy=`pN0aDXct~(($wIR zIUFDn*x#5glBM)}XHfK7p1xK&;aVgoD_dz+x+j8wb|_wrFl2~)Rse>bsGs1v=}S7y z92}UZ%I@pTXZzhm&S(2R&?pGb4ARy^AOr>wIz+?I3ee5~#JR>`!*sNr{2To#+VWUL zQf#z)2l-|C0I00SkOAoB0p@=LLI-j>%*p`d2#^if!3H4X1aL-$8vYX?yWrV+by$=V zbP6ub{?Sc}dyeHK6%6)LZr;lJ=SHK5kIdKEH(-DA33@=$c)O?VQU?zbb;)E28X-~w8Uk>4 zcPa++$9Iga}BWesj|Q7LjE>V7be4Rg%i*VUHPer#$ZQ<6#iLOP!Fv)&CE+F zjSdrb0N%KZFFeH{d%k4MCUO60176_HWDu6N0Fl1)>dm~6vPiJhSW4%ZF#bXi$Y)-% z83q2;Y`5lNhB!8678VmDhtZyA^A|2uM#qGxI{@-afgUn~g%*Um0=R-4Sc;WsZlUwR zUXc#!veNrkuh?@3$wnsF!v6$aoM$|5x0-2vxBWMO={ScR0hGCBqFGq2=H_O(o)<5~ z$-cnY#v^EQ%JqSwL)NSkr(h)u%cEzFHZl|mZ!rtEfUQ-W3(87hI>K^DA%m$;_Bj3az4Xj92k_|w>O4&2QaU- zpnDVms(YD%GTJOVoM-&_Uq7+CySq|c+v?|63_$#mJ%i>il;WEko@k|-mjM+A(DQ(> z6sxOu$=!YM;K9ObncATKUU`|9Z{}UiW)l%fJNAlVvh1&w53jFZ&rM^9n|TDBhKe=0 z=pL_Gr*;%mJ(NVYZ+}}EhZ7jp)86j=(FP>xdmibDi73hc#a!^vj8O2SAtOV_m)pVO zpCHz%kc>y~1EWMr`|t}U+1RD?-}%S_~b{d*Z_p-8v->-rkBoxj6aOQIze{IilHMRx8=wYOz^GKzQAX*f=KwMsd zb_bP4wmhb%C=KFwmFZ}XM4+8rX>xbT-XG$0<;Wu&_IxYaVK}n&FT{58*i`}}0&~cl zS{%6NC_z!7b(b3luTurjk&%^kE9jYa{rMb*r;-fHB)|gnrrHg+dXID`mXZz(ZT^A< z6*2Ld(P$^g(Fb0yJ%_1@rmDC8Bq!74=d3_apNNboglvM&KcO3v6BSJve!I zWTB_=qzVZO8)2jlGr!KwS|%VuHsGZsgO^UrAG&K}YuiP-HO|hk8AAsimxec>#Hv#| zMEF3Z;h#%YSXM5y5*cZ;nr~fT`>l5jp5;@2eD1wxR=Zq}33CCOnipp~iiCAyr2?!T44P0VeaHbNa5Yw(pGQaYH+W0l z=-d46#bpk6&@V$^X~ma=^JQUK8Q*(8g6V=oi-{4ylv_BL$hEBC4q0FbmFiH+}*M>kqWr)KmjQHc# zj&90PdC{-ez-I{qaYJ*1a^O)dk}0cK1UCI27ms=iV8ua71kH$NC;3qMR!3YYZ_#t2XsmzUMtp`_%oPK$ZOmbjP|=0dr6Z+1umWMA3X}ELEWG}Rc17*Vk^cW>*J;;D9|3_bWq zSeXVtuIb->d{T7}KXH2U6SjyBKO7JZG~JK&TlSpwp%drUTj{JX!qxvP}~?* zx84lrxnkK_$t4FZbJpI8F|&#;#HdKZtG{r8FJqVe*hPlfvo&|fP5;`?f@p>^WPE;g zB&=1CSscL6W$5s>jYeEDMmGpl9S?BBsWhAxO-Qo@E2bClM93q}rTF!@sGgplfN%$7 z0%*ps1_BM82)!Z-7k~8VK{SvIiUp?kMh$1oDoLev7Pby~?RMbKg|mi|X}f7@w$VQk zs(k7|lYV^4Yx+}~c&aQ21|A)5i3!SzgJ7W?ByImt-A^xAtHGu~I8#aLpp$}hsH_qF|+ zxLSm6bk6D*`CX?iAZ&YS{r)pXzSEh_BLf@{V&hT zMkfx0Xq-2WIdi^QR3r0=!4ooU6ub1}F}m;@X-2gPVPLeK{vS_^9mL?Xb!k@dfi|67 zXcQHqe}O!*9GvArG=6;oc0kjvG#X&l_{8$|^g@|=yCCE+O zsGvTVNNY^5&tnT~#ihk@IK;RD|N4FLIM*UOR6NO`5n|w*#>Vy2fBD$w*lI~gBSXPM zDg?IT~|Ka;UU2?!<`sQ!E=-3k7$opK|r`Qd3rxnbY$Wd z1Y_n`8F+8Xc5z*W%jXM2dbZGcR236hxBmZG(8`en*>pKLN?CmEPc zU&fF*G<90w)5En#8H7gMDJIM-n3+k(#t`}nv}Y>evLAOYbNC?gz9%}|(=T+Q_KJZt zey$4rr9p%kPGoIaWBqpm?Y?U;@*b}j*o+zh&$J4(N)@E%f(CEJVE|GNji#;ubF3%x zpOJ-vjj4m3hvP~MF@b3N*Xe3pG7r5 z5oH`c@6S$raS*@wnE*k+oM0|+1YT;LNP?s7k#{dz!~e3o9(a;}9g3%CdVYU?je^*2 zbu%?B%@{}4^!#>hD{||DSmxO_4pMd(;to9&TRR%`gmUt{r#&HeAK}=^`oQ$z>4Wxk0IO( z!Zm`X2=3S;55_+~NIB}w!BE!CjYw`EG5v2}%vK+N^9PWnP~PzV(`$YtD#pJH(hcAV zsu=YIu1BIMxw}-0rmrN&tp)B^3{X=+h86~1Fk!;fa$jo%-zU~>&GcIx)f>F>8LTEa z?d{Uayi{8!Gkw{Kb0gh=3Njj50*DMoetW@AMECHh8Rh`}{=;w|IS&HAtjaIa8Mv1@}y!zN)qA(YyzW^e8f0R{4}b6{sg4hL>6N}fA6IKm#t%L0bdcreGLD{zg(|O zE*{+UfniUe8lro9tA#hmX=laZ3`|k$2)zR`2jAHAzX@kA#0Y8#H(O>QQys6@#KSN5 z3heyU$Q)S01UVQ|HjoK+5!e?GFxEvAOqJ5}i~d23RWI}046lq);{~-f{Cyu%@Uu+- zbSj~E8Xg@jJh43zARJkHQr8l+7T1?Brah5s3k9FX*2x>|L$=%hpxUx2F=Vohe1Cr~ z2K&D@5WBkyXe3@V1{`EuY2;cv(+SUy6fjjovBx@Fy|v%r3VZaM2ZzNIAL;J5A*$0Z z6w2>l?-#dP`;r365f!AX{ld=2j;5G~foKObsKs4gk&&N03h*_vh~=ZumYg=EW`9U^ zlP}i)h|LDhhmfLZkZ}=Ua}VH6i8GTX2Bkpz>uc2QF29!xASO!c&v+&!F)I^wA=jf* z@$VL`6Rqj?fo7xN#P&n$*X-FC4dJN4=np1Xt$`7p$zy3Nf&Rs0-q8R5s5*D5BBM) zR5Ms9$Ag{<#24O}IKG-&vw)Ww0mJbFFOYD@$nF6MohtyWr+b|BIG}q!);x)iR13U5U}k`Mb|%8 z5ncTv*IA$G;>E9HGnecNJV`!`HQE<`aXv-!a8w+|zDj{%C_{(8_#>$M2yI+W&4GBT{RkX#AyT; z`Kp~hb|5%vK**a4no>9)W#|_}?}e9gN#B3{GfKCc(DvD0x3)iNeDOMoxG(xanMC4z zzj+vu4F|KaG7!(e>&`>EGV~!zqsV_(6%%ZU>}tFLbFrtwl@N}!zA@Hckpco&x$N-{GgTl}Qq$d0-V zex{e8)!yo>>J`ePh&A7-5`;DjOs&~zq-mf?AKrft_?1P=n`JabJd3UCrf($i*SL{- z-o^Sy?LlUN__E*BG9JbQ;KdI?CAxPMF6^mXW;^(hobs0M2ko7^caeR z@P@FVyraUCPlK!##c_aqLeLF?PRh|&u37~|N4SF!9ZOO!G+!s9@qB!kXo{rggXHW^ zD+;F=I8}tEZQ215DPck*T=8&OdlNjS;_h0}i;tme;iVx|n)ulggRN+kKHBsa9I&U6 zMt|?h@^;*ufd2fT?gUi!h=_PFbr=k6s6aBxSte5TikS(^8Q_|VfkEUXDH$!rCkBUs zoS<9jiS7W2SEM`vSOAqPp*s&kFVg`CgB}V&0C88)bN!A9(eH4LS7NxT7J8!?l=XH2 zZ~6nY)w|$S3yt?ZxT-0`m>0^FYLjjM^cN;k_N4C{@hPe>ctBPvOV0^y6oPd_&#Nim zjposwuP;}SRZJa*nQ6_cN`f238eqB zXRuNU(=pvpMEd&y;DM#Y$$RQJlbB;Z2|dUE)pQK?(q~ZM8a`ZxbqyDuP4Zp1T|9B)4qu}r3BA95awdL~ zoeRJzrXCU2cY-Jf1T3WYgc69I#;q87Gr4^Fz@Es(fd$->G})QL0(ruwBHRoBO%;^< zXKE7;&|7fB;F?zd7x~Qg%Z59iF@A&8?`fn-f)y}EJRoqX$X*=1(!HaPq9~o{hhW6A z3i9w|QUR^JcrZjkYqmq^ehyw4`Xy=axAz2td|B77T>h}r)wNi$KEBaTq4c}S>8HFr z8>Md?KKAbN3h$(enggtRTO+{ppp`!wx)BZjh1nO{)c0l+#FvH&F9}*!?3%-H+vjv=3tKU|c(59qTJK!sS}jY!T18m>6kJsk?|NS0Cn^JFtS zJC}ecx6Fmp&CSJ`K|Vgz((wJB9y$W=*RKw83;Qwku5N3{Qs-Aysh|>3LHL3GCITnN z7&8P&@~7my7VApzLar1yh=48)cGp(abYv>0hr?4Pjm42MsJW|HO6!XqK_fj~ox3O%cDWC%`QDDWH#W z4P(sLT-(p3b`gmxOD!xecuAGz)E=+S&OSb4j)Sblsh6(bq2ih|$q7@sD_PKP%iVX#n zBqmSTUD`XfjOzOX870}HU>N2HyCx*WR5Z{5;c;#F7$lMdOe`E(XPd4xc8+@WxjCIZ zrQV0=da`!3ZBF<5g*HvS%Hwp=CO_hcWryA&-6NVw7`}Lro$$p&Xz<{L&2U2>zs>8MNRC03V zDw=F|=3TH?f2yKgdNdx?CGM7PYEU`*fI$Vk_=fmYc-jrK?@pDfZ2D-v6c=j+p&4ar zZtj6Q6g&@p=RtsuZ5Ou!h;Th>byeVJ#Q}P#Jcnou{neee-S09(Rx4&?T5b_fTM|&w zE*4i7T9z^R{Y`|K=a#8Y+@gvGm)Q7#QrrrRB&x2vJy;5n`=Rc#;GW*5J_*GM^n9@sM+xz5%y zI<9fyTd72Ql`Ia;XwZjKfn&e{dvem>zm-Q;=%%?XcK{ z!^8aSgFP)9I{C(SfEjD}@ti|PZuJ~>U91r0h2ruO-ePDm?(etAl*cw?*pDx-Z8zxY zRF1!Xw{y^+asI+vM*@sm1S+QERFv(F*CWFfY6;0+Kpv$%2ae!XX71l;dQb58(k~_q z`uRRJ&o91u{YroLYXbpp0a0oCN=8=y5Q~`-IqbCFvvERxzsY`H;L_pH3j3*H+#{RlDfOF>XlF~AJ~_4DkzMiw^x;v z34dHxW-~aR5?(zkb+nBW^ByPvs7v6`!4xD88{dG?j$DRe1p%>UWwNoF9f&&7NXm|f zVZ{?Z-}2t8`VApyxa@lLk?_!T>F0%wjfb-zF01=d%_ldm=LBNsCYnuD(`h5sAZ_!h zkMHKtAIg?`B75c{3`XMxFb;|Xhe!dlrwXnvoIw#+Wfikk@wZ~FzOh4< zCfz8f`KFeuSSuuVj%ZwblPzDv?oAxyk=pP21TzjFk`+kS_8oJ7ZNI&AAl1x&w&p-< zjMwngm!iyH8M9h);zu*7R@HBD*}s92Wh8nQQTPwJ_&{&K?JC47WGV@ikRFO0Ipa&t z0&a{&LhYi{gD#u+sxv=^SxA#cBMCfZGGi#sT(`%^|`>Vuqc0osG_4F81ZQA~t5LT-2xlFljz}*?spL)7ebx5J1b1(C*M>j%S zDp-SqPpi}c_)j2cgS?h{fyvnDtg!^tM}u2d!-S#Z7kuXoP3U8)i+YzL~d6H_gA9XbrpCiN1TJ;DO2H&h38wud^5{aOa`<4=tC zRYI>@83!=*X&Vf$-kY$H6^e599Z9t87CU3p@Kj@KVzWy?Ms!}#s5stDZ&dBv-Umzc z9}xYbMcp{HotMM7XLN#-FMk6ToYVe`o*I zYvh{hHC~KF4e6(7=^!H7dJz+DI|Chp%j1h|p`(Gi-{QBwKO~G=n-fF58MCUcj zRX#tf%eX~TUF{r$!Ov*1!RpJ2nu$Gb!b^T0a@ZPR^gXT#%82N^o28)bQ(c3d20IsM zRR*85c*Q|A=n8cAOVqjZv))Qme(|`Zh_`HU+4xKHeJK>LQ&m?Yku4J!F`6()l|WBC zT2p`&$}y6wUEFBKzLU(O*`l=aIq^&0)(kn6m>9ZP>=NSM>{ws}q(eXDAvfO|0l)jv77fT&4R) zO87#r-(KHD&Gnfj=i(>#h~wC4{o)TOqIM<24kjoz1wE?m$!i4;jpmZZ*@FkRJ-OEX zOXgQNp3Us6OtlYhURxCvYPTiAck6ZO{D`DWqG#RGWm0A6Ua<}SdPQL3TMX(sQafUo z?NMUf-v^ICm%y&!NNR`wumk5t{>_C!zpl0y<;ZpIKHWlz45FbS>|r$go>_FM7T*`D z7)V7Dx2hB}QOZT1@yyl-Ua{|EKCL@Da_Wz_CAw#yI8|_UjP7Ssc^20aKNtfv7~|paZ!A zfvO6uN=)z;02(4qC@}v1{v;y83=-T*DSVp$i@o=ZiZa`}MzL-6nC&*Af~bX(5dl%6 z5iLNFj6_8x2gyl5Z6ktUDJWUV8Of5hm7wG-AfS@7WXX4KXtmEf?)~l^_s93^V;uX8 zZmFuLo@ej1*P3gtIm0Hwk|GHsD2D$uOU~KN2G94TI^Sb7R+8i%H_@&5c<~R}scF4o zu5!Ve524-MZGCzMrwj7m$6no?WCn7%RDac06BWMGxRcoJeJ@GBf^dj%DLu%n`1_~e zt67b-m{<6C9R@Yg(LM(e&{%(D;&k82KfY{@*b+ADM5`zVXBSrVO$xw0cSf^!zKnu(4jm<09MpYNP;MquFgNF~q2?P4>m z5At)a&BiUy8BYQB)ZW(?aa-%*qol`|e9Tg!z2xIdOF(6JGH$R^Cnuv!!FoB_v^GO7 zGWzn(CZ}$aJ?*T;pMO5)@L5b5k6O2#KQ@{J04W+{u6EfRxbko38d2{!spgyo4W74U z7u){*MUL#w2)Z(BCRLWa`I4BHR)r8J_f~at6@ysk+b+eJ=cXrU14k6?VZIP*P2zH204!?^PW!3g1P}(nZ z-^89cdkdBI#(n~_n6aQykb`b1TAb@)$|P6=xLje@B|i1@FV32DlAVys#3g!6*|@=MaX++R9}thi9y)l}`tAo^N$XE0U9r z<0|#x3v=J+*1b2kccN|A&aj}{8VaFVX*(GhHf`G(XPEXIh3eL;RN8ZT@e98^jFQl> zChCg84!3;aK#~W3^g+PX5$wS5mqsoF3w70+l>$Qh*aVeC#IJ>6;E75TWfV9U9MJC< z?8i1~8%c9lj+#jAJ95z=%~W>(X|u&@f@62t-~wgPLj)V6AxF>r&Lxwk)wR6y6K^BA8$ zn^&tmUD(`Q8Z@+cafz)rIEfkZvj(6hlg~)7y|4pd_vqK}^s1S&FDd$c4`jNK)L#}V z9naox)%S^iZC|ZKGqZ@);ELsUdy4|=x<__7J*9|@*>1!rQS+Yk#W2=Zy;N;Ow$m)= zxdQ_M?%w2zQ!m!!&#I=H#8W_pkY4dnORn*T{6W8@KHoZp4^?|>9O_I~^pv=7?ZThJ z=##QADL-DpV3$pz3D#rI?|AP@TkfTs&&ys%pr1s*386o&E6+GJJU< z9BRnSi!7z3jIQ}i2qs5G*WIsJx@KNJ3BUeaw zUHzrXv1EkZ^qySqg9qgRMN6cdNOp91SWn*lpj@1SnXwk%d746w6gBCRV2b{Q?T-~h zvxSRe6psf=+lrGvIFd80h0c{quVYh=y7lS7D;*)5g?FbjOjJ2$i4JvokBmF;Dm1CR zgVXU>M~}%C)>C7BtBbF?>J9}Bb7)ov^?IRVW9HnskiMOJ`41y%F-y7qTLD+@#=s-#L*mDE6`&9^l{kP+qj_g`Ugy9z1+Zm-I7EWj83jpz&_yWe z^NnPSd40|qC&p8PU0D0*$Yq7Ua*5}-8VIzN4@76m1teCYz?%i1}#t$49Fup==G*{qg zQ||L+;r83=%}!3R&c3?lOXk;1>$V`XlMJ)S2}o;JSTkED#I096q9NtjdS|?Rq&0^u z$g#k5!prsTdp5aGF^0C^blr!_?7#%Gp{bdI7G|okPc540o)iZMp^~W!5hyCXC}(}s zKN$Iy<;B_{)Wp~9TXm`e(KlMEeGKuS%IEk6X@mv|!1BZnboz>NE>Tou2uui^eN-Z($ z{I+nVuix$kn(*#j@t{D$of*}I(0~0?0}>w{t_Bv}{ijR(1Qjq7BRN9ydxNy1=N|Pr zrx>Vm$BzeDGYI-dnT`y!BC&i-5vdBcvX&>w@AbQKfE197)`3*P;VJ0_Ne3mgLW-*R z7aqpTThBfoUK@D$G=?s-`M^#L79Gobmw)95H@WM!7#U=WzPs~YHt4xa< zgV^4K>pdUKkIzvV8R_>@+4@vj-kO=PP7Ri&bF=Ur2ispSzLfh~MC06UgPMpt87Za| zWo2Fe!!~ChI895Y&7P<>Eq=aE6&E*+>?fhH&8`oBcmA7Ls|s% zWYGao0&)v_x{%0l;a(K)m&1l;W@oKnYBY-hDZ*2P!=YbPE}G2tt!@INr9y*k`;76n zpKokF+|!7rg^ZmkBWYUL?ZLKwyT247A~emOiDzV=^l!u3r%_I?m|jzWsvkUnLcly+ zqN0jg3YIYxj1HzB5VVupl`WKE6U7ssQ?BSP!mD^=_aF9GD&2cNh20>yi**ipT;HA- zDjutbP7R0?FT`E9y?}%c>9|XrIpaYA-SG?@X-JEiLLD9-R|4-&Axw&XGncvw2Ds3q zfZFISBD_QNpdjqr@!0ZeIhlTg_J1@6Ez+s<q}t2S17wQzs3Fe46ar zy`_(QSa?&$-_zOdHFrX?ia<4l+DPc>I%GA54!Gw{3EpX=z-LOEB zv1K7}PT>42aoT90jb`DHNzon?E?UU9dtGY^Zi}DvZa6hBQ1Gc&R717!!hvUQ72%Q= zZay=Q(H|e*DJZ#+j=VlmvS(SvG(nZQbG{!Z6KAzC=NHBkK=A^-rP$mRs%cT^cl9)7 zra=LP3RR&iq6=l-^%d1<7(lLx<>_(hhQ#SxsePi4bc!EUIfacsiFcN**b@iXHo|n@ z+C$k|$mNh(h8d>u&IycGohA--iPYD-qRONsiV9j_m?p)G`wx37Et5f)jMxZSfT0;LrF!*sj- zg-`>mNl!OWV)fOLDmon%DPd=-kzuWzwAHxw+njsRd)6_rUT&@2=$14_8bzcj(tFjB z@SHngt~O>OwTTZ?2aZJMT_ zoO&AD7+mH&qd`;1)mb*yRsK;TaG{nQUM{pKF=l&a)FCL;dO$4ph-9kXn|PrVtBI8< zCUWw~EE~orsHep9z>Dc3h*5n(qme5{#Ji$xw5p>A=VP=4`s)IxrHSN&xs_S%P?EA0 zDKq5gw^Bm9Y+QRCAo-BjfG!DyQr# zA3zbp=?;&$XccvHksV2Nd;D5BW1FwNu7-nic#eizPtwY=h-77+XvYPxw~+lGmR_N5 z6Qf~FVyKIxT-d2I2-_89Rux4{g?a{qw<4i^zLR06LW(vkf5iFSMTXYK)AukEeR6-P zMpa(4%_jQI?D2n|s);$KXt#;eFS=|YOIrWuj{`nQ+=?wy)?cFDr}gs%Qr>mg_uKP- zh{jq?f4KO&Y+S!BEPC?f{&$Gh9BJ$P3P^*rm66>Cqp|AAB72-{4WI%TKzq_T`TL3$ z(U>~Tu;_e9>3+XIuo@?@S9cc5^zOU6oS?machja#6mD*APzRwMD+_-A6|dT#5^pM} z&!~hT(TLSpQPz{+e-zRsLGMB+c3aqkPNQo#$W@{k?jw^iGVMBb+(}?2DNEB$A~fZD zbb`*bcI(%%X|KN;5<739RtvsYxGfzbARk1V4O)O*4OB~U-5#FPHN>4-A9aoO9vYI} z;AU0m20Dh7mzVR+LLQXRs-SYcS12>9@f{ibY09#z8Wkk8ry(f8Y!|eQaGIO!R2bAs zH$Me_dj$d}8D=``!}mVwpW5ewh-BSB?SocbHXfe$M)M8*cgNn?u#bJJayQ6-^0au% z-ZIUMPz$HD#H=vgp*=b+Vi1rBn9Mb`>~3h{b0iWOS)R@YmC8keYoKbvN|0)@hzNJj`&9U}x?K+f$kItxSFL+~&?k3y&lWVVpuxqQQKQRc8I|JEwYz+tsF=-s@<*GC#fAsVUN<{7kG5Ki65; z&Agh*QtH?4^j_ngU(fm7Dx9K!&b?M2vQ(>;>UQs9JN=G+JQkXq)7&-a?-;g#^FwD>)=`tWUsj1%@#%8-M|JxN0a6a|3ts%Aeqg z=%SS_1vxA8ha*KU#4%wT55}uuAK7zzt7c&`lRXst96_X~g+SC9y`EEH#AH(+JsL7z z0={1eS~di{XH!j%$h$;#6CLBue*mQ?KLDlWR;ICEmNE)VOHTXmGLDcfvy^lHBT&C+ zU?V}4#~F{rd%1@+`=3ves9#eD_0Wd&`D9AunTuP;20jUwJfmMNz=m&)fpfL8?wYL` zVLgNO36ZLNF=5L4W+U!JY;zPb&CaC?YJE8|wdLsh?sE58ySh8e^51fgk z;N&(KPe>O)`(Ugb(QMz#iCIsdDMC#K%cJd3$(rn;{2j1gS>W*?$1&Tu?E)k(* zL?Oel2ph8u>js;vg4sK%eqU4!SsE!$_ z=PuSA*U#T6+CGsx);Bi%`k_v1R&`qTr#XR%Ia_2-H%o(-RiHO?<*+r@9c1zjWHTNy z5AovCV8bN6C!E0v4Tp@FfsFVpA4=_nr<8t{R|5>UTFlr|z+ZuH``)4vG{iYs)BmH6 zfjw08)_p|HAoSM=Z=o}^hNO)57xeA=aA@B7bh||5>g)y|2KkS9=V@)Kh3BQEJL=%K zDK9JVv~4J^w7wL}`Sq6e3?_SYapW_R?$JM-1^H0Q7-EG|A+U2GjOU zn=TtXWs${OgVz_i%HF%_?j+_#d>ep-mR<4caN8xO37(W9(nmg>;%6LXLg+n2`Zt%B zPD5#w0w`?F6AW*iT3p0*OwOlH?rW(zGTGZs3wbds)b*&rGja2!ygRv|DX_E7LY!%I z`1!*qtt7Ca>+5oB9y*50j|~diD$R;Jvx1h8?wX(T{iF|>nO!D3UN-psDKrq|g@SK( z-B`K;=)0+X6B6(>tzlX2qtL%hFD%c(K=k!0OLm@_E+?XW2U4xVxk5z^b72xoVMG0=qmY%WmK51y)XE zrd}9P-JEiw^K;7i8+qoZd)HBE+1dW+m+NJ;`uXonxiqvXTf{MewAkIn=5GMti2~X= z@cS(`3WuCY+-<%PQwK-EIr^jtkFa;YKJDr_y<7g8LCZ2JOiTe{4ok(jJfvoJ9`@+5 z5-)fsAqVkVss5KQU+&b=)l^V;qG{jFOz$Q>+cG~JNZO3CldOZsI2Bl0O*!C;Rs8=Z zB#;{NTqO5T5JM$@5<_p?Rc_~amzr>3%@OtS`aO;a1yh}mxOr0s&V=|xEqmF%z-wE!Foy08kM9a^Z%jtc9ovws zC0{UJ)3`UV)I4i`F5?6qv|N5cyM!F1sx<8lz0e0@M>AcsuSr@&=#Z!KN}FZN?;4h* zo|x)>YHZ&fTTiNb&hOf;>Z?L?Z!BS90c3CW`5m)yi-X*0X{`!un&I_p$$6Dm%oc%r?7m!@TiGT&rrjS>S?-&~qdxN+%URI7E}iS${%4y{H4(b${15>$fDV z&amJ-^Nt(i^w`ZG#_)4$Npdzb4KbM(0BDO0Oh@AYGjeg@fjmAR?@ya>n zrfTbY{JFil#0IlGOoRh2ex$hue2F_DS{XQ>VKI@z`_#nXZD}wmUXa!w4TA+&EQiBC-1c?2r1X2 z7Pq#*)oc3(*md_8SQg~xG3$+gEuj$wxiHo{&2?Bf_CZa1Rnul)5~9+s9MPk3TpjLD z7v8sv{?N8HJs&f28V*H=HYgTFqSk_RSWUmfn}&qJWF3xA5!XLob(^-m@oCl8N=e6E zygNF9vx>}pS}~kyCW9hsJI^;WR7fY=kyc@cNNc@%(UQ8eyy^P6q>)^oG|fQA`AkZr z_IVk%T|-Z{*Tr(w9hc6y+Mb%D_ENTSJfM2uL}#^bwex7_*Z186o7%y=X4Mlg7+#~R zWnxp>nZY#m@shhfaynWFd;(x+CJT9R?I@lJo6F60n0S(EWrY5DH5UYQOse=+u3AN- zfUhqFRv(%w;8M_%;RSQ#Fva;kQCP7G70VVjFx{;9)i1l)7szb6lY~rUJ z#`-(j115ycWW7u9X~)Bg(c*clyOI9&X6d?IEeZ=9p^C5+-ksB+d6D}{T2eiKujP-` zVwRkwOEs94F)e!YMzmyFk^PCBZ&=FcwfIzn$`MWgoCXc9irXv09es9X?Qj#%{?@T( zvguX0_Q35Fvt}Xt5bIEdO9slE;i4L9P@Ul^QAPC8noCxqDI}`=(XVpw)D^i;^y*0x z=dB!K5&{^HX!NG;B3f<9o!{&bQ!n1=D9Jo-rnNENMATpZ6K?Q=8VMqSQt~4gbY$8CTTBZY}2Gc{DGz~1>gb<7az?!%=x70`#^w30vb&8bFAvGUQ)X6 z?+-YT{dAaDqEcQ)gqbP`ThOT1>+*zg1?voEDrn?VRi-}};f2ok?^6M^Wl$oIpD&Xt zi2wAeEV-3835kh9LvP^5>e9)k4;Ml#2ZkZr>zdXst+~|bJe2>!QVm}u7f6*&wLGdJ z@qPYrY$8K$gzp1P`8%Hvx+++pA}NPdH7phMQzLLB4wUV=yD4|F!xt5kR#>a1UHfJ$ zOh8we0e7n{@twccs)WG?_4cPoDYAF&+<9=uA`mKP*%N~!ep*hGS1=974$;qz`=zCl zq@j=8>@X-oeSfMItjbXOOTWHO%gn_3rqyd6Q^vYHCRw%SHB#i|hT|%aHI=li@R2(| z_%%v1A(_%$cXrxp*7|w985B5^FPE55-)AHF~SS>WYuU%p)YF`V==PIqo_2d7%v>0`V9(4U%#YpL3?YgT!qs0M`^buKkT zT%?itt*x@jcf-ybytEJ#cYZktY9>7?@O|rWhPEeIyQV!)Kde<~u+gS3h!Al|<2ht# ze#iRPSr9$#KXOeIa&|#Dh(460c;wg*=J6pH%SGL1cuw(;M?6#J1FBZG1CBL)-+i`Q z?~1>1DYp{Got#GYxT#jo^i8CFJ%4)&^$1N#`U^WZS!QS-J@+ikY2*QkmkTJJC%1fM z;;wCUs}9^PzI)jX4;*SZ;b2r6fHVO_-UWk}?X9hQQLv zEkrzw$RSI5aa3TEIwsJ0?nucqqVj|yG*R)N-21h?SA0u!d+&q<4KVTh%v+hOG6jpo zSc;R`6=(*EtkGJ!xvXh_->zKEOjUhpe>d_!G|yJ^L5BnzAp+#L6wjaNjQ^xGts53~ zIbqK*S9id^r++NnRd??OL(&M(jo)7A6x40nxFi17yOWbEZnLt!XW0)4(w8X>Vm>@-e!(zf2kD-kSUQdL zZqptMYjcHQK~KJynem!E)6ZaEF=ytB?irFbDvfC9TKL|n|Iq&6ep$49@6?q2+^(tK z*YwMNzst^aSe>R&ST7s9y{0^x+p^|y|BfqzCw^P@&Q0i;i`xU4v4EtKW#Grp4Gf_NbN#*NC5(7707poM+i;{X9o2<5W>?f6beXPpdDz(``uUbKGoh zEwQoaeZ?_N<|cAicoK%f#Augwbv=OJm~;Q#^n&h!{ub_g4L5;2ppf93xDytRLNd}i zx#QN4`H9!V_BQ%)%^EZtL$p4-q>o1Yy(dT^qsJ^wij+Y?LffOV>Ge618590=dahC3 zhqcqAMQQWl*f84G!&lZqGyBIX)|%7r0tc*Mrx z7{s8AqnG**p6_*a)>aMm!Q5Ic+9nKGc^t#<2^f zyQ`Z$a!LvONA&5y&yE-Nj{>{%bnEgD_3sgi3@gHnoVUvC?-kawOUx6xR|w|l!g{@4}xd_jx$ zL)f#WbcGCeKP^|mFc(qCi-&$i406=>r2~PghUA=vzF-wsC8o7F;T#KfsSo|bKR&!o z1qe-;QxfT?c0J$UC?LVx(SD2}1&wqf`Fbk1zcNXrF@zoijMhzS&ouw1I$Ux@!I43= zhBqbmyMAnjaKnL`vhqRS6x*cZZ-XgR_wZ*+nI_L?+ZSnpjlnK_P@M6UI-zxW4v!XE z4&ad9mk)PXV z{WygU0sHFbLnGA?BeQUQdkN|-I~AT#JATT~U$2R2bW(wau(JC)x0)CyMY}EB27+^k zFz@3#Cm6RrN9{$`R#ORD{`$ceiOu>pI-+B?@>WI~T7)HDNEKs&I5PLB0XrIeF0L*W zpEf)9#FlNL(vXzsk6LKRmBa%hvbWR=LT9PEbXK>z*OPnnOYc&Jv<&D7A2_UT;bm02 z0&H`dgzYQ+HeUajajPYaTOP^s_{_C#hw!c33TgF&4o@rz3D!wv{g`9Wl?*e@>8k;l zb11aTzBnuNjPst!M>7SI#s7kr!q>Wh3seml7|riJjIhSm=N>l~KsL zxEfUp*uEHHHfFmQ=C{zx$Ai5digAse+-U1A{5b6~6jAG+fF z!S2VxR#BZ;=LXg4iiYIgSJu0PqkV1u+Pq;W7tmDIU^)5})AP!2r)+K!<&Nb_hGHHO6y+GHL&E2nN0G?T4_C3Rlku+|f7?kvSZ+iR%fnVYPV?ISD z`a!U?kGdr9W|#8-a|~*uy>!AcpVa-wDt0WbNom>}CC@q5ltr`r_Kuk-wUbUZ?-rwK z@RA}MFS%8@O}sL7BsI!I^YmPc=};&4NcG4+!zK_^#%juX`Gy{ZX4nawna#fzxir^_ z!(lqPKzr{zfl7&5#8!4Cj<*TK7z7CF!q4kUmyC3SqXJrMyE+W*xmhU3fiaqMfA#4X zn%}vzkAFOGKOg>}mmTrE{6B;=zFRk(qXfae$|I+r9|$36m8=vFiIw;753|UDHt%w>#j@Yi~ZIyr4Ws8Z6PzPNQlY9?CUr_{MC6M-0Kf{9)n6?5K z@|Y8tFTuzd6ylTA*USpQzh5lAUOqqd5!|@ z%me^$G#fqDHq$z($ zv|Zzu{0Gb7afa*&T`d`hpc9!LDf|;v%5Cz3BicIYa(Rk5uHm3`AY+ueD>KP@kWEP} z{$m_LTYBnEquTe@B>6XfZLuLu9a_kMJHOJb&*nsirL}pF(tmyj+vlF_8pme?^jN7b z5ARn25VM^6dUiH_Fj;lXQQ1O_KJsB~udn~~lw?y)d&TmonRp@by?ZW=f=|Y8+`+c& z_Q{C4!fwF?rwoZL+&=ADWLVG8rW+QSn`73mfuct+_1es^)FaLjB@e z){)eX6*Sq0568g`+U7aN70_R{ytl~em?e5JUWGsQ5Oin~0XpbaW!WgE_3j{J1Fzh- zB0TGK)O2R{6b052_Ij;}(38-cUD>bCSuxB!GWoO;b1}!noTnr*gcKglBNreu>&!n7 zTZ}!0zPjr>B~mX(_CGiUgsJFxEh};JTd+G!rJa3lNqBcA5WSqfioq3c9PmTADBJ;FgjyqUTp67%WTqDzr4QR zt_eK?2~OQozM{&5Bf&b}`iG1nLl5yQ>!+W7Xl^)=JyDqbhwSD5+`Q`K{H=BN_UD~< zRy-1LpYHGIxAwQt7_6JNa(S>ctJF$ktLYh9b5AozMNes)_D(@N*`^ha+*I?cSIeK* zsWNXL`F1f`lX2#+T4SL-)Tz%UsoN(@CH0*&Y-j$xbnlPM)j8|cy>PCXsh42uS39#T z%bZoFv~N}Rt`}Itdfge>8T1<{RQovTWo!Ez56QZ&&-D(Db&!eF&vs!`3-VZH+pZMg zSHF4NrXb^K>F#*vadtt$@#6CMJC=%LtC%H>RQ{ZwQaXs>ztXbuHm?yar z!VV-vv=oP!e3pUMP@WSG5v?OBw7wSSzN`vS`>90xEJFuHd*6NhEcZk&{?~Wq?fC2I zSp9S9hKz|P5B)Mib}Tuz>eWUL8-cBxM?b_ywL8`%o1D5baW!dsYxG1$ck2>lnq}0z z`Td!@KMV{+Y2VIP$dsVxx9x~Lsn(xgm*sTDlS}hJo?TtUGh0vAga5V~5`OeaU8B0M zUczA9DD<>T1%8)5UhBx&I%m;Wl0JiO`~Mm<7g$_N%A9?{v}cscBO7}5Lh3Hy1%v$oh`+h^WH#1 zN>RX)qi;)CI9Q@(H-FYx1^~QyPEj^pRcRhFF7hSx~^7C)${>QsA+7+oo?$Dv_*Y(p`PZTbzz|30x-XCM6c==1A)L%A?bQe_r z>lXd{X48fJmy?q|{6c1!yJ2N>id@j3`qD zh&Uu;YKS;0(iWz8h(2um&lSj&M7~a{Q9U0#c+q4(4gpjJFcil?=O98{=2li`%hZ;- zpN0JZvGN5Kg7x6RgYS9ZO-7ihe4y>L7e0Dx5Yw1u8VyYjRbWs-qL5p194QeXESFeK zvCD-U2(6VU_5_KpS=sWGjxJ$6_2e-Ha1VanaHb;MzpfpoS-XL2zPL%GG9W4z37!a1 z)kY5_5)+{xN=i6SC19l9Dz0P6pQ+o z7yrVgu)u?@Cg!|6q)+Og@cFoT{s%xnM;D;dt-1l?clNY0ydkkbh;+dcmSF?$3Vk-o zGT{+dK-_!zdk2GUX@KgI8aKQXqm1NYs6_9X0Oi4^pf4Lh^9}WAIHnhl zLj0FJ(LeLH37GgnNh}hs?Ppvx%p?}5rVC5#@|B15{&XJocoruRD zy688h(Sw3%eb5~41iP@XI&HL8&6Jrw7{cE~BzhFne}rUa|LyY<-JzfB zoU&JAbr59RG{{@6#TfX)dW4)z=*d}#13jTYsq`$!Q)%17fhJ5rgaPITLFfWSg5B2r zctLWAe9!AtL^zL9jDdzJYucTH6XFMN-f(E+HJ!W!2%U>c1Z&Vhs!@Slh(ZvdpDq4e z(8!3}uq@FB;^GF7K%#|!O$RNZQ$!936ayktg@MXQe2e>WnSi9hBzm;xP%G4BtM!N? z)%Vb|aBpOefPPW3!B7s=;eCucpquDzFkh6fLR zoQr)C=xRZ|kVav}t;6=J0i`Y3tW_KeF;v147KJDRRGKKSxQJy1(R=01fvE7mzev+1 z&U<%m-O@v6obbeOFNq;f_#-d8*dmVR`EXtA4Sk15^k zB*1%Gi<4UJ?6@yc!iB1P_C&M8%6wZ}TNXfI>luZ~wf|9Fdp`Kbt)sg)ro!fiH%C~I z2b+tist`qQKmj$_bs!jqV4pH?6C)N7*dlEHnu_lpBm81ceGTt=P z+E(l>PlMp85wFtB+^he3L|cg)2Jwm}7j=JCU@eRx_vd>BX&3dzI!LfTi)9C| zh^5p@(9khb889ln@r(_a;rtB~gDo=U$`F*MR-!L%*;&dE%>%;PW%LjtF0}mq^Nf>X z!Ft98)PizzL6{NTKG0X5Lu4%is&R+n`^#VaRiTMX%V__PFc+x)ybYknOO)3Iomx#z zXp{t_tk%F4H|h*+)g~5VTKrLr`tyE2CUKBV*MIwRf+~l*&JW@4BZ6UlUbx+d8bJP( zcsg^vfEkk4hu0JPW;m8o%Uhs;g|Q|ojV#CW99}1RFhUT6Q3DC0wdwUWePN=7e29l< zsNtzA5L^)_Mwt2O&so1fH!6AFp?S|YiZ>KYtOuL@FaIHriNnK0(RaJr7bi%P+|Pdn zvCp=`M>K6Qx&xM7=RkkkHcQx1IA?IGw!v)@x8neV=FiZ0ee$wo*=G=24mN<*Laj-$ z8*bxiNG(kI|?0MIk49}_scK>(C+D~2!`~@EgowKutC8{jb=t9 z5E19~K%OJAG%4tvVdO*^ir}i5*0G4y#IXS9H@oxP3^o5dVo)+<8(Pf}sBY$ zhtV+LqaCtZKSW#TUT@^YL&STC3n3r4Kg9s~q_`-DZ=Y{eV1_u=+5EbH*357bBH0b= zzK<`!v!e9aT)j#_O+M{+{{a>Loxx57{1RUQ8HCSJvbR9zXFhC`3^h5j%hCHZ<X6N#c zlIWDxBqc-@?+ zTNVfJ+y-VZjO&%ugw!fPpa

$&x={dOA9t&(8wh?BF-JoD8IU;v~T9bxQI65caPy z`S|jM#>mgZfX;(I#PC=$5=3za5SyW~r{NN(2EkiUB6dSh#9WRfF3e&-ZV(+^hgY0h z2UdbI>gFn}W~{bqRFw$(He`SSx`nhX9oUCJknHKuE6`ZTNKHmt=is2JU)44n9eu1W zMRZ;fS_(Q=)qaOjY6_w*C`MXv%8*4irx>p(LgsPu(m(F~mvGyy$`sh1K%%{hw@i31 zIk$41wQwvERqYytkn(`=Y zB7l$RvY?EH=^hQl2nByZE8?F!)>Ux=?15@bv+z4kjast9HUYaJ219<4xYgTvv~Q6^ zSor;MoJYJwL4dlT@WG;qHAbErwG#WFYEdC|aV7RL7OkrIzG@3c1hM^%&zb6J&9RF5 zJ~>I>KRq$Hzb8=4IVa>}-KBoN3J{;s#ut^dv`mI~b3DFHy~wX+Mc2#NxT>PK13Otf z^Z4ymE2kighwrXL1wFuxF#9jNB7~4hHd+Yg{XhQuwT&$I_FaVb*VNYs+!U&DL$WM3 z_{Pyd5r^8tD~JKAKCNGCo(mINprF61^I2tvI-h$j8OSb$hiUDkX!Q z{CgQjrn({Cf}QvmC=-vme89>N_TPJYo$pZVL_uyTBTr01>ep)Mn_=PA;DC7ScH^Cn zdZA|ptFY2iUcnm$u69d!->bI7YV+K^xbM92I$r0Chl(#UjeIZ|QPy7fRz0y{r_S8p zM$g`!k-mJnZEwx$DyyPTcqDr0U^e79W&PUjS6cTE6wlnQKXCVvU)z?g`}X4qy}N$bp$5{%trkOP z6CsgWiG>n`Jpf1xydA_f!+pRvzx_|6WH+y?0n_E6jS9OO~2+_UR*H z3KE^17P$Y$$zYmn9E(4nBu;xG=Ik^!(9~$#j8R6?BOS7ZKxp@~E0s8-K1>ww)kLs=Lh{*|7#xsg9srP%iVP%teWjc3R`hNwqR7=i#`7J#) z_4P-NwIE_|-nelu)c_U`L55J4(M$g8=FQ6e$9CY{B78XNK_!=bGfPVa(9$VX5cyN> zmlNe^tFP}?E~tq1{Y%gEx-qI#H?<|KRInt@mZyPX8ms}t;ACC?W=UrHPFGQB!mV4C zV0OFcyC%v&YC?RkyV%^BgZcQUUK<+n6PEun#Cn}Uu2pPs7-`zdDOlFAP)QAj0oNn7yD2H44a!`*0?bs<7e5Nb@0kTfZ!P7?y6kC9u4K!Ge8r=dBD5hQCV7M;>fugp%!86{YwQvHDY7jzi1 z4?YOrNH?QVO+`STm8f=OvN#Nfh#uAoEd=pRA3KlQq5ba_5kpo4f&oA55vfE6oD0|o zILC^39kJe`bVEBzVz>bxyrVxPHRf!z-w_KUXhRhKp@~N9uJ&zj{qp4nDoVm0QwDV` zCLcSx3P*Ac95)c+sd!!ory$uu?4KYBC=2#y_x7I^Dc#Y^%4{Z8xOfm@?smf}%$mc^GO?Rycv20&KwQ zqMTu=Z~W<%EU8hULuolNpaIvHXw34mslWPm>A}xRh>j@%>p@UZum_-u0x~5lBpS9$<5=eD43P;!dL+h^(C7UNBD`+Nbx$IpQwyDD1xV_Y zqskxs{AMZ9{emBXdriL{w3fJma8WBE5nCyA>em9<=Q#jb#}AM8-fhNQPWDKt5bO5b z3vX^(mM$vlkl#G2^!ZuE&(Z|{ZvWz6E>Exjm-uL7@NyPRp!vjGp#XqW&J&a>goTS` zEDJ|91t`v*KKp~wfzM*hX0o3e)puyyO~n|I4-!;z$`XP`PEL;XzW`vSUc0ZU#cmHD z66IWl6a&Rr&1}J72tNM!@whKCys82yNa$n(_R;NIw}_Jq5sijaX2n(%-$a`lYgPfX z1H1Uo{&*c7-TCspTTkj1Kf8FBPrLtP!U=@@I{2nBQ=zF%?kB`lucSHr`u#$|fCx?t zEOnBl5#7yv3qsaJS;KpP{nxK~*KAbW;KmC#Em8s5&LHo)a$!-iY~hyoVNAfsn_VbzU9EwJ@E-W5wMu>gSE;t4De=mK3NDla|Kl3=EJ)iC zEX=I{pi|jvqhIfd>rg+&2lxbb7j>AQbnPS~er79wJ(0rX#4DD+nUU~gP{T$DJu6*a z5Qq*(;6aWBVT8rE^RB}KfF_rJhv6k{Z6f=O%7m!*-k(~%nJolan6QNXuTM^Vd7A4U z%D+ZNJBl2p=s7jBRG@w<{q*~<-zgrvFLVNTNsTz{!1hQH7DcYoKgbwZ)BDB?*-t_# z%D2N1_3RT72kZ7ye>E(+C`g?I*mckY?-Mpq{q>IDx|UUdFA^$-28uT2mCLZ4FBh4bpNyO^8fjk#2NwJrK6K^L7fE?26lu1z>Mb*YTJ8y ze0mOJW)?;wQDA&S51SMIRsa!&$Ay19<`}e4E?)$M2biJ8*EEL}K{P!*J(3J2z&aGt z6i;|{rHO(Ew!fq-acXr0eHo$tCE?@&Um%*DY^XSMCOdY(AWqEd=X=lJ-5f-Boue1V zQ9Z~NZg#HP$`$k5vQ=GJ=!D#hDlVMNBhOB>)S*I+%_o+5D07In6;&?h+N1)C2vVJr zLXH4-C{~FsJ{k<&UO!eg({3enYZHa>%^b&Slr!L>NHAbnT67^fT$U69dSjZQG%a)2mGY({7L#`g^4&`iJm2~Q=9^) zw!OF4_jxX=?*?=L@FDxD&>U2tnOVbH6UjGZz}CK=NeNtIMM0@?k zoJBQ$cH$}GzO6IRXg{xv9aY{0z|rb`*xowXN;>#S(q$7<1-9oi&6*-FAhuC z1G4ii&hwnqC>UfxzK!N83oi&yj`IiK*gAABxqJ$?W8`b0gk?uGmL~qvIMoP(hi4ti zR*S>ACd10G{r%(95C_LK%ah48gK8GwbE?_M0<7RYCRLLQ%v_{La`7o0U3I0X3m~Iy zCFqX%0xb^Iqax9X1lv4NO%Dgyz0Zs+J3?`K@HXkxeef%oKPHGV~iMJR##ON!=fTWzYe(;GvZm2fp zjChifLb}g=T|72C_K+R;`i@PDqcUziW@v@i;; zqb7b8zFUX}I8agw6$&CjP{L;c*hrhAG-w2Oif#p&*NyjgMaq- z6C+(J(5E5+jK&AR1-Px)F=Codd~~RC=oAAB4!3;2^k@S-drx7vKi-tdZ$jX=_V!SM z)}r{vIDI+r(kH;xsd6ZyHKH&^T+Q}4r>&_kG{hnVRI6vJo zuYejD?^)EtPa2z+bGmpnX+T9FRFN(}aNt-jG@w#*?0LD`F%B|b!#F!LGs7cZ{t*}(l#lEs5*xG!*aR_1AT2r~=7$2Q2VY~jvus{vnC^lzW`!P6 z(lavlO}@D^@1n-QmY~6Pp~x^kJM5qRa(@cW8}bSp+@R$8^5WZ3#C-MzVgteBAttDR zhWn;YYm~@D<|AL}`yyY3X=E{3z*%Bqq1KShq7?HCvRIG4*wMWV1Vm42ZCVHlZOZhG zTLkyOIZj&uAP`o&1NM5H2ZK^#Zf=eQNWy6VQXi7hg>a_adSdmmRh!Ns4T6meK_T>j z@;0D^L1+ob!`6fCCi>Lsi9DQJ9%wqxT9IA^vLqK57m2Fg1u#IEelh=6LST=?BY?ZP zJfhSnL~EHHN+6$VHP)j@tz?hS+JnH1wr627R1IE0lOzJ|Zp)ct z_6K=E2pqr9YeS%{|G^;OT4K#k+OmKqrOe;HX?4CnY60VPAuSgxjwIPbfKj9nh7XP&Tc}O!0(4AOf{s(kaIcM|OydCGD{%f?hmI0#Xbk;B_2EYDa{7va++=J372B5&$@H73_(xpxKw# z`nVxyoZKw5BZ&9~;rU<_d*ZxA3=GA6@4{0hX^Z?OXTVtE((s70cNzK31ea~O8}>7U zY(a*7P>L|;&GD3g#)2J4ss>HRF-4L~0*Lnd>#q|TdE~9H$B>*Jhfo!X7qEs=Bi=Je z1;PQJ5Kli)!YXiVDMCo}tiPZl+5ySidt=j@D*=^`LMuB0txfkKMY@iwP{JXQ92_-i zA^7wn7|Hk8Vn?8W2=8IUKS@XPF~inB{&)erkT{`}SbH*bywI!#3q z6rNb(0b(Sj4gZx7o8ZBuj0H`M_bL~gT$dFA%~lcK0feOVDnqEmQ|sm&o{ZttXg9#< z8#UYD>vnf;ZrYU4jBwOj*oUuq_BUu9=zfrymM--W~hzo12Rv>qJszHs9}r$ebA7Gj); zbbPB`T>0>XC$(G@M19l-q2!b!K+4fAo><6;vV_@S{BQ^+IEas9a7zR|Og_W9-v>Rq zKV7{cmPL6F>pQejWe%3YI4KBUr3V-g`tn|6>?*Thaq$|!pveFZ;@GRChU0``Gu~Z% z{@a(gDF|_>Y!m@m(ny${9jp7Dr`c)RxGCM@G;Uhm^(9LgCnV_(#*)MWm2xH8-odDW zSwXBI(kqbbdK}ctz`#I4lJ6ysEe{Zkn5iEEL^5G6=GXQLvf0`zpwz5Iywa1%n^*XT75?Yulk-!(+*CQVeUWtCpj;9jDdr`IEAik& zRr(v<9e;Et_H`1&`*Ow11kKgmk593vph5{y~;5vFX}WMI29+v7Xu^&Opkd@Vf#oAH{dvw zpy9+Oix+^2HK=knGct0q4&7CmXLoXjsd&&F?(umvv=US}@}J?l zz&5Tt5A)xIP!rI^6d+PSDabSaOCwoN7I1G*Li>u#)+og)A=u_>9FK~Ha4ouUNU>(* zAhk4T%19~UA>B^=w(AgSckqSbBOt!_h(v0=yE%l&gaehx)e`0ewr>IgAI`JC-fB)gMeg`2N*d zgffvSiDmf&{cl2}!hv@sAIlA#P62@x_&{O*_bK4+hQzR&NE z=lfjOb6uzFI%l8uF|74j@AqrC@B4MX66G9-k)Ulnh*#u^%eL5A+Lg?AWo))*iA4f; zM?gTpmf>OJC;QG~c2tN(OKr0)SabKXNu&VoY+R(^^D`7%=@6)hW21b9HGmQ8=RCEX zeP}lrU9GOcjJ-4Z2df96BlbCoPm=Ak9>`eifJ=YXA;QgxFNXGtUJFk6z+0!lNj;0Z zFJ8VZhSr-6Bx*?WYbVqpbY$S46#c3nv?#)g=JDV&%q;%xN>{R%PnmLg&E6eqy(%QA zg#=4OuiYrw-gJ5P;sIX`y~H{$R0X~#KhOEm?`^&fSNDcX?SDuCB8*jfmjo{DdEFk6 zaWI$vQsDmNPUFJ8S&PR;5;HS1_h%>*dx+por8nVagdjDg0l4SxK!D`2@A+Et7y`NK z7rCf?g;-EfOv~9-;Je-@t{tGm=AWn+=p{SErDYV!V}>8@6)d-S>i-emt}#*KQole- z4o3ug&PD=TNuG(c{?w@?PR?DZS!zCt%li?Puk*v*Y}GY1tY9O{CPznqbfrboVt9dw zx2?F1a+c%Vd{H$m`y~|s0=~lWCePwfrC6*9VwC}lxKEi!v=*0Wt^248aIH7Sg;Y0D zTMf~G4uAr5@$9B#QedH=j6rQfTg4y;As1MbdF3z9oC5K~)L2gvfVXeoZV~O*pX+-F zL5cL>%o*jyg!FPPSt3Cu zcQC~SbVkr1`mP8POvA3e3{9Otfej<^8L$DKrR>AVh**pK^X`4PNinlShh_=V7S;~E zb)MedWn_?tfcyTmBwBE2-pkRpCY!&ROnaD70G?rgo!p83A9(Yclb;=rA7{tL#sYAB zj7M*U(1{Q4hVs}*&z4s_)8oGJI_d;HqhQGuq z>mlr9m!JnS!L3FMwOwv9``9wUa9LFai#-ASJr%tHZ=`O-^)Igl8Q^w zTmapx()Jz?c=I+Nxymxh)z#H7Li75XY=RLJlz-OP&nX)fZwxK_)Uh{KQzC*43ipr;-;E4pS6+I%v?E z>g(^J*`lmQDiXx%`Pqtq&PY6le^0;d2NOKA zUod3WHA`J{l6HTZ0*w^2vRLjFsvsMj@|;Fo9U>gG*4W_*)7C*k+==YKVcBuc4H(>s z&f48TzOSdIo{Dks8DpJ%wP!(N#+s?CS>*l@&) z#ecG}5){xcO?z7_{5FFrmb3MFG6sar03E z6cDl6z?EqK7>2>6Iv{Q8yRjlO5^HjJTuE$23LxMgRAk`o>C(o}cp-u2V{J-pqK)Ev z=e`7Rjo>c|O(kF&Yn)%|Fp6x@p^JdLdTWu;g3m8`eLsLGwneI==NOB1X1v3ke(7;{ zNeh=i>{)PPUEBmuqwf*n{r2U{f#go#b+zwk(>UswHd4x->I{WOi#Vq7LR7Tm1IbXZ z98rICABW;fTzBcw*=6vr2Onm#gwK7k1U?j;vkiIn^ft(>90$gFZ1fkMOwd*B!SakZ zWbnCgp%BZtBe_D)u?Us80b-7N$xQSA7ur*Fuz;!T~iYs)yUwH1~Qd5LXC>&OTlIzBg9rlf4%T65 z9do;=WJ%2qgAwotzdyAZPZSpw=C zR>G9?DlUUkYVN>Tlt;QC@<$)}0qxJdEc{l?5Mdw4K{2fV-ed3lwhq;9{abWP6Wk)ZR~sB7|(s&mNw6b2pa)O%x0?`#gN%+`Y4eP3F8)ZJ zPqURnFH;U3JlF2`ky{tD$|%(WC1>U;q9Abp)d$hsoDhtmjg;P)bnPvk_}W9o{Hz3r zj`XjgXPN>aT&frw9Z+HEfj^NuT08Pqnj=eKl`tg?0IGqLXY)Tq7^J1P=opDRMG!8> z;DzPvoSl$WSSD0ln#C7a`<1$DP&(a%Kt)cv3905ENI%ztg56hky-+Jch!I-uBco`K zcC!=chk_E*Mj1nhEU?jdXu}9dP;eVbLzzjP?#sc3D@HvDTYhV9Jqp`#(dkoRMx!yv zhhWhA(Kle!)G>>kT=fVQrn=>&*-NbB<}Q)i=W$17a@SrlqqukIYO&)7Rd;x}>u7KL z*RO|?8olKiWs*R##>+sXRJpNqjKYBF7Ty3^*qr zR5i-J_n6DO4_buDSNFG4Rp7y207OnBOgmS#6o9GfPD2P0g~qYm7-iFQ1d@mkIKd!> zaH?Vu;OKFIs;8yi`6VSg&YnF>vOAPBD_Pc<62Nl~XsBf9t+Mk^1rtp0-1F>hMJ5;V zub|0UC~EKyhtPM4D8wxW=!gFTg+2hgpuU_25DAO0?|7&i4vFw|&?T*qmw!ZVEKheh zL{$Mz?Vx}p=^%jCn#zl^rYYh(*{>Hc3$TE z5r)VD02_d_G`)Xysez5^ShT^ezy{XE@j*2W6HNRuW)Tt+LKQBK%EgGQ-wIFzP;Eoy zEthLS?u2LT&tvaOu((Jo+HqDXfsjt&95-H%K1x#p^EBCnMO=M6^NB>k!r$uv z7+ozm#CMt&C9}LTzI<^f8V?_*5kQ-Am`<*V7XX?*9=7mrF^g#X%&8{|=iKm-+4^Lw z_c_gN1`Izt_zr`8VBh>fO&F4E%_OR&cc! znJmcYL}s3b?dE_ZcGGrq!tC()98pz5J`9}#!8v+&m0fAj#`4AE$B$k2PQ5n$?~&na z*7$~$tsy95S330@TO+8&p!&=SJjjTfF?;q$uLY~-D=y~YvFb%u6jR%a^K1L)$<{oS*MGb`GMJD?B{7WFUe;G)89YBjW+Bd+X zjrxT8bllyC&?d63s=!d38gW(|ikx$@lWn_y{3rwHMFP%B{t5tRR6bE(plim+i-m$q z?ntE|3+YM+g$SE}@!YwnOz+ML9Uqv%g~hHTU?-cRJbn@DH~z;%oQZ}257)>$_jy{H zamOc&bj+iRB64*ICzy1yG*XjvX26cbpa8}(fy0J&nPwu9S7-`B~IM~hy_0b-?0 zKqL}y@>WTPUKC0e=kMuXdx44`2Dj5z7I_7_mH9pxcR5G?`Yb(ExrP{hQ^$fv3pkqv zG$~O%04I5f%%S=8U%j`s`0<3_Iv8O1j{k}MF!nj45kfOAB~bVL)M~XlWF6mJkfqQr z9}43pjSxPZaOgK;kgJDrQ8CrF8%ie|e0WsBk%$0)#;@@4<62zd7+xhXp5_!lzG5zU z8*%p3UEwDZY5_*Pf@qeJy$nsIe+cLTvoNl(1~VzlpBNU1gKmn?FL;d=EpS@JyA!1y z#dkaynpeP-TdKMx5XSp;Za=qC&G=&vCc)|F#xGJtczy&rwcyR06+)|LXzu*i1ILE8 zX=pnEf3DWB#6dlf{1{j?#onmxcVZ}PBQ<$@dFfK!LBlfNDt3T-eB%5IweFguwTkh3 zGxjW&G}8gtP6LYIs9O+5w;vDQEg1f!x#GiI~; zfMHb&Ef!P~a;=v5eH`h(1K@4`@4x+z)BaU53V#VN6`lTB_T$^jW{Le3%Eo`@e>w7?W}nQFGqV_(WB-$f!T%sy{7+{D<2Mz9ha9K-)ntO*jLMz^uLL{-#>ZZ?ptw>z`m|f zlU^f1ioWroVT@;Q3BShk03zMt9|fz0SfBvkBP72U`sdJL;(oOc0@$0*99KgUo!1nA zv{q1Fz8133arW@9U$SIU=^9TU6@+*mL5biN3QmfM>!Rg$hd!jzT;n=nF1BF>On_nP{KPJ8!a7V3$ zw&sj0*%n+D93xBrO>E5AZ&zqe^MVrt7xyfF`#IoE^dXFz*VkWb{wCrXr8Z(11YA-s z7l46M-uPr2-u9}Y7rAKLF}Uew)~d&A`w#B_kqu@cBJQTp1285}%!ILhYr zylx7}h|1-cTi^U78qgtw{&9I5e!Fv_`3<~2-kyrzU$I?j@#5QCo`s$}%sZ}KoHA)r z6+d5w%Fah)rp}+d+e?B=u zE0YvK(uq%+GUdVHBWO0_`_Xr81&UIJ(ObgQYF?h%2|NQOcnC;~D)&9((C1%g?wawf z_!Lxz6^}@^8~=#u8uE_|IkNOG>D6CfT{h$Yy4w10y8B;AcmFN;5G4OU_WSzpQT|^! z%2c^A#>hg8C{|$UG+3?F_d)--y?tIw(!Y|ThxV0)VLLp~pi;+A)f@2h`(W2%5jI7Q zpWGwZTj_L{V7NZ#zrIOE$jI}c6bK>mb|E5wNpKtnPu^q2Fez7kVJtyw%Yfg)`Xuy^ zs^7YL`T>D6#MEVt!MV{;D-lXSjPA}T5On}bXOk#cd{w; zICpq*y30@^76>v~s4AbXPr-NqI|2eh;gu^_zSRHs*J`hTao;sVWV4TlftY`8u7p61 z#ReBakE_w1%ox05ROkOc88KXk_A%)31HmA*2BXVbdcg6HT!la|we;1g&OM0gi-|SGcza6$_t3<>!EmX4Ew6nxCYjl=54D1O#XSmlLe*WxPHUGEo-|dp#ynZdv`U-$dUhN##h7GQ& z(Ct))AWg>#`AhCr2-kNy60op2XJoAk{qW00zC$){^uzIc$A37c|M7btryqzM5|h4s zT?Ri1V@q3m`nc3(J)w0gb%TteC;@BUHg6Cr@5C_d10`wIvDEz2@P^?Dndb4~cb&vd zHKuBF_Q<4hS{J{nTVDR4BOqBQZ%en^l^+@z9jy2Zl)xwa#1|hI_OVD;1D8o%-w$}2 z%>-2q4U+snJ#Pvx(AS|d#vf)Y;8?lbP-WL$Y#8ZCli;_;;ol!?;W>OQth4ledDk*o zsE`sAByr{NO;WK@t&>dBA6^ik}D!c4wFkY4}`LlY9kJA~rn&fG9tLn24nTI(c~lO9`Hj zahN3Y6dWzt9MZyAGGsxh6a{C5*)m=9w$z_0L3SWlHwGCT&60rU+=fQ+#_t)uFQ6I8 zObQG>p|99neN^@~CeJpKSd>Z7BiSUDCqR70t9beeWJs9u!hZGU;ybbUzNkMlChVJy zUKdo`l_bb}$YKGm3Ltv|2uVig-WL5JRLf8i@@GKRs|M4D@K}*ISjR_5WeVbe=o!pC zB=P(NWd|#UV7t0k5-f5;#E6>&)LsU#Q_GVyOS@Kos0mLW3evZr`QSgYz26x>7P4zA zToO{&LH#L-Oaf3r_-O45Y~Ttb&JHI#uT=Vg4d?a{`TO8o;U(t^Kcbxwg#v9hz~)Qb z7+tV+$<7@hSbnP?@jxQsYxb`BB>@sxnQ4H}5NI@5^qpW zj}M&vtqC&7eStYY00A+WfouVYfuW2WkRFqX3n=b~(1hiU7PX>T8E*7bNpk&p5^Hev zAr->fSs%J(M8G!xY=PjA7%f&AzS4H|h`_>yk?1^lJHKK==mQ9so8$qIVJEoaGjJ8K z4pPlsz=p>?gRkQ|d@N*+ftg5? zbw_JIfIapZ`h?R=N*~;}RNfU}{-HWyGa)`RLRLt1DnZLfKB1(iW|BMy9Q(Fzo6RfD zIreEm_>mxw4W3M)p`*}~i+Jn9Vxp1P6vx~`1g(v9KQw1gE?mC+sWFlfrJsKMK&*p4 zTL_-chTU|ph=M8#0OxuOxsUnmB;fD!hXH3541Sz;NDWb1{vnyIZ2+{fj(eG##*}|9N?|KoIetka#w|9%uKqRJcVS$%t zRrr=UTMY~hVvbJ?byQYXHu@R{=)l{8uChkwV_g3feg((32Wg1a7|Q(MBgCtiU|9Qx zPd9YtJdL0|5BIp?QOdnqg*jp4MXUFOgjLK`oQA~{Huf;1Q`mulpFV@!5RBXi?Fdzr z`??p(AKRQkG?)@d%tnv#fW5w;z%qK=iQ6j;3%{2}xuAyY%5d*3k01a@Kufo^&t|fW z7UY;mijzStL0)kZNbYIFT`<7p7O3jME@fs z=>RUwB8U$yM=-k~flwggT=dqgQf9GKH14luvWYV-NF||15-^3dt5O9{Rk2aQ5-O2Zn6$R<#ng|wCX_^fwIHw5vTN# zBA__VQ+I`|LnPD>uZsp7eU}K7Vdixs0%iqhG$9rZYk*LWLY#iEe&w}5yo?=a&fkM` zgsqOHLpv}=9{<@J$F3(y^8l7a#>?^4Xwyr_II*-YX?&KK>=vo)Ni%y08!DDi&A5>gp6lI}EFU zElo6i0m>(KZ-yKBZxz|6)IM+@`m%Tyw^U|xNhzP)5gnMl&`z3WGeQa>T)@Dc9-G7| zwzV%zkmR>JXxH06t*DEpDG!J=cfkhLD3_CyljcrxLWUmSe04jw!(b3{lSq*Yc|2(& zJUZXq#>!Zv1&^K3-NxG9b@vzxdY}TqYd3u9d;ELm&m=GOJc6hbj#8aO9UjzQ3bKwW?7mycGe!7MwN%e@_E{SkL+bN8S-8jQICkG2o$X z_!*&E1nNLSld!!EzomOi@d+dxQhNHR=+Pc*IF)9v$ek3b4t*fJXt#CA{vb6P^mY&A zFCnCmybb8@USO#@XD3XeIZ-k(Lpd`H1~mzy1XjY*t1VE9r`Xj52^Zra6Cc-TkB8D* zyrGkx5ft?LB;&p~AqV zAGbp#v~~Dp+fRgG#K8ovymz|O-~>T{70npX5Dn>RvG)d9%X)1N0vOGP{&rD?k5eqb z!0FScmm$%wff5*1U<)|`Vn0k!O7ARasqkr2*`&$FbB}|xC-41x+42Ou0w|3UdwCsB zqnncZTIEhY52Q8eP6PkT3lR*4$HY)PlO{mrSMLqdpy1BMK=3e(n{*>&HIDyB%!S2* zk))AD;_cB+oxnH+sV)!>^x@_UuzqwPW2}`FitjtqAga+pyGmMI{A@C8!LP!WM8O4l zm6Ku^Wd%rd;GGWCZ{X!HOcORcpC@Cx~TPk<9ck|Mg<(w;oqz@He9V8H+!QJVP%>Cr%>b;}aQww)vdL$bNw5KfOzFc1>>^EZl@-cgH zy4%1uFql1%j)Gv-pz<+@E!8b)WGFlXb)A=6>Qz~IUBgwf;Nz`yJcwIvzlM2!VN5Uy zqc=>A#|uuEVW3oK*p=LYqHARaP@dhwkzG|ezv5$O?#J2(jblD$C}5d&R}K{g9q{Sy z>U#1SG#KjWa1-g&qF2JhnK_W{;7qWS=&k~5pB1hyPG{GCe--_ zDKhD_voo(f%_gZ^vOme3!db~Nszbr`6uJ`Gs-MeyEDOvsBVbx7vL&}SxB;(8A2q5^ z$5slyWDnAMQYBLxqh)<>rD+NXgZ$in55ScGH`ukLv#E)RCX_2T)mYs5>zz|RhQgq} z-{CgVa7@ywSpgoGFKoVq}q7bY~l&+8e$+*rF5XLC803BnRI*K_X-f z$wnzs5?R;+=lE6wlZd9Y%o~LQDk~l|xjq7Zf;)4oTRwWA616{X2R4E4`c*^YbUS_o zHIZnJtQjHNVX~lurNy6Ab2o5j-pGP5z%b+JE?IiPKX*YZk10q^@~2i%jvi=52Z6u8 zJv~BxVNMb6@sA_zF5>__ez-_+55n%YV~z&!%}>I(?$xUsqYn5D)T3=+dDqOcYOhQ% za?io6T$1#u#<+h68#k&O-5Nw0&%=%X^hA!S{c`?ziv zq3J!BD|dGW9DS|%zfo!qUb@C}Y!rIJVXkZy3|&xC8t)H>2RcitYTy1KgN3ImR~CvL z>`{^i?MX)E%q1#rW>l5ESGsX#bt;6S(ljH414lDFl=i7b)1Up;1^uh0KaR~qkV)R4 zrT}PA-?gsx^osEJ6B0wxcGR*GzdfiVfj2rW=paql5x4;~oIriV_=AGl6sLE&$jK8Y zc8z87@bbohd}Mk5vfQr44_Q7{Kj1cnSa9N?xyr$bv+iB!yaUSgx}RUkid;p=M?hhk zl93IX7rC9mQCTP#dUN=Wl7l>GxLvPY(n*@WrFenrMw$)c`gt~d&|SxOLF)VSTxVVd zti|2GU7C_?x-|r(D>XK0k5ERDq;mJGr5u4Z2=6|!=am!{%SJ{QkIn2HbXYIa{Zs#V z{{Tqf@+an-!s`_v3B&W>;WsimXa7lpc}||Z#UwHr<)~5aF7ZW^I61lf{2&^RI+OGA z_EwQGS0~6)i0_2_Od96;YUEH6^xdz}>zuF$*H5H0tQ+m*bidb%0&??%N&8v&#AT|1 zzOb3}DInp~@Rz~!leQ>xwm~7b6MpI57MI}pfWnF1HZvL-jNXSgY?M2DIrP0@>4q># zo-$fJ4S)boPpD z&TEy>WhFcH9AxPMXS!jQU50`nTxS@9o;JwaF$8z5bSYe6X@CkUI)CPulDm7U0xtSF z8HF88qiIMnqH;nT*lr999k{%6(tfw@x(7CZbdd#!pC`c-Y3e~`n?S(!k(yNq%3|%| zlX)}=t%^-gC>k@FlqTQ3W-P7GrMmWYGw3iidh#$5mOi{4fe*kMdC@4Bbnb($CR)F` zrS#h6Dv;7|aBV2TJfO9M3k0mS&fg@+M^CT)a~&QxEX@K9>i7c9asoFc71OwEHBmDip_$J?9PMRQH%@ELADw`CZs9Xn85CT3m-zY4K1s986^D$X#Q+vYvETQ&K8GK zG6AMX%oZ_}p7XF;;j^_5yxoPKodT3CfdiRXfZpvnqMs?#1ghPR`K#sSEA{SgoYZ(| zKcJ3S%cC{V4mbg!T2QnDK3UYjcAFt>Vmna?OC|?-*S~2X09@^-ky_~H@jrYY58)Lk zRdw5U#RdaQng>mtnRi`&cuCh{RzL=hQPg<*j$$650P|<&_NPwmkb1!W z8TiS_4(O24SS0rpW7qc}H3Y$LTE%Q2e1~ddmEXCmTt^TQ=ASEve2ippBuxo%qP&uMU752Ee!GU?I2Vw;{w@^6AMj z@}T$%Q(!&U7wXX0m3OMB8Z8`it;VcAGNja1Kh1o6$PoDQS#I%{x6csUDzagx)c24w z4BK)2n#~V^|Fvt^_UCe#6hTPnvm^)IZ!d5FG2jAJ50gPPA^^=D>772EhF5mfY%v*z z{iNB;AZ6EQfU!@sxYMj998U}@G8HMkB#?oCUOV3wqzrhlJ)2oP^(`4!V!oZpqH$iR zO_rZJb&4kQVepgV<6{79z;^Uc(wYtO?weIL10;crZ zmtHVUI&AK+L6s4`U|>T#u}HuDf|zGg>hXcle1t2STB9pWQ1N;=BUn%nwBG?<8s{+r zEyAjP;Y71Fkx!mH`H;b#8PSlj1JB$9-H9QB8x4`;Jd#MYjHJcwPfK_kk#NHZB=E$w zu}B&-72lSR0W4^lzR1&$CyBQRt>Ko^P(V1X2t!9{`$^9YgEJ97OX znp`7GXD6ptJZN1I15916No-lDA!wErJ77OR37zEJXYGnT5!gndi$G27DXC#kC-9YVqJVxzYa~k#=MKklWw`LO0|kq`!&DW=Td4^ z(Gan};%1t(eTJp!dyLYOKhQMqumvl%HAgRUm}r9Sq~}i0(d*<6VWa{ANJw_QYI}%O zsNc_Ef4Ki>VDzI577)u>&gf>b9%PW~>ld75T)k+NjBJZ5VJLCnIq=xD%UfotyEd*m5{kB$J46WcNG zuC2rM3&TRf-V?_5jtmr|XljFb&9>tq5Fo))!cy_-SS=4G;7~rZ*yNcGgrQ181>HIc z;7JM>QK|AS<#6e_FyXGL)YFk45)LJOD9FVG>8w0cLt{afw=biJw1%jLSPas1liL{N z)D-yO-6(+ZW4oX#`kcO`6w7LhFEmtd zd;5o(VBZiUS`JyHu`nVn?66kK)Cv1kfK#rk#xH-eUe3+^^XfAxS@B$`RTCpB_eznP z9MeT>4$jyw0U`kp;AXdcxwW;VI&Cbj!`VVziCRWC0a#`-YNLEK&ta{{KULT zD>?2)hrU3j{Ln*=gA0%qaT*DTQ22<<3cnhd6vCzv51QIl?dp?8=S_1CK&&yF#il1-uis(ShjQ5lK^w>S2G^NA z4X`#Ed~1KUun^C^yJYzWSSyvg<9l+_aDcc=aNfX9AZBi!l0SD z0z_~Bh|2KK(9jLN9l!^OP_Rd@eSnQIy{ln|7pbXnsRp?M#>w%%k>(!;?4r0n&Li$m z_@T4$ULq=G_2Ygea`W&6)@%Ex#UImO{zw=<6aHlAS46xgh|uW@-yLV1z|3_4GDb8r zT4MtUWNGXR^0<=)_Yv_BKD$*4IrnSec;x)7N0VY}=eHIQKKxG5!1gm%vB&O`3`iC=xcl~&|x>bbcSmR$#G2AxObH46HaX?ie!Q$)IE#~0hnCU&gRSist zwB2WUhU)(PObk!i1L#lq>LoZe?R58Y`uop*TSoSq*+WA^R=vs}KYpA#d-g3(4ObTz zZm!66Bp&K|?EY&88o$k$2@VY110O&6GyUSwYD-S@yfrgdf8$(ES^(Bi#Dlu?!i5W- zJ%U#Hm@eu<*W#r{w$(DiT1S)Ed?NVPuD-r(G}=OQ79zaYpP!?giPns1Czf72ArsNt z9=cP1cOj11_Xev?5?_K|w7y-QnaLouN*cqP4Ap}NS7N{}^Pcea`FKWeP+NDkm5WtY zRV_oTZ`E=6VQgaJmEr1e{CNGBFBgH0Edg?G5%l`?Mxdme0F&lTjJkE})-kw8w7k?h z7bev6;EsyB0Q?H}>D;U}M%8;cJ$*f+9`N-WNMVI|o{a*9GDeF>?6%#FmvH(<&6xEC zr3mubE7e`Qg3=_IOipw!i@#2vJ$0%oD6h4WoiJEEkBf>YfY8bh8n!}yX;=^(12S@P zbLTBsFi~x8RtIq0L701G&C_A*gWU}Gn!`GQim-60|Ni|dnwQLX@8&{T@(29BlJ^){ zT3WW(zT`vto}i<6&d1Mh&iSylA!h}D<}t%~8LvO)Y>{mUNHb-p};@gF^z0? zF!g>UoPF&+rOX2Bd-KS1mmwMF{`P5%)15B| zvYdFdwz61i$Rtp@%wlA}JOI9DpN&oI!FaD2m8#Ae3DJqrWu`?j!9-P!pW^W}@qCkp7aR@Z9h6&D)~E=f&H z-01AAaOB7luHqhvsT06&UPXD5!4RB1Rz*9K3N#ErqNmJ67V$rFqDpESm+uxzL28R%?t^7lv{%LaZ!i+PC7>2k69x;DOturag_wu>|hN>cgM>#wO-^44T?3#*&MB-*?b76bHt?VLy zK1_W&bJZ%bUv9TPPZiOX@zb&bGWZ(J!k^HyDYst>4mLe#yn4lo72+_N)|N6@Af%-6 zYh>gY4k!Sf%!-QJYP!zhAc2-ITc-Bi>;h1=0D1f6H-0qp234i4f5X8I=0uSfu4LtCt#-QH*a2mI)IU#5xI5jd&-k1 zhz(PkCU07+D-&6SdRJFR$0IXSb?P~t`zOz8-lbg^y~>7pZ)OSYUn;5tr&#zg_3Q}j zz)bvW06||SZ{lbjqY~NwR;)VCmtmA{7V;d<+c`V|v}{-KT27xivtrGfq>V#*T&d6COws$dOr42c1^WD23ZFAmz;`htI1OtwS$W;28wq;Lp0 z7(iIEh5|inIU&@_0$?bGRzK-T)qKb?*%h;9&H583(@%MLDhxWa@rKQAk z7K6Q9be9c9&HAHd@A&e{%I2fEK%n8vpLx{k;K9idT(s9bUyO)(O>x@ZKuQ~LF8Xt7 ztTs#2$rD-;la40$d{@^oSad`db{@HxcB+uVKceZCIT8jJa5n&Z_yStwU_ciI`@YG7 zva(M%4$U{*0t-gIC!+x&nApHdAH~+zR?Y3( zH>311KXmB9j~tjWTpi2`jf@lmN@ehBi;>YH%wey5^r@-}31D;lk9p;5u$*3V+qR3y zyK(XH<-Efwy{9yHqk$D(aKUxxM`Y#GNTd<#q8;3e7q4>KSce`XWWcj!8)DA+!NJoA zte$VojXogPOU&L~htB_EMI-Dg?>~7mHzz0O7}T(0o#^1~MXw<7#;*0@);)18AmCxC zV&4z9=TL&FHV&M?nx@J(+D4l(D*ty{MyyN%`V>4}ezH$Qsx@yDNU``rvv zUDW-z_3yc+@Znd-E@NX)$R3ujUOfi^k+&#ULJV_9tM|TAr#EkD8UAt8%!Gsl;DHPs z#p5>=GHBnMWWSpP&-dqHbF}BS21Si+G5nfpl6rYwy8nGiJO2*UrH}1#2o4h%F#ATAn3XRv*Z+D@kT>o+DBZbMR#s5EYy?znZfiRWmlJZ@$>{E) z+b!ltU$os`E{*00u+1cf6g21~;KTe)1sRZSKsvINu@AS=7%q}IPWulW0BAAe^ytX? z`SanM-rwJUxbn#|NLwI@SQ39LFeYkb#6^Ac<_6Txnp#>Laf`&o#k0O0f@G)^5d(L? z;H0jX0TAHXz#dmTLiY!=@UZz?dz%4kC_;U~!9dxoxpiyo=LuH2c6O4KEIMnI6rFpg zB6>F11nuzy4sfRF>(_}4NS3Z(G2FxCI?w_MHWiZgH3;44a@FxxIT(=i9kaJr!;65G z&0N5By6JdwP3CiSmoE=nmj!|#`@s?Tl<|Z-+=y541lrjUdHV_2%AobWy0S8}ynG=j zF%D8?KA1n>>ZDl>{C{9)g3r|HgxnK2W%Pa@%P6*j>YBr<3t!(V2EQC14yG9_WX-fX(RJWFS9k z2S{-MQq&2QLzDr;`WkllIO~2uU1uTaEEGMZ@!_V!1MgRDD0RJIq%0^R!iw)jqcwHT zoLszH?zuwFZWha zSDCVQ=o+GOxqu{j(`q70&T}}4nR$8Bz~g%Q`ep;|>h5jTmo9?1>m6!zI#~pE?m4hV zXF>5Dg4V089>*d%$^k&7l$fPY|jS7;j0(}z4+40>*2wzT3YPh zXRi7a!x9S%3+quM06JK{cCF#jH>J@2^P<7X#?xRh(D3D;=*JsgweDqnyaZO(U%)|Y zL>$67nF5c6d$$wk&(kf+8V}97La1%C(6(ov078lt#jCl&ZI}Jh)7Pi46%Z8E_RLdV zeOD?FAZULkFrwUn%yZZZYJuksW*9Ana0r!jHfiz6j zH2}49k&%&$tT+090hKiW{C%ohH=L5B^{|e{rEI6oTb1Xis~-(Q$@-?CUkA03(k-XI(9dAh}A`UlYY36PB%1uqLUNumR?($chW=p7uWb}#bte3-9y z$Zp5(^~dTNP}#$JM3Ip_c6Ytg4<_jMsN;1A`+q_*mxG3V9uJQf#0$lGy4SG2-`tP#_#!Y z@ZiBptfhdIjcNA@YAM>mAi2SCR$|3=Mu4)bvylGtor^J2&H`SVKoE+|n#)@*>}bKx0DH-IL`9343s=pt4CKmc~T4u?Y*91M02 z^C&s@dU$!U(tEaT-MR@KFsi0Ys9X^jIZ*M9SJcfsOP8*meSg1�Eqc6$`AcJ*d zqf;19o<4npayh#q6h|G9KMx}--IlQrXffa&$r~ukWod~2htd~#elEHVEIr@<@-t2~ zRdsbvh|TL7F#;*=FVPuKUF1Y#3{QT)ibY~wwUX99c2E+5} z)g0U#G^f*)@$O}u4*%pA4UccU7u)Hnh?kY-+$TDJ!GdnWYO1SW;~Am6#IB$f=%#yj z4a6{)E#pAVz3Ipi@v^eAeFqL)1O@a5VCB|#fr%h7J#ktmGJxn#a(8#{PDcLy{OrUG zfDlr$vYVhQtMBNT#mC3zTfn<`@nzW05n`cgFZBdZ3&@WqHw&-}>YHYsfu)J5~9FvI8(%uOH#M;#jp3lo9`FMFPpk?B9y^BUa z0YnlM>9fuWGK7^sBD5TT53vddaw5a~adh+Fw1bWpp^&_vGe1p(qi)*v-+xLyQr1IAL{^ zLr_R)5{Q9J$l0!ig!rJ4eS?F5p?_6O)9Io}NbV`iK6Fn8%8x(6n4KQ%T88&$ZecM2 z!6)tc^R3F~^Z%}ayuY5qv3NDOYvaZV2*g-VHUso=W^L`Bj=H`d`Lq%dHm}Q&7pM-w zi=^?o*#o@%{5~iF7!1f7gnAZTzIgFZ3`iZQo~6;D2c)O`yGC@8^T+)JSHXtP1MM$E z6+P5Ed>gKr&CL>cz0T$4p2`@6@-zXYTtINJ&C3_+>b;ZaqI_uaeFvv%y}%Z@60gKR8ZFMI%1jjIlyS=_~$D>*gCAi?7cXE+^LAq$G=(PoFliZo$hZB z%zpvGK>YhURETPGIVm)r!^YvyFScoGUjVPAjx=Tz8;5l88voa%O`A5Q4*?X`N8Gw4 zIQ85+D3k1ef^HQWO0q833Qvi4tU0lJ$BDcYKD63;foS8l9}k@}lUwKeK!11lydG&O zskv-6yD91$Y_2vR;Nnz!l8`WOK*Y7CEIa)Zxbym!7U5IwVq#+E$ep4YC5G{#ZNpG4 z3k{h;LS;7eFs$fgl)9N&NEe7Stw+ngSdAalwqf~}@FmlyO`C@^&6Sd_+I@5uw2Xfu z+B(NRUy0Zp>EEekwR+EsxymrGqZ)m>-!|O&i-dC(lA-kH+_Ew~fGMbh)1@FFyLh(~ zmB3Umcz6H+s9%F^r^YPZsj8}K8nBePQ0iX*(|6_5D(wKlysr?8e0aE<%rW|wQtSco3|EDGe?vF(O;V2qyT0+7N5 z%2vpb-|ZTO=s*pTvm0WfD@vM8UVeUn`(+96yZF`kBQn-IWKGOh?*0A^Z^t;pO$j!r zlKpnk+Rqeb3nl$p;)F|#FS~?S(TM+Oz0b+bwFH|qeQ(8yn!fmZ_ZaZ~^tgL>DZ)K#gCcqC1hm9xRU26MS+x+#bCDvEq=4$}{do8z^0n%;W{Q2jgT6pWad+CXaRMd`P zNZ*xd_AB;exW!A=0FIEt&L~2Qc)vgzcNM)+Bd~UZ(~p39&ji`WNf%1V~yw`4XdZ2YiqzvQ(5I*w-ib-D#@`z$0)tW@zklsx3SS#)NC^< z2U+{KW_W0NhcQlobx@NS_wZV}dV4Rte5nN76M_#DK&}&2yNkY{OtZ-J;IN*RAqU;P zwCZ~dhxB%QT8w&~kdecOg`xjsFoe;+p_=kTne^~P&xJp$)jvdtt5uCr|Z;+InjdGjK*PN1)0!&B^!4FaY93X@c zMW^{dBRCm&&is%rWCE32*n9@M1>))0wv^d%n#`@NyfBqqk62D`469AApb@(nJ5n7M z+bFm-xb8ZB76O-`w6yN$gVxs5!SaA2pGVy!Fn&PyfQx)!aI@38rq33V(@HE{PqW*< zpA(+|4AZ9>85sfH$;Nw|D*2-X5>o{fWyK&^Y zE(qK&Mno`yooVs2laLnJ_c;!j?Ap~985)FIBqqY8E#wAT4$Y6Cr%k>OV7u0qz!#Ti z@fm#|`4Cy;Fo00vSy7Bz5r$VtNf5IX$H0|ggP8Z6kNk?K=i-HB0kuvxTQQVmvf}`T zJ!V2aonZSC1tmN^r5syyfW*#4fY^drp?sy8X_D1}K0Y&hJL{GL7iy3L+B1_|ih(qw zfmRx4)(5Y1nTpC%vSO|lXxVha`smpPn{M(g{Di^CS$OjBrzgM(g%9;YxaisD=f%Rm z4=+NN1=GQra2j(bEj7>A5$+Gn0P_wE^o<+A9WeF*V?bf3|Lv67_U*I5qIo1Gt>Nb7 zMI@UDb_aSXReYKlx;%?doRM935w?BxNQBcbYu}Z&oJ>APkAedNGIMjMpz`;KjuxR$ z9`}-3NZ74-U}*YvB2-*_c(Cf|)(TRjxNHwe0)+O~Aen|6@&TZ{ii?Bb09!p?u8W#)o=fC~h{typH*+#7*+IMv4Mux15F{Kwy&JQk6v*U0yb%EymmS z%<&b9zFQmG8}S00X8XoHED8jB^!h%Bf2pC;+HM$Jc{(AP8pgYV;}`i(O04NG-F0=Z zH$9U9@tfZ2xOQf=TuBY6Qn0fZWUR1Padi@{i$t(~S{{ZA0Y%^IW%(Rafjca~i0uUk zX|?2?enecmefxpYt7q1SC}Xo;p|DN3=I?K~5{qcQHqE{m*SX9g*7VGCpl+;C^x$(* zO)k8(V%@r(=rR_CDmsOwKC{^vyr~&b%5=mhi#;e<=h8V^XY1nf0zk&

2y6ELgEF zzFj9#$Q=sq>pxA`u3o((K;c9TGd`?GU!e}&?hI6E2g)KNBBtVIi0Dt&uq{UB5^DVr z8S6a#oZw8}HJavpM%PtNTzW0J4ZrTz=(fh@=BQ(y&f7X_p6|5Dn1=GQD4W{yQob-n z4&4O{7W@g2N2})!l*j?EFd1w8WrnITT%^lf+>zf|&Q53Vwh7E#$f<^W6aIQVTWbu# zXd0G-?F^~|24f(QS_U&K38y(4U&L9K`}PTPPMIQB)Pv!Sn>RPrap>+|GJY(YK%%gJ zxIxh=S`h0g`R&m(+ezv@Jv|qp=J0mHT$uLv^IVCzfRn)O+TR5zk>1)gNz#8kv?ZV` z`I4tio5l`4PXh8X#qh8&PVfP&SI=g9T8D=Q4P$x=av>|{0W6xr@y3k`iF zrLsF&7TgoNa{$1Ae>3!6x(Dq}xK38b2mN!n&xSP@#fk|MnNiT=<+f2h zm!Ge7dSo!6jIgJTm?p_`1t+p%-R@S^KTVIm6auIrTDVR{W$$=5hEdAf$KTIds%)m+ zh=H13Gjv1KN!5OHQw4{vZMptjYdckg{t^%bt=(lGXx?FH)ykFD5=W7W9gfW=Fk(Cv zA87t23Ym$0!*Oi(*)^1c3iykZ8IqkUDzEH5b>-8tP;6H%IlwwvT^34cj zwU*cj1m-a9b-KVh+iIrd93B}?S}kO93AcMT(4Iwgn`&K-Bs!!|=HvuFxYH^Wu$x4C zd%MA_)JH9C@;9U6JKrMgWPgEO+dlAsqIIV8T+4Uw#THsG-0=y*0t%h5<2PE*9|fng zrnIFBj+$S78UtrR;@=sRgZ5UYDfmT?6LPZs!(6l}=72r4dVm-U(Wq^rOLbVhv$ZJb zL9>*E1YMiE2u1ek$ejx%H-Tr%`T{}k^shU4z${I|nZEcKtO~1n46)caIX+efqUtTD zPs`)N`2cib728^|SS%VwE=5CQ1}Mai7oRpn>+>%!Tf25`HXe_Ea}{)7+VByu;@ur% z*Ug0G5*UT8yhi`)qy|G0ykL@vWe&mE}AUOUjzK0;Wz+S2nWpZ;XB?19w-{3zvbhZ z&XjXG=so%^f>(p+`B^QN7kI!)*x~YE?kT%wK?fsBYlLbn$V1 zRJSPMvik5IcE%4p-uYMn!Uh797@7E9Dd*n9-F?WcdE54%__0=2_6-__J~{V0;<)lo zN!?3CP}kwtW%t320Q}RJpMj4`o_*VjvNo+9^RoT?1^~?7kO-2I-KQJ5KxOw{T-1HXcje}dUi|JEcm)4mTk+e2KNo13jWR`50H_0IKNxqo6D zF3!T@9d+nyqCUD*wkJfDY0BrtbZP2EI5>X!lod8lq*vcwmKw*OtzZSFJjHr&Rq0f|3FXQV~|BX#+T zpOS5G<`n7?fkfY1pOmo==dMPL2=zQKO?9wn_&*B?z@gFz|9 zLV`+nsgl^^clGL`+q-w~F3*38-bzYN4*HR^yIRFIn)by_t?G&LInsI0d-8!vYU3)R zM|P{rW7G|g_RhzrzXgSSc4sDThRuFegIm|H&jSn2%!RkmWQbAds%CpQL%X8?$ zzDy0ZN|^@<&db~UWAl=qLY8ek(0c*`sjaF8c%iLm^1MN1heNQ~aCimA>K5R(i>?`- zrqiE#USUoJ2!X#?5Rr%@#>yHW(o5sN>%^cxVfTIb@PT*fT!|uR9{`o1Vah@e-$gC) z<6nTpA(CqijY!<|z&gya{(i5sQ_0;|h^*s}yY+{{`t?>J55itJnw)aI9{(P&u08on z^}MS{7yq@V&26WTudn%m147dde)E1*eZp}6(g`{PEA63=%K8!)Cyquxmj8$Db#T8} z7o1o0{Mozx+{Yosk=D`oe^oM!Y0L#KFhJ(sa8g`RQ86o}3FZT*hkHfvhDD>w4L9hA z_GjWCAMH@HDvy8ZFL6}k(VfVivIF$dD48X2?s#olu!>OAoxU0N6u+8OH-0Y5c z#q#v;KMcOsO+5NK*#WQMz@@&Amv96Oy{5Q<^qj}bI~`6E_MB`a;Wt@XoJfJT=|9AO zeNyW`HEG_w>(9V2AsfF$d7&eAr~UI5Mt))T&d}f&iAon@R-jdt77PjwzVhLNfqlys zQ&Uq`HAEzN{4Q-+ZgDg2w(@sWI7FA_yJ+22adNbuec}YaV`T5UJGju+OZGa&nAVgp zGhL;kl5xCf-h+~%#u^fJ4uyxP5L!^mQqmZaj!G&^i=`pH&#TYO})^fDbwMf`np(bJ7y{ch?@#lCVyS`<7+4U>9eCh!%JR`Tjb%P19Xai zqpI{9_1vqgVQqQZ;X9X2a~)biRsB-NZ`$q0L3r`btmjoAl`~GOj+d1!@bg$YFD5MF zLDXsf2Ia1EJGO6c0x7MGUd?lfb?j1Qs6l%N;u&>9mdt31SI}} zwl)d z=Z+kNpKOkLby3KVLoqhZexqscbmh}!U^1wp!Q|C%&xVvbtx$5?QFrqj0Uvc~>;|CR znO+!~p7!bcI`%L0X8Pbl;~fdP(^Tz6@ItiZlTL9xa)XPV_%PgfXKjm&pM_rrEluy) z6`VOhug6sd9AI_v)9b6QOi#X5oByLWEY7T#&D_O{drotHZSoZ^Ij7r+>iM2ZT}OIO z+nLc0XMR6r6$qF=)oR#+9EKZw;F8I9fRcqbmV)MmLdGT1YPsjd#aT>?Q5MHM@Uj~}UJq_C#^PxC z>ooXupYm~zMFFLO<+05aG*O1ra+QOOzCL4=&&YGLPSQ0_?y=P6NHV9!-+%uddJo0c z#>2xSzDkxz1O%#Pf!MqtOri>#c7I)5Y~neL2{z|^1h?cd3w2&XmcEVS=F6UtWK$tw zpZtK-n)y`OBb?JuM^|@ubv=e_nKLd&Z9f!Dueo@wtSvQU;6F8l`=WQvon`IMPz}@g z4K>iq`^UO3SkMqAeVL1La)I=g`DS~XSaPqzlyhq=MOr}M@w_yNzTwojh+u{GdIJ zGf}d-nDx``#Y?#6WH^dQr}t5!dc$`;QdXqv=;#=%DqvuaX_G{)n_?x%|MA`V+%>=F z=5M$T#)!S#HG1~7mGIRbs&wMMdx0KE6j`?Nb^)u%8UH!b4;$!R9rlVYDxbT6!S+AH< z#}!J}rm_A;e6)4}zpHoe+I1rL#Q~bl)&qSb;}W?>HX^xzUegUwx3I`1p~+d@+y~eV z<{+I-|KUY&d!F-zXdDGE8FF|2(xv8CGP&~lZT>Kuio6HLG65&267YR^W1WtyYEH;A zl_GdAZ&6ciEmFftqBk!VgPCd~Hd-=vimoM9EI3Mn_($e%rV)i6JZ1|5)d z3NP{2q%?{ca*V4m&M$dB5TnBQBjfaUHb}Jl^7ZT14O502$irU^NU>X*W!$x^JH?TB z%MfNd6`l5n&@Lb~8NH+ZfTjTZUVF@0+~gK-A_TQaob8!l9Sjklpn3nEFp~bVHMqK3 zTPBQwb5*8;g9r)?KzE#!7XEIS-%2UtL=} zP*Pu!Eq@qKHYvY_1K77eg<@IBs0t9&Qqd6)oNKvRVet^U<0qoR^wI(`B7F3-%R7SgD?oeH~t@2lIWXMQkYyvsX_FBz{eXzH%dIx$b zc=gKYN<$BrMDgDLH0E6B(W4!3ain~tg>`6`YTSg>P7C}oJ2Ms!bQ_b6HsX}T1~`g<0JnL1DVMOh{K*APN&d!Y z*jwN;g3FVom|; zhgf;hFNk@}r&BQWuQ)-z6djl?mK$ih6P!=yFJB%_|Gn$r!7X&~uh4Jj71y?bIXP| z)JRuAxW}{0Ju!j_;SatMYW!N~sXoV)k0^qv%N|BxE4ax8Dc5e@prgV4pfA4yq&9~a zptM3~zG6+U{P0o>r+Z<8E$ZpdNw4=iwqr4Ds^*)kpm++8Y(>RQ6v=|(;s!`6qE*nT z^v25pT7ne}=dIz)OQ~7~Ew`=AzYP@DK7~58(Q`Z^|8C;~ z7Xd-%jw}O7)n~ota5()EI`2pX5Q>282z-MD1hVJV3t~5zFE?`PC_2lVtp6}%4`b4F z*?f}b!Pt&@w;A(%grqjqsR!RCpImwQqKKM>uk!X?ibAv+iN^+~P($87GBW0qdTjRH z!P|sLP21QqC*qh&Cs4YkW$*6NRGy#@@%Hh-%45t&%oEv)<;ZUBBt>dfRy2O?B|^ed z=C^UQZ%rOa1JQ`xGK!SV!+#VI0HNE`YgunMbV(Q(3|cvAF;5q(iWqk4(;R+XDzXEa zf#9oG2et3efh2A)2w03qbev=G0}4Jqc%^CNAwPC#eEM(WoOm{=4Kw+yy%{h{fJ>*EdIer6olyA3~Ta+A;fPjAD^#HC3H^95D z=xp>f|4?Z3QJad-5rtZJ$V(N_N6HG#$jnT&q?DffR2AI=MPVa$rF~Cl1*~1@e=*m2Qt~NBVFC-5^`w@ z8-*b^P@9flKo;usRao!z)g-=Csui0c*7nsLV@n8~HFuc*I-J8v@ZBda>LF%09&DvJ zL6H?>d+JBnw(c2yafo?v7iQMCy&c9V(JsTC4i?o)vJbAq`OrF0_3if%Q@>;!wwmOb zC}K6rmSeHZ`6{4|Yo5!or?Q;wa>N$m$YsqK?coMwbT+ez)h@slKZ3Q8L_f5*IA-f_ zqwzA(F^>R^KVG6wekbzD=aJ5@&nLuQ5h))j+|sm+Z}{l6VnR7rH06aYQ~1Ob(CW05 zp-%Qg5g3z5_kjJdGt35r;p}cXQ1w8m^)N+}e8L zMd`FQ7^&{n#>=3Qq}7=j8O=M|j-p~4J&tynk@o<9(4X9|l~fHs+#)+@(2aqF%$+$(oM0aq%a^exVmvwh-VE3fVHs=8h6@)eHw?uJZ-BTBN;nCbV;6R4g|BZ4 z(+DKxKs;rnr${Y*P*Wc9QOC#dfj7kClthnUS^DVs*5x`S@2pGG;ng=&N?n1I9w3UW zOUR_j?Dv=tAEkw5x%Gxa6`vH9`9O7Pw@mQ6ZIGTg+?GKvStPrc+F}xs2+X#2Ruh-# zub>XD^!G1-fTJ<7CEE7oP)+KiyxX^LzrfHwV$Tg2ZD68}sgeK)$F)}QpQUuQvonmrOix-hoOwf?oN4I%b6a+1N>DGe=5f-z(rd9K$qAo*dQ?cFsX}n#F zEo9nQ>*Nm*_KZ7rns>*(8LsxYbw7rJqIY6-M<%llXY4z2q`h?055k7E3NF1&6`}S$WbEuYy{J~4d)UV{DE8u51c*F3GpSSB zg5`Luq}frt1VdU$m9(5R;r?@m4W-sREJ+oOimKL^2kxOpp{b& z?d{N_Y+6SOD!1Px@e5MC7C;Ka&(3jm-GIn=l19BPYJ?sX(jKfC0n}l9ip!R@TC)ATcvW*tQ)qH9|fLT!-q1 z?GIo*(XC%-Y_~QSiC4qlD0~!?3^rl(mE@&qzBrd~gbZ4*Y51vmrqXQOWf6gIGW*H# zodc$S%teDqQnYE)25W*XMhSUf6;@k=gBuc+QYfTVX4*@qZkY}S+ORJ5S;r{ha;logBIJq*MaPJMGDNuX^Bt-aYQr z3}N8$%V;SW*VK>5TCknDeNKeDW=7p>4gn7(^sY&6iTHsf>s;ZKUHy&u>|j5<0hOte*g6riNCs%&fwR8MJU>lO~-|I5+Rds)l$$|je+g2OQ@(0(=Bbe^^h{! z{mA_dsu=bgCdqa-6*@ijf-sQnb7fIjm?;X)kCvIHbaoEQI$qWR}j&i(RcPna`z zuK3sMckB39C2p6qg-{&!N|I3_pp7nw_^yXo-x74Nxby)QBY2)mhN)Y}>1ioS!8oK8hC_#TL~KryjMmfF47(AZQjhSs zfY)!|ZfDCIVPk@A!24HL^(SxPV`XL0IL7X@#(iCq715X_r44caSnaG&b>f4P{gCQu ztvC*RTA2BMY|cF(;YNH5x{41>u2uT*O_8VT%iL zSp?R;B?b^ncMHCYe|0|fkE2zs=Z&H+Cht@L!Utx+y@~SW**HGG5Gn`#I(saJ z83*G7LDjctqb2^JAiJ(y;u=zI28Ba@pLKp44Oi+RZQ0!U^J(bo0i{YfZ|LEBT#aQ>=6kdSp^BwEju*OQVUHF{OPwfK{Rx|MhEG|Kz{_uKVx* e$8V_}X0oS?snxzVWS|v_v7;uAj2$%HkzcMy5lv zvW3N#r#u&vwbvYbdeG*Y3$x9(tqM2pNx6FrmCv8t!NejvHtu?Oq$m8bU4-3lscWBt z`oy-ys#2G&`TI-FOg1Q&vhDA$>bqCP|Ni%Fv;_qmfB#It5<^4(TYr6hpzKXPF7wxy z31w5m_P;+vh_a&p?~ho+iq-w=6K>sFL7{c|>l0*T=sf=){}{P-b8~ZBS$qOp9_=?% zZYZIZ7QAz;iE<;;RxCzj*3pX>_q@{2z5V{Z%nojDE)kK9 zXHzx>2M5RK=Zcxv##}x3>CrQ%(VsDD@#>}mA-Cc@i;ASb<$G|}CmEEEc2^CLbSeDk z>MH#9js5G_uQ57Vd$hAI`u_YB_U`@lMHhuXt7k2-5W!4VIw)+j$VJ2EOe+Q^rgu>a zAxfH>F?CLt`@SAa($D?;_KNk2En@3FJvvb1c*@>>|F>`7R5K@k)f~5OEvVlglA{o^ zr#*MFg|BYvwgf`VtPJ5NL@=lVtsxMp4a zUNk-0ZTtJ1=;K3IRt*miJC64TmDzC|c2qJ870b)ZV_31Gw(ntPX6AQ=2D=uc=Lpgu`{M}80G%Gg?^N(sXZxOrx^yyai!;Y&34DxCl@}0+fHyw7| zyYtlJTWhwW5|~hmrh_(H z#V+@%{78RNkB_+NlBzZzWarm=#}b;+TO2UA58RerVPzZ+x0i5(U1Q8 zbQJ%5NjrafZgw!g>vhLS=is10uie5-pFrX@=NFAPR&H+l{o9Z2fcaA_>BGZLLhiEz zBJWE|;#T$sIJ{8i)z5SFDfQ>vE+N4YCSqOkE!*zPYg^+LtU^mS?$PC`bHs98i;w5K zv1-fhwl+h2Cgt9P2VUYX6NW!N1oSjz@V(N_Rt_%b)mXM{ncV5qrJbezwiEqYmd%+f z?`+sD-CY?@J}*HhYt`A5*Qp^V{%pP)`#)#sg-By#<3mxqoz7!DSB*>kep!7}d3lO! z#||0mPPG?S&6!-gcc1KAK=64iOrIBh)iZEusO^Z%v143(eDSFvBkH_VsP>eaSyr<5%&!o6KLHA7`GMpKJ}v4Wx1l2rwfyYu_xkV`8dZpEEqpa-WpvyTdFFClsYF^sTP1(+k zR!M3PCaKkN@7U26;jwV_<;#~%?%h=p$1N=_i}~eG%lfe|7Z6A))0d5{P1M;Ke(1^# zRci6Gqc^E8qIAYTD?)qf;zNpzaT?wS@o6Mm)bH_abTDuoUMB0ywv37C4vUq`ME@C= ziwKS3n$wbn*dSS1Sss3VJnzyC{FPGi{PZ~b`}>E=1@d_gPxRLD z;V2o_ztVec*X`Fdnq}2|wDvEn zoL>oRd3HTR`k+SeNlHq(4i(aydxb8}PZ~Jw_WiT1EB-w;s^@%se5TrOa@J17s>N^c zV;7}4f4*DU*UHSyXZhNlh6uu`$ySeyGKPe*ZQHlo4m2G|w`x{5bGdX$$l>=lt-^Tc z(Pc}Qs!p)9Fe1%0WLRuu-?Am(@#BQlR_tRS-Gs7=irSOTU%z5k*FMldo!TU18yp-8z}qA3S)_^7BD}+>V1zBRi0&Jzu=op{cDM={`Tx^Zl(zZ*A=J`fOC3 zuJ-)Jg`?8aG}YDBN^c#k?D4Yk`6-iwt);OlFNbhg9zJ?h)Y8K7{mtd0<-xlzYpbZM zZ}|N!+w0}&h>pgz^B*cJW%czr?%us?XlzWodGqG|X4OjvnzL^D`ud)-GdoHR>gXuG zbIafVIF@o`bkxYo$~Sc1`8ZbrE^MXHj(@aseJ59;N2bxJ(Ru?9E=fs)zLZ;WY!((4cR3~X2P6XGU#4Z1;d5HtjW} zs3+ zvYLZK|J4zFY>u-FZ<5!al=$MR%(f#+O81&1p3C|cp-NLP98)X3%epTxm>q!;U(Ia7 z&9Z!9ZuE4N@qFvzd_+tAaaDahV^romkb^v@#zxAS_3ZbvU8hsaGJk)68!w)f`lGu$ z{?pF=f`Uf|nlduj_zoS~jFtC#{(QUp{J2J9Y8WTN7Fm6R?`Dyb-nw|MAo&cvH`-NC z#E(Wuxa~DAS&<=`b(5L%?9AA&hj`cbXKiDDHoUpa$Ii~)*4gRR({sW1h&Tl&zFDrQ!cjT_?&qzYnICym7gz}H`T$!e$eKrDt;Lj zl5T^5{@o3`wcD`HHO+(ir%tW9!OZEWm3EGIW$!PABS)6V`g0@-$8`7h-o_)+jU!RT zeESz=n_faa$!Oe38A4st_-q~*4B*K7NvPuE$HEfd%?gL}&R209cAH6m-#M5&$$>Xl z*<(=~a_bg@qqB3-moICOoc8Y9cQjn=^5;}j1p>O_HIig?b#+Yz9#G|g@R2M--G#^#up zs*naQKZzhZ36>|LQ=XI*25diGrYno3xFxsw zzmm%1!%oUpF1gO*pPwK1XVgAfH~Qr`C$F0;0*+h4L{06DQr#xCj~^fGO7U292UcL# z($Yd6mOF8x5Gic9N^-FcRrGpV+CE)9y)`%(jpb379QwZQ&KQUj(sXro{hVTaJjzNr zcAas_-6M+&^C{+aRLh3sC4A}$o_FpbrD#6Aw1<U0R$S0XRV1#I5A;LjrvE>{)VhGEaR50eSzT0@F+5y=oZ) z0Ivy8gH%t>&rTli`a)=q|{PE+(n5~M63Qmud6>k-ym|s6<)4p>R z1*xXxHZx;pL2lXXr(gR~~|xw*D>9UirIlYLobWl8XEowD?=NL$Y@et&bJDV>*1*fM6^ z-2pX}&n{DdA|opsJgsYmlF<(ET-yHbrrJcPxQlS6RWqH6O3fubCIW*0ZO?-OqAOOI z9!adnx~({JB5pHdA|pDB7WJ>Zv$C>UCe$M1 z0Q;HHCLEKOzun|sCSO~&|H5Yq>QQNtH&gl8TLBgyg46IFM)Nb{dj$n~$2SFiY-rfT z=|07UQ+t4}9S?K^M=;*lZlB*8xEJ_)DRbHP3@EIi>5d0xp(gD=#KDEcX3B{Aw2==>Kai|QSx2) zA3O*^uY``M(_?X#(;P)Jx-a1T*|XQs=60ZxIL?gmR@c$#}SlM zP&n$~AbkJ+eH(-{`A&ku@xE)=+3!xaxSa0N6tQkmeM@g>rW&ull9p~o0e)(@@q7e1 ziD)uUhL)CN8HU>5)h*0+qCOWPcRdrlbrOa1g@`iB=(b3q3w3cp>paVv1@h-2ZC5Ze zGiMm*xcAk(^e;;Rye)IRXlC}v0&S@>D$=LQ%HXD^rr--3ePpW9V9G~GdhAI=*W=@0 zY;0_H_H3lO-LYfGehs{-HT(A3Hr8GJ(M8~fqr$}*gJ!w%{|}gFZlvn z;ykrHX0ol2Hh0lgVqwbo7k*8~qt-89&U76F4!Dma6L(nT&>@|sHg{r(eCJK0XV;GiJx`HnjRs+j^h znTnm`%joE+;v&Mr>H|*5*Xp84(UXpp>ZH6G`Cr*ybStYJf1{0*f2^UZy2Gk5HLA=G zV7XQz1g~~K&v{(lDxyBuby~T%t?lWhQTR^Lk+l~Nura_gH%Ewk)X?N) zDQaazqkQOdLjkV&4CA>ZV#|e7RKJjgXLy4udVe0OPz?)*xr%G?HG8JA%z1 z+z(DcU?!Re;3OWR5~0?;N<9s} z&V6zIN`iJqoT^1a+04YioyCROB-L(kB?S0)r9GvZpnyHxZ+5>)Vs>J{D=SOn(xprG zkN8AISvg&YmSA(1qw5SWiMvo0AzAuR(0B-ai4j&>el~`0IZu%MmzXo#!@sWHy!j65 zKDfOc)$WxXhcyQ_vOE&8el;;yQBlF-ub1b_dg8!TA9R0gI}Ikuf8=2ZZ02l0b}nSgGJ1KFXWq1yASzU8`X#N!KebavX#PFRsV zhVzTw?1m=)SI4}&(L#B>55vMbfoN(icq>@ECY$)jvvAA^T^vJdeXu9=!B}0UYC> zay?&ERMZkvz!7AEQ-R;b4j`S;6P_ov>hBArn*85)^R+LkNQwi@gB98ME{HEXsWmdHRvpo zSvHHkdLxrE5yI0{%R2E>mIl)TkGGSpf59W07r!Y56Y ztZ}*y+IB3Jy}N12p+korK79BTFGJR|@`*STnAIR-0o~WDy}iA~3{IP=CFyT@u=Awy z>yu9ov)Xl67K4xSL6$iC`RNgARfyaU%}mSCYGJh{N8IF3o!T=l`bcO*)zt5!TvRoS zzgby_c}=wMPw!txK;Z1pqmO$lLBbD|Xq;tDplH%>-OEWQq~;%m2pFtF70X>VPWgT< zqF3Ivw;A8FsTRUOqU}fA{WTkRB8$ke{=-SeC^}E{$7YJ-SAq>+pZ+NBB+?+V1=}pR>62ah@;Bsb=wGc5aFdyN zMpYO>P&JsZ+yZ39C;{sm+48Ote!+pFrUpCtvuu$o+fnc z`0>q_lqeye;C?2mFQx? zvolIcPg8SGg^QW*yF^1n11U6^r8*mT;F{5y?p+OT3TnvrIn1jm3 zTOg&(6+C2RWZHlXx>6D@83A(xj=aO`m_|StdWQ``m0JH=2vM``UuK>j!CriA8B^_q z)7%uu-=3>OLqm+(YIR*bJ-4`y-UxKeK`OVgvx~pJ-YhsGf*IvVY zpvn*6XBA=e?uObgOKRMv-ignAPz%t$s%9!gDWe_K9GKdzoz64bRHkt5+)knq<>VX= za+BE>P<=N8y-A!~CHj;U+fLcU`uhO^R{?o8gV*;C3DNK#(0REsKyK;j)2B1s2o6X) zUuDkDM+gkpVecpE6AnVanvFoAQ9?@d{f+w+BP4Qjg4HfoeELKP0YoQW9+;VIYG1xD z3B?)1e7dZR#x77;q%tl+1Z87&6`P+479p#hHU6@KlUTw>a!C@ z@nC!^t7A_`NI*dR+AJn5EqyN{B5CMhY;0^xiGTq9YYH4_iv2I;9F+1+`!B2}pak*D zR246?tq^Ey2+3&M-n~j@tVf)B0%~b>!nNM|6VA5|#54 z5)y2BK68P1)#z%y4?U#d-8)*~4(j4LLTCJ1`>%Yq7UlT*jT=RfQ2aS1)tg1AR7QLo z#S^_|VP)kibXI6$OA(>ev)MtFwY9ZdX`JG26`$<~mw@xVOiA}u1?X_>sT=_Bbo*>y zi9aW^hDQB}E|p@4#uv!nDYQ~8#CHar6Wzqv3nY>Yt$Ddkx$T+Ot$yH@&^xg~96jdj z{6;<0lrTq$+8JZnx`{wF=HILpA5=-qqA~z)LRDF_apTKW|NhHV7vD#Z9&HmB&s8+N z%=f^{i&kR#$6D9XPa@P=P!-0|d(i}}L^(VI8Xpo?8|1xvay!<)ZW7S}SgFmFbe&p# z^yVt>&`{RiuZed80|P}mtX=SPwm&|s8SAZMy7K$mnWk>^5IDtBCMG+~AsQ9(p-e<( zt^8MAtz;9qco(cCR9G{&Q||5(gt|Ik^(1!<=Ui^$4wmIl#9eNHCb_Vs)}lFcH=e)T z$<%~LX4{&tuk_*_Jobd%y}M$^&YkA#Ytu3_OMu6c{JGV^nwAunmn)dQ#pW(dcXN{7 zzEd-`z6bi^$~`f=(k^w$-(?q7j?c@^&c>FnL4;I3ee&dTZ|!n%=dsTXNg(3=ImDHx zn$b!x-?%YkL{~pM4l=smmP2%)?1S7$R3f)+#{$x5AkoQKZ~r%o2>^`yA(oB%8&bN2 zZw3XaW&567-wBNzdf?LXpnz+5DRS%vIP{~q_#$4$+8e6<7#Prty!%pUCENZ29Jj#V z?~+ zjpZfh7+5cMUMa9eALvgS&)t-6_PpjGsyeVh;yvZ0nc55s4R+B>ORrwNx>xYa#rnJ8 zA-C_@vkFP^YxaBVcQ;lJ|9Tk_iHalUtD=t-NEncUf&%K>lB^@>e<1taessXXypCWu z(Cil3?!R7cL%D2+YHO}Ar~le!2`&AqT58%uQvLYb}uN$y$QV4H|oOcKXj@Iy?-Om*>ldu8^?gpeR<@DIf#kB-Xmn;ZR zE*773Z3XZQLv-o;tvkZ`>P-1r;`ey@l~z2hmIERfzl z04u5mjIFL91&-?KvZD?q2r*chm|O>-B_0AiVnW;c;3T3hLoiIv%v5je29AQm;b~!T zJ~xgZ&}EQ(R8TGhslCplQ7>M!;oS*g2KwMC>fxr%o8L7xvB5f$oR)SJE`rYqTKnjR zR!AY)60Hu2i%VR53wmf|gyfVI0`J?}+K4YB_%wI;I?$nzeqTEbu#pc0y5EK(3HJ&f zMwUnkAox(X72^%RHbWhyMc;H(S(ybyTtHCJGGx?tQH#b@?#GWG+gFanGrm77NcvFl zX`thnxK~?QT7E|0r&~6%pp>@_9vA(HPHwofbQl@ZS`7&)={*CM%LB_Fx!yV z-2N|oB2|}snNT!uKR{&t=j;DP)}); z7a&mh!S(ige0sJ1r!{-5IVf)f@I$nuWE9m_mB z1ok4wB-9!hR-aEOmg`>gUm?(w_y7oSb#1)L%T->^_^o5If(H&9!-*qOJVgpxAry1I z%P0TaVRt6h)YNo&DfztjK?KdxbSKFtca2+mgv8puS{_Vi`oPW`PVW%wP_LN|GYan zkpGu`d}G0V%~=A#1_aDc098EuUR9NJcH?o?VyJ}6ll<{l;iXIr4694<$7iR*+nm$> zdPx&Q78aK4%nO%u)iMgMzOg`;e+}e5vd~_KeoYi}TC~NC-R1OJ_PJ^s5MfX>&o(3* z{TmQBfRGCT5;qH5ddh73(yR+;fm-PeMVF}NNO>p~JqMJc5|N<`K75dS_!hpI^}<)2 z%07I6Ovwp@AVhQR#`0Jeld78%tseRw17VLI5t%|eJna5`8Z5c@XjlIGt&3gsdKXdP z)|Uh_x8=F-`$zKrl5KasB20uC9dT{*sLYM37pWVi{#jU{lbhRO*U1KL;*Y&x-8`K5r#ni;aaC)SRLV-{TIZ*xgwKz)S=$DtLIj3f3 zg3H-qK7r;$I%DI`GJOs; zp?~ldiJlngO+GRWZb$EiEVFK*pG-;42} zdYNkd@>F=)fY)Wc#^z>1s290m3C70kAB)=2!TNXU#X-%-IZ2{AavPwLV z)Z~quHpSSq^yMwgSOCq(9rl2ctSIHpul|M$zt2yVi8w;QiWTnLwSD{1gAO(}dmC18snv_Z0Lc&T z4+3`)B&*?}_M@Q1jK4%4gW$9tc{A!#1u_LgDPSOD)G)Of)WG$p`dC?Hj;lymj01p; z-T+KR@7~!$<|hkA1k+P?esB&D9}%SQ$Kl~d=g-p{7#MJUKC_14Xt+o?D9LAw853y) zWqMbz?PmaGL#%5yP=jF=i0|Lb$?1>2X9ZZ*ScAH&ds4wQ-60{Ka`@ z5H{ixi*t6A;=wh*pt9Ih1#$XZ(buK_T@6{;;ea8Ig4c^+lT|y+?RN#pB|~GUcmh=e z8?J-9((vL%KRoJFoM#FJF+{9|FP&&#UjL_Cn(|pQQ5Uud>(4o-;?s`{V9$!>1 zWz8WVAix!-RwLn_>Hx;eyW(wJM_QGH7o>a2)0h|rz~1;D>(7|}wJSZl>wz`lz`8Y- zzN-zN1qjK6rzGAhf@8u&ZYCuOoO*olS%@g*DR}>)VYGbX!mlzRfnM((K(MmSD(qk( zMnL#vaX{60rZ?iKg1@17-n@CU?CxV2TMg%?hAFovqnh(Q@;P3;7)j7dqXVgp?qogb zr0P@h5n=7!cA!k7+yJ4yCGcUSqiq^QC~cg%e)A?@I?19|;6#f$moHxqR+;EILz;$$ zPm5erb)==QH37rVi$QPj#*uqhR78_uQGes+P5NtIUKuSRLxAhVX{w!Rxg0XLDwt*R zcK-gSeZ;pIY}~SCcY>Nq}}T!n`CoN`y1ss2Ow*LS6;z!coW)`*DjNLQq$4`LPBoN zP7ZG0yZ29^gKUKL?edc(#COp!8g+Y)q;#Naes=n0%gp+5;MVNewTqS_G1W$k49=7O z5dkVUaByS*UZJnfxoV$KvkA*WDR${UU|uVi)L9;)<$L?Ka#Qzs>4EWm2M+93(~@;# zU|^8KNlA8}cP2({I9jBErU8EiK}INs3LP7mMq&ifAp<&($bOyZ{fHY5$`U#|e}p9A zAb=c>!l$wf_4C2NmKn&=Fg`1pMIhHOx^%16<;q^|n3pyEp|HEA*~jI-o%8{fNUm)i z9oK-}mr$CrteFw32gSsSK73$6R2`F&qLAJj-YDs90T=iKUVg;pT{lLr8(u=a1KKAN zk}*-Yku2afOBC)qBQdp$bKO!;EE!{I?*uKaCon&sc-7|$BtsNG8wZDyOa~tT92=^H?ABi#=m!w5Y*qZdk_UK(#j(JoYt3uDk#U*#@R0$%K zLV=O(p@{X9>b$b+-rhz4^x!#)Z*scdLLV-@gM|wz50W-SDOp3qt%L@G$&k8*kf{wR z#`K;YmtILoNDx8iuCFg~5LkYDO*E7DXx?C(^Z{9wYdV;>2PKVi{eB2|zG6sjdXWZr zV*4lYAR=roT`EGIU%h(u)AWk?e({)NcQ%mELw6Q4n$vY8_!E;9kD$D@z<(E~TXMv5 z_zt=z@Fj808ZOLDbI-UA7jN8jzBb5)*dYupA46EIs$cjtIRR)eK*9gF*}5km)wSUkuRxfv zRZ)s@DWK=w@xJ;vMf=FhI?=H^~5nFK}h8O)ho!Xf;hABBhOosGx^ zjFgK~MlLV}b%~@}?BbSxXxcC$uE5F(c)J+%PT;vQUv^Oz0IH=Fm?~(&B~ib0L%Lt< z2m40?JiYK*2s-pPa4o&*G1X3wP3-Msp$4JvheK#GBH_dZ9OQj z{p=i3>w#~UJ$3Y#HVwTw=I|G^zGa_!xlf%^%qivp7Snw8Yl7BVxY(9ZpfzoUwbis0 zlO(7z?dQN0Ak!HQwG|Q-(6gi9I5b;*`N^PTI9^}S=Ti#6Ua0*Z-^E154-gNQjEjRq zrj6yri?3|;nV6W`r@#|OY1^5G9lEjyO?!J=nM_EPB#sORL*ht%ck7TYXx@gq6hCw7J5RiV~bA{YmwBOK(r}V?8aoZ<{9QR1jrc#IgTOhb~^IWwqBw6AY%a<3Fg^69J z;*oTY@I}V`J(yMPlVERz6M?Y-tvEXaX4wyESy{w+;ysJRLaHJJA-Lg?9~hP{TV{kq zX{wz?W#*KS(bauiHi5+d3=MD5Jo=U8q340!xoKPq3k#*qj!svHkANz_-rWHZi=(Lg zwgmK1dqY!`wpD8!irdkKw^o??fS#q9UB%0pdQSa)=!b{hQ`N_fmz~n(NaD3SDPmz2DXTQG#kwvwHfp_I5n+~ zVWf2<98~lUh4;A?l%PhUiltRpIO>_0yttPVRRITKTX*-aZ`NNTmyux?6zZ+~{3`&X zF@qKSc@N(^vx^t?xAp8&L9B6k20nTeC!7p)c@K;hhVIl_w5(@FK!_5LJ1Jshyu@g< zjK%?#l2C;}o7^USA89;N-6;gn!+A-MrXc7|J_ltJYml4B?rE09ra8Y4i4l}8+qW9uzkbAKlE%pm>vo%|t-(~%D-TW$mbOI2b ztFUYL0s@wxzJopwo^?gEHl&&|f#&tX+8GsITL!|=Fe>;pL_i_9B4~khNy9tQ-$)_z zAq=yRS8__QLv59>B5Q(~0Ls&Yw90x;Uoh;$#OjH52oTXZ=l)2V(K|Yd6be%^RJABK zAt9mbofQ>|uJhydl-pjB==S()a*_tq>f<#8xD3hZ;Ogn>?_U9s2qz>9q8nA1g~k(B z1|VsL(4R^CBcViCWt%M@9F0-fZ~@;ETD4mMDlPhPT8vHzx=dK2Ut;uM%gTnzEa$Rq z`_^AOwtlSnR=+w_lQO8)>RBuH1 zu?ThAclGt@^fJ$t`fk3tjA?@u|Ay^6JaP>ER^Q*K_3SgN-VxND1r8dMJrqy86VKA; z`d0usyhm4@L=%WIdtjpc!-oQd7iC?DBEN-U5}GBP8EN4}YNcjXq}`L5a8Bm;wIC-Z*1e8aCpk^Y=O z{e_f+0%n8%zx_aCJ<|&5gR6yTqdw-f2L#IhvGKx)aE$?AWhk(dJ#L-h~U+HZ|fme50fGfh= zSj+Qt5abo6(p1qC2eb-S@vNe@0Ow{|x6)BwzkSO$7PM?8^7->MoSXtCt9zAisN_A4 zR;BJ9Js4Go9dDn)W3tV1ie8fWl`Ul(s~L)Ukhm?~$bf9!_f139e~h+wc0Qkw2zd7Q zM5%qpz`$;HAUa*1wOTDW|HMhTglE-ieuKOm>7Qz*K!QcKN=mC?(%KWabBm;VPLrZ6 zEDP}8&{B|4Rq1%)m;hi}MJev$J$`r7v!t$&L-4f?@8mnqNC6z)z!4(wMQcN9fiC1J z7@ap$%E~RuW`GWiFsi`KA;2`9+n3Jk6Dx4?(f%Vq_!J7jtn%h>hddSzBLgm3CplPI zS;;tn0|6^NtrGi~>iT;m63}n%e|2)=u`1qeV89nHoaG|Zle@PaE&6MS_SXvt#soJ~ z+CWG=#9TsZYO3l4oHq?w)|{jQjeY~hm2NBm*l2^;Cg!iDKH|~?EPL|V{+KsHkELwY z5eIiD>*xzGv=H#*$*a%Z$ks!{!)vZ_Aq04%%um%X)K%tMt- z5j{07gQopxmMG{Rlp}M_mHKV*j(C<3;q6Tlq598ki@G@OyH2PYjGvTep}Be+Bl&>X zNRx>m;}qyTi&`9k`gyTG>t`_k!`SFc#;4#@R=A7R_hOcql8RS;3@K+B1wXge$!WdH zv3S_I4q`IF#sX*ByF%fG^|!y+YWqwwjhE-PS>u&%A^HxO>HV;ak$9gN$XOVR8iITK zeQ9YY;);-20H*d&>M{EXJFVab3|5*947B9xFl(qf|NQwA_OSiJ7r))YS}$)rkDVg5 zxnvX#fG;Y22Uh;pojXTyCy*vZ z)Ko!bYJ!DkpkOv@HAl@2kjfK-t^R0swhIZd5XqG- zr>{14E!+&(`SJE?#nJP;e)Fc|^RuHx#&Km9Z?d!7$>h~+ zP)T)^0vY{-2i^!hH21YCW{$&B&@b(u91q3tTTxjVBbp{DjF+L6dZ~0y^&FBTs@|Q| zx-Q}1DT4X6t^oit&K@tDxVLnx$;}CjIU5@eJ~%CbX{FV$W!UsL2(rUx^-LVDU#{}| zw-z}68*N518e8qWmciO`Eo_9jq}5(nSeSI75G#iWK?STLgQEr~3=JP29~AGxnwqr) z)04TC^?d3*ruz7M;@+W9J(6+}8qc9)Gy0oR-|;4aR6#TqpwroF-=l=A&j~4l%yobJ z{++Blp)tYA&gjoR~yzN?4bmOdoYM<_;yK*kp!sR8d?=1|cA!#IVp$Te`1 z!U|l7IUgj1n>f&qgspA^<+bCvK?-M;BoyMowjMsr4msBg!`BcsJ|`O;y}Matm7t&? z@&2h>m}3+c??|loM1zH&kxF}8=7rpj#1Gf8VkE znf^a_3j`dIQ&f}z!+{4=SSqN>dtcsYY=fK4Y^I*Tnl)d-68IM<0=}G2@Mitb#J?(6 zOXk-4W`|tVZ`FnWUtAvWr%?ajf0crDcAncTSd}9?Sa_}c{romG&CktI9c*RP9%G7X zXomQVoN{0|G$Mb5mu{bbW7AH9qz*aT1{_D3{;F7;j`z#aH6qD(pm7PxWzYR@dF5Q(?;nijPcWRyG+P-n^L!94&-}NLa%#aPr0Csk{!R$2u9ti}} zg;#89dK%n8dTPk?z@((4qCs~YM}?eB)ojP%$fn64^bI{-<2`+Se0O7iU%q(m932Ke z$tXJ+2>IQT8(-gT4)vRiaAG`z$SsLgqeDZhzg{pk^~UIcP*OW85pnGc+O$dSw{Bra z28dBlWFy=N@hLy|mncG}JEN+q`cstBh73_0`s0ddoFxUV1-mbS|bp)gzD8y zHCYboFRAA62$-Q1%yC!1=1(SPNglx(0OdssEdfRe)Ymhwjc5moF#O@7KuPkVjTJ1L1$CT>LHuxuKx2@JRPGt~I!xp1wc4^ottc(YcC;OJY^E8x_91 zg)avfjGY3v1?}n6Rb_hjHh_S%X)9QAm+gSOaVxRJ7TiwXwc!qNYruyImYPv=erzpJ z5xGgA@9+eok+{GqJZck{Fi{MQe0>(EABjmMS}{~PZW@yqw}~qpa8!dWUE$OzS-G(w~R7KxkqRmQ)nqoST1Z2{-% zkGY_206;Q2u)CsBv4IriKUyhj&t+``wL%Xq9$R=) zmeSL2#{@2F_A%5MjCy#%o%QU>V9O>U69!B#)cKJT1@js!0B#9X7Idt%;QQ}#h_ey$ z7cripe;N(L0t|L!MA>nopB2de&sZ@irNv2)MS4JTJ%Lct$l)vIn~$R6y-dSZj4CQr z7v8SQhDweM_vz=~N|K-hqmZg*9+#CZf}loe`}s3kScmXgjG=qge2~z;VlLzxOsfPU zVR@3RnuX!&BRG-_6~nDZ1I$9knZf@liT=dswRfr|pT;X0gi6`0P`Wxs1#!s+!KUct zF5(TYqkrI#biWHU*NzAWZh8l9hn!PGB;wBOoj+er)c?50i>V2biQj*3b5- zE=whC8BiHt!Y^2xFw$6podwmIjQM9wDj2sHd7CF4e*k$CBGO8>>mbt1-1H3%4Sk-D zL# z8CqFceF0Xo?{8Rxus#m&$r(+}TcEMvh-E;XL|chc$qGB0dV=PeXeSDp%#>D2f}OUf zK8cguXo4MuxuuiTT>vH+2w*tBjyFUXSolHC$+k>C*O}bYK*APO#HI$9wNcU0q#JL* z)gwNc0NNc8kH~aBrogYmXGGny9ZVN#iBpVk7ImifjDs5|bLaidS`c~0y&yLr!dpl-lmf)0EdVi#8{LXcPe+7>Be1ym&Ige8*1I(Cp_U4Pc+zuNZ1#eGZV|~2ptHr=~p^)8^W&r$B(6J z*Ehn4yC6yiBQf1bZa5(KSU~^4jD?rbr(G;Q1oOGh{Ce2~81FNIb4q?0NhpxBh}&{0 z9bMeYd(JK{ov?!wHVM(~%^~g_XlA9he6nG7QQTvJofrnLIg|TGLJZt}5U9og$y2b% z3SRlGi)y7iaF8^TN=bm~f1JZ^vkqj^gj_L0A&xd;A;yQka7e)1Axr+;2ovNRj9vMG z)#c*hVZaSByWn^eNN{<7YYmn5kH7g(@K|sAAIYx8oENPdQ;%`Q&{NQb5h$*-vI3b( zKtBh%9%K^N@j8t^M3gEWF><4e11BzsF2&KwNo|TvF5o|mIdK_hCRqyPVqi=)+tUXp z7$o?1lbNm`Kfu%_RCMCiS5a_SeUY#(xas3ABSKnHHc7f4jy@GmO*cy`H6uge{v5lP zj)N@q!O&pb%HAc#OX{%hoe*+WLcc_x0CB8FH*P}MVw^Zy9(PVAW@dpJT`61MO7J1n zmAxN{`?b}5HqpkQ#_pi=*aZS!h&107a>5{BQ^Qf)_pSgu#2*Lr)IK=q4-Ks&%sTHX zLK?r`7m1+|tAMJZHj_w8X~_4Ggvzq~=~S2!rn;tnhFRlk63#Z@326)$uNHMjjUW@1 zK^C|D{Ws=CowcyIgEQ5R`d5sIAWbW*nlf^;7zQl?rPBIgrsm&c4GaxiI9h+zXz_=E zqOMbW;Z`AG3HA+c#7OKdh}2?K(J>)? z>JRkX0skO9eauS#Wf_?m!!;E&6f(K~y?)^Gh|6l6BxA4+(x2TiUM)Xa_5j4*9^wb3 z5GYEDH|SFBPcS0hC8L%Cv$k=N{5H31h6v>)d4wJbf*XdR^pSfsUi3XO%@PWldT6z)37OEM7- zqcZFyGSD#gy3N?beIDNZz|@FR@VDA+QD#YVTcMNtzqD>|#fR_-GnB$3uYAbQbr!Ln zp>M5b2OB22utaK@!zE7Q%5J`Cp?IP{%;qe-j&{eg*j&C`S4ESt4ip2j5E%o<#YYk- zU*5opm{LEBIp?URmnU^n-9(Zw4FDM<%I(^JXHw)Wy8j>w4`Ai zFkNC0q7$i&o*nm}fruu|0ij4J#J&x1TkFSVAc57Cypfn_#j9bkfUbHMdp-trx+Eep zGBRu)hlIRL$?rcO4Dc{KExgp)&5+z+@-o7mhVm!-$rkm3bU)5>^Vg3b*D-CvGjQI> z=qg4V878k_sx`6-!!+L3Vhs%qG2e1D;jzK-WBFa4y6n%zM`RbM+EDayHwK(i%PALH z^TdM%4E#XEaUDFkiEzjhdQBi`iq4yuUdDw$?XPq8DF?D*rWD5OWfX1GQy=LUe|TM^ ztYe|TBU9&BgAOCAoAnX${Wh@BghY(6>s3I#@vfli(YT;N!reril&5gUNb{btET-p&>dK!63`z6U*LlowRGT&Xh=7Epl(qejG0<= zAe5LrmwC#6qP3Ai+dnRrFY!^xXbD5Tsy38?^?s%wg_|0(?RKI2TEdg}vy1lG2&-EX z2u-CA!r_>1{c5KXh)Xp#f-R#}(%qw}W+@Z#qjeHa|AZ6|lpJqkJ$Y~S^}TnV=PfhY ztV&aWi#^J8Ez}o6j!0x^gyN!y<>UDfpYDQs;7MNu1pZJ}MNc7@rC4bv&FS8l8(_zV z|M?QK{68VJw-y&iRagk{!B2-PXXrfU$2X#^U5Bh2)Oh&+nAGmf_gwo7aeBd1Ux@1s zpl060%uGu+xhw@N7V$&D8iq!U$K(VK1>+JLuyoL(sVo0aNbSkH+Ts0=Nxl;H>Mz*` z3fvR~rFCVJC8ecIU0iD1(W=W>d}O#bPkg`9y1VA!`O3#jlpgbbg||Mj>cO%B*dnEs zDsj0A_)UUTkZsY!Tty2Pcu*3POFVVIdU|5+xoLi>jF1!vjN6Tyd;oH7FcQOcf?VQ& znfJtdcXQz@U@FBxL0v;W{v-`9{D0mNQCu#4`zB1d4_@t%bHNQ7n4G=~=mRcs6>9qh z=-tFZ<1OM@AM|tc0rV5A7nJn!-E-fIILYq|&dtqT9dPQzi8ad8|9zh1VQ|M&E}`J& z6D77^j9A?6(ac=*^QTsPrLIk+vCDa~CSAy-kjFA@8k$=;0?@_NM< zQ!SAZZW>IS*H+@bgKZ`pCw8!)%LzR9F=!7j?};fVfFWkMWT{!%CRe|rlV_Rj$7mmr zSP?+H+VD{_mB zbQu4gDoh3-iPrYz%xnvihXp|zVjB(5D)nIQFn-iSOvo%z(oH=^NjM!uOQjB`~*!1`GX!d6G)gTaVVL;?8e1K$TmsEYOn0oX8q`)T?nyMk` zaxYc-r;?p8_SV(fC7(l`?s7u7955M$bO!ScJ zr_j<0o~zi@&YYZ6?fSd~Yrt~|cMd)u>fj&H^gJ79W@^duUu}Vmbrx_H4Ns-~Gz^`! z6Ny@B(L?RdxKv6hTRs#&`ad)s>;*ye@?|`4I4T??=cd8$Z#BB~JyM@Wz={XZqL7Yh z0;W=g4qJCda`I|F1Bl&m2Q)I#AYg213)oK0>`SKEl~ETX#2el4&0~G_r+HGQo%3;$ zDaD0Lw6-QjuGlQRf=~^Oh>tz&xW`d1$!-!`25FVDuH5!^@mb5(ye$+WrFyp(m)VT| zJc)5G#w^S#cEV>Q1$LYG3mE4?@asn3%|ZrT{!uiw`fbxa0sN|8dX2`ZiaKghs-~fl z6u3UrmcfBn3|GNtNG_Km_;`7jBSoyIu|v;X`(}kDTAU^8&Tp2$1vuZnYOZOSQGSdu z$f9AOeH#qg<6anAqWD2$fhldx+O=L-TxtFd#-RHNV_yB!RDRT#jY4jALx08dm~XLM zBRC~6JbtB>%2-##F6Tq6n>TBi$6Yxlf8xXu%zu~)c>MtgG4kb~J`s;ChIj#?GsKSP z-ED4`Ztj+C{jr#@9u^VcdW_f9jhn@%s3B=GK&?3L%<~)fo<*Jli{aU!_z=nnZ(t_h zC%dot9v-AGg4*?E=J2#dklYUEX;xoQ1g);4E5TEd*-)@teCd)d6N(KkpV93hV`v;o z!qow<4O;u=$EV-!jQA9p)GfCYP98D@N14OSD^jP*rp57W`|!6Ut>B_lGCK$Z^1Vi0 zrnd3l-#1fm?GiJ)xbr$J&)QV#9gHJS)WiMEo3eAdJ}k9cVIfaA>G@bR;Jaq~?wEsZU5eQ`GWk8L=Dqmc14nfNEnsEO8NT z6RZ;RV>LqJk|Sy(xN-Ga-6q)giA3(%v2_ktmdQV^2Nr~YK#Na&50f^OO@j_5#wa%d zfzw#6^?C2XQ`~`*wGDJs@oCBV>mVQ6(R&ds0bQrkp3!blEl|NWvP6+<5ddG!<{y8c zagiFXQmxAi?#2B~xI$?)90EaX`<(h{T}G}ZBTu{oUk}K|cvToW8=*kRgeYb!$)sC9 zUINO~+L@lw%?GS&vh4nBKK}SH^ilqM6t1(WcqkYprty7a-N3*A8B1cA#5E?igDv8g z-`}hUEGEtqGFXUfQsNZ*)VqNDaNC|nV@s;30-3dcEeXFOnG>XVq7w*V=*POLOnK^S zWpXI#p5({Ll(IFGKz_-v2wZmjYKL3?3QVltP>Qa@L|%MuioL11Idy42=}%3yn6HDf*x zZrvHc+|Noq**X`HyO(ht)^Le_U?W$3Ub4Tfz7>rhcx&EI#$gW=LMJw?+ia z-F#j)^XTDRzW*9rho~_2rHi&3a~IyjKm0Ldrm*xQ#SmO3sSBX}U{|>bSBVj%K?Xk5@DvMBWOYEg^Y z7_O!FaI)8@D(L^0rDbXw2|_kjqV21UJF}o#)fp|)suQvjZk%B(Cf}CH@zOEDtha9u zkvV-#^TN9K4BU|P-eOcx+%0t#qlRGf2nE{-<_=ZX6Kb&1$a)I?_NBe}NRo(YRmhy1 zmA?N-XDAFc0|KoYjTS08IMB$WUif5yK=l1E$XUanOv}^ab=o~IH6bcN(t0{u-$T4N zEJ21(X|9XL>MKvVCGG9CH&w$&(K#E!-hw-a*ONFW>{9!DCSIm!7+P=Kk5--I$B&bl zKtk|=cVw2Fdn)XenhtGZFK(LUdY6gKglkcSm0#_c1aDbW(;LOAD9{b)<&DhDv|x29|`k}bG8Bo8u-$;;eMm3pLgd(E~NWR#Lde3N|gSKWiel54WbcZ zswX(Ld$}4<6i=Q^!JRrFJ?MZbi!d3#mk0|GoxgRf6KeyZdN3VF20URaqNU*f;WAnL zU-@zk@wzFQDtAZXc@6FE0_TTC=wM^kf<07P!S0t zQ2|LxMkHfETNKoiARs}>IZF~yY(SD^kRYf?PD+&g=0?T-ec!wH{c-PjZ@h6Ct=%nB zr|O)&*IsL`Ip>O`)#)+5sjY=8$Ldt`&NXPQFS&PgK04tb&nS*maNM?iBnAsj%^70y z1)OVjpMj5;_p@gYNR6=L_=8ozZ~hLmq|)22fQ;q*Jo zU-a~Am_17yANMwdJeX{{08?g}j4S|(pgPMSqrU_asH71(7f;G~!FRi^vhEY-eRN}% z7BD0@`t@WJcANOxwQJF{rtgImEob^Ml8ajXh9Chj(LGn4d^{y)bNxDdwC(6lQ)A;* ze4CR0>bZ)f1OzqhSr;$*QK+--k(V58ZB~w0Gj@B2MXS3MiVHX|imkdC1Pp?Q7`Pqq zQDUkq#Zz1cD7~_16K9b!%ZDe6x+r0jFCU1Ei4CwVzL2E5ZA%f|q#LYUa~3~D+jsr+ z=W`zM6n5LClG7@4H_3TxgG~>^{qX4}}iZ}pQ&54hydd>mpb)9P~zxK(4V~(omX%HSe4zLC6Je9l8tm^M>nQ6bKE6e;3c4R`tX6PaNPSY zCFgC_j`g|ZhA29N`1B`Q_7lzA>{sU#K#|iwT2@kCgUMJ9QHPUgT;YBA`iV{S@Dolv z>UEr2`QV-x+NsGC0|_WuDD3Z};QRdWTb*x2=CBr^4h!QUg};I=eV;ue#*fi? zrI6U7-3V$j0`6^8A?J0$0^CAuXYo>>knj;NEJUL4YGxB;O6QYwSL5VaKu1RxT_d{= z1@d}Vo1u>9mFKYso(fq+#GO$GxMl)Eh3_R6#0G{0_6&>ekjn9hn<-+NXQq|;yl?8F z-Ir?7z0UEAP*Z~V9!LA^vz559vvBIFwQB?8?lyIQZU6Eo%L(H%@2*)6C~ZE45PykW zTS392a2m=Az!7h8KP~(k-~oh$l#W9D+i@OMTNNkJnvWRaM$>B#R=Iy(=$u%n)p|5o zcK3hdOC15fNZ#wsm)~Tmpd@o$D_zj>_k@a#)>Y7r*^QtjF^{2NeZ<~EtZ*FbO-#dC9N{>R^FrWnxcYAruQkT5hqaivnKk4U6k$=&iY-TI|^g0LNVNC z0uOxZ@8=acp!0r9u-$_B^KXyRjnxKcoFv>^z~IP4<7sj|Fv^n!$rboZ4ZpO zSc>+_8QJBjA(8wH#RxP6xEvGGT;j6jKJpRNE}0#T)z$JJ2@Z!<6g z#$=uPqJm#QfJhjLl}lr~#Zq*BC^D0u&GFs+eke_@5C;yBpiTf|fE?x_!Q7Ep6C(iN zir7@kDY@>i=XhPSO30^NElVhbCq{zE2>8DzD?#i*V0meCCDZ6 zU=(-T9<6-Ak{*riuBunwvbxj^RqKzV@E@K zY6veHARxv0NhI>@*aMF^__6wcBSu{D>R=$jXvek z(b2JHq6ts#^u+~DD1Vav5yXPK1$Cg~7x5glMd>@E$s&a#6j2&dUgKfOhxG6$B`kz> zEc7g-2}K4gI3+xvRR?aXXbhHGk6{!^Zi+-xJ^K~J0+Iz$GF*QD>>e%#g2Dp4BSrS| z5h0}z-x#7s19_d;6`<10#VwJy?^9?Z(HH0Yd`Fq_Q(~J>FaZnfY0hRr$LBblN=TP$ z-}Tz?bGtxvV86mBr$E?F$@=bv*T$lbcwyIWR?#328A3=w^NvT~o<>k^L#ZPpE3cYe zaX#IAH`FH&z=ZDsjqvEVF&v8|NW34E83S8<9Vzz8%4CRUQ$Qf)=+;gEE^vQYB(r51 zo|RR2+f%UQh{p{jmwJ5QhA0JWL}8r&cVXnR3?=GOplgH;5#Y`TY$Knr9I#|P^|=Q? zeU3-l?qar#Har;PAux5Cq7;#eB_|%KERkbXTXm_C&Mh+71r%a}vh8A#%-*=WUyP&( zy9C{>w>W5tUKy3*0+7~jLtO=o2u9A^wYp25?meP+pb7Oa$%RR^iITAcggtVfiRA=3 zcZAq_yTcSJ(r-ZNB2~_zTqnHT&(Oadf$0)4cET+TL>i5rNFF+iq=!k#-Dux6edG6Y zm6PAlP^UPuhU1?GQ{UJ`=y4HEzd_)miAD4S{nl?-GsNrxBTkU*93fQ5UVbZn22wa3^hx%U-dE0&sqSKDqQ!t6m-Kw*!5p!@ex*&|#X$}5m;6T%*V9EGbcE%3Ztf`0D(!V-tU z!9gfcIj?9$O2a@l9?q3(HeQc~TOtXt1PY5#IR~h8)s`(pU6riq?%s2(RDJCl5mySm zeE)=Q*#mCJJ9qq@iC--UdaKv0UF-B<(N+F=L2LdxYIJtbm#Z0S7_Xv=#5v5* z&+jQrk^%V^nnx2`ILSG7c|gNmWqg7=>V}K{lUQ8w_3Rf>aR9|Gm(sI%RGM@9G5I1M7G(ZXKbKwttV1 z)F}Ozg9AZpn&u;;6^0DV7=C8Y#2vB{JdbdYD`Feng7qrK$D|ahxvIs#6ZoBb;@^+g zfi!j-=Wn6fV!cmYUE_HQ+qM3i5y&4kIJwgQj~q|_I9AMl=KsZ+Q02-oSlE!K;2+6` zK(v;xf%N69y5%fmJ48OO#L#WFVCLPr_0R_LTl@Eu**q$#6u>Y*vwuF4!8}6Uf!PF_ z8n+77B;9}6RFb#%_y#-;SBd?o9agBGPuCSw0zeT$8KNac7Ei=8rKCGaG z@`+?RIA)*Y*DPPKI+6~uWuomtKgY;sO%s?`8gOl{J>>op9PD|hv@iA% zCF6(2*4;S#5KYi9KccIvtId02H&pgc%~QyGJ+EI4H^qF9|V!jLmrRT&3lp3VbexTCMf^!noJ}S>j+d-8UWEZP2eC+o39Wg3e*LB9|To$ z2$F<24s|2(0c?*S?QOUVuzkVn%|4l1(5{B1-O59`YMN-16BBdD{)lE*5T*!DC0BL- zFpXb0Z{wR!J3XKFs!)wO#ve~<>Qpj(i)?{NmpK~nKtSnONg)E*S5|{^BfVAOPb79q z%U?oaM|x+Ys4R%-I3|3MAx)&l0j&s#up=s|*KnCa=kNqN-=Isuu2UuB+Hy)9PI83( zm{+_<43H#lfL}BeEh_>9I}Vt&N&glkBWPPcf}z%{@Gtq!8W$K@G?QerrEWQ2=&J$-s;`{S)x@|qfF&~-qNrf<(QL=S&< zK8`);l{h&U1$^zEvWS>phx*_x@I=O%As8=#_}Bt^Ae<`*Z550~l8qg~(h=%8)Iys0j(cVh$GzWibNluPL7GUwH|`}p?)?A9vCP&ZFAw%x zKq2-S7?eP9Oi8iy%G)}OB!m!8MC>Ow5W&C#8;-bjb8bhg|M$3BXs@{5@dDijBptVq zDgtKv+}bMt*VQj0TqfJ8Oyl>&Is>6+iAOX(JW^(d==mN@LS($;~4cYlxYcF8DPpEAu5Un>WD&ttr8NAk7$mlYKaWP<7gT{0l zNCt-+H;&JvD19h|4+C6YUWoXP-A=d#?k?C=|HT5s{gF~IFX+9=-v$plOuwoSL;WGa zg4aTM$k{szFng&T9tmSTEP8IgM0fG?ckl&q6zscNFL#{_^5lzRb*IG4PI)22&;;v@ z7+s@NPh6L-U%$RpQ1E>J#F_*8RZGx?%(0dSD6@LLHpq6?s<*O;j+_8yfVuyZj077q zS#88`^=O9}@A1V87cS{d*xCh#AjvGrR3rfO*V}z9iJi3XO^??z6T({?)K@xIG|S6t z>r_@$kf1?~G+{4E#3G8Z%)r&@h@%P2?ISIalb#i<8-;Lo3^C-m997hC0z1BW=|0hOpi;S!3j@Bm-AV}8p*_$+Dfw@~Vsb>}I%gMP_ zh{cM>g!`i<0Fv}@;vw!u3l|2UJ$*jgUX=U{%3h43TMohk#=Xo3|NF)0LG1}cwKUU3 zpu|Z|K)8IEYu(#-@4_TKLy}Tb*1NVAF?i;|xZ?IG7ihe8T#?cQtU|T119TizEnt=8 zBc~%%Sv25@O{__f6?53Cf~9tX;Ws37|#Lk*p#miiVwIu zY!9T^OdkFjgju2VlLlmrLK-;pduuBzf-Hd1MdCcP)AAA%^4{Q3cE)yNWSx0TZcbc^OKv9X-zh)!L-6$-$pA(12e2>fs{AbeOByP?NT z`TgK&V!M#0BlPBkF3L8}zO*NEI`Oe&u*AA^|MwOf40Ob-B{V6P4+uNB#4Abr21dd! z(6&Z)1eV$xe4jL7yO%cu0GIKQf)aeQad?M0N$ zn|=6P$Xt!jsrY@`m%I?9$(ukIqr;BJD56DJ6ujl_7#{RG>};U% zPkJp;F>Xq%lmjY5&!U!*n(AiNtE~;G2AF=I@LqF(9IRquistj#g$B<%F+H&O?tSvL z-Cd5M`nFc82V#tK_beGicTV8Jwy>AaUdu$G*<%|BOW(Py3kWULrO`;ae_k;i!XG?NmuC->1SE?J*zj-2I3)TGs1w zRA2KbFdjdjYJ9Pd_YhsZM@u7$Z9-n75Oq*FkNWm9$Q^s1?cBLjq79$@Fg}=`he{_b z&u;tIB7<|ge5>>R|IGa(mVZn`{g~&)eg8xAkBaWb1V>dgzE^A!3@F=K^b(B3T=<+0 z3&m!Eg-=Yk5Rssua0gVndsq<16oFGIuL}zPI)2=Jy+R!rdJ-~cLV#frfzdn)kh5TW zBko8jv7W#BhDR#3aMe5If7zS0jPl9V2D+VlU14MH78n^A~Y z803v-8&Z(i<=pKC&V{(BSV!FNaU7@qnvh9mYLH?hK=2!}aRo!Xvr}I(6#7`ATB2y6 zbmtfyf3j!_1rc!T^DjiP3j`0qw^T3|%>d&`v_xd1(-KO2g^8*N1243EMc*Z{%fRW22x-p#NaTOefF_W(z_d`Aj$Xs4->q+!e+#jU2S8nKUDhLj)| zpqu>5GrmFu_gP3GGo#+U;5>trcdaXuEHn2hY)eMT&xQV>4|;?sOKvEQJN(-*)SQe~ zqa1=mi7^m+mnrQxuRx)KwNrL+a4*>paMY60>?|BOVDjZCF+u~v3SJn6Oin4AhC4X8 zu=6QEZJdDJDdzD--X_XkVA2#K(uf|73M1QS`ZuAtAb8ejnc&FIVgcj##K~YiF{=tcV+wAZWjwouRt$@1@RkObZ zm+CY?ymPZo$pmpEO?>T5;%L|k#uUHygMV24HKk8u;4r_#$T&QLA{A@7&6q319`MygfUbC*Yf;N+ec68m*T?QW(ir1pp(JL(QeXeed~XWGWFk77H*X#h zfWluG3}DQI`N9{i?2etgormYNwhbgBWfAh6pQka^;&yRjKtRBn8=0VWZ?id`WgaR5 zj>+dV(#D6EMODmR&~5}#Wh*qKL=ZtT*jq~|BS z+itBZyAx5?Lf`n;s4{YMA~pogAL;m!3m3d4E!@wCk`VB~n*;6CRP9>a6KLC38LR@GtQ1(&uuBDR&e{pVgvn) z23qZkZrT2&akUvelPFkmg~!{*O*Xp~@(f>K{Ey`m4}%P7@yK)}Epg&nMY08SW8dJw zB~N787Qr}KDHS~1jlf@TpmG6FPV7G@uYvEcVqsZH*&t>si09<)*9diW_ z$VDYy#~`hdkRj<8Y?8s6VFM!n>FZZGiABmsE{0_|Ux`r%v=ro)U=dw}01dSzSC`7PgvPXm+$j(#kR<8 zy}zi42uj(vl4?r1WbE=!-bWIBPl3*l@S>B8xAw?~xd5>Ch&Th(A(=UYi%4QIRhP^a ztMQ=GwrhRSyWlf9IC-F-8X?)*_WhTS2~f8;-2j9gzldgO-FW1@2Sw7wGvPQQT7Z5r z9vJNg$V;XwP+pTkv;Zt{7|e1%NtFh?#8yDPtbmRD7?^~m;fKPJ_QV<`sI4q>JtK*vy>+MERJ`_x9Uwya6sXLJ{K5e8Q>m7 zFpVNl$T0!yzj`Foq~(tBBv=aJy%PuiV@+&Vry0R>Rl@7t02ypd8byb`)0Kg55c|kV z-V5^|jkln0kFC2jn2NGC_P~>J^b`R6Q5@0CbU$2(HUjaLIAmf~2z^wC!C32Q4tM=m z1%;32=Xwbxtx$zL%v zem@?mwV?(hMk!)f{NZ&5TjpqU5me+T{qYaNt@|)dG8i<%oM#D{oe}h76(LYlP$<~_ zYg+f?^RZ7Z)1CqnaQpRXC1RzqVGt16Vy|RkaztuN^f^dx4Sxeh)Qk}hGt)WW!Q|RM z;=ZVwc)OT4m!leKfzpIySO_9DXXJmFTny#{na=|zaU=KnxPU6nB7#H+y57HkuAvrO zEX*+g^MC&obcF65=TK$goTuMtrD!68%UyF{AZourScSCr(3}Aqhl`|Bpnnh_cg)c_3=1+s>VtnS87kFoRZ?6`Cd?9*twc=|YN=u4 zW5GHwQUx!B%wPkF!{d=%8%|1m&p^b1=vJZS0fy-cv@*%4nK|)t$<1HNVZ?2Ewl3mC zMRee=+Hjh$Q96c-JLfx6CB)O`Wm&)$olI}tc@5Urhh3-t2ZkzkmH$IS6(iNS;z!Fi z@+|~1LU2)t0dadH5{wXa2);t9NqBp8R0J=@QoDsiVJqedqx%G9@o{KGNhFB>C!`pS zP%=8iyrXR1e=lkgoo9UtpC7%f1RQ9|ymYv25Nj=nA>vm>nVXw4NN$Azf+o<0>yYvg z)i7H7geHK~?GEJG;CTOyaQy}8v&0Nos$sq}h`KqmN*`<2cj^O~B4jsU&t&GAN~Sd=+P^lxxO@Lt>&_mlP#GSqELrPu#IHp6E<-G~Sx&pw#IC`6!v(TPy*Ti*LL z2z^>v5Ou9@`}2XbnDR=V0suFtl@9+j0+AoV%UK^f!PoOgFS!L6U9GjW_MH_>^zc9+cLC@CaS9D9%R*-4#Ptaj3sRHX8&SlZ`o`cT0$wKu*o16Z-_ki z98K>+<3ciX16Lp~20{Y^p3_|j+6aV+)*zYcNRk=QK(joj^Pa3M3+Id-v)+0p>w%7v z1(q68k=KxKl!KC;e0W=gHz8V4h*zwxZdiR5h(uo)(PbBGhXM{{o_7ief0)%LD z^pB%`y#wy4I1ui_l$w~;qdNhV;xl}8h&RkdtsvzO`Jgy}Xp;+CB_RSK0O&Wy8UfbL zv9U(*ASWlBKFEv@V@u0?z#z^j!(ITj)6WOygsrkzMx{c|W8+EZ*|xJ3<8jtC^BfP? z*oBx52+n7Rp{l&w%KvT%!@##rZRMpfR_^&njmk%hn_qUbZMS}#j3@z|AP?dSGE=;7 z5D1|LDSsS6f_Mvwc4K>#G!}i%v<-4kh?Qm7Vo#p+n(iH(sf2^fOv2V*kMxD}_m_*( zntr}&fusyGc;UFCVU4-3Zp>2Wawje*BnPO;=Qya&(fbf~^!9*h13;DHm?LHulmT+t zC)C{(N1PFg1NspM{$&A+Seo<=cM*8xtd})*HK)x%qh(3~<^Ym9kQB7?ggEnu_bmGN z1vcKqog_n(6Fk4?0rCjzVcW3m@*fXhyoc2rDgXHg@_PZ6qb(TSqbni>2nLDaBS5)Uem*i za_=u%we;Gd_#1yS?3P@;YRR!q)u2)DL(8@~J(4)-V>RA!Uzn-nQfp)ZLV$qmy zN8{!}t^NfR3WZ_bLL2<+=hvathvMjF|I;z!GM^WH`a=9qBqx{n;HsW^vw!}{weCaX zB;@$k*7FqSQz#mCDfVq?!~8Obw|yWlN1?oCGBY=KV}+#$`~LmOj*3(ge`B)w{aI2q7vDMz9E={m^BBQ|$ifIOYrGD| z&J=3wVKBdkAGgfa;@d4M`q6eWgq8exetzg{hrGGZp=yH90h1jb)pV`t+}hr33O*qF z3RFE`TUyS2${1$7DM~&th0?eH?DR1VD?E5mi2&cHPkWT>a&;u&TYX5|UA8bWp=R58 z;za)Z=5WiUZxd6(_em|bi|p|3)j z^*$crU2ZKs&6qi;Qkhm(PHWkOLTe9McjqSk+83^VeKi-{ZHrp1 zemy@5g?<6<=OUi%0};#{f4GjYhyfx34`m%qBD>EoQjDQfhzS z;8^ebmdYk2*eN-*zgN`!!hOa5M{hJz?+Py>ivnK(8rZ0DeC{B=^QO&jZMriR{RgiIf|n7{99fh#TOs?9^a(i`Wz@;0~}U(Pb>%Nj1#aL0NkdHP^qp_y@9%AwX@ zuSEIEOYV8wKVDKoRy@+Qs!%B@AZuC&4PURlTGFd+V0pP~Wz(eRdnYvy*35&i9x`&w zeacU(d*-q4X&7bVPf^|<7SFbvZ2Hrp*y-#D_n|8dKB%6QDxE!?(6!e&L`h0-Sb-Y+8M8)$7@4$ zmJG9Ye)>MRX>xquu?OLti0Zfq)dR4%H< zNH}2Q(%Ri(+o*}2lh6MN+C1LcJ)`|}v;R{q!wf;;*6Y3v6Ki936_l5&(agE0(;plU zWYw!?ap*d3>KrkqSQcs|5_;g&0_{B;lB`BoXO2&Ek&jeV+g~#y<{entY~Li?jDz9l z*TAAT%#Jo|UK)5vdUAMmPQAanN%u#hYE?|La=LM+^Saok=fQ7;s}jx32A^&&nCbaq zX8dlJcY8_tNsxopbKb|sL*3V6qJawwF;*;+|wUxhK(J|+a#!5%|WZm;p z$@6X5s0|~`{474ZsWCk7+pb-zaFI4Mvb1q1STj?cVYi&x^&vv-rGa^Rv{C=_$~3tZ zzh30fXB>U6y9^iE6zmRn&e~*T-yHhtZkpY05uyD3hQl`L8QOv#mF4APZ&!#mZ*i?1 z;_lZrJaJsTNOSo22V+=)pc3_7V~^IDeMy8^+sM5{&jeq8_=oau`|Q=$C>t5kFRp3S zu==TAKSFHGCe=%dZPhA{`GgQ2+H z(+;Ogq^F|2s}7`;I;p+C=9@UfvO-eU$&|Uy_2&VPD6?n#hXoq)!Xr$z<>#K$4DXm8 zI50C=dfFvIETT{SFfFMgh>dn!v2ort;-|D~Bxq3Ywslhz2ITATQ-Pg+uC<)oiA{m?QDnHFE^ zTl2uFoAu+emQA(M9(vLr>!-VZ$cR!a+*lNz-ukD3N!;wL|N9^E6(#LMqTf#r_82Xj zy9w_!_`IYSsV=0CRW3c=qQ3U2YW>#WT~r!J@`fn$z_Vw}4MQ#u8-#sXpk{i_ROZ)) z>0DaFA@;fL1GBHUenE;Rzo-!}H@E7_;c5S$!L{qV0*THls?U-w`uAo|v~EYd@x)qt z$vrKz{AR>urJC>j>#LLU4V>?1$5a^dY#Nj}BihJyl9r@hde-ho$r{y6kF8X1af>9s zt{S1i?JwWc&e+_y_4)nWA9494uj<)$Vxi5uuj+5qPJVD#30>AzsaCl&Ypc$#e6EQd z+he;P-{{w!DK}FZ@iaH7X)IMgJ9_$**{k;Sem z7oQGc+Rq6q0|V)?*;aJzw;r=>zpMckBsoE;OZN-_#?ma zL}Z40q%LzG7=1MtR*;nL_}`KZQGr@NexvcOQZZQ4eIV&FZ1>^3!wC&0j-qhD+WO@$ zoX9von4Stp7(scy31b!_f38yXfCmXWIY`*u&0O!6o1a{n{Q0}liC-(T0wRB3m0^cR@r9l~ zE2m4B>k_mNj(D=4-JU-^5BBxIo@uOLFD_>il7m$|C>0gxp(epr&=)OvF!XBx0l`Y! zr}qLV2GxK+*{G06SVO&Boob?qkpQ|-)Lnov1PT~TwGklx+URO#Ci*PuyW?wiSW!b1 zO0C~-Lvj+*V@n+EO$Qg{Ux~CJ~^1x7i1lhkkbd%@zXK`G%WP}wwTq@>SxMgPAg=B$Sq>r6s z-R7lZ4vc&XO@1t;heI%Yy1c>j(IX8*;e!WTtxv({!c169tV!SC_kl(UxW4tnlPWk! zY?&YmI{@$<;JJFR+QA6V8vR^M&wk3%s#Klr-#==Ua&5@U#-{zm!9&lDvwNg}FJ}gB z5lvpg><9(cBSnKSA_O3?P6w}2LQt9=5L}hoj435M=v@?)|MNLB!T#DM%?|XuWpz6Mt=ovX0 zhcD0=)~unms?0qk29||3mRm!8O|X&Lh%czAN;{&OW7^P~zm{tJu;5 zn32TNgRVb|dc4G;@}qHaaW8t+ZKGjL52<)^g~ti%YA`EG*>lxA=$X7A^#?W=#tMhu8p`6Qvk4h{hn0 zc{`a7n*pypH!dYvC@aEPccYt)8916A242!TA#f|1Ob7kkDtp`91sHaZ+Qp6Lu2jH<6fhcvPoK}j z4rmIZpignrD+XDzBv}B{sDSwwd>5O$kc|08#QpGq`|KwX;A*jvP`}4PveNqL)1y{vh{A#(lV+t3eEU|ml=l@n@>@aJFo5u-K2hf( zYq;&$Ks|h;(RfUZ?gc{XPYe&JwEOq(V{ls((0k5p+YU>4aFt`EMgX33-IzmcjL0f+$yOufa}L0k>+y=mkoCI;8FF~4BLff%_&*T8zN{w zSYxMrh4xoI1*csfu)M-wbIC77=6d6(@CeLgBhii^ldFZUe+vL6{Q@mcbg{z`-69mC zy)n!zyxq^{G+Bj%>9Fe=2X_ukZ5}dEiHylxvP1@`^L`jOrlqESwTi|8s1J!=J**Ci z!z+&dg63v*@+{C5{o2`SC6MB`s|WEvUURM>c$KJCW7GuW~H3J zZw|5-q5mVWn*V>mQvqRnH!e+|-<`qIxBBNJNSn*dpXAmO5BEvB~s;u?i<-fCLt$Trja{=FRyW`v<08BXlPi{^5|MFWs1KkpU zBgh9JWE!2>zSRagshbK!`r2CJ}h=(kjX!;pNXP8KJ|``9)3WRAj` ze3ExAO{bWSz7?HKzGaS{-~ZwM-+z}O?k0i6)mxW%0u`HA-c#wFR2mW)B)CcFW10B_ z_9Ph_$?n2K{bbPxk5=NrZ8p|fcA%oQSN4MJVZ}>K*WS}C@8691nb=Q?M=rEMo4OiI znq*9#G<#~v`MT(|!W6DRBG|S5rfBFYRl%mL#t644IOZt_UcqaAdLPV2?abZxt0Zu8 zK6WdULuU+#%rU&f>={iG->wD1bI9N^! z;5}QppovP%9nmh5K@)Tz*kZ|78)|5+zjFl&#mVE?%}ks!-R_ENtp@>c*1TwYE3wRi`AOxHU@Im2im`GUD;0sDMF8;Y3 z@w<%fM(H%)$6j^WID#V+3{zI|fUWA7K!!=4uz5465&-9PGyeCM5y)Mn@YatBgmPT=Oh z6Bc_od-1JwC?Kx%SuOgw>|;-F?|!X!tH^>uo_-ek8?JR-R9a?cCZwXzVWfHrw#Q{? zOd*pAgEhi^%wqlh#U91Arf2zvJ%^wRuYv$I6ohQ0O9flxqc6x43sy2YQ^((~rmObP z&(M1x^NSaNDXJ8gSw1-Z^gZ>zUygFfKH(h@t+}t7V{@h9e`{c9Jl5$^l10zWerC$@ z*wLe;NJHJB06pr`bgsFV2sor9`hkCw%Q%v9?R}fH4GhR^lL7!M2m6w1)hbbuknC`o z-5HVZx5*TE=t-3_{$xJ^G6~jkDOQ7*T{iL`N3Q`!0L6S&PlBLP*TbQwG{TLErs&h7 z!}kO{r=M&8=hbD{m8Le@VCco36tLR;Tc4tXOVYx&vQl_#f+p}dJ3E`TK#NZghdqWr zcqbSAM)m+yvqjR`IG`>oSz7lY3~NZq)oZWj9V`VlD)NIGP&<;)`zDJSwCUg=pU+L$ zP0^}k>&;zW^aJq}U?qnf3@nvgmD0__(?l^X3EjJ=>acli+_L1(ZI~~X2P#CZ`~8Cy z_GeMdJ;#n8M*}5HCCyZ61nu!=A(Jn+S;MsoXpmK&2-%DKurFoh7C{Ynp+BK9r?~5l z8g?41c7`&sbfB>!Js=yTsY$HzX@liA9L%}&@_)*EC!gwDnaVeskpCp~G$VGUeMC*e zl@C#scSFieiMhA*$^O!?1{_OGQ8c0%?k2tST*x%oh-R>@s8NbBv0*>|;^<#R>;Ibj z2?kSzl_SWfcXeJCB>dWCb|m6}-@!%&1Rx7yc7jR^ly+^`u9bHES)R`PxbLFYLwiy{ z(uhEC;b3Ds2I5CGM6awLZL`!s9RwtrprVP)0$P|-j57ND)wdp(M6*12HjRg9r8>*b z<{(DsqDX4Zz5Xe1=Ah3#l|jMWj}PfwPPK&{<8sb>@p|6eL=)dA<)9Bk=+lCXOr#VDaZr8j)*-e2tvVXhw6_6e%RVqLlXh^wIXt2 zkGbXsl80+t>w2b?QQ!^tW%PG}=Zl#hHwhEiqUXa|(8Jmu@rsG5k(3<;S8H!?^j&7_ zlbE3t5AKd(G-*5j{Bqo0p#P?+mq0J5OgJHVf<*lS)El5eZ-qp`?42i&V}y8JRQmMG zmdU}_9}r0}S>B?eqT+>rZV-L6?O!8?iV0fO;hws4I1^=$9=#1*Duf)!AU6~mYqg_l z+XrDLI00(lgIITkhwg)%JvT3}&2p~uLOFD!XjG&B8bT%(PAy2p<{(WI6bhDHXAv8; zQB1EGBegDksgwW-DWU=&h95;YdIS~Z(fgq+*)uRUdZHtc3*0|I+EN<(irK9f=c@2VR!rRDLN zinp=+zLON=Y|@Jy$jQq)zEX7P5wms2Hao~Sgg_>S9e%hoBj-AEy}7@AD2sg!&=!PK zaB$BqQvGwbKQf|Mja$EIQv?(is_6ckT3RX+^KvNHZ}R*p%-LlSGO+Z>@a&B)$I+dJ z-Qiij4a<~VZF-Yyhgv<=$rC!#y!MR9bF$b#La*WZgP^dxAyi*&dZnb9_W<~&45Cxw zvSC&XI5okvCsPkQES|F%9g85$3bK=d$Dd0xNFJ!kdJTehaW6azbPmF^>!`o&z`cV* zi76?HQ0#q$dkq!wwjMJK5Ns~@n*J+H6?DC6!h^FKlsIsp3)z8(CX9`qqy0~n#n!RF zv+ixqRtJ%$oY3bxFQ*D%#Ta=0`||kYs9kqZ2S!Id6BErmG*Kn$8=e6$15~Z87Fr?! zx|>L1g{+6d7$g4#3Hn_K(cqfkE@AtscuYa6l7duY8QdeXQLCB2}Oo)11f z_pk!?BwELLy1B7Z>-uW20)1GK2800;z#xIreQhI5ws^I83HP}lIQ04QutvW^034{3 zxhBB5(Kk#1%viJ+`C8jZbik^)znSWPUi1T$K_r#{tz&yq;1(_}DEOffWN-@)gGOe0 z5B9S;1IPY#@;$@JhqHp3zVSU^!ktH&k>y%I%s`bz!Hf+37FR7%Dog@(d-#_BtV2_ z0fZDGMnBh?5l`kx5HYx-Y4LegAuX6>|JPzSCshuXSLJ$j+vhl#zPLgjC&2r5)7%?; zJ;mO9Gc%*Lys91lDT0#-V~HE{O^Cw4yBEza{Nd@)#!t?3!=Z&t@N zFxGc}&g(uJo(M^?VX=(<{v%ePg9ea80*nl{wY8N)N@L(@T#APfgfz!Xz0`ZZ6c%-{ zg&NQ^f{UU;+P!_d1cCy=I*_~X`-OF+-62>9}C`v{}4jOBkxY%h$#E_++vD zNLa`k@8aZEO_y6K_Cv#!0BkPIq8AY03*uV#!rYnMH_!xwu@>|&qe{QwWiM0pT#DxS z;%#32ddMD2Nsg-vZQe1CstnUsXBH_kyZh(5?zi&24?0M)E9pm9o{|@y<55;_5GnE5bMa={REa>d){@~MAK~@ccs_#@U~-arAwk6eKyh1(fH+(; zxE}Vxr0tc}dNN!D*=U!@uLq;SI17GEam3uf24d&K`67?j6;OHH$C@m+T<|YD@f_S( z$qtqWDmu8YAAHERmObz|8v<;8`QUpngQ2C{n~nO#pWo%)!-ut`a;9RPZo*(&4+CI? zq#rRkUSoK8ZX+`AViEe;4p6BT09=P*QKmd$x{jD(gSKCgpML^_bJYCWqg69%8v9V} zxN1U;d&LlBFP*MtEORKI*{Fn0dT$L|f@=4Ofp+qrn#NfShsJ`%%VZEfwE4UUw0~j3 zbo2_)WN$IMbXGkSaHbX(C4-F(537i0f~zKuYXh;SLC_4_b_n!e{;$muk=ozQW{TsB zHweqI?N@vByOx0gCoga8O`aXbbZnqB0F$b6^#sR666iGSzx z*^hBa+A`0h9D0L8K@q3cvd{R06}ad=3|ALc!zn<5C9Tzqbm%aHG@P{)g`_}#NXA+i zD7L-Cys4NLjDIibMXxL}dhX*oz9>a@!Hp%i_|~?{74Z(@$~>I-&oeE33LyFUA?+Y2 zF!tFdG27JY82JxhN9;0(OK}T?#S7=rJ599mO5ezFE!{42{hQT7dOETDhd#e)2M);K zMzUL{6PBCbEAgX>hg3t_VQSbD@|1m=dCNd~{1`}wlu6I98N=afze8ft_7b76x@Iom zA1c=w)+p88cVS}9=vN3Kys_n2I{^-zMatA?9Sw@FPXp5LG*?XVdv6*OT>Gyd^)y23 zP=WZBqFIJ;qXik?Xp;?Z5s?J&r6B(ZM~|3F1GhO0fV~tA)o=*1bV(Ze0*_iOU@QPf z&7D+J6O)1=>1@C)UJYr{Y*b`PO7AhhmF3HqSc7U7o8gZD z{j1mmB{%yzynpV{*Hb-ND@P2)cbdb{)5<#$)N&H~D@u0o8!fUS&7r-}Z&spGNmM0s zRev{#I4cwBW-=8{qo-K%#t-;hp$E55=mZ`Hi_Ho@*=$d-q6FUU915!&E+t=VIBdsf z7w%!pui(ioyRsQ!+b=i8CZ=m`vUdJ_GsfZ>{kKG=NZ&9m6GEyj`HSi3 zd>U}*y{~G9S>Hh+Ar7=i5p&N1k9g+G-yip5bhb@u>=1VTjnq{i(Xhpb<-aO7oog*h z=VaS|YTHG{eGX&mqU(~*teD*ZR0m{@ci&(&$gU&)*tk7Y}a=Y@RsW{j$%K zb+~`b*sqIk6C4OfII$QbqjYww-b=q`DX`?{8*FU9LZ9w<^8n-wUeX~d*{H-c{P+QT z4NhP?CX7uvI#S5dLNkZC7D)h$44M!n_%9dl$?gN(7_yg5%A<;CPpHoK?~};`Pg^fw zN1$WPpWO!^SHFHHij1uYxFy>fbo6UNc{{Tgq(>;{hv=)McwaaW5geM4QXZkyvQN8F zgDQJP<%O7U7S~%_8yn$83Zq5eBNL1cUTwBip?y^^SLWVaU3l(lbFp%u>cp9d+K{|Ch}0htG@JwA`}OnZZBP!^)oEv1eM6*%M7=u+vk3>|y`1V` zOHarlRn9 z4DPMR%*VyqQ!^9S~yW>a^_7(tOo${}YLm;Kq6oKX{_{B4F~sJD|K<)l_1DJseM zKp%-jJsf>On5#S{Q~R>*?J~dRECrMzYi^zd;5869#7SBqBwetq5NSvT4igId0UUQ? z6-W=NAeCMY9UJzlf`r6P46}WKjE$O}_SOgur!muqI3o(c$vN#hB6(6;)>N{KsnkEi z{7Kr!Y#n@n6C~Q57H%DjXoC6nuYQ{@ocyfRMZfh6m&D@AEBOSZ2jvN zsV<^?%|IYSvEgUcwh?u3*`!dcn*EqIH@uZlaU;N-d%kjnRS!}hR7n>w1sTlfVVG}g zZ8`^Noa|dL*AK?*2NsB-Lk8GKlG4&uI8^yvb}PS(TE}%h08jKn7Y@+evOK1{42)}c zQNj88?Amrm{a$&$@Cr$qcXjOZ2zTap7Vis}+a!VPC7F@dQ7Pb;F?e&fP#oO}f*o%N zSU-)S1Fh`uNtYdszP9!X04!8wZ`Ghh%yJmo(I8gA8$vnJ_EFEt7u{g;4uE)tLAWt6 zJnW7?Avj8u62BdC3epZ_w*B@-r52wKb#4MIkYn&-g#bEXAi$xTp43$v8w(X$7obg~ zoma&gH&Oz^s3)Z>z|Y!m(j{Tx@Y!E{-^PP+-AvK^4r7;tLz_ju1UWoBV3pqY&7@h9 z2D)o@p_O}DVE3=8_gyivL=<=0Vulr9O#o!5!ZB>nXL;?~dkYt6ehiRqJKwukj^sr? z@$#w2Y9C)iDzhQ` zXhqCH*JjHf53-(3`2fEj>Yd;%%-gBH;@1@@&Z5XWl|cw%Ex^oC%q8ejjW(Gu>wwuE zmn=%NT={In=;{!?N34W<4R%=NvtS3^w=ZvX@lR@=^53UYviYms=tN`i=8|3=n>O1a zGo0XeT&E;EJCB1wD?hv-vhm}opMi$*?w>9P36@QpH~?>=yY0??_Df6+J2*mVz93dJkc+YQ*doD62Fs(m4DE6O8-3vLhN(8!Zajrx}Tl7z4EpkWYjmDi64B# zb3)CX6)a1$x(4N`{57w;Mz)9q+w6&bJdFLz{Dpo!wRHr1!>|yQ?nT3shmfazuOb<@BW5;g&+UESuBayWKcI zBu}1{DWF+SOWjp2?R*jhUAvSA9p$whsJkVNE0*7vyML$d;i>e;QNae5lC=38;sGJU z247RJnMy{mc7iV~x__Wj#lULx{;{@^>RW? zZ^8-p&bGZ@bEQ7?zQc)d%^4G+1Ve9*>o!^!cI-T$*6U|4b@$BecQrM~&sZ8n=hq7; zu=ywOc?Auw-;s22Vjj1024}4`p3BHP4>?NLyzB3U!w~zeS9>F@KHN zx5mC2C}b)T2e4y5HLOS#u$&F$Wf!mFFy0$ z|87H+K*ZVrF;(TRZp*s`H3!=BFMD1yJF8)losqqJ_WjA|ew|dNEQOV#e@~BL+g+eZ zl|YpSLIgSjHoPi22ZNnv)rALVB;YQ#I@P_-or^kc4bGhnGEccetqf;ykv z(V-tU1Psi1>y5Ez_3UnRe36Gr3YhFZFmQk;fI1d|gVg(=SqPMqX3qe^3E|K>N|JU2 zJSfltp+&5&Z=Fw(VJ%Uc%%D9vm)I+Syo|r|1e4h6?NluB5RM6J9bqwuK5DLh3P0@Drjzck1C6w;&w&Ds(HhlmV+| zSie3Lx2OvgAw?KufH)`a5Ip(iK5rND2+{-q)WSwZng%a9l1c!Yp6#zVKl}b@Q5R`a0PHM! zC_9hBp70_zue-tj=5X(sH+8Omg-trP*-txn)r^#yM3dv`pitNuPY(~SwW+*ptUEg| zU3arrN>=r^n~5Bodb}Xx@&?OqKNb!0WCe8Dr`xth82LYg4VouB}{G? zht2{O@uFFFbhX`AHs8Pg@(gPPk>0)rslpw3(dwKs=Sc_55`q*(CPmjFpOxL=NL>#>l51 z79JEn7-p+vSsmFL+sEV+rqh1e)Hn_w3Nnzc*d1sPK1?c!-VJ->zgvHM{Mh+HmdC~7;UKN4bVE;ttCQ!%ZABzX zza~VEOy>SA9b2mYplq_$XZWPgVs8O$HymJ11<|vgXrNHbZJB}t$AvfPEPe5-3P$;D zQ6~g|$=d-qv4SANm{$!zm2>ajv!rX8X4+bX0h|Q5ZE6z1U_QrJVP~sP?2j1NMXzG? zl|o5G7N6?*i#HtE%uC1s#YYd_6tRC-kkm@wC`-F!mwP% zg=1;%T+cf`T)$dpC)Yfgbo_C7ZPMjRG0~C9jQZ41Qz}Os$m4pgy63}5&lD*=>LS&9 z79S2uHHol@i8Fq9dywR2U^_OKKZA@j3}~N11W7mIp4orA5P7N+K(=L_ugLn4gHkhP zcdpV^>-@;~7Sl`h76|jYc=}SsaulBgLgW38Jl0yDgzJ&-T)KH#)JKnS$-MguCuLIbi2fXRiOr~RW{1BIAhRAKa7UerRNdkE!_M* z9=EG1NzH;SG)TW=v-)ms?vIV_U0<#k)<(-KwZ1CU$~YaWAaEvo&nB1p(V5cRS8Ltv zGOM$WsZuNI9u~okbR?@`Xg+va6ce-VWAp<)puibR0W;x!J^tbvk@Ca-0o#jpk8tnuS8<*G*@^hSNP7>k zDzhwW7|T-INR?tj0mDT=KvYm9WA=gyk~1bivJxd@!ZLwg36dp=d4hYsv&o?vQ|NC_J(`CWEZ#ZY4z1LoA?Ik>#V8R&HAHvKaH5$gH1?Mu`oCYd%a9b;&!}Qu-|qcEm3iQF zK)$wG=-o4qWg>f)Efd1=31ME8jBNS-sKIQ-tPX=bEo#{tjoy}yEB*lvy=_@$%_k(1 z%#WSWl#SUaCL{5oDDkey;HgHVxyBX)d)j1bDIv(fb(1t(!4`nL+?RAk~#UKCyY#*udyX$K#Lu)5U|DW)5et z7zS)ut^Xk_Fj>vVmn4M~E`H@btPVugLNj8{7U=cexIgM#ezFfqCm@dWt1=dO?w&Cn zs_5sSRyoPZzJJ`SZ7P*I=njqvFq%@Kn4yAK?}KADc6WM+m9O*lj?fBH8HiCjyQlc< z{ge$cMQP4mbz}NwtX*D|FlvJyw1x(vIP}Qqo@Gr@pLFK&y zJc|$n;JX)te`Mda>jVul4%R8SZ9p0#WAFYiI`IIVy*2C>UsWZt7-dUCl}x*_0bnQhIQW0OR_YhMV3i*hCT7u zW|XZ>_E9KeWaDv>sjb>)m#uO}GxlJeAhUjxl9>HC&Bte}NeU;{I9L^%TJ&R)Ifa74 z7mg+nNj*@3;^hc-!rxxLDbvmz9=25eql6##p`?UI43*&Xyi+1}GVlK?An#>UfmOTc zzu1Fs>PVG_Ke9=jDrpv6w>J8(mr6N*@5-yqKMY(fLWA4Eyuap{h!FqNTes|-_3g&n zwwZUA9wSgu?}T%=NsD~SA)zpBd6wax`qS_HtmIgIm=ZiEd=452z4UePEe#uV;Ox@# zx!An6^=l8se}H-ZTvs^rThupox)pb!g;g2O)#w$tt}@~5bFQX!)A9FevPk1Uymph5 zUM(iln0e2yyWuRWdc5e;@mx*&YSWlAMdiZ>_gvRB^jgw-s7&(=JkSc@5P{}Xn zc7N%4sprP|QZre>IzUsWwywR~A&}&A>^BFdvKTQegrq?tL#0OrQ6R%?C=w41B&^we z2EMv{P`Hbp-0;Va?IK+9jYl8pcL%&r?7yz@ZrS3BZ`pJYD(D#G|9jk0X_PmC>w2gk zlgRPq_o!{7G^5}v?Un03ja4r%8gI3X8oQPbjdGG66M2*OlYSdBeY0Qd!xiLRM1({{ z4gHGYa#ZMvWDs^%2F{Y%#P`V{lzlpALu6)o$?2gVJj@j(^t_I|jlMEHz^#&?>%V83 zDa3gyw=*wQH~e$&lU^15_Vgf>ayoUIwaIEpb`H#fQK;8mY{}=?-P#q%KU1ele;_>M z$KRBxZ-d}$r^rJZ7-4j@k?4e7eSLdH8u)RAj$b3*6py`>4!BtrqZxzo61#WrmLL=~ zZ~}-xUYd0_hTgw-*Lw-a<%vSBiFx#KV}e3gh^03B!9C~v&uYHNZ(gdCrjhu8{-w;C zFY0GC{C;E@Q9{8DJG;f+>)C~qhIj9F<<~_#d@&h&gp}JotXbV-nN0m%g0l}w?7{Cg zIG-}QQ=nLavWW&v$j@%1c55! zRWn?@+;C>_HAS~WmMKAa*XXT%QwJ<-PaM~LRIS77zHZj}Adk>J!3h90puU`uP#=LJ z#rN`io*uPbO+KE^C@vwv?P*ZP+O@?J{qv z!;7a9n5x6C^`15*U-!9*EB=x(dW-H#bL^4`0ElHd-`J<(_0oWeD-KTVTxL;ptsH*A~yOFkSiF#{sKzmy-L1F@(;QDo;OcQ52 z$~PAbP2JsD5vpHs+SI82)VZO`>r@&fx?8KWGG=71%Q?gU6DUd+Xri-=M7rV?0E?KW z2^YO3a)lPB*NYj8d`w9JlDNHTnXsFmQAOC`1;`8|h@SuZaw<{Do0z<+Zf5ej{7bp4 z^gwd?NmHROUk5(v6kIwo#B(hu8SWmc2x- z^=9hfEF=82<6Jmzz$cYZW++Ot&J8ajf2YNSbs!b}JN4 zTan~X{IJA-IE6NvEqAO$l#heNHVgs$lilZF6~0LGlY{z8et6Yg@ZfctJbQM{0DE&d z&eK!|X18X+jHuXvxy0 zdO-2VPzFhN2r2dfw8?>&9swc)_<>~K znCv_-$L)M$_vQ+pUgswV=5>E6O%<~&-ghZm&@@fH`IF~(nx(--qnXz~M!Bld@jU*( zzHWDC+nTH{`I$cN^kpPCr_luRdtx7RFzRcOL z;5#Gl#;Ed#cld8*#QTN7Dj?zXCD=0{Q#9^- zP<LPTyUgcBtV!BQJkXq4O zLI3N+NJmjUT-W!%vnJn6*Fw2I@_W>$*NWsjU`BBNulw?+4Q~4QZk5Z7OzWx~P#kqW zgSW>kaHvMdt7+TI`)y}7hv2lh()5LQg-d$)%V&o|Lbu2Rsa#U|O_qqr)%r(v%Z6W$ z9v|wb^RQ$0 z)>>w~?=^+RPJhbm1{eRX@GFh$)63M)Uu&IevqhvK<57+t-gFetu2qROvp3Wy1{$O| zxTj!+n6;g5TXws>5~gNFhTzRgx-M`M=omP#6X4>;K^=#;R3h4&X)2e%Uu?4gV1gkv3p&Ycbqq3a2nHX6YW}sk*&Nf7k!MPh+!E+6HYz0yolgjOT>njc0HYQu z79udjw0ktV0kVN@6$qB+lisb6sxWodQdBvB3N>)3ZSSXg(425 zl1V@0*A804pFj%&PnGZ$TcIV=oGsYNZ2+(+PlrP*8cV=X12GOPgM_~vUq@^7BM zjyK)Yu_7|)bt9n?ItnRl`)Nerei_>GZ${@zwUi#^Uv|xYY%Enyena{cG$VBo2@5wa2CP}V=#Tb{~2 zG2H%zS8$7o@;xfQ+=s;5dP{zU7rcAK*AnJz!qe+@iQ7$Ol>)Jq&^KzJNPBOvyQ(;f z^>p6wcpLlKr}=X8tjW)7dIYXZOlgdB6=b3&JZ|uy5>VW5{!oa@K#|gc@O6(b6b3Z6 zR|Zzkz~aVL*B+^#VT0*u)zPix?Ys_^lVLI8fZP;P&^d5?8<;n5*u`$WTj85boi=^= zVd?c_?xS*J@qH51GqR^@))#4J9w#?JI-PMdhnST4&#X@uDKu?fODhb}h;SurrWGp@aJo=k+TP?MG-UVf%2C0`L@xHIvJv@B2NR1~T2Ds05AG`}4+jSPF` z{QwAI!$c9jm4VLt=ALal5VCSo(UdmJ-Z0zbpmP*XNK7m;FMKe?azISexV!Pv=5e5# zVRyy>CI`w3ellk9=J!Y379o@?K zxr&5>Z|5r?7^a*C1!!;I?M0` z`K>XO(L|Y~{x{I8oP)9~A z!n+hjJ>pKvk3R7*+eJ~s2et}JMkr4)b~Ob(5=_>;sz~s{Jf01!M}oaC)cdCUZvl%P zX6y3cd$w(cR+IcWUh%3nOZV>|oysF1(9d?!D`!~1X|6JwmR#oS>p?Nslb14zW;NM5 z(;p;A@c6L@f1>M`z^q#?KKSAx%|^F+Hrd-Iro>m{eL!EqsT$V;jS!@c(&x`V2S)7{ zY7PTR$mDmx=NUSW{7N^L7jxf#Kjl9KYB*=AbZQ0-gh)~!AL<9_($$eh6JMi?&c z&)_HQZW&Mn<9kR_hUM_RI!0ALQJ0ZIP4#n)MA^ru)5-EFw)gq?su`tnw$+M*OAidD zBkq+urxSDz#6T}gnb?Pvw{Q6ffbQ`p>MkE3E^mb)K;mxJtKl% zljV6(=#kmlM!PvvIA-+2-~EXf-!W79T|WDK8;})nE&<6mry$7He$4ve)~!aC@|LL86o2j_PZ!nESfph28J`q|2=r+jCcE0VV}eL`kMQWX1rDoZe3!Zd90yIZ z-QCI(^1H1g5Jjz1Rl9{W*3sSPomZGMu+JgrTr%8@+9F2i)LVb=HD;yCPo^i@_Z@S~ z|MFLnkk(!$1CHmVwWi-($1{eM-R=H77k9sAL&WO3HC<0TB@uC= z93jiCP|nuGUHQ7Cz59fTgwyEJcRSWIZU_U@p&+OB9FJI2(Z%ZD9RrJZWH-XqMILtg zl3en^(Q~kfqs>N(G5|f-0P>`ah%@Y<^HhY|%{9((hC+U2*IF z`t*vV+FN>m$Q{pY6_H2kC?iwOu-4-Epoj5CuQL;7-}~9kd&JJ3tU4pRr2k=>!kHHi zpAa!tg!hG3yy8P^*CaPFeJo$i?pq}fEf~IVd@rSJM3jPP67E;a+&c6>*>t>u?b!Ck z_k`#!pMP>b297gbb;&W{$QZ&A{M}l~wk|T&3PYTTz&Yf_i?)ITa}n^oGWh(G)qQ+f zYa%uFl|SQ7b9q@EOYi38wF}U!ikM|!j!RqGe*pCd{5GhwuwS9)Un!F};;B56Xdovk zWoaD5qa9Tt*lIA#v_3i!^#1ht$E_<~T~U(7CgFWJ20Dz$!5<>54X4hb zP>AvsIgA6Y{v4A&|4hghewp}Qmxq}zGA$rX0}?kq8ezbG$+Yeo^uB3)ow!1 z2Nj5jssJX2!!d>k$trL@fxR^2SJeao3&6$~OIB`R$7~UDpn;ei0p-WXwaIf)fo|m% zF{>LVvSw~si+wu6%~<8?aNHPpbf=1KAgmojYbB>kQOi*pgm0RD65_f1%;6wK&3D;p zY2;S5H8ma0af>e-8;9;>OK4(hY3>auo3ZX|{D12q+{s!e{_#903~VQDP{?!CN(i;y z#KSWU@L;7Huj#+!rjLmS+D~SCSQW&z1EfKA2=r)XaQ!DHG)Puypy!Z`7N-pKiqi7o za_121an#T*_+r)>*zcVn8Vh2G*GOMO1QKI(BHjzXADUSvUfB>- zTz?FRX0q54(!5V#S|>fWsAy_(ePvXy|CgyK8)uIF?emp(i-pm&S1X9ax3vepwaSM}LH zR}j8m@@j9*-G5=0h$sT7z}FV#X!ffWy8u5 zBi*Vu;r%LSk8(_)p|J^K^nC!r$w)z0fiYj6>#a3_Le++Lq&MbfKjm6$% zl(&;Z#d?Ci^P1w|vJoSX*1j~MO7t5J*zW!8SH0-|1Kk9N^1r^9C-fk7{HSBRCSIb{ zkdM*)tHCo$KHx>aA0-sRMJj)NE4M`dTX*>@lXn_Zla-tN5|+^=$6ZHR1e=s$bgsGy zWffc&VpKqsgV3fcs;Q|tn>N(f6W%M@BNE_ID_~)o*gDRtdU2JX8~=gazWOj;{WYfG z3ZDc`Vd{G*Q2du(4omFD-<#@{^GO4O^0s987|~x+2G_z@wo^t^{Xqi6lr&0)9_AVf zcvky1twJsB2~~);eLv7j;@=BH1`I5oH6$At7@)zCceh+@@mSH%&3-kjyE-cI6W?kY zMO{jwD&O>iSHcjzVU2~fos&Z(*hCRlx4brF8GRkz(l!Fr#mdI*V-gw}9w#{lC)7V3 zlzEz3Dc}C&30mMM%G)B#US>D6$TpUL9i|gMhrEM_6K`nX`*)7NB!8yWZ|sWjU;*EH zY4S&_o|&={jdaOW4xF*$Q#JIrmMqA$IqD)@|0b~7+{7&Jn|0wz5#HO@$=6fUIv;(> z7;d`OM5qXow=DHqOkI>;ZJ=WhzdpeJ^Jc_j=xvUi>6TYSql>_;m^#Ouni=uH)Tx?D zwX(p%Ahd3Lgjvw=qafXLcFxC4KQmpFQtjVI9uan+`U;n_zQt^nffu+RMD=!D;la@a zBFZl6K+%`$XPO2ypiulSe(leV4(bupH4$hs>Nh>%U6HmmDDRoR#96BR=49H6@GqfA zg+7{Q)P=QV{r24fb%9r!S&%l**wK?`?{uito4fI-o*-%BUyIb`Lls(utIfwg$QMmo zWhg{81r2)aAl+Myz7V7o>&4TvTT}vKC(x#I1ujh{F0RFf2W@`Kq!&zO1`Rpc!%U%$oU*_Wx8V3iHGE9%%r~_rk{4AwranfeeCpL8$aNc6<}C73D=%`#Joyc zmw#i9+JkDGUW8a}U~oNE^}@VmH>h}MO_U`H6~!!$P<-{{fVG9*3(G7e3RNY^LNO`3 zvFx)$*Ol5lliVA>-N}Aebdw?8PTSoMB>%GVhPsXi%ACVxLc9!HJYGtL@|3q+yT$uZ zHOoAUBuiB5uTR>>$0GK%eU3WkODBSn8eQgA%5m53F$!Er6-*3gvzaJ-Qu<}x#-XO~ zg3M-kM!%(oe{nQTwUcP>qYzi{Qfg?aOD;&$8*2Xv~X|A_`SuW_JM<< ziOEQo>wm$~Q%!eHO`NLy9spl24zb8mc#eX9y5R8{blCrTa7s=$-3nTiVP9@RX3fmG z!RXp38yBsvnVuQ3ofZ~J;{(0{o)+cu1KgqfPNVJ=xb+6WVOYz`_s`}~PV7f>J%y@5 zmy_C?ryIV4wX5F{D{Ar6^8C$?)4PIJHF6Wzzx|#M_iy^4YQ-S(Kf}ZhLq5UYC$F9b zOw3h1_T8h2H(m5pG#RtMBey@I)6&X!}tuiqC>q+hJ2-T5V7N00o zi9Xz_DZ)&1d2cr*CSv8M_2=Pv2bBkVHyJPpedi9*e8kZq6ZtmjVVZ=CO$?-6eJa3i zz)^?=o_wfo-y%1bQkaV7?|=L>xLw2BtGB*d^qJe2;U6A;Fpu`a=CvL7?h={Y2|8lu zBdIoR#!EUn5&6bqF`=o6#Us|@*OGp66=~s6@02Ewmg>UQok~+74$h3AC=r)p!tVlu zK&NX)tI~ov3gvxUmHDjZI-0Q_CpRmmcvxh#+uYECUF>WSd2>-l%HQ>;!_9jWcQN-r z@_4cn*sfH*&-gUmehtJyMX-=TZENXic@Ao?e2jZms-;1opRado!8IE&1v)!Yb;rNv zHody|&Pe^W*>PRnP9y_i7DX%-1-F(|I@4 zc^?X)o13B@B=No!y&}2qA94B+bQ(Uou%phePln!#Dg$D$3AgeQR4`Q}i_FXaWAK^Y z^m^i#P535TnFQ`9>|L+fBt4eqU>}oTq05+(u zW7m68G<|00d#RH65>Sx+GXzdWdHt?^9nIBODZR3dyt%`6A8bz1faHRwrS-2u7vIdW zhbTSxGx#nzW&XIP^|f?Vj=z1@_#}U$ati+FQjfY7WM};7^{3J;hre_$HXD+A z`AKxxy~?28DtPrla_*Hw4cgc~L4ztYz@i5Nu-1_aO^ zX0Id_AaH*W{Crrmc!M~s1zkNczy!W_4*iIgsk87kc}I@6!~e$HccMdPFb8|^fwr%!JQ7rgESTj%%*&9v@`8(IuV2q8d) z2=@pKit;LS_HKeKmH&*NurhSTT#jAyOM`B}AKvdNOuzxqh)dE}7(S5YfTT0EHTFF= z0X(5^uo~9|uu&7k$MyQNy@kG&l|K@wq8X*vI_S4EuW&x@@-iOe5}W1FBo zN>{zI;Ct}#AiQHBhkZyq&|??}uey>|(Fjm7Om~|zCF0^na-bc_} zqRDpfQOJCs#qn#=&ttVXrAbWs)ExCap(*b&AL9Y-G8aMH9mi;dXf8i8T)bC%=y4w{ z7(8*p=_q#!Te)%P5D0;%XuBueWnIpm;Z@ItBl<1tWDg2Jb@?n~v1hu4t(=O(RXSo+ z8UV?m`Wcs#*>N?zBg9RVjN_hRar?D_qX zZFIy3Yu75^nL~XZ50K{I!=HyW-vu+i`_PrIFYQ$_z?HglGOYW#CGoEw>Ug)h6^91N zkjttc5KNH9_a!8h6bIH#n_BMSwZVUYN(1~`5qP|K<}Pm2fqnPx59z9eGrIXmM%@{V z*G0#<6i5c9QEexS2wqk0ySMz@M)#R+WuJ&!*CaDu8R^o&*=11hI%4ScC^C)D(Kzw` z>tbDNrl&+h0mU^EUU8^St5c&Uc!%oh>U6taXIn?wH5u$FHN_8#z`b#CZmSa#d4W`yi%+s8jDmJRDkk<$L?*-Z4^uztV$Th#3rxa8aQ0P zB*r*!Aft3bxJyK}cf#ji)K`w-A?oIxdp4SFR=B{`?mMyV`QsISB8MXm_EnCExpbdC z^?FC(#r5mF;ZMGdXsfIfCo3~9ts{iMHETv{qnp8LB1|*_GLF#1%zj+{3sT1j+uHql zb(4o5ZT<1At0LS-Uj(t*jQz(-y8}9j?!Wh!danE;k@{qZam;?bhTx=Wq303M6vehA z(~Z8lnB9oBB3!syT3g+^Yv612tPbkIaCnnKfon4PP zZ)taT&t)YfGz8{fl-d9jxL^v$u3hcC-Y!f+F96yMc0-rnlA7e(AU1YMLgItE!n~pm zZ6S5Q1#l-OJsKFPLO!e|)K|672Q7!PqDu!&O+no>oltQ6^&9n4aEgLbND&GV0k~>l z5(Wq_-gQLqzhZ~v(a;Efx=Jt3FZ}lNf5j=qRs*RR+~)}noHX$^2mQGL3T6ClHDH`!i~e5@;TL&6Ux-$S*LVRNX zq1+2Reg0(Vv5oUhSgHB>`NT*%X$nKINFN?HEttss60XoM;XWr2O&a}RQbRui0~MHa zR)9z|L1$DUI8VS_lKdGPIOZQXX#df#b7zc9~Q+7#=q+~BHow)A3ws3o*f?e zIZ91+1DwuLL%dmz$5cxOR zVfsuyB)RbDkG&p+Jpsq4E6yc--jJ3P7Z>Bqu@kTsB?l0U)BJSn;k;Zl$LYuS%SS4G ztQWKWah2$|iQyUSvVg>fz*dp9K)(+zR>X0Q5(dIIjDsbe`Wpz0yL{cQ4p?lQgV-BU zG#nnnP)ycWjxGcSCD_rpX2V<7VZ>n{-lBVy$^fv9Yls&$Q0h&HNN!lQ9&~|lVz30eIPyxcT-h8PW>z)tI{oatk;7mSKaL#L>hZ3Km?1O( zIDwA;ba-LdwCP!=7eqCDELmej?Y;Rg+^`0GJwM_d5x-Vq>_l9tyQ6-BTy%7t`~NS6 z;p-SB?ix-b7=*6>8ezm%>ar2B8f>~5b0|yb~99` zbun(0D)Y;Uj-5GjZ4WCeD=~QK0xJjh*f;~1yFo^V<4_h7hT-m*ES2VXg-d`ls#J|T z2U@-FS)-l8%k<6)MpdRl$vTT5#ZrVN0WBDrfTv4+x_2VtHzW^iRI-b4?3<*_pY@3E z$2W-GH-&u+={I4J0wsT|3JIOCgcY&CK)=$=7V~u1>{xSxP6_ zg|O^|$SoX>LK3r^4Q4W@2Z`|9i0~Qf0~igT~f z6zn1I?w{rjcn%EQXrF_m7eX`)aI-8c(s|R+SEUbJ8_Tvk;4X6+Oqy6B|C4mTCIy?P zD{EMd+hP8NG)IlBeWHN~6Px$l9~OjRKMC0-86q`l-ih&E-WzA8bzFT_)7D>`!F(k* z4-dt|9#42+_~GW~rvY}U^4ElaF)-UQ-+0?>KPEh&eC&c{mhA>sQ5W_};wZD&|G3hv z9OtimjE}mSva;aSTX^9K@^s?w9qMN-Aq8tzG&s0B{r7gl1&WbZ`(cPgY&cj;FCkN{ z?Q@bmb!u<2uaA$3dI`|PdS@4xH)BF6H`-JEc9X2NBUT`WQY}9SStgL zE`vZ;dy^*Y8cipvc*jj3Bf@m2)KnEi?u8w|K#z0-og*95!&|o4;gk_@Hbw1BsW zyO5ZRVKYmlTHWq|<=Orr_-}rycfNoBgLy_rM+aBCiuHf`C{n8K-6V%lDa=$?1CIHRPs{@PyuM|7T9K*-cK z0@;kI8XNKH%e340nVYb+{@XzWN-+NHLvvHR*awtc$lF#;Aq0ji#`sY>VS$BkcgDS& zm_7X$HZGlQM(?qi_7#xXFT_sKz9HGTDZOFAM2had&xB}=_kOMW*sE^M&)|tfEaT9g z@u0v7wmYBAzmU%eR@ZGK;MOrPGI|AIUYOpGZ(rI*ZHBwtzSI=eWlW9Kwe1wdVIKh= zjus;?46O+@oCZ_Lo#vfotho~l%R>C4Te2zdgE+jgS^FY*f)bBdDDADRt;t*vJl%1G zM#BBf`TOrvsa6#oq}-#YcR8&OP4*e=y-JKR!MKP(gspBEn@LMcKZeoB0iEKl#4wyD zmuYK7{5oo@u%<ln$RtBLNqU~JLd}f!dU;8GIAVpk==@@J|&73byPE8FrST==`VBqX2Vv~f-aRX+G z;>aS)-Shp=**k8?Oq{DxAkbhHMp)t4JbI{hFf9V)OL*`}BkG%~{i0VM_m;Ql0gW8q zSmqTizJ+nGloPadk-m@#0`QZ24igg&OsfQ40OJPWQ$p-LhmaXZU_L#3ck`}^U|=NS zZNqu26s7)ioMD~V_i!4M$MXgd5rw22ur6!$CtNfcKqP86P6ERJbw+`1nDMWNvw{QJ zpCx1QdbwwoX%G>SIxQ&UN5INcE6;G0?O|H)>U>3dvH02exGN_;2+*B`ql8Eqg(S&@BXWF|G@Gb7cT&__^qvU3$v=w`lM4cX!wGb zC`ocvE%kSUqwPMt8b3Dor-svenGM)=vN&Q48=@%r7vtCw#w5pomeBaA9?zm%BJcJZ zG_cP965ZqWxLoFifBH;kAbKeVugQ99xZ1Gc*Q!d_K#ox zhoL3ivR|DdJdJ-&5dCkT#=+GitKP+GSuT7k-L%+7RdZsANF{@sqGqSW&i&hOGqqeb zh|RC$760+$H$;BfYW{z{ivR69`42bznO4bv9Z#;PnfU)|eQ?e1QvJ0)Q_i~2>Zv_) zEsn2?{u5N5@ zOT?-#ouQ+9VN5-CSdRSUX17*dRm2XhO-}HkeNvhH>$^>&xadd7s4?madirX_$%Ju> zg%1~xc5ThpK)PR%WT?8?DFD0}PUMRb#y^Q__`w#z+W7AcFkg-$F&5yUUg`523Xw|N zBPzX^l7Fo*lO-OJnc4XFus~Ezu0r_`Mm#c6b&}#3CX~^j9fzOKzG-42q5A$14EiFu z7HTfKSV?8(o<}GZHay6&HHcHXyvYftJW}d#v(c0D9VCDq>FfnC7~j8Yr+P^!<%0Mx zfTD#J9{`b!_VBNz>!`+qM+YOH$jl`RxBR$Up!J`Dh1vNZ(=`6mj?5bn8Idt*FJ@vA zG&=?NNh12I)T>1K^;oub;S=dBxK(e7L{x^uUZ2$HHX}XfV4fN{n7_mZb}RRhhO$zL z`yd{GJ9px*S2IGXyP1K35W}72CrRB(+3fVBBR%ILbwm(Kbyze>Zg_AUUT0+(+7+L! zu<(U%8ZQ?x+W-QsB&N)y^4q&=v(t8rU8@X}J$d{-19kWlGO7>xo%WunP1P;tWI(gO zU*l`Ae8hUB2N3L@yuGX3Fr|nLN+3rfnTiHgL70YW`t@rEFg?iW0vSkjA1;Ei6b+LfDsb1UhQIXo8Zv8QE&wh;8Af>8nB-wYYwvGJBXaVi4wK*SB3TF}`c(PKn|hJE#&;gFtwW5b4tsh%GS6 zgBI+9zaamf7%0~zMUkl;;BbXW2OI(LQ;yt_Kqm-EntB+~MHJ-tyx_1*AVvBer2#bJ zGJEIl-MYSuYG1LaEC3$~DCj=1c>~tPCI}<-b;ErPs$^N> zMhKrr|MQz0;Yx8A11<0-tjAhChB1$dZF$dcaAPgxE$JYV1CYkoV+gquZ^`_HIC8nQ+Vo%!T>jpNr+e$nzkNPDcxb21&?12gs6Ak%t4fpZ zzX1$I9~vcEF#0G_YAO4v-pnG_S0IQ9D}GB-ED3im4ysDr#XFG--ADC>;Mhi7YEfiD zva6$OB{2U|w%ygTidBqI;HDBfIvGW`YB&xl=iqDLR|msZ#x6t8gx^KVSI{j$R83vI z=-Qv_*AJK4u}3M)TA$+U@h~J@ga=MJ^(~}s@rnrfkUs8r$#r4z@9WMQYuiGi2zE3w zcC6Qg3+Cd*#1qg()~i;h+g`iGFDk=kG#rJsH8u{ftT;~NJfq%5r^q?DB^Zf zk&C$xw3qk?*v#(<#~u$H0We1%Mt+ZoKrKTIAO3rt)D>vI5%46+rt!eZL`EcqK}7Qc zR0vy(NW}?STG{$rZ5t3GFAN8ng<6oO6ibVdnIHstpkg&kCH5kMnGLp`A;ft+g}lP`s7mLh z_WTz=HjTG^-?-Q~V;BxX$~!Q4aI@2eX{9IyWv&g>GF?CLXY6mjO3qc)&WQ8f?;TAHB^Zj*F1X?$?r#C8_E<+iY{eQ1W+ zeiTx=5CqGWLsl;6#A;M2ki$MEKeC31CQ{6m6(Xl1CTXX^tco;1b3DVlnstQc&ez|2}`S+j1`#$oMSB zcsyMcVnl6)n^dz|`VvyNq z@3d-jAa=ea4PKdn@Ic>@lSIaozzzvfjSrPq%InJ$zmDvKkAob(@W=%JMcO-0S?Oxq zP&Sc(g)7*lsHlhof$!+dR2B+HG8usq26q)QIReZ~S%aC6Ct33K62noD66^#z7I*~G zN2(-BfGY<~q8tW@-hJcqw<%tAzH80n)fn6njB=m@a5Lz@Wh8gOqkWO)PkNm!w%=}0 zI+6O8BO9a#8Mz(tkiLM zZhr-L`Jj&TDEiKRZi>kx2=2W6T;(vtrjo*^2 z{+xgPZ;UQm#U`7!pB?<+spCgYD#DKtQ7~@>3I#d9N`&W28aRL_WRajWTD3EAYi0^Z zNG>&iXZ8BF?|8$+id?RaAAX>yvZzC9@TrC|{7!1B&1iJ0Melg&gb_Ha#Dm>3W8sT4 zafDdno2Z0I&1}{3rtF^^x9qO>DTww z4x&-F-vt6DcLVR58l}OSi*YrvKl#)6b|?5kZ0 z+&De7{+VEZK)P3oakqA862z-Sd5Jk>l1Kpbi;N5*7;+t^hi%IdO~Pyq=4xP6lHI>b zg`RV`Gti_zZZS>|z@+57i*Z+gF@8!h)d@U^1i4g_C&RU87=Kj`wf&9ag(KJ70mqJMvpL80CzlXxYQ)CIurv80xM?6MZ^vl|{{3Ry_;UwY)ThgI^&`q-DU ztkpih#?=ZxI8PrN;2q~=^uEn(z4E*A1C~qO=Wp=Picp_lU@G04ceMuZ8<9frnr9$W zz0bfgD_6s`RP3;W;S1kQch&abECj^!Am?I4UrTp)golPT6*E^9zkkg!tm2vfe7etn z^%!xSeYY1SA+bfmqz7?iIpx^frvDhTw(VpsUyaQlAGF%q{js?!GHfjEMjI@XxE2C2 z-BoscW2A!q3wvznpuhh5S8qd_A{L(P?2rZ|W=%LJ8DcVE%{w8UA*{jzhc??Vg2B1j0FF;U#WxQ#Y&CvGa`RoLEdHy zmL8JCksBC$4cVuF(l6-gF>jA(%@*Vn=mH^gBMHRV_;{Ncb^a>p#FG;0&i4~m2LM5W zTZ@kR&rGr{nW<@?buqt+{3^W zNMirT3qAB}2|djuTPPvR(UFe4_k-OtWEuQJVo;s`l+7IRAJ}p@8MhAl*Z63E3>jXK zQbNW%FIhr{W_(0ygXZ*h_=_o%$VJO>P-AlEe)OdDG3%W`N+>BnB%2($ z4V^k#D^Dn7pFp`PgOq{Pd}y@lVj2=Y!xgKWpav1AJ@<1|jiA*@dZM)8-j4!`cWUNI;%fBxxlD!K{8MK2;c-VLV!N{6BMTZ~^~ z>I$`(o{{eqid;n;^5oM9tPiY14t>_AuWJq>aaLj`!6na9dc++jLmP zdb#L5@s1ThlQDQX`i9i_yLz)G}6vRlRj{WL>qRnf~ALcAh6{#Rje`%uOk#Z24nY|6aMox*;(J2#a@aKtKSs*tj9JvVJrMpvmlljk!H4 z5&bK2x2*ZyM%VRf{%Md0=LUL$CQ;9P^8B6gcGx+brm)`3Cx_X<} zEoO_HKcj|JKu2iSMIk0*M1^(`W zzA&@N#nJ9O*AC9Tm-@Gp5(IEWo?fB^t=vHdHp^`_k|zudB4KK#OxP~8ir5QBr|==WrJsx6uV&bzRwL~E z@S$yo2vSU1Ph!$9xPvZbadx9vJ&p-{@7$~%1Qo&%2sm;_CLgeKR z6g@A2TIr@{5|Y|?L`_-4EsTtM%p^(99|K>DT{F{`MsO9@Pe=Rx*EVvdWG=j|$3lPf zP?1tvBUrCryLJcFy5-49P}JU=71>9qrNjgLqENwH+v_nf847S$uQA=Ktr<&#qT38C z0$Hd8&8Yr$z~|4OFHh`Wbrj9){VbzBwY){c3yN(;WWWc#X+WP6q8Cphw#LN zwkRCM`Ed72-xb0$>kx;B;wX7`0$BFXGG%(lJEe*e6epO z^tE>|@fCVf^ai5IV0eTh0z(iwQDLT}D|-OyKuXRuhB*<#nb{#vC^;DUjUvHScnO3W zgCX1r%8Kw#O%;1*jPz^s6dOz}vf(pX*FToG(}iNyhi@R$`I6?B)|>CGW9R!35)Qf$ zY!^hJ;LcRQHf$4jYXPo&Ko~^knTqjWjSe$Y6hku15=Rg@til?b!OD<^aW$DSh%E{y z^_|o$$gu4I)rI3(dUY+OI|UMeJW|slo>Rfc5k;V7oRX{CSbP3yUoq(Ssiz&j-D z!S(FYp9AcGR-u1>1_NVD@rE!v`XK7CbEHVX*-a3d?MIz(b_;FM*i3gQ2`550a0t@E z0o;R0J};0)hmMVd;}n66Ff6GSK2_M%G0bT-n|QhwCR{2*lf; z8p=j0?L$8vQ)hn0pT`r2X8STNOOfSOA||s^(MCH5tluPQ+nck1s`atW8HFq&fpl+! z1qSXM|3S}4v|$SLfGgNwK+Ak3JoZGvv?{;4S`iTlm;J0NTHU*A7|hCX3`?oN1P|nD z1(lWZu=2ssx>8VUkY0qX%9NSg+;s(mydx-r??Bdk7{5ge&a|7r>S(!o+|iBzx5}Di z%#dk`NGJ+#HfvGLze8_`j07P*`+%xgZKiOi;Q&Ru@nG-yWr10>2C)i+sxAOl)nO^SNMG z!${iQp2Pv*U;V;I=7Y%r7kPDn)>=R8R8 zxCKe>k^5wH?XzH!*%csub~}_q+tz`(7(F|5hmeI?Sp2s}|0}P7yke3kmQ8Iorw40C zl}sLo3GU@h?~hI(KHs;R1l|}@L*~QHwsUn}U7lN4-gBSIT!;aa8#S(PU&gs;z-igx zR0@X5$O6=nM;$liyF0k!Cc$ ztDnWUWUhE4di($1_UJKJh>$0h4b1Gxj2qN&ALe!lf}693P45B7L3TWi*+6c^WZ4LL zKDSoAD+T+8Ld8BJ&Iu;3cWfJ-zz6)4j_K#DWRojYc)9qs1;bY7{WGvE6A`uvdjTD`$uDr!av zs0=6o+Ve8BdAEU^>?>>~LmUon-nvByO!)6{9F#K1YiL6UzzZg{XJA=X1Up0la;zO> ziC_|F!T&~|M0xs?qmNvYfijs%4@#!wXd-S#7XNQO?7Yx+Ag4QahAZB;QB!6V4q{yd zI*#JN47i;IPe`^{pkF}7I$^3H0WdLsaCY_&w%{-->61%VFqGny@CmD)^{R0uUZ9Sv ztoN!)UEF+jTiUUcC%tpY*2kEN5Ja9v+h{<4@YSPXCN~;45qJzmp1dbfDr(a*NjZw+ zMuuRYBZL@Gh6qN0OTrOui&>JVOA-ZCcNh@67#%teF0Ru!B&8HtkToe_wjeypLXb-7 zqf(%1|8@|QGRVwBC|Ba`ghy&FF=FH+$zWR45ZH-A=Kz3K{Av|hcxZ}To-_H&TUm37 z+{Isj18_)EsQ3;=5YSDr#*9xL;!Q#da_>mSZJgA0ZFs6vWD31HDuWE z>uWH4e*X3AS3WFl>S)u-si~=i9*u>Dq}CkW+a_1W2kFlRl)Qpg1Hog5_U|tf7R7;t zHH*QXetx898a`!5An9uoyiPJ}F&7%xzo*L#xMIi5cgdmSkC_gl>QjqTls)%r(L~Nh zMSX3&D;z;^Ps9b>(4sK2Mjr*oP_lQ-uG4SV1=vJHI@HuL?+7mb~? zCsUcF4%CowtmJ7?%Ls)8SyX6*JD|2J*gyDDI9y;(*LhJu@&eh+kHxWZikg4?_#s!f zOSu2*fJ4>@Gp6Svw$T{=_~Wr2%kt5x2>>YN@|=(BKoHCojz(=kJWXJVas>i`Hgp-k zpTja9LX%NjFTZ4=se1h*x!BL2mF};jb2J0a@e%b2CBz|#*|Cs%D6J0%zC>d0t=4f& zT#bh~tyA(5W)`;?|ZSJQR0M#bPzaZ~g^M)3R@sk6;ItobD8l zCbFNwADH;Qi&xn%3qHbTl8C?Q_w4BWE8%BHE^)8vC+CGE|)?@ZB&J0gqBf)J0iUj zDh)j=^oO((s_9ymqS@T`9W`a*`Rv~ae-k<369g_$UZXLf3xbPNG^DZG&S9^{v6KlX znmVe}N=@kttcB!VJd z=)>$+`|5X0c@0NmeQ;b6`{FilJAv%J4pd-wWDDj3p(+LMj_x7 z>NUf_+GMig9&emgDQv^gEip2^`SeIVHbJ6EtHfYjR+dA2wjbUVLL^~YTtS2+ttUB*6qD0&UY*`=Tu7hFA z^CYn}jhJ(Db3;EJ{9?7y!&S_QS;*ov+RLQ~+iu|d5I`1zs=zkR(RXlgu%L4-&Ns{y zM^Ba}?0^%Vw4(9k%gBTwVqFAyS3r6@9`8@eVJ9Lm@8FNQkVC++h9-{~@I3O95E^9B zfn(Ub8C)zyWEET=MI6%ufp>?iB!=S<@CfL}blEU~dD*XmzB_Zc+FUF4VSmhF;llS;59zSQI7#*M+rNCuxoU5+9|zf z{Y9f@wIe9mx!6L3g^sADn$3~|vBO$02YhC#-+}bs6taGQWAqT_ve?Gb$SKGK4`d^N z)kTlm;bnwF1+_VKg!Jk$kgXD3J4!o-S_ObP5ROcXTpxiWKv)h=?KyyJGjRFI|J*!) zjVD0^87PQMx6LQ5RH6kYCjqLc&lE$#f50h?96=UI8qrpu^X!*+XR(DgYE|ojd^S^C zDk?b3U0b?Dq_G_U7yTdh-aM@5wOt?nCgT#BH6TNV2AU(9D3nYMWGXbN6e-c1u?$U8 znxhoWR2nsk=0r(JN~Bq7*8HAV3+vf?AA7&YZ~ycD-s5;2dq2lo7T-SK`*YvdeO>2u zp69jOow6JJgO0iZe%jCRiEY7PSiV6iMncM-bC_LYX@g`|e)yyn8V!gEibKs6kzRDz zX<6~vgo!AEQf2~UY4;LzIp$w$^aifY?#UJ7) z3oK%E5y(f%E%g5HK06Ig1}FzQgL&4yXmA)$wRfki0~5J6R-`gBGta;7!iZBxG_drq z>3)7`6fFz7mCw3To6&&#C#1 zdhV}RG{iJ&yr?WiWRC zjxU+xqx%RG1IcisM!MAjbQunSJA;J{!){(ZqmQBEj7%;J(DY{0h8qwCKxQX21${{5 zJ#eq{KpFXHs1C*lmd9`)i|@^>@{Bq%djRo~Ls&;4HD(YIjM@(x$tEk__JDHm29|?# zj}m=sXr!kUKT5yIl!2oDl8DRILmA_(d^5xeLnaTcfotCcQhEHid5}_x6|>sGEXd zVm{EJk^b|`ZIJbXsejx8%81M5U-Z|kY@jK``rIeg=Yd*gEhwm;P~W1gegn_DpjA6v zn$f}~b<#T6KVdaETP52u*Q7)6pqA@eaF;l1Q92G$4x1~}DJ#Nq*o$+5f75|Ts0 zst7?qb|I?&=7FQ8i2az2$h)@M^ylQs)@*|m7Av4cQcfIJ#Qe8-7LfyM8CmE4uIX{` z82SFdFIrH{poNqkiet=+`k&ATSS*--&SE0jiaUZ}xVon0>QAFcy7Q=C7BU>~2s!4- z@*yT`L9nx=1{>~*A9XLsMSywhm8HTWd-;x1Ah?S>9?xzv_K@ZIjAwO&G-H&pVjoHs9KiiQ%@McwJv4(0QXl zcbJNPn?M{K0!(K$uBJQnj8SB1A%v^u}pD~+$2!d)vhWZeatcJh*5&Tsp z-({> z%p)3n%XJYm)<7fB3b`Bm8)~t^+ugF+obpr9b#xMrfL0L=fHhpF!2PhshR;9HXn8=R z?aD?k2}mhbVp8S=k%hYj*ATV(?i!PyhvKfD-LhHA-x>nFW{|lc0_fhZG{|1`ikQo; zO>DKpje1c*ekr{(Uk1jch&-LlXwbxMz0pf|!aszenJK#}95*XcJaF%e-c->B27J z;1gq_|65vMnT11-Zq_>Gm9H)k4Hm320CnG5TO*M5ha+ZvP+s}@2dbKIF!GfqI!G)6 z8e9Hf(1w=3zF$t`eQ2#YssBcR3$6wHGKKtk6QYQZtLk5^)RjUKK&T`U0PlpkN)foAI@^=bk(HGtAGFHN zi#qJMCEU=<1^^#DaSZR8+%Lk7;cXmf@nwbxccxWeghb;<06$;iQt$^VL1^lIy`{4m z*fli?(Je6=iD2aEMM??WUceK4>b~tkQ0(^8`-Eu^_GKHEi3&ELIem5GW9pDH6P#}} zIPa3q-#TvDWhNr;-;`AO$E-YUk=pn=Yo0>n*ywN(YB)T284eP`K=`)#KsL#(1FA{t zbRx*@iHW)PS@s~%raRo=y&=PE5Blzf4iiqJqB~aO_)7|NEOOsW@tqWYh6$~ZdA>n3CbuY3uTY@E`ih~8 z3}z!@rA^PCg-|aj88R)o6OF*&I!(m*a{C`e+VWCAumF*dd$*@hd`IQgs11Sw|IB_= zF`)!1pruM^CYS?Pu3T|9-lFw#dO9c}hgGIsdUn&*wGyq-)rutbuQzmC{Pd5AuIdh8 z{jG=>T%_{dC1gBts>9V?K7iCK0x@JG&&Z6&i;)F_WF#?W)v^Q@+c zaNwo7D_mFo+FjUT7jiWF)7pXB0Bek*$_f2HqL-qkr(qw|`vt5ZI?!v~SD{`3l#@T4 zS>eX7q!53Le0aXxZvkE0<$oz#r0~co82sHxW5CNY-WMQ8i>-~3Qep!N1*UPp_bQ3uP zrKkU2Dl-3kE1)0$lRov|KkD}u`Ckfr|8H49MVO{gUW3Mg7{VUagD~3wLW2K8wP}+& z%!2D0w2p6`odTdhqEQ$D2XU$SW|1X5DWDlKwFy=DQLEas)+98x;0H1mJMe zyA=JIokU8+H4m*FzbznV0$BI7{=;^()yfU*D6LWo13pGItHThAe^I$~JYP$6Qf=AB zI-7@>@Dl-AiF*jjZ=%{#A2Grz%eBqKMQFzS9k@s=w@IrIpPCY>?yb#pIN?Cy}QKJlZ z&e+)SSrT24&dL3I8)_L}kO{fLxKRS6t0bCp7WfiX{&3@Y0D#E=wQDsJa0nBvj!XaE zhft7<^mCIP!_eoetP__dV6~i4?wZ?Y0gK&)-3OHbKm|&gkbOlUS>yg#3EpWtFWGp;Ut*@oFXo}}u`Pc-yPY#T(AggziN|KgBd}uxuXfyb$jY9e=JIv*LLnbg zf@~uJ0z^e>ltK)WsWVJlNJ#72;LmKMt$#kML3*;acpSuRJ;|V9g(qS~(k++3TcTkE znG?#uRpfo}jwok=&fLdT;$gcMo*0crH8PCVii9wwH0bRZg6{ZE zW@aWg4+t8dMYKe4{`sjuJxX_=zyFI-n21W4?nG|#QlQ+#n96fRy$7v_ug8eOmcaPM5#XiVX?>1TsKb#aYd-XxD=@02;mc zyt#9O(3HYIvV)zc!FJolxy$>x;IgHhUxfxv@NX8+)#`X*ao)%&ujYDJb28iQl$qK> z!-CqSI{;EA61KsioTOXtK8PThiU|1OWcZdGM+is%B)UPH~o){ z#`rtvmk>Y1{rs1gGoa5}^Wg%u!y%9@&O`K3D9c5b3Hg-=H(4Yh8!zaGqzdVnu5<@- zm>os`1e1VORq;Up%6;;P(2f5%KOpHyG;xduZNToFxBj`$o9CSwr#HMEBp)Ymx|PcG zAg-a~$sLZw3vlz@J(_<$kkRgn?BWC~A?8pzb6agV>6U0vz6Lav)tl?k9XE5RWd?+D z5CIpxoM3<*gl?jP_7sOA9IvH0@<~P+a3OBWN8do5>8`bhXiumJ$A;_1X;LmAllP#H z5SNpvfv7CYL7?OQbiaNhSfIN{|2Y*|8FSl-j_HJK+yBcT67|!7V{QgU0(EjfvCkbz z^ExmC5uw|#rdC8zeunxt_@2P=7!nsBshq}T^8h_9(BtILxSObp#ZhVvK@f~`HbCv4 z9dt<>6*=c`rW?uTP(!Gr_3VK#1%ejQ$A-|CCpSxQ>e~J=-N104Al3zKn*t{gVyzL6 zkWB87m7?A__~A$4pL6bZ*E@DCG0=rsavs?xo~_fwmr7DR4q1i@x@|^fe^NWflKI5? z`u*$Q`O@!|VEiXfg#W>%x4Tat;PHA4bTaz3E04%w-Ihlmcj7us+h zIm9IR``2E9lQ_G*X6M>Ob;fmC`2XjQl_&A_yqQASJ z@A|SG!>Cv88Hhc^@5A?xw3$`+wt~y=v5ok^OUw$k9BigeNQhCnu9;AIOG-(p8r#G5 zrzaWYcY_Sp`QmpDvg%#zu48eF|D9G}?V4USEqCSbw3<{urF(%pPAFt{Cm4h71S-V} zMKW4L`4|`X$4eiYPOKRrp*@@U`S~jnVaK747S@EiquuXMa_Ba5E*9ip5@s?XY&ZY2 z3i(pg``aP7{4Z(oy^23C@RGl_P*uX)4M-@dkq6UTH_(Luh4%qFUt_QsfH<{?QT0GE zuo)&HWKqPta8;7_L-j0a!C(5KXT;DXJ-^GY0Weg(b-s_PF%QUAz| zKioV~$^|aH^`P=?Ol3-HU@((D-ko-l2 znfQjEe{*vt{_X$dLd}<>vB)?K5{q&0jco$ zjy6>S3X$SWwyeD$4E;yT&t{H3Ldwj8LI)!KLMTWm0ul&~{%k6G1b4qApjr~_%mm)x zSlb5v#FG!T=!AfWTe!bOcCQeN98&9dFz4v2F)O8^%;-lup{PeogB3m|M~J}`bvXV! zSQcu-sVE!4!3`x>B=q!QZ1@C#GI(7pkb&<(=-byb%%MFu`7C<5)P}>i3IfZI7fvGJ zbrO}AOoC41g|~E=0l;XqP%Ii=|8;JPbF|&Ggm1&O%pjP)`{hKFkER&e8}|09p&JVV z-bIk+*5PYbNVCP8aQFZ85ey< z9EDLsYpbEA2*`w@6*F}1Mdy8i0c|Kbt3na7|IW(o=pt^gx3`z-DHb`Bdfvcy=myvU z;I+Y3MxT%orao0oYWN&J{* zX%;bH4V6(zB2Xj|>deJhFAbZ`W-#9?dw#<}erBwnx&qdIDNc zDdqAnxhZA0Mih2@N1}5r}-48a?#g_fV|t*WJ*YLpcmb;I+IZzVfS;B3N`C}({43fR;qS3vACFC5Uiz~s zN$S?3^#mHW;GK(S63vLbDZxxta))7Z0sta-o8>hQOlrTmKJNosj=d-jiNEI_LX;Rh zHzpE0Aee=Og{`AG5X1a)zcT?i$n z64>F?=_e9C8Gn(hH+4P0(Iwj=Sbizv_ME3-Kh$eLg|xdGT$~dO%6AaY1^XZPX95i3 zNfAa~Ay7L|-hP`&?8Mqj`wAEHQ`SDx%A!pPHPN$*bg}~PzvsFwCh{nK6E4!BBCzY3IDQk zral}_D1i`nG1|~!Y!+Mm)5ZNmK0gTd1fvy^n@b?QcS6}kFd~FUr9YhnKOhtQ7wNUN8sZ?@?bAuSuO$4V|@8WYrH}g9!yg_2l_R^`lR4^> zssxTYi{uD8H3eF~PV4xdR40yJy^0Zh`aUcum1^gFjTB=C7IeOCIXc?`1~K!WZUhdV ziyxzEJb&@xQ!MPlFGo5vfGsOlry(ob0eOfM0v$X4VnM$dj9(ri&(xEFHeyd#pwsq- z0|z53^Z{G0C@{HjL7cL&RR~wF`oq&7kCuliKV(td0VTDUFYZ)*J@o>i)lZ}kccT+>%hd)VSTFhb>+YQK4tEj{ zZ^HwKeN0YCVF$3XZ}j@4!&!^8yC86D+`SQ;BW>(*#p;*2xg5BsHx35~p4=8^OQL`s zU|~2zAd#Dlx%Y9#=wqLiM0Z{9IBxcBw-SUF^@v}QTJ9Qe{`4Z~sXeS4-JP&$*}+^% z6j)-ioSpp?PKVGFOB^#1^YM3MbGl29%VN}|6Q1@|tKFrCH{f%IOIdGhz#b5od=su) z-$!=Xp1S?YxTqI=M5I!O3Z*H?{#oe_#NrBCH*R)-RZ6|n3W_INJMC;()H1sC-Mvn0 zQq_L^IXD=~(YUs~6PAB_5@%^Ia@p`$X8whivRCbVp{sv_r2H0uSNtdRUGEwXJQZ9~ zzt|YpsuOYry@(50_sX^`LZYr4KZ?~+HM$6+fE;jk-Eu@A=s~ zHAB}MVPCV))m3^(fw(Ei?+MYpE6wgITOIPvwiLf4$B+nu>89E)P_Ma@ZUD2TPKO)x z5=V9?V=M!hxC-uIhTm(odxI!0;)>Fz+{P7jDye}QGqMumEJj5)k!;X`64xaM_e z3_cz3x^`U6@<*+QG9X=1*Yci0_gL?XU^?VK1Ag0lb8X<838ODGNGc)WhFIk7k{A|~ z-INrcWOENema)s7AXP;0FhKgI7cY{t9GP-~Eb!9E?;u)URFoy?7Qv|w!&D>irJ*@D zO|)UZq#?p0M?*Lyl%60CLsPd69WPc^!q&M9pl(7KGDvlay^;1gd)Mr;(E(BTmLjZ^ z3=3@y{46iDOz7&xOAie%1qWUqoMm_LHP?hg3GEYew`+Qc#*x|q(l4VpQAizM{RYbN zHN#ulaD(+Ct5b5vHTM`40EsUADXpril0R;x*tSdG=~db*bPXAwt4N!P zjo*x;ur=FX3B)D4V0ygd-#e37xgRyrPQYz#@D{X%gM44 zwS#+PM}i4#aqsNG3r1o8{A!ZEC) z{6j(yc3~uBUVG#VQ$9wpwLQk|wbrs0_OBGw;KAZs&nih%dq~R(E!t{!DXa6oG1~kq zpD6qc6bA!yDO6s>&O@IALaztlq8p$iGz9l}bL1AV7*vamjVyHm6J$Idwo2FOtkAX2D~E;2Ple%eQNthlJ$LqIhn zB9K8wKNF&6ynV)Fe2AhWE7yBRF1s)&D%N3qHW>xtY*AIGSVq(yH#awKTM**2N96L* zcruLFz0QVss?%Wm1S)WH0HuccgmE27N=f63M=m>-$Bejo5OLK4_iRtqr%$6n1ACJv zl_LbI8vB_@60dYQQ_5@fx=X%7u67#vh+;LliC}}MxQCvGruR;aHB5&o z+6fh$sj~CywS_t%t#;#K?cZ|)79Ti{;DBF-THD0tV5Hbv`anPJ^+AX(EmW*t?mSkB-FUvaY031gXnNkJNh1RL2#*w=4z)I%L{$gBGyEP^>1I9GzH%RkHSp$5GG9 z(QU950O*1ZZ~$@%qO8-L0GhRf%=A2@{MQ{ECL}K4O=AB?;41JAI%&ujpFcCMTO?@x z>Zpn7A)^g<=V!2dd0~a!Ue1|2JE99|3a0-+@Z^>j7V*d4-X`2cHzZ-lAUJ(S8 zu%gB+6{4WOX3yT+lYpq|c!E_HL&L6-gOdB$S>_)54NoR5x1DF#Rt4GtMDV*~jnTg= zKC4CW8me}sgib6gy$$z{vr#p;n|c=ST%H7N39-@1tD{tUcsDs6q9;!tj3|`WyiHo7 zs-1A80t&3rA33b2@$|G}BEZ_Km3(h%onh!#2EM`!~{VEZ;<3@G+Dtwn<_Nn+P zLU}b*Vx~ta8)DP73U_BL=HhySmC;9GOn?Uxh$y7&kuXW2aV4m+D`GUZkj4e;?8$9h zldO!zw$_)zW}Je}*xb@$uuP`v;$|`QK@}5S8l09$sEiMm+H;TANhhP74Wk>z&TgOz z-t~bMh80lRiJr(}(9A}1jfIIjD;*jPDy%PL@a+W$pop*FH3oq)q=TWzivD}%s2B`3q=|(}AIuBnvT2f@Fq(?AImCUM_{Q2E*S)0NTA5Y_UcMk#e zT^~^(Jdq0k7H%*Bk|bB6qAj-FQ+AtDs5?8Cy~V$C>yAhn4ebUVtLan*{Hzwz%TEg@-Z*AW-nlLz=TU!5GdML=&Q_oqV}q8bHb)wI?TwS0=^d zPQS*KeN~_&1X?^1EUYigIuAh|$?Wo(5=&T(-T5*I{>~@K zy=}E^BqstNfv?4mE!l2#NXP5b9==?yT2Lyp)&S)ueh%uPy@a*`r0IKNKdOY)D}2W} z&u#JQ5xE^luB2WwF6$23=MH$i7lm^nq%t^dg#HCuwek>o6sxq`{_Ul!yOPu1K=-O@ zJlttNev~{dxx-O|({~3r;TJxPIQj9z7pW3KmY$Sp+CNUxam59ol$v?GAm@tI&93Pq zs(Q5p8#L4Omo4U1YYzH8e{8nj`_NeS`1}&1K_9{TIvqO{t7^kHZ`=^PDg|=Q`RH9j z#?_*z2TJ@URIxO$paZK(1(c=onrWZ&CtN)rFJeezc$;WCE@<}-*N^zd5lD@FVy?z| zZ^V7e@aqDwh-44hEO*P8z_oEeH1lk=r)1^es2103X$FJ>yVzs5jNxAk+_*Te<#0^) zt5@48tpj(8Tags*ux9sZ_EKJ+v;_)$OaJO+Q_rS#IlwPyg(B%HL0#pAk*6|ike#&wCGp@Uu_H`t4j(yc%I2D7>tkD|#MX#gs3Yh*s2``7EyhUPk^$Nl zao83(-4oVS9i{sFl0kZbBgFd&LQHB~Anssflp9*c;;KnF06|~bC)UsUxK`->?Mju2 z9LWx};h`ZlDMotO=hy_kqNUwRGeJTEF*fTFv6bP6D24du9)cQ@6fp&jB@C5D;Hpq5 z94Q+;tNEzEs1n;TY=ZPd!MDS_N<%{<0dsUnx>YtR<8Z$9T|v$KDfuNhYbh*`xy8=Z zZ2EA4Y7^<9=2eOV8-?qzH^53ZDnEkiHX2E8iGb}W%(}^7^_KC~D_4rs6Od3Cj%|%c zTS=JZYmf{AxX3WB7yrQ##JutYB46iP2CT0gnD=@Pl+WiPjNvmA1i1iq70Y2{^5y0N zx%{T9j@qs1#Y@J;wOSFE@qQB%lekgj;ZWN6xB>_A396O>NJA3QvQgO}ICl{HPWav8 z&+8~O^`uAPrXzjUE#syVzf&Dt8V_Wds5WivEJ_55`3V8O>KNKpn2HoJQNR(5M(`UJ zDW3wo5WcE{^MT*E6mvSj*lCvX=U(oe#n@L|;xfshYuqvbVD48&HA= ze!aV)AIi*Dupu{QemVws0gwv$6rZCeGl1&Yy+eSU$95o;RP=90V5;G(53~SGFEakq z`wj9HTo{9KVduZNEJ ze4VS%?V9|1_h$fU(*VM!SoUA7#5y`z-$Q`RQYj7kW%@{W(S#d6&9gugwZz0;b}u(ii0NG`3D3e}Lz; z)};0d*j)8ipMEkyXb0C6jg1vBqrz+i3{KgGSy`39x#)IR9PsJ_NMPq4nhBSc(`R%x zkhyIo88m_srq+qgU#{lWw|kcM+!SR*eX{zdq?txc0zn~XA4pA_@%m^wcg9dFpIKrp z66E>o>wQ##7*2q}K*NxPh7lOl(>`e|5_J-|;Z5I}mN2q4;OBbQQ<2hUyo zMq?`>y^{j%T_PuO?b4JdtJbi^S={q7f@ars$X#Ubnf0b;u14kZE?VO=P59}|jz0=;pOyr!n+5Om8b zBmBv@d`OJiIQEua3fU$An3t}oRK|~I>HPlsQ1L5jra&+@OR;5W#$SK!Dsx2=Z8?U6 zA70yo#C->;aQe~F6Ghn?C?+uHDKqs_;LY&C#N2#O-w=K_CwOQ){cq7miT{p|QjH$R zZ|*?aRoqYT49XVIZ2MWuPo3|mktq541SN7VD$u*%*9r;>e)=kra4z?dyyhYDU-r~IOo%A91BT;ecr98c!29S@T=jC$4$k7tJ zpj3VIbLdyE>Wv^vj$r}^t-GqNK!0DK7$s?jGBd`H-#PM$wL=1LA)dUG;lprC?Fp07 ztApC-;OxhmzG3h#XxJwk<6zsv7KYxtC)D7dw}oh4>Jw-34mc^Nv4kXkM_-5+*fsuQ7;Z>N zE1%;|dj{p8Uv&Em@9k%_BTKJgYm$r}umaTe41MY9V4=b2b`u(eMHRZYp^v!GC>Z1d%b*D;)>K3avGNz z_{X07(x*Ry9khP`_ifZXY zr|6db)iVP+N2dPZ{1UIV^+Wt)s&;`jw;=9GXb8LUj@`fNaSSR*gb)vI%1HU-jSUQ)LdZ6royeY~NYx%D zkvfWMBmrV0`UrJ)g7*XB-$%|O@X8>z4X$XvrrBK5Q2+o95nMnCP866d@j{S=t|4j> z&%bVkCZ#%l8X9kWKq2H&-`}~b`b9>}AMfmW(UeO|E`nL~e0m>Uv*d227p9KNol1R0 zOimu>Cv^($%v1Kj0Y4n!V=Ol&Qmky?`T7@AsJ<2w5_0Fpj^E=gxRn?VNS=r)#3Mq=+2$ zEoCMMwPd~Z9u*_GP{n@-C0y%u^(F0LB&+jFEHFAHf_e+MEJ#^GxG)JXQ30w#g-AVf zFxk9Yn3k6U%Atk}8IqIS2F1*d#7|qmu0sYUCes7;=V`=TD_AS2rZv)hZQw1d74d*^ zw@v0T+{m(uk$;!6XS3YP@KQ!UoQxqB3TR3y9!2^Eqc*vTn+bhtq+n!f@d0BKbPiz+ z*bC8ylSO#8YZimK?ExAf0`Cf7oQzmhBSd)vbZ+O~k8+zNB?QI5aU#pc4i7;s8)MDs zrt%0y!M1tuZXk;iVsZc#QYxzj+n=&M^0YvqKwjz}4y|WYIntnCYUZe^sRi5ilR9|) z0PF18@5$&hdAzbVAn$$NkF@-{X50h!*dbXI7RywsqSArtR`>-)gPx?0=lF6Y+XpoL*ZL+}ZRs z`>POhQ$|y950Qn(MYx;?H`m)`Whwz*TjN}1`;LZ+*?$G7w0^*P%(cpxmzRep z8~Y}?5BEpgf}JekYPZnq)@56ju9&j?|%~GfQFo0N{s{9xFF@DjH|r z;>H|!XdYgdI9vCkZ^+UZD8pS)^=G|%x4^~41*?{gd-O&`1g~=JAgYyX$Xg$vsx&!w z?s9lI5C8i0^(`$okz1ZX<#xqPApn$?li*}De|o+V!sIEcs;Z)AYnR~ky#hpk1OL?^ zAaf}ANzdU5%ZuR8?GD(y5D)V7*)w?@LfSfzq)*0oJMyg$sOX_|D>01dn=x(LG(0tp zfn}4ilLWsdJ2cXDGmV7x-g0pR%wwi=a&jIzdX$C}(eDr&ro!%4f_C2QGZ;pGhok(y zgxOTEBWi{KEepbt;_XMA?i}RSDcb6qnl4W=hIZ*%jh$Oy#)7}Q{-;wg7D0C9ofF8Y zYC7Rs;Bn6{&iK8J=E^&t5&jqv*6T-*fT3f{N?#MZJ^A@_ZFqNWLMN&5YVgXKHLcx; zRo8YqZW+ayQ0sV%*Q^rtn8or_npxBA3>cyGcGhuW^qioG$kl*=rSKXPgpJ$?@|7cb zk{3IcU|QZJaM-;4{2l^dL@GE9&iu|JBQ^}*(r}KU4RQzE-QAz+zg;eOJ=P5P)b zk-EMCSQ>RGc2Y`83Ws)B_2Vm7Cc){Es1uG4AFjH%VW0n|xv;mJ>nS7cv)ch^^;ZyU zt~_}n0Gt04T3VB^m(I7npM?@W4-t7ot5q>u4z>>gRuHu?_4oJF14l;k!2{a~mIEd| z*Tkj{ZNBdra1lKSa1dS~K-agn{)IRH%;>{BNZ6`U%pSk*@dHGX>5U5sCD1e_B_*iD zAE4a+35S6@0RbhUk!_m`<_uVtJ_n;?D)z#c*GxD8f?>Zg3WGuUf@i zP*5`PQ%3#c5aLtjx)|8UW&RWODVg+gQIDV~fD76yJACMk^_N-NudTb53DPZ+`W* z0U1#gn*SUF;959RCqb_`!elE3ZHLDD0x}9Spxh zvU=0=-aU?Id%xIAZ`$N7lv{Jv}`mV6Q8cw)+F1GwE`6a;n0&5Vh08XtgV) zFlK0d_8quccE-X53)bz)lbL$>@L`Uxdb+wZq#wd4Q6n-=wPwvYj3&F&>T z&~IE^oN&V6K_@hTG2Ke0x2=-3v9U3s3|=Q~&z4$STVoA7*1t#2|Al5O@!aFZM4v1~waMvT zIRBqpT4v0jKVL^rZ{^yx(-C}RajMqYXOe6U!5$M^H$Y7ud#PU8xP-R7-p*Cuyd#2ot{eSOyvwcv4e$Jh5(Vc{-? zP!*7Qr$WQWg%Kvv=Q7)_u6TVa139%8E^|gi8#kDxm{7AEaRT;12-BZYw=IC7{$-R- zg@=$PyntCmvSmHv0lptTpFERkkt8jCFmN#Bb#+6_QbQdw+CuPkzW^R*+AmeA0f+#! zvCBxbzMx#k$0j;+$t(*D4CH9anj6`;c@hJtl|yuNban560|(&9QvV)SolXR$KKqVr z*)$dwjdM9Ui%LFfD}d^lY(Gx!uQ!pHFfqu$BhKkvz%(@UvJvRJK4&vCD$Ei2IbmIX zCou4}h7#OfZ|`(nR*x!1P)O))&oZ7$U;y4eJ}WkETx?)qz=+6{i^#CnYCWinKt8z* zc-%|ilo6`@x=>RJB86SD`TULH0Qr@dU0qpGpWVc{4G9emdiHFAlaT!O?JM~BW+IC0 zYWf-31zt&|FK6M$kDN&9=uy@4z$KhaUU?tKnt8Wu;e;I~S~Hsmw%`S0_UK&HGC3du zyWP7t6}JjL8or-g1D$nwXi;r{1?~)4uC4Igby7!XGMbE4A3v@V5MV`QK9-7}8wWCJ zBH~u$evH$dNk${H`lhLsmV9e#n~D3#!^7h<(5)INK70M++rgvDf)CH+RmxQ#K4jw= zaXRt~3bN6ZVN$bS3yka*ZZ++nKk9?~(>JiU+Qi)P@^S377M7OT6%`9H2s0bo zHF=p4qS=ibH?oL^E*=DREJoj)3JB>OPFtLfYUUat+DAKUQbcgx z>zkXyRMTKHgq33o%V4FsioNdp5EZQUKPkC&ZO*p*ncKXUS-*gzI%h0 zSqsh`sDHL=EI7>oTJe7Ew$4L-6j|x@`H|Vic_({j%4s-yj?4eW_BscBg;ltrV zF&Wm@T?2b|cg`}VR>f0iJo7;lr~9v<=ba*DilI=Y^XAR-c+~bTe7+aWu82xbK4ee~ zZbTgL5EBEiRdP`?Wk*CrB#UAJP68EW7ZDcLns@Ws7RO6NEwM>RowJ3g%zqB1$8u*i zUR`i{v~c4eO9iPT!l}Ui?sL6yq+y6kJsKccVkiK3nHp*7{yI-pfMpi*&CV%6NBies4Cp} zlg$quII!>|2L4pu2JuEzKH#POs=C_RKt=9(dfM7K09A@{{-zkPF5UbxDmpr)7wK_u z;)(G2_Z8^>EB<`*A5wP7fuNMl>KXPu-}KSEF@P?zMDn!`F9k_TxiB zxVNT)^%jrX{jcsgSlTL|>-+Fg>2T+M+g*x^a~pC>N}NMOLYmi0XJ&|ti|gTj(!)_$ z9Ks_Ya0(Ud;@(R{!z=Vg+@8~r0yvsqN=n165nDXUetayUb&`3LM^NF@;NWd1Lo<4m#Aw8Xv`eZvy zU2r%4lIf-*>U*5STt*Roq@Nnr`36d5oaQ|ot>jA?n)`o9C!f+Mumkn`*iyR25v~+c+mX#PTZm$$~nuB z%&AN}Z&1aYg{H`p3;Nr}WS+Ht>E<$;bOH&@Nz{AlK6ViPU+F~wQNflJE7nf4JfAxI zeoe;+DgfP2dIYtRX~45*hRf~L6^oR1?eYffo;&RYzO%H~BS_=gf#E2#o1uhD=#P%! z><4zgLEmW*AV5X#ZY3pcT+_#fz;#@==kcbOswg-sW*>iL|2d{3y9@&H+Xh|xyGl1pEyX#1(e=0R{@AqVOKin z{BqHvMY~+LXgk>gc{sKiW>NiksS!>--rnmhZf4IhcmHyA3yO#f09-saTtM`ZdLS&6+j*Yu4;A9zrT1j3?@_ zj%lCb)ZJ~`*wDbesZ(v%-6P7%+1QopZK>h6f(lI#u#R!Sp!Z%tK(zQ8`{Z<-P(E7; z#V+gM@Y%_T84rRKE7Ik*Z=YgjZvL$d*tdSMfcEZh=?gPs&p<7cplS0tG9p3?m&dUo zybyz$7&fgZQ+(OJ*yu%F2oOcT&$BKCrRE}#;bJ@P7U*0_hATijXk@%Sg_(=R)B<$E zh9z?kMW?lW-C1ePZ!6x8GA9=I>tA2M(rDAoZ@}NyGc+uGb>QOh?buvr(yMIW0eNtn zdIqn;&@s$M#$GzfKOn#v>8SEK+>%ac8hhGvp*MTtMl>`*H*bCPP6M-&4O7xSUj$;6 zhroObfMd@4_lq!0gi2p z-+j>Mzkf{W1O1*A75+hXGtjGGiN475xZEI`waGybFq+L3Um;>o9hk$Yey<%J2W%^L zCMQ3CI^v9@kB?8G=t!r6mR9hIbp;$qF0ZTzLx=t{2^_xm`-vJ%nmP0Kk<4*`ypxN3 z9aNoe-1zKZZPVL47vwpyGtXnNzy}*qh3)`NLT$_zt&5udH+6M&lQFm?T2$D_iSzaR zuw|3t5Pf*H`(Uz?0=0F(N-HD*M0|^!r!%4^=yegk09X(0hF`S@DQV@B{;Vm8lPgte>9uGLd)nh!^N?+&^x2OM?lee60%w{dyNi1sL`|oQ1sI`nxyO@Dn2<3}VpLUEuiU(u zyxMG>oHGkg1GBCX?4ETQ+2{7e#6-OG_`)_^^0A+X9z`s1UAo1gz1`@{DE7`3 zz*vRfV~!;Jh2)r{=dr+0YN>VYw}yu49ri#M=fW$->ef_qCuip=b>G2Bmo(drG-7D$ zENf&D3d}^Hj6DalkNL>OYBt;3Eie}?@d@@`vuc&x`S#owRaI=`1NP&NNP&561y_0% zAzyUCch^E64ei8N$adZQ{iDP?5c8A2A?FRZ-E@G8Y3nLPRE=yoiw9P(1;^4vwHlch zsOCE06JlW+edzG+N8%`FU9xEo8U=>sy{#3u`RWiQ&e%z2z<%dJNXTT_ zJ~?hX=7)&@B-{c6XFsso>yuP#rSz`p=kORoyJ@zzA} zCp60aclnBfkEYw6`v8bnGkgTAMbXz|!zjWeJJ1s4PDQ@Pn2&nol)O!!Y`Ys~;Fb~q zR(H$Ks|#cXu8o<#!l*PiU1(>!)HJ^V(W3oMKQdGq3FQwBs7cFuC6zF{J0PMz(Cx64L-SnLq zh5NO+@(BvQ)6b(8W^zxS99&|xflarmzM8%nh{Z4X? zIwOHTty+#F@5mtlF7rPzzpSgffAsx3i(~36=F)sHzLLkyC6;?0m&!d)&#&l%CF`Or zaF$wqxz%eGXxnWKTWJi@+c;e%=N8fLj}s{$YP)dZ0*4Ix11kI$0;Z|Ir-p2 zp7B)brz=}7Md*_rZWmEDxbK34?QzTZ`ME7egV=&KQnR4jh!AC;rP(g+zBO?Fk>Z*50dye5=VWna_07Vco zj92XFxMrS$^}XwmtF zc}M&h(D0k)EHfysTkz|@ld55TkKd5xs@J+q) zXWoi6Yo-F~Uf3sL$9hPpa<)*P+oR3v)~?;T|A}Og`CTE0XU{~>j3Oyqz~US|d%QL* z=-9?t=cM@_8cR&_ed4Eeyf(7PYoxU;^+Od()p~3z3j~OixzcbXnvQA)C|{G$*Pc$! z1)V&2U+=gKt5)GebF7PHeKYv7_qC~s$(*IG_wUIpXy_@ftXzzDRpW1h`@bsFUIx&@ z_2S~)8$O{KErj-Hc8#i7j()`)5&sb3TRHc-IMtu6&m7XP`pV^qzQYnI4IXc5bvVo1 zJFv7~hRyhG@b&jXgFlA@@7`5yV@hK4?MIr>YI88;iD^Tn@~v5_t&(kmceU6f_HLBl z{VF#JXojR}&a&}iR`)jbZ8u7f^zPW6e8v~}0|?}TOEh-un2IXP39e11=`vh0c1|cu zD}ATA*q_0i6Y284;Kt`s7B%MUGgDv6%%-dgq{{@C6^+$F#!?|IzB430n#ZGH{n3r- z7)|G0%I8qd@<{rLpSgL-WRBP7@j}hj349>TxPbqEPN2LvV9R1;YYW9hQ#K2<3a!58 z=HSvev3#0>)gyO{xuo};(@$J{Jvw?lvI7USXJ$I^p!R)%HtJ-9zP?3~$2KgE>Tykv z`KIp0V!njsy1mW)KJOOoI?h67kKBVUsVd@mjXf{1c28e-E{(YL#eavF!14E%!%jkH zY87>3IhkYoVzfrPf;BhV+S+zOP+AX$&(pZLFLw5D?8=@geCcE>`>U@prw1!m+b(e& zO4gfHdZ8vTsjavtQKnF`v$8lh3WyG{qPg#$1Ic}XGkBk8!+6HluD~m}`A376Q3jAd z^HE5M{Nrb+B&|M3kC(LP;+f_l+Ff;YoHmg44V&nS=&iFtb&wtO3T|utL0{?dKHow& z`z25dZD-{L$taGBGroMRbCjx>7Rv}WB2dVNZ+zDeD$mUGBEFJ^_RziN0`*mNW) zhB9>3WLqKGeBGl@BMJl;G|}jA)g{}_E!WRI>c#u0JkLBnR#W71t79?n=@(!&q|^mS znC*CzTr^(maQ%9V?ri>vFWz$7TeyMC5uv{xdM{xG&EBoCr+{0zz5%f*K#NE76hRiR zUsWB#+ZN|Odh{q}X)w>R|728Lxm&I{r@5iw8g$X=c@+mhXE}$v7z5)TP9RokJ+HR_ z2+Jmtoa$~nAHG^u+=byXe}}%b6VKRv?~LoE#|7Q{c{W@9z=4~$K6pE>!E|v)v^pw> z8_;OGlDS1fV#a&SGU+Ru^TL%cafD6#iiCrjkUV>L&(WT&uoYju-p0RP{h_K~V<|3g zhcXDG&m8v4ZE&dlF%pF$;-r?=3)@UHGqbrUSo#62ZursL`w~|>gMrd82iTW+YdB?7 z*_-Y2cSzeUd-d+`MeOA5-$AP=bXd2J!D79(l_5f&tM=P|iYi!ZpK;Qhqeo1& zGDx}N9Gq3Gi%(Uc6TU>+h7FCeS9rB6yo$PD@|OMbq37O}cT9?A>FRHjO)kad za3xb>^X9*-R$^Ljv#uXn`}Ox!RaE4q6x7;RKfJY~&}}@?7T7MCmGGXit#2B@wk;K% zyh!F=jM)^i1rD3<&k6;+SksNZ?>UUCbG~0tSU9hOFKkKqHn*{RQlCPQCp6}B>FWTsxnec5i;G!g`+mfLVlWj1Z}sRe0H<|Moib}9X8&nv zRGrN(6a8+t#k&Dd5tmKq6H?g}T_82kp*=jvF?Q5XYuPb?MbpP5)Es3TuX;-awO%>4 zp~>ngLl#^L!_a!xo*erJqx_h~QM6vhZiEd~3=Wwb0U2`^We1$sEA}IOh+eBjMLmvY zSPyJKE*EST1-kbORFAUeUoGUg9F*n*zi}voG2hmG^MrC9z%)ql{D{M$wKqv4cjmB> z+*^b$;M>{JWRu=D_z->#D=f<4dNIgb-rcrR$ZUwfvEc3puBOAtSmmW2)Wx4-O)7qp zG|16y$MFKed`jD`$`WM3XY_ZXdyO=qRrfj4xv%$daohlMpU|Q!%#KocEpH;W!|_UN znlEVI26w9*|5;fQ78Z8Ocb|J^C%XT^J6ZK_AoJrabXy^p{LRB-0r#3U5qpj+D_;i1 z?e?utbK1Cby=(1XW-L+Tf>-%9%v>@_cf!ZRTJ#d$%uN?6O=HG96u5UC@8w3aEc16i z-;&7|)5zBe2&fMZeEFhD=dNA1u!9}JJj_1Y4@jciGG;YoeNLPk_s&t;U0UE!IHxH- z2sNg5p$AXR5WvkJa0>|Eec!|58jj&Dm$Uhs3*P*6UIM?*IeqMC2id^Wzs3r>a~fkD zdpTss`SYSA-a8|;ZryZDc;zkcX$uDQ5aDbKvgli>>3zy)R8J1=4rhS>vFK;&C(CEYkNzW>0LeZ*$PbRWX)pKbU9Fk zN0~(d2s)R(+|1PUDO$AY4YATlL&QC(6X_obDc=6!c{%h(G9R33-hA`M{AJ5rKtc0t zTd#58z;!UbK7*bxr_XXsN)DB}VaaVppgv>!Cm_Y{NqCe+fLEUM&Z~}3$X~+DLwb%|fsXgav&+!o;@#{zvT++Y_dj=^y~+n_gty?yNodhO zwd~PT?x|ZMX)yV?VCGH-z$D&X+uyxJdaOd$K^>Q|7RlkB&9S%6ir69W>*%OIb@(tlVFH|-kzHnBu)V;1vmXE` zZ&$ly{Agk=DlUGBEV4q{3D{T_+&RlM7s6HJGsg=!6WoG2ce<4oZw^}ya<4ADN;LgC z*!$%(C(aVAWj?pJvph(30@2?A{ecrgVOg|!X@50T@;_F-!s(0d3VjI3J5)dwY5a`FlQ4B&C_VB+&e`0^ zz<>u5B+m?;1NT|ZNps$UTlD8CPWZ}c;FKq5=34#u%!D4vbsFQ@D^~u-eBRu%AVotS z$f~}~)6)~}V~;S2Y5M{;zgq8j{W?4EZKQnX9}LOx5w$_m3+p~nlA^rXDJ#n$)DJD< ztsA^n@$k&x+<0bzs6Ia=lRqw`2D3{4yku=*k75+0aCXF}>wwmf@dCEhmEW;rB^uG_ zqFe(g9_DZxDs7V-EvUU&fGRzvW3uUol#^%VH5u%c;$0?`#Bx4Cfk*nZ-~a!@s0 za0gJ2wWXy+K|w**#zvA#ty9bD(9xvLs*UD6;}R&4Q&50fGzDDz+|=X(m50IqW9sO9m=qg7%=I=^obw+{kgy%yC4L3h;$X*d9)O_YH6*+t$hP+ zte1ItmA+?R0)?YyJIps$+%0hP@R*Elb6S7(|I^;Nf9062ar`YcW6)$w5;Z0@V&1Hd zLnkHPgbre+k`cQkyoD*%j2i5aSEi$Bp;A)h5F-hr5NfJHQ&W@V*l~)=AtV#AKezVU zYxY|EPuRb(w4S{0b3gZUU*GHd{a)7<$Zj4^<~s;JL}Sz_eS~*tEkQ2zUTCRWeNY{= za!1E-X=z#6+6(93Z4~knW-@PQw&A|4EM?B)Wa}^~d0{lZT5Ly+V8@A*CaFkb$J&m; zqkykQS+eG=)W0KZu9*e_1HNb|suf?9CM6@U-i zoWM!t?%~n&n?-|0&%S!HQ9`_-hl|MvFeui;Nhr0_nk4& zEeh|9Z$+|cP)LZ(*LR%o4*=?qPcOk@hM=xlVO%Q4?f~sWWKzCU`D`brVrmpPu3||$ zb?xlNB9qUQ%a7deKb4nvS?GvruPno#E)$^-#I2Eb1Ku)> zL8DsqNEenl%5kyRWkPhK)s=FL1e4$$9~bj>?GT*Cok0wcV!~c#PR<<$@ zJzlI|tE$RT2~Us#@U=vYH%ZHpa@f;-%NIB2I%1*3{YD9=aVbLN@DU@F;qj7FQ=75A z30DZj1=M3%+-LJ{*k%ci2J2@?$*YY_&*AvD;5kKOK)+mB`5`-qYF6=Di5!^|wF%=? z?np74_H0m-@`7?k#=a-^F1!3A4UH%?VSA(9`+1oW;o(T=Up3*$^9Ns9T3J2V&>*@_ zDslS0MLm>Mr8zy0&62Pvk^3qZ?Vhm zC|o!y>`TAQuCXrNs-OGIq2niXWLug##_MdSoq|iUiIQ2$Ftlzh)1@i`u$~M~Gc`Ed zAegZwPdWG;+_X>xXACpa)7z}%BT1%}_~B5Y6_2#R()7xOafLNs@b8uy`f;4VgT8#t zn#%sU*Q>a>6c>X(lDCN(vsnKXE6PbQPu-g38mzCU=kSUCOe8{KkoMcVv!7)?$+nQ+ zapaKDCP3G9gNBZnEnm1WW#4&^emn~Kt*y)Ew>zrIcKR+~ZZLH4;NSa5I-0gQwaD|` zCPO?HRBXf){Hheu<;q|s8V?)Cj>Nz{SR~*U5+MV8+L~zf8;flLW zJB;NdwRP5Spp8AsggG()%$YhZKQz5Aew0=9xoqb+zgR}ECV?I08*~UD0i_t(MqF^4 zOyyO<^NvR#z%H6T9K*?*_3W_GUfcyjWs10%o|fkDaJNC>-iDkPVwI-!nq7S{=Wh6f zq$~9}26JD$DaQ%+us&~p-CwU;uY9~SIP6H$unidtJ`J_at-Wgz-QVzKHN2$i+d&Jx zyiQzv)8{O|g@MRuF=xQ(JAv#cg`+HZ1b-Xv;~TUf3Q(VRuI3_}Nw`*{Tz&17b0)RBNz|o3=jA@5Vtb*AHqH)-S*_Do5C_fjVX2Xk{tb#(GVYLX$@x6 zVkf#>QZfib;&d06VdOn(Ag$@Sdaklqh`^ko6hbt1#EB}W$-VpY@=S~RUo1Ls=uitm zAqp$l)l;m9aKvIx-@+4!Yxus1jK7)WQu=+wnU&SbsP{7&6Yg1VYhp4S6ybGp*#`_O z^7#hQ-)o*fP)b1O%hlEMA2jeXZvW*M2MXA{{U)K+0nr^z9Hbe*PEj2q#JuYa{@pW$ z{!AkT8%n8oCQjLa?=u60r?r^$A40ib9qa3;u-8PWCjz5j&ja?g8R6=DEjw z5*^Y9$voTMn?j_%n0@GwzLnKzqTJT(?D1GJ641Rvl8y!LC-$L|PB1e!pYP$J%F*%s z(cMKY!&mjhnJR#xjd*&62MbCGg2{LFYBkKUNJkiEXRVec7^r`0_X`4L;+{P*Sy_4l zMIuBBriG_cYd3Mh!DGjU;p-Z%(@A9fAU+I@w)c`D+q$+yPqy3F(^Heh*pZQuKo-a3 z24y^>EHw}(vQFJz?E%r!azo!2v^Cly{`=Hap*B6~?cD|>@KhVRwm=6j>TG9c+4Sj3 z_<(rU;uc=^^%c`@$z4A}mihQuK@lYZ!jpXVtd@OGT69l$(D~y60V8`4lDwu4#d_Ld z(k>BAS%%=ZZ#GbJA?*F2Y6tSK=U={IM+c zdh{rsd@-mTUg`Gc!1VfA;}w%+5H|pf79OVfPvES$lG3O^jNj9bDxMVzpQJFT3tYn& zMamz9LmmHlfbFt2TqHqsa1u%kql%TNxKxJFT^A8i+|clk2kYb1Qi`hD`<`M$Xl6$3 z`w|sZRa{t^dqW(CWdeaIm@4LE5$_?w+@-p@Ge!A0IHCqhMiNIw7AkC?zOKpkO-LMd z1!huIrc170=)roR?W#U?F%y~vBH<+E!Iz7sS2oUSyK>(B(rNKabJnf1v}o@D?00is z)rQ1Jk6U%Oa3s>=pn0~BPfrP%ONcn49oK%^N910-A=UO>B`^6hJ~W-X(+AW1m$yZS zQY5xaG5lL%9}pm=K|zPrEH3WYQ{UN!ws6+bR`R^KKI8{eg)lZsd{(cHBQx8_ZGvk? z4=6k$wX+bz+M^VTr7+u0?K=#Ojc2*LYtR!q9^|QXc&~b=0B|j7bqVwZS0TbnBH#7O z3z0!^6p5ZyqRs($qr3ZBV81P%OtST+h#)0lj;yUTcNvol>C;ReA&CUC2Bu9&6{5oL zYs!NDeS7jtT7LA0m3e#Xh=;|Kw6w7)ffyEVE9TdE_{_5{%*={G9ZhX*o(ToVZP1pz z3PV2^i3-uXiDz9-n^n(-n|iQpZn&9a0eHt2{rn+*rZLZn! z^z`&R`p40o=+yC(H+0E;B^1LSe!PmuC6_=X3Rmla1BYFR*Y+4WN~){#K3wbAkxI1U z<(c1bQi6(jZ ztpbOL&Gnx@I$S^g4btkrel~bqPuxJM)ZZ=~ELRzzG5>yTqGRO$FC19C|GhkoZu`LA&*pytaGE(| From c9abe2b145c6f5e0cb2d140977ff32908dd297ea Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 19:00:53 +0000 Subject: [PATCH 04/10] bump version to v0.1.5 --- README.md | 132 ++++++++++++++++++++++++++++++++++++++++++----- pyproject.toml | 5 +- requirements.txt | 2 +- 3 files changed, 123 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2fb1ee2..49da6e1 100644 --- a/README.md +++ b/README.md @@ -20,19 +20,21 @@ ______________________________________________________________________ ## 📰 News +- 🔀 **2026-07-13 · [v0.1.5](https://github.com/tile-ai/TileRT/releases/tag/v0.1.5) Released**. Introduce **PD (prefill–decode) disaggregation** — vLLM prefill + TileRT decode, behind an OpenAI-compatible endpoint. Supported on GLM-5 and DeepSeek-V3.2. + - 💥 **2026-06-08 · [Breaking 1000 TPS on a 1T Model](https://www.tilert.ai/blog/breaking-1000-tps.html)**. In collaboration with [Xiaomi MiMo](https://mimo.xiaomi.com/blog/mimo-tilert-1000tps), TileRT pushes [**MiMo-V2.5-Pro-UltraSpeed**](https://platform.xiaomimimo.com/docs/en-US/model-intro/mimo-v2.5-pro-ultraspeed) past **1000 tokens/s** on a **1-trillion-parameter** model through extreme model–system co-design — a first without custom silicon, all on a single 8-GPU node. - 🚀 **2026-06-01 · [v0.1.4](https://github.com/tile-ai/TileRT/releases/tag/v0.1.4) Released**. A major performance upgrade for both DeepSeek-V3.2 and GLM-5, with model quality unchanged. See the benchmark charts for details. - 🏭 **2026-05-22 · [TileRT in Production](https://www.tilert.ai/blog/speed-as-the-next-scaling-law-zh.html)**. [**GLM-5.1-highspeed**](https://docs.bigmodel.cn/cn/guide/models/text/glm-5.1-highspeed) is now live on Z.ai, powered by TileRT — from experimental prototype to real production. +

+ Key Milestones + - :fire: **2026-02-14 · [Try the Online Demo](https://www.tilert.ai/)**. Our online demo is now live! Experience ultra-low-latency inference with **GLM-5** and **DeepSeek-V3.2**. [Try it now !](https://www.tilert.ai) - 🎉 **2026-02-14 · [v0.1.3](https://github.com/tile-ai/TileRT/releases/tag/v0.1.3) Released**. The v0.1.3 release introduces full support for the latest GLM-5 model, achieving up to 500 tokens/s on GLM-5-FP8 and up to 600 tokens/s on DeepSeek-V3.2. -
- Key Milestones - - 🚀 **2026-01-26 · [v0.1.2-alpha.1](https://github.com/tile-ai/TileRT/releases/tag/v0.1.2-alpha.1)**. **Multi-Token Prediction (MTP)** is now available in TileRT! With mtp=3, we achieve decoding rates of up to **590 tokens/s** under synthetic workloads. - ⚡ **2025-12-23 · [v0.1.1](https://github.com/tile-ai/TileRT/releases/tag/v0.1.1)**. Achieved ~**35% further reduction** (3 ~ 4x speedup over baseline) in end-to-end token generation latency on a single node with **8× NVIDIA B200**. @@ -54,9 +56,9 @@ To achieve this, TileRT introduces a **tile-level runtime engine**. Leveraging a The project is actively evolving, and the underlying compiler techniques will be gradually shared with the community as they are integrated into **TileLang** and **TileScale**.

- GLM-5.1-FP8 token generation speed on 8× B200 with TileRT v0.1.4 + GLM-5.1-FP8 token generation speed on 8× B200 with TileRT v0.1.5
- GLM-5.1-FP8 token generation speed on 8× NVIDIA B200 with TileRT v0.1.4. Output length 1K, input length 1K–192K. Bars compare TileRT without MTP, with MTP at average acceptance length 3.2, and the peak under best-case MTP acceptance. + GLM-5.1-FP8 token generation speed on 8× NVIDIA B200 with TileRT v0.1.5. Output length 1K, input length 1K–192K. Bars compare TileRT without MTP, with MTP at average acceptance length 3.2, and the peak under best-case MTP acceptance (4.0).

______________________________________________________________________ @@ -64,11 +66,11 @@ ______________________________________________________________________ ## Installation > \[!IMPORTANT\] -> TileRT v0.1.4 is distributed as a **pre-built binary wheel**. The wheel is linked against the exact ABI of the versions listed below. Other combinations of Python, CUDA, or PyTorch versions are **untested and not guaranteed to work** — please reproduce this environment for a supported setup. +> TileRT v0.1.5 is distributed as a **pre-built binary wheel**. The wheel is linked against the exact ABI of the versions listed below. Other combinations of Python, CUDA, or PyTorch versions are **untested and not guaranteed to work** — please reproduce this environment for a supported setup. -### Build environment of the v0.1.4 wheel +### Build environment of the v0.1.5 wheel -The official `tilert==0.1.4` wheel on PyPI was compiled against the following stack. Treat these as **hard requirements**, not lower bounds. +The official `tilert==0.1.5` wheel on PyPI was compiled against the following stack. Treat these as **hard requirements**, not lower bounds. | Component | Pinned version | | ---------------- | --------------------------------------------------- | @@ -83,7 +85,7 @@ The official `tilert==0.1.4` wheel on PyPI was compiled against the following st ### Recommended: pre-built Docker image The pinned build environment above is preinstalled in our official image -— this is the **recommended way to run v0.1.4** and avoids any version +— this is the **recommended way to run v0.1.5** and avoids any version drift on the host. The image is mirrored to two registries; pull from whichever is reachable: @@ -104,18 +106,18 @@ docker run --rm -it --gpus all --ipc=host \ ghcr.io/tile-ai/tilert:cu132-latest # Inside the container — install from PyPI: -pip install tilert==0.1.4 +pip install tilert==0.1.5 # Or pin the exact wheel from the GitHub Release page directly # (same artifact, useful when PyPI is unreachable): -pip install https://github.com/tile-ai/TileRT/releases/download/v0.1.4/tilert-0.1.4-cp312-cp312-manylinux_2_28_x86_64.whl +pip install https://github.com/tile-ai/TileRT/releases/download/v0.1.5/tilert-0.1.5-cp312-cp312-manylinux_2_28_x86_64.whl ``` Verify the install: ```bash python -c "import tilert, torch; print('tilert', tilert.__version__, '/ torch', torch.__version__, '/ cuda', torch.version.cuda)" -# Expected: tilert 0.1.4 / torch 2.11.0+cu130 / cuda 13.0 +# Expected: tilert 0.1.5 / torch 2.11.0+cu130 / cuda 13.0 ``` Proceed to [Getting Started](#getting-started) to download and convert model weights. @@ -178,7 +180,7 @@ python -m tilert.generate --model deepseek_v3_2 --max-new-tokens 1000 ``` > \[!NOTE\] -> v0.1.4 ships **two independent backend libraries** (`libtilert_dsv32.so` +> v0.1.5 ships **two independent backend libraries** (`libtilert_dsv32.so` > and `libtilert_glm5.so`) and loads exactly one per Python process via > `tilert.load_backend(model_type)`. Run DeepSeek-V3.2 and GLM-5 in > separate processes — they cannot coexist in a single interpreter. @@ -306,6 +308,110 @@ This example highlights how MTP enables TileRT to efficiently generate longer ou For the full list of CLI flags (sampling, batching, benchmark modes, …), run `python -m tilert.generate --help`. +## Disaggregated Serving: vLLM Prefill + TileRT Decode + +TileRT can run as the **decode engine behind a vLLM prefill**, integrated through vLLM's V1 `KVConnector` interface. The connector, decode server, and router all ship inside the `tilert` wheel under `tilert.pd_vllm` — no vLLM fork or patch is needed (the connector loads via vLLM's standard `kv_connector_module_path`). Latency-critical requests are routed to the TileRT decode pool; other traffic can stay on native vLLM decode. + +**Prerequisites** + +- Convert the model weights for TileRT decode (see [Step 2](#step-2-shard-weights-with-weight_converter)). +- On the **prefill** node, a vLLM build with V1 disaggregation and support for the GLM-5 / DeepSeek-V3.2 (DSA) model and the `fp8_ds_mla` KV-cache dtype. Install `tilert` in the same environment so the connector plugin is importable. +- **The KV-cache dtype must match on both ends.** These examples use fp8: `--kv-cache-dtype fp8_ds_mla` on the vLLM prefill and `--kv-cache-dtype fp8` on the TileRT decode (a mismatch is rejected at the connector handshake). +- The examples use the **NIXL** transfer engine. On multi-NIC hosts, pin NIXL to the RDMA NICs via `UCX_NET_DEVICES` (otherwise UCX may pick the wrong interface). Mooncake is also supported (`--transport mooncake` on the decode, `"tilert_transport": "mooncake"` on the prefill). + +Commands below use GLM-5. For DeepSeek-V3.2, use `--model deepseek_v3_2`, the DeepSeek-V3.2-TileRT weights, and `--parser none`. + +### Topology A: vLLM prefill → TileRT decode + +Three processes — a TileRT decode server, a stock vLLM prefill, and an OpenAI-compatible router: + +```bash +# 1) TileRT decode node +python -m tilert.pd_vllm.decode_server \ + --engine tilert --model glm5 \ + --model-weights-dir /path/to/GLM-5-FP8-TileRT \ + --with-mtp --max-seq-len 202752 \ + --kv-cache-dtype fp8 --transport nixl \ + --ctrl-port 5556 --http-port 5557 + +# 2) vLLM prefill (stock vLLM; the TileRT connector loads as a plugin). +# The MTP speculative config is required: the prefill populates the +# draft-layer KV that decode-side speculation resumes from. +export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... # pin NIXL to the RDMA NICs (multi-NIC hosts) +vllm serve /path/to/GLM-5-FP8 \ + --served-model-name glm5 --port 8000 \ + --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ + --return-tokens-as-token-ids --gpu-memory-utilization 0.85 \ + --kv-cache-dtype fp8_ds_mla \ + --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}' \ + --kv-transfer-config '{ + "kv_connector": "TileRTConnector", + "kv_connector_module_path": "tilert.pd_vllm.prefill_connector", + "kv_role": "kv_producer", + "kv_connector_extra_config": { + "tilert_host": "", "tilert_ctrl_port": 5556, + "tilert_model": "glm5", "tilert_max_seq_len": 202752, + "tilert_transport": "nixl"}}' + +# 3) Router — OpenAI-compatible ingress for the TileRT pool +python -m tilert.pd_vllm.pd_router \ + --vllm-url http://:8000 \ + --decode :5556:5557 \ + --model-path /path/to/GLM-5-FP8 \ + --parser glm47 --port 23333 +``` + +Send OpenAI requests to `http://:23333/v1/chat/completions`. The router runs the prefill on vLLM (first token), hands the attention state to the TileRT decode node over RDMA, and streams the completion back. + +### Topology B: shared prefill → TileRT decode **and** native vLLM decode + +One prefill pool feeds two decode pools side by side, composed under vLLM's `MultiConnector`. Each request is claimed by exactly one connector — the TileRT connector claims requests marked with `tilert_host`, and vLLM's native connector handles the rest — so latency-critical traffic goes to TileRT while general traffic stays on native vLLM decode, behind the same OpenAI surface. + +```bash +# 1) TileRT decode node (identical to Topology A) +python -m tilert.pd_vllm.decode_server --engine tilert --model glm5 \ + --model-weights-dir /path/to/GLM-5-FP8-TileRT --with-mtp \ + --max-seq-len 202752 --kv-cache-dtype fp8 --transport nixl \ + --ctrl-port 5556 --http-port 5557 + +# 2) Native vLLM decode node — vLLM's standard disaggregation (NixlConnector consumer) +export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... +vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8001 \ + --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ + --return-tokens-as-token-ids --kv-cache-dtype fp8_ds_mla \ + --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}' \ + --kv-transfer-config '{"kv_connector": "NixlConnector", "kv_role": "kv_consumer"}' + +# 3) Shared vLLM prefill — MultiConnector[ NixlConnector + TileRTConnector ] +export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... +vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8000 \ + --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ + --return-tokens-as-token-ids --gpu-memory-utilization 0.85 \ + --kv-cache-dtype fp8_ds_mla \ + --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}' \ + --kv-transfer-config '{ + "kv_connector": "MultiConnector", "kv_role": "kv_producer", + "kv_connector_extra_config": {"connectors": [ + {"kv_connector": "NixlConnector", "kv_role": "kv_producer"}, + {"kv_connector": "TileRTConnector", + "kv_connector_module_path": "tilert.pd_vllm.prefill_connector", + "kv_role": "kv_producer", + "kv_connector_extra_config": { + "tilert_host": "", "tilert_ctrl_port": 5556, + "tilert_model": "glm5", "tilert_max_seq_len": 202752, + "tilert_transport": "nixl"}}]}}' + +# 4a) TileRT router — latency-critical traffic → TileRT pool +python -m tilert.pd_vllm.pd_router --vllm-url http://:8000 \ + --decode :5556:5557 --model-path /path/to/GLM-5-FP8 \ + --parser glm47 --port 23333 + +# 4b) General traffic → native vLLM decode pool, via vLLM's standard NixlConnector +# disaggregation proxy, pointing prefill :8000 → native decode :8001. +``` + +**Note.** Running NIXL end to end (both the native and TileRT connectors in NIXL mode) lets the shared prefill use a single transfer library. Only the prefill's `--kv-transfer-config` differs from Topology A; the TileRT decode node is unchanged, and the native decode instance plus its proxy follow vLLM's usual `NixlConnector` disaggregation setup. + ## Status & Future Work TileRT is currently offered as a preview release, and we’re just getting started. diff --git a/pyproject.toml b/pyproject.toml index 13aa491..cfa0bcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ description = "TileRT" readme = "README.md" requires-python = ">=3.11" license = {text = "MIT"} +authors = [{name = "TileRT Team", email = "contact@tilert.ai"}] classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -14,7 +15,7 @@ classifiers = [ ] dependencies = [ - # Pinned to the exact ABI the v0.1.4 wheel was built against. ``torch`` must + # Pinned to the exact ABI the v0.1.5 wheel was built against. ``torch`` must # come from PyTorch's cu130 index (``--index-url # https://download.pytorch.org/whl/cu130``); installing from PyPI yields a # CUDA build that does not match the cu130-linked tilert binary. @@ -64,7 +65,7 @@ dev = [ Homepage = "https://github.com/tile-ai/TileRT" Issues = "https://github.com/tile-ai/TileRT/issues" -# Note: this repository ships the public sources that match the v0.1.4 wheel. +# Note: this repository ships the public sources that match the v0.1.5 wheel. # The wheel itself is built in the development repo (TileRT-dev/TileRT) with # scikit-build-core; no [build-system] block is declared here on purpose so # nobody accidentally runs ``pip wheel .`` against this presentation copy. diff --git a/requirements.txt b/requirements.txt index fd4a9ba..c22551d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# Runtime dependencies for the v0.1.4 wheel, pinned to the exact ABI the +# Runtime dependencies for the v0.1.5 wheel, pinned to the exact ABI the # wheel was built against. ``torch`` must be installed from PyTorch's cu130 # index — PyPI's default ``torch`` is a different CUDA build and will not load # the cu130-linked tilert binary: From e02101f12e01b8e3bb58155f548431ce8eecf1eb Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 19:07:14 +0000 Subject: [PATCH 05/10] update --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cfa0bcd..fc42915 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ description = "TileRT" readme = "README.md" requires-python = ">=3.11" license = {text = "MIT"} -authors = [{name = "TileRT Team", email = "contact@tilert.ai"}] +authors = [{name = "TileRT-team", email = "contact@tilert.ai"}] classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", From 11d64463613f402e7a22355fb85e3cc18ceff896 Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 19:08:11 +0000 Subject: [PATCH 06/10] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49da6e1..caa406e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ ______________________________________________________________________ ## 📰 News -- 🔀 **2026-07-13 · [v0.1.5](https://github.com/tile-ai/TileRT/releases/tag/v0.1.5) Released**. Introduce **PD (prefill–decode) disaggregation** — vLLM prefill + TileRT decode, behind an OpenAI-compatible endpoint. Supported on GLM-5 and DeepSeek-V3.2. +- 🔀 **2026-07-14 · [v0.1.5](https://github.com/tile-ai/TileRT/releases/tag/v0.1.5) Released**. Introduce **PD (prefill–decode) disaggregation** — vLLM prefill + TileRT decode, behind an OpenAI-compatible endpoint. Supported on GLM-5 and DeepSeek-V3.2. - 💥 **2026-06-08 · [Breaking 1000 TPS on a 1T Model](https://www.tilert.ai/blog/breaking-1000-tps.html)**. In collaboration with [Xiaomi MiMo](https://mimo.xiaomi.com/blog/mimo-tilert-1000tps), TileRT pushes [**MiMo-V2.5-Pro-UltraSpeed**](https://platform.xiaomimimo.com/docs/en-US/model-intro/mimo-v2.5-pro-ultraspeed) past **1000 tokens/s** on a **1-trillion-parameter** model through extreme model–system co-design — a first without custom silicon, all on a single 8-GPU node. From 321c6fd88a2b62b9d121f60580866bea6d70fcd1 Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Mon, 13 Jul 2026 19:09:33 +0000 Subject: [PATCH 07/10] update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index caa406e..2cf401b 100644 --- a/README.md +++ b/README.md @@ -341,7 +341,7 @@ export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... # pin NIXL to the RDMA NICs (mult vllm serve /path/to/GLM-5-FP8 \ --served-model-name glm5 --port 8000 \ --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ - --return-tokens-as-token-ids --gpu-memory-utilization 0.85 \ + --return-tokens-as-token-ids --gpu-memory-utilization 0.75 \ --kv-cache-dtype fp8_ds_mla \ --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}' \ --kv-transfer-config '{ @@ -386,7 +386,7 @@ vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8001 \ export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8000 \ --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ - --return-tokens-as-token-ids --gpu-memory-utilization 0.85 \ + --return-tokens-as-token-ids --gpu-memory-utilization 0.75 \ --kv-cache-dtype fp8_ds_mla \ --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}' \ --kv-transfer-config '{ From cfc9c5bbcf532a3c03cc35766ad7487276553ccd Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Tue, 14 Jul 2026 06:46:12 +0000 Subject: [PATCH 08/10] update --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2cf401b..4242f04 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ ______________________________________________________________________ ## 📰 News -- 🔀 **2026-07-14 · [v0.1.5](https://github.com/tile-ai/TileRT/releases/tag/v0.1.5) Released**. Introduce **PD (prefill–decode) disaggregation** — vLLM prefill + TileRT decode, behind an OpenAI-compatible endpoint. Supported on GLM-5 and DeepSeek-V3.2. +- 🔀 **2026-07-14 · [v0.1.5](https://github.com/tile-ai/TileRT/releases/tag/v0.1.5) Released**. Introduce **PD (prefill–decode) disaggregation** — vLLM prefill + TileRT decode, behind an OpenAI-compatible endpoint. Supported on GLM-5/5.1 and DeepSeek-V3.2. - 💥 **2026-06-08 · [Breaking 1000 TPS on a 1T Model](https://www.tilert.ai/blog/breaking-1000-tps.html)**. In collaboration with [Xiaomi MiMo](https://mimo.xiaomi.com/blog/mimo-tilert-1000tps), TileRT pushes [**MiMo-V2.5-Pro-UltraSpeed**](https://platform.xiaomimimo.com/docs/en-US/model-intro/mimo-v2.5-pro-ultraspeed) past **1000 tokens/s** on a **1-trillion-parameter** model through extreme model–system co-design — a first without custom silicon, all on a single 8-GPU node. @@ -145,7 +145,7 @@ python -m tilert.models.preprocess.weight_converter \ --save_dir "/path/to/DeepSeek-V3.2-TileRT" ``` -For **GLM-5**: +For **GLM-5/5.1**: ```bash python -m tilert.models.preprocess.weight_converter \ @@ -315,11 +315,11 @@ TileRT can run as the **decode engine behind a vLLM prefill**, integrated throug **Prerequisites** - Convert the model weights for TileRT decode (see [Step 2](#step-2-shard-weights-with-weight_converter)). -- On the **prefill** node, a vLLM build with V1 disaggregation and support for the GLM-5 / DeepSeek-V3.2 (DSA) model and the `fp8_ds_mla` KV-cache dtype. Install `tilert` in the same environment so the connector plugin is importable. +- On the **prefill** node, a vLLM build with V1 disaggregation and support for the GLM-5/5.1 / DeepSeek-V3.2 (DSA) model and the `fp8_ds_mla` KV-cache dtype. Install `tilert` in the same environment so the connector plugin is importable. - **The KV-cache dtype must match on both ends.** These examples use fp8: `--kv-cache-dtype fp8_ds_mla` on the vLLM prefill and `--kv-cache-dtype fp8` on the TileRT decode (a mismatch is rejected at the connector handshake). - The examples use the **NIXL** transfer engine. On multi-NIC hosts, pin NIXL to the RDMA NICs via `UCX_NET_DEVICES` (otherwise UCX may pick the wrong interface). Mooncake is also supported (`--transport mooncake` on the decode, `"tilert_transport": "mooncake"` on the prefill). -Commands below use GLM-5. For DeepSeek-V3.2, use `--model deepseek_v3_2`, the DeepSeek-V3.2-TileRT weights, and `--parser none`. +Commands below use GLM-5/5.1. For DeepSeek-V3.2, use `--model deepseek_v3_2`, the DeepSeek-V3.2-TileRT weights, and `--parser none`. ### Topology A: vLLM prefill → TileRT decode @@ -329,7 +329,7 @@ Three processes — a TileRT decode server, a stock vLLM prefill, and an OpenAI- # 1) TileRT decode node python -m tilert.pd_vllm.decode_server \ --engine tilert --model glm5 \ - --model-weights-dir /path/to/GLM-5-FP8-TileRT \ + --model-weights-dir /path/to/GLM-5.1-FP8-TileRT \ --with-mtp --max-seq-len 202752 \ --kv-cache-dtype fp8 --transport nixl \ --ctrl-port 5556 --http-port 5557 @@ -338,7 +338,7 @@ python -m tilert.pd_vllm.decode_server \ # The MTP speculative config is required: the prefill populates the # draft-layer KV that decode-side speculation resumes from. export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... # pin NIXL to the RDMA NICs (multi-NIC hosts) -vllm serve /path/to/GLM-5-FP8 \ +vllm serve /path/to/GLM-5.1-FP8 \ --served-model-name glm5 --port 8000 \ --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ --return-tokens-as-token-ids --gpu-memory-utilization 0.75 \ @@ -357,7 +357,7 @@ vllm serve /path/to/GLM-5-FP8 \ python -m tilert.pd_vllm.pd_router \ --vllm-url http://:8000 \ --decode :5556:5557 \ - --model-path /path/to/GLM-5-FP8 \ + --model-path /path/to/GLM-5.1-FP8 \ --parser glm47 --port 23333 ``` @@ -370,13 +370,13 @@ One prefill pool feeds two decode pools side by side, composed under vLLM's `Mul ```bash # 1) TileRT decode node (identical to Topology A) python -m tilert.pd_vllm.decode_server --engine tilert --model glm5 \ - --model-weights-dir /path/to/GLM-5-FP8-TileRT --with-mtp \ + --model-weights-dir /path/to/GLM-5.1-FP8-TileRT --with-mtp \ --max-seq-len 202752 --kv-cache-dtype fp8 --transport nixl \ --ctrl-port 5556 --http-port 5557 # 2) Native vLLM decode node — vLLM's standard disaggregation (NixlConnector consumer) export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... -vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8001 \ +vllm serve /path/to/GLM-5.1-FP8 --served-model-name glm5 --port 8001 \ --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ --return-tokens-as-token-ids --kv-cache-dtype fp8_ds_mla \ --speculative-config '{"method": "mtp", "num_speculative_tokens": 1}' \ @@ -384,7 +384,7 @@ vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8001 \ # 3) Shared vLLM prefill — MultiConnector[ NixlConnector + TileRTConnector ] export UCX_NET_DEVICES=mlx5_1:1,mlx5_2:1,... -vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8000 \ +vllm serve /path/to/GLM-5.1-FP8 --served-model-name glm5 --port 8000 \ --tensor-parallel-size 8 --enforce-eager --trust-remote-code \ --return-tokens-as-token-ids --gpu-memory-utilization 0.75 \ --kv-cache-dtype fp8_ds_mla \ @@ -403,7 +403,7 @@ vllm serve /path/to/GLM-5-FP8 --served-model-name glm5 --port 8000 \ # 4a) TileRT router — latency-critical traffic → TileRT pool python -m tilert.pd_vllm.pd_router --vllm-url http://:8000 \ - --decode :5556:5557 --model-path /path/to/GLM-5-FP8 \ + --decode :5556:5557 --model-path /path/to/GLM-5.1-FP8 \ --parser glm47 --port 23333 # 4b) General traffic → native vLLM decode pool, via vLLM's standard NixlConnector From 559a4019642f9f359157322d474a1b8cfa5f59ef Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Tue, 14 Jul 2026 08:49:13 +0000 Subject: [PATCH 09/10] fix lint --- .flake8 | 4 + pyproject.toml | 10 + .../_dsa_v32/ops/expert_down_allreduce.py | 6 - .../_dsa_v32/ops/expert_sel_up_gate_silu.py | 2 +- .../models/glm_5/_dsa_v32/ops/projo_wkvb.py | 4 +- .../glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py | 4 +- tilert/models/glm_5/modules/__init__.py | 1 - tilert/pd_vllm/decode_server.py | 169 +++++++++----- tilert/pd_vllm/engine_iface.py | 16 +- tilert/pd_vllm/oai_parser.py | 32 +-- tilert/pd_vllm/pd_router.py | 205 +++++++++-------- tilert/pd_vllm/prefill_connector.py | 125 +++++++---- tilert/pd_vllm/profiles/base.py | 64 ++++-- tilert/pd_vllm/profiles/dsv32.py | 16 +- tilert/pd_vllm/profiles/glm5.py | 16 +- tilert/pd_vllm/profiles/mla_nsa.py | 212 ++++++++++-------- tilert/pd_vllm/receive_server.py | 72 +++--- tilert/pd_vllm/transport.py | 32 +-- tilert/pd_vllm/wire.py | 33 ++- 19 files changed, 615 insertions(+), 408 deletions(-) diff --git a/.flake8 b/.flake8 index 63396a2..a247806 100644 --- a/.flake8 +++ b/.flake8 @@ -35,3 +35,7 @@ per-file-ignores = __init__.py:F401 tests/*:D100,D101,D102,D103 setup.py:D100,D101,D102,D103,B009 + # pd_vllm implements vLLM connector / abstract-profile interfaces, so many + # method params are unused by design (U100); black collapses the ``...`` + # interface stubs onto one line (E704). + tilert/pd_vllm/*:U100,E704 diff --git a/pyproject.toml b/pyproject.toml index fc42915..407c497 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,16 @@ ignore_missing_imports = true exclude = ["3rd-party/"] explicit_package_bases = true +# pd_vllm is glue over vLLM's connector API; its interface methods take +# ``Any``-typed params by contract, so full def-level annotation is low value. +# Relax annotation-completeness there, but keep real type checks (arg-type, +# union-attr, etc.) on so genuine mistakes still surface. +[[tool.mypy.overrides]] +module = "tilert.pd_vllm.*" +disallow_untyped_defs = false +disallow_incomplete_defs = false +warn_return_any = false + [tool.bandit] exclude_dirs = ["tests", "3rd-party"] skips = ["B101", "B311", "B404", "B603", "B607"] diff --git a/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py b/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py index 2da6d3f..19e98e6 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py +++ b/tilert/models/glm_5/_dsa_v32/ops/expert_down_allreduce.py @@ -18,8 +18,6 @@ ] - - def expert_down_allreduce( vec_in: torch.Tensor, mat_in: torch.Tensor, @@ -49,8 +47,6 @@ def expert_down_allreduce( ) - - class ExpertDownAllReduceAlgorithm(Enum): """ExpertDownAllReduce algorithm.""" @@ -280,8 +276,6 @@ def convert_to_glm5_fp4_hmma( return down_packed, dummy - - @dataclass class ExpertDownAllReduceTilertWeightsAlias: diff --git a/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py b/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py index fa1ab6e..c5c3fbd 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py +++ b/tilert/models/glm_5/_dsa_v32/ops/expert_sel_up_gate_silu.py @@ -4,7 +4,6 @@ from enum import Enum import numpy as np - import torch import torch.nn.functional as F @@ -336,6 +335,7 @@ def convert_to_glm5_fp4_hmma( moe_inter_pd = self.model_args.moe_inter_dim // self.num_devices with torch.inference_mode(): + def _ensure_unpacked(t: torch.Tensor) -> torch.Tensor: if t.shape[-1] == dim: return t.to(torch.uint8).contiguous() diff --git a/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py b/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py index 7128c39..36fdf73 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py +++ b/tilert/models/glm_5/_dsa_v32/ops/projo_wkvb.py @@ -98,9 +98,7 @@ def convert_to_fp16mma(self, weights: list[torch.Tensor]) -> torch.Tensor: scales_per_cta = wkv_b_b_scales.repeat_interleave(ctas_per_scale_row, dim=1) scales_per_cta = scales_per_cta.reshape(num_ctas, n_scale_k) else: - scales_per_cta = wkv_b_b_scales.squeeze(1).repeat_interleave( - ctas_per_head, dim=0 - ) + scales_per_cta = wkv_b_b_scales.squeeze(1).repeat_interleave(ctas_per_head, dim=0) scale_dtype = torch.float32 scales_per_cta = scales_per_cta.to(scale_dtype) diff --git a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py index b7bff81..9e5bbc2 100644 --- a/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py +++ b/tilert/models/glm_5/_dsa_v32/ops/rmsnorm_projx_wqkva.py @@ -121,9 +121,7 @@ def _bcast_block_scale(scale: torch.Tensor, rows: int) -> torch.Tensor: _bcast_block_scale(w_pe_scale, qk_rope_head_dim), ], dim=0, - ).clamp( - min=1e-12 - ) + ).clamp(min=1e-12) assert C.MAT_BYTES == C.SCALE_OFFSET, "Layout mismatch: scales must follow mat" diff --git a/tilert/models/glm_5/modules/__init__.py b/tilert/models/glm_5/modules/__init__.py index 8b13789..e69de29 100644 --- a/tilert/models/glm_5/modules/__init__.py +++ b/tilert/models/glm_5/modules/__init__.py @@ -1 +0,0 @@ - diff --git a/tilert/pd_vllm/decode_server.py b/tilert/pd_vllm/decode_server.py index ddaa039..3372694 100644 --- a/tilert/pd_vllm/decode_server.py +++ b/tilert/pd_vllm/decode_server.py @@ -1,5 +1,4 @@ -"""PD decode server (W6): HTTP orchestration around receive -> convert -> -inject -> decode. +"""PD decode server (W6): HTTP orchestration around receive -> convert -> inject -> decode. Internal token-level API (the client-facing OpenAI layer lives in pd_router / a later serving layer): @@ -15,11 +14,14 @@ """ import argparse +import contextlib import json import logging import queue as queue_mod +import socket import threading import time +from typing import Any import uvicorn from fastapi import FastAPI @@ -43,7 +45,7 @@ class DecodeBody(BaseModel): def build_app(server: ReceiveServer, engine) -> FastAPI: app = FastAPI() lock = threading.Lock() - state = {"current_rid": None} + state: dict[str, Any] = {"current_rid": None} @app.get("/health") def health(): @@ -52,8 +54,7 @@ def health(): @app.get("/decode_status") def decode_status(): busy = lock.locked() - return {"status": "busy" if busy else "idle", - "current_rid": state["current_rid"]} + return {"status": "busy" if busy else "idle", "current_rid": state["current_rid"]} @app.post("/pd/cancel") def pd_cancel(body: dict): @@ -69,9 +70,10 @@ def pd_cancel(body: dict): ev.set() logger.info("cancel requested for %s", rid) return {"cancelled": rid} - return JSONResponse({"error": "no matching in-flight request", - "current_rid": state["current_rid"]}, - status_code=404) + return JSONResponse( + {"error": "no matching in-flight request", "current_rid": state["current_rid"]}, + status_code=404, + ) def _cleanup(): try: @@ -84,16 +86,20 @@ def _cleanup(): lock.release() def _log_reqstat(body, req, n_tokens, timing): - logger.info("REQSTAT rid=%s seq=%d completion=%d %s", - body.rid, req.seq_len, n_tokens, - " ".join(f"{k}={v}" for k, v in timing.items())) + logger.info( + "REQSTAT rid=%s seq=%d completion=%d %s", + body.rid, + req.seq_len, + n_tokens, + " ".join(f"{k}={v}" for k, v in timing.items()), + ) @app.post("/pd/decode") def pd_decode(body: DecodeBody): if not lock.acquire(blocking=False): return JSONResponse( - {"error": "busy", "current_rid": state["current_rid"]}, - status_code=429) + {"error": "busy", "current_rid": state["current_rid"]}, status_code=429 + ) state["current_rid"] = body.rid t0 = time.time() # phase 1: wire wait + convert + inject (common to both modes) @@ -104,33 +110,32 @@ def pd_decode(body: DecodeBody): deadline = time.time() + body.timeout_s while time.time() < deadline: try: - cand = server.completed.get( - timeout=max(0.1, deadline - time.time())) + cand = server.completed.get(timeout=max(0.1, deadline - time.time())) except queue_mod.Empty: break if cand.rid == body.rid: req = cand break - logger.warning("dropping unmatched request %s " - "(waiting for %s)", cand.rid, body.rid) + logger.warning( + "dropping unmatched request %s " "(waiting for %s)", cand.rid, body.rid + ) server.release() if req is None: _cleanup() return JSONResponse( - {"error": "kv_transfer_timeout", "rid": body.rid}, - status_code=504) + {"error": "kv_transfer_timeout", "rid": body.rid}, status_code=504 + ) t_recv = time.time() conv = server.profile.convert( - server.buffer, server.base_ptr, server.max_seq_len, - req, server.profile.num_ranks) + server.buffer, server.base_ptr, server.max_seq_len, req, server.profile.num_ranks + ) t_conv = time.time() engine.inject(conv) t_inj = time.time() except Exception as e: logger.exception("prepare failed for %s", body.rid) _cleanup() - return JSONResponse({"error": str(e), "rid": body.rid}, - status_code=500) + return JSONResponse({"error": str(e), "rid": body.rid}, status_code=500) pre_timing = { "wire_wait": round(1000 * (t_recv - t0), 1), @@ -150,16 +155,21 @@ def pd_decode(body: DecodeBody): sampling=body.sampling, cancel_event=cancel, ) - timing = {**pre_timing, - "decode": round(1000 * (time.time() - t_inj), 1), - **getattr(engine, "last_stats", {})} + timing = { + **pre_timing, + "decode": round(1000 * (time.time() - t_inj), 1), + **getattr(engine, "last_stats", {}), + } _log_reqstat(body, req, len(tokens), timing) - return {"rid": body.rid, "token_ids": tokens, - "seq_len": req.seq_len, "timing_ms": timing} + return { + "rid": body.rid, + "token_ids": tokens, + "seq_len": req.seq_len, + "timing_ms": timing, + } except Exception as e: logger.exception("decode failed for %s", body.rid) - return JSONResponse({"error": str(e), "rid": body.rid}, - status_code=500) + return JSONResponse({"error": str(e), "rid": body.rid}, status_code=500) finally: _cleanup() @@ -218,22 +228,28 @@ async def _gen(): if done_msg is None: if drained: last_activity = time.time() - elif time.time() - last_activity > 600: + elif time.time() - last_activity > 600: # noqa: R505 (exclusive branches) yield json.dumps({"error": "decode stalled"}) + "\n" return else: await asyncio.sleep(0.005) kind, payload = done_msg if kind == "done": - timing = {**pre_timing, - "decode": round(1000 * (time.time() - t_inj), 1), - **getattr(engine, "last_stats", {})} + timing = { + **pre_timing, + "decode": round(1000 * (time.time() - t_inj), 1), + **getattr(engine, "last_stats", {}), + } _log_reqstat(body, req, len(payload), timing) - yield json.dumps({ - "done": True, "n": len(payload), - "seq_len": req.seq_len, - "finish_reason": timing.get("finish_reason", "stop"), - "timing_ms": timing}) + "\n" + yield json.dumps( + { + "done": True, + "n": len(payload), + "seq_len": req.seq_len, + "finish_reason": timing.get("finish_reason", "stop"), + "timing_ms": timing, + } + ) + "\n" else: yield json.dumps({"error": payload}) + "\n" finally: @@ -244,18 +260,16 @@ async def _gen(): with anyio.CancelScope(shield=True): await run_in_threadpool(worker.join, 120) if worker.is_alive(): - logger.error("decode worker failed to stop for %s", - body.rid) + logger.error("decode worker failed to stop for %s", body.rid) _cleanup() return StreamingResponse(_gen(), media_type="application/x-ndjson") - return app + return app # noqa: R504 (assembled across the function) def main() -> None: - logging.basicConfig(level=logging.INFO, - format="%(asctime)s %(name)s %(message)s") + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s") ap = argparse.ArgumentParser() ap.add_argument("--engine", choices=["stub", "tilert"], default="stub") ap.add_argument("--model", default="glm5", help="model profile") @@ -264,41 +278,72 @@ def main() -> None: ap.add_argument("--http-port", type=int, default=5557) ap.add_argument("--model-weights-dir", default="") ap.add_argument("--with-mtp", action="store_true") - ap.add_argument("--transport", choices=["mooncake", "nixl"], - default="mooncake", help="RDMA data-plane backend " - "(must match prefill's tilert_transport)") - ap.add_argument("--kv-cache-dtype", default="fp8_ds_mla", - help="MLA cache dtype (must match vLLM prefill); " - "MLA-family profiles only") + ap.add_argument( + "--transport", + choices=["mooncake", "nixl"], + default="mooncake", + help="RDMA data-plane backend " "(must match prefill's tilert_transport)", + ) + ap.add_argument( + "--kv-cache-dtype", + default="fp8_ds_mla", + help="MLA cache dtype (must match vLLM prefill); " "MLA-family profiles only", + ) args = ap.parse_args() from tilert.pd_vllm.profiles import base as profiles + profile = profiles.get_profile(args.model) # MLA-family profiles (glm5/dsv32) need the cache dtype to size the receive # buffer. if hasattr(profile, "configure"): profile.configure(args.kv_cache_dtype) - logger.info("profile %s MLA cache dtype = %s (layout v%d)", - profile.name, args.kv_cache_dtype, profile.layout_version) + logger.info( + "profile %s MLA cache dtype = %s (layout v%d)", + profile.name, + args.kv_cache_dtype, + profile.layout_version, + ) if args.engine == "stub": from tilert.pd_vllm.engine_iface import StubEngine - engine = StubEngine() + + engine: Any = StubEngine() else: - logger.info("loading TileRT engine (profile=%s, weights=%s)...", - profile.name, args.model_weights_dir) + logger.info( + "loading TileRT engine (profile=%s, weights=%s)...", + profile.name, + args.model_weights_dir, + ) engine = profile.build_engine( model_weights_dir=args.model_weights_dir, - max_seq_len=args.max_seq_len, with_mtp=args.with_mtp, - ar_steps=8) + max_seq_len=args.max_seq_len, + with_mtp=args.with_mtp, + ar_steps=8, + ) logger.info("TileRT engine ready (cache window %d)", engine.max_seq_len) - server = ReceiveServer(profile, max_seq_len=args.max_seq_len, - ctrl_port=args.ctrl_port, transport=args.transport) + server = ReceiveServer( + profile, max_seq_len=args.max_seq_len, ctrl_port=args.ctrl_port, transport=args.transport + ) app = build_app(server, engine) - logger.info("decode server on :%d (profile=%s, engine=%s, ctrl=:%d)", - args.http_port, profile.name, args.engine, args.ctrl_port) - uvicorn.run(app, host="::", port=args.http_port, log_level="warning") + logger.info( + "decode server on :%d (profile=%s, engine=%s, ctrl=:%d)", + args.http_port, + profile.name, + args.engine, + args.ctrl_port, + ) + # Bind dual-stack (IPv4 + IPv6) explicitly. uvicorn's host="::" is + # IPv6-only under some uvicorn/OS combinations, which leaves the decode + # HTTP endpoint unreachable from an IPv4 router. Mirror the control plane + # (receive_server) by clearing IPV6_V6ONLY on an AF_INET6 socket. + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + with contextlib.suppress(OSError): + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", args.http_port)) + uvicorn.Server(uvicorn.Config(app, log_level="warning")).run(sockets=[sock]) if __name__ == "__main__": diff --git a/tilert/pd_vllm/engine_iface.py b/tilert/pd_vllm/engine_iface.py index 9719e50..6ed9133 100644 --- a/tilert/pd_vllm/engine_iface.py +++ b/tilert/pd_vllm/engine_iface.py @@ -5,7 +5,8 @@ ``StubEngine`` runs the whole serving path with no GPU / no tilert. """ -from typing import Any, Callable, Protocol +from collections.abc import Callable +from typing import Any, Protocol class PDEngine(Protocol): @@ -20,10 +21,12 @@ def decode( on_token: Callable[[int], None] | None = None, cancel_event=None, ) -> list[int]: - """AR/MTP decode from first_token_id; returns completion ids - (incl. first_token_id, excl. stop token). on_token never fires for - stop tokens; cancel_event stops early; last_stats['finish_reason'] is - 'stop' | 'length' | 'cancelled'.""" + """AR/MTP decode from first_token_id; returns completion ids. + + Includes first_token_id, excludes the stop token. on_token never fires + for stop tokens; cancel_event stops early; last_stats['finish_reason'] + is 'stop' | 'length' | 'cancelled'. + """ def reset(self) -> None: """Release per-request state.""" @@ -40,8 +43,7 @@ def __init__(self, fixed_tokens: tuple[int, ...] = (11, 22, 33)): def inject(self, req: Any) -> None: self.injected = req - def decode(self, first_token_id, max_tokens, sampling, on_token=None, - cancel_event=None): + def decode(self, first_token_id, max_tokens, sampling, on_token=None, cancel_event=None): out = ([int(first_token_id)] + list(self._fixed))[:max_tokens] if on_token: for t in out: diff --git a/tilert/pd_vllm/oai_parser.py b/tilert/pd_vllm/oai_parser.py index 3840f43..931daa9 100644 --- a/tilert/pd_vllm/oai_parser.py +++ b/tilert/pd_vllm/oai_parser.py @@ -14,7 +14,6 @@ node never imports vllm. """ -import json import logging import uuid from dataclasses import dataclass, field @@ -24,14 +23,14 @@ @dataclass class ToolCall: - id: str + call_id: str name: str arguments: str # JSON string (OpenAI convention) def to_openai(self, index: int) -> dict: return { "index": index, - "id": self.id, + "id": self.call_id, "type": "function", "function": {"name": self.name, "arguments": self.arguments}, } @@ -53,21 +52,21 @@ def _new_call_id() -> str: # builder + a `_*_arg_converter(raw, partial)`); the adapter picks the engine # by family name. _FAMILIES = { - "glm47": ("vllm.parser.glm47_moe", "glm47_moe_config", - "_glm47_arg_converter"), + "glm47": ("vllm.parser.glm47_moe", "glm47_moe_config", "_glm47_arg_converter"), } def make_parser(family: str, tokenizer, thinking: bool = True) -> "OaiParser": if family not in _FAMILIES: - raise KeyError(f"unknown parser family {family!r}; " - f"known: {sorted(_FAMILIES)}") + raise KeyError(f"unknown parser family {family!r}; " f"known: {sorted(_FAMILIES)}") return OaiParser(family, tokenizer, thinking) class OaiParser: - """Family-parameterized parser; one instance per model, ``stream()`` per - request. Family is a vllm.parser engine (glm47).""" + """Family-parameterized parser; one instance per model, ``stream()`` per request. + + Family is a vllm.parser engine (glm47). + """ def __init__(self, family: str, tokenizer, thinking: bool = True): import importlib @@ -191,12 +190,17 @@ def _flush_tool(self, index: int, partial: bool) -> dict | None: return None self._emitted.add(index) args = self._p._convert("".join(s["args"]), partial) - return {"kind": "tool", "index": index, "id": _new_call_id(), - "name": name, "arguments": args} + return { + "kind": "tool", + "index": index, + "id": _new_call_id(), + "name": name, + "arguments": args, + } class IncrementalDetok: - """Incremental token→text for byte-level BPE tokenizers. + r"""Incremental token→text for byte-level BPE tokenizers. Decodes a bounded trailing window; holds output while the window ends in a partial multi-byte sequence (\\ufffd). Window folding is safe for @@ -218,9 +222,9 @@ def push(self, ids: list[int]) -> str: text = self._tok.decode(self._ids, skip_special_tokens=False) if text.endswith("�"): return "" - delta = text[self._emitted:] + delta = text[self._emitted :] self._emitted = len(text) if len(self._ids) > self._FOLD: self._ids = [] self._emitted = 0 - return delta + return delta # noqa: R504 (self._emitted mutated after delta is computed) diff --git a/tilert/pd_vllm/pd_router.py b/tilert/pd_vllm/pd_router.py index d373261..87a61e5 100644 --- a/tilert/pd_vllm/pd_router.py +++ b/tilert/pd_vllm/pd_router.py @@ -1,6 +1,7 @@ -"""PD router (W6): client-facing entry that orchestrates vLLM prefill and -TileRT decode, with OpenAI-semantics output parsing (reasoning + tool calls), -streaming and non-streaming. +"""PD router (W6): client-facing entry over vLLM prefill + TileRT decode. + +Does OpenAI-semantics output parsing (reasoning + tool calls), streaming and +non-streaming. Flow per request (phase-1 hybrid, see design doc): 1. pick a free decode node (in-memory busy tracking; all busy -> 429) @@ -87,7 +88,8 @@ def first_token_from_logprobs(resp: dict, is_chat: bool) -> int: return int(tok.split(":", 1)[1]) raise ValueError( f"cannot extract first token id from logprobs ({tok!r}); launch vLLM " - f"with --return-tokens-as-token-ids and request logprobs") + f"with --return-tokens-as-token-ids and request logprobs" + ) def _thinking_enabled(body: dict) -> bool: @@ -108,11 +110,10 @@ def __init__(self, vllm_url: str, pool: Pool, tokenizer, parser_name: str): if tokenizer is None: raise SystemExit("--parser requires --model-path (tokenizer)") from tilert.pd_vllm.oai_parser import make_parser - self._parsers[True] = make_parser(parser_name, tokenizer, - thinking=True) + + self._parsers[True] = make_parser(parser_name, tokenizer, thinking=True) self._parsers[False] = self._parsers[True].with_thinking(False) - logger.info("parser '%s' ready (thinking variants cached)", - parser_name) + logger.info("parser '%s' ready (thinking variants cached)", parser_name) def parser(self, thinking: bool): return self._parsers.get(thinking) @@ -124,13 +125,11 @@ def build_app(ctx: RouterCtx) -> FastAPI: @app.get("/health") def health(): - return {"status": "ok", - "decode_free": sum(1 for n in pool.nodes if not n.busy)} + return {"status": "ok", "decode_free": sum(1 for n in pool.nodes if not n.busy)} @app.get("/pool_status") def pool_status(): - return {"nodes": [{"host": n.host, "busy": n.busy} - for n in pool.nodes]} + return {"nodes": [{"host": n.host, "busy": n.busy} for n in pool.nodes]} # ── shared prefill step ────────────────────────────────────────────── def _prefill(path, body, node): @@ -146,26 +145,22 @@ def _prefill(path, body, node): "tilert_host": node.host, "tilert_ctrl_port": node.ctrl_port, } - r = requests.post(f"{ctx.vllm_url}{path}", json=prefill_body, - timeout=600) + r = requests.post(f"{ctx.vllm_url}{path}", json=prefill_body, timeout=600) r.raise_for_status() return r.json() def _sampling_of(body): - return {k: body[k] for k in ("temperature", "top_p", "top_k") - if k in body} + return {k: body[k] for k in ("temperature", "top_p", "top_k") if k in body} def _max_tokens_of(body): - return int(body.get("max_tokens") - or body.get("max_completion_tokens") or 256) + return int(body.get("max_tokens") or body.get("max_completion_tokens") or 256) # ── non-streaming ──────────────────────────────────────────────────── def _handle(path: str, body: dict): is_chat = path.endswith("chat/completions") node = pool.acquire() if node is None: - return JSONResponse({"error": "all decode nodes busy"}, - status_code=429) + return JSONResponse({"error": "all decode nodes busy"}, status_code=429) t0 = time.time() try: prefill = _prefill(path, body, node) @@ -173,12 +168,16 @@ def _handle(path: str, body: dict): rid = derive_rid(prefill["id"]) first_token_id = first_token_from_logprobs(prefill, is_chat) - dr = requests.post(f"{node.http_base}/pd/decode", json={ - "rid": rid, - "first_token_id": first_token_id, - "max_tokens": _max_tokens_of(body), - "sampling": _sampling_of(body), - }, timeout=600) + dr = requests.post( + f"{node.http_base}/pd/decode", + json={ + "rid": rid, + "first_token_id": first_token_id, + "max_tokens": _max_tokens_of(body), + "sampling": _sampling_of(body), + }, + timeout=600, + ) dr.raise_for_status() decode = dr.json() token_ids = decode["token_ids"] @@ -190,44 +189,44 @@ def _handle(path: str, body: dict): choice: dict = {"index": 0, "finish_reason": finish} parser = ctx.parser(_thinking_enabled(body)) if is_chat else None if parser is not None: - text = ctx.tokenizer.decode(token_ids, - skip_special_tokens=False) + text = ctx.tokenizer.decode(token_ids, skip_special_tokens=False) parsed = parser.parse_complete(text) msg = {"role": "assistant", "content": parsed.content or ""} if parsed.reasoning_content: msg["reasoning_content"] = parsed.reasoning_content if parsed.tool_calls: - msg["tool_calls"] = [c.to_openai(i) for i, c - in enumerate(parsed.tool_calls)] + msg["tool_calls"] = [c.to_openai(i) for i, c in enumerate(parsed.tool_calls)] choice["finish_reason"] = "tool_calls" choice["message"] = msg else: - text = (ctx.tokenizer.decode(token_ids, - skip_special_tokens=True) - if ctx.tokenizer else None) + text = ( + ctx.tokenizer.decode(token_ids, skip_special_tokens=True) + if ctx.tokenizer + else None + ) if is_chat: choice["message"] = {"role": "assistant", "content": text} else: choice["text"] = text choice["token_ids"] = token_ids - return JSONResponse({ - "id": prefill["id"], - "object": ("chat.completion" if is_chat - else "text_completion"), - "created": int(time.time()), - "model": prefill.get("model"), - "choices": [choice], - "usage": { - "prompt_tokens": (prefill.get("usage") or {}) - .get("prompt_tokens"), - "completion_tokens": len(token_ids), - }, - "pd_timing_ms": { - "prefill": round(1000 * (t_prefill - t0), 1), - **timing, - }, - }) + return JSONResponse( + { + "id": prefill["id"], + "object": "chat.completion" if is_chat else "text_completion", + "created": int(time.time()), + "model": prefill.get("model"), + "choices": [choice], + "usage": { + "prompt_tokens": (prefill.get("usage") or {}).get("prompt_tokens"), + "completion_tokens": len(token_ids), + }, + "pd_timing_ms": { + "prefill": round(1000 * (t_prefill - t0), 1), + **timing, + }, + } + ) except Exception as e: logger.exception("pd request failed") return JSONResponse({"error": str(e)}, status_code=502) @@ -240,8 +239,7 @@ async def _handle_stream(path: str, body: dict, request: Request): node = pool.acquire() if node is None: - return JSONResponse({"error": "all decode nodes busy"}, - status_code=429) + return JSONResponse({"error": "all decode nodes busy"}, status_code=429) try: prefill = await run_in_threadpool(_prefill, path, body, node) @@ -259,10 +257,11 @@ async def _handle_stream(path: str, body: dict, request: Request): def _chunk(delta: dict, finish=None, usage=None) -> str: payload = { - "id": chunk_id, "object": "chat.completion.chunk", - "created": int(time.time()), "model": model, - "choices": [{"index": 0, "delta": delta, - "finish_reason": finish}], + "id": chunk_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], } if usage is not None: payload["usage"] = usage @@ -273,16 +272,20 @@ def _event_delta(ev: dict) -> dict: return {"reasoning_content": ev["text"]} if ev["kind"] == "content": return {"content": ev["text"]} - return {"tool_calls": [{ - "index": ev["index"], "id": ev["id"], "type": "function", - "function": {"name": ev["name"], - "arguments": ev["arguments"]}, - }]} + return { + "tool_calls": [ + { + "index": ev["index"], + "id": ev["id"], + "type": "function", + "function": {"name": ev["name"], "arguments": ev["arguments"]}, + } + ] + } def _fire_cancel(): try: - requests.post(f"{node.http_base}/pd/cancel", - json={"rid": rid}, timeout=5) + requests.post(f"{node.http_base}/pd/cancel", json={"rid": rid}, timeout=5) except Exception: logger.warning("cancel POST failed for %s", rid) @@ -303,11 +306,16 @@ async def _gen(): try: yield _chunk({"role": "assistant"}) async with client.stream( - "POST", f"{node.http_base}/pd/decode", - json={"rid": rid, "first_token_id": first_token_id, - "max_tokens": _max_tokens_of(body), - "sampling": _sampling_of(body), - "stream": True}) as resp: + "POST", + f"{node.http_base}/pd/decode", + json={ + "rid": rid, + "first_token_id": first_token_id, + "max_tokens": _max_tokens_of(body), + "sampling": _sampling_of(body), + "stream": True, + }, + ) as resp: resp.raise_for_status() async for line in resp.aiter_lines(): # Deterministic client-liveness check: writes to a @@ -315,8 +323,7 @@ async def _gen(): # poll the ASGI disconnect state every line. if await request.is_disconnected(): client_gone = True - logger.info("client disconnected, cancelling %s", - rid) + logger.info("client disconnected, cancelling %s", rid) break if not line: continue @@ -338,8 +345,7 @@ async def _gen(): if finish_reason == "cancelled": finish_reason = "stop" elif "error" in msg: - yield _chunk( - {"content": f"\n[decode error: {msg['error']}]"}) + yield _chunk({"content": f"\n[decode error: {msg['error']}]"}) finish_reason = "stop" if client_gone: logger.info("client gone mid-stream for %s", rid) @@ -351,10 +357,14 @@ async def _gen(): yield _chunk(_event_delta(ev)) if saw_tool: finish_reason = "tool_calls" - yield _chunk({}, finish=finish_reason, usage={ - "prompt_tokens": prompt_tokens, - "completion_tokens": n_tokens, - }) + yield _chunk( + {}, + finish=finish_reason, + usage={ + "prompt_tokens": prompt_tokens, + "completion_tokens": n_tokens, + }, + ) yield "data: [DONE]\n\n" completed_ok = True except Exception: @@ -390,27 +400,31 @@ async def completions(request: Request): body = await request.json() if body.get("stream"): return JSONResponse( - {"error": "streaming is supported on /v1/chat/completions"}, - status_code=400) + {"error": "streaming is supported on /v1/chat/completions"}, status_code=400 + ) return await run_in_threadpool(_handle, "/v1/completions", body) - return app + return app # noqa: R504 (assembled across the function) def main() -> None: - logging.basicConfig(level=logging.INFO, - format="%(asctime)s %(name)s %(message)s") + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s") ap = argparse.ArgumentParser() ap.add_argument("--vllm-url", required=True) - ap.add_argument("--decode", nargs="+", required=True, - help="decode nodes as host:ctrl_port:http_port") - ap.add_argument("--host", default="0.0.0.0") + ap.add_argument( + "--decode", nargs="+", required=True, help="decode nodes as host:ctrl_port:http_port" + ) + ap.add_argument("--host", default="0.0.0.0") # nosec B104 (bind-all by design) ap.add_argument("--port", type=int, default=23333) - ap.add_argument("--model-path", default="", - help="tokenizer path (required unless --parser none)") - ap.add_argument("--parser", choices=["glm47", "none"], - default="glm47", - help="output parser (reasoning + tool calls)") + ap.add_argument( + "--model-path", default="", help="tokenizer path (required unless --parser none)" + ) + ap.add_argument( + "--parser", + choices=["glm47", "none"], + default="glm47", + help="output parser (reasoning + tool calls)", + ) args = ap.parse_args() nodes = [] @@ -421,13 +435,20 @@ def main() -> None: tokenizer = None if args.model_path: from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained( - args.model_path, trust_remote_code=True) # nosec B615 + args.model_path, trust_remote_code=True + ) # nosec B615 ctx = RouterCtx(args.vllm_url, Pool(nodes), tokenizer, args.parser) app = build_app(ctx) - logger.info("router on :%d -> vllm=%s, %d decode node(s), parser=%s", - args.port, args.vllm_url, len(nodes), args.parser) + logger.info( + "router on :%d -> vllm=%s, %d decode node(s), parser=%s", + args.port, + args.vllm_url, + len(nodes), + args.parser, + ) uvicorn.run(app, host=args.host, port=args.port, log_level="warning") diff --git a/tilert/pd_vllm/prefill_connector.py b/tilert/pd_vllm/prefill_connector.py index 0237958..abd4819 100644 --- a/tilert/pd_vllm/prefill_connector.py +++ b/tilert/pd_vllm/prefill_connector.py @@ -59,6 +59,7 @@ class TileRTMetadata(KVConnectorMetadata): @dataclass class _Pending: """Scheduler-side chunked-prefill accumulation.""" + req_id: str prompt_token_ids: list total_tokens: int @@ -75,8 +76,7 @@ def __init__(self, vllm_config, role, kv_cache_config=None): self._default_host = extra.get("tilert_host") self._default_port = int(extra.get("tilert_ctrl_port", 5556)) self._sync_send = bool(extra.get("tilert_sync_send", False)) - self._max_seq = int(extra.get("tilert_max_seq_len", - vllm_config.model_config.max_model_len)) + self._max_seq = int(extra.get("tilert_max_seq_len", vllm_config.model_config.max_model_len)) self._profile = profiles.get_profile(extra.get("tilert_model", "glm5")) self._transport_name = extra.get("tilert_transport", "mooncake") @@ -85,16 +85,21 @@ def __init__(self, vllm_config, role, kv_cache_config=None): # worker-side (lazy) self._kv_caches: dict = {} - self._reg = None # profile registration (layer map) + self._reg = None # profile registration (layer map) self._tp_rank: int | None = None self._transport = None self._staging = None self._send_q: queue.Queue = queue.Queue() self._sender_thread: threading.Thread | None = None - logger.info("TileRTConnector: role=%s profile=%s target=%s:%s sync=%s", - role, self._profile.name, self._default_host, - self._default_port, self._sync_send) + logger.info( + "TileRTConnector: role=%s profile=%s target=%s:%s sync=%s", + role, + self._profile.name, + self._default_host, + self._default_port, + self._sync_send, + ) # ══════════════════════ scheduler side ═════════════════════ @@ -137,13 +142,15 @@ def build_connector_meta(self, scheduler_output) -> KVConnectorMetadata: groups = [list(g) for g in new_req.block_ids] n = num_sched.get(new_req.req_id, 0) if new_req.num_computed_tokens + n >= len(token_ids): - meta.requests.append(self._emit(new_req.req_id, token_ids, - groups, params)) + meta.requests.append(self._emit(new_req.req_id, token_ids, groups, params)) else: self._pending[new_req.req_id] = _Pending( - req_id=new_req.req_id, prompt_token_ids=token_ids, - total_tokens=len(token_ids), block_ids_per_group=groups, - params=params) + req_id=new_req.req_id, + prompt_token_ids=token_ids, + total_tokens=len(token_ids), + block_ids_per_group=groups, + params=params, + ) cached = scheduler_output.scheduled_cached_reqs for i, req_id in enumerate(getattr(cached, "req_ids", []) or []): @@ -157,25 +164,33 @@ def build_connector_meta(self, scheduler_output) -> KVConnectorMetadata: p.block_ids_per_group[gi].extend(g) n = num_sched.get(req_id, 0) if cached.num_computed_tokens[i] + n >= p.total_tokens: - meta.requests.append(self._emit(req_id, p.prompt_token_ids, - p.block_ids_per_group, p.params)) + meta.requests.append( + self._emit(req_id, p.prompt_token_ids, p.block_ids_per_group, p.params) + ) del self._pending[req_id] return meta def _emit(self, req_id, token_ids, groups, params) -> _ReqMeta: + host = params.get("tilert_host") or self._default_host + assert host is not None, "claimed a request with no tilert_host" m = _ReqMeta( req_id=req_id, rid=derive_rid(req_id), num_tokens=len(token_ids), last_prompt_token=int(token_ids[-1]), block_ids_per_group=groups, - tilert_host=params.get("tilert_host") or self._default_host, - tilert_ctrl_port=int(params.get("tilert_ctrl_port", - self._default_port)), + tilert_host=host, + tilert_ctrl_port=int(params.get("tilert_ctrl_port", self._default_port)), sampling=params.get("sampling"), ) - logger.info("claimed %s (rid=%s, %d tokens) -> %s:%d", - req_id, m.rid, m.num_tokens, m.tilert_host, m.tilert_ctrl_port) + logger.info( + "claimed %s (rid=%s, %d tokens) -> %s:%d", + req_id, + m.rid, + m.num_tokens, + m.tilert_host, + m.tilert_ctrl_port, + ) return m def request_finished(self, request, block_ids): @@ -197,7 +212,8 @@ def _ensure_worker_ready(self) -> None: return import torch from vllm.distributed import get_tensor_model_parallel_rank - self._tp_rank = get_tensor_model_parallel_rank() + + self._tp_rank = int(get_tensor_model_parallel_rank()) from tilert.pd_vllm.transport import make_transport @@ -212,11 +228,16 @@ def _ensure_worker_ready(self) -> None: if not self._sync_send: self._sender_thread = threading.Thread( - target=self._sender_loop, name="tilert-pd-sender", daemon=True) + target=self._sender_loop, name="tilert-pd-sender", daemon=True + ) self._sender_thread.start() - logger.info("worker ready: rank=%d transport=%s staging=%.1f MB profile=%s", - self._tp_rank, self._transport.name, total / 1e6, - self._profile.name) + logger.info( + "worker ready: rank=%d transport=%s staging=%.1f MB profile=%s", + self._tp_rank, + self._transport.name, + total / 1e6, + self._profile.name, + ) def start_load_kv(self, forward_context, **kwargs): pass @@ -232,12 +253,14 @@ def wait_for_save(self): if not isinstance(metadata, TileRTMetadata) or not metadata.requests: return self._ensure_worker_ready() + assert self._tp_rank is not None # set by _ensure_worker_ready if self._tp_rank not in self._profile.sender_ranks: return # this rank does not participate (e.g. replicated MLA) for m in metadata.requests: try: sections = self._profile.extract( - self._reg, m, self._tp_rank, self._staging, self._max_seq) + self._reg, m, self._tp_rank, self._staging, self._max_seq + ) except Exception: logger.exception("extraction failed for %s", m.rid) continue @@ -264,6 +287,9 @@ def _send(self, job: dict) -> None: import socket as _socket import time as _time + # _send only runs after wait_for_save() -> _ensure_worker_ready() + assert self._transport is not None and self._staging is not None + assert self._tp_rank is not None m: _ReqMeta = job["meta"] seq = job["seq"] t0 = _time.time() @@ -274,31 +300,42 @@ def _send(self, job: dict) -> None: conn.connect((m.tilert_host, m.tilert_ctrl_port)) hello = wire.recv_msg(conn) assert hello.get("magic") == wire.MAGIC, f"bad hello: {hello}" - assert hello.get("layout_version") == self._profile.layout_version, \ - (f"layout version mismatch: {hello.get('layout_version')} " - f"vs {self._profile.layout_version}") - assert hello.get("transport") == self._transport.name, \ - (f"transport mismatch: decode={hello.get('transport')} " - f"vs prefill={self._transport.name}") + assert hello.get("layout_version") == self._profile.layout_version, ( + f"layout version mismatch: {hello.get('layout_version')} " + f"vs {self._profile.layout_version}" + ) + assert hello.get("transport") == self._transport.name, ( + f"transport mismatch: decode={hello.get('transport')} " + f"vs prefill={self._transport.name}" + ) remote_max_seq = int(hello["max_seq_len"]) - assert seq <= remote_max_seq, \ - f"seq {seq} exceeds decode max_seq_len {remote_max_seq}" - - wire.send_msg(conn, { - "rid": m.rid, "rank": self._tp_rank, "seq_len": seq, - "last_prompt_token": m.last_prompt_token, - "sampling": m.sampling, - }) + assert seq <= remote_max_seq, f"seq {seq} exceeds decode max_seq_len {remote_max_seq}" + + wire.send_msg( + conn, + { + "rid": m.rid, + "rank": self._tp_rank, + "seq_len": seq, + "last_prompt_token": m.last_prompt_token, + "sampling": m.sampling, + }, + ) base = self._staging.data_ptr() srcs, dsts, lens = self._profile.rdma_plan( - hello, job["sections"], self._tp_rank, seq, base) + hello, job["sections"], self._tp_rank, seq, base + ) self._transport.write(hello, srcs, dsts, lens) - wire.send_msg(conn, {"done": True, "rid": m.rid, - "rank": self._tp_rank}) - logger.info("sent %s: rank=%d seq=%d %.1f MB in %.1f ms", - m.rid, self._tp_rank, seq, sum(lens) / 1e6, - 1000 * (_time.time() - t0)) + wire.send_msg(conn, {"done": True, "rid": m.rid, "rank": self._tp_rank}) + logger.info( + "sent %s: rank=%d seq=%d %.1f MB in %.1f ms", + m.rid, + self._tp_rank, + seq, + sum(lens) / 1e6, + 1000 * (_time.time() - t0), + ) finally: conn.close() diff --git a/tilert/pd_vllm/profiles/base.py b/tilert/pd_vllm/profiles/base.py index 08302c4..f1231d7 100644 --- a/tilert/pd_vllm/profiles/base.py +++ b/tilert/pd_vllm/profiles/base.py @@ -16,58 +16,73 @@ class ModelProfile(Protocol): name: str num_ranks: int - layout_version: int sender_ranks: frozenset + @property + def layout_version(self) -> int: ... + # ── receive side (decode node) ─────────────────────────────────────── def buffer_bytes(self, max_seq_len: int) -> int: """Total receive-buffer size for one request slot.""" def hello_layout(self, base_ptr: int, max_seq_len: int) -> dict[str, int]: - """Region base addresses (merged into the hello message) so the - sender knows where to RDMA-write each section.""" + """Region base addresses, merged into the hello message. + + Tells the sender where to RDMA-write each section. + """ - def convert(self, buffer: Any, base_ptr: int, max_seq_len: int, - received: Any, num_devices: int) -> Any: + def convert( + self, buffer: Any, base_ptr: int, max_seq_len: int, received: Any, num_devices: int + ) -> Any: """Received buffer -> native per-device tensors (ConvertedRequest).""" # ── prefill side (vLLM connector worker) ───────────────────────────── def classify_layers(self, kv_caches: dict, kv_cache_config: Any) -> Any: - """Inspect registered kv_caches; return an opaque registration the - framework passes back to ``staging_bytes``/``extract``. Raise on an - unexpected layer set (e.g. missing speculative layer).""" + """Inspect registered kv_caches and return an opaque registration. + + The framework passes it back to ``staging_bytes``/``extract``. Raise on + an unexpected layer set (e.g. missing speculative layer). + """ def staging_bytes(self, reg: Any, tp_rank: int, max_seq_len: int) -> int: """Per-rank staging-buffer size.""" - def extract(self, reg: Any, req_meta: Any, tp_rank: int, - staging, max_seq_len: int) -> Any: - """Copy this rank's KV out of the paged caches into ``staging`` - (inside the forward window); return opaque ``sections``.""" + def extract(self, reg: Any, req_meta: Any, tp_rank: int, staging, max_seq_len: int) -> Any: + """Copy this rank's KV out of the paged caches into ``staging``. + + Runs inside the forward window; returns opaque ``sections``. + """ - def rdma_plan(self, hello: dict, sections: Any, tp_rank: int, - seq_len: int, staging_base: int) -> tuple[list, list, list]: + def rdma_plan( + self, hello: dict, sections: Any, tp_rank: int, seq_len: int, staging_base: int + ) -> tuple[list, list, list]: """(src_ptrs, dst_ptrs, lengths) for one mooncake batch write.""" # ── engine (decode node) ───────────────────────────────────────────── - def build_engine(self, model_weights_dir: str, max_seq_len: int, - with_mtp: bool, ar_steps: int) -> Any: + def build_engine( + self, model_weights_dir: str, max_seq_len: int, with_mtp: bool, ar_steps: int + ) -> Any: """Construct the decode engine adapter (inject/decode/reset).""" -_REGISTRY: dict[str, "ModelProfile"] = {} +_REGISTRY: dict[str, ModelProfile] = {} _ALIASES = { - "glm5": "glm5", "glm_5": "glm5", "glm-5": "glm5", - "dsv32": "dsv32", "deepseek_v3_2": "dsv32", "deepseek-v3.2": "dsv32", - "dsv3.2": "dsv32", "v32": "dsv32", + "glm5": "glm5", + "glm_5": "glm5", + "glm-5": "glm5", + "dsv32": "dsv32", + "deepseek_v3_2": "dsv32", + "deepseek-v3.2": "dsv32", + "dsv3.2": "dsv32", + "v32": "dsv32", } -def register(profile: "ModelProfile") -> None: +def register(profile: ModelProfile) -> None: _REGISTRY[profile.name] = profile -def get_profile(name: str) -> "ModelProfile": +def get_profile(name: str) -> ModelProfile: canon = _ALIASES.get(name, name) if canon not in _REGISTRY: # lazy import so a profile's heavy deps load only when selected @@ -76,6 +91,7 @@ def get_profile(name: str) -> "ModelProfile": elif canon == "dsv32": from tilert.pd_vllm.profiles import dsv32 # noqa: F401 if canon not in _REGISTRY: - raise KeyError(f"unknown model profile {name!r}; " - f"accepted keys (incl. aliases): {sorted(_ALIASES)}") + raise KeyError( + f"unknown model profile {name!r}; " f"accepted keys (incl. aliases): {sorted(_ALIASES)}" + ) return _REGISTRY[canon] diff --git a/tilert/pd_vllm/profiles/dsv32.py b/tilert/pd_vllm/profiles/dsv32.py index 0beebf2..5691772 100644 --- a/tilert/pd_vllm/profiles/dsv32.py +++ b/tilert/pd_vllm/profiles/dsv32.py @@ -8,12 +8,13 @@ MlaNsaProfile, ) -NUM_LAYERS = 62 # 61 main + 1 MTP draft (HF: 61 hidden + 1 nextn) -LAYOUT_VERSION = 11 # dsv32 wire family (distinct from glm5's 10) +NUM_LAYERS = 62 # 61 main + 1 MTP draft (HF: 61 hidden + 1 nextn) +LAYOUT_VERSION = 11 # dsv32 wire family (distinct from glm5's 10) def _build_engine(model_weights_dir, max_seq_len, with_mtp, ar_steps): import tilert + if hasattr(tilert, "load_backend"): tilert.load_backend("deepseek_v3_2") # multi-backend builds only from tilert.models.deepseek_v3_2.generator import DSAv32Generator @@ -30,6 +31,11 @@ def _build_engine(model_weights_dir, max_seq_len, with_mtp, ar_steps): return MlaNsaEngineAdapter(gen, with_mtp) -base.register(MlaNsaProfile( - name="dsv32", num_layers=NUM_LAYERS, layout_version=LAYOUT_VERSION, - engine_factory=_build_engine)) +base.register( + MlaNsaProfile( + name="dsv32", + num_layers=NUM_LAYERS, + layout_version=LAYOUT_VERSION, + engine_factory=_build_engine, + ) +) diff --git a/tilert/pd_vllm/profiles/glm5.py b/tilert/pd_vllm/profiles/glm5.py index b0245b1..c98d7e9 100644 --- a/tilert/pd_vllm/profiles/glm5.py +++ b/tilert/pd_vllm/profiles/glm5.py @@ -13,12 +13,13 @@ MlaNsaProfile, ) -NUM_LAYERS = 79 # 78 main + 1 MTP draft -LAYOUT_VERSION = 10 # glm5 wire family +NUM_LAYERS = 79 # 78 main + 1 MTP draft +LAYOUT_VERSION = 10 # glm5 wire family def _build_engine(model_weights_dir, max_seq_len, with_mtp, ar_steps): import tilert + # multi-backend builds (tilert>=0.1.x) load the per-model .so on demand; # single-backend builds auto-register on import and lack load_backend. if hasattr(tilert, "load_backend"): @@ -38,6 +39,11 @@ def _build_engine(model_weights_dir, max_seq_len, with_mtp, ar_steps): return MlaNsaEngineAdapter(gen, with_mtp) -base.register(MlaNsaProfile( - name="glm5", num_layers=NUM_LAYERS, layout_version=LAYOUT_VERSION, - engine_factory=_build_engine)) +base.register( + MlaNsaProfile( + name="glm5", + num_layers=NUM_LAYERS, + layout_version=LAYOUT_VERSION, + engine_factory=_build_engine, + ) +) diff --git a/tilert/pd_vllm/profiles/mla_nsa.py b/tilert/pd_vllm/profiles/mla_nsa.py index c8c98c9..a270f7c 100644 --- a/tilert/pd_vllm/profiles/mla_nsa.py +++ b/tilert/pd_vllm/profiles/mla_nsa.py @@ -16,7 +16,7 @@ QK_ROPE_HEAD_DIM = 64 INDEX_HEAD_DIM = 128 KI_QUANT_BLOCK = 128 -KV_QUANT_BLOCK = 128 # per-128 fp8 scale on the kv latent +KV_QUANT_BLOCK = 128 # per-128 fp8 scale on the kv latent PAGE_SIZE = 64 # The MLA KV cache dtype is a launch choice (vLLM ``--kv-cache-dtype``), NOT @@ -28,16 +28,17 @@ # the decode side. (recommended, aligns with SGLang fp8) # bf16 : cache tensor [nblk, page, 576] bf16; per token 512 bf16 kv_c + # 64 bf16 k_pe = 1024-B kv plane + 128-B pe plane, no dequant. -KV_FP8_BYTES = KV_LORA_RANK # 512 (fp8, 1 B each) -KV_SCALE_BYTES = KV_LORA_RANK // KV_QUANT_BLOCK * 4 # 16 (4 fp32 scales) -KV_BYTES_FP8 = KV_FP8_BYTES + KV_SCALE_BYTES # 528 B/token -KV_BYTES_BF16 = KV_LORA_RANK * 2 # 1024 B/token -PE_BPT = QK_ROPE_HEAD_DIM * 2 # 128 B/token bf16 (both) -MLA_BPT_FP8 = KV_BYTES_FP8 + PE_BPT # 656 (fp8 cache stride) -MLA_BPT_BF16 = (KV_LORA_RANK + QK_ROPE_HEAD_DIM) * 2 # 1152 (bf16 cache stride) -_VERSION_BF16_OFFSET = 40 # bf16 layout_version = base + 40 -KI_PAGE_BYTES = (PAGE_SIZE * INDEX_HEAD_DIM - + PAGE_SIZE * INDEX_HEAD_DIM // KI_QUANT_BLOCK * 4) # 8448 +KV_FP8_BYTES = KV_LORA_RANK # 512 (fp8, 1 B each) +KV_SCALE_BYTES = KV_LORA_RANK // KV_QUANT_BLOCK * 4 # 16 (4 fp32 scales) +KV_BYTES_FP8 = KV_FP8_BYTES + KV_SCALE_BYTES # 528 B/token +KV_BYTES_BF16 = KV_LORA_RANK * 2 # 1024 B/token +PE_BPT = QK_ROPE_HEAD_DIM * 2 # 128 B/token bf16 (both) +MLA_BPT_FP8 = KV_BYTES_FP8 + PE_BPT # 656 (fp8 cache stride) +MLA_BPT_BF16 = (KV_LORA_RANK + QK_ROPE_HEAD_DIM) * 2 # 1152 (bf16 cache stride) +_VERSION_BF16_OFFSET = 40 # bf16 layout_version = base + 40 +KI_PAGE_BYTES = ( + PAGE_SIZE * INDEX_HEAD_DIM + PAGE_SIZE * INDEX_HEAD_DIM // KI_QUANT_BLOCK * 4 +) # 8448 def _max_pages(max_seq_len: int) -> int: @@ -45,15 +46,19 @@ def _max_pages(max_seq_len: int) -> int: def _hadamard(x: torch.Tensor) -> torch.Tensor: - """Hadamard rotation of the last dim (scale d^-0.5), matching TileRT's - indexer. fast_hadamard_transform if present, else a scipy matmul.""" + """Hadamard rotation of the last dim (scale d^-0.5), matching TileRT's indexer. + + Uses fast_hadamard_transform if present, else a scipy matmul. + """ d = x.shape[-1] try: from fast_hadamard_transform import hadamard_transform - return hadamard_transform(x, scale=d ** -0.5) + + return hadamard_transform(x, scale=d**-0.5) except Exception: from scipy.linalg import hadamard as _h - H = torch.from_numpy(_h(d).astype("float32")).to(x.device) * (d ** -0.5) + + H = torch.from_numpy(_h(d).astype("float32")).to(x.device) * (d**-0.5) return (x.float() @ H).to(x.dtype) @@ -69,26 +74,30 @@ class ConvertedRequest: @dataclass class _Reg: - mla_layers: list # [(lid, name, kv_t, gi)] sorted - ki_layers: list # [(lid, name, ki_t, gi)] sorted + mla_layers: list # [(lid, name, kv_t, gi)] sorted + ki_layers: list # [(lid, name, ki_t, gi)] sorted class MlaNsaProfile: - """Config-driven MLA+NSA profile. ``engine_factory(weights, max_seq, - with_mtp, ar_steps) -> adapter`` builds the model-specific engine.""" + """Config-driven MLA+NSA profile. + + ``engine_factory(weights, max_seq, with_mtp, ar_steps) -> adapter`` builds + the model-specific engine. + """ num_ranks = wire.NUM_RANKS - sender_ranks = frozenset({0}) # MLA latent replicated across TP + sender_ranks = frozenset({0}) # MLA latent replicated across TP - def __init__(self, name: str, num_layers: int, layout_version: int, - engine_factory, mla_fp8: bool = True): + def __init__( + self, name: str, num_layers: int, layout_version: int, engine_factory, mla_fp8: bool = True + ): self.name = name self.num_layers = num_layers self._base_version = layout_version self._engine_factory = engine_factory - self.mla_fp8 = mla_fp8 # fp8_ds_mla (True) vs bf16 (False) MLA cache + self.mla_fp8 = mla_fp8 # fp8_ds_mla (True) vs bf16 (False) MLA cache - def configure(self, kv_cache_dtype: str) -> "MlaNsaProfile": + def configure(self, kv_cache_dtype: str) -> MlaNsaProfile: """Select the MLA cache dtype (decode side; prefill auto-detects).""" d = (kv_cache_dtype or "").lower() if d in ("fp8_ds_mla", "fp8", "fp8_e4m3"): @@ -96,8 +105,9 @@ def configure(self, kv_cache_dtype: str) -> "MlaNsaProfile": elif d in ("bf16", "bfloat16", "auto"): self.mla_fp8 = False else: - raise ValueError(f"unknown kv_cache_dtype {kv_cache_dtype!r}; " - f"want fp8_ds_mla or bf16") + raise ValueError( + f"unknown kv_cache_dtype {kv_cache_dtype!r}; " f"want fp8_ds_mla or bf16" + ) return self @property @@ -126,8 +136,9 @@ def _ki_plane(self, max_seq_len: int) -> int: # ── receive side ── def buffer_bytes(self, max_seq_len: int) -> int: - return (self._kv_plane(max_seq_len) + self._pe_plane(max_seq_len) - + self._ki_plane(max_seq_len)) + return ( + self._kv_plane(max_seq_len) + self._pe_plane(max_seq_len) + self._ki_plane(max_seq_len) + ) def hello_layout(self, base_ptr: int, max_seq_len: int) -> dict[str, int]: kv = base_ptr @@ -145,35 +156,53 @@ def convert(self, buffer, base_ptr, max_seq_len, received, num_devices=1): layers = [] for lid in range(self.num_layers): ko = lid * max_seq_len * kv_bpt - kv_raw = buffer[ko:ko + seq * kv_bpt].view(seq, kv_bpt) + kv_raw = buffer[ko : ko + seq * kv_bpt].view(seq, kv_bpt) if self.mla_fp8: - kv = self._dequant_kv(kv_raw, seq) # fp8+scale -> bf16 512 + kv = self._dequant_kv(kv_raw, seq) # fp8+scale -> bf16 512 else: - kv = kv_raw.view(torch.bfloat16).view(seq, KV_LORA_RANK) \ - .contiguous() # already bf16 + kv = ( + kv_raw.view(torch.bfloat16).view(seq, KV_LORA_RANK).contiguous() + ) # already bf16 po = pe_base + lid * max_seq_len * PE_BPT - pe = buffer[po:po + seq * PE_BPT].view(torch.bfloat16) \ - .view(seq, QK_ROPE_HEAD_DIM).contiguous() + pe = ( + buffer[po : po + seq * PE_BPT] + .view(torch.bfloat16) + .view(seq, QK_ROPE_HEAD_DIM) + .contiguous() + ) io = ki_base + lid * _max_pages(max_seq_len) * KI_PAGE_BYTES - ki_raw = buffer[io:io + npages * KI_PAGE_BYTES].view(npages, - KI_PAGE_BYTES) + ki_raw = buffer[io : io + npages * KI_PAGE_BYTES].view(npages, KI_PAGE_BYTES) layers.append((self._dequant_ki(ki_raw, seq), kv, pe)) torch.cuda.synchronize() return ConvertedRequest( - rid=received.rid, seq_len=seq, + rid=received.rid, + seq_len=seq, last_prompt_token=received.last_prompt_token, first_token_id=received.first_token_id, - sampling=received.sampling, layers=layers) + sampling=received.sampling, + layers=layers, + ) @staticmethod def _dequant_kv(kv_raw: torch.Tensor, seq_len: int) -> torch.Tensor: - """kv_merged [seq,528] u8 (512 fp8 + 4 fp32 scale) -> bf16 [seq,512]. - Per-128-block scale: kv[:, b*128:(b+1)*128] *= scale[:, b].""" + """Dequantize kv_merged [seq,528] u8 (512 fp8 + 4 fp32 scale) -> bf16 [seq,512]. + + Per-128-block scale: kv[:, b*128:(b+1)*128] *= scale[:, b]. + """ nblk = KV_LORA_RANK // KV_QUANT_BLOCK - fp8 = kv_raw[:, :KV_FP8_BYTES].reshape(-1).view(torch.float8_e4m3fn) \ + fp8 = ( + kv_raw[:, :KV_FP8_BYTES] + .reshape(-1) + .view(torch.float8_e4m3fn) .reshape(seq_len, KV_LORA_RANK) - scale = kv_raw[:, KV_FP8_BYTES:].reshape(-1).contiguous() \ - .view(torch.float32).reshape(seq_len, nblk) + ) + scale = ( + kv_raw[:, KV_FP8_BYTES:] + .reshape(-1) + .contiguous() + .view(torch.float32) + .reshape(seq_len, nblk) + ) fp32 = fp8.float().view(seq_len, nblk, KV_QUANT_BLOCK) deq = (fp32 * scale.unsqueeze(-1)).view(seq_len, KV_LORA_RANK) return deq.to(torch.bfloat16) @@ -182,19 +211,26 @@ def _dequant_kv(kv_raw: torch.Tensor, seq_len: int) -> torch.Tensor: def _dequant_ki(ki_raw: torch.Tensor, seq_len: int) -> torch.Tensor: npages = ki_raw.shape[0] fp8_bytes = PAGE_SIZE * INDEX_HEAD_DIM - ki_fp8 = ki_raw[:, :fp8_bytes].reshape(-1).view(torch.float8_e4m3fn) \ + ki_fp8 = ( + ki_raw[:, :fp8_bytes] + .reshape(-1) + .view(torch.float8_e4m3fn) .reshape(npages * PAGE_SIZE, INDEX_HEAD_DIM) - scale = ki_raw[:, fp8_bytes:].reshape(-1).contiguous() \ - .view(torch.float32).reshape(npages * PAGE_SIZE, - INDEX_HEAD_DIM // KI_QUANT_BLOCK) + ) + scale = ( + ki_raw[:, fp8_bytes:] + .reshape(-1) + .contiguous() + .view(torch.float32) + .reshape(npages * PAGE_SIZE, INDEX_HEAD_DIM // KI_QUANT_BLOCK) + ) deq = (ki_fp8[:seq_len].float() * scale[:seq_len]).to(torch.bfloat16) return _hadamard(deq) # ── prefill side ── def classify_layers(self, kv_caches: dict, kv_cache_config) -> _Reg: group_of = {} - for gi, g in enumerate(getattr(kv_cache_config, "kv_cache_groups", - []) or []): + for gi, g in enumerate(getattr(kv_cache_config, "kv_cache_groups", []) or []): for ln in getattr(g, "layer_names", []): group_of[ln] = gi @@ -204,8 +240,8 @@ def lid_of(name): return self.num_layers - 1 if name.startswith("mtp.") else base_i mla, ki = [], [] - for name, val in kv_caches.items(): - t = val[0] if isinstance(val, (tuple, list)) else val + for name, cache in kv_caches.items(): + t = cache[0] if isinstance(cache, (tuple, list)) else cache gi = group_of.get(name, -1) if "indexer" in name.lower() or "index_k" in name.lower(): ki.append((lid_of(name), name, t, gi)) @@ -217,7 +253,8 @@ def lid_of(name): raise RuntimeError( f"{self.name} classify: {len(mla)} MLA + {len(ki)} KI layers " f"(expected {self.num_layers} each); check --speculative-config" - f" and the vLLM layer naming") + f" and the vLLM layer naming" + ) # auto-detect MLA cache dtype from the actual cache stride (the prefill # cache is ground truth; the decode side is told via --kv-cache-dtype) t0 = mla[0][2] @@ -229,10 +266,15 @@ def lid_of(name): else: raise RuntimeError( f"{self.name}: unexpected MLA cache stride {bpt} B/token; " - f"expected {MLA_BPT_FP8} (fp8_ds_mla) or {MLA_BPT_BF16} (bf16)") - logger.info("%s registered %d MLA + %d KI layers, MLA cache=%s", - self.name, len(mla), len(ki), - "fp8_ds_mla" if self.mla_fp8 else "bf16") + f"expected {MLA_BPT_FP8} (fp8_ds_mla) or {MLA_BPT_BF16} (bf16)" + ) + logger.info( + "%s registered %d MLA + %d KI layers, MLA cache=%s", + self.name, + len(mla), + len(ki), + "fp8_ds_mla" if self.mla_fp8 else "bf16", + ) return _Reg(mla_layers=mla, ki_layers=ki) def staging_bytes(self, reg, tp_rank, max_seq_len): @@ -248,8 +290,7 @@ def extract(self, reg: _Reg, m, tp_rank, staging, max_seq_len): mla_ids = m.block_ids_per_group[reg.mla_layers[0][3]] bt = torch.tensor(mla_ids, dtype=torch.long) offs = torch.arange(PAGE_SIZE) - slots = (offs.reshape(1, -1) - + bt.reshape(-1, 1) * PAGE_SIZE).flatten()[:seq] + slots = (offs.reshape(1, -1) + bt.reshape(-1, 1) * PAGE_SIZE).flatten()[:seq] ki_ids = m.block_ids_per_group[reg.ki_layers[0][3]] ki_bt = torch.tensor(ki_ids[:npages], dtype=torch.long) @@ -261,20 +302,21 @@ def extract(self, reg: _Reg, m, tp_rank, staging, max_seq_len): # (fp8 656 -> 528+128, bf16 1152 -> 1024+128), no conversion here kv_t = reg.mla_layers[lid][2] raw = kv_t if kv_t.dtype == torch.uint8 else kv_t.view(torch.uint8) - flat = raw.reshape(-1, mla_bpt) # [ntok, mla_bpt] u8 - rows = flat[slots.to(flat.device)] # [seq, mla_bpt] u8 - kv_merged = rows[:, :kv_bpt].contiguous() # kv_c (+scale) - pe = rows[:, kv_bpt:].contiguous() # 64 bf16 (128 B) + flat = raw.reshape(-1, mla_bpt) # [ntok, mla_bpt] u8 + rows = flat[slots.to(flat.device)] # [seq, mla_bpt] u8 + kv_merged = rows[:, :kv_bpt].contiguous() # kv_c (+scale) + pe = rows[:, kv_bpt:].contiguous() # 64 bf16 (128 B) ko = lid * max_seq_len * kv_bpt po = pe_base + lid * max_seq_len * PE_BPT - staging[ko:ko + seq * kv_bpt].copy_(kv_merged.flatten()) - staging[po:po + seq * PE_BPT].copy_(pe.flatten()) + staging[ko : ko + seq * kv_bpt].copy_(kv_merged.flatten()) + staging[po : po + seq * PE_BPT].copy_(pe.flatten()) ki_t = reg.ki_layers[lid][2] ki_pages = ki_t[ki_bt.to(ki_t.device)].reshape(npages, -1) io = ki_base + lid * _max_pages(max_seq_len) * KI_PAGE_BYTES - staging[io:io + npages * KI_PAGE_BYTES].copy_( - ki_pages.contiguous().view(torch.uint8).flatten()) + staging[io : io + npages * KI_PAGE_BYTES].copy_( + ki_pages.contiguous().view(torch.uint8).flatten() + ) torch.cuda.synchronize() return {"seq": seq, "npages": npages, "stage_max": max_seq_len} @@ -286,8 +328,7 @@ def rdma_plan(self, hello, sections, tp_rank, seq_len, base): s_pe = self._kv_plane(stage_max) s_ki = s_pe + self._pe_plane(stage_max) kv_bpt = self._kv_bpt - r_kv, r_pe, r_ki = (int(hello["kv_base"]), int(hello["pe_base"]), - int(hello["ki_base"])) + r_kv, r_pe, r_ki = (int(hello["kv_base"]), int(hello["pe_base"]), int(hello["ki_base"])) for lid in range(self.num_layers): srcs.append(base + lid * stage_max * kv_bpt) dsts.append(r_kv + lid * remote_max * kv_bpt) @@ -301,8 +342,7 @@ def rdma_plan(self, hello, sections, tp_rank, seq_len, base): return srcs, dsts, lens def build_engine(self, model_weights_dir, max_seq_len, with_mtp, ar_steps): - return self._engine_factory(model_weights_dir, max_seq_len, with_mtp, - ar_steps) + return self._engine_factory(model_weights_dir, max_seq_len, with_mtp, ar_steps) class MlaNsaEngineAdapter: @@ -316,12 +356,12 @@ class MlaNsaEngineAdapter: def __init__(self, generator, with_mtp: bool): import torch as _torch + self._torch = _torch self.gen = generator self.with_mtp = with_mtp self.mtp_seq_len = getattr(generator, "mtp_seq_len", 4) - self.max_seq_len = getattr(generator.decode_layer, "max_seq_len", - 200000) + self.max_seq_len = getattr(generator.decode_layer, "max_seq_len", 200000) self.last_stats: dict = {} self.stop_ids = self._resolve_stop_ids(generator) @@ -340,26 +380,25 @@ def inject(self, req) -> None: self._last_prompt_token = req.last_prompt_token self._seq_len = req.seq_len - def decode(self, first_token_id, max_tokens, sampling, on_token=None, - cancel_event=None): + def decode(self, first_token_id, max_tokens, sampling, on_token=None, cancel_event=None): sampling = sampling or {} temp = float(sampling.get("temperature", 1.0)) if temp < 1e-5: - self.gen.update_sampling_params( - temperature=1.0, top_p=1.0, top_k=1, use_topp=False) + self.gen.update_sampling_params(temperature=1.0, top_p=1.0, top_k=1, use_topp=False) else: self.gen.update_sampling_params( - temperature=temp, top_p=float(sampling.get("top_p", 0.95)), - top_k=int(sampling.get("top_k", 256)), use_topp=True) + temperature=temp, + top_p=float(sampling.get("top_p", 0.95)), + top_k=int(sampling.get("top_k", 256)), + use_topp=True, + ) budget = min(int(max_tokens), self.max_seq_len - self._seq_len - 1) if budget <= 0: self.last_stats = {"finish_reason": "length"} return [int(first_token_id)] if self.with_mtp: - return self._decode_mtp(first_token_id, budget, on_token, - cancel_event) - return self._decode_standard(first_token_id, budget, on_token, - cancel_event) + return self._decode_mtp(first_token_id, budget, on_token, cancel_event) + return self._decode_standard(first_token_id, budget, on_token, cancel_event) def _decode_mtp(self, first_token_id, budget, on_token, cancel_event): dl = self.gen.decode_layer @@ -373,16 +412,14 @@ def _decode_mtp(self, first_token_id, budget, on_token, cancel_event): self.last_stats = {"finish_reason": "stop"} return [] dl.set_prefill_valid_tokens(0) - draft = torch.full((1, T), int(self._last_prompt_token), - dtype=torch.int32, device="cuda:0") + draft = torch.full((1, T), int(self._last_prompt_token), dtype=torch.int32, device="cuda:0") accepted, finish, fwd, finished = [], "length", 0, False while not finished and len(tokens) < budget: if cancel_event is not None and cancel_event.is_set(): finish = "cancelled" break if fwd == 1: - draft = torch.full((1, T), int(first_token_id), - dtype=torch.int32, device="cuda:0") + draft = torch.full((1, T), int(first_token_id), dtype=torch.int32, device="cuda:0") elif fwd > 1: draft = dl.get_next_draft_tokens(0).reshape(1, T) dl.forward(draft) @@ -425,8 +462,7 @@ def _decode_standard(self, first_token_id, budget, on_token, cancel_event): self.last_stats = {"finish_reason": "stop"} return [] finish = "length" - cur = torch.tensor(int(first_token_id), dtype=torch.long, - device="cuda:0") + cur = torch.tensor(int(first_token_id), dtype=torch.long, device="cuda:0") while len(tokens) < budget: if cancel_event is not None and cancel_event.is_set(): finish = "cancelled" diff --git a/tilert/pd_vllm/receive_server.py b/tilert/pd_vllm/receive_server.py index fb6f36d..3ad06f2 100644 --- a/tilert/pd_vllm/receive_server.py +++ b/tilert/pd_vllm/receive_server.py @@ -1,5 +1,6 @@ """Decode-side receive server (W4): Mooncake buffer + TCP control plane.""" +import contextlib import logging import queue import socket @@ -44,8 +45,12 @@ def __init__( self.request_timeout = request_timeout total = profile.buffer_bytes(max_seq_len) - logger.info("allocating receive buffer: %.2f GB on %s (profile=%s)", - total / 1024**3, device, profile.name) + logger.info( + "allocating receive buffer: %.2f GB on %s (profile=%s)", + total / 1024**3, + device, + profile.name, + ) self.buffer = torch.zeros(total, dtype=torch.uint8, device=device) self.base_ptr = self.buffer.data_ptr() self._hello_layout = profile.hello_layout(self.base_ptr, max_seq_len) @@ -60,8 +65,9 @@ def __init__( self._transport.init(hostname) self._transport.register(self.base_ptr, total, dev_id) self._transport_meta = self._transport.local_meta() - logger.info("transport=%s ready, buffer registered (%.2f GB)", - self._transport.name, total / 1024**3) + logger.info( + "transport=%s ready, buffer registered (%.2f GB)", self._transport.name, total / 1024**3 + ) self._lock = threading.Lock() self._current: ReceivedRequest | None = None @@ -71,15 +77,14 @@ def __init__( # (e.g. an IPv6-only decode node reached over fe80::.../bond0) self._srv = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) self._srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: + with contextlib.suppress(OSError): self._srv.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except OSError: - pass self._srv.bind(("::", ctrl_port)) self._srv.listen(32) self._stop = threading.Event() - self._thread = threading.Thread(target=self._accept_loop, - name="pd-recv-accept", daemon=True) + self._thread = threading.Thread( + target=self._accept_loop, name="pd-recv-accept", daemon=True + ) self._thread.start() logger.info("control plane listening on :%d", ctrl_port) @@ -92,10 +97,8 @@ def release(self) -> None: def close(self) -> None: self._stop.set() - try: + with contextlib.suppress(OSError): self._srv.close() - except OSError: - pass # ── accept / per-connection handling ───────────────────────────────── @@ -105,8 +108,7 @@ def _accept_loop(self) -> None: conn, addr = self._srv.accept() except OSError: break - t = threading.Thread(target=self._handle, args=(conn, addr), - daemon=True) + t = threading.Thread(target=self._handle, args=(conn, addr), daemon=True) t.start() def _handle(self, conn: socket.socket, addr) -> None: @@ -116,12 +118,18 @@ def _handle(self, conn: socket.socket, addr) -> None: # `busy` in hello is advisory (a same-rid rank must still proceed); # the authoritative accept/reject happens once the rid is known. with self._lock: - advisory_busy = (self._current is not None - and self._current.t_complete == 0.0) - wire.send_msg(conn, wire.hello_msg( - self._transport.name, self._transport_meta, self.max_seq_len, - self.profile.layout_version, self._hello_layout, - busy=advisory_busy)) + advisory_busy = self._current is not None and self._current.t_complete == 0.0 + wire.send_msg( + conn, + wire.hello_msg( + self._transport.name, + self._transport_meta, + self.max_seq_len, + self.profile.layout_version, + self._hello_layout, + busy=advisory_busy, + ), + ) req = wire.recv_msg(conn) rid, rank = req["rid"], int(req["rank"]) @@ -132,8 +140,11 @@ def _handle(self, conn: socket.socket, addr) -> None: with self._lock: cur = self._current if cur is None or cur.rid != rid: - if cur is not None and cur.t_complete == 0.0 and \ - time.time() - cur.t_first_conn < self.request_timeout: + if ( + cur is not None + and cur.t_complete == 0.0 + and time.time() - cur.t_first_conn < self.request_timeout + ): # busy with a different in-flight rid wire.send_msg(conn, {"error": "busy", "busy_rid": cur.rid}) logger.warning("rejecting %s (busy with %s)", rid, cur.rid) @@ -158,14 +169,21 @@ def _handle(self, conn: socket.socket, addr) -> None: if cur is None or cur.rid != rid: return cur.done_ranks.add(rank) - logger.info("request %s: rank %d done (%d/%d)", - rid, rank, len(cur.done_ranks), - len(self.profile.sender_ranks)) + logger.info( + "request %s: rank %d done (%d/%d)", + rid, + rank, + len(cur.done_ranks), + len(self.profile.sender_ranks), + ) if cur.done_ranks >= set(self.profile.sender_ranks): cur.t_complete = time.time() self.completed.put(cur) - logger.info("request %s: all ranks done in %.1f ms", - rid, 1000 * (cur.t_complete - cur.t_first_conn)) + logger.info( + "request %s: all ranks done in %.1f ms", + rid, + 1000 * (cur.t_complete - cur.t_first_conn), + ) except Exception: logger.exception("connection from %s failed", addr) finally: diff --git a/tilert/pd_vllm/transport.py b/tilert/pd_vllm/transport.py index 0f99dee..7d5e779 100644 --- a/tilert/pd_vllm/transport.py +++ b/tilert/pd_vllm/transport.py @@ -11,7 +11,7 @@ class Transport: def init(self, host: str) -> None: ... def register(self, ptr: int, nbytes: int, dev_id: int) -> None: ... - def local_meta(self) -> dict: ... + def local_meta(self) -> dict: ... # type: ignore[empty-body] def write(self, remote_meta: dict, srcs, dsts, lens) -> None: ... @@ -22,6 +22,7 @@ class MooncakeTransport(Transport): def init(self, host: str) -> None: from mooncake.engine import TransferEngine + self.engine = TransferEngine() ret = self.engine.initialize(host, "P2PHANDSHAKE", "rdma", "") if ret != 0: @@ -37,26 +38,27 @@ def local_meta(self) -> dict: return {"session_id": self.session_id} def write(self, remote_meta: dict, srcs, dsts, lens) -> None: - ret = self.engine.batch_transfer_sync_write( - remote_meta["session_id"], srcs, dsts, lens) + ret = self.engine.batch_transfer_sync_write(remote_meta["session_id"], srcs, dsts, lens) if ret != 0: raise RuntimeError(f"mooncake write failed: {ret}") class NixlTransport(Transport): - """NIXL agent over the UCX backend (GPUDirect RDMA). Registers VRAM - regions with 4-tuple descriptors, exchanges agent metadata via the hello, - and issues WRITE transfers built from (src,dst,len) triples.""" + """NIXL agent over the UCX backend (GPUDirect RDMA). + + Registers VRAM regions with 4-tuple descriptors, exchanges agent metadata + via the hello, and issues WRITE transfers built from (src,dst,len) triples. + """ name = "nixl" _MAX_POLL = 2_000_000 def init(self, host: str) -> None: from nixl._api import nixl_agent, nixl_agent_config + # agent name must be globally unique across the two peers - self._agent = nixl_agent(f"{host}:{os.getpid()}", - nixl_agent_config(backends=["UCX"])) - self._remotes: dict[bytes, str] = {} # remote meta -> remote name + self._agent = nixl_agent(f"{host}:{os.getpid()}", nixl_agent_config(backends=["UCX"])) + self._remotes: dict[bytes, str] = {} # remote meta -> remote name self._dev = 0 def register(self, ptr: int, nbytes: int, dev_id: int) -> None: @@ -65,8 +67,7 @@ def register(self, ptr: int, nbytes: int, dev_id: int) -> None: def local_meta(self) -> dict: return { - "nixl_meta": base64.b64encode( - self._agent.get_agent_metadata()).decode(), + "nixl_meta": base64.b64encode(self._agent.get_agent_metadata()).decode(), "nixl_dev": self._dev, } @@ -78,9 +79,11 @@ def write(self, remote_meta: dict, srcs, dsts, lens) -> None: self._remotes[meta_b] = rname rdev = int(remote_meta.get("nixl_dev", 0)) ld = self._agent.get_xfer_descs( - [(int(s), int(n), self._dev) for s, n in zip(srcs, lens)], "VRAM") + [(int(s), int(n), self._dev) for s, n in zip(srcs, lens)], "VRAM" + ) rd = self._agent.get_xfer_descs( - [(int(d), int(n), rdev) for d, n in zip(dsts, lens)], "VRAM") + [(int(d), int(n), rdev) for d, n in zip(dsts, lens)], "VRAM" + ) h = self._agent.initialize_xfer("WRITE", ld, rd, rname) try: st = self._agent.transfer(h) @@ -102,6 +105,5 @@ def write(self, remote_meta: dict, srcs, dsts, lens) -> None: def make_transport(name: str | None) -> Transport: key = (name or "mooncake").lower() if key not in _BACKENDS: - raise ValueError(f"unknown transport {name!r}; " - f"choices: {sorted(_BACKENDS)}") + raise ValueError(f"unknown transport {name!r}; " f"choices: {sorted(_BACKENDS)}") return _BACKENDS[key]() diff --git a/tilert/pd_vllm/wire.py b/tilert/pd_vllm/wire.py index c3e5a76..284850c 100644 --- a/tilert/pd_vllm/wire.py +++ b/tilert/pd_vllm/wire.py @@ -24,16 +24,18 @@ def local_ip(probe_addr: str | None = None) -> str: def derive_rid(request_id: str) -> str: - """vLLM request/response id -> client-visible rid. Shared by the prefill - connector (internal id) and the router (response id) so both agree.""" + """Map a vLLM request/response id to the client-visible rid. + + Shared by the prefill connector (internal id) and the router (response id) + so both agree. + """ rid = request_id for prefix in ("chatcmpl-", "cmpl-"): if rid.startswith(prefix): - rid = rid[len(prefix):] + rid = rid[len(prefix) :] break parts = rid.rsplit("-", 1) - if len(parts) == 2 and len(parts[1]) <= 8 and \ - all(c in "0123456789abcdef" for c in parts[1]): + if len(parts) == 2 and len(parts[1]) <= 8 and all(c in "0123456789abcdef" for c in parts[1]): rid = parts[0] parts = rid.rsplit("-", 1) if len(parts) == 2 and parts[1].isdigit() and len(parts[1]) <= 3: @@ -64,12 +66,21 @@ def _recv_exact(sock: socket.socket, n: int) -> bytes: return buf -def hello_msg(transport: str, transport_meta: dict, max_seq_len: int, - layout_version: int, layout: dict, busy: bool) -> dict: - """Common hello envelope. ``transport`` names the RDMA backend and - ``transport_meta`` carries its connection info (mooncake: session_id; - nixl: nixl_meta/nixl_dev). ``layout`` carries profile-specific region base - addresses (e.g. gdn_base / gqa_k_base / kv_base).""" +def hello_msg( + transport: str, + transport_meta: dict, + max_seq_len: int, + layout_version: int, + layout: dict, + busy: bool, +) -> dict: + """Build the common hello envelope. + + ``transport`` names the RDMA backend and ``transport_meta`` carries its + connection info (mooncake: session_id; nixl: nixl_meta/nixl_dev). + ``layout`` carries profile-specific region base addresses (e.g. gdn_base / + gqa_k_base / kv_base). + """ return { "magic": MAGIC, "layout_version": layout_version, From 0313a16a76c918292e78994e9d618715f3a43bc1 Mon Sep 17 00:00:00 2001 From: Lingxiao Ma Date: Tue, 14 Jul 2026 13:48:29 +0000 Subject: [PATCH 10/10] bug fix --- tilert/models/glm_5/generator.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tilert/models/glm_5/generator.py b/tilert/models/glm_5/generator.py index 25ee616..18c422a 100644 --- a/tilert/models/glm_5/generator.py +++ b/tilert/models/glm_5/generator.py @@ -471,9 +471,11 @@ def inject_cache( kv_src = kv[:cache_len].to(f"cuda:{device_id}") pe_src = pe[:cache_len].to(f"cuda:{device_id}") - caches[base_idx + 0][0, start_pos:end_pos, :].copy_(ki_src) - caches[base_idx + 1][0, start_pos:end_pos, :].copy_(kv_src) - caches[base_idx + 2][0, start_pos:end_pos, :].copy_(pe_src) + for _off, _src in ((0, ki_src), (1, kv_src), (2, pe_src)): + _dst = caches[base_idx + _off] + if _dst.size(1) < end_pos: + continue + _dst[0, start_pos:end_pos, :].copy_(_src) torch.cuda.synchronize(device_id)