diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index c931ead34d..3e49e0f66a 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-14, macos-15] + os: [ubuntu-22.04, windows-2022, macos-15] steps: - uses: actions/checkout@v6 @@ -181,7 +181,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 892787ee1f..186e0913bf 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-14, macos-15] + os: [macos-15] steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 8908ccdd6f..b29faafc39 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml index 416cd0cddf..6a25eb7065 100644 --- a/.github/workflows/test-pypi.yaml +++ b/.github/workflows/test-pypi.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: build-linux: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 82f7a8a7de..5579138b89 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,7 +25,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.9" + python-version: "3.10" - name: Install huggingface-hub run: pip install huggingface-hub - name: Restore model cache @@ -44,7 +44,7 @@ jobs: build-linux: needs: download-model - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b337ad923..c582001020 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): remove unsupported macOS 14 release runners by @abetlen in #2416 + +## [0.3.37] + +- feat: update llama.cpp to ggml-org/llama.cpp@a956a852c +- fix(ci): use supported Python versions on Ubuntu by @abetlen in #2407 + +## [0.3.36] + +- feat: update llama.cpp to ggml-org/llama.cpp@0c1e57098 + +## [0.3.35] + +- feat: update llama.cpp to ggml-org/llama.cpp@4df29be4f +- fix(example): retain recurrent state for server MTP rollback +- feat: update llama.cpp to ggml-org/llama.cpp@adb55e514 +- fix(server): show falsey defaults in CLI help by @cupkk in #2355 +- fix: release multimodal contexts when closing Llama by @Anai-Guo in #2343 + +## [0.3.34] + +- feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 + +## [0.3.33] + +- feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 + +## [0.3.32] + +- feat(example): support chained NextN heads for server MTP drafting +- feat: update llama.cpp to ggml-org/llama.cpp@b3fed31b9 +- fix: preserve recurrent/hybrid model state when the full prompt is already cached by @allthatido and @abetlen in #2306 + +## [0.3.31] + +- feat: update llama.cpp to ggml-org/llama.cpp@f449e0553 + ## [0.3.30] - feat: update llama.cpp to ggml-org/llama.cpp@e3a74b299 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5feaaca5b9..89c12958c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,25 +187,10 @@ if (LLAMA_BUILD) add_compile_definitions(GGML_USE_METAL) endif() - # Upstream mtmd expects LLAMA_INSTALL_VERSION to be set by llama.cpp's - # top-level CMakeLists.txt. When we include tools/mtmd directly from the - # Python package build, that directory scope is skipped. - if (NOT DEFINED LLAMA_INSTALL_VERSION OR "${LLAMA_INSTALL_VERSION}" STREQUAL "") - set(LLAMA_INSTALL_VERSION 0.0.0) - find_package(Git QUIET) - if (Git_FOUND) - execute_process( - COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp - OUTPUT_VARIABLE LLAMA_MTMD_BUILD_NUMBER - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE LLAMA_MTMD_BUILD_NUMBER_RESULT - ) - if (LLAMA_MTMD_BUILD_NUMBER_RESULT EQUAL 0) - set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_MTMD_BUILD_NUMBER}) - endif() - endif() - endif() + # tools/mtmd is added outside llama.cpp's directory scope, so inherit + # its library version variables explicitly. + get_directory_property(LLAMA_VERSION_BASE DIRECTORY vendor/llama.cpp DEFINITION LLAMA_VERSION_BASE) + get_directory_property(LLAMA_VERSION_MAJOR DIRECTORY vendor/llama.cpp DEFINITION LLAMA_VERSION_MAJOR) # Building llava add_subdirectory(vendor/llama.cpp/tools/mtmd) diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 20f7a158ac..43b0b31bb8 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,8 +76,14 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap + if self.params.use_mmap and self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif self.params.use_mmap: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE self.model = llama_cpp.llama_model_load_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/examples/server/server.py b/examples/server/server.py index 16f8c9f7e5..70a4a6e04d 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -1288,7 +1288,14 @@ def __init__( raise RuntimeError("failed to create MTP draft context") ctx_other = llama_cpp_ext.llama_get_ctx_other(self.ctx) self.is_mem_shared = bool(ctx_other and ctx_other == self.target_ctx) - self.sampled_batch_draft = not self.is_mem_shared + self.n_mtp_layers = max( + 1, + int(llama_cpp.llama_model_n_layer_nextn(self.model)), + ) + self.chain_heads = self.n_mtp_layers > 1 and not self.is_mem_shared + if self.chain_heads: + self.num_pred_tokens = min(self.num_pred_tokens, self.n_mtp_layers) + self.sampled_batch_draft = not self.is_mem_shared and not self.chain_heads self.n_batch = int(llama_cpp.llama_n_batch(self.ctx)) mem = llama_cpp.llama_get_memory(self.ctx) if mem is None: @@ -1451,6 +1458,17 @@ def _try_decode_batch(self) -> bool: return False return True + def _set_nextn_layer_offset(self, offset: int) -> None: + if self.chain_heads: + llama_cpp_ext.llama_set_nextn_layer_offset(self.ctx, offset) + + def _try_decode_batch_for_mtp_head(self, head: int) -> bool: + self._set_nextn_layer_offset(head) + try: + return self._try_decode_batch() + finally: + self._set_nextn_layer_offset(0) + def _decode_batch(self) -> None: n_tokens = int(self.batch.n_tokens) if n_tokens <= 0: @@ -1464,6 +1482,22 @@ def _decode_batch(self) -> None: self.decode_failures_total += 1 raise RuntimeError(f"MTP draft decode failed with code {result}") + def _decode_batch_for_mtp_heads( + self, + start_pos_by_seq: Dict[int, int], + ) -> None: + if not self.chain_heads: + self._decode_batch() + return + try: + for head in range(self.n_mtp_layers): + for seq_id, start_pos in start_pos_by_seq.items(): + llama_cpp.llama_memory_seq_rm(self.mem, seq_id, start_pos, -1) + self._set_nextn_layer_offset(head) + self._decode_batch() + finally: + self._set_nextn_layer_offset(0) + def metric_definitions( self, ) -> List[Tuple[str, str, str, Union[int, float]]]: @@ -1577,7 +1611,8 @@ def _process_rows( target_rows_by_seq: Dict[int, List[int]], aligned_by_seq: Dict[int, bool], ) -> None: - added_pos_by_seq: Dict[int, int] = {} + added_start_pos_by_seq: Dict[int, int] = {} + added_end_pos_by_seq: Dict[int, int] = {} self._clear_batch() for index in range(start, end): if int(batch.n_seq_id[index]) != 1: @@ -1609,15 +1644,85 @@ def _process_rows( self._set_batch_embedding_row(slot, self.pending_h[seq_id]) else: self._set_batch_embedding_row(slot, h_tgt_rows[previous_row]) - added_pos_by_seq[seq_id] = pos + added_start_pos_by_seq.setdefault(seq_id, pos) + added_end_pos_by_seq[seq_id] = pos previous_row_by_seq[seq_id] = index target_rows_by_seq.setdefault(seq_id, []).append(index) if int(self.batch.n_tokens) > 0: - self._decode_batch() - for seq_id, pos in added_pos_by_seq.items(): + self._decode_batch_for_mtp_heads(added_start_pos_by_seq) + for seq_id, pos in added_end_pos_by_seq.items(): self.context_pos[seq_id] = max(self.context_pos[seq_id], pos + 1) + def _draft_chain_heads( + self, + *, + seq_id: int, + first_pos: int, + token: int, + n_predict: int, + ) -> np.ndarray: + if self.context_pos[seq_id] > first_pos: + self.truncate(seq_id, first_pos) + if self.context_pos[seq_id] < first_pos: + self.ready[seq_id] = False + return np.array([], dtype=np.intc) + + drafted: List[int] = [] + chain_tokens = [token] + chain_embeddings = [self.pending_h[seq_id].copy()] + self._reset_sampler(seq_id) + + try: + for head in range(min(n_predict, self.n_mtp_layers)): + llama_cpp.llama_memory_seq_rm(self.mem, seq_id, first_pos, -1) + self._clear_batch() + for offset, (chain_token, embedding) in enumerate( + zip(chain_tokens, chain_embeddings) + ): + slot = int(self.batch.n_tokens) + self._add_batch_token( + token=chain_token, + pos=first_pos + offset, + seq_id=seq_id, + logits=offset == len(chain_tokens) - 1, + ) + self._set_batch_embedding_row(slot, embedding) + + output_index = int(self.batch.n_tokens) - 1 + if not self._try_decode_batch_for_mtp_head(head): + break + self.context_pos[seq_id] = max( + self.context_pos[seq_id], + first_pos + len(chain_tokens), + ) + sampled_token = self._sample_token(output_index, seq_id=seq_id) + if sampled_token is None: + break + drafted.append(sampled_token) + if len(drafted) >= n_predict: + break + h_row = llama_cpp_ext.llama_get_embeddings_nextn_ith( + self.ctx, + output_index, + ) + if not h_row: + break + chain_tokens.append(sampled_token) + chain_embeddings.append( + np.ctypeslib.as_array( + h_row, + shape=(self.n_embd,), + ).copy() + ) + finally: + self._set_nextn_layer_offset(0) + self.truncate(seq_id, first_pos) + + if not drafted: + return np.array([], dtype=np.intc) + return np.asarray(drafted, dtype=np.intc) + def draft( self, input_ids: np.ndarray, @@ -1649,6 +1754,13 @@ def draft( token = int(input_ids[-1]) drafted: List[int] = [] + if self.chain_heads: + return self._draft_chain_heads( + seq_id=seq_id, + first_pos=first_pos, + token=token, + n_predict=n_predict, + ) if not self.is_mem_shared and self.context_pos[seq_id] > first_pos: self.truncate(seq_id, first_pos) if not self.is_mem_shared and self.context_pos[seq_id] < first_pos: @@ -1709,6 +1821,15 @@ def draft_many( /, ) -> List[np.ndarray]: results = [np.array([], dtype=np.intc) for _ in requests] + if self.chain_heads: + for result_index, (input_ids, seq_id, max_tokens) in enumerate(requests): + results[result_index] = self.draft( + input_ids, + seq_id=seq_id, + max_tokens=max_tokens, + ) + return results + active: List["MTPDraftProvider.DraftManyState"] = [] for result_index, (input_ids, seq_id, max_tokens) in enumerate(requests): if ( @@ -10178,6 +10299,7 @@ def __init__( llama_cpp.llama_sampler_chain_add( self._sampler, llama_cpp.llama_sampler_init_penalties( + n_vocab, 64, 1.0, frequency_penalty, @@ -10626,6 +10748,7 @@ def _create_loaded_media( buffer, len(media_bytes), False, + mtmd_cpp.mtmd_helper_init_opt_default(), ) bitmap = wrapper.bitmap if bitmap is None: @@ -10909,7 +11032,9 @@ def _build_prompt_plan_locked( "multiple videos require MTMD to report frame counts" ) input_text = mtmd_cpp.mtmd_input_text() - input_text.text = prompt.encode("utf-8") + input_text_bytes = prompt.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = False input_text.parse_special = True chunks = mtmd_cpp.mtmd_input_chunks_init() @@ -11189,6 +11314,7 @@ def __init__( vocab_only=vocab_only, use_mmap=use_mmap, use_mlock=use_mlock, + load_mtp=draft_model == "draft-mtp", kv_overrides=kv_overrides, ) ) @@ -11237,6 +11363,11 @@ def __init__( "speculative decoding is only supported for attention models" ) n_ctx_train = int(llama_cpp.llama_model_n_ctx_train(llama_model)) + target_n_rs_seq = ( + max(1, draft_model_num_pred_tokens) + if normalized_draft_model == "draft-mtp" + else None + ) context_params = self.build_context_params( n_ctx=n_ctx if n_ctx is not None else n_ctx_train, @@ -11264,7 +11395,7 @@ def __init__( type_k=type_k, type_v=type_v, kv_unified=kv_unified, - n_rs_seq=None, + n_rs_seq=target_n_rs_seq, ctx_type=None, ) ctx = llama_cpp.llama_init_from_model(llama_model, context_params) @@ -11294,6 +11425,15 @@ def __init__( "MTP requires runtime n_batch to fit the pending token plus draft tokens " f"(required {required_mtp_batch}, got {self.n_batch})" ) + if ( + target_n_rs_seq is not None + and self.exact_checkpoints_only + and self.n_rs_seq < target_n_rs_seq + ): + raise RuntimeError( + "MTP requires retained recurrent-state slots for rollback " + f"(required {target_n_rs_seq}, got {self.n_rs_seq})" + ) self.n_ctx_train = n_ctx_train self.n_vocab = int(llama_cpp.llama_vocab_n_tokens(self.vocab)) self.n_embd = int(llama_cpp.llama_model_n_embd(self.llama_model)) @@ -11467,6 +11607,7 @@ def build_model_params( vocab_only: Optional[bool], use_mmap: Optional[bool], use_mlock: Optional[bool], + load_mtp: bool, kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]], ) -> Tuple[Any, Optional[Any], Optional[Any]]: model_params = llama_cpp.llama_model_default_params() @@ -11488,10 +11629,17 @@ def build_model_params( model_params.tensor_split = tensor_split_ref if vocab_only is not None: model_params.vocab_only = vocab_only - if use_mmap is not None: - model_params.use_mmap = use_mmap - if use_mlock is not None: - model_params.use_mlock = use_mlock + model_params.load_mtp = load_mtp + if use_mlock and use_mmap is not False: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif use_mlock: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap is not None: + model_params.load_mode = ( + llama_cpp.LLAMA_LOAD_MODE_MMAP + if use_mmap + else llama_cpp.LLAMA_LOAD_MODE_NONE + ) kv_overrides_ref = None if kv_overrides is not None: diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b72459f653..7fd2b4c492 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.30" +__version__ = "0.3.37" diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b0fe94d01f..b45d34b2c4 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -276,6 +276,8 @@ def free_ctx(): self.ctx = None self._exit_stack.callback(free_ctx) + # The native context must be freed before its model. + self.model._exit_stack.callback(self.close) def close(self): self._exit_stack.close() @@ -784,12 +786,14 @@ def add_grammar_lazy_patterns( def add_penalties( self, + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, ): sampler = llama_cpp.llama_sampler_init_penalties( + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, @@ -800,7 +804,6 @@ def add_penalties( def add_dry( self, model: LlamaModel, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -814,7 +817,6 @@ def add_dry( sampler = llama_cpp.llama_sampler_init_dry( model.vocab, - n_ctx_train, dry_multiplier, dry_base, dry_allowed_length, diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4a09b55ee5..14e2f8500f 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -244,8 +244,15 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap if lora_path is None else False - self.model_params.use_mlock = use_mlock + use_mmap = use_mmap and lora_path is None + if use_mmap and use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE # kv_overrides is the original python dict self.kv_overrides = kv_overrides @@ -471,6 +478,8 @@ def free_lora_adapter(): self._candidates = internals.LlamaTokenDataArray(n_vocab=self._n_vocab) self.n_tokens = 0 + # Restored or truncated state must decode before sampling. + self._requires_eval = True self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc) self.scores: npt.NDArray[np.single] = np.ndarray( (n_ctx if logits_all == True else n_batch, self._n_vocab), dtype=np.single @@ -647,6 +656,7 @@ def set_seed(self, seed: int): def reset(self): """Reset the model state.""" self.n_tokens = 0 + self._requires_eval = True if self._is_recurrent or self._is_hybrid: mem = llama_cpp.llama_get_memory(self._ctx.ctx) @@ -689,6 +699,7 @@ def eval(self, tokens: Sequence[int]): pass # Update n_tokens self.n_tokens += n_tokens + self._requires_eval = False def _init_sampler( self, @@ -733,7 +744,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_custom(apply_func) sampler.add_penalties( - # n_vocab=self._n_vocab, + n_vocab=self._n_vocab, # special_eos_id=self._token_eos, # linefeed_id=self._token_nl, penalty_last_n=self.last_n_tokens_size, @@ -900,41 +911,53 @@ def generate( grammar=grammar, ) + tokens = list(tokens) + # Check for kv cache prefix match if reset and self.n_tokens > 0: longest_prefix = 0 - for a, b in zip(self._input_ids, tokens[:-1]): + for a, b in zip(self._input_ids, tokens): if a == b: longest_prefix += 1 else: break - # Recurrent and hybrid models cannot rewind state; reset if needed - if ( - self._is_recurrent or self._is_hybrid - ) and longest_prefix < self.n_tokens: - longest_prefix = 0 - reset = True + prompt_consumed = longest_prefix == len(tokens) + exact_prompt_cached = self.n_tokens == len(tokens) and prompt_consumed + + # Exact cache hits can sample immediately only when the current + # logits were produced by a live decode, not restored state. + if exact_prompt_cached and not self._requires_eval: + reset = False + tokens = [] + reuse_prefix = 0 if self.verbose: print( - "Llama.generate: recurrent/hybrid model requires full state reset", + "Llama.generate: full prompt already cached, skipping reset", file=sys.stderr, ) - - if longest_prefix > 0: - if self._ctx.kv_cache_seq_rm(-1, longest_prefix, -1): + else: + # If there is no suffix to decode, replay one token to refresh + # logits after truncating to a valid prefix. + reuse_prefix = longest_prefix - 1 if prompt_consumed else longest_prefix + + # Prefix hits can reuse memory because the suffix decode refreshes + # logits before sampling. + if reuse_prefix > 0: + if self._ctx.kv_cache_seq_rm(-1, reuse_prefix, -1): reset = False - tokens = tokens[longest_prefix:] - self.n_tokens = longest_prefix + tokens = tokens[reuse_prefix:] + self.n_tokens = reuse_prefix + self._requires_eval = True if self.verbose: print( - f"Llama.generate: {longest_prefix} prefix-match hit, " + f"Llama.generate: {reuse_prefix} prefix-match hit, " f"remaining {len(tokens)} prompt tokens to eval", file=sys.stderr, ) elif self.verbose: print( - f"Llama.generate: {longest_prefix} prefix-match found " + f"Llama.generate: {reuse_prefix} prefix-match found " f"but partial kv removal not supported, re-evaluating full prompt", file=sys.stderr, ) @@ -948,7 +971,6 @@ def generate( # grammar.reset() sample_idx = self.n_tokens + len(tokens) - 1 - tokens = list(tokens) # Eval and sample while True: @@ -988,6 +1010,7 @@ def generate( if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]: self.n_tokens = sample_idx self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1) + self._requires_eval = True break if self.draft_model is not None: @@ -2126,8 +2149,16 @@ def __getstate__(self): main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_mlock=self.model_params.use_mlock, + use_mmap=self.model_params.load_mode + in ( + llama_cpp.LLAMA_LOAD_MODE_MMAP, + llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK, + ), + use_mlock=self.model_params.load_mode + in ( + llama_cpp.LLAMA_LOAD_MODE_MLOCK, + llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK, + ), kv_overrides=self.kv_overrides, # Context Params seed=self._seed, @@ -2217,6 +2248,7 @@ def load_state(self, state: LlamaState) -> None: rest[rest > 0] = 0.0 self.input_ids = state.input_ids.copy() self.n_tokens = state.n_tokens + self._requires_eval = True self._seed = state.seed state_size = state.llama_state_size LLamaStateArrayType = ctypes.c_uint8 * state_size diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0034bdae98..62e2364157 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -9,7 +9,6 @@ import string from datetime import datetime -from contextlib import ExitStack from typing import ( Any, Dict, @@ -2779,7 +2778,6 @@ def __init__(self, clip_model_path: str, verbose: bool = True): self.clip_model_path = clip_model_path self.verbose = verbose self._mtmd_cpp = mtmd_cpp - self._exit_stack = ExitStack() self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None if not os.path.exists(clip_model_path): @@ -2825,7 +2823,7 @@ def mtmd_free(): self._mtmd_cpp.mtmd_free(self.mtmd_ctx) self.mtmd_ctx = None - self._exit_stack.callback(mtmd_free) + llama_model._stack.callback(mtmd_free) def load_image(self, image_url: str) -> bytes: return self._load_image(image_url) @@ -2936,7 +2934,9 @@ def __call__( # Create input text structure input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True @@ -3276,7 +3276,6 @@ def __init__( self.verbose = verbose self.use_gpu = use_gpu self._mtmd_cpp = mtmd_cpp - self._exit_stack = ExitStack() self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None if not os.path.exists(clip_model_path): @@ -3319,7 +3318,7 @@ def mtmd_free(): self._mtmd_cpp.mtmd_free(self.mtmd_ctx) self.mtmd_ctx = None - self._exit_stack.callback(mtmd_free) + llama_model._stack.callback(mtmd_free) def load_image(self, image_url: str) -> bytes: return self._load_image(image_url) @@ -3485,7 +3484,9 @@ def raise_exception(message: str): bitmap_cleanup.append(bitmap) input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 21f85c81c3..0073ffafd0 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -4,6 +4,7 @@ import ctypes import pathlib import warnings +import re from typing import ( Callable, @@ -89,7 +90,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # GGML_TYPE_MXFP4 = 39, # GGML_TYPE_NVFP4 = 40, # GGML_TYPE_Q1_0 = 41, -# GGML_TYPE_COUNT = 42, +# GGML_TYPE_Q2_0 = 42, +# GGML_TYPE_COUNT = 43, # }; GGML_TYPE_F32 = 0 GGML_TYPE_F16 = 1 @@ -122,7 +124,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 GGML_TYPE_Q1_0 = 41 -GGML_TYPE_COUNT = 42 +GGML_TYPE_Q2_0 = 42 +GGML_TYPE_COUNT = 43 # from ggml-backend.h # typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); @@ -160,13 +163,13 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 9 -LLAMA_SESSION_VERSION = 9 +# define LLAMA_SESSION_VERSION 10 +LLAMA_SESSION_VERSION = 10 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ -# define LLAMA_STATE_SEQ_VERSION 2 -LLAMA_STATE_SEQ_VERSION = 2 +# define LLAMA_STATE_SEQ_VERSION 3 +LLAMA_STATE_SEQ_VERSION = 3 # struct llama_vocab; llama_vocab_p = NewType("llama_vocab_p", int) @@ -211,6 +214,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram # LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization # LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming +# LLAMA_VOCAB_TYPE_TEST = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex # }; LLAMA_VOCAB_TYPE_NONE = 0 """For models without vocab""" @@ -226,6 +230,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: """RWKV tokenizer based on greedy tokenization""" LLAMA_VOCAB_TYPE_PLAMO2 = 6 """PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming""" +LLAMA_VOCAB_TYPE_TEST = 7 +"""Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex""" # NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp) @@ -411,6 +417,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -452,6 +459,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_MOSTLY_Q1_0 = 40 +LLAMA_FTYPE_MOSTLY_Q2_0 = 41 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { @@ -516,6 +524,32 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +LLAMA_LOAD_MODE_AUTO = -1 +LLAMA_LOAD_MODE_NONE = 0 +LLAMA_LOAD_MODE_MMAP = 1 +LLAMA_LOAD_MODE_MLOCK = 2 +LLAMA_LOAD_MODE_MMAP_MLOCK = 3 +LLAMA_LOAD_MODE_DIRECT_IO = 4 + + +# enum llama_lazy_mode { +# LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front +# LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap) +# LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap) +# }; +LLAMA_LAZY_MODE_OFF = 0 +LLAMA_LAZY_MODE_AUTO = 1 +LLAMA_LAZY_MODE_ON = 2 + + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -524,6 +558,21 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_CONTEXT_TYPE_MTP = 1 +# // the head on the block of a DFlash draft model +# enum llama_dflash_variant { +# LLAMA_DFLASH_VARIANT_NONE = -1, // not a DFlash draft model +# LLAMA_DFLASH_VARIANT_DFLASH = 0, +# LLAMA_DFLASH_VARIANT_DFLASH2 = 1, +# LLAMA_DFLASH_VARIANT_DSPARK = 2, +# LLAMA_DFLASH_VARIANT_LILICORR = 3, +# }; +LLAMA_DFLASH_VARIANT_NONE = -1 +LLAMA_DFLASH_VARIANT_DFLASH = 0 +LLAMA_DFLASH_VARIANT_DFLASH2 = 1 +LLAMA_DFLASH_VARIANT_DSPARK = 2 +LLAMA_DFLASH_VARIANT_LILICORR = 3 + + # typedef struct llama_token_data { # llama_token id; // token id # float logit; // log-odds of the token @@ -694,6 +743,14 @@ class llama_batch(ctypes.Structure): LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA = 11 +# enum llama_process_type { +# LLAMA_PROCESS_TYPE_ENCODE, +# LLAMA_PROCESS_TYPE_DECODE, +# }; +LLAMA_PROCESS_TYPE_ENCODE = 0 +LLAMA_PROCESS_TYPE_DECODE = 1 + + # struct llama_model_kv_override { # enum llama_model_kv_override_type tag; @@ -785,8 +842,11 @@ class llama_model_imatrix_data(ctypes.Structure): # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# int32_t n_gpu_layers; // number of layers to store in VRAM +# int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model + +# enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch # // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; @@ -808,13 +868,11 @@ class llama_model_imatrix_data(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -822,21 +880,21 @@ class llama_model_params(ctypes.Structure): Attributes: devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern - n_gpu_layers (int): number of layers to store in VRAM + n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model + lazy_mode (int): on-demand reading of tensors marked by the arch main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io (bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -845,38 +903,38 @@ class llama_model_params(ctypes.Structure): ] # NOTE: unused n_gpu_layers: int split_mode: int + load_mode: int + lazy_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), + ("lazy_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] @@ -898,14 +956,15 @@ class llama_sampler_seq_config(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations # // https://github.com/ggml-org/llama.cpp/pull/7544 # struct llama_context_params { -# uint32_t n_ctx; // text context, 0 = from model -# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode -# uint32_t n_ubatch; // physical maximum batch size -# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) -# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] -# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) -# int32_t n_threads; // number of threads to use for generation -# int32_t n_threads_batch; // number of threads to use for batch processing +# uint32_t n_ctx; // text context, 0 = from model +# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode +# uint32_t n_ubatch; // physical maximum batch size +# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) +# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] +# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) +# uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max) +# int32_t n_threads; // number of threads to use for generation +# int32_t n_threads_batch; // number of threads to use for batch processing # enum llama_context_type ctx_type; // set the context type (e.g. MTP) # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` @@ -921,7 +980,6 @@ class llama_sampler_seq_config(ctypes.Structure): # float yarn_beta_fast; // YaRN low correction dim # float yarn_beta_slow; // YaRN high correction dim # uint32_t yarn_orig_ctx; // YaRN original context size -# float defrag_thold; // defragment the KV cache if holes/size > thold, <= 0 disabled (default) # ggml_backend_sched_eval_callback cb_eval; # void * cb_eval_user_data; @@ -964,6 +1022,7 @@ class llama_context_params(ctypes.Structure): n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_rs_seq (int): number of recurrent-state snapshots per sequence for rollback n_outputs_max (int): max outputs in a ubatch, 0 = n_batch + n_outputs_max_per_seq (int): max outputs per sequence, 0 = n_outputs_max n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing ctx_type (int): context type, from `enum llama_context_type` @@ -978,11 +1037,12 @@ class llama_context_params(ctypes.Structure): yarn_beta_fast (float): YaRN low correction dim yarn_beta_slow (float): YaRN high correction dim yarn_orig_ctx (int): YaRN original context size - defrag_thold (float): defragment the KV cache if holes/size > thold, <= 0 disabled (default) cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval type_k (int): data type for K cache type_v (int): data type for V cache + moe_cache_size (int): device cache size in bytes for host-resident MoE experts + moe_cache_cpu_share (float): share of MoE cache misses computed on the CPU abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback embeddings (bool): if true, extract embeddings (together with logits) @@ -1003,6 +1063,7 @@ class llama_context_params(ctypes.Structure): n_seq_max: int n_rs_seq: int n_outputs_max: int + n_outputs_max_per_seq: int n_threads: int n_threads_batch: int ctx_type: int @@ -1017,11 +1078,12 @@ class llama_context_params(ctypes.Structure): yarn_beta_fast: float yarn_beta_slow: float yarn_orig_ctx: int - defrag_thold: float cb_eval: Callable[[ctypes.c_void_p, bool], bool] cb_eval_user_data: ctypes.c_void_p type_k: int type_v: int + moe_cache_size: int + moe_cache_cpu_share: float abort_callback: Callable[[ctypes.c_void_p], bool] abort_callback_data: ctypes.c_void_p embeddings: bool @@ -1041,6 +1103,7 @@ class llama_context_params(ctypes.Structure): ("n_seq_max", ctypes.c_uint32), ("n_rs_seq", ctypes.c_uint32), ("n_outputs_max", ctypes.c_uint32), + ("n_outputs_max_per_seq", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), ("ctx_type", ctypes.c_int), @@ -1055,11 +1118,12 @@ class llama_context_params(ctypes.Structure): ("yarn_beta_fast", ctypes.c_float), ("yarn_beta_slow", ctypes.c_float), ("yarn_orig_ctx", ctypes.c_uint32), - ("defrag_thold", ctypes.c_float), ("cb_eval", ggml_backend_sched_eval_callback), ("cb_eval_user_data", ctypes.c_void_p), ("type_k", ctypes.c_int), ("type_v", ctypes.c_int), + ("moe_cache_size", ctypes.c_size_t), + ("moe_cache_cpu_share", ctypes.c_float), ("abort_callback", ggml_abort_callback), ("abort_callback_data", ctypes.c_void_p), ("embeddings", ctypes.c_bool), @@ -1106,6 +1170,7 @@ class llama_context_params(ctypes.Structure): # const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides # const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides # const int32_t * prune_layers; // pointer to layer indices to prune +# size_t max_buf_size; // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB) # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -1125,6 +1190,7 @@ class llama_model_quantize_params(ctypes.Structure): kv_overrides (ctypes.Array[llama_model_kv_override]): pointer to kv overrides tt_overrides (ctypes.Array[llama_model_tensor_override]): pointer to tensor overrides prune_layers (ctypes.Array[ctypes.c_int32]): pointer to layer indices to prune + max_buf_size (int): max bytes of tensor rows kept in memory at once, 0 = default (8 GiB) """ if TYPE_CHECKING: @@ -1142,6 +1208,7 @@ class llama_model_quantize_params(ctypes.Structure): kv_overrides: CtypesPointer[llama_model_kv_override] tt_overrides: CtypesPointer[llama_model_tensor_override] prune_layers: CtypesPointer[ctypes.c_int32] + max_buf_size: int _fields_ = [ ("nthread", ctypes.c_int32), @@ -1158,6 +1225,7 @@ class llama_model_quantize_params(ctypes.Structure): ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("tt_overrides", ctypes.POINTER(llama_model_tensor_override)), ("prune_layers", ctypes.POINTER(ctypes.c_int32)), + ("max_buf_size", ctypes.c_size_t), ] @@ -1220,6 +1288,13 @@ class llama_chat_message(ctypes.Structure): llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p) +# LLAMA_API const char * llama_version(void); +@ctypes_function("llama_version", [], ctypes.c_char_p) +def llama_version() -> bytes: + """Get the llama.cpp version.""" + ... + + # // Helpers for getting default parameters # LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( @@ -1272,8 +1347,29 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int, /) -> Optional[bytes]: + """Get the model load mode name.""" + ... + + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(value: bytes, /) -> int: + """Get the model load mode from a string.""" + ... + + +# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" +# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); +@ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) +def llama_ftype_name(ftype: int, /) -> Optional[bytes]: + '''Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"''' + ... + + # // Initialize the llama + ggml backend -# // If numa is true, use NUMA optimizations # // Call once at the start of the program # LLAMA_API void llama_backend_init(void); @ctypes_function( @@ -1316,7 +1412,8 @@ def llama_backend_free(): ... -# //optional: +# // Optional: enable numa optimizations +# // TODO: deprecate and make part of llama_backend_init() # LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); @ctypes_function( "llama_numa_init", @@ -1342,19 +1439,6 @@ def llama_numa_init(numa: int, /): ... # const char * path_model, # struct llama_model_params params), # "use llama_model_load_from_file instead"); -@ctypes_function( - "llama_load_model_from_file", - [ctypes.c_char_p, llama_model_params], - llama_model_p_ctypes, -) -def llama_load_model_from_file( - path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: ... - - -_llama_load_model_from_file = llama_load_model_from_file - - def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / ) -> Optional[llama_model_p]: @@ -1362,7 +1446,7 @@ def llama_load_model_from_file( "llama_load_model_from_file", "use llama_model_load_from_file instead", ) - return _llama_load_model_from_file(path_model, params) + return llama_model_load_from_file(path_model, params) # // Load the model from a file @@ -1408,6 +1492,8 @@ def llama_model_load_from_splits( # // Load a model from an open FILE pointer +# // The GGUF is read from the current position, so it can be embedded in a larger file +# // mmap needs the GGUF data section at a file offset to be aligned to the CPU tensor alignment (32 bytes) # LLAMA_API struct llama_model * llama_model_load_from_file_ptr( # FILE * file, # struct llama_model_params params); @@ -1419,7 +1505,11 @@ def llama_model_load_from_splits( def llama_model_load_from_file_ptr( file: ctypes.c_void_p, params: llama_model_params, / ) -> Optional[llama_model_p]: - """Load a model from an open FILE pointer.""" + """Load a model from an open FILE pointer + + The GGUF is read from the current position, so it can be embedded in a larger file + mmap needs the GGUF data section at a file offset to be aligned to the CPU tensor alignment (32 bytes) + """ ... @@ -1438,20 +1528,9 @@ def llama_model_save_to_file(model: llama_model_p, path_model: bytes, /): # DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), # "use llama_model_free instead"); -@ctypes_function( - "llama_free_model", - [llama_model_p_ctypes], - None, -) -def llama_free_model(model: llama_model_p, /): ... - - -_llama_free_model = llama_free_model - - def llama_free_model(model: llama_model_p, /): _warn_deprecated("llama_free_model", "use llama_model_free instead") - return _llama_free_model(model) + return llama_model_free(model) # LLAMA_API void llama_model_free(struct llama_model * model); @@ -1510,19 +1589,6 @@ def llama_init_from_model( # struct llama_model * model, # struct llama_context_params params), # "use llama_init_from_model instead"); -@ctypes_function( - "llama_new_context_with_model", - [llama_model_p_ctypes, llama_context_params], - llama_context_p_ctypes, -) -def llama_new_context_with_model( - model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: ... - - -_llama_new_context_with_model = llama_new_context_with_model - - def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / ) -> Optional[llama_context_p]: @@ -1530,7 +1596,7 @@ def llama_new_context_with_model( "llama_new_context_with_model", "use llama_init_from_model instead", ) - return _llama_new_context_with_model(model, params) + return llama_init_from_model(model, params) # // Frees all allocated memory @@ -1624,68 +1690,33 @@ def llama_n_rs_seq(ctx: llama_context_p, /) -> int: ... # DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); -@ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: ... - - -_llama_n_ctx_train = llama_n_ctx_train - - def llama_n_ctx_train(model: llama_model_p, /) -> int: _warn_deprecated("llama_n_ctx_train", "use llama_model_n_ctx_train instead") - return _llama_n_ctx_train(model) + return llama_model_n_ctx_train(model) # DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); -@ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: ... - - -_llama_n_embd = llama_n_embd - - def llama_n_embd(model: llama_model_p, /) -> int: _warn_deprecated("llama_n_embd", "use llama_model_n_embd instead") - return _llama_n_embd(model) + return llama_model_n_embd(model) # DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); -@ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: ... - - -_llama_n_layer = llama_n_layer - - def llama_n_layer(model: llama_model_p, /) -> int: _warn_deprecated("llama_n_layer", "use llama_model_n_layer instead") - return _llama_n_layer(model) + return llama_model_n_layer(model) # DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); -@ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_head(model: llama_model_p, /) -> int: ... - - -_llama_n_head = llama_n_head - - def llama_n_head(model: llama_model_p, /) -> int: _warn_deprecated("llama_n_head", "use llama_model_n_head instead") - return _llama_n_head(model) + return llama_model_n_head(model) # DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); -@ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_vocab_p, /) -> int: ... - - -_llama_n_vocab = llama_n_vocab - - def llama_n_vocab(model: llama_vocab_p, /) -> int: _warn_deprecated("llama_n_vocab", "use llama_vocab_n_tokens instead") - return _llama_n_vocab(model) + return llama_vocab_n_tokens(model) # LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); @@ -1744,6 +1775,11 @@ def llama_model_n_embd_out(model: llama_model_p, /) -> int: def llama_model_n_layer(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_layer_nextn(const struct llama_model * model); +@ctypes_function("llama_model_n_layer_nextn", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_layer_nextn(model: llama_model_p, /) -> int: ... + + # LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); @ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32) def llama_model_n_head(model: llama_model_p, /) -> int: ... @@ -1772,7 +1808,8 @@ def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ... # LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); @ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32) def llama_model_n_cls_out(model: llama_model_p, /) -> int: - """Returns the number of classifier outputs (only valid for classifier models)""" + """Returns the number of classifier outputs (only valid for classifier models) + Undefined behavior for non-classifier models""" ... @@ -1838,7 +1875,7 @@ def llama_model_meta_count(model: llama_model_p, /) -> int: # LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key); @ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p) def llama_model_meta_key_str(key: int, /) -> Optional[bytes]: - """Get sampling metadata key name. Returns None if the key is invalid.""" + """Get sampling metadata key name. Returns None if the key is invalid""" ... @@ -1905,6 +1942,14 @@ def llama_model_desc( ... +# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 +# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); +@ctypes_function("llama_model_ftype", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_ftype(model: llama_model_p, /) -> int: + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0""" + ... + + # // Returns the total size of all the tensors in the model in bytes # LLAMA_API uint64_t llama_model_size(const struct llama_model * model); @ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64) @@ -2014,6 +2059,46 @@ def llama_model_quantize( ... +# // +# // NextN / speculative draft model queries +# // + + +# // the DFlash variant the model loader picked from the head of the draft +# LLAMA_API enum llama_dflash_variant llama_model_dflash_variant(const struct llama_model * model); +@ctypes_function("llama_model_dflash_variant", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_dflash_variant(model: llama_model_p, /) -> int: ... + + +# // DFlash2 selector: number of candidate tokens the draft keeps per block position from its logits +# // (the packed lattice also holds the top_k x top_k pairwise transition scores); 0 if the model has no selector head +# LLAMA_API int32_t llama_model_dflash_selector_top_k(const struct llama_model * model); +@ctypes_function( + "llama_model_dflash_selector_top_k", [llama_model_p_ctypes], ctypes.c_int32 +) +def llama_model_dflash_selector_top_k(model: llama_model_p, /) -> int: ... + + +# // returns pointer to the target-model layer indices +# LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); +@ctypes_function( + "llama_model_target_layer_ids", + [llama_model_p_ctypes], + ctypes.POINTER(ctypes.c_int32), +) +def llama_model_target_layer_ids( + model: llama_model_p, / +) -> CtypesPointer[ctypes.c_int32]: ... + + +# // returns the number of extracted layers from target model +# LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); +@ctypes_function( + "llama_model_target_layer_ids_n", [llama_model_p_ctypes], ctypes.c_uint32 +) +def llama_model_target_layer_ids_n(model: llama_model_p, /) -> int: ... + + # // # // Adapters # // @@ -2033,6 +2118,22 @@ def llama_adapter_lora_init( ) -> Optional[llama_adapter_lora_p]: ... +# // Load a LoRA adapter from an open FILE pointer, reading from its current position +# LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init_from_file_ptr( +# struct llama_model * model, +# FILE * file); +@ctypes_function( + "llama_adapter_lora_init_from_file_ptr", + [llama_model_p_ctypes, ctypes.c_void_p], + llama_adapter_lora_p_ctypes, +) +def llama_adapter_lora_init_from_file_ptr( + model: llama_model_p, file: ctypes.c_void_p, / +) -> Optional[llama_adapter_lora_p]: + """Load a LoRA adapter from an open FILE pointer, reading from its current position""" + ... + + # // Get metadata value as a string by key name # LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size); @ctypes_function( @@ -2480,24 +2581,17 @@ def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the *actual* size in bytes of the state (logits, embedding and memory)""" + """Returns the *actual* size in bytes of the state + (logits, embedding and memory) + Only use when saving the state, not when restoring it, otherwise the size may be too small.""" ... # LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), # "use llama_state_get_size instead"); -@ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t) -def llama_get_state_size(ctx: llama_context_p, /) -> int: - """Returns the size in bytes of the state (DEPRECATED)""" - ... - - -_llama_get_state_size = llama_get_state_size - - def llama_get_state_size(ctx: llama_context_p, /) -> int: _warn_deprecated("llama_get_state_size", "use llama_state_get_size instead") - return _llama_get_state_size(ctx) + return llama_state_get_size(ctx) # // Copies the state to the specified destination address. @@ -2532,29 +2626,11 @@ def llama_state_get_data( # struct llama_context * ctx, # uint8_t * dst), # "use llama_state_get_data instead"); -@ctypes_function( - "llama_copy_state_data", - [ - llama_context_p_ctypes, - ctypes.POINTER(ctypes.c_uint8), - ], - ctypes.c_size_t, -) -def llama_copy_state_data( - ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / -) -> int: - """Copies the state to the specified destination address (DEPRECATED)""" - ... - - -_llama_copy_state_data = llama_copy_state_data - - def llama_copy_state_data( ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / ) -> int: _warn_deprecated("llama_copy_state_data", "use llama_state_get_data instead") - return _llama_copy_state_data(ctx, dst) + return llama_state_get_data(ctx, dst, ctypes.c_size_t(-1).value) # // Set the state reading from the specified address @@ -2583,26 +2659,11 @@ def llama_state_set_data( # struct llama_context * ctx, # const uint8_t * src), # "use llama_state_set_data instead"); -@ctypes_function( - "llama_set_state_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)], - ctypes.c_size_t, -) -def llama_set_state_data( - ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / -) -> int: - """Set the state reading from the specified address (DEPRECATED)""" - ... - - -_llama_set_state_data = llama_set_state_data - - def llama_set_state_data( ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / ) -> int: _warn_deprecated("llama_set_state_data", "use llama_state_set_data instead") - return _llama_set_state_data(ctx, src) + return llama_state_set_data(ctx, src, ctypes.c_size_t(-1).value) # Save/load session file @@ -2640,30 +2701,6 @@ def llama_state_load_file( # size_t n_token_capacity, # size_t * n_token_count_out), # "use llama_state_load_file instead"); -@ctypes_function( - "llama_load_session_file", - [ - llama_context_p_ctypes, - ctypes.c_char_p, - llama_token_p, - ctypes.c_size_t, - ctypes.POINTER(ctypes.c_size_t), - ], - ctypes.c_bool, -) -def llama_load_session_file( - ctx: llama_context_p, - path_session: bytes, - tokens_out: CtypesArray[llama_token], - n_token_capacity: Union[ctypes.c_size_t, int], - n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], - /, -) -> bool: ... - - -_llama_load_session_file = llama_load_session_file - - def llama_load_session_file( ctx: llama_context_p, path_session: bytes, @@ -2673,7 +2710,7 @@ def llama_load_session_file( /, ) -> bool: _warn_deprecated("llama_load_session_file", "use llama_state_load_file instead") - return _llama_load_session_file( + return llama_state_load_file( ctx, path_session, tokens_out, n_token_capacity, n_token_count_out ) @@ -2708,28 +2745,6 @@ def llama_state_save_file( # const llama_token * tokens, # size_t n_token_count), # "use llama_state_save_file instead"); -@ctypes_function( - "llama_save_session_file", - [ - llama_context_p_ctypes, - ctypes.c_char_p, - llama_token_p, - ctypes.c_size_t, - ], - ctypes.c_bool, -) -def llama_save_session_file( - ctx: llama_context_p, - path_session: bytes, - tokens: CtypesArray[llama_token], - n_token_count: Union[ctypes.c_size_t, int], - /, -) -> bool: ... - - -_llama_save_session_file = llama_save_session_file - - def llama_save_session_file( ctx: llama_context_p, path_session: bytes, @@ -2738,7 +2753,7 @@ def llama_save_session_file( /, ) -> bool: _warn_deprecated("llama_save_session_file", "use llama_state_save_file instead") - return _llama_save_session_file(ctx, path_session, tokens, n_token_count) + return llama_state_save_file(ctx, path_session, tokens, n_token_count) # // Get the exact size needed to copy the state of a single sequence @@ -2839,6 +2854,7 @@ def llama_state_seq_save_file( ) -> int: ... +# // If tokens_out is None, only the token count is reported through n_token_count_out and no state is loaded # LLAMA_API size_t llama_state_seq_load_file( # struct llama_context * ctx, # const char * filepath, @@ -3041,9 +3057,12 @@ def llama_batch_free(batch: llama_batch, /): # struct llama_batch batch); @ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: - """Process a batch of tokens using the encoder. + """Process a batch of tokens. + In contrast to llama_decode() - this call does not use KV cache. + For encode-decoder contexts, processes the batch using the encoder. + Can store the encoder output internally for later use by the decoder's cross-attention layers. 0 - success - < 0 - error""" + < 0 - error. the memory state is restored to the state before this call""" ... @@ -3065,104 +3084,342 @@ def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: @ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: """Process a batch of tokens. + Requires the context to have a memory. + For encode-decoder contexts, processes the batch using the decoder. + Positive return values does not mean a fatal error, but rather a warning. + Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + Upon other return values, the memory state is restored to the state before this call 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) - 2 - aborted (processed ubatches will remain in the context's memory) + 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)""" ... -# // Set the number of threads used for decoding -# // n_threads is the number of threads used for generation (single token) -# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) -# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); -@ctypes_function( - "llama_set_n_threads", - [ - llama_context_p_ctypes, - ctypes.c_int32, - ctypes.c_int32, - ], - None, -) -def llama_set_n_threads( - ctx: llama_context_p, - n_threads: Union[ctypes.c_int32, int], - n_threads_batch: Union[ctypes.c_int32, int], - /, -): - """Set the number of threads used for decoding - n_threads is the number of threads used for generation (single token) - n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) - """ - ... +# // +# // Extended batch API +# // +# struct llama_batch_ext; +llama_batch_ext_p = NewType("llama_batch_ext_p", int) +llama_batch_ext_p_ctypes = ctypes.c_void_p -# // Get the number of threads used for generation of a single token. -# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); -@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) -def llama_n_threads(ctx: llama_context_p, /) -> int: - """Get the number of threads used for generation of a single token""" - ... +# struct llama_embd { +# const float * data; +# size_t n_rows; // number of embedding rows in data +# size_t n_embd; // size of one row +# }; +class llama_embd(ctypes.Structure): + if TYPE_CHECKING: + data: CtypesPointer[ctypes.c_float] + n_rows: int + n_embd: int -# // Get the number of threads used for prompt and batch processing (multiple token). -# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); -@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) -def llama_n_threads_batch(ctx: llama_context_p, /) -> int: - """Get the number of threads used for prompt and batch processing (multiple token)""" - ... + _fields_ = [ + ("data", ctypes.POINTER(ctypes.c_float)), + ("n_rows", ctypes.c_size_t), + ("n_embd", ctypes.c_size_t), + ] -# // Set whether the context outputs embeddings or not -# // TODO: rename to avoid confusion with llama_get_embeddings() -# LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); -@ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) -def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): - """Set whether the context outputs embeddings or not""" - ... +# LLAMA_API struct llama_batch_ext * llama_batch_ext_init (struct llama_context * ctx); +@ctypes_function( + "llama_batch_ext_init", [llama_context_p_ctypes], llama_batch_ext_p_ctypes +) +def llama_batch_ext_init(ctx: llama_context_p, /) -> Optional[llama_batch_ext_p]: ... -# // Set whether to use causal attention or not -# // If set to true, the model will only attend to the past tokens -# LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); -@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None) -def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /): - """Set whether to use causal attention or not - If set to true, the model will only attend to the past tokens""" - ... +# LLAMA_API void llama_batch_ext_free (struct llama_batch_ext * batch); +@ctypes_function("llama_batch_ext_free", [llama_batch_ext_p_ctypes], None) +def llama_batch_ext_free(batch: llama_batch_ext_p, /): ... -# // Set whether the model is in warmup mode or not -# // If true, all model tensors are activated during llama_decode() to load and cache their weights. -# LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); -@ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None) -def llama_set_warmup(ctx: llama_context_p, warmup: bool, /): - """Set whether the model is in warmup mode or not - If true, all model tensors are activated during llama_decode() to load and cache their weights.""" - ... +# LLAMA_API void llama_batch_ext_clear(struct llama_batch_ext * batch); +@ctypes_function("llama_batch_ext_clear", [llama_batch_ext_p_ctypes], None) +def llama_batch_ext_clear(batch: llama_batch_ext_p, /): ... -# // Set abort callback -# LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); +# // Add an input token to the batch, with default values: +# // id = LLAMA_TOKEN_NULL +# // embd = None +# // pos = not set, the caller must set it with llama_batch_ext_set_pos() +# // Returns the batch index (>= 0) +# // On error: +# // -1: batch is full +# // -2: token is invalid (id == LLAMA_TOKEN_NULL or invalid embd) +# // -3: invalid sequence id +# LLAMA_API int32_t llama_batch_ext_add(struct llama_batch_ext * batch, llama_seq_id seq_id); @ctypes_function( - "llama_set_abort_callback", - [llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p], - None, + "llama_batch_ext_add", [llama_batch_ext_p_ctypes, llama_seq_id], ctypes.c_int32 ) -def llama_set_abort_callback( - ctx: llama_context_p, - abort_callback: Callable[[ctypes.c_void_p], None], - abort_callback_data: ctypes.c_void_p, - /, -): - """Set abort callback""" - ... +def llama_batch_ext_add(batch: llama_batch_ext_p, seq_id: int, /) -> int: ... -# // Wait until all computations are finished -# // This is automatically done when using one of the functions below to obtain the computation results +# // Add an input token to the batch, with a specified token ID or token embedding +# LLAMA_API int32_t llama_batch_ext_add_token(struct llama_batch_ext * batch, llama_seq_id seq_id, llama_token id); +@ctypes_function( + "llama_batch_ext_add_token", + [llama_batch_ext_p_ctypes, llama_seq_id, llama_token], + ctypes.c_int32, +) +def llama_batch_ext_add_token( + batch: llama_batch_ext_p, seq_id: int, id: int, / +) -> int: ... + + +# LLAMA_API int32_t llama_batch_ext_add_embd(struct llama_batch_ext * batch, llama_seq_id seq_id, struct llama_embd embd); +@ctypes_function( + "llama_batch_ext_add_embd", + [llama_batch_ext_p_ctypes, llama_seq_id, llama_embd], + ctypes.c_int32, +) +def llama_batch_ext_add_embd( + batch: llama_batch_ext_p, seq_id: int, embd: llama_embd, / +) -> int: ... + + +# // Add the token at index idx in the batch to another sequence id. The position will stays the same. +# // Note: this should be called before other _set() functions +# LLAMA_API bool llama_batch_ext_add_seq( +# struct llama_batch_ext * batch, +# int32_t idx, +# llama_seq_id seq_id); +@ctypes_function( + "llama_batch_ext_add_seq", + [llama_batch_ext_p_ctypes, ctypes.c_int32, llama_seq_id], + ctypes.c_bool, +) +def llama_batch_ext_add_seq( + batch: llama_batch_ext_p, idx: int, seq_id: int, / +) -> bool: ... + + +# // Set the token embedding for the token at index idx in the batch +# // use it after llama_batch_ext_add_token() to have an entry with both a token id and an embedding +# LLAMA_API bool llama_batch_ext_set_embd_token( +# struct llama_batch_ext * batch, +# int32_t idx, +# struct llama_embd embd); +@ctypes_function( + "llama_batch_ext_set_embd_token", + [llama_batch_ext_p_ctypes, ctypes.c_int32, llama_embd], + ctypes.c_bool, +) +def llama_batch_ext_set_embd_token( + batch: llama_batch_ext_p, idx: int, embd: llama_embd, / +) -> bool: ... + + +# // Set the "state" embedding for the token at index idx in the batch +# // "state" here means extra hidden state carried over from a previous stage, e.g.: +# // - MTP: state from N layers of the target model +# // - Qwen3 VL (deepstack): state from N layers of the vision encoder +# LLAMA_API bool llama_batch_ext_set_embd_state( +# struct llama_batch_ext * batch, +# int32_t idx, +# struct llama_embd embd); +@ctypes_function( + "llama_batch_ext_set_embd_state", + [llama_batch_ext_p_ctypes, ctypes.c_int32, llama_embd], + ctypes.c_bool, +) +def llama_batch_ext_set_embd_state( + batch: llama_batch_ext_p, idx: int, embd: llama_embd, / +) -> bool: ... + + +# // Add an entry whose embedding is row `row` of the layer input device buffer of ctx_other. +# // All entries of the batch must be of this kind. Positions are set with llama_batch_ext_set_pos(), n_pos_per_embd of them. +# // Returns the batch index (>= 0), or the llama_batch_ext_add() error codes. +# LLAMA_API int32_t llama_batch_ext_add_embd_row(struct llama_batch_ext * batch, llama_seq_id seq_id, int32_t row); +@ctypes_function( + "llama_batch_ext_add_embd_row", + [llama_batch_ext_p_ctypes, llama_seq_id, ctypes.c_int32], + ctypes.c_int32, +) +def llama_batch_ext_add_embd_row( + batch: llama_batch_ext_p, seq_id: int, row: int, / +) -> int: ... + + +# // The state of entry idx is row `row` of the h_nextn device buffer of the context set with llama_set_ctx_state_src(). +# // All entries of a batch have a state row, or none. +# LLAMA_API bool llama_batch_ext_set_embd_state_row(struct llama_batch_ext * batch, int32_t idx, int32_t row); +@ctypes_function( + "llama_batch_ext_set_embd_state_row", + [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.c_int32], + ctypes.c_bool, +) +def llama_batch_ext_set_embd_state_row( + batch: llama_batch_ext_p, idx: int, row: int, / +) -> bool: ... + + +# // Set if output embeddings should be available for the token at index idx in the batch +# // Note: for now, this is equivalent to setting the output logits +# LLAMA_API bool llama_batch_ext_set_output_embd( +# struct llama_batch_ext * batch, +# int32_t idx, +# bool value); +@ctypes_function( + "llama_batch_ext_set_output_embd", + [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.c_bool], + ctypes.c_bool, +) +def llama_batch_ext_set_output_embd( + batch: llama_batch_ext_p, idx: int, value: bool, / +) -> bool: ... + + +# // Set output logits for the token at index idx in the batch +# // Note: for now, this is equivalent to setting the output embd +# LLAMA_API bool llama_batch_ext_set_output_logits( +# struct llama_batch_ext * batch, +# int32_t idx, +# bool value); +@ctypes_function( + "llama_batch_ext_set_output_logits", + [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.c_bool], + ctypes.c_bool, +) +def llama_batch_ext_set_output_logits( + batch: llama_batch_ext_p, idx: int, value: bool, / +) -> bool: ... + + +# // Set custom position for the token at index idx in the batch +# // For M-RoPE models: +# // - Embedding tokens must have multiple positions per token +# // - Text token only requires one single position per token +# LLAMA_API bool llama_batch_ext_set_pos( +# struct llama_batch_ext * batch, +# int32_t idx, +# const llama_pos * pos); +@ctypes_function( + "llama_batch_ext_set_pos", + [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.POINTER(llama_pos)], + ctypes.c_bool, +) +def llama_batch_ext_set_pos( + batch: llama_batch_ext_p, idx: int, pos: CtypesPointerOrRef[llama_pos], / +) -> bool: ... + + +# // TODO: implement get_embeddings() and get_logits() for llama_batch_ext + + +# // Return values are the same as llama_decode() +# LLAMA_API int32_t llama_process( +# struct llama_context * ctx, +# enum llama_process_type type, +# struct llama_batch_ext * batch); +@ctypes_function( + "llama_process", + [llama_context_p_ctypes, ctypes.c_int, llama_batch_ext_p_ctypes], + ctypes.c_int32, +) +def llama_process(ctx: llama_context_p, type: int, batch: llama_batch_ext_p, /) -> int: + """Return values are the same as llama_decode()""" + ... + + +# // Set the number of threads used for decoding +# // n_threads is the number of threads used for generation (single token) +# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) +# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); +@ctypes_function( + "llama_set_n_threads", + [ + llama_context_p_ctypes, + ctypes.c_int32, + ctypes.c_int32, + ], + None, +) +def llama_set_n_threads( + ctx: llama_context_p, + n_threads: Union[ctypes.c_int32, int], + n_threads_batch: Union[ctypes.c_int32, int], + /, +): + """Set the number of threads used for decoding + n_threads is the number of threads used for generation (single token) + n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) + """ + ... + + +# // Get the number of threads used for generation of a single token. +# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); +@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) +def llama_n_threads(ctx: llama_context_p, /) -> int: + """Get the number of threads used for generation of a single token.""" + ... + + +# // Get the number of threads used for prompt and batch processing (multiple token). +# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); +@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) +def llama_n_threads_batch(ctx: llama_context_p, /) -> int: + """Get the number of threads used for prompt and batch processing (multiple token).""" + ... + + +# // Set whether the context outputs embeddings or not +# // TODO: rename to avoid confusion with llama_get_embeddings() +# LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); +@ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) +def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): + """Set whether the context outputs embeddings or not + TODO: rename to avoid confusion with llama_get_embeddings()""" + ... + + +# // Set whether to use causal attention or not +# // If set to true, the model will only attend to the past tokens +# LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); +@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None) +def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /): + """Set whether to use causal attention or not + If set to true, the model will only attend to the past tokens""" + ... + + +# // Returns whether the context is currently using causal attention +# LLAMA_API bool llama_get_causal_attn(const struct llama_context * ctx); +@ctypes_function("llama_get_causal_attn", [llama_context_p_ctypes], ctypes.c_bool) +def llama_get_causal_attn(ctx: llama_context_p, /) -> bool: + """Returns whether the context is currently using causal attention""" + ... + + +def llama_set_warmup(ctx: llama_context_p, warmup: bool, /): + """Deprecated compatibility no-op; perform warmup runs manually instead.""" + _warn_deprecated("llama_set_warmup", "perform warmup runs manually instead") + + +# // Set abort callback +# LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); +@ctypes_function( + "llama_set_abort_callback", + [llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p], + None, +) +def llama_set_abort_callback( + ctx: llama_context_p, + abort_callback: Callable[[ctypes.c_void_p], None], + abort_callback_data: ctypes.c_void_p, + /, +): + """Set abort callback""" + ... + + +# // Wait until all computations are finished +# // This is automatically done when using one of the functions below to obtain the computation results # // and is not necessary to call it explicitly in most cases # LLAMA_API void llama_synchronize(struct llama_context * ctx); @ctypes_function("llama_synchronize", [llama_context_p_ctypes], None) @@ -3268,7 +3525,172 @@ def llama_get_embeddings_seq( ... +# // +# // NextN / speculative decoding API +# // + + +# // Set whether the context outputs nextn embeddings or not +# // If masked == true, output the embeddings only for the tokens with batch.logits != 0 +# // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits +# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); +@ctypes_function( + "llama_set_embeddings_nextn", + [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], + None, +) +def llama_set_embeddings_nextn( + ctx: llama_context_p, value: bool, masked: bool, / +) -> None: ... + + +# // Select which appended NextN block the DECODER_MTP graph runs (offset past +# // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to +# // chain multiple trained NextN heads. Default 0 (first head). +# LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); +@ctypes_function( + "llama_set_nextn_layer_offset", [llama_context_p_ctypes, ctypes.c_int32], None +) +def llama_set_nextn_layer_offset(ctx: llama_context_p, offset: int, /) -> None: ... + + +# // Get all nextn embeddings, mirroring llama_get_embeddings() +# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); +@ctypes_function( + "llama_get_embeddings_nextn", + [llama_context_p_ctypes], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_embeddings_nextn( + ctx: llama_context_p, / +) -> CtypesPointer[ctypes.c_float]: ... + + +# // Get the nextn embeddings for the ith token, mirroring llama_get_embeddings_ith() +# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_embeddings_nextn_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_embeddings_nextn_ith( + ctx: llama_context_p, i: int, / +) -> CtypesPointer[ctypes.c_float]: ... + + +# // Set whether the context outputs the input embeddings of a specific layer +# LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value); +@ctypes_function( + "llama_set_embeddings_layer_inp", + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_bool], + None, +) +def llama_set_embeddings_layer_inp( + ctx: llama_context_p, lid: int, value: bool, / +) -> None: ... + + +# // Get the input embeddings of a specific layer, mirroring llama_get_embeddings() +# LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); +@ctypes_function( + "llama_get_embeddings_layer_inp", + [llama_context_p_ctypes, ctypes.c_uint32], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_embeddings_layer_inp( + ctx: llama_context_p, lid: int, / +) -> CtypesPointer[ctypes.c_float]: ... + + +# // Write the enabled layer inputs to a device buffer [n_layer_inp*n_embd, n_batch] instead of host memory. +# // Row r holds the token at batch index r, layers in ascending layer id order. +# LLAMA_API void llama_set_embeddings_layer_inp_dev(struct llama_context * ctx, bool value); +@ctypes_function( + "llama_set_embeddings_layer_inp_dev", [llama_context_p_ctypes, ctypes.c_bool], None +) +def llama_set_embeddings_layer_inp_dev( + ctx: llama_context_p, value: bool, / +) -> None: ... + + +# // Whether decode accepts entries added with llama_batch_ext_add_embd_row() +# // (needs a DFlash/DSpark decoder context whose ctx_other has a layer input device buffer). +# LLAMA_API bool llama_supports_embd_row(const struct llama_context * ctx); +@ctypes_function("llama_supports_embd_row", [llama_context_p_ctypes], ctypes.c_bool) +def llama_supports_embd_row(ctx: llama_context_p, /) -> bool: ... + + +# // Write the unmasked h_nextn rows (see llama_set_embeddings_nextn) of each decode to a device buffer +# // [n_embd_out, n_batch + n_seq_max] instead of host memory: row r < n_batch is the token at batch index r of the last decode, +# // row n_batch + s is free for the caller, e.g. for seq s (see llama_set_embeddings_nextn_rows()). +# // Returns false if the buffer cannot be used, the rows are then extracted to host memory. +# LLAMA_API bool llama_set_embeddings_nextn_dev(struct llama_context * ctx, bool value); +@ctypes_function( + "llama_set_embeddings_nextn_dev", + [llama_context_p_ctypes, ctypes.c_bool], + ctypes.c_bool, +) +def llama_set_embeddings_nextn_dev(ctx: llama_context_p, value: bool, /) -> bool: ... + + +# // Copy rows of the h_nextn device buffer to or from host memory, ordered after the last decode and its readers +# LLAMA_API bool llama_get_embeddings_nextn_rows(struct llama_context * ctx, int64_t row0, int64_t n_rows, float * out); +@ctypes_function( + "llama_get_embeddings_nextn_rows", + [ + llama_context_p_ctypes, + ctypes.c_int64, + ctypes.c_int64, + ctypes.POINTER(ctypes.c_float), + ], + ctypes.c_bool, +) +def llama_get_embeddings_nextn_rows( + ctx: llama_context_p, row0: int, n_rows: int, out: CtypesPointer[ctypes.c_float], / +) -> bool: ... + + +# LLAMA_API bool llama_set_embeddings_nextn_rows(struct llama_context * ctx, int64_t row0, int64_t n_rows, const float * data); +@ctypes_function( + "llama_set_embeddings_nextn_rows", + [ + llama_context_p_ctypes, + ctypes.c_int64, + ctypes.c_int64, + ctypes.POINTER(ctypes.c_float), + ], + ctypes.c_bool, +) +def llama_set_embeddings_nextn_rows( + ctx: llama_context_p, row0: int, n_rows: int, data: CtypesPointer[ctypes.c_float], / +) -> bool: ... + + +# // Read the state rows of batches (see llama_batch_ext_set_embd_state_row()) from the h_nextn device buffer of ctx_src. +# // Returns whether decode accepts batches with state rows. +# LLAMA_API bool llama_set_ctx_state_src(struct llama_context * ctx, struct llama_context * ctx_src); +@ctypes_function( + "llama_set_ctx_state_src", + [llama_context_p_ctypes, llama_context_p_ctypes], + ctypes.c_bool, +) +def llama_set_ctx_state_src( + ctx: llama_context_p, ctx_src: llama_context_p, / +) -> bool: ... + + +# // The paired context (llama_context_params::ctx_other), e.g. the target context of a draft context; may be None +# LLAMA_API struct llama_context * llama_get_ctx_other(struct llama_context * ctx); +@ctypes_function( + "llama_get_ctx_other", [llama_context_p_ctypes], llama_context_p_ctypes +) +def llama_get_ctx_other(ctx: llama_context_p, /) -> Optional[llama_context_p]: ... + + # // Get the backend sampled token for the ith token. +# // With multiple outputs, sampler state advances when the token is accepted, +# // not when it is read through this function. +# // When accepting multiple outputs, accept a contiguous prefix in output order. # // Returns LLAMA_TOKEN_NULL if no token was sampled. # LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); @ctypes_function( @@ -3493,6 +3915,20 @@ def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [llama_vocab_p_ctypes, ctypes.POINTER(ctypes.c_int32)], + ctypes.POINTER(llama_token), +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: CtypesPointer[ctypes.c_int32], + /, +) -> Optional[CtypesPointer[llama_token]]: ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", @@ -3549,101 +3985,36 @@ def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... # DEPRECATED functions # DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); -@ctypes_function( - "llama_token_get_text", - [llama_vocab_p_ctypes, llama_token], - ctypes.c_char_p, -) -def llama_token_get_text( - vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bytes: ... - - -_llama_token_get_text = llama_token_get_text - - def llama_token_get_text( vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> bytes: _warn_deprecated("llama_token_get_text", "use llama_vocab_get_text instead") - return _llama_token_get_text(vocab, token) + return llama_vocab_get_text(vocab, token) # DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); -@ctypes_function( - "llama_token_get_score", - [llama_vocab_p_ctypes, llama_token], - ctypes.c_float, -) -def llama_token_get_score( - vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> float: ... - - -_llama_token_get_score = llama_token_get_score - - def llama_token_get_score( vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> float: _warn_deprecated("llama_token_get_score", "use llama_vocab_get_score instead") - return _llama_token_get_score(vocab, token) + return llama_vocab_get_score(vocab, token) # DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); -@ctypes_function( - "llama_token_get_attr", - [llama_vocab_p_ctypes, llama_token], - ctypes.c_int, -) -def llama_token_get_attr( - vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> int: ... - - -_llama_token_get_attr = llama_token_get_attr - - def llama_token_get_attr( vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> int: _warn_deprecated("llama_token_get_attr", "use llama_vocab_get_attr instead") - return _llama_token_get_attr(vocab, token) + return llama_vocab_get_attr(vocab, token) # DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); -@ctypes_function( - "llama_token_is_eog", - [llama_vocab_p_ctypes, llama_token], - ctypes.c_bool, -) -def llama_token_is_eog( - vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bool: ... - - -_llama_token_is_eog = llama_token_is_eog - - def llama_token_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool: _warn_deprecated("llama_token_is_eog", "use llama_vocab_is_eog instead") - return _llama_token_is_eog(vocab, token) + return llama_vocab_is_eog(vocab, token) # DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); -@ctypes_function( - "llama_token_is_control", - [llama_vocab_p_ctypes, llama_token], - ctypes.c_bool, -) -def llama_token_is_control( - vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bool: ... - - -_llama_token_is_control = llama_token_is_control - - def llama_token_is_control( vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> bool: @@ -3651,281 +4022,105 @@ def llama_token_is_control( "llama_token_is_control", "use llama_vocab_is_control instead", ) - return _llama_token_is_control(vocab, token) + return llama_vocab_is_control(vocab, token) # DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); -@ctypes_function( - "llama_token_bos", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_bos(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_bos = llama_token_bos - - def llama_token_bos(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_bos", "use llama_vocab_bos instead") - return _llama_token_bos(vocab) + return llama_vocab_bos(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); -@ctypes_function( - "llama_token_eos", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_eos(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_eos = llama_token_eos - - def llama_token_eos(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_eos", "use llama_vocab_eos instead") - return _llama_token_eos(vocab) + return llama_vocab_eos(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); -@ctypes_function( - "llama_token_eot", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_eot(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_eot = llama_token_eot - - def llama_token_eot(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_eot", "use llama_vocab_eot instead") - return _llama_token_eot(vocab) + return llama_vocab_eot(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); -@ctypes_function( - "llama_token_cls", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_cls(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_cls = llama_token_cls - - def llama_token_cls(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_cls", "use llama_vocab_cls instead") - return _llama_token_cls(vocab) + return llama_vocab_bos(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); -@ctypes_function( - "llama_token_sep", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_sep(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_sep = llama_token_sep - - def llama_token_sep(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_sep", "use llama_vocab_sep instead") - return _llama_token_sep(vocab) + return llama_vocab_sep(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); -@ctypes_function( - "llama_token_nl", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_nl(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_nl = llama_token_nl - - def llama_token_nl(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_nl", "use llama_vocab_nl instead") - return _llama_token_nl(vocab) + return llama_vocab_nl(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); -@ctypes_function( - "llama_token_pad", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_pad(vocab: llama_vocab_p, /) -> int: ... - - -_llama_token_pad = llama_token_pad - - def llama_token_pad(vocab: llama_vocab_p, /) -> int: _warn_deprecated("llama_token_pad", "use llama_vocab_pad instead") - return _llama_token_pad(vocab) + return llama_vocab_pad(vocab) # DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); -@ctypes_function( - "llama_add_bos_token", - [llama_vocab_p_ctypes], - ctypes.c_bool, -) -def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: ... - - -_llama_add_bos_token = llama_add_bos_token - - def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: _warn_deprecated("llama_add_bos_token", "use llama_vocab_get_add_bos instead") - return _llama_add_bos_token(vocab) + return llama_vocab_get_add_bos(vocab) # DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); -@ctypes_function( - "llama_add_eos_token", - [llama_vocab_p_ctypes], - ctypes.c_bool, -) -def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: ... - - -_llama_add_eos_token = llama_add_eos_token - - def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: _warn_deprecated("llama_add_eos_token", "use llama_vocab_get_add_eos instead") - return _llama_add_eos_token(vocab) + return llama_vocab_get_add_eos(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); -@ctypes_function( - "llama_token_fim_pre", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_token_fim_pre = llama_token_fim_pre - - def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_token_fim_pre", "use llama_vocab_fim_pre instead") - return _llama_token_fim_pre(vocab) + return llama_vocab_fim_pre(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); -@ctypes_function( - "llama_token_fim_suf", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_token_fim_suf = llama_token_fim_suf - - def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_token_fim_suf", "use llama_vocab_fim_suf instead") - return _llama_token_fim_suf(vocab) + return llama_vocab_fim_suf(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); -@ctypes_function( - "llama_token_fim_mid", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_token_fim_mid = llama_token_fim_mid - - def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_token_fim_mid", "use llama_vocab_fim_mid instead") - return _llama_token_fim_mid(vocab) + return llama_vocab_fim_mid(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); -@ctypes_function( - "llama_token_fim_pad", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_token_fim_pad = llama_token_fim_pad - - def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_token_fim_pad", "use llama_vocab_fim_pad instead") - return _llama_token_fim_pad(vocab) + return llama_vocab_fim_pad(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); -@ctypes_function( - "llama_token_fim_rep", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_token_fim_rep = llama_token_fim_rep - - def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_token_fim_rep", "use llama_vocab_fim_rep instead") - return _llama_token_fim_rep(vocab) + return llama_vocab_fim_rep(vocab) # DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); -@ctypes_function( - "llama_token_fim_sep", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_token_fim_sep = llama_token_fim_sep - - def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_token_fim_sep", "use llama_vocab_fim_sep instead") - return _llama_token_fim_sep(vocab) + return llama_vocab_fim_sep(vocab) # // CLS is equivalent to BOS # DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification # "use llama_vocab_bos instead"); -@ctypes_function( - "llama_vocab_cls", - [llama_vocab_p_ctypes], - llama_token, -) -def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: ... - - -_llama_vocab_cls = llama_vocab_cls - - def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: _warn_deprecated("llama_vocab_cls", "use llama_vocab_bos instead") - return _llama_vocab_cls(vocab) + return llama_vocab_bos(vocab) # // @@ -4241,7 +4436,7 @@ class llama_sampler(ctypes.Structure): llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes) llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) llama_sampler_i_backend_init = ctypes.CFUNCTYPE( - ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p + ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p, ctypes.c_uint32 ) llama_sampler_i_backend_accept = ctypes.CFUNCTYPE( None, @@ -4258,6 +4453,10 @@ class llama_sampler(ctypes.Structure): ctypes.POINTER(llama_sampler_data), ) llama_sampler_i_backend_set_input = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_backend_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_copy_state = ctypes.CFUNCTYPE( + None, llama_sampler_p_ctypes, llama_sampler_p_ctypes +) llama_sampler_i._fields_ = [ ("name", llama_sampler_i_name), @@ -4270,6 +4469,8 @@ class llama_sampler(ctypes.Structure): ("backend_accept", llama_sampler_i_backend_accept), ("backend_apply", llama_sampler_i_backend_apply), ("backend_set_input", llama_sampler_i_backend_set_input), + ("backend_reset", llama_sampler_i_backend_reset), + ("copy_state", llama_sampler_i_copy_state), ] @@ -4346,6 +4547,15 @@ def llama_sampler_reset(smpl: llama_sampler_p, /): ... def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... +# LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst); +@ctypes_function( + "llama_sampler_copy", + [llama_sampler_p_ctypes, llama_sampler_p_ctypes], + None, +) +def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p, /): ... + + # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) # LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); @ctypes_function( @@ -4392,11 +4602,11 @@ def llama_sampler_chain_get( ) -> llama_sampler_p: ... -# LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); +# LLAMA_API int32_t llama_sampler_chain_n (const struct llama_sampler * chain); @ctypes_function( "llama_sampler_chain_n", [llama_sampler_p_ctypes], - ctypes.c_int, + ctypes.c_int32, ) def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... @@ -4559,34 +4769,6 @@ def llama_sampler_init_grammar( # const llama_token * trigger_tokens, # size_t num_trigger_tokens), # "use llama_sampler_init_grammar_lazy_patterns instead"); -@ctypes_function( - "llama_sampler_init_grammar_lazy", - [ - llama_vocab_p_ctypes, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.POINTER(ctypes.c_char_p), - ctypes.c_size_t, - ctypes.POINTER(llama_token), - ctypes.c_size_t, - ], - llama_sampler_p_ctypes, -) -def llama_sampler_init_grammar_lazy( - vocab: llama_vocab_p, - grammar_str: bytes, - grammar_root: bytes, - trigger_words: CtypesArray[bytes], - num_trigger_words: int, - trigger_tokens: CtypesArray[llama_token], - num_trigger_tokens: int, - /, -) -> llama_sampler_p: ... - - -_llama_sampler_init_grammar_lazy = llama_sampler_init_grammar_lazy - - def llama_sampler_init_grammar_lazy( vocab: llama_vocab_p, grammar_str: bytes, @@ -4601,12 +4783,17 @@ def llama_sampler_init_grammar_lazy( "llama_sampler_init_grammar_lazy", "use llama_sampler_init_grammar_lazy_patterns instead", ) - return _llama_sampler_init_grammar_lazy( + # Match the literal-trigger conversion used by the removed native wrapper. + patterns = (ctypes.c_char_p * (1 if trigger_words and num_trigger_words else 0))() + if patterns: + words = [re.escape(trigger_words[i]) for i in range(num_trigger_words)] + patterns[0] = rb"[\s\S]*?(" + b"|".join(words) + rb")[\s\S]*" + return llama_sampler_init_grammar_lazy_patterns( vocab, grammar_str, grammar_root, - trigger_words, - num_trigger_words, + patterns, + len(patterns), trigger_tokens, num_trigger_tokens, ) @@ -4648,16 +4835,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# int32_t n_vocab, +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty) +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, @@ -4669,18 +4864,16 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 # LLAMA_API struct llama_sampler * llama_sampler_init_dry( # const struct llama_vocab * vocab, -# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, -# int32_t dry_penalty_last_n, +# int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty) # const char ** seq_breakers, # size_t num_breakers); @ctypes_function( "llama_sampler_init_dry", [ llama_vocab_p_ctypes, - ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -4692,7 +4885,6 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_dry( vocab: llama_vocab_p, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -4754,6 +4946,7 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... # /// @details Sample and accept a token from the idx-th output of the last evaluation +# // For multiple outputs from one sampler, call this function in output order without gaps. # LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); @ctypes_function( "llama_sampler_sample", @@ -4863,6 +5056,8 @@ def llama_log_set( # int32_t n_p_eval; # int32_t n_eval; # int32_t n_reused; // number of times a ggml compute graph had been reused +# int32_t n_reallocs_same; // reallocations with unchanged graph size +# int32_t n_reallocs_diff; // reallocations with changed graph size # }; class llama_perf_context_data(ctypes.Structure): _fields_ = [ @@ -4873,6 +5068,8 @@ class llama_perf_context_data(ctypes.Structure): ("n_p_eval", ctypes.c_int32), ("n_eval", ctypes.c_int32), ("n_reused", ctypes.c_int32), + ("n_reallocs_same", ctypes.c_int32), + ("n_reallocs_diff", ctypes.c_int32), ] diff --git a/llama_cpp/llama_cpp_ext.py b/llama_cpp/llama_cpp_ext.py index 284811086a..257ce50c2c 100644 --- a/llama_cpp/llama_cpp_ext.py +++ b/llama_cpp/llama_cpp_ext.py @@ -1,117 +1,13 @@ -"""Experimental bindings for non-public llama.cpp APIs from `llama-ext.h`. +"""Compatibility imports for formerly non-public llama.cpp APIs. -This module is not part of the stable llama-cpp-python public API. -Downstream code should not import or depend on it directly. +These functions are now exposed by llama_cpp.llama_cpp. +This module remains experimental; new callers should use the public bindings. """ -from __future__ import annotations - -import ctypes -import functools - -from typing import Any, Iterable, Union - -from . import llama_cpp - -_lib = llama_cpp._lib - - -def _ctypes_function_from_names( - names: Iterable[str], - argtypes: list[Any], - restype: Any, -): - """Decorator for extension functions whose exported symbol name can vary by ABI.""" - - def decorator(f): - missing: list[str] = [] - for name in names: - try: - func = getattr(_lib, name) - except AttributeError: - missing.append(name) - continue - func.argtypes = argtypes - func.restype = restype - functools.wraps(f)(func) - return func - raise AttributeError( - f"None of the shared library symbols were found: {', '.join(missing)}" - ) - - return decorator - - -# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); -@_ctypes_function_from_names( - ( - "llama_set_embeddings_nextn", - "_Z26llama_set_embeddings_nextnP13llama_contextbb", - "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", - ), - [llama_cpp.llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], - None, -) -def llama_set_embeddings_nextn( - ctx: llama_cpp.llama_context_p, - value: bool, - masked: bool, - /, -): - """Set whether the context outputs nextn embeddings or not.""" - ... - - -# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); -@_ctypes_function_from_names( - ( - "llama_get_embeddings_nextn", - "_Z26llama_get_embeddings_nextnP13llama_context", - "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", - ), - [llama_cpp.llama_context_p_ctypes], - ctypes.POINTER(ctypes.c_float), -) -def llama_get_embeddings_nextn( - ctx: llama_cpp.llama_context_p, - /, -): - """Get the nextn embeddings from the last evaluation.""" - ... - - -# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); -@_ctypes_function_from_names( - ( - "llama_get_embeddings_nextn_ith", - "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", - "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", - ), - [llama_cpp.llama_context_p_ctypes, ctypes.c_int32], - ctypes.POINTER(ctypes.c_float), -) -def llama_get_embeddings_nextn_ith( - ctx: llama_cpp.llama_context_p, - i: Union[ctypes.c_int32, int], - /, -): - """Get the nextn embeddings for the ith output row from the last evaluation.""" - ... - - -# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); -@_ctypes_function_from_names( - ( - "llama_get_ctx_other", - "_Z19llama_get_ctx_otherP13llama_context", - "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", - ), - [llama_cpp.llama_context_p_ctypes], - llama_cpp.llama_context_p_ctypes, +from .llama_cpp import ( + llama_get_ctx_other as llama_get_ctx_other, + llama_get_embeddings_nextn as llama_get_embeddings_nextn, + llama_get_embeddings_nextn_ith as llama_get_embeddings_nextn_ith, + llama_set_embeddings_nextn as llama_set_embeddings_nextn, + llama_set_nextn_layer_offset as llama_set_nextn_layer_offset, ) -def llama_get_ctx_other( - ctx: llama_cpp.llama_context_p, - /, -): - """Get the context linked through llama_context_params.ctx_other.""" - ... diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 78f068aa9a..2c41febdb3 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -5,8 +5,10 @@ from ctypes import ( CFUNCTYPE, c_bool, + c_char, c_char_p, c_int, + c_int32, c_int64, c_uint8, c_uint32, @@ -20,6 +22,7 @@ ) import pathlib from typing import ( + Callable, Union, NewType, Optional, @@ -67,6 +70,9 @@ mtmd_helper_video_p = NewType("mtmd_helper_video_p", int) mtmd_helper_video_p_ctypes = c_void_p +mtmd_helper_gen_audio_p = NewType("mtmd_helper_gen_audio_p", int) +mtmd_helper_gen_audio_p_ctypes = c_void_p + mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) mtmd_image_tokens_p_ctypes = c_void_p @@ -79,10 +85,43 @@ mtmd_batch_p = NewType("mtmd_batch_p", int) mtmd_batch_p_ctypes = c_void_p +mtmd_audio_stream_p = NewType("mtmd_audio_stream_p", int) +mtmd_audio_stream_p_ctypes = c_void_p + +mtmd_gen_state_p = NewType("mtmd_gen_state_p", int) +mtmd_gen_state_p_ctypes = c_void_p + # Enums MTMD_INPUT_CHUNK_TYPE_TEXT = 0 MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 +MTMD_INPUT_CHUNK_TYPE_COUNT = 3 + +MTMD_VIDEO_TEMPLATE_DEFAULT = 0 +MTMD_VIDEO_TEMPLATE_LFM3 = 1 +MTMD_VIDEO_TEMPLATE_LFM3_OMNI = 2 +MTMD_VIDEO_TEMPLATE_QWEN3VL = 3 + +MTMD_GEN_AUDIO_TYPE_NONE = 0 +MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 +MTMD_GEN_AUDIO_TYPE_POCKETTTS = 2 +MTMD_GEN_AUDIO_TYPE_SOPRANO = 3 +MTMD_GEN_AUDIO_TYPE_LFM3A = 4 +MTMD_GEN_AUDIO_TYPE_CHATTERBOX = 5 +MTMD_GEN_AUDIO_TYPE_DRAMA3 = 6 + +MTMD_GEN_PROCESS_TYPE_GEN_CODE = 0 +MTMD_GEN_PROCESS_TYPE_GEN_WAV = 1 + +MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM = 0 +MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV = 1 + +MTMD_HELPER_GEN_AUDIO_OUT_AUTO = 0 +MTMD_HELPER_GEN_AUDIO_OUT_TEXT = 1 +MTMD_HELPER_GEN_AUDIO_OUT_AUDIO = 2 +MTMD_HELPER_GEN_AUDIO_OUT_MIXED = 3 + +mtmd_progress_callback = CFUNCTYPE(c_bool, c_float, c_void_p) # Structures @@ -95,6 +134,7 @@ class mtmd_context_params(Structure): if TYPE_CHECKING: use_gpu: bool + device: c_void_p print_timings: bool n_threads: int image_marker: Optional[bytes] @@ -106,9 +146,12 @@ class mtmd_context_params(Structure): cb_eval: llama_cpp.ggml_backend_sched_eval_callback cb_eval_user_data: c_void_p batch_max_tokens: int + progress_callback: Callable[[float, c_void_p], bool] + progress_callback_user_data: c_void_p _fields_ = [ ("use_gpu", c_bool), + ("device", c_void_p), ("print_timings", c_bool), ("n_threads", c_int), ("image_marker", c_char_p), @@ -120,19 +163,46 @@ class mtmd_context_params(Structure): ("cb_eval", llama_cpp.ggml_backend_sched_eval_callback), ("cb_eval_user_data", c_void_p), ("batch_max_tokens", c_int), + ("progress_callback", mtmd_progress_callback), + ("progress_callback_user_data", c_void_p), ] class mtmd_input_text(Structure): """Text input passed to `mtmd_tokenize`.""" + if TYPE_CHECKING: + text: Optional[bytes] + text_len: int + add_special: bool + parse_special: bool + _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] +# struct mtmd_input_part { +# // only text or bitmap can be set, not both +# const struct mtmd_input_text * text; +# const struct mtmd_bitmap * bitmap; +# }; +class mtmd_input_part(Structure): + """An input part with only text or bitmap set, not both.""" + + if TYPE_CHECKING: + text: _Pointer[mtmd_input_text] + bitmap: Optional[mtmd_bitmap_p] + + _fields_ = [ + ("text", POINTER(mtmd_input_text)), + ("bitmap", mtmd_bitmap_p_ctypes), + ] + + class mtmd_decoder_pos(Structure): """Decoder attention position for M-RoPE models.""" @@ -144,9 +214,52 @@ class mtmd_decoder_pos(Structure): ] +# struct mtmd_video_config { +# enum mtmd_video_template tmpl; +# float fps; +# int32_t n_frames; +# int32_t frame_width; +# int32_t frame_height; +# }; +class mtmd_video_config(Structure): + if TYPE_CHECKING: + tmpl: int + fps: float + n_frames: int + frame_width: int + frame_height: int + + _fields_ = [ + ("tmpl", c_int), + ("fps", c_float), + ("n_frames", c_int32), + ("frame_width", c_int32), + ("frame_height", c_int32), + ] + + +# struct mtmd_audio_stream_params { +# int32_t step_ms; +# const llama_token * prompt; +# size_t n_prompt; +# }; +class mtmd_audio_stream_params(Structure): + if TYPE_CHECKING: + step_ms: int + prompt: Optional["_Pointer[llama_cpp.llama_token]"] + n_prompt: int + + _fields_ = [ + ("step_ms", c_int32), + ("prompt", llama_cpp.llama_token_p), + ("n_prompt", c_size_t), + ] + + # struct mtmd_caps { # bool inp_vision; # bool inp_audio; +# bool gen_audio; # }; class mtmd_caps(Structure): """Capabilities exposed by an mmproj file.""" @@ -154,13 +267,135 @@ class mtmd_caps(Structure): if TYPE_CHECKING: inp_vision: bool inp_audio: bool + gen_audio: bool _fields_ = [ ("inp_vision", c_bool), ("inp_audio", c_bool), + ("gen_audio", c_bool), ] +# struct mtmd_gen_audio_info { +# enum mtmd_gen_audio_type type; +# int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# const char * model_variant; // name of the weight variant, can be None if not applicable +# int32_t n_voices; // predefined voices, see mtmd_gen_audio_voice_name() +# }; +class mtmd_gen_audio_info(Structure): + if TYPE_CHECKING: + type: int + sample_rate: int + model_variant: Optional[bytes] + n_voices: int + + _fields_ = [ + ("type", c_int), + ("sample_rate", c_int32), + ("model_variant", c_char_p), + ("n_voices", c_int32), + ] + + +# struct mtmd_gen_inp { +# enum mtmd_gen_process_type type; +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# int32_t code0; // the sampled codebook 0 entry from backbone +# float * embd; // the hidden state from backbone, must have n_text_embd elements +# int32_t top_k; +# float top_p; +# float temp; // sampling temperature, or noise scale for flow-matching decoders (0 = no noise) +# float penalty_repeat; // repetition penalty of the sampled codes, <= 1 is off +# int32_t penalty_last_n; // frames the repetition penalty looks back on +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# // pass either codes (discrete) or feats (continuous), depending on the pipeline +# int32_t * codes; +# size_t n_codes; +# const float * feats; +# size_t n_feats; +# }; +class mtmd_gen_inp(Structure): + if TYPE_CHECKING: + type: int + code0: int + embd: Optional["_Pointer[c_float]"] + top_k: int + top_p: float + temp: float + penalty_repeat: float + penalty_last_n: int + codes: Optional["_Pointer[c_int32]"] + n_codes: int + feats: Optional["_Pointer[c_float]"] + n_feats: int + + _fields_ = [ + ("type", c_int), + ("code0", c_int32), + ("embd", POINTER(c_float)), + ("top_k", c_int32), + ("top_p", c_float), + ("temp", c_float), + ("penalty_repeat", c_float), + ("penalty_last_n", c_int32), + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), + ] + + +# struct mtmd_gen_out { +# // note: output memory is owned by the state, valid until the next process() call with this state +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# const int32_t * codes; +# size_t n_codes; +# const float * feats; // continuous counterpart of codes +# size_t n_feats; +# const float * embd; // the generated hidden state, to be fed back to backbone +# // it must have n_text_embd elements +# bool is_eos; // only set by pipelines having the EOS head inside mmproj +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# const float * audio; +# size_t n_samples; +# }; +class mtmd_gen_out(Structure): + if TYPE_CHECKING: + codes: Optional["_Pointer[c_int32]"] + n_codes: int + feats: Optional["_Pointer[c_float]"] + n_feats: int + embd: Optional["_Pointer[c_float]"] + is_eos: bool + audio: Optional["_Pointer[c_float]"] + n_samples: int + + _fields_ = [ + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), + ("embd", POINTER(c_float)), + ("is_eos", c_bool), + ("audio", POINTER(c_float)), + ("n_samples", c_size_t), + ] + + +# struct mtmd_gen_state_params { +# uint32_t seed; // UINT32_MAX for random +# }; +class mtmd_gen_state_params(Structure): + if TYPE_CHECKING: + seed: int + + _fields_ = [("seed", c_uint32)] + + mtmd_bitmap_lazy_callback = CFUNCTYPE( c_int, c_size_t, @@ -169,9 +404,41 @@ class mtmd_caps(Structure): POINTER(c_char_p), ) + +# // one decoded sub-batch of embeddings, passed to mtmd_helper_post_decode_callback +# struct mtmd_helper_embd_batch { +# int32_t n_tokens; +# const float * embd; // [n_tokens, n_embd] +# int32_t n_embd; +# const llama_pos * pos; // [n_pos, n_tokens], section-major +# int32_t n_pos; // 4 for M-RoPE models, 1 otherwise +# llama_seq_id seq_id; +# }; +class mtmd_helper_embd_batch(Structure): + """One decoded sub-batch of embeddings, passed to mtmd_helper_post_decode_callback.""" + + if TYPE_CHECKING: + n_tokens: int + embd: "_Pointer[c_float]" + n_embd: int + pos: "_Pointer[llama_cpp.llama_pos]" + n_pos: int + seq_id: int + + _fields_ = [ + ("n_tokens", c_int32), + ("embd", POINTER(c_float)), + ("n_embd", c_int32), + ("pos", POINTER(llama_cpp.llama_pos)), + ("n_pos", c_int32), + ("seq_id", llama_cpp.llama_seq_id), + ] + + +# typedef int32_t (*mtmd_helper_post_decode_callback)(const struct mtmd_helper_embd_batch * batch, void * user_data); mtmd_helper_post_decode_callback = CFUNCTYPE( - c_int, - llama_cpp.llama_batch, + c_int32, + POINTER(mtmd_helper_embd_batch), c_void_p, ) @@ -206,6 +473,73 @@ class mtmd_helper_video_info(Structure): ] +# struct mtmd_helper_video_stream_info { +# uint32_t width; // as displayed (rotation and pixel aspect ratio applied) +# uint32_t height; +# float fps; +# int32_t n_frames; // -1 if unknown +# float duration; // in seconds, -1 if unknown +# }; +class mtmd_helper_video_stream_info(Structure): + """Source video metadata, with rotation and pixel aspect ratio applied.""" + + _fields_ = [ + ("width", c_uint32), + ("height", c_uint32), + ("fps", c_float), + ("n_frames", c_int32), + ("duration", c_float), + ] + + +# Video decoder: the built-in one runs ffmpeg/ffprobe (MTMD_VIDEO), an application can provide its own. +# Call order: open, start, read_audio, read_frame..., close +mtmd_helper_video_open_callback = CFUNCTYPE( + c_void_p, + c_void_p, + c_char_p, + POINTER(c_uint8), + c_size_t, + POINTER(mtmd_helper_video_stream_info), +) +mtmd_helper_video_start_callback = CFUNCTYPE( + c_bool, + c_void_p, + c_uint32, + c_uint32, + c_float, + POINTER(c_bool), +) +mtmd_helper_video_read_frame_callback = CFUNCTYPE( + c_bool, + c_void_p, + POINTER(c_uint8), + c_uint32, + c_uint32, +) +# ctypes callbacks cannot return a typed pointer; return the sample buffer's address. +mtmd_helper_video_read_audio_callback = CFUNCTYPE( + c_void_p, + c_void_p, + c_int, + POINTER(c_size_t), +) +mtmd_helper_video_close_callback = CFUNCTYPE(None, c_void_p) + + +class mtmd_helper_video_decoder(Structure): + """Keep the callbacks and their buffers alive while the decoder is in use.""" + + _fields_ = [ + ("open", mtmd_helper_video_open_callback), + ("start", mtmd_helper_video_start_callback), + ("read_frame", mtmd_helper_video_read_frame_callback), + ("read_audio", mtmd_helper_video_read_audio_callback), + ("close", mtmd_helper_video_close_callback), + ("user_data", c_void_p), + ] + + class mtmd_helper_video_init_params(Structure): """Parameters for initializing an MTMD helper video stream.""" @@ -213,11 +547,104 @@ class mtmd_helper_video_init_params(Structure): fps_target: float ffmpeg_bin_dir: Optional[bytes] timestamp_interval_ms: int + no_soundtrack: bool + decoder: "_Pointer[mtmd_helper_video_decoder]" _fields_ = [ ("fps_target", c_float), ("ffmpeg_bin_dir", c_char_p), ("timestamp_interval_ms", c_int64), + ("no_soundtrack", c_bool), + ("decoder", POINTER(mtmd_helper_video_decoder)), + ] + + +# // opt for mtmd_helper_bitmap_init_from_*() +# struct mtmd_helper_init_opt { +# struct mtmd_helper_video_init_params video_params; +# }; +class mtmd_helper_init_opt(Structure): + if TYPE_CHECKING: + video_params: mtmd_helper_video_init_params + + _fields_ = [("video_params", mtmd_helper_video_init_params)] + + +# struct mtmd_helper_gen_audio_inp { +# bool stream; // if true, output() must be called after each step_gen() +# llama_seq_id seq_id; +# +# const char * prompt; +# size_t prompt_len; +# +# int32_t n_past; +# enum mtmd_helper_gen_audio_out_mode out_mode; +# +# mtmd_bitmap * speaker_ref; // optional, can be NULL +# const char * lang; // optional, can be NULL +# +# int32_t top_k; +# float top_p; +# float temp; // audio frame temperature if the model supports it, < 0 = model default, 0 = greedy +# uint32_t seed; // UINT32_MAX for random (default: random) +# +# enum mtmd_helper_gen_audio_outtype out_type; +# }; +class mtmd_helper_gen_audio_inp(Structure): + if TYPE_CHECKING: + stream: bool + seq_id: int + prompt: Optional[bytes] + prompt_len: int + n_past: int + out_mode: int + speaker_ref: Optional[mtmd_bitmap_p] + lang: Optional[bytes] + top_k: int + top_p: float + temp: float + seed: int + out_type: int + + _fields_ = [ + ("stream", c_bool), + ("seq_id", llama_cpp.llama_seq_id), + ("prompt", c_char_p), + ("prompt_len", c_size_t), + ("n_past", c_int32), + ("out_mode", c_int), + ("speaker_ref", mtmd_bitmap_p_ctypes), + ("lang", c_char_p), + ("top_k", c_int32), + ("top_p", c_float), + ("temp", c_float), + ("seed", c_uint32), + ("out_type", c_int), + ] + + +class mtmd_helper_cap(Structure): + """Capabilities exposed by an MTMD model and its projector.""" + + if TYPE_CHECKING: + chat: bool + vision: bool + audio: bool + video: bool + audio_stream: bool + text_gen: bool + audio_gen: bool + audio_gen_spk_ref: bool + + _fields_ = [ + ("chat", c_bool), + ("vision", c_bool), + ("audio", c_bool), + ("video", c_bool), + ("audio_stream", c_bool), + ("text_gen", c_bool), + ("audio_gen", c_bool), + ("audio_gen_spk_ref", c_bool), ] @@ -297,6 +724,13 @@ def mtmd_support_audio(ctx: mtmd_context_p, /) -> bool: ... +# MTMD_API bool mtmd_support_audio_stream(const mtmd_context * ctx); +@ctypes_function("mtmd_support_audio_stream", [mtmd_context_p_ctypes], c_bool) +def mtmd_support_audio_stream(ctx: mtmd_context_p, /) -> bool: + """Check whether the current model supports realtime audio input.""" + ... + + # MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); @ctypes_function("mtmd_get_audio_sample_rate", [mtmd_context_p_ctypes], c_int) def mtmd_get_audio_sample_rate(ctx: mtmd_context_p, /) -> int: @@ -304,6 +738,134 @@ def mtmd_get_audio_sample_rate(ctx: mtmd_context_p, /) -> int: ... +# MTMD_API struct mtmd_audio_stream_params mtmd_audio_stream_params_default(void); +@ctypes_function("mtmd_audio_stream_params_default", [], mtmd_audio_stream_params) +def mtmd_audio_stream_params_default() -> mtmd_audio_stream_params: + """Get default realtime audio stream parameters.""" + ... + + +# MTMD_API mtmd_audio_stream * mtmd_audio_stream_init( +# mtmd_context * ctx, struct mtmd_audio_stream_params params); +@ctypes_function( + "mtmd_audio_stream_init", + [mtmd_context_p_ctypes, mtmd_audio_stream_params], + mtmd_audio_stream_p_ctypes, +) +def mtmd_audio_stream_init( + ctx: mtmd_context_p, + params: mtmd_audio_stream_params, + /, +) -> Optional[mtmd_audio_stream_p]: + """Initialize realtime audio processing, or return None on failure.""" + ... + + +# MTMD_API void mtmd_audio_stream_free(mtmd_audio_stream * stream); +@ctypes_function("mtmd_audio_stream_free", [mtmd_audio_stream_p_ctypes], None) +def mtmd_audio_stream_free(stream: mtmd_audio_stream_p, /): + """Free a realtime audio stream.""" + ... + + +# MTMD_API int32_t mtmd_audio_stream_push( +# mtmd_audio_stream * stream, const float * samples, size_t n_samples); +@ctypes_function( + "mtmd_audio_stream_push", + [mtmd_audio_stream_p_ctypes, POINTER(c_float), c_size_t], + c_int32, +) +def mtmd_audio_stream_push( + stream: mtmd_audio_stream_p, + samples: Optional[CtypesArray[c_float]], + n_samples: Union[c_size_t, int], + /, +) -> int: + """Push PCM F32 samples into a realtime audio stream.""" + ... + + +# MTMD_API int32_t mtmd_audio_stream_flush(mtmd_audio_stream * stream, bool end); +@ctypes_function( + "mtmd_audio_stream_flush", [mtmd_audio_stream_p_ctypes, c_bool], c_int32 +) +def mtmd_audio_stream_flush(stream: mtmd_audio_stream_p, end: bool, /) -> int: + """Flush pending audio, optionally ending the stream.""" + ... + + +# MTMD_API int32_t mtmd_audio_stream_step(mtmd_audio_stream * stream, llama_token last_token); +@ctypes_function( + "mtmd_audio_stream_step", + [mtmd_audio_stream_p_ctypes, llama_cpp.llama_token], + c_int32, +) +def mtmd_audio_stream_step( + stream: mtmd_audio_stream_p, + last_token: llama_cpp.llama_token, + /, +) -> int: + """Produce the next decoder rows, using the previously sampled token.""" + ... + + +# MTMD_API const float * mtmd_audio_stream_get_output_embd(const mtmd_audio_stream * stream); +@ctypes_function( + "mtmd_audio_stream_get_output_embd", + [mtmd_audio_stream_p_ctypes], + POINTER(c_float), +) +def mtmd_audio_stream_get_output_embd( + stream: mtmd_audio_stream_p, + /, +) -> Optional[CtypesArray[c_float]]: + """Get decoder embedding rows produced by the last stream step.""" + ... + + +# MTMD_API bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * stream); +@ctypes_function("mtmd_audio_stream_is_seq_start", [mtmd_audio_stream_p_ctypes], c_bool) +def mtmd_audio_stream_is_seq_start(stream: mtmd_audio_stream_p, /) -> bool: + """Check whether the last stream step starts a new sequence.""" + ... + + +# MTMD_API int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * stream); +@ctypes_function("mtmd_audio_stream_get_n_rows", [mtmd_audio_stream_p_ctypes], c_int64) +def mtmd_audio_stream_get_n_rows(stream: mtmd_audio_stream_p, /) -> int: + """Get the number of audio rows whose text has been sampled.""" + ... + + +# MTMD_API int64_t mtmd_audio_stream_get_n_rows_flushed(const mtmd_audio_stream * stream); +@ctypes_function( + "mtmd_audio_stream_get_n_rows_flushed", [mtmd_audio_stream_p_ctypes], c_int64 +) +def mtmd_audio_stream_get_n_rows_flushed(stream: mtmd_audio_stream_p, /) -> int: + """Get the number of audio rows produced through the last flush.""" + ... + + +# MTMD_API struct mtmd_video_config mtmd_get_video_config( +# const mtmd_context * ctx, int32_t src_width, int32_t src_height, +# int32_t src_n_frames, float src_fps); +@ctypes_function( + "mtmd_get_video_config", + [mtmd_context_p_ctypes, c_int32, c_int32, c_int32, c_float], + mtmd_video_config, +) +def mtmd_get_video_config( + ctx: mtmd_context_p, + src_width: int, + src_height: int, + src_n_frames: int, + src_fps: float, + /, +) -> mtmd_video_config: + """Get recommended video sampling and frame dimensions for the current model.""" + ... + + # MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); @ctypes_function("mtmd_get_marker", [mtmd_context_p_ctypes], c_char_p) def mtmd_get_marker(ctx: mtmd_context_p, /) -> Optional[bytes]: @@ -388,6 +950,13 @@ def mtmd_bitmap_is_audio(bitmap: mtmd_bitmap_p, /) -> bool: ... +# MTMD_API bool mtmd_bitmap_is_voice(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_is_voice", [mtmd_bitmap_p_ctypes], c_bool) +def mtmd_bitmap_is_voice(bitmap: mtmd_bitmap_p, /) -> bool: + """Check whether the bitmap contains a predefined voice representation.""" + ... + + # MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); @ctypes_function("mtmd_bitmap_get_id", [mtmd_bitmap_p_ctypes], c_char_p) def mtmd_bitmap_get_id(bitmap: mtmd_bitmap_p, /) -> Optional[bytes]: @@ -402,7 +971,22 @@ def mtmd_bitmap_set_id(bitmap: mtmd_bitmap_p, id: Optional[bytes], /): ... -# MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, +# // if true, this bitmap can be merged (temporal merge) with an adjacent mergeable bitmap by certain video input models +# MTMD_API void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable); +@ctypes_function("mtmd_bitmap_set_mergeable", [mtmd_bitmap_p_ctypes, c_bool], None) +def mtmd_bitmap_set_mergeable(bitmap: mtmd_bitmap_p, mergeable: bool, /): + """Allow temporal merging with an adjacent mergeable bitmap.""" + ... + + +# MTMD_API void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video); +@ctypes_function("mtmd_bitmap_set_is_video", [mtmd_bitmap_p_ctypes, c_bool], None) +def mtmd_bitmap_set_is_video(bitmap: mtmd_bitmap_p, is_video: bool, /): + """Mark a bitmap or lazy bitmap as part of a video input.""" + ... + + +# MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx, # const char * id, # void * user_data, # mtmd_bitmap_lazy_callback callback); @@ -448,10 +1032,10 @@ def mtmd_input_chunks_get( ) -> Optional[mtmd_input_chunk_p]: ... -# MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, +# MTMD_API int32_t mtmd_tokenize(const mtmd_context * ctx, # mtmd_input_chunks * output, # const mtmd_input_text * text, -# const mtmd_bitmap ** bitmaps, +# const mtmd_bitmap * const * bitmaps, # size_t n_bitmaps); @ctypes_function( "mtmd_tokenize", @@ -474,6 +1058,40 @@ def mtmd_tokenize( ) -> int: ... +# // same as mtmd_tokenize(), but takes an array of mtmd_input_part +# // use cases: +# // - when you don't want to use media markers (they will be tokenized as normal text) +# // - when you want to control parse_special for each text part +# // note: per-part add_special will be ignored +# // return 1 if a part has both text and bitmap set (or neither) +# MTMD_API int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx, +# mtmd_input_chunks * output, +# const mtmd_input_part * const * parts, +# size_t n_parts, +# bool add_special); +@ctypes_function( + "mtmd_tokenize_from_parts", + [ + mtmd_context_p_ctypes, + mtmd_input_chunks_p_ctypes, + POINTER(POINTER(mtmd_input_part)), + c_size_t, + c_bool, + ], + c_int32, +) +def mtmd_tokenize_from_parts( + ctx: mtmd_context_p, + output: mtmd_input_chunks_p, + parts: CtypesArray[POINTER(mtmd_input_part)], + n_parts: Union[c_size_t, int], + add_special: bool, + /, +) -> int: + """Tokenize text and bitmap parts without substituting media markers.""" + ... + + # MTMD_API size_t mtmd_input_chunk_get_n_tokens(const mtmd_input_chunk * chunk); @ctypes_function("mtmd_input_chunk_get_n_tokens", [mtmd_input_chunk_p_ctypes], c_size_t) def mtmd_input_chunk_get_n_tokens(chunk: mtmd_input_chunk_p, /) -> int: ... @@ -540,6 +1158,58 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p, /): ... +# // similar to mtmd_input_chunk_copy, but returns a placeholder chunk +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_input_chunk_get_placeholder", + [mtmd_input_chunk_p_ctypes], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_get_placeholder( + chunk: mtmd_input_chunk_p, / +) -> Optional[mtmd_input_chunk_p]: + """Like mtmd_input_chunk_copy, but returns a placeholder chunk.""" + ... + + +# // save/load an input chunk to/from a buffer (useful for KV save/load) +# // important: only chunk's metadata will be saved, the actual image/audio data will not be saved +# // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() +# // out_buf can be nullptr (to query expected_out_len) +# // returns 0 on success, non-zero on failure +# MTMD_API int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len); +@ctypes_function( + "mtmd_input_chunk_save", + [mtmd_input_chunk_p_ctypes, POINTER(c_char), c_size_t, POINTER(c_size_t)], + c_int32, +) +def mtmd_input_chunk_save( + chunk: mtmd_input_chunk_p, + out_buf: Optional[CtypesArray[c_char]], + out_len: Union[c_size_t, int], + expected_out_len: "_Pointer[c_size_t]", + /, +) -> int: + """Save an input chunk's metadata to a buffer.""" + ... + + +# // returns nullptr on failure +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len); +@ctypes_function( + "mtmd_input_chunk_load", + [c_char_p, c_size_t], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_load( + buf: bytes, + length: Union[c_size_t, int], + /, +) -> Optional[mtmd_input_chunk_p]: + """Load an input chunk placeholder from saved metadata.""" + ... + + # MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); @ctypes_function( "mtmd_image_tokens_get_n_tokens", [mtmd_image_tokens_p_ctypes], c_size_t @@ -685,6 +1355,154 @@ def mtmd_get_cap_from_file(mmproj_fname: bytes, /) -> mtmd_caps: ... +# MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +@ctypes_function( + "mtmd_gen_audio_get_info", + [mtmd_context_p_ctypes], + mtmd_gen_audio_info, +) +def mtmd_gen_audio_get_info(ctx: mtmd_context_p, /) -> mtmd_gen_audio_info: + """Get audio generation information for an MTMD context.""" + ... + + +# MTMD_API size_t mtmd_gen_audio_read_tensor( +# mtmd_context * ctx, const char * name, float * out, size_t n_max); +@ctypes_function( + "mtmd_gen_audio_read_tensor", + [mtmd_context_p_ctypes, c_char_p, POINTER(c_float), c_size_t], + c_size_t, +) +def mtmd_gen_audio_read_tensor( + ctx: mtmd_context_p, + name: bytes, + out: Optional[CtypesArray[c_float]], + n_max: Union[c_size_t, int], + /, +) -> int: + """Read a named audio-generation tensor as F32 and return its element count.""" + ... + + +# MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx); +@ctypes_function( + "mtmd_gen_audio_voice_name", + [mtmd_context_p_ctypes, c_int32], + c_char_p, +) +def mtmd_gen_audio_voice_name( + ctx: mtmd_context_p, + idx: int, + /, +) -> Optional[bytes]: + """Get the name of a predefined voice by index.""" + ... + + +# MTMD_API mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name); +@ctypes_function( + "mtmd_gen_audio_get_voice", + [mtmd_context_p_ctypes, c_char_p], + mtmd_bitmap_p_ctypes, +) +def mtmd_gen_audio_get_voice( + ctx: mtmd_context_p, + name: bytes, + /, +) -> Optional[mtmd_bitmap_p]: + """Create an owned voice bitmap for a predefined voice name.""" + ... + + +# MTMD_API struct mtmd_gen_state_params mtmd_gen_state_params_default(void); +@ctypes_function("mtmd_gen_state_params_default", [], mtmd_gen_state_params) +def mtmd_gen_state_params_default() -> mtmd_gen_state_params: + """Get default audio generation state parameters.""" + ... + + +# MTMD_API mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, struct mtmd_gen_state_params params); +@ctypes_function( + "mtmd_gen_state_init", + [mtmd_context_p_ctypes, mtmd_gen_state_params], + mtmd_gen_state_p_ctypes, +) +def mtmd_gen_state_init( + ctx: mtmd_context_p, + params: mtmd_gen_state_params, + /, +) -> Optional[mtmd_gen_state_p]: + """Initialize audio generation state, or return None if unsupported.""" + ... + + +# MTMD_API void mtmd_gen_state_free(mtmd_gen_state * state); +@ctypes_function("mtmd_gen_state_free", [mtmd_gen_state_p_ctypes], None) +def mtmd_gen_state_free(state: mtmd_gen_state_p, /): + """Free audio generation state.""" + ... + + +# MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state); +@ctypes_function("mtmd_gen_state_reset", [mtmd_gen_state_p_ctypes], None) +def mtmd_gen_state_reset(state: mtmd_gen_state_p, /): + """Reset audio generation state while preserving its RNG stream.""" + ... + + +# MTMD_API int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state); +@ctypes_function( + "mtmd_gen_state_set_ref", + [mtmd_context_p_ctypes, mtmd_gen_state_p_ctypes], + c_int32, +) +def mtmd_gen_state_set_ref( + ctx: mtmd_context_p, + state: mtmd_gen_state_p, + /, +) -> int: + """Copy the most recently encoded speaker reference into generation state.""" + ... + + +# // defaults tuned for the loaded pipeline, callers override only what they care about +# MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); +@ctypes_function( + "mtmd_gen_inp_default", + [mtmd_context_p_ctypes], + mtmd_gen_inp, +) +def mtmd_gen_inp_default(ctx: mtmd_context_p, /) -> mtmd_gen_inp: + """Get default audio generation input parameters for an MTMD context.""" + ... + + +# // run one step and update the state in place +# MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, +# mtmd_gen_state * state, +# const struct mtmd_gen_inp * inp, +# struct mtmd_gen_out * out); +@ctypes_function( + "mtmd_gen_audio_process", + [ + mtmd_context_p_ctypes, + mtmd_gen_state_p_ctypes, + POINTER(mtmd_gen_inp), + POINTER(mtmd_gen_out), + ], + c_int32, +) +def mtmd_gen_audio_process( + ctx: mtmd_context_p, + state: mtmd_gen_state_p, + inp: "_Pointer[mtmd_gen_inp]", + out: "_Pointer[mtmd_gen_out]", + /, +) -> int: + """Process one audio generation step.""" + ... + + # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function("mtmd_test_create_input_chunks", [], mtmd_input_chunks_p_ctypes) def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: @@ -697,7 +1515,7 @@ def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: ################################################ -# MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); +# MTMD_API bool mtmd_helper_support_video(const mtmd_context * ctx); @ctypes_function( "mtmd_helper_support_video", [mtmd_context_p_ctypes], @@ -708,30 +1526,57 @@ def mtmd_helper_support_video(ctx: mtmd_context_p, /) -> bool: ... -# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); +# MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void); +@ctypes_function("mtmd_helper_init_opt_default", [], mtmd_helper_init_opt) +def mtmd_helper_init_opt_default() -> mtmd_helper_init_opt: + """Get default options for initializing media from files or buffers.""" + ... + + +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file( +# const mtmd_context * ctx, +# const char * fname, +# bool placeholder, +# struct mtmd_helper_init_opt opt); @ctypes_function( "mtmd_helper_bitmap_init_from_file", - [mtmd_context_p_ctypes, c_char_p, c_bool], + [mtmd_context_p_ctypes, c_char_p, c_bool, mtmd_helper_init_opt], mtmd_helper_bitmap_wrapper, ) def mtmd_helper_bitmap_init_from_file_wrapper( - ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / + ctx: mtmd_context_p, + fname: bytes, + placeholder: Union[c_bool, bool], + opt: mtmd_helper_init_opt, + /, ) -> mtmd_helper_bitmap_wrapper: """Initialize an MTMD bitmap wrapper from a file.""" ... def mtmd_helper_bitmap_init_from_file( - ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / + ctx: mtmd_context_p, + fname: bytes, + placeholder: Union[c_bool, bool], + opt: Optional[mtmd_helper_init_opt] = None, + /, ) -> Optional[mtmd_bitmap_p]: """Initialize an MTMD bitmap from a file.""" - return mtmd_helper_bitmap_init_from_file_wrapper(ctx, fname, placeholder).bitmap + if opt is None: + opt = mtmd_helper_init_opt_default() + return mtmd_helper_bitmap_init_from_file_wrapper( + ctx, fname, placeholder, opt + ).bitmap -# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( +# const mtmd_context * ctx, +# const unsigned char * buf, size_t len, +# bool placeholder, +# struct mtmd_helper_init_opt opt); @ctypes_function( "mtmd_helper_bitmap_init_from_buf", - [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool], + [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool, mtmd_helper_init_opt], mtmd_helper_bitmap_wrapper, ) def mtmd_helper_bitmap_init_from_buf_wrapper( @@ -739,6 +1584,7 @@ def mtmd_helper_bitmap_init_from_buf_wrapper( buf: CtypesArray[c_uint8], length: Union[c_size_t, int], placeholder: Union[c_bool, bool], + opt: mtmd_helper_init_opt, /, ) -> mtmd_helper_bitmap_wrapper: ... @@ -748,11 +1594,14 @@ def mtmd_helper_bitmap_init_from_buf( buf: CtypesArray[c_uint8], length: Union[c_size_t, int], placeholder: Union[c_bool, bool], + opt: Optional[mtmd_helper_init_opt] = None, /, ) -> Optional[mtmd_bitmap_p]: """Initialize an MTMD bitmap from a buffer.""" + if opt is None: + opt = mtmd_helper_init_opt_default() return mtmd_helper_bitmap_init_from_buf_wrapper( - ctx, buf, length, placeholder + ctx, buf, length, placeholder, opt ).bitmap @@ -902,6 +1751,49 @@ def mtmd_helper_decode_image_chunk( ... +# streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate() +# it keeps state between calls, use one per stream +mtmd_helper_resampler_p = NewType("mtmd_helper_resampler_p", int) +mtmd_helper_resampler_p_ctypes = c_void_p + + +# returns NULL on failure +# MTMD_API mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out); +@ctypes_function( + "mtmd_helper_resampler_init", [c_int32, c_int32], mtmd_helper_resampler_p_ctypes +) +def mtmd_helper_resampler_init( + rate_in: int, rate_out: int, / +) -> Optional[mtmd_helper_resampler_p]: ... + + +# MTMD_API void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx); +@ctypes_function("mtmd_helper_resampler_free", [mtmd_helper_resampler_p_ctypes], None) +def mtmd_helper_resampler_free(ctx: mtmd_helper_resampler_p, /) -> None: ... + + +# *out is valid until the next call +# returns the number of output samples (can be 0), < 0 on error +# MTMD_API int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out); +@ctypes_function( + "mtmd_helper_resampler_process", + [ + mtmd_helper_resampler_p_ctypes, + POINTER(c_float), + c_size_t, + POINTER(POINTER(c_float)), + ], + c_int64, +) +def mtmd_helper_resampler_process( + ctx: mtmd_helper_resampler_p, + samples: "_Pointer[c_float]", + n_in: int, + out: "_Pointer[_Pointer[c_float]]", + /, +) -> int: ... + + # MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); @ctypes_function( "mtmd_helper_video_init_params_default", [], mtmd_helper_video_init_params @@ -912,7 +1804,7 @@ def mtmd_helper_video_init_params_default() -> mtmd_helper_video_init_params: # MTMD_API mtmd_helper_video * mtmd_helper_video_init( -# struct mtmd_context * mctx, +# const struct mtmd_context * mctx, # const char * path, # struct mtmd_helper_video_init_params params); @ctypes_function( @@ -931,7 +1823,7 @@ def mtmd_helper_video_init( # MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( -# struct mtmd_context * mctx, +# const struct mtmd_context * mctx, # const unsigned char * buf, size_t len, # struct mtmd_helper_video_init_params params); @ctypes_function( @@ -990,6 +1882,244 @@ def mtmd_helper_video_read_next( ... +# // return true if model can be used for chat +# MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_model_can_chat", + [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], + c_bool, +) +def mtmd_helper_model_can_chat( + lctx: llama_cpp.llama_context_p, + mctx: mtmd_context_p, + /, +) -> bool: + """Return whether the model can be used for chat.""" + ... + + +# MTMD_API struct mtmd_helper_cap mtmd_helper_get_capabilities( +# const struct llama_model * model, const struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_get_capabilities", + [llama_cpp.llama_model_p_ctypes, mtmd_context_p_ctypes], + mtmd_helper_cap, +) +def mtmd_helper_get_capabilities( + model: llama_cpp.llama_model_p, + mctx: Optional[mtmd_context_p], + /, +) -> mtmd_helper_cap: + """Get model and projector capabilities.""" + ... + + +# MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void); +@ctypes_function( + "mtmd_helper_gen_audio_inp_default", + [], + mtmd_helper_gen_audio_inp, +) +def mtmd_helper_gen_audio_inp_default() -> mtmd_helper_gen_audio_inp: + """Get default audio generation helper input parameters.""" + ... + + +# MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( +# struct llama_context * lctx, +# struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_gen_audio_init", + [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], + mtmd_helper_gen_audio_p_ctypes, +) +def mtmd_helper_gen_audio_init( + lctx: llama_cpp.llama_context_p, + mctx: mtmd_context_p, + /, +) -> Optional[mtmd_helper_gen_audio_p]: + """Initialize an audio generation helper context.""" + ... + + +# MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_free", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_free(ctx: mtmd_helper_gen_audio_p, /): ... + + +# MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_reset", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_reset(ctx: mtmd_helper_gen_audio_p, /): ... + + +# MTMD_API int32_t mtmd_helper_gen_audio_set_input( +# mtmd_helper_gen_audio * ctx, +# const struct mtmd_helper_gen_audio_inp * inp); +@ctypes_function( + "mtmd_helper_gen_audio_set_input", + [mtmd_helper_gen_audio_p_ctypes, POINTER(mtmd_helper_gen_audio_inp)], + c_int32, +) +def mtmd_helper_gen_audio_set_input( + ctx: mtmd_helper_gen_audio_p, + inp: "_Pointer[mtmd_helper_gen_audio_inp]", + /, +) -> int: + """Set the audio generation helper input.""" + ... + + +# // processes at most n_batch prompt tokens per call +# // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error +# MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( +# mtmd_helper_gen_audio * ctx, +# int32_t n_batch); +@ctypes_function( + "mtmd_helper_gen_audio_step_prompt", + [mtmd_helper_gen_audio_p_ctypes, c_int32], + c_int32, +) +def mtmd_helper_gen_audio_step_prompt( + ctx: mtmd_helper_gen_audio_p, + n_batch: int, + /, +) -> int: + """Process up to n_batch prompt tokens.""" + ... + + +# // generates one frame; must only be called after step_prompt() has returned 0 +# // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token +# // out_stop (optional) is set on end-of-speech, the caller must then stop the loop +# // h_state_out is valid until next step_gen() or reset() call, None if no frame is generated +# MTMD_API int32_t mtmd_helper_gen_audio_step_gen( +# mtmd_helper_gen_audio * ctx, +# llama_token sampled, +# const float * h_state_in, +# const float ** h_state_out, +# bool * out_stop); +@ctypes_function( + "mtmd_helper_gen_audio_step_gen", + [ + mtmd_helper_gen_audio_p_ctypes, + llama_cpp.llama_token, + POINTER(c_float), + POINTER(POINTER(c_float)), + POINTER(c_bool), + ], + c_int32, +) +def mtmd_helper_gen_audio_step_gen( + ctx: mtmd_helper_gen_audio_p, + sampled: llama_cpp.llama_token, + h_state_in: Optional["_Pointer[c_float]"], + h_state_out: "_Pointer[_Pointer[c_float]]", + out_stop: Optional["_Pointer[c_bool]"], + /, +) -> int: + """Generate one audio frame.""" + ... + + +# MTMD_API bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_next_is_text", + [mtmd_helper_gen_audio_p_ctypes], + c_bool, +) +def mtmd_helper_gen_audio_next_is_text(ctx: mtmd_helper_gen_audio_p, /) -> bool: + """Check whether the next audio generation step expects a sampled text token.""" + ... + + +# MTMD_API const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias( +# mtmd_helper_gen_audio * ctx, size_t * n_bias); +@ctypes_function( + "mtmd_helper_gen_audio_get_logit_bias", + [mtmd_helper_gen_audio_p_ctypes, POINTER(c_size_t)], + llama_cpp.llama_logit_bias_p, +) +def mtmd_helper_gen_audio_get_logit_bias( + ctx: mtmd_helper_gen_audio_p, + n_bias: "_Pointer[c_size_t]", + /, +) -> Optional[CtypesArray[llama_cpp.llama_logit_bias]]: + """Get the logit bias for tokens sampled by the audio generation helper.""" + ... + + +# MTMD_API int32_t mtmd_helper_audio_stream_get_logit_bias( +# const mtmd_context * mctx, const struct llama_model * model, +# llama_logit_bias * out, int32_t n_max); +@ctypes_function( + "mtmd_helper_audio_stream_get_logit_bias", + [ + mtmd_context_p_ctypes, + llama_cpp.llama_model_p_ctypes, + POINTER(llama_cpp.llama_logit_bias), + c_int32, + ], + c_int32, +) +def mtmd_helper_audio_stream_get_logit_bias( + mctx: mtmd_context_p, + model: llama_cpp.llama_model_p, + out: Optional[CtypesArray[llama_cpp.llama_logit_bias]], + n_max: int, + /, +) -> int: + """Get token biases for text sampled while transcribing an audio stream.""" + ... + + +# // out_data valid until next get_output() or reset() call +# // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +# MTMD_API int32_t mtmd_helper_gen_audio_get_output( +# mtmd_helper_gen_audio * ctx, +# int32_t * out_sample_rate, +# const char ** out_data, +# size_t * out_data_len, +# int64_t * out_n_samples); +@ctypes_function( + "mtmd_helper_gen_audio_get_output", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(c_int32), + POINTER(POINTER(c_char)), + POINTER(c_size_t), + POINTER(c_int64), + ], + c_int32, +) +def mtmd_helper_gen_audio_get_output( + ctx: mtmd_helper_gen_audio_p, + out_sample_rate: "_Pointer[c_int32]", + out_data: "_Pointer[_Pointer[c_char]]", + out_data_len: "_Pointer[c_size_t]", + out_n_samples: Optional["_Pointer[c_int64]"], + /, +) -> int: + """Get accumulated PCM or WAV audio output.""" + ... + + +# MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_flush", [mtmd_helper_gen_audio_p_ctypes], c_int32 +) +def mtmd_helper_gen_audio_flush(ctx: mtmd_helper_gen_audio_p, /) -> int: + """Flush buffered audio codes through the decoder.""" + ... + + # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( "mtmd_log_set", diff --git a/llama_cpp/server/cli.py b/llama_cpp/server/cli.py index 171b8db30c..3a5cde9cd2 100644 --- a/llama_cpp/server/cli.py +++ b/llama_cpp/server/cli.py @@ -84,7 +84,7 @@ def add_args_from_model(parser: argparse.ArgumentParser, model: Type[BaseModel]) for name, field in model.model_fields.items(): description = field.description - if field.default and description and not field.is_required(): + if description and not field.is_required() and field.default is not None: description += f" (default: {field.default})" base_type = ( _get_base_type(field.annotation) if field.annotation is not None else str diff --git a/tests/test_llama.py b/tests/test_llama.py index 336d6a6122..8c1512d496 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -1,4 +1,5 @@ import ctypes +import itertools import multiprocessing import numpy as np @@ -64,6 +65,14 @@ def llama_cpp_model_path(): return model_path +@pytest.fixture +def llama_cpp_transformer_model_path(): + repo_id = "ggml-org/models" + filename = "tinyllamas/stories15M-q4_0.gguf" + model_path = hf_hub_download(repo_id, filename) + return model_path + + @pytest.fixture def llama_cpp_embedding_model_path(): repo_id = "CompendiumLabs/bge-small-en-v1.5-gguf" @@ -94,8 +103,14 @@ def test_real_model(llama_cpp_model_path): assert os.path.exists(llama_cpp_model_path) params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_mlock = llama_cpp.llama_supports_mlock() + if llama_cpp.llama_supports_mmap() and llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif llama_cpp.llama_supports_mmap(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE params.check_tensors = False model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params) @@ -146,6 +161,9 @@ def test_real_model(llama_cpp_model_path): assert len(output) == 4 assert output_text + model.close() + assert context.ctx is None + def test_real_llama(llama_cpp_model_path): model = llama_cpp.Llama( @@ -339,6 +357,285 @@ def test_hybrid_model_prompt_cache_reset(llama_cpp_hybrid_model_path): ) +def _create_test_model(model_path): + return llama_cpp.Llama( + model_path, + n_ctx=64, + n_batch=64, + n_ubatch=64, + n_threads=multiprocessing.cpu_count(), + n_threads_batch=multiprocessing.cpu_count(), + logits_all=False, + verbose=False, + ) + + +def _generate_test_tokens(model, tokens, max_tokens=3): + return list( + itertools.islice( + model.generate( + tokens, + temp=0.0, + ), + max_tokens, + ) + ) + + +MODEL_CACHE_CASES = ( + ("llama_cpp_transformer_model_path", False, False), + ("llama_cpp_recurrent_model_path", True, False), + ("llama_cpp_hybrid_model_path", False, True), +) + +RESTORED_CACHE_CASES = MODEL_CACHE_CASES + + +def _eval_alternate_same_length_prompt(model, tokens, expected_next_token): + replacement_tokens = ( + model.token_eos(), + model.token_nl(), + 0, + 1, + 2, + model.n_vocab() - 1, + ) + + for replacement_token in replacement_tokens: + alternate_tokens = list(tokens) + alternate_tokens[-1] = replacement_token + if alternate_tokens == tokens: + continue + + model.reset() + model.eval(alternate_tokens) + if model.sample(temp=0.0, idx=len(tokens) - 1) != expected_next_token: + return + + raise AssertionError("failed to find an alternate same-length prompt") + + +def _assert_exact_cached_prompt_reuse_matches_fresh( + model_path, + *, + is_recurrent: bool, + is_hybrid: bool, +): + prompt = "The quick brown fox" + fresh = _create_test_model(model_path) + tokens = fresh.tokenize(prompt.encode(), add_bos=True, special=True) + + assert fresh._is_recurrent is is_recurrent + assert fresh._is_hybrid is is_hybrid + + expected_tokens = _generate_test_tokens(fresh, tokens) + + cached = _create_test_model(model_path) + assert cached._is_recurrent is is_recurrent + assert cached._is_hybrid is is_hybrid + + cached.eval(tokens) + assert cached.n_tokens == len(tokens) + assert cached.input_ids[: cached.n_tokens].tolist() == tokens + assert cached.sample(temp=0.0, idx=len(tokens) - 1) == expected_tokens[0] + + reset_calls = 0 + original_reset = cached.reset + + def reset_tracker(): + nonlocal reset_calls + reset_calls += 1 + original_reset() + + cached.reset = reset_tracker + + cached_tokens = _generate_test_tokens(cached, tokens) + assert reset_calls == 0 + assert cached_tokens == expected_tokens + assert cached.n_tokens == len(tokens) + len(cached_tokens) - 1 + + +def _assert_loaded_exact_cached_prompt_reuse_matches_fresh( + model_path, + *, + is_recurrent: bool, + is_hybrid: bool, +): + prompt = "The quick brown fox" + fresh = _create_test_model(model_path) + tokens = fresh.tokenize(prompt.encode(), add_bos=True, special=True) + expected_tokens = _generate_test_tokens(fresh, tokens) + + source = _create_test_model(model_path) + assert source._is_recurrent is is_recurrent + assert source._is_hybrid is is_hybrid + + source.eval(tokens) + state = source.save_state() + + loaded = _create_test_model(model_path) + assert loaded._is_recurrent is is_recurrent + assert loaded._is_hybrid is is_hybrid + + _eval_alternate_same_length_prompt( + loaded, + tokens, + expected_tokens[0], + ) + loaded.load_state(state) + + assert loaded.n_tokens == len(tokens) + assert loaded.input_ids[: loaded.n_tokens].tolist() == tokens + + loaded_tokens = _generate_test_tokens(loaded, tokens) + assert loaded_tokens == expected_tokens + assert loaded.n_tokens == len(tokens) + len(loaded_tokens) - 1 + + +def _assert_ram_cache_exact_prompt_hit_matches_fresh( + model_path, + *, + is_recurrent: bool, + is_hybrid: bool, +): + prompt = "The quick brown fox" + fresh = _create_test_model(model_path) + tokens = fresh.tokenize(prompt.encode(), add_bos=True, special=True) + expected = fresh.create_completion( + tokens, + max_tokens=1, + temperature=0.0, + seed=1337, + ) + + cache = llama_cpp.LlamaRAMCache() + writer = _create_test_model(model_path) + writer.set_cache(cache) + writer.create_completion( + tokens, + max_tokens=1, + temperature=0.0, + seed=1337, + ) + + cached = _create_test_model(model_path) + assert cached._is_recurrent is is_recurrent + assert cached._is_hybrid is is_hybrid + cached.set_cache(cache) + + load_state_calls = 0 + original_load_state = cached.load_state + + def load_state_tracker(state): + nonlocal load_state_calls + load_state_calls += 1 + original_load_state(state) + + cached.load_state = load_state_tracker + + actual = cached.create_completion( + tokens, + max_tokens=1, + temperature=0.0, + seed=1337, + ) + + assert load_state_calls == 1 + assert actual["choices"][0]["text"] == expected["choices"][0]["text"] + assert ( + actual["usage"]["completion_tokens"] == expected["usage"]["completion_tokens"] + ) + + +def _assert_shorter_prompt_prefix_reuse_matches_fresh( + model_path, + *, + is_recurrent: bool, + is_hybrid: bool, +): + prompt = "The quick brown fox" + history = " jumps over the lazy dog" + fresh = _create_test_model(model_path) + tokens = fresh.tokenize(prompt.encode(), add_bos=True, special=True) + history_tokens = fresh.tokenize(history.encode(), add_bos=False, special=True) + expected_tokens = _generate_test_tokens(fresh, tokens) + + cached = _create_test_model(model_path) + assert cached._is_recurrent is is_recurrent + assert cached._is_hybrid is is_hybrid + + cached.eval(tokens + history_tokens) + assert cached.n_tokens > len(tokens) + assert cached.input_ids[: len(tokens)].tolist() == tokens + + cached_tokens = _generate_test_tokens(cached, tokens) + assert cached_tokens == expected_tokens + + +@pytest.mark.parametrize( + ("model_path_fixture", "is_recurrent", "is_hybrid"), MODEL_CACHE_CASES +) +def test_exact_cached_prompt_reuse_matches_fresh( + request, + model_path_fixture, + is_recurrent, + is_hybrid, +): + _assert_exact_cached_prompt_reuse_matches_fresh( + request.getfixturevalue(model_path_fixture), + is_recurrent=is_recurrent, + is_hybrid=is_hybrid, + ) + + +@pytest.mark.parametrize( + ("model_path_fixture", "is_recurrent", "is_hybrid"), RESTORED_CACHE_CASES +) +def test_loaded_exact_cached_prompt_reuse_matches_fresh( + request, + model_path_fixture, + is_recurrent, + is_hybrid, +): + _assert_loaded_exact_cached_prompt_reuse_matches_fresh( + request.getfixturevalue(model_path_fixture), + is_recurrent=is_recurrent, + is_hybrid=is_hybrid, + ) + + +@pytest.mark.parametrize( + ("model_path_fixture", "is_recurrent", "is_hybrid"), RESTORED_CACHE_CASES +) +def test_ram_cache_exact_prompt_hit_matches_fresh( + request, + model_path_fixture, + is_recurrent, + is_hybrid, +): + _assert_ram_cache_exact_prompt_hit_matches_fresh( + request.getfixturevalue(model_path_fixture), + is_recurrent=is_recurrent, + is_hybrid=is_hybrid, + ) + + +@pytest.mark.parametrize( + ("model_path_fixture", "is_recurrent", "is_hybrid"), MODEL_CACHE_CASES +) +def test_shorter_prompt_prefix_reuse_matches_fresh( + request, + model_path_fixture, + is_recurrent, + is_hybrid, +): + _assert_shorter_prompt_prefix_reuse_matches_fresh( + request.getfixturevalue(model_path_fixture), + is_recurrent=is_recurrent, + is_hybrid=is_hybrid, + ) + + def test_real_llama_embeddings(llama_cpp_embedding_model_path): model = llama_cpp.Llama( llama_cpp_embedding_model_path, @@ -348,7 +645,7 @@ def test_real_llama_embeddings(llama_cpp_embedding_model_path): n_threads=multiprocessing.cpu_count(), n_threads_batch=multiprocessing.cpu_count(), logits_all=False, - flash_attn=True, + flash_attn=False, embedding=True, ) embedding = model.embed("Hello World") diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3a74b2990..a956a852c9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3a74b299085cd00013804f7fca2e03441b2da20 +Subproject commit a956a852c9c8bc1fb07206364838ddef299178c0