Source code for actrone_memory.tokens

"""Token counting for the context budget, kept in step with the TypeScript library.

The default is a dependency-free heuristic, about 4 characters per token (the well-known rough GPT
ratio), identical to the TypeScript ``heuristicTokenCounter``, so both libraries allocate a budget
the same way and neither touches the network. Exact ``cl100k_base`` counts are opt-in through
``MemoryConfig(token_counter="tiktoken")`` and the ``[tiktoken]`` extra.
"""

from __future__ import annotations

import math
from collections.abc import Callable
from typing import Literal

from actrone_memory.exceptions import ConfigurationError

TokenCounter = Callable[[str], int]
"""Returns how many tokens a text consumes."""


[docs] def heuristic_token_counter(text: str) -> int: """Estimate tokens as ``ceil(len(text) / 4)``. Never returns less than 1 for non-empty text, so a turn or memory always consumes some budget. """ if not text: return 0 return max(1, math.ceil(len(text) / 4))
[docs] def tiktoken_token_counter() -> TokenCounter: """Return an exact ``cl100k_base`` counter. The encoding is loaded here, once, rather than on every call. tiktoken downloads the encoding file on first use unless ``TIKTOKEN_CACHE_DIR`` already holds it, so this is the one opt-in path in the library that may reach the network. Raises: ConfigurationError: If tiktoken is not installed or the encoding cannot be loaded. """ try: import tiktoken except ImportError as exc: raise ConfigurationError( 'token_counter="tiktoken" needs the tiktoken extra: ' 'pip install "actrone-memory[tiktoken]"', details={"token_counter": "tiktoken"}, ) from exc try: encoding = tiktoken.get_encoding("cl100k_base") except Exception as exc: # network or cache failure while fetching the encoding raise ConfigurationError( "Could not load the cl100k_base encoding for tiktoken. It is downloaded once on first " "use; check network access, or set TIKTOKEN_CACHE_DIR to a directory that holds it.", details={"token_counter": "tiktoken", "error": str(exc)}, ) from exc def count(text: str) -> int: return len(encoding.encode(text)) if text else 0 return count
[docs] def build_token_counter(kind: Literal["heuristic", "tiktoken"]) -> TokenCounter: """Return the counter ``MemoryConfig.token_counter`` names. Raises: ConfigurationError: If ``kind`` is "tiktoken" and tiktoken cannot be loaded. """ if kind == "tiktoken": return tiktoken_token_counter() return heuristic_token_counter