plamo-3-translate-8b-MLX-4bit / serving_memory.py
kaitos255's picture
Initial release of PLaMo 3 Translate 8B MLX-4bit
9d87684
Raw History Blame Contribute Delete
916 Bytes
"""Materialize extracted prompt caches before sharing them with queued requests.
In mlx-lm 0.32.0, extract() leaves contiguous/slice operations lazy. A pending
request can therefore retain the entire old generation batch through its small
prefix cache, even after the LRU evicts that entry. Evaluate at the ownership
boundary so the queue holds the extracted tensors, not the extraction graph.
"""
import mlx.core as mx
from mlx_lm.models.cache import LRUPromptCache
def install():
original = LRUPromptCache.insert_cache
if getattr(original, "_plamo_materialized", False):
return
def insert_cache(self, model, tokens, prompt_cache, *, cache_type="assistant"):
mx.eval([cache.state for cache in prompt_cache])
return original(self, model, tokens, prompt_cache, cache_type=cache_type)
insert_cache._plamo_materialized = True
LRUPromptCache.insert_cache = insert_cache