From 07349731f0f2ae31e46cac93a5a90cc15751d705 Mon Sep 17 00:00:00 2001 From: Travis Johnson Date: Fri, 10 May 2024 14:13:06 -0600 Subject: [PATCH] fix: check for tokenizer eos_token in ModelInfo response (#93) Use consistent logic to determine the eos_token_id in ModelInfo as it is in other functions by falling back to the tokenizer's `eos_token_id` attribute if the model config does not have an `eos_token_id`. Fixes the behavior for a model that does not have an eos_token_id in the model config Resolves https://github.com/IBM/text-generation-inference/issues/91 Signed-off-by: Travis Johnson --- server/text_generation_server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 3a2e5c2c..f70cb5df 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -79,7 +79,7 @@ async def ModelInfo(self, request: generate_pb2.ModelInfoRequest, context) -> ge return generate_pb2.ModelInfoResponse( model_type=ModelInfoResponse.ModelType.SEQ2SEQ_LM if isinstance(self.model, Seq2SeqLM) else ModelInfoResponse.ModelType.CAUSAL_LM, - eos_token=self.model.config.eos_token_id, + eos_token=getattr(self.model.tokenizer, 'model_eos_token_id', self.model.tokenizer.eos_token_id), batch_padding=not isinstance(self.model, FlashCausalLM), memory_scaling_model=self.memory_scaling_model, )