wuyingwen/comfyui
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Fix hard crash when the spiece tokenizer path is bad.

Browse Source
This commit is contained in:
comfyanonymous 2025-04-19 15:55:43 -04:00
parent dc300a4569
commit 636d4bfb89
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
comfy/text_encoders/spiece_tokenizer.py
View File

@ -1,4 +1,5 @@
import torch import torch
import os
class SPieceTokenizer: class SPieceTokenizer:
@staticmethod @staticmethod
@ -15,6 +16,8 @@ class SPieceTokenizer:
if isinstance(tokenizer_path, bytes): if isinstance(tokenizer_path, bytes):
self.tokenizer = sentencepiece.SentencePieceProcessor(model_proto=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos) self.tokenizer = sentencepiece.SentencePieceProcessor(model_proto=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos)
else: else:
if not os.path.isfile(tokenizer_path):
raise ValueError("invalid tokenizer")
self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos) self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos)
def get_vocab(self): def get_vocab(self):