Skip to content

Commit bcf29ef

Browse files
committed
convert_hf_to_gguf: rwkv: Avoid using eval
Signed-off-by: Molly Sophia <mollysophia379@gmail.com>
1 parent 9bf958f commit bcf29ef

File tree

1 file changed

+8
-5
lines changed

1 file changed

+8
-5
lines changed

convert_hf_to_gguf.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
from __future__ import annotations
55

6+
import ast
67
import logging
78
import argparse
89
import contextlib
@@ -2716,12 +2717,14 @@ def set_vocab(self):
27162717
with open(self.dir_model / "rwkv_vocab_v20230424.txt", "r", encoding="utf-8") as f:
27172718
lines = f.readlines()
27182719
for line in lines:
2719-
x = eval(line[line.index(' '):line.rindex(' ')])
2720-
x = x.encode("utf-8") if isinstance(x, str) else x
2721-
assert isinstance(x, bytes)
2722-
assert len(x) == int(line[line.rindex(' '):])
2720+
parts = line.split(' ')
2721+
assert len(parts) >= 3
2722+
_, token, token_len = int(parts[0]), ast.literal_eval(' '.join(parts[1:-1])), int(parts[-1])
2723+
token = token.encode("utf-8") if isinstance(token, str) else token
2724+
assert isinstance(token, bytes)
2725+
assert len(token) == token_len
27232726
token_text: str = ""
2724-
for b in x:
2727+
for b in token:
27252728
token_text += f"\\x{b:02x}"
27262729
tokens.append(token_text.encode("utf-8"))
27272730
toktypes.append(gguf.TokenType.NORMAL)

0 commit comments

Comments
 (0)