Removed uneeded vocab.tokenizer_clean_spaces assignment

m18coppola · m18coppola · commit ea215c7db6d3 · 2024-07-18T22:05:33.000-04:00
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -5520,7 +5520,6 @@ static void llm_load_vocab(
             } else if (
                 tokenizer_pre == "tekken") {
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_TEKKEN;
-                vocab.tokenizer_clean_spaces = true;
                 vocab.tokenizer_ignore_merges = true;
                 vocab.tokenizer_add_bos = true;
             } else if (
