fix: fix clip tokenizer (leejet#383)

stduhpf · web-flow · commit 14206fd48832 · 2024-09-02T22:31:46.000+08:00
diff --git a/clip.hpp b/clip.hpp
@@ -388,7 +388,7 @@ class CLIPTokenizer {
                 std::string token_str = token.str();
                 std::u32string utf32_token;
                 for (int i = 0; i < token_str.length(); i++) {
-                    char b = token_str[i];
+                    unsigned char b = token_str[i];
                     utf32_token += byte_encoder[b];
                 }
                 auto bpe_strs = bpe(utf32_token);

Original file line number	Diff line number	Diff line change
`@@ -388,7 +388,7 @@ class CLIPTokenizer {`
`388`	`388`	`std::string token_str = token.str();`
`389`	`389`	`std::u32string utf32_token;`
`390`	`390`	`for (int i = 0; i < token_str.length(); i++) {`
`391`		`- char b = token_str[i];`
	`391`	`+ unsigned char b = token_str[i];`
`392`	`392`	`utf32_token += byte_encoder[b];`
`393`	`393`	`}`
`394`	`394`	`auto bpe_strs = bpe(utf32_token);`