TinyChatEngine
Loading...
Searching...
No Matches
Encoder Class Reference

Public Member Functions

 Encoder (std::map< std::string, int > encoder, std::vector< std::pair< std::string, std::string > > bpe_merges)
 
std::unordered_map< int, std::string > bytes_to_unicode ()
 
std::set< std::pair< std::string, std::string > > get_pairs (std::vector< std::string > word)
 
std::string bpe (std::string token)
 
std::vector< int > encode (std::string text)
 
std::string decode (std::vector< int > tokens)
 

The documentation for this class was generated from the following files: