59friend GncTokenizerTest;
61 GncTokenizer() =
default;
62 GncTokenizer(
const GncTokenizer&) =
default;
63 GncTokenizer& operator=(
const GncTokenizer&) =
default;
64 GncTokenizer(GncTokenizer&&) =
default;
65 GncTokenizer& operator=(GncTokenizer&&) =
default;
66 virtual ~GncTokenizer() =
default;
68 virtual void load_file(
const std::string& path);
69 const std::string& current_file();
70 void encoding(
const std::string& encoding);
71 const std::string& encoding();
72 virtual int tokenize() = 0;
73 const std::vector<StrVec>& get_tokens();
76 std::string m_utf8_contents;
77 std::vector<StrVec> m_tokenized_contents;
80 std::string m_imp_file_str;
81 std::string m_raw_contents;
82 std::string m_enc_str;