Inheritance diagram for GncTokenizer:

Public Member Functions
	GncTokenizer (const GncTokenizer &)=default

GncTokenizer &	operator= (const GncTokenizer &)=default

	GncTokenizer (GncTokenizer &&)=default

GncTokenizer &	operator= (GncTokenizer &&)=default

virtual void	load_file (const std::string &path)

const std::string &	current_file ()

void	encoding (const std::string &encoding)

const std::string &	encoding ()

virtual int	tokenize ()=0

const std::vector< StrVec > &	get_tokens ()

Protected Attributes
std::string	m_utf8_contents

std::vector< StrVec >	m_tokenized_contents

Detailed Description

Definition at line 57 of file gnc-tokenizer.hpp.

Member Function Documentation

◆ current_file()

const std::string & GncTokenizer::current_file ( )

Definition at line 99 of file gnc-tokenizer.cpp.

{
    return m_imp_file_str;
}

◆ encoding() [1/2]

const std::string & GncTokenizer::encoding ( )

Definition at line 117 of file gnc-tokenizer.cpp.

{
    return m_enc_str;
}

◆ encoding() [2/2]

void GncTokenizer::encoding ( const std::string & encoding )

Definition at line 105 of file gnc-tokenizer.cpp.

{
    m_enc_str = encoding;
    m_utf8_contents = boost::locale::conv::to_utf<char>(m_raw_contents, m_enc_str);
 
    // While we are converting here, let's also normalize line-endings to "\n"
    // That's what STL expects by default
    boost::replace_all (m_utf8_contents, "\r\n", "\n");
    boost::replace_all (m_utf8_contents, "\r", "\n");
}

◆ get_tokens()

const std::vector< StrVec > & GncTokenizer::get_tokens ( )

Definition at line 124 of file gnc-tokenizer.cpp.

{
    return m_tokenized_contents;
}

◆ load_file()

void GncTokenizer::load_file ( const std::string & path )

virtual

Definition at line 65 of file gnc-tokenizer.cpp.

{
    if (path.empty())
        return;
 
    m_imp_file_str = path;
    char *raw_contents;
    size_t raw_length;
    GError *error = nullptr;
 
    if (!g_file_get_contents(path.c_str(), &raw_contents, &raw_length, &error))
    {
        std::string msg {error->message};
        g_error_free (error);
        throw std::ifstream::failure {msg};
    }
 
    m_raw_contents = raw_contents;
    g_free(raw_contents);
    
    // Guess encoding, user can override if needed later on.
    const char *guessed_enc = NULL;
    guessed_enc = go_guess_encoding (m_raw_contents.c_str(),
                                     m_raw_contents.length(),
                                     m_enc_str.empty() ? "UTF-8" : m_enc_str.c_str(),
                                     NULL);
    if (guessed_enc)
        this->encoding(guessed_enc);
    else
        m_enc_str.clear();
 
}

Field Documentation

◆ m_tokenized_contents

std::vector<StrVec> GncTokenizer::m_tokenized_contents

protected

Definition at line 77 of file gnc-tokenizer.hpp.

◆ m_utf8_contents

std::string GncTokenizer::m_utf8_contents