Inheritance diagram for GncFwTokenizer:

Public Member Functions
	GncFwTokenizer (const GncFwTokenizer &)=default

GncFwTokenizer &	operator= (const GncFwTokenizer &)=default

	GncFwTokenizer (GncFwTokenizer &&)=default

GncFwTokenizer &	operator= (GncFwTokenizer &&)=default

void	columns (const std::vector< uint32_t > &cols=std::vector< uint32_t >())

std::vector< uint32_t >	get_columns ()

uint32_t	get_column (uint32_t num)

bool	col_can_delete (uint32_t col_num)

void	col_delete (uint32_t col_num)

bool	col_can_narrow (uint32_t col_num)

void	col_narrow (uint32_t col_num)

bool	col_can_widen (uint32_t col_num)

void	col_widen (uint32_t col_num)

bool	col_can_split (uint32_t col_num, uint32_t position)

void	col_split (uint32_t col_num, uint32_t position)

void	load_file (const std::string &path) override

int	tokenize () override

Public Member Functions inherited from GncTokenizer
	GncTokenizer (const GncTokenizer &)=default

GncTokenizer &	operator= (const GncTokenizer &)=default

	GncTokenizer (GncTokenizer &&)=default

GncTokenizer &	operator= (GncTokenizer &&)=default

const std::string &	current_file ()

void	encoding (const std::string &encoding)

const std::string &	encoding ()

const std::vector< StrVec > &	get_tokens ()

Additional Inherited Members
Protected Attributes inherited from GncTokenizer
std::string	m_utf8_contents

std::vector< StrVec >	m_tokenized_contents

Detailed Description

Definition at line 49 of file gnc-tokenizer-fw.hpp.

Member Function Documentation

◆ col_can_delete()

bool GncFwTokenizer::col_can_delete ( uint32_t col_num )

Definition at line 53 of file gnc-tokenizer-fw.cpp.

{
    auto last_col = m_col_vec.size() - 1;
    if (col_num >= last_col)
        return false;
    else
        return true;
}

◆ col_can_narrow()

bool GncFwTokenizer::col_can_narrow ( uint32_t col_num )

Definition at line 71 of file gnc-tokenizer-fw.cpp.

{
    // Can't narrow the last column, it always sticks to the end of the parseable data
    auto last_col = m_col_vec.size() - 1;
    if (col_num >= last_col)
        return false;
    else
        return true;
}

◆ col_can_split()

bool GncFwTokenizer::col_can_split	(	uint32_t	col_num,
		uint32_t	position
	)

Definition at line 117 of file gnc-tokenizer-fw.cpp.

{
    auto last_col = m_col_vec.size() - 1;
    if (col_num > last_col)
        return false;
 
    uint32_t col_end = m_col_vec[col_num];
    if (position < 1 || position >= col_end)
        return false;
    else
        return true;
}

◆ col_can_widen()

bool GncFwTokenizer::col_can_widen ( uint32_t col_num )

Definition at line 94 of file gnc-tokenizer-fw.cpp.

{
    // Can't widen the last column, it always sticks to the end of the parseable data
    auto last_col = m_col_vec.size() - 1;
    if (col_num >= last_col)
        return false;
    else
        return true;
}

◆ col_delete()

void GncFwTokenizer::col_delete ( uint32_t col_num )

Definition at line 62 of file gnc-tokenizer-fw.cpp.

{
    if (!col_can_delete (col_num))
        return;
 
    m_col_vec[col_num + 1] += m_col_vec[col_num];
    m_col_vec.erase (m_col_vec.begin() + col_num);
}

◆ col_narrow()

void GncFwTokenizer::col_narrow ( uint32_t col_num )

Definition at line 81 of file gnc-tokenizer-fw.cpp.

{
    if (!col_can_narrow (col_num))
        return;
 
    m_col_vec[col_num]--;
    m_col_vec[col_num + 1]++;
 
    // Drop column if it has become 0-width now
    if (m_col_vec[col_num] == 0)
        m_col_vec.erase (m_col_vec.begin() + col_num);
}

◆ col_split()

void GncFwTokenizer::col_split	(	uint32_t	col_num,
		uint32_t	position
	)

Definition at line 130 of file gnc-tokenizer-fw.cpp.

{
    if (col_can_split (col_num, position))
    {
        m_col_vec.insert (m_col_vec.begin() + col_num, position);
        m_col_vec[col_num + 1] -= position;
    }
}

◆ col_widen()

void GncFwTokenizer::col_widen ( uint32_t col_num )

Definition at line 104 of file gnc-tokenizer-fw.cpp.

{
    if (!col_can_widen (col_num))
        return;
 
    m_col_vec[col_num]++;
    m_col_vec[col_num + 1]--;
 
    // Drop next column if it has become 0-width now
    if (m_col_vec[col_num + 1] == 0)
        m_col_vec.erase (m_col_vec.begin() + col_num + 1);
}

◆ columns()

void GncFwTokenizer::columns ( const std::vector< uint32_t > & cols = std::vector<uint32_t>() )

Definition at line 41 of file gnc-tokenizer-fw.cpp.

{
    m_col_vec = cols;
}

◆ get_columns()

std::vector< uint32_t > GncFwTokenizer::get_columns ( )

Definition at line 47 of file gnc-tokenizer-fw.cpp.

{
    return m_col_vec;
}

◆ load_file()

void GncFwTokenizer::load_file ( const std::string & path )

overridevirtual

Reimplemented from GncTokenizer.

Definition at line 140 of file gnc-tokenizer-fw.cpp.

{
    GncTokenizer::load_file(path);
 
    std::string line;
    m_longest_line = 0;
    std::istringstream in_stream(m_utf8_contents);
    while (std::getline (in_stream, line))
    {
        if (line.size() > m_longest_line)
            m_longest_line = line.size();
 
        line.clear();
    }
 
    if (m_col_vec.empty())
        /* Set a sane default for the offsets
         * That is, assume one column with all the data */
        m_col_vec.push_back(m_longest_line);
    else
    {
        /* Adjust existing last column(s) so the total column width
         * equals the width of the longest line
         * This may mean
         * - widening the last column to widen to the longest line or
         * - deleting columns/narrowing the last one to reduce to the longest line
         */
        uint32_t total_width = 0;
        for (auto col_width : m_col_vec)
            total_width += col_width;
 
        if (m_longest_line > total_width)
            m_col_vec.back() += m_longest_line - total_width;
        else if (m_longest_line < total_width)
        {
            while (total_width - m_col_vec.back() > m_longest_line)
                col_delete (m_col_vec[m_col_vec.size() - 2]);
            m_col_vec.back() -= total_width - m_longest_line;
        }
    }
}

◆ tokenize()

int GncFwTokenizer::tokenize ( )