ParquetFormatter::Impl Struct Reference

Public Member Functions
void	checkAttr (const SumoXMLAttr attr)
template<class ATTR_TYPE , class BUILDER >
void	checkBuilder (const ATTR_TYPE &attr, const std::shared_ptr< arrow::DataType > &(*dataType)())
	Impl (const std::string &columnNames, const int batchSize)

Data Fields
const int	myBatchSize
	the number of rows to write per batch
std::vector< std::shared_ptr< arrow::ArrayBuilder > >	myBuilders
	the content array builders for the table
bool	myCheckColumns = false
	whether the columns should be checked for completeness
parquet::Compression::type	myCompression = parquet::Compression::UNCOMPRESSED
	the compression to use
SumoXMLAttrMask	myExpectedAttrs
	the attributes which are expected for a complete row (including null values)
std::vector< std::string >	myFullHeader
	the column names if we write the full name
bool	myHaveRootAttrs = false
	whether any root attribute have been encountered
const std::string	myHeaderFormat
	the format to use for the column names
int	myMaxDepth = 1000
	the maximum depth of the XML hierarchy
bool	myNeedsWrite = false
	whether there is still unwritten data
std::unique_ptr< parquet::arrow::FileWriter >	myParquetWriter
	the output stream writer
std::shared_ptr< arrow::Schema >	mySchema = arrow::schema({})
	the table schema
SumoXMLAttrMask	mySeenAttrs
	the attributes already seen (including null values)
std::vector< std::shared_ptr< arrow::Scalar > >	myValues
	the current attribute / column values
bool	myWroteHeader = false
	whether the schema has been constructed completely
std::vector< std::pair< const std::string, int > >	myXMLStack
	The name and number of attributes in the currently open XML elements.

Detailed Description

Definition at line 105 of file ParquetFormatter.cpp.

Constructor & Destructor Documentation

Impl()

ParquetFormatter::Impl::Impl ( const std::string &  columnNames, const int  batchSize  )

inline

Definition at line 106 of file ParquetFormatter.cpp.

Member Function Documentation

checkAttr()

void ParquetFormatter::Impl::checkAttr ( const SumoXMLAttr  attr )

inline

Definition at line 157 of file ParquetFormatter.cpp.

References myCheckColumns, myExpectedAttrs, myMaxDepth, mySeenAttrs, myXMLStack, TLF, and toString().

checkBuilder()

template<class ATTR_TYPE , class BUILDER >

void ParquetFormatter::Impl::checkBuilder ( const ATTR_TYPE &  attr, const std::shared_ptr< arrow::DataType > &(*)()  dataType  )

inline

Definition at line 167 of file ParquetFormatter.cpp.

References myBuilders, myFullHeader, myHeaderFormat, myNeedsWrite, mySchema, myValues, myWroteHeader, myXMLStack, and toString().

Field Documentation

myBatchSize

const int ParquetFormatter::Impl::myBatchSize

the number of rows to write per batch

Definition at line 119 of file ParquetFormatter.cpp.

myBuilders

std::vector<std::shared_ptr<arrow::ArrayBuilder> > ParquetFormatter::Impl::myBuilders

the content array builders for the table

Definition at line 128 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

myCheckColumns

bool ParquetFormatter::Impl::myCheckColumns = false

whether the columns should be checked for completeness

Definition at line 143 of file ParquetFormatter.cpp.

Referenced by checkAttr().

myCompression

parquet::Compression::type ParquetFormatter::Impl::myCompression = parquet::Compression::UNCOMPRESSED

the compression to use

Definition at line 116 of file ParquetFormatter.cpp.

myExpectedAttrs

SumoXMLAttrMask ParquetFormatter::Impl::myExpectedAttrs

the attributes which are expected for a complete row (including null values)

Definition at line 152 of file ParquetFormatter.cpp.

Referenced by checkAttr().

myFullHeader

std::vector<std::string> ParquetFormatter::Impl::myFullHeader

the column names if we write the full name

Definition at line 113 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

myHaveRootAttrs

bool ParquetFormatter::Impl::myHaveRootAttrs = false

whether any root attribute have been encountered

Definition at line 149 of file ParquetFormatter.cpp.

myHeaderFormat

const std::string ParquetFormatter::Impl::myHeaderFormat

the format to use for the column names

Definition at line 110 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

myMaxDepth

int ParquetFormatter::Impl::myMaxDepth = 1000

the maximum depth of the XML hierarchy

Definition at line 137 of file ParquetFormatter.cpp.

Referenced by checkAttr().

myNeedsWrite

bool ParquetFormatter::Impl::myNeedsWrite = false

whether there is still unwritten data

Definition at line 146 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

myParquetWriter

std::unique_ptr<parquet::arrow::FileWriter> ParquetFormatter::Impl::myParquetWriter

the output stream writer

Definition at line 125 of file ParquetFormatter.cpp.

mySchema

std::shared_ptr<arrow::Schema> ParquetFormatter::Impl::mySchema = arrow::schema({})

the table schema

Definition at line 122 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

mySeenAttrs

SumoXMLAttrMask ParquetFormatter::Impl::mySeenAttrs

the attributes already seen (including null values)

Definition at line 155 of file ParquetFormatter.cpp.

Referenced by checkAttr().

myValues

std::vector<std::shared_ptr<arrow::Scalar> > ParquetFormatter::Impl::myValues

the current attribute / column values

Definition at line 134 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

myWroteHeader

bool ParquetFormatter::Impl::myWroteHeader = false

whether the schema has been constructed completely

Definition at line 140 of file ParquetFormatter.cpp.

Referenced by checkBuilder().

myXMLStack

std::vector<std::pair<const std::string, int> > ParquetFormatter::Impl::myXMLStack

The name and number of attributes in the currently open XML elements.

Definition at line 131 of file ParquetFormatter.cpp.

Referenced by checkAttr(), and checkBuilder().

---

The documentation for this struct was generated from the following file:

• ParquetFormatter.cpp