Line data Source code
1 : /****************************************************************************/
2 : // Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.dev/sumo
3 : // Copyright (C) 2012-2026 German Aerospace Center (DLR) and others.
4 : // This program and the accompanying materials are made available under the
5 : // terms of the Eclipse Public License 2.0 which is available at
6 : // https://www.eclipse.org/legal/epl-2.0/
7 : // This Source Code may also be made available under the following Secondary
8 : // Licenses when the conditions for such availability set forth in the Eclipse
9 : // Public License 2.0 are satisfied: GNU General Public License, version 2
10 : // or later which is available at
11 : // https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html
12 : // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-or-later
13 : /****************************************************************************/
14 : /// @file ParquetFormatter.h
15 : /// @author Michael Behrisch
16 : /// @date 2025-06-17
17 : ///
18 : // Output formatter for Parquet output
19 : /****************************************************************************/
20 : #pragma once
21 : #include <config.h>
22 :
23 : #include <memory>
24 : #include <ostream>
25 : #include <utils/common/ToString.h>
26 : #include "OutputFormatter.h"
27 :
28 :
29 : // ===========================================================================
30 : // class definitions
31 : // ===========================================================================
32 : /**
33 : * @class ParquetFormatter
34 : * @brief Output formatter for Parquet output
35 : *
36 : * All arrow/parquet types are hidden in the Impl so that callers
37 : * (notably OutputDevice.h) do not pay the include cost.
38 : */
39 : class ParquetFormatter : public OutputFormatter {
40 : public:
41 : /// @brief Constructor
42 : // for some motivation on the default batch size see https://stackoverflow.com/questions/76782018/what-is-actually-meant-when-referring-to-parquet-row-group-size
43 : ParquetFormatter(const std::string& columnNames, const std::string& compression = "", const int batchSize = 1000000);
44 :
45 : /// @brief Destructor (out-of-line: Impl is incomplete here)
46 : ~ParquetFormatter() override;
47 :
48 : /** @brief Writes an "XML header"
49 : *
50 : * For Parquet output the header is only relevant if it contains additional attributes.
51 : *
52 : * @param[in] into The output stream to use
53 : * @param[in] rootElement The root element to use
54 : * @param[in] attrs Additional attributes to save within the rootElement
55 : * @return whether something has been written
56 : */
57 : bool writeXMLHeader(std::ostream& into, const std::string& rootElement,
58 : const std::map<SumoXMLAttr, std::string>& attrs, bool /* writeMetadata */,
59 : bool /* includeConfig */) override;
60 :
61 : /** @brief Keeps track of an open XML tag by adding a new element to the stack
62 : *
63 : * @param[in] into The output stream to use (unused)
64 : * @param[in] xmlElement Name of the element to open
65 : */
66 : void openTag(std::ostream& into, const std::string& xmlElement) override;
67 :
68 : /** @brief Keeps track of an open XML tag by adding a new element to the stack
69 : *
70 : * @param[in] into The output stream to use (unused)
71 : * @param[in] xmlElement Enum identifier of the element to open
72 : */
73 : void openTag(std::ostream& into, const SumoXMLTag& xmlElement) override;
74 :
75 : /** @brief Closes the most recently opened tag
76 : *
77 : * This is where the main action starts. This function determines whether a row is completed and can be written.
78 : *
79 : * @param[in] into The output stream to use
80 : * @param[in] comment A comment to write after the tag (ignored for Parquet)
81 : * @return Whether a further element existed in the stack and could be closed
82 : */
83 : bool closeTag(std::ostream& into, const std::string& comment = "") override;
84 :
85 : /** @brief Writes a named attribute
86 : *
87 : * @param[in] into The output stream to use (unused)
88 : * @param[in] attr The attribute (name as enum value)
89 : * @param[in] val The attribute value
90 : * @param[in] isNull whether this actually a null value (adds nullptr to myValues)
91 : */
92 : template <class T>
93 112464 : void writeAttr(std::ostream& /* into */, const SumoXMLAttr attr, const T& val, const bool isNull, const bool /* escape */) {
94 112464 : if (isNull) {
95 75582 : writeNullAttr(attr);
96 : } else {
97 73764 : writeStringAttr(attr, toString(val));
98 : }
99 112464 : }
100 :
101 : /** @brief Writes a named attribute
102 : *
103 : * @param[in] into The output stream to use (unused)
104 : * @param[in] attr The attribute (name as string value)
105 : * @param[in] val The attribute value
106 : * @param[in] isNull whether this actually a null value (adds nullptr to myValues)
107 : */
108 : template <class T>
109 22963 : void writeAttr(std::ostream& /* into */, const std::string& attr, const T& val, const bool isNull, const bool /* escape */) {
110 22963 : if (isNull) {
111 432 : writeNullAttr(attr);
112 : } else {
113 45062 : writeStringAttr(attr, toString(val));
114 : }
115 22963 : }
116 :
117 : /// @brief typed overloads (non-template) -- picked by overload resolution over the template
118 : void writeAttr(std::ostream& into, const SumoXMLAttr attr, const double& val, const bool isNull, const bool escape);
119 : void writeAttr(std::ostream& into, const SumoXMLAttr attr, const int& val, const bool isNull, const bool escape);
120 : void writeAttr(std::ostream& into, const std::string& attr, const double& val, const bool isNull, const bool escape);
121 : void writeAttr(std::ostream& into, const std::string& attr, const int& val, const bool isNull, const bool escape);
122 :
123 : /** @brief Writes a time value
124 : *
125 : * Currently this writes a string if human readable times are activated and a double otherwise
126 : *
127 : * @param[in] into The output stream to use (unused)
128 : * @param[in] attr The attribute (name as enum value)
129 : * @param[in] val The attribute value
130 : * @todo use one of Parquet's time types
131 : */
132 : void writeTime(std::ostream& into, const SumoXMLAttr attr, const SUMOTime val) override;
133 :
134 : /** @brief Whether a complete row has been encountered and triggered writing
135 : *
136 : * @return Whether the Parquet writer has been initialized and a first row has been written
137 : */
138 : bool wroteHeader() const override;
139 :
140 : /** @brief Which elements are expected and which maximum depth the XML tree has.
141 : *
142 : * This is not necessary for the functionality but very useful for debugging and tracking whether expected attributes
143 : * are still missing (triggers an error in checkAttr). If expected is empty, no tracking takes place.
144 : *
145 : * The depth parameter is only for performance. If a tag at this depth is closed for the first time,
146 : * the header is being written and buffered rows may be flushed. Setting it to a large value (which is also the default)
147 : * means you are on the safe side if more attributes or tags show up later but it may result in buffering the complete
148 : * output before writing the first line.
149 : * Setting it to 0 triggers auto detection which means the first time a tag is closed the maximum depth will be determined.
150 : *
151 : * @param[in] expected The enum values of the attrs which should be present before a row can be written.
152 : * @param[in] depth The maximum expected depth of nested XML elements.
153 : */
154 : void setExpectedAttributes(const SumoXMLAttrMask& expected, const int depth) override;
155 :
156 : private:
157 : /// @brief non-template helpers; defined in the .cpp where arrow/parquet are available
158 : void writeStringAttr(const SumoXMLAttr attr, const std::string& val);
159 : void writeStringAttr(const std::string& attr, const std::string& val);
160 : void writeNullAttr(const SumoXMLAttr attr);
161 : void writeNullAttr(const std::string& attr);
162 :
163 : /// @brief opaque arrow/parquet state
164 : struct Impl;
165 : std::unique_ptr<Impl> myImpl;
166 : };
|
