Repository: ndsev/mapget (language: python)

Issue:
Implement TileLayerStream
Both to transfer data from sources to the mapget cache, and to receive tile data from the mapget cache, the TileLayerStream protocol is required. It will be implemented in two classes:

* The `TileLayerStreamReader` is constructed with a callback for `onParsedLayer`. It has `pushBytes` and `numUnparsedBytes` methods, which may be used to insert binary data to parse, and to check whether there is any unparsed data.
* The `TileLayerStreamWriter` is constructed with a callback for `onWrite`, which may be called when something is ready to be sent over the wire, and `onGetFieldCacheOffset`, which may be called when the stream must decide how much of the field cache must be re-sent. It has a `pushFeatureLayer` method, which is used to insert a (Tile-)FeatureLayer into the stream.


Discussion / hints:

Fix patch:
diff --git a/libs/model/CMakeLists.txt b/libs/model/CMakeLists.txt
index d588cc4f..73385556 100644
--- a/libs/model/CMakeLists.txt
+++ b/libs/model/CMakeLists.txt
@@ -10,6 +10,7 @@ add_library(mapget-model STATIC
   include/mapget/model/attrlayer.h
   include/mapget/model/tileid.h
   include/mapget/model/featureid.h
+  include/mapget/model/stream.h
 
   src/fields.cpp
   src/layer.cpp
@@ -19,7 +20,8 @@ add_library(mapget-model STATIC
   src/attr.cpp
   src/attrlayer.cpp
   src/tileid.cpp
-  src/featureid.cpp)
+  src/featureid.cpp
+  src/stream.cpp)
 
 target_include_directories(mapget-model
   PUBLIC
diff --git a/libs/model/include/mapget/model/attr.h b/libs/model/include/mapget/model/attr.h
index 04d6feca..2c626375 100644
--- a/libs/model/include/mapget/model/attr.h
+++ b/libs/model/include/mapget/model/attr.h
@@ -44,15 +44,25 @@ class Attribute : public simfil::ProceduralObject<2, Attribute>
 
 protected:
 
+    /** Actual per-attribute data that is stored in the model's attributes-column. */
     struct Data {
         Direction direction_ = Empty;
         simfil::ModelNodeAddress validity_;
         simfil::ArrayIndex fields_ = -1;
         simfil::FieldId name_ = 0;
+
+        template<typename S>
+        void serialize(S& s) {
+            s.value1b(direction_);
+            s.value4b(validity_.value_);
+            s.value4b(fields_);
+            s.value2b(name_);
+        }
     };
 
     Attribute(Data& data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a);
 
+    /** Reference to the actual stored data for the attribute. */
     Data& data_;
 };
 
diff --git a/libs/model/include/mapget/model/feature.h b/libs/model/include/mapget/model/feature.h
index ea533761..412f3989 100644
--- a/libs/model/include/mapget/model/feature.h
+++ b/libs/model/include/mapget/model/feature.h
@@ -84,6 +84,15 @@ class Feature : protected simfil::MandatoryDerivedModelNodeBase<TileFeatureLayer
         simfil::ModelNodeAddress attrLayers_;
         simfil::ModelNodeAddress attrs_;
         simfil::ModelNodeAddress children_;
+
+        template<typename S>
+        void serialize(S& s) {
+            s.value4b(id_.value_);
+            s.value4b(geom_.value_);
+            s.value4b(attrLayers_.value_);
+            s.value4b(attrs_.value_);
+            s.value4b(children_.value_);
+        }
     };
 
     Feature(Data& d, simfil::ModelConstPtr l, simfil::ModelNodeAddress a);
diff --git a/libs/model/include/mapget/model/featureid.h b/libs/model/include/mapget/model/featureid.h
index b150245c..f993c27f 100644
--- a/libs/model/include/mapget/model/featureid.h
+++ b/libs/model/include/mapget/model/featureid.h
@@ -50,6 +50,13 @@ class FeatureId : protected simfil::MandatoryDerivedModelNodeBase<TileFeatureLay
         bool useCommonTilePrefix_ = false;
         simfil::FieldId typeId_ = 0;
         simfil::ModelNodeAddress idParts_;
+
+        template<typename S>
+        void serialize(S& s) {
+            s.value1b(useCommonTilePrefix_);
+            s.value2b(typeId_);
+            s.value4b(idParts_.value_);
+        }
     };
 
     FeatureId(Data& data, simfil::ModelConstPtr l, simfil::ModelNodeAddress a);
diff --git a/libs/model/include/mapget/model/featurelayer.h b/libs/model/include/mapget/model/featurelayer.h
index 8818cbbb..1da32bf3 100644
--- a/libs/model/include/mapget/model/featurelayer.h
+++ b/libs/model/include/mapget/model/featurelayer.h
@@ -27,6 +27,12 @@ using KeyValuePairs = std::vector<std::pair<
     std::string_view,
     std::variant<int64_t, std::string_view>>>;
 
+/**
+ * Callback type for a function which returns a field name cache instance
+ * for a given node identifier.
+ */
+using FieldNameResolveFun = std::function<std::shared_ptr<Fields>(std::string_view const&)>;
+
 /**
  * The TileFeatureLayer class represents a specific map layer
  * within a map tile. It is a container for map features.
@@ -61,6 +67,20 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool
         std::shared_ptr<Fields> const& fields
     );
 
+    /**
+     * Constructor which parses a TileFeatureLayer from a binary stream.
+     * @param inputStream The binary stream to parse.
+     * @param layerInfoResolveFun Function which will be called to retrieve
+     *  a layerInfo object for the layer name stored for the tile.
+     * @param fieldNameResolveFun Function which will be called to retrieve
+     *  a Fields dictionary object for the node name for the tile.
+     */
+    TileFeatureLayer(
+        std::istream& inputStream,
+        LayerInfoResolveFun const& layerInfoResolveFun,
+        FieldNameResolveFun const& fieldNameResolveFun
+    );
+
     /**
      * Set common id prefix for all features in this layer.
      */
@@ -136,6 +156,9 @@ class TileFeatureLayer : public TileLayer, public simfil::ModelPool
     Iterator begin() const;
     Iterator end() const;
 
+    /** (De-)Serialization */
+   void write(std::ostream& outputStream) override;
+
 protected:
     /**
      * The FeatureTileColumnId enum provides identifiers for different
diff --git a/libs/model/include/mapget/model/fields.h b/libs/model/include/mapget/model/fields.h
index b0ae1030..eb584687 100644
--- a/libs/model/include/mapget/model/fields.h
+++ b/libs/model/include/mapget/model/fields.h
@@ -17,6 +17,9 @@ namespace mapget
  * to work with compared to strings. When querying fields, the Fields dictionary is used to look up the
  * field ID for a given name. Once the ID is obtained, subsequent searches are performed using this 16-bit
  * integer, resulting in improved efficiency.
+ *
+ * Note: A fields dictionary is always unique per datasource node. Therefore,
+ * the Fields object must be constructed with a datasource node id.
  */
 struct Fields : public simfil::Fields
 {
@@ -30,7 +33,22 @@ struct Fields : public simfil::Fields
         PropertiesStr,
     };
 
-    Fields();
+    explicit Fields(std::string nodeId);
+
+    /**
+     * Write is overloaded, because it prepends the stream with
+     * this dictionary's data source node id. On the read side, the
+     * consumer must call readDataSourceNodeId() before calling read().
+     */
+    void write(std::ostream& outputStream, simfil::FieldId offset) const override;
+
+    /**
+     * Call this before calling read() to figure out which Fields-
+     * object to call read() with.
+     */
+    static std::string readDataSourceNodeId(std::istream& inputStream);
+
+    std::string const nodeId_;
 };
 
 }
diff --git a/libs/model/include/mapget/model/info.h b/libs/model/include/mapget/model/info.h
index 03f1d11f..3d62f1b1 100644
--- a/libs/model/include/mapget/model/info.h
+++ b/libs/model/include/mapget/model/info.h
@@ -9,6 +9,35 @@
 
 namespace mapget
 {
+
+/**
+ * Version Definition - This is used to recognize whether
+ * a stored blob of a TileFeatureLayer should be parsed by this
+ * version of the mapget library. When a TileFeatureLayer is serialized,
+ * the current Version value for the map layer and and the stream protocol are
+ * also stored. Upon deserialization, the Major-Minor version values
+ * must match the parsed version Major-Minor value, for both the for the map layer
+ * and and the stream protocol.
+ */
+struct Version {
+    uint16_t major_ = 0;
+    uint16_t minor_ = 0;
+    uint16_t patch_ = 0;
+
+    [[nodiscard]] bool isCompatible(Version const& other) const;
+    [[nodiscard]] std::string toString() const;
+
+    bool operator==(Version const& other) const;
+    bool operator<(Version const& other) const;
+
+    template<typename S>
+    void serialize(S& s) {
+        s.value2b(major_);
+        s.value2b(minor_);
+        s.value2b(patch_);
+    }
+};
+
 /** Enum to represent the possible data types */
 enum class IdPartDataType {I32, U32, I64, U64, UUID128, STR};
 
@@ -19,50 +48,54 @@ enum class LayerType {Features, Heightmap, OrthoImage, GLTF};
 struct UniqueIdPart
 {
     /** Unique identifier part */
-    std::string partId;
+    std::string partId_;
 
     /** Description of the unique identifier */
-    std::string description;
+    std::string description_;
 
     /** Datatype of the unique identifier */
-    IdPartDataType datatype;
+    IdPartDataType datatype_;
 
     /** Is the identifier synthetic or part of a map specification? */
-    bool isSynthetic = false;
+    bool isSynthetic_ = false;
 
     /** Is the identifier actually optional? */
-    bool isOptional = false;
+    bool isOptional_ = false;
 };
 
 /** Structure to represent the feature type info */
-struct FeatureTypeInfo {
+struct FeatureTypeInfo
+{
     /** Name of the feature type */
-    std::string name;
+    std::string name_;
 
     /**
      * List of allowed unique id compositions (each id composition is a list of id parts)
      * A single id composition must never have more than 16 parts.
      */
-    std::vector<std::vector<UniqueIdPart>> uniqueIdCompositions;
+    std::vector<std::vector<UniqueIdPart>> uniqueIdCompositions_;
 };
 
 /**
  * Structure to represent a list of coverage flags
  * as a rectangle between a minimum and a maximum tile id.
  */
-struct Coverage {
+struct Coverage
+{
     /** Minimum tile id */
-    TileId min;
+    TileId min_;
 
     /** Maximum tile id. Must have same zoom level as min. */
-    TileId max;
+    TileId max_;
 
     /**
      * Bitset indicating where the associated layer is filled.
      * Must have size (max.x() - min.x() + 1) * (max.y() - min.y() + 1).
      * Bits are stored row-major: [ y0x0..., y0xn, y1x0..., y1xn, ... ]
+     * If the vector is completely empty, the rectangle is considered
+     * to be fully filled.
      */
-    std::vector<bool> filled;
+    std::vector<bool> filled_;
 };
 
 /**
@@ -71,28 +104,31 @@ struct Coverage {
 struct LayerInfo
 {
     /** Unique identifier of the layer. */
-    std::string layerId;
+    std::string layerId_;
 
     /** Type of the layer */
-    LayerType type = LayerType::Features;
+    LayerType type_ = LayerType::Features;
 
-    /** List of feature types */
-    std::vector<FeatureTypeInfo> featureTypes;
+    /** List of feature types, only relevant if this is a Feature-layer. */
+    std::vector<FeatureTypeInfo> featureTypes_;
 
     /** List of zoom levels */
-    std::vector<int> zoomLevels;
+    std::vector<int> zoomLevels_;
 
     /**
      * List of Coverage structures.
-     * Multiple coverages may exist for the same map layer.
+     * Multiple coverages may exist for the same zoom level.
      */
-    std::vector<Coverage> coverage;
+    std::vector<Coverage> coverage_;
 
     /** Can this layer be read from? */
-    bool canRead = true;
+    bool canRead_ = true;
 
     /** Can this layer be written to? */
-    bool canWrite = false;
+    bool canWrite_ = false;
+
+    /** Version of the map layer. */
+    Version version_;
 };
 
 /**
@@ -101,22 +137,22 @@ struct LayerInfo
 struct DataSourceInfo
 {
     /** Unique identifier of the node */
-    std::string nodeId;
+    std::string nodeId_;
 
     /** Unique identifier of the map */
-    std::string mapId;
+    std::string mapId_;
 
     /** List of layers */
-    std::map<std::string, std::shared_ptr<LayerInfo>> layers;
+    std::map<std::string, std::shared_ptr<LayerInfo>> layers_;
 
     /** Maximum number of parallel jobs */
-    int maxParallelJobs = 8;
-
-    /** Version of the protocol */
-    int protocolVersion = 0;
+    int maxParallelJobs_ = 8;
 
     /** Extra JSON attachment. May also be used to store style-sheets. */
-    nlohmann::json extraJsonAttachment;
+    nlohmann::json extraJsonAttachment_;
+
+    /** Used mapget protocol version */
+    Version protocolVersion_;
 };
 
-}  // End of namespace mapget
\ No newline at end of file
+}  // End of namespace mapget
diff --git a/libs/model/include/mapget/model/layer.h b/libs/model/include/mapget/model/layer.h
index 9466db9e..5048f385 100644
--- a/libs/model/include/mapget/model/layer.h
+++ b/libs/model/include/mapget/model/layer.h
@@ -9,10 +9,17 @@
 #include <chrono>
 #include <optional>
 #include <memory>
+#include <tuple>
 
 namespace mapget
 {
 
+/**
+ * Callback type for a function which returns a layer info pointer for
+ * a given (map-name, layer-name) combination.
+ */
+using LayerInfoResolveFun = std::function<std::shared_ptr<LayerInfo>(std::string_view const&, std::string_view const&)>;
+
 /**
  * Tile Layer base class. Used by TileFeatureLayer class and other
  * tile-specific data containers.
@@ -26,10 +33,19 @@ class TileLayer
      */
     TileLayer(
         const TileId& id,
-        std::string node_id,
-        std::string map_id,
+        std::string nodeId,
+        std::string mapId,
         const std::shared_ptr<LayerInfo>& info);
 
+    /**
+     * Parse a tile layer from an input stream. Will throw if
+     * the resolved major-minor of the TileLayer is not the same
+     * as the one read from the stream.
+     */
+    TileLayer(
+        std::istream& inputStream,
+        LayerInfoResolveFun const& layerInfoResolveFun);
+
     /**
      * Getter and setter for layer's tileId. This controls the rough
      * geographic extent of the contained tile data.
@@ -69,22 +85,22 @@ class TileLayer
      * Getter and setter for 'timestamp' member variable.
      * It represents when this layer was created.
      */
-    [[nodiscard]] std::chrono::time_point<std::chrono::steady_clock> timestamp() const;
-    void setTimestamp(const std::chrono::time_point<std::chrono::steady_clock>& ts);
+    [[nodiscard]] std::chrono::time_point<std::chrono::system_clock> timestamp() const;
+    void setTimestamp(const std::chrono::time_point<std::chrono::system_clock>& ts);
 
     /**
      * Getter and setter for 'ttl_' member variable.
      * It represents how long this layer should live.
      */
-    [[nodiscard]]std::optional<std::chrono::seconds> ttl() const;
-    void setTtl(const std::optional<std::chrono::seconds>& timeToLive);
+    [[nodiscard]]std::optional<std::chrono::milliseconds> ttl() const;
+    void setTtl(const std::optional<std::chrono::milliseconds>& timeToLive);
 
     /**
-     * Getter and setter for 'protocolVersion_' member variable.
+     * Getter and setter for 'mapVersion_' member variable.
      * It represents the protocol version that was used to serialize this layer.
      */
-    [[nodiscard]] int protocolVersion() const;
-    void setProtocolVersion(int version);
+    [[nodiscard]] Version mapVersion() const;
+    void setProtocolVersion(Version v);
 
     /**
      * Getter and setter for 'info' member variable.
@@ -95,15 +111,18 @@ class TileLayer
     void setInfo(const nlohmann::json& info);
 
 protected:
+    Version mapVersion_{0, 0, 0};
     TileId tileId_;
     std::string nodeId_;
     std::string mapId_;
     std::shared_ptr<LayerInfo> layerInfo_;
     std::optional<std::string> error_;
-    std::chrono::time_point<std::chrono::steady_clock> timestamp_;
-    std::optional<std::chrono::seconds> ttl_;
-    int protocolVersion_ = 0;
+    std::chrono::time_point<std::chrono::system_clock> timestamp_;
+    std::optional<std::chrono::milliseconds> ttl_;
     nlohmann::json info_;
+
+    /** Serialization */
+    void write(std::ostream& outputStream);
 };
 
 }
diff --git a/libs/model/include/mapget/model/stream.h b/libs/model/include/mapget/model/stream.h
new file mode 100644
index 00000000..bbc75518
--- /dev/null
+++ b/libs/model/include/mapget/model/stream.h
@@ -0,0 +1,121 @@
+#pragma once
+
+#include "featurelayer.h"
+
+#include <map>
+#include <sstream>
+
+namespace mapget
+{
+
+/**
+ * Protocol for binary streaming of TileFeatureLayer and associated
+ * Fields dictionary objects. The general stream encoding is a simple
+ * Version-Type-Length-Value one:
+ * - The version (6b) indicates the protocol version which was used to
+ *   serialise the blob. This must be comatible with the current version
+ *   which is used by the mapget library.
+ * - The type (1B) must be either Fields (1) or TileFeatureLayer (2).
+ * - The length (4b)  indicates the byte-length of the serialized object,
+ *   which is stored in the value.
+ */
+class TileLayerStream
+{
+    enum class MessageType : uint8_t {
+        None = 0,
+        Fields = 1,
+        TileFeatureLayer = 2,
+    };
+
+public:
+    struct CachedFieldsProvider;
+
+    /** Protocol Version which parsed blobs must be compatible with. */
+    static constexpr Version CurrentProtocolVersion{0, 1, 0};
+
+    /** Map to keep track of the highest sent field id per datasource node. */
+    using FieldOffsetMap = std::map<std::string, simfil::FieldId>;
+
+    /** The Reader turns bytes into TileFeatureLayer objects. */
+    struct Reader
+    {
+        /**
+         * Construct a Reader with a callback for parsed result layers,
+         * a CachedFieldsProvider which can supply and receive Fields
+         * dictionaries for a node id, and a layerInfoProvider which
+         * can provide LayerInfo objects for a (map-id, layer-id) combination.
+         */
+        Reader(
+            LayerInfoResolveFun layerInfoProvider,
+            std::function<void(std::shared_ptr<TileFeatureLayer>)> onParsedLayer,
+            std::shared_ptr<CachedFieldsProvider> fieldCacheProvider = nullptr);
+
+        /**
+         * Add some bytes to parse. The next object will be parsed once
+         * sufficient bytes are available.
+         */
+        void read(std::vector<uint8_t> const& bytes);
+
+        /** end-of-stream: Returns true if the internal buffer is exhausted. */
+        [[nodiscard]] bool eos();
+
+    private:
+        enum class Phase { ReadHeader, ReadValue };
+
+        Phase currentPhase_ = Phase::ReadHeader;
+        MessageType nextValueType_ = MessageType::None;
+        uint32_t nextValueSize_ = 0;
+
+        bool continueReading();
+
+        std::stringstream buffer_;
+        LayerInfoResolveFun layerInfoProvider_;
+        std::shared_ptr<CachedFieldsProvider> fieldCacheProvider_;
+        std::function<void(std::shared_ptr<TileFeatureLayer>)> onParsedLayer_;
+    };
+
+    /**
+     * The Writer turns TileFeatureLayer objects and associated Fields
+     * dictionaries into bytes.
+     */
+    struct Writer
+    {
+        /**
+         * Construct a Writer with a callback for serialized messages,
+         * and a reference to a dict which stores the current highest
+         * sent FieldId offset per data source node id. Note, that this
+         * dictionary will be updated as Fields dictionary updates are sent.
+         * Using the same FieldId offset map for to Writer objects will
+         * lead to undefined behavior.
+         */
+        Writer(
+            std::function<void(std::string)> onMessage,
+            FieldOffsetMap& fieldsOffsets);
+
+        /** Serialize a tile feature layer and the required part of a Fields cache. */
+        void write(std::shared_ptr<TileFeatureLayer> const& tileFeatureLayer);
+
+    private:
+        void sendMessage(std::string const& bytes, MessageType msgType);
+
+        std::function<void(std::string)> onMessage_;
+        FieldOffsetMap& fieldsOffsets_;
+    };
+
+    /**
+     * Cache for Fields-dictionaries. Fields dictionaries are unique per data source node,
+     * since each data source node may have a uniquely-filled field cache.
+     * The default implementation just places an empty Fields-dictionary
+     * into the cache if there is no registered one for the given node id.
+     * Derived CachedFieldsProviders may handle uncached Fields dicts differently,
+     * e.g. by initializing the Fields-object from a cache database.
+     */
+    struct CachedFieldsProvider
+    {
+        virtual std::shared_ptr<Fields> operator() (std::string_view const&);
+    private:
+        std::map<std::string, std::shared_ptr<Fields>> fieldsPerNodeId_;
+    };
+};
+
+}
\ No newline at end of file
diff --git a/libs/model/src/featurelayer.cpp b/libs/model/src/featurelayer.cpp
index 6b232fce..0cfd4868 100644
--- a/libs/model/src/featurelayer.cpp
+++ b/libs/model/src/featurelayer.cpp
@@ -3,11 +3,17 @@
 #include <map>
 #include <shared_mutex>
 
+#include <bitsery/bitsery.h>
+#include <bitsery/adapter/stream.h>
+#include <bitsery/traits/string.h>
+
+#include "simfil/model/bitsery-traits.h"
+
 namespace mapget
 {
 
 struct TileFeatureLayer::Impl {
-    std::optional<model_ptr<Object>> featureIdPrefix_;
+    simfil::ModelNodeAddress featureIdPrefix_;
 
     sfl::segmented_vector<Feature::Data, simfil::detail::ColumnPageSize/4> features_;
     sfl::segmented_vector<Attribute::Data, simfil::detail::ColumnPageSize> attributes_;
@@ -24,6 +30,18 @@ struct TileFeatureLayer::Impl {
     // Mutex to manage access to the expression cache
     std::shared_mutex expressionCacheLock_;
 
+    // (De-)Serialization
+    template<typename S>
+    void readWrite(S& s) {
+        constexpr size_t maxColumnSize = std::numeric_limits<uint32_t>::max();
+        s.container(features_, maxColumnSize);
+        s.container(attributes_, maxColumnSize);
+        s.container(featureIds_, maxColumnSize);
+        s.container(attrLayers_, maxColumnSize);
+        s.container(attrLayerLists_, maxColumnSize);
+        s.value4b(featureIdPrefix_.value_);
+    }
+
     explicit Impl(std::shared_ptr<Fields> fields)
         : simfilEnv_(std::move(fields))
     {
@@ -43,6 +61,25 @@ TileFeatureLayer::TileFeatureLayer(
 {
 }
 
+TileFeatureLayer::TileFeatureLayer(
+    std::istream& inputStream,
+    const LayerInfoResolveFun& layerInfoResolveFun,
+    const FieldNameResolveFun& fieldNameResolveFun
+) :
+    TileLayer(inputStream, layerInfoResolveFun),
+    simfil::ModelPool(fieldNameResolveFun(nodeId_)),
+    impl_(std::make_unique<Impl>(fieldNameResolveFun(nodeId_)))
+{
+    bitsery::Deserializer<bitsery::InputStreamAdapter> s(inputStream);
+    impl_->readWrite(s);
+    if (s.adapter().error() != bitsery::ReaderError::NoError) {
+        throw std::runtime_error(stx::format(
+            "Failed to read TileFeatureLayer: Error {}",
+            static_cast<std::underlying_type_t<bitsery::ReaderError>>(s.adapter().error())));
+    }
+    ModelPool::read(inputStream);
+}
+
 TileFeatureLayer::~TileFeatureLayer() = default;
 
 simfil::shared_model_ptr<Feature> TileFeatureLayer::newFeature(
@@ -101,7 +138,9 @@ TileFeatureLayer::newFeatureId(
 
 std::optional<model_ptr<Object>> TileFeatureLayer::featureIdPrefix()
 {
-    return impl_->featureIdPrefix_;
+    if (impl_->featureIdPrefix_.value_)
+        return resolveObject(simfil::ModelNode::Ptr::make(shared_from_this(), impl_->featureIdPrefix_));
+    return {};
 }
 
 model_ptr<Attribute>
@@ -248,7 +287,7 @@ void TileFeatureLayer::setPrefix(const KeyValuePairs& prefix)
             idPrefix->addField(kk, x);
         }, v);
     }
-    impl_->featureIdPrefix_ = idPrefix;
+    impl_->featureIdPrefix_ = idPrefix->addr();
 }
 
 TileFeatureLayer::Iterator TileFeatureLayer::begin() const
@@ -261,4 +300,12 @@ TileFeatureLayer::Iterator TileFeatureLayer::end() const
     return TileFeatureLayer::Iterator{*this, numRoots()};
 }
 
+void TileFeatureLayer::write(std::ostream& outputStream)
+{
+    TileLayer::write(outputStream);
+    bitsery::Serializer<bitsery::OutputStreamAdapter> s(outputStream);
+    impl_->readWrite(s);
+    ModelPool::write(outputStream);
+}
+
 }
diff --git a/libs/model/src/fields.cpp b/libs/model/src/fields.cpp
index 7dad48a0..52c294da 100644
--- a/libs/model/src/fields.cpp
+++ b/libs/model/src/fields.cpp
@@ -1,9 +1,13 @@
 #include "fields.h"
 
+#include <bitsery/bitsery.h>
+#include <bitsery/adapter/stream.h>
+#include <bitsery/traits/string.h>
+
 namespace mapget
 {
 
-Fields::Fields() {
+Fields::Fields(std::string nodeId) : nodeId_(std::move(nodeId)) {
     addStaticKey(IdStr, "id");
     addStaticKey(TypeIdStr, "typeId");
     addStaticKey(LayerStr, "layer");
@@ -13,4 +17,20 @@ Fields::Fields() {
     addStaticKey(PropertiesStr, "properties");
 }
 
+void Fields::write(std::ostream& outputStream, simfil::FieldId offset) const
+{
+    // Write the node id which identifies the fields dictionary
+    bitsery::Serializer<bitsery::OutputStreamAdapter> s(outputStream);
+    s.text1b(nodeId_, std::numeric_limits<uint32_t>::max());
+    simfil::Fields::write(outputStream, offset);
+}
+
+std::string Fields::readDataSourceNodeId(std::istream& inputStream) {
+    // Read the node id which identifies the fields dictionary
+    bitsery::Deserializer<bitsery::InputStreamAdapter> s(inputStream);
+    std::string fieldsDictNodeId;
+    s.text1b(fieldsDictNodeId, std::numeric_limits<uint32_t>::max());
+    return fieldsDictNodeId;
+}
+
 }
diff --git a/libs/model/src/info.cpp b/libs/model/src/info.cpp
index c8b91a2a..99bb5edb 100644
--- a/libs/model/src/info.cpp
+++ b/libs/model/src/info.cpp
@@ -1,8 +1,31 @@
 #include "info.h"
+#include <tuple>
 
 namespace mapget
 {
 
+bool Version::isCompatible(const Version& other) const
+{
+    return other.major_ == major_ && other.minor_ == minor_;
+}
+
+std::string Version::toString() const
+{
+    return stx::format("{}.{}.{}", major_, minor_, patch_);
+}
 
+bool Version::operator==(const Version& other) const
+{
+    return (
+        std::tie(other.major_, other.minor_, other.patch_) ==
+        std::tie(major_, minor_, patch_));
+}
+
+bool Version::operator<(const Version& other) const
+{
+    return (
+        std::tie(other.major_, other.minor_, other.patch_) <
+        std::tie(major_, minor_, patch_));
+}
 
-}
\ No newline at end of file
+}
diff --git a/libs/model/src/layer.cpp b/libs/model/src/layer.cpp
index 9afef4f3..51a5edba 100644
--- a/libs/model/src/layer.cpp
+++ b/libs/model/src/layer.cpp
@@ -1,21 +1,75 @@
 #include "layer.h"
 
+#include <bitsery/bitsery.h>
+#include <bitsery/adapter/stream.h>
+#include <bitsery/traits/string.h>
+
+#include "simfil/model/bitsery-traits.h"
+
+#include <istream>
+
+#include "nlohmann/json.hpp"
+
 namespace mapget
 {
 
 TileLayer::TileLayer(
     const TileId& id,
-    std::string node_id,
-    std::string map_id,
+    std::string nodeId,
+    std::string mapId,
     const std::shared_ptr<LayerInfo>& info
 )
     : tileId_(id),
-      nodeId_(std::move(node_id)),
-      mapId_(std::move(map_id)),
+      nodeId_(std::move(nodeId)),
+      mapId_(std::move(mapId)),
       layerInfo_(info),
-      timestamp_(std::chrono::steady_clock::now())
+      timestamp_(std::chrono::duration_cast<std::chrono::microseconds>(
+          std::chrono::system_clock::now().time_since_epoch()))
 {
-    // constructor body
+}
+
+TileLayer::TileLayer(
+    std::istream& inputStream,
+    const LayerInfoResolveFun& layerInfoResolveFun
+) : tileId_(0)
+{
+    using namespace std::chrono;
+    using namespace nlohmann;
+
+    bitsery::Deserializer<bitsery::InputStreamAdapter> s(inputStream);
+    s.text1b(mapId_, std::numeric_limits<uint32_t>::max());
+    std::string layerName;
+    s.text1b(layerName, std::numeric_limits<uint32_t>::max());
+    layerInfo_ = layerInfoResolveFun(mapId_, layerName);
+
+    s.object(mapVersion_);
+    if (!mapVersion_.isCompatible(layerInfo_->version_)) {
+        throw std::runtime_error(stx::format(
+            "Read map layer '{}' version {} "
+            "is incompatible with present version {}.",
+            layerName,
+            mapVersion_.toString(),
+            layerInfo_->version_.toString()));
+    }
+
+    s.value8b(tileId_.value_);
+    s.text1b(nodeId_, std::numeric_limits<uint32_t>::max());
+
+    int64_t timestamp = 0;
+    s.value8b(timestamp);
+    timestamp_ = time_point<system_clock>(microseconds(timestamp));
+
+    bool hasTtl = false;
+    s.value1b(hasTtl);
+    if (hasTtl) {
+        int64_t ttl = 0;
+        s.value8b(ttl);
+        ttl_ = milliseconds(ttl);
+    }
+
+    std::string infoJsonString;
+    s.text1b(infoJsonString, std::numeric_limits<uint32_t>::max());
+    info_ = json::parse(infoJsonString);
 }
 
 TileId TileLayer::tileId() const {
@@ -38,16 +92,16 @@ std::optional<std::string> TileLayer::error() const {
     return error_;
 }
 
-std::chrono::time_point<std::chrono::steady_clock> TileLayer::timestamp() const {
+std::chrono::time_point<std::chrono::system_clock> TileLayer::timestamp() const {
     return timestamp_;
 }
 
-std::optional<std::chrono::seconds> TileLayer::ttl() const {
+std::optional<std::chrono::milliseconds> TileLayer::ttl() const {
     return ttl_;
 }
 
-int TileLayer::protocolVersion() const {
-    return protocolVersion_;
+Version TileLayer::mapVersion() const {
+    return mapVersion_;
 }
 
 nlohmann::json TileLayer::info() const {
@@ -74,20 +128,38 @@ void TileLayer::setError(const std::optional<std::string>& err) {
     error_ = err;
 }
 
-void TileLayer::setTimestamp(const std::chrono::time_point<std::chrono::steady_clock>& ts) {
+void TileLayer::setTimestamp(const std::chrono::time_point<std::chrono::system_clock>& ts) {
     timestamp_ = ts;
 }
 
-void TileLayer::setTtl(const std::optional<std::chrono::seconds>& timeToLive) {
+void TileLayer::setTtl(const std::optional<std::chrono::milliseconds>& timeToLive) {
     ttl_ = timeToLive;
 }
 
-void TileLayer::setProtocolVersion(int version) {
-    protocolVersion_ = version;
+void TileLayer::setProtocolVersion(Version version) {
+    mapVersion_ = version;
 }
 
 void TileLayer::setInfo(const nlohmann::json& info) {
     this->info_ = info;
 }
 
+void TileLayer::write(std::ostream& outputStream)
+{
+    using namespace std::chrono;
+    using namespace nlohmann;
+
+    bitsery::Serializer<bitsery::OutputStreamAdapter> s(outputStream);
+    s.text1b(mapId_, std::numeric_limits<uint32_t>::max());
+    s.text1b(layerInfo_->layerId_, std::numeric_limits<uint32_t>::max());
+    s.object(mapVersion_);
+    s.value8b(tileId_.value_);
+    s.text1b(nodeId_, std::numeric_limits<uint32_t>::max());
+    s.value8b(duration_cast<microseconds>(timestamp_.time_since_epoch()).count());
+    s.value1b(ttl_.has_value());
+    if (ttl_)
+        s.value8b(ttl_->count());
+    s.text1b(info_.dump(), std::numeric_limits<uint32_t>::max());
+}
+
 } // namespace mapget
diff --git a/libs/model/src/stream.cpp b/libs/model/src/stream.cpp
new file mode 100644
index 00000000..26b37565
--- /dev/null
+++ b/libs/model/src/stream.cpp
@@ -0,0 +1,141 @@
+#include "stream.h"
+
+#include <bitsery/bitsery.h>
+#include <bitsery/adapter/stream.h>
+#include <bitsery/traits/string.h>
+
+namespace mapget
+{
+
+TileLayerStream::Reader::Reader(
+    LayerInfoResolveFun layerInfoProvider,
+    std::function<void(std::shared_ptr<TileFeatureLayer>)> onParsedLayer,
+    std::shared_ptr<CachedFieldsProvider> fieldCacheProvider)
+    : layerInfoProvider_(std::move(layerInfoProvider)),
+      fieldCacheProvider_(
+          fieldCacheProvider ? std::move(fieldCacheProvider) :
+                               std::make_shared<TileLayerStream::CachedFieldsProvider>()),
+      onParsedLayer_(std::move(onParsedLayer))
+{
+}
+
+void TileLayerStream::Reader::read(const std::vector<uint8_t>& bytes)
+{
+    std::string_view strBytes((char const*)bytes.data(), bytes.size());
+    buffer_ << strBytes;
+    while (continueReading());
+}
+
+bool TileLayerStream::Reader::eos()
+{
+    return (buffer_.tellp() - buffer_.tellg()) == 0;
+}
+
+bool TileLayerStream::Reader::continueReading()
+{
+    bitsery::Deserializer<bitsery::InputStreamAdapter> s(buffer_);
+    auto numUnreadBytes = buffer_.tellp() - buffer_.tellg();
+
+    if (currentPhase_ == Phase::ReadHeader)
+    {
+        // Version: 6B, Type: 1B, Size: 4B
+        constexpr auto headerSize = 6 + 1 + 4;
+        if (numUnreadBytes < headerSize)
+            return false;
+
+        Version protocolVersion;
+        s.object(protocolVersion);
+        if (!protocolVersion.isCompatible(CurrentProtocolVersion)) {
+            throw std::runtime_error(stx::format(
+                "Unable to read message with version {} using version {}.",
+                protocolVersion.toString(),
+                CurrentProtocolVersion.toString()));
+        }
+        s.value1b(nextValueType_);
+        s.value4b(nextValueSize_);
+        currentPhase_ = Phase::ReadValue;
+        return true;
+    }
+
+    if (numUnreadBytes < nextValueSize_)
+        return false;
+
+    if (nextValueType_ == MessageType::TileFeatureLayer)
+    {
+        onParsedLayer_(std::make_shared<TileFeatureLayer>(
+            buffer_,
+            layerInfoProvider_,
+            *fieldCacheProvider_));
+    }
+    else if (nextValueType_ == MessageType::Fields)
+    {
+        // Read the node id which identifies the fields dictionary
+        std::string fieldsDictNodeId = Fields::readDataSourceNodeId(buffer_);
+        (*fieldCacheProvider_)(fieldsDictNodeId)->read(buffer_);
+    }
+
+    currentPhase_ = Phase::ReadHeader;
+    return true;
+}
+
+TileLayerStream::Writer::Writer(
+    std::function<void(std::string)> onMessage,
+    FieldOffsetMap& fieldsOffsets)
+    : onMessage_(std::move(onMessage)),
+      fieldsOffsets_(fieldsOffsets)
+{
+}
+
+void TileLayerStream::Writer::write(std::shared_ptr<TileFeatureLayer> const& tileFeatureLayer)
+{
+    auto fields = tileFeatureLayer->fieldNames();
+    auto& highestFieldKnownToClient = fieldsOffsets_[tileFeatureLayer->nodeId()];
+    auto highestField = fields->highest();
+
+    if (highestFieldKnownToClient < highestField)
+    {
+        // Need to send the client an update for the Fields dictionary
+        std::stringstream serializedFields;
+        fields->write(serializedFields, highestFieldKnownToClient+1);
+        sendMessage(serializedFields.str(), MessageType::Fields);
+        highestFieldKnownToClient = highestField;
+    }
+
+    // Send actual tileFeatureLayer
+    std::stringstream serializedFeatureLayer;
+    tileFeatureLayer->write(serializedFeatureLayer);
+    sendMessage(serializedFeatureLayer.str(), MessageType::TileFeatureLayer);
+}
+
+void TileLayerStream::Writer::sendMessage(std::string const& bytes, TileLayerStream::MessageType msgType)
+{
+    std::stringstream message;
+    bitsery::Serializer<bitsery::OutputStreamAdapter> s(message);
+
+    // Write protocol version
+    s.object(CurrentProtocolVersion);
+
+    // Write message type
+    s.value1b(msgType);
+
+    // Write content length
+    s.value4b((uint32_t)bytes.size());
+
+    // Write content
+    message << bytes;
+
+    // Notify result
+    onMessage_(message.str());
+}
+
+std::shared_ptr<Fields> TileLayerStream::CachedFieldsProvider::operator()(const std::string_view& nodeId)
+{
+    auto it = fieldsPerNodeId_.find(std::string(nodeId));
+    if (it != fieldsPerNodeId_.end()) {
+        return it->second;
+    }
+    auto [newIt, _] = fieldsPerNodeId_.emplace(nodeId, std::make_shared<Fields>(std::string(nodeId)));
+    return newIt->second;
+}
+
+}
\ No newline at end of file


Test patch:
diff --git a/test/model.cpp b/test/model.cpp
index 602a94ef..0a5607f8 100644
--- a/test/model.cpp
+++ b/test/model.cpp
@@ -1,6 +1,7 @@
 #include <catch2/catch_test_macros.hpp>
 
 #include "mapget/model/featurelayer.h"
+#include "mapget/model/stream.h"
 #include "nlohmann/json.hpp"
 
 #include <sstream>
@@ -12,16 +13,17 @@ TEST_CASE("FeatureLayer", "[test.featurelayer]")
     // Create layer info which has a single feature type with
     // a single allowed feature id composition.
     auto layerInfo = std::make_shared<LayerInfo>();
-    layerInfo->featureTypes.emplace_back(FeatureTypeInfo{
+    layerInfo->featureTypes_.emplace_back(FeatureTypeInfo{
         "Way",
         {{
             UniqueIdPart{"areaId", "String which identifies the map area.", IdPartDataType::STR, false, false},
             UniqueIdPart{"wayId", "Globally Unique 32b integer.",
               IdPartDataType::U32, false, false}
         }}});
+    layerInfo->layerId_ = "WayLayer";
 
     // Create empty shared autofilled field-name dictionary
-    auto fieldNames = std::make_shared<Fields>();
+    auto fieldNames = std::make_shared<Fields>("TastyTomatoSaladNode");
 
     // Create a basic TileFeatureLayer
     auto tile = std::make_shared<TileFeatureLayer>(
@@ -76,6 +78,95 @@ TEST_CASE("FeatureLayer", "[test.featurelayer]")
             REQUIRE(feature->id()->toString() == feature1->id()->toString());
         }
     }
+
+    SECTION("Serialization")
+    {
+        std::stringstream tileBytes;
+        tile->write(tileBytes);
+
+        auto deserializedTile = std::make_shared<TileFeatureLayer>(
+            tileBytes,
+            [&](auto&& mapName, auto&& layerName){
+                REQUIRE(mapName == "GarlicChickenMap");
+                REQUIRE(layerName == "WayLayer");
+                return layerInfo;
+            },
+            [&](auto&& nodeId){
+                REQUIRE(nodeId == "TastyTomatoSaladNode");
+                return fieldNames;
+            }
+        );
+
+        REQUIRE(deserializedTile->tileId() == tile->tileId());
+        REQUIRE(deserializedTile->nodeId() == tile->nodeId());
+        REQUIRE(deserializedTile->mapId() == tile->mapId());
+        REQUIRE(deserializedTile->layerInfo() == tile->layerInfo());
+        REQUIRE(deserializedTile->error() == tile->error());
+        REQUIRE(deserializedTile->timestamp().time_since_epoch() == tile->timestamp().time_since_epoch());
+        REQUIRE(deserializedTile->ttl() == tile->ttl());
+        REQUIRE(deserializedTile->mapVersion() == tile->mapVersion());
+        REQUIRE(deserializedTile->info() == tile->info());
+
+        REQUIRE(deserializedTile->fieldNames() == tile->fieldNames());
+        for (auto feature : *deserializedTile) {
+            REQUIRE(feature->id()->toString() == feature1->id()->toString());
+        }
+    }
+
+    SECTION("Stream")
+    {
+        // We will write the same tile into the stream twice,
+        // but expect the Fields object to be sent only once.
+        // Then we add another feature with a yet unseen field, send it,
+        // and expect an update for the fields dictionary to be sent along.
+
+        auto messageCount = 0;
+        std::stringstream byteStream;
+        TileLayerStream::FieldOffsetMap fieldOffsets;
+        TileLayerStream::Writer layerWriter{[&](auto&& msg){
+            ++messageCount;
+            byteStream << msg;
+        }, fieldOffsets};
+
+        layerWriter.write(tile);
+        REQUIRE(messageCount == 2);
+        layerWriter.write(tile);
+        REQUIRE(messageCount == 3);
+
+        // Create another Feature
+        auto feature2 = tile->newFeature("Way", {{"wayId", 43}});
+        feature1->attributes()->addField("new_shiny_attr_name", "Salsa");
+
+        layerWriter.write(tile);
+        REQUIRE(messageCount == 5);
+
+        // Now, read the stream in small chunks to verify that the Reader
+        // always waits until the full message is received before trying
+        // to parse an object.
+
+        std::vector<std::shared_ptr<TileFeatureLayer>> readTiles;
+        TileLayerStream::Reader reader{
+            [&](auto&& mapId, auto&& layerId) { return layerInfo; },
+            [&](auto&& layerPtr) { readTiles.push_back(layerPtr); },
+        };
+
+        std::string byteStreamData = byteStream.str();
+        for (auto i = 0; i < byteStreamData.size(); i += 2) {
+            // Read two-byte chunks, except if only one byte is left
+            if (i < byteStreamData.size() - 1)
+                reader.read({(uint8_t)byteStreamData[i], (uint8_t)byteStreamData[i+1]});
+            else
+                reader.read({(uint8_t)byteStreamData[i]});
+        }
+
+        REQUIRE(reader.eos());
+        REQUIRE(readTiles.size() == 3);
+        REQUIRE(readTiles[0]->fieldNames() == readTiles[1]->fieldNames());
+        REQUIRE(readTiles[1]->fieldNames() == readTiles[2]->fieldNames());
+        REQUIRE(readTiles[0]->numRoots() == 1);
+        REQUIRE(readTiles[1]->numRoots() == 1);
+        REQUIRE(readTiles[2]->numRoots() == 2);
+    }
 }
 
 // Helper function to compare two points with some tolerance