feat(versioned): Roundtrip-track changed types

crates/stackable-versioned-macros/src/codegen/container/struct/conversion.rs

@@ -176,8 +176,11 @@ impl Struct {
         // Ideally we would integrate with the serde(rename) functionality to produce these field
         // names.
         let inserts = self.generate_tracking_inserts(direction, next_version, mod_gen_ctx);
+        let post_inserts =
+            self.generate_tracking_post_inserts(direction, next_version, mod_gen_ctx);
         let removals = self.generate_tracking_removals(direction, next_version, mod_gen_ctx);
         let json_paths = self.generate_json_paths(next_version, mod_gen_ctx);
+        let tracked_values = self.generate_tracked_values(direction, next_version, mod_gen_ctx);
 
         // TODO (@Techassi): We know the status, so we can hard-code it, but hard to track across structs
 
@@ -196,10 +199,18 @@ impl Struct {
                     // the upgrade or downgrade section. Only then we can convert the spec.
                     #inserts
 
+                    // Fields which changed their type are consumed by the conversion, so their
+                    // values need to be serialized before converting the spec.
+                    #tracked_values
+
                     let mut spec = Self {
                         #fields
                     };
 
+                    // Fields which changed their type can only be inserted into the status after
+                    // the spec is converted, because the downgraded value is tracked as well.
+                    #post_inserts
+
                     // After the spec is converted, depending on the direction, we need to apply
                     // changed values from either the upgrade or downgrade section. Afterwards
                     // we can return the successfully converted spec and the status contains
@@ -241,6 +252,12 @@ impl Struct {
                     })
                     .collect();
 
+                // This is the case if only fields which changed their type need to be tracked.
+                // These are inserted after the conversion, see generate_tracking_post_inserts.
+                if inserts.is_empty() {
+                    return None;
+                }
+
                 Some(quote! {
                     let upgrades = status
                         .changes()
@@ -254,6 +271,42 @@ impl Struct {
         }
     }
 
+    fn generate_tracking_post_inserts(
+        &self,
+        direction: Direction,
+        next_version: &VersionDefinition,
+        mod_gen_ctx: ModuleGenerationContext<'_>,
+    ) -> Option<TokenStream> {
+        match direction {
+            Direction::Upgrade => None,
+            Direction::Downgrade => {
+                let next_version_string = next_version.inner.to_string();
+
+                let post_inserts: TokenStream = self
+                    .fields
+                    .iter()
+                    .filter_map(|f| {
+                        f.generate_for_status_post_insertion(direction, next_version, mod_gen_ctx)
+                    })
+                    .collect();
+
+                if post_inserts.is_empty() {
+                    return None;
+                }
+
+                Some(quote! {
+                    let upgrades = status
+                        .changes()
+                        .upgrades
+                        .entry(#next_version_string.to_owned())
+                        .or_default();
+
+                    #post_inserts
+                })
+            }
+        }
+    }
+
     fn generate_tracking_removals(
         &self,
         direction: Direction,
@@ -274,6 +327,10 @@ impl Struct {
             Direction::Upgrade => {
                 let next_version_string = next_version.inner.to_string();
 
+                // Multiple (nested) containers can track values for the same version. As such, only
+                // the values matching the JSON paths of this container are applied. All other values
+                // are kept, so that they can be applied by the other containers. The entry for the
+                // version is only removed once all values have been applied.
                 Some(quote! {
                     if let Some(upgrades) = status.changes().upgrades.get_mut(#next_version_string) {
                         #removals
@@ -300,17 +357,37 @@ impl Struct {
             .collect()
     }
 
+    fn generate_tracked_values(
+        &self,
+        direction: Direction,
+        next_version: &VersionDefinition,
+        mod_gen_ctx: ModuleGenerationContext<'_>,
+    ) -> TokenStream {
+        let from_struct_ident = &self.common.idents.parameter;
+
+        self.fields
+            .iter()
+            .filter_map(|f| {
+                f.generate_for_tracked_value(
+                    direction,
+                    next_version,
+                    from_struct_ident,
+                    mod_gen_ctx,
+                )
+            })
+            .collect()
+    }
+
     pub(super) fn needs_tracking(&self, version: &VersionDefinition) -> bool {
         self.fields.iter().any(|f| {
             f.changes.as_ref().is_some_and(|c| {
                 c.value_is(&version.inner, |s| {
-                    // For now, only added fields need to be tracked. In the future, removals and
-                    // type changes also need to be tracked
+                    // For now, only added fields and fields which changed their type need to be
+                    // tracked. In the future, removals also need to be tracked.
                     match s {
                         ItemStatus::Addition { .. } => true,
-                        // TODO (@Techassi): Support tracking for changed fields
-                        ItemStatus::Change { .. }
-                        | ItemStatus::Deprecation { .. }
+                        ItemStatus::Change { .. } => s.is_type_change(),
+                        ItemStatus::Deprecation { .. }
                         | ItemStatus::NoChange { .. }
                         | ItemStatus::NotPresent => false,
                     }

crates/stackable-versioned-macros/src/codegen/item/field.rs

@@ -330,6 +330,89 @@ impl VersionedField {
                             upgrades.push(#versioned_path::ChangedValue {
                                 json_path: #json_path_ident,
                                 value: #serde_json_path::to_value(&#from_struct_ident.#ident).unwrap(),
+                                downgraded_value: ::core::option::Option::None,
+                            });
+                        })
+                    }
+                    _ => None,
+                }
+            }
+        }
+    }
+
+    /// Generates code which serializes the value of this field before it is converted. This is only
+    /// needed for fields which changed their type, because the conversion consumes the value.
+    ///
+    /// - When downgrading, this is the value of the newer type, which is tracked in the status.
+    /// - When upgrading, this is the value of the older type, which is compared against the tracked
+    ///   downgraded value to detect if a user changed the field in the older version.
+    pub fn generate_for_tracked_value(
+        &self,
+        direction: Direction,
+        next_version: &VersionDefinition,
+        from_struct_ident: &IdentString,
+        mod_gen_ctx: ModuleGenerationContext<'_>,
+    ) -> Option<TokenStream> {
+        let changes = self.changes.as_ref()?;
+        let next_change = changes.get_expect(&next_version.inner);
+
+        match next_change {
+            ItemStatus::Change {
+                previous_ident,
+                current_ident,
+                ..
+            } if next_change.is_type_change() => {
+                let serde_json_path = &*mod_gen_ctx.crates.serde_json;
+                let value_ident = current_ident.tracked_value_ident();
+
+                let field_ident = match direction {
+                    Direction::Upgrade => previous_ident,
+                    Direction::Downgrade => current_ident,
+                };
+
+                Some(quote! {
+                    let #value_ident = #serde_json_path::to_value(&#from_struct_ident.#field_ident).unwrap();
+                })
+            }
+            _ => None,
+        }
+    }
+
+    /// Generates code needed when a tracked type change of this field needs to be inserted into the
+    /// status. In contrast to added fields, this can only be done after the conversion, because the
+    /// downgraded value is tracked as well.
+    pub fn generate_for_status_post_insertion(
+        &self,
+        direction: Direction,
+        next_version: &VersionDefinition,
+        mod_gen_ctx: ModuleGenerationContext<'_>,
+    ) -> Option<TokenStream> {
+        let changes = self.changes.as_ref()?;
+
+        match direction {
+            Direction::Upgrade => None,
+            Direction::Downgrade => {
+                let next_change = changes.get_expect(&next_version.inner);
+
+                let serde_json_path = &*mod_gen_ctx.crates.serde_json;
+                let versioned_path = &*mod_gen_ctx.crates.versioned;
+
+                match next_change {
+                    ItemStatus::Change {
+                        previous_ident,
+                        current_ident,
+                        ..
+                    } if next_change.is_type_change() => {
+                        let json_path_ident = current_ident.json_path_ident();
+                        let value_ident = current_ident.tracked_value_ident();
+
+                        Some(quote! {
+                            upgrades.push(#versioned_path::ChangedValue {
+                                json_path: #json_path_ident,
+                                value: #value_ident,
+                                downgraded_value: ::core::option::Option::Some(
+                                    #serde_json_path::to_value(&spec.#previous_ident).unwrap()
+                                ),
                             });
                         })
                     }
@@ -359,8 +442,6 @@ impl VersionedField {
                 let next_change = changes.get_expect(&next_version.inner);
 
                 match next_change {
-                    // NOTE (@Techassi): We currently only support tracking added fields. As such
-                    // we only need to generate code if the next change is "Addition".
                     ItemStatus::Addition { ident, .. } => {
                         let json_path_ident = ident.json_path_ident();
                         let expect_message = format!(
@@ -374,6 +455,38 @@ impl VersionedField {
                             }
                         })
                     }
+                    // The tracked value is only applied if the field still contains the value it
+                    // was downgraded to. Otherwise, a user changed the field in the older version
+                    // and that change takes precedence over the tracked value.
+                    //
+                    // A downgraded value of null is serialized as `downgradedValue: null`, which is
+                    // deserialized as None. As such, a missing downgraded value is treated as null.
+                    //
+                    // There is one notable edge case:
+                    // Let's start with an enum with variants "a", "b" and "unknown" in v1, and
+                    // change that to a string for maximum flexibility in v2. Now let's say we have
+                    // a v2 object with the content "foo". It get's downgraded to v1 as "unknown".
+                    // Now something wants to change the value in v1 to "unknown" (effectively
+                    // change it from "foo" to "unknown"). When upgrading back to v2, this mechanism
+                    // only sees the old value it converted to: "unknown". It therefore restores
+                    // "foo" into v2. This example sounds a bit constructed and very unlikely. The
+                    // solution in this case would be for the editor to change it to "unknown" in
+                    // v2, that change takes effect.
+                    ItemStatus::Change { current_ident, .. } if next_change.is_type_change() => {
+                        let json_path_ident = current_ident.json_path_ident();
+                        let value_ident = current_ident.tracked_value_ident();
+                        let expect_message = format!(
+                            "internal error: {{{json_path_ident}}} must be serializable for spec.{current_ident}"
+                        );
+
+                        Some(quote! {
+                            if let Some(#versioned_path::ChangedValue { value, downgraded_value, .. }) = upgrades.extract_first_if(|u| u.json_path == #json_path_ident) {
+                                if downgraded_value.unwrap_or_default() == #value_ident {
+                                    spec.#current_ident = #serde_json_path::from_value(value).expect(#expect_message);
+                                }
+                            }
+                        })
+                    }
                     _ => None,
                 }
             }
@@ -409,17 +522,18 @@ impl VersionedField {
             (Some(changes), _) => {
                 let next_change = changes.get_expect(&next_version.inner);
 
-                match next_change {
-                    ItemStatus::Addition { ident, .. } => {
-                        let field_ident = ident.json_path_ident();
-                        let child_string = ident.to_string();
+                let ident = match next_change {
+                    ItemStatus::Addition { ident, .. } => ident,
+                    ItemStatus::Change { current_ident, .. } if next_change.is_type_change() => current_ident,
+                    _ => return None,
+                };
 
-                        Some(quote! {
-                            let #field_ident = #versioned_path::jthong_path(parent, #child_string);
-                        })
-                    }
-                    _ => None,
-                }
+                let field_ident = ident.json_path_ident();
+                let child_string = ident.to_string();
+
+                Some(quote! {
+                    let #field_ident = #versioned_path::jthong_path(parent, #child_string);
+                })
             }
         }
     }

crates/stackable-versioned-macros/src/codegen/item/mod.rs

@@ -82,4 +82,11 @@ impl ItemStatus {
             Self::NotPresent => unreachable!("ItemStatus::NotPresent does not have an ident"),
         }
     }
+
+    /// Returns `true` if this status is a change which modified the type of the item.
+    ///
+    /// Changes which only rename the item are lossless and as such don't need to be tracked.
+    pub fn is_type_change(&self) -> bool {
+        matches!(self, Self::Change { previous_type, current_type, .. } if previous_type != current_type)
+    }
 }

crates/stackable-versioned-macros/src/lib.rs

@@ -885,8 +885,8 @@ mod utils;
 ///
 /// <div class="warning">
 ///
-/// Currently, only tracking of **added** fields is supported. This will be
-/// expanded to removed fields, field type changes, and fields containing
+/// Currently, only tracking of **added** fields and **field type changes** is
+/// supported. This will be expanded to removed fields and fields containing
 /// collections in the future.
 ///
 /// </div>
@@ -932,6 +932,29 @@ mod utils;
 /// The final upgrade to `v1` will apply the tracked value for the field `baz`.
 /// Again, it is removed from the status afterwards.
 ///
+/// #### Tracking Type Changes
+///
+/// Fields which changed their type via `changed(previous_type = "...")` are tracked
+/// as well, because converting to the older type might lose data. Renaming a
+/// field without changing its type is lossless and is not tracked. In addition
+/// to the original value, the value the field was downgraded to is tracked:
+///
+/// ```yaml
+/// status:
+///   changedValues:
+///     upgrades:
+///       v1:
+///         - jsonPath: "$.gender"
+///           value: "It's complicated"
+///           downgradedValue: "Unknown"
+/// ```
+///
+/// Unlike added fields, the field also exists in the older version and can be
+/// changed by users there. When upgrading again, the tracked value is only
+/// applied if the field still contains the downgraded value. Otherwise, the
+/// change made by the user in the older version takes precedence and the
+/// tracked value is discarded.
+///
 /// ### Tracking Nested Changes
 ///
 /// To be able to automatically track values of changed fields in nested sub

crates/stackable-versioned-macros/src/utils/mod.rs

@@ -78,6 +78,7 @@ pub trait ItemIdents {
 
 pub trait ItemIdentExt {
     fn json_path_ident(&self) -> IdentString;
+    fn tracked_value_ident(&self) -> IdentString;
 }
 
 impl ItemIdentExt for IdentString {
@@ -88,6 +89,14 @@ impl ItemIdentExt for IdentString {
         )
         .into()
     }
+
+    fn tracked_value_ident(&self) -> IdentString {
+        format_ident!(
+            "__sv_{lowercase_ident}_value",
+            lowercase_ident = self.as_str().to_lowercase()
+        )
+        .into()
+    }
 }
 
 pub fn path_to_string(path: &Path) -> String {