Serialize user sortables with MessagePack (#1555)

Author: louispt1

app/models/heat_network_order.rb

@@ -16,6 +16,7 @@ class HeatNetworkOrder < ApplicationRecord
     message: 'already exists for this scenario'
   }
 
+
   def self.default_order(temperature = :mt)
     Etsource::Config.public_send("heat_network_order_#{temperature}")
   end

app/models/user_sortable.rb

@@ -7,7 +7,7 @@ module UserSortable
   extend ActiveSupport::Concern
 
   included do
-    serialize :order, type: Array
+    serialize :order, type: Array, coder: MessagePack
     belongs_to :scenario
 
     validates_with Atlas::UserSortableValidator,

config/initializers/msgpack.rb

@@ -0,0 +1,8 @@
+require 'msgpack'
+
+module MessagePack
+  class << self
+    alias_method :load, :unpack
+    alias_method :dump,  :pack
+  end
+end

db/migrate/20250417090853_serialize_user_sortables.rb

@@ -0,0 +1,41 @@
+class SerializeUserSortables < ActiveRecord::Migration[7.1]
+  require 'msgpack'
+  require 'yaml'
+
+  TARGET_MODELS = {
+    forecast_storage_orders: 'ForecastStorageOrder',
+    hydrogen_supply_orders: 'HydrogenSupplyOrder',
+    hydrogen_demand_orders: 'HydrogenDemandOrder',
+    households_space_heating_producer_orders: 'HouseholdsSpaceHeatingProducerOrder',
+    heat_network_orders: 'HeatNetworkOrder'
+  }.freeze
+
+  def up
+    TARGET_MODELS.each do |table_name, model_name|
+      say "Serializing #{model_name}#order → MessagePack binary"
+
+      rename_column table_name, :order, :order_old
+      add_column    table_name, :order, :binary, limit: 64.kilobytes
+
+      record_class = Class.new(ActiveRecord::Base) do
+        self.table_name = table_name.to_s
+      end
+      record_class.reset_column_information
+
+      record_class.find_each do |record|
+        json = record.read_attribute_before_type_cast('order_old')
+        next if json.blank? || !json.is_a?(String)
+
+        array = YAML.safe_load(
+          json,
+          permitted_classes: [Array, String, Symbol],
+          aliases: true
+        )
+        next unless array.is_a?(Array)
+
+        msgpack_blob = array.to_msgpack
+        record_class.where(id: record.id).update_all(order: msgpack_blob)
+      end
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2025_04_15_093227) do
+ActiveRecord::Schema[7.1].define(version: 2025_04_17_090853) do
   create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.string "name", limit: 191, null: false
     t.string "record_type", limit: 191, null: false
@@ -41,33 +41,38 @@
 
   create_table "forecast_storage_orders", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.integer "scenario_id"
-    t.text "order"
+    t.text "order_old"
+    t.binary "order", size: :medium
     t.index ["scenario_id"], name: "index_forecast_storage_orders_on_scenario_id", unique: true
   end
 
   create_table "heat_network_orders", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.integer "scenario_id"
-    t.text "order"
+    t.text "order_old"
     t.string "temperature", default: "mt"
+    t.binary "order", size: :medium
     t.index ["scenario_id", "temperature"], name: "index_heat_network_orders_on_scenario_id_and_temperature", unique: true
     t.index ["scenario_id"], name: "index_heat_network_orders_on_scenario_id"
   end
 
   create_table "households_space_heating_producer_orders", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.integer "scenario_id"
-    t.text "order"
+    t.text "order_old"
+    t.binary "order", size: :medium
     t.index ["scenario_id"], name: "index_households_space_heating_producer_orders_on_scenario_id", unique: true
   end
 
   create_table "hydrogen_demand_orders", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.integer "scenario_id"
-    t.text "order"
+    t.text "order_old"
+    t.binary "order", size: :medium
     t.index ["scenario_id"], name: "index_hydrogen_demand_orders_on_scenario_id", unique: true
   end
 
   create_table "hydrogen_supply_orders", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.integer "scenario_id"
-    t.text "order"
+    t.text "order_old"
+    t.binary "order", size: :medium
     t.index ["scenario_id"], name: "index_hydrogen_supply_orders_on_scenario_id", unique: true
   end