[GR-42999] Fix Marshal.dump with 4-8 bytes Integer

PullRequest: truffleruby/3582

Author: eregon

CHANGELOG.md

@@ -3,6 +3,7 @@
 Bug fixes:
 
 * Fix `rb_thread_fd_select()` to correctly initialize fdset copies and handle the timeout (@eregon).
+* Fix `Marshal.dump` when big Integer (that cannot be expressed with 4 bytes) is serialized (#2790, @andrykonchin).
 
 # 22.3.0
 

spec/ruby/core/marshal/dump_spec.rb

@@ -185,6 +185,20 @@
         [Marshal, -2**64, "\004\bl-\n\000\000\000\000\000\000\000\000\001\000"],
       ].should be_computed_by(:dump)
     end
+
+    it "increases the object links counter" do
+      obj = Object.new
+      object_1_link = "\x06" # representing of (0-based) index=1 (by adding 5 for small Integers)
+      object_2_link = "\x07" # representing of index=2
+
+      # objects: Array, Object, Object
+      Marshal.dump([obj, obj]).should == "\x04\b[\ao:\vObject\x00@#{object_1_link}"
+
+      # objects: Array, Bignum, Object, Object
+      Marshal.dump([2**64, obj, obj]).should == "\x04\b[\bl+\n\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00o:\vObject\x00@#{object_2_link}"
+      Marshal.dump([2**48, obj, obj]).should == "\x04\b[\bl+\t\x00\x00\x00\x00\x00\x00\x01\x00o:\vObject\x00@#{object_2_link}"
+      Marshal.dump([2**32, obj, obj]).should == "\x04\b[\bl+\b\x00\x00\x00\x00\x01\x00o:\vObject\x00@#{object_2_link}"
+    end
   end
 
   describe "with a String" do

src/main/ruby/truffleruby/core/marshal.rb

@@ -542,7 +542,9 @@ def const_lookup(name, type = nil)
     end
 
     def add_non_immediate_object(obj)
-      return if Primitive.immediate_value?(obj)
+      # Skip entities that cannot be referenced as objects.
+      # Integers that are bigger than 4 bytes also increase the object links counter.
+      return if Primitive.immediate_value?(obj) && !serialize_as_bignum?(obj)
       add_object(obj)
     end
 
@@ -1060,13 +1062,18 @@ def serialize_instance_variables(obj, ivars)
     end
 
     def serialize_integer(n, prefix = nil)
-      if Truffle::Type.fits_into_int?(n)
+      if !serialize_as_bignum?(n)
         Truffle::Type.binary_string(prefix.to_s + serialize_fixnum(n))
       else
         serialize_bignum(n)
       end
     end
 
+    # Integers bigger than 4 bytes are serialized in a special format
+    def serialize_as_bignum?(obj)
+      Primitive.object_kind_of?(obj, Integer) && !Truffle::Type.fits_into_int?(obj)
+    end
+
     def serialize_fixnum(n)
       if n == 0
         s = n.chr