[CBOR] Improve an error message for byte string / array type mismatch (#3052)

Author: fzhinkin

formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Decoder.kt

@@ -208,8 +208,17 @@ internal class CborParser(private val input: ByteArrayInput, private val verifyO
             skipByte(unboundedHeader)
             return -1
         }
-        if ((curByte and 0b111_00000) != boundedHeaderMask)
+        if ((curByte and 0b111_00000) != boundedHeaderMask) {
+            if (boundedHeaderMask == HEADER_ARRAY && (curByte and 0b111_00000) == HEADER_BYTE_STRING) {
+                throw CborDecodingException(
+                    "Expected a start of array, " +
+                        "but found ${printByte(curByte)}, which corresponds to the start of a byte string. " +
+                        "Make sure you correctly set 'alwaysUseByteString' setting " +
+                        "and/or 'kotlinx.serialization.cbor.ByteString' annotation."
+                )
+            }
             throw CborDecodingException("start of $collectionType", curByte)
+        }
         val size = readNumber().toInt()
         readByte()
         return size
@@ -221,8 +230,17 @@ internal class CborParser(private val input: ByteArrayInput, private val verifyO
 
     fun nextByteString(tags: ULongArray? = null): ByteArray {
         processTags(tags)
-        if ((curByte and 0b111_00000) != HEADER_BYTE_STRING)
+        if ((curByte and 0b111_00000) != HEADER_BYTE_STRING) {
+            if (curByte and 0b111_00000 == HEADER_ARRAY) {
+                throw CborDecodingException(
+                    "Expected a start of a byte string, " +
+                        "but found ${printByte(curByte)}, which corresponds to the start of an array. " +
+                        "Make sure you correctly set 'alwaysUseByteString' setting " +
+                        "and/or 'kotlinx.serialization.cbor.ByteString' annotation."
+                )
+            }
             throw CborDecodingException("start of byte string", curByte)
+        }
         val arr = readBytes()
         readByte()
         return arr

formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborDecoderTest.kt

@@ -374,4 +374,24 @@ class CborDecoderTest {
         )
     }
 
+    @Test
+    fun testMismatchedByteStringRepresentationError() {
+        @Serializable class BytesHolder(@ByteString val bytes: ByteArray)
+        @Serializable class ArrayHolder(val bytes: ByteArray)
+
+        assertFailsWithMessage<CborDecodingException>("Expected a start of array, but found 40, " +
+            "which corresponds to the start of a byte string. " +
+            "Make sure you correctly set 'alwaysUseByteString' setting " +
+            "and/or 'kotlinx.serialization.cbor.ByteString' annotation.") {
+            Cbor.decodeFromHexString<ArrayHolder>(Cbor.encodeToHexString(BytesHolder(ByteArray(0))))
+        }
+
+        assertFailsWithMessage<CborDecodingException>("Expected a start of a byte string, but found 9F, " +
+            "which corresponds to the start of an array. " +
+            "Make sure you correctly set 'alwaysUseByteString' setting " +
+            "and/or 'kotlinx.serialization.cbor.ByteString' annotation.") {
+            Cbor.decodeFromHexString<BytesHolder>(Cbor.encodeToHexString(ArrayHolder(ByteArray(0))))
+        }
+    }
+
 }