Added option for enabling backwards compatiblity decoding for obvious use cases (i.e. has a .)

Author: passuied

codec.go

@@ -49,6 +49,15 @@ type CodecOption struct {
 	// When true, the string literal "null" in textual Avro data will be coerced to Go's nil.
 	// Primarily used to handle edge cases where some Avro implementations allow string representations of null.
 	EnableStringNull bool
+
+	// EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding controls backwards compatibility for decimal
+	// binary decoding. When true, the decoder will first check if binary bytes look like an
+	// ASCII decimal string (e.g., "40.20") and parse them as such. This handles legacy data
+	// that was incorrectly encoded as ASCII strings instead of two's-complement bytes.
+	// WARNING: This can cause incorrect decoding if a valid two's-complement value happens
+	// to look like ASCII digits. Only enable this if you have legacy incorrectly-encoded data.
+	// Default: false (use correct two's-complement decoding only)
+	EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding bool
 }
 
 // Codec supports decoding binary and text Avro data to Go native data types,
@@ -83,6 +92,7 @@ type codecBuilder struct {
 func DefaultCodecOption() *CodecOption {
 	return &CodecOption{
 		EnableStringNull: true,
+		EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding: false,
 	}
 }
 
@@ -739,9 +749,9 @@ func buildCodecForTypeDescribedByString(st map[string]*Codec, enclosingNamespace
 	case "record":
 		return makeRecordCodec(st, enclosingNamespace, schemaMap, cb)
 	case "bytes.decimal":
-		return makeDecimalBytesCodec(st, enclosingNamespace, schemaMap)
+		return makeDecimalBytesCodec(st, enclosingNamespace, schemaMap, cb)
 	case "fixed.decimal":
-		return makeDecimalFixedCodec(st, enclosingNamespace, schemaMap)
+		return makeDecimalFixedCodec(st, enclosingNamespace, schemaMap, cb)
 	case "string.validated-string":
 		return makeValidatedStringCodec(st, enclosingNamespace, schemaMap)
 	default:

logical_type.go

@@ -258,7 +258,7 @@ func precisionAndScaleFromSchemaMap(schemaMap map[string]interface{}) (int, int,
 
 var one = big.NewInt(1)
 
-func makeDecimalBytesCodec(st map[string]*Codec, enclosingNamespace string, schemaMap map[string]interface{}) (*Codec, error) {
+func makeDecimalBytesCodec(st map[string]*Codec, enclosingNamespace string, schemaMap map[string]interface{}, cb *codecBuilder) (*Codec, error) {
 	precision, scale, err := precisionAndScaleFromSchemaMap(schemaMap)
 	if err != nil {
 		return nil, err
@@ -275,16 +275,20 @@ func makeDecimalBytesCodec(st map[string]*Codec, enclosingNamespace string, sche
 	decimalSearchType := fmt.Sprintf("bytes.decimal.%d.%d", precision, scale)
 	st[decimalSearchType] = c
 
+	// Check if backwards compatibility mode is enabled
+	enableBackwardsCompat := cb != nil && cb.option != nil && cb.option.EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding
+
 	c.binaryFromNative = decimalBytesFromNative(bytesBinaryFromNative, toSignedBytes, precision, scale)
 	c.textualFromNative = decimalTextualFromNative(scale)
-	c.nativeFromBinary = nativeFromDecimalBytes(bytesNativeFromBinary, scale)
+	c.nativeFromBinary = nativeFromDecimalBytes(bytesNativeFromBinary, scale, enableBackwardsCompat)
 	c.nativeFromTextual = nativeFromDecimalTextual()
 	return c, nil
 }
 
-// nativeFromDecimalBytes decodes bytes to *big.Rat with backwards compatibility
-// for incorrectly encoded ASCII decimal strings.
-func nativeFromDecimalBytes(fn toNativeFn, scale int) toNativeFn {
+// nativeFromDecimalBytes decodes bytes to *big.Rat.
+// If enableBackwardsCompat is true, it first checks if bytes look like an ASCII decimal
+// string (for backwards compatibility with incorrectly encoded legacy data).
+func nativeFromDecimalBytes(fn toNativeFn, scale int, enableBackwardsCompat bool) toNativeFn {
 	return func(bytes []byte) (interface{}, []byte, error) {
 		d, b, err := fn(bytes)
 		if err != nil {
@@ -296,7 +300,7 @@ func nativeFromDecimalBytes(fn toNativeFn, scale int) toNativeFn {
 		}
 
 		// Check if bytes look like ASCII decimal string (backwards compat)
-		if looksLikeASCIIDecimal(bs) {
+		if enableBackwardsCompat && looksLikeASCIIDecimal(bs) {
 			r := new(big.Rat)
 			if _, ok := r.SetString(string(bs)); ok {
 				return r, b, nil
@@ -362,7 +366,9 @@ func nativeFromDecimalTextual() toNativeFn {
 
 // looksLikeASCIIDecimal checks if the bytes look like an ASCII decimal string
 // (for backwards compatibility with incorrectly encoded data).
-// Returns true if all bytes are printable ASCII and form a valid decimal pattern.
+// Returns true if all bytes are printable ASCII, form a valid decimal pattern,
+// and contain a decimal point. Requiring a decimal point reduces false positives
+// since 0x2E ('.') is unlikely to appear in valid two's-complement encoded numbers.
 func looksLikeASCIIDecimal(bs []byte) bool {
 	if len(bs) == 0 {
 		return false
@@ -386,10 +392,11 @@ func looksLikeASCIIDecimal(bs []byte) bool {
 			return false // non-decimal character
 		}
 	}
-	return hasDigit
+	// Require both a digit and a decimal point to reduce false positives
+	return hasDigit && hasDot
 }
 
-func makeDecimalFixedCodec(st map[string]*Codec, enclosingNamespace string, schemaMap map[string]interface{}) (*Codec, error) {
+func makeDecimalFixedCodec(st map[string]*Codec, enclosingNamespace string, schemaMap map[string]interface{}, cb *codecBuilder) (*Codec, error) {
 	precision, scale, err := precisionAndScaleFromSchemaMap(schemaMap)
 	if err != nil {
 		return nil, err
@@ -405,9 +412,13 @@ func makeDecimalFixedCodec(st map[string]*Codec, enclosingNamespace string, sche
 	if err != nil {
 		return nil, err
 	}
+
+	// Check if backwards compatibility mode is enabled
+	enableBackwardsCompat := cb != nil && cb.option != nil && cb.option.EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding
+
 	c.binaryFromNative = decimalBytesFromNative(c.binaryFromNative, toSignedFixedBytes(size), precision, scale)
 	c.textualFromNative = decimalTextualFromNative(scale)
-	c.nativeFromBinary = nativeFromDecimalBytes(c.nativeFromBinary, scale)
+	c.nativeFromBinary = nativeFromDecimalBytes(c.nativeFromBinary, scale, enableBackwardsCompat)
 	c.nativeFromTextual = nativeFromDecimalTextual()
 	return c, nil
 }

logical_type_test.go

@@ -276,15 +276,18 @@ func TestDecimalFixedTextualRoundTrip(t *testing.T) {
 
 func TestDecimalBytesBackwardsCompatibility(t *testing.T) {
 	// Test that binary data incorrectly encoded as ASCII decimal strings
-	// can still be decoded correctly (backwards compatibility)
+	// can still be decoded correctly when backwards compatibility is enabled
 	schema := `{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}`
-	codec, err := NewCodec(schema)
+
+	// Create codec with backwards compatibility enabled
+	opt := &CodecOption{EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding: true}
+	codec, err := NewCodecWithOptions(schema, opt)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Simulate incorrectly encoded data: "40.20" as ASCII bytes
-	// Length prefix (10 = 0x14 in varint) + ASCII bytes for "40.20"
+	// Length prefix (10 = 0x0a in varint) + ASCII bytes for "40.20"
 	incorrectlyEncodedBytes := append([]byte{0x0a}, []byte("40.20")...)
 
 	native, _, err := codec.NativeFromBinary(incorrectlyEncodedBytes)
@@ -386,12 +389,14 @@ func TestLooksLikeASCIIDecimal(t *testing.T) {
 		{[]byte("40.20"), true},
 		{[]byte("-40.20"), true},
 		{[]byte("+40.20"), true},
-		{[]byte("0"), true},
-		{[]byte("123456"), true},
 		{[]byte(".5"), true},
 		{[]byte("5."), true},
-		{[]byte(""), false},
-		{[]byte("-"), false},
+		{[]byte("0.0"), true},
+		{[]byte("0"), false},        // no decimal point - could be valid two's complement
+		{[]byte("123456"), false},   // no decimal point - could be valid two's complement
+		{[]byte(""), false},         // empty
+		{[]byte("-"), false},        // no digit
+		{[]byte("."), false},        // no digit
 		{[]byte("40.20.30"), false}, // multiple dots
 		{[]byte("40-20"), false},    // sign not at start
 		{[]byte("40a20"), false},    // non-decimal char
@@ -409,10 +414,46 @@ func TestLooksLikeASCIIDecimal(t *testing.T) {
 	}
 }
 
+func TestDecimalDefaultNoBackwardsCompat(t *testing.T) {
+	// Test that by default, ASCII-looking bytes are NOT interpreted as ASCII
+	// but rather as two's-complement (which is the correct spec behavior)
+	schema := `{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}`
+
+	// Create codec with default options (no backwards compat)
+	codec, err := NewCodec(schema)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// "09" as ASCII bytes is [0x30, 0x39] which is 12345 in decimal
+	// This could be misinterpreted as the string "09" if backwards compat were on
+	// But with default settings, it should be decoded as 12345/100 = 123.45
+	asciiLookingBytes := []byte{0x04, 0x30, 0x39} // length=2, bytes="09"
+
+	native, _, err := codec.NativeFromBinary(asciiLookingBytes)
+	if err != nil {
+		t.Fatalf("NativeFromBinary: %v", err)
+	}
+
+	rat, ok := native.(*big.Rat)
+	if !ok {
+		t.Fatalf("NativeFromBinary: expected *big.Rat, got %T", native)
+	}
+
+	// 0x3039 = 12345, with scale 2 = 123.45 = 12345/100
+	expected := big.NewRat(12345, 100)
+	if rat.Cmp(expected) != 0 {
+		t.Errorf("NativeFromBinary (default, no backwards compat): got %v, want %v", rat, expected)
+	}
+}
+
 func TestDecimalNegativeBackwardsCompatibility(t *testing.T) {
 	// Test backwards compatibility with negative numbers encoded as ASCII
 	schema := `{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}`
-	codec, err := NewCodec(schema)
+
+	// Create codec with backwards compatibility enabled
+	opt := &CodecOption{EnableDecimalBinaryToTextualBackwardsCompatASCIIDecoding: true}
+	codec, err := NewCodecWithOptions(schema, opt)
 	if err != nil {
 		t.Fatal(err)
 	}