Branch: refs/heads/master
Home: https://github.com/mono/mono
Compare: https://github.com/mono/mono/compare/779127e0a5ca...394591e20315
Commit: 1b318701b99d1cd6e9f9c6305384c13fa9ef44fe
Author: Martin Potter <[email protected]> (martinpotter)
Date: 2013-10-13 21:37:45 GMT
URL: https://github.com/mono/mono/commit/1b318701b99d1cd6e9f9c6305384c13fa9ef44fe
Fixing bugs in UTF8Encoding's GetMaxByteCount and GetMaxCharCount.
The previous implementation of GetMaxByteCount has two bugs: it incorrectly took into account the byte order mark and it was taking the max of a byte count (3) and a character count (EncoderFallback.MaxCharCount). This could result in returning the max number of bytes required to convert a string that was less than the number bytes used when converting the string.
GetMaxCharCount did not previously take into account DecoderFallback.MaxCharCount or the fact that there may still be part of a surrogate pair left on the buffer from a previous call to convert.
I removed the #if NET_2_0 conditionals and NotWorking category since these methods now return results consistent with Microsoft's implementation. I also added two simple tests for custom EncoderFallback and DecoderFallback cases.
This commit is licensed under MIT/X11.
Changed paths:
M mcs/class/corlib/System.Text/UTF8Encoding.cs
M mcs/class/corlib/Test/System.Text/UTF8EncodingTest.cs
Modified: mcs/class/corlib/System.Text/UTF8Encoding.cs
===================================================================
@@ -852,8 +852,14 @@ public override int GetMaxByteCount (int charCount)
if (charCount < 0) {
throw new ArgumentOutOfRangeException ("charCount", _("ArgRange_NonNegative"));
}
- var max = Math.Max (3, EncoderFallback.MaxCharCount);
- return max * charCount + GetPreamble ().Length;
+
+ // Add 1 to charCount since there may be a lead surrogate left from the previous call to GetBytes/Encoder.Convert
+ charCount = charCount + 1;
+ if (EncoderFallback.MaxCharCount > 1) {
+ charCount = charCount * EncoderFallback.MaxCharCount;
+ }
+
+ return charCount * 3;
}
// Get the maximum number of characters needed to decode a
@@ -863,7 +869,14 @@ public override int GetMaxCharCount (int byteCount)
if (byteCount < 0) {
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_NonNegative"));
}
- return byteCount;
+
+ // Add 1 to byteCount since there may be the bytes from part of a surrogate pair left from the previous call to GetChars/Decoder.Convert
+ int maxCharCount = byteCount + 1;
+ if (DecoderFallback.MaxCharCount > 1) {
+ maxCharCount = maxCharCount * DecoderFallback.MaxCharCount;
+ }
+
+ return maxCharCount;
}
// Get a UTF8-specific decoder that is attached to this instance.
Modified: mcs/class/corlib/Test/System.Text/UTF8EncodingTest.cs
===================================================================
@@ -125,34 +125,36 @@ public void TestDecodingGetChars1()
}
[Test]
-#if NET_2_0
- [Category ("NotWorking")]
-#endif
public void TestMaxCharCount()
{
UTF8Encoding UTF8enc = new UTF8Encoding ();
-#if NET_2_0
- // hmm, where is this extra 1 coming from?
+ Encoding UTF8encWithBOM = new UTF8Encoding(true);
Assert.AreEqual (51, UTF8enc.GetMaxCharCount(50), "UTF #1");
-#else
- Assert.AreEqual (50, UTF8enc.GetMaxCharCount(50), "UTF #1");
-#endif
+ Assert.AreEqual (UTF8enc.GetMaxByteCount(50), UTF8encWithBOM.GetMaxByteCount(50), "UTF #2");
+ }
+
+ [Test]
+ public void TestMaxCharCountWithCustomFallback()
+ {
+ Encoding encoding = Encoding.GetEncoding("utf-8", new EncoderReplacementFallback("\u2047\u2047"), new DecoderReplacementFallback("\u2047\u2047"));
+ Assert.AreEqual (102, encoding.GetMaxCharCount(50), "UTF #1");
}
[Test]
-#if NET_2_0
- [Category ("NotWorking")]
-#endif
public void TestMaxByteCount()
{
UTF8Encoding UTF8enc = new UTF8Encoding ();
-#if NET_2_0
- // maybe under .NET 2.0 insufficient surrogate pair is
- // just not handled, and 3 is Preamble size.
+ Encoding UTF8encWithBOM = new UTF8Encoding(true);
+
Assert.AreEqual (153, UTF8enc.GetMaxByteCount(50), "UTF #1");
-#else
- Assert.AreEqual (200, UTF8enc.GetMaxByteCount(50), "UTF #1");
-#endif
+ Assert.AreEqual (UTF8enc.GetMaxByteCount(50), UTF8encWithBOM.GetMaxByteCount(50), "UTF #2");
+ }
+
+ [Test]
+ public void TestMaxByteCountWithCustomFallback()
+ {
+ Encoding encoding = Encoding.GetEncoding("utf-8", new EncoderReplacementFallback("\u2047\u2047"), new DecoderReplacementFallback("?"));
+ Assert.AreEqual (306, encoding.GetMaxByteCount(50), "UTF #1");
}
// regression for bug #59648
Commit: 394591e203151b57178ca15a75b8af9e89df2af1
Author: Marek Safar <[email protected]> (marek-safar)
Date: 2013-10-14 07:32:13 GMT
URL: https://github.com/mono/mono/commit/394591e203151b57178ca15a75b8af9e89df2af1
Merge pull request #779 from LogosBible/UTF8Encoding
Fixing bugs in UTF8Encoding's GetMaxByteCount and GetMaxCharCount.
Changed paths:
M mcs/class/corlib/System.Text/UTF8Encoding.cs
M mcs/class/corlib/Test/System.Text/UTF8EncodingTest.cs
Modified: mcs/class/corlib/System.Text/UTF8Encoding.cs
===================================================================
@@ -852,8 +852,14 @@ public override int GetMaxByteCount (int charCount)
if (charCount < 0) {
throw new ArgumentOutOfRangeException ("charCount", _("ArgRange_NonNegative"));
}
- var max = Math.Max (3, EncoderFallback.MaxCharCount);
- return max * charCount + GetPreamble ().Length;
+
+ // Add 1 to charCount since there may be a lead surrogate left from the previous call to GetBytes/Encoder.Convert
+ charCount = charCount + 1;
+ if (EncoderFallback.MaxCharCount > 1) {
+ charCount = charCount * EncoderFallback.MaxCharCount;
+ }
+
+ return charCount * 3;
}
// Get the maximum number of characters needed to decode a
@@ -863,7 +869,14 @@ public override int GetMaxCharCount (int byteCount)
if (byteCount < 0) {
throw new ArgumentOutOfRangeException ("byteCount", _("ArgRange_NonNegative"));
}
- return byteCount;
+
+ // Add 1 to byteCount since there may be the bytes from part of a surrogate pair left from the previous call to GetChars/Decoder.Convert
+ int maxCharCount = byteCount + 1;
+ if (DecoderFallback.MaxCharCount > 1) {
+ maxCharCount = maxCharCount * DecoderFallback.MaxCharCount;
+ }
+
+ return maxCharCount;
}
// Get a UTF8-specific decoder that is attached to this instance.
Modified: mcs/class/corlib/Test/System.Text/UTF8EncodingTest.cs
===================================================================
@@ -125,34 +125,36 @@ public void TestDecodingGetChars1()
}
[Test]
-#if NET_2_0
- [Category ("NotWorking")]
-#endif
public void TestMaxCharCount()
{
UTF8Encoding UTF8enc = new UTF8Encoding ();
-#if NET_2_0
- // hmm, where is this extra 1 coming from?
+ Encoding UTF8encWithBOM = new UTF8Encoding(true);
Assert.AreEqual (51, UTF8enc.GetMaxCharCount(50), "UTF #1");
-#else
- Assert.AreEqual (50, UTF8enc.GetMaxCharCount(50), "UTF #1");
-#endif
+ Assert.AreEqual (UTF8enc.GetMaxByteCount(50), UTF8encWithBOM.GetMaxByteCount(50), "UTF #2");
+ }
+
+ [Test]
+ public void TestMaxCharCountWithCustomFallback()
+ {
+ Encoding encoding = Encoding.GetEncoding("utf-8", new EncoderReplacementFallback("\u2047\u2047"), new DecoderReplacementFallback("\u2047\u2047"));
+ Assert.AreEqual (102, encoding.GetMaxCharCount(50), "UTF #1");
}
[Test]
-#if NET_2_0
- [Category ("NotWorking")]
-#endif
public void TestMaxByteCount()
{
UTF8Encoding UTF8enc = new UTF8Encoding ();
-#if NET_2_0
- // maybe under .NET 2.0 insufficient surrogate pair is
- // just not handled, and 3 is Preamble size.
+ Encoding UTF8encWithBOM = new UTF8Encoding(true);
+
Assert.AreEqual (153, UTF8enc.GetMaxByteCount(50), "UTF #1");
-#else
- Assert.AreEqual (200, UTF8enc.GetMaxByteCount(50), "UTF #1");
-#endif
+ Assert.AreEqual (UTF8enc.GetMaxByteCount(50), UTF8encWithBOM.GetMaxByteCount(50), "UTF #2");
+ }
+
+ [Test]
+ public void TestMaxByteCountWithCustomFallback()
+ {
+ Encoding encoding = Encoding.GetEncoding("utf-8", new EncoderReplacementFallback("\u2047\u2047"), new DecoderReplacementFallback("?"));
+ Assert.AreEqual (306, encoding.GetMaxByteCount(50), "UTF #1");
}
// regression for bug #59648
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.