Use TranscodingStream in JSON and XML formatters

- remove `ReadOnlyStreamWithEncodingPreamble` and its tests
  - was insufficient in general and doesn't work correctly in `netstandard1.3`
  - see dotnet/runtime#80160

Now support a much broader set of encodings in JSON and XML formatters
- not just UTF8, UTF16BE, and UTF16LE
- JSON formatter already supported every encoding when `UseDataContractJsonSerializer` was `false`
  - no longer have this downside when property is `true`
  - however, quotas remain unsupported in `netstandard1.3` version of this formatter when property is `true`
- XML formatter requirement for an XML declaration in non-UTF8 content was general and is no longer an issue

Author: dougbu

src/System.Net.Http.Formatting/Formatting/JsonMediaTypeFormatter.cs

@@ -22,9 +22,9 @@ namespace System.Net.Http.Formatting
     /// </summary>
     public class JsonMediaTypeFormatter : BaseJsonMediaTypeFormatter
     {
-        private ConcurrentDictionary<Type, DataContractJsonSerializer> _dataContractSerializerCache = new ConcurrentDictionary<Type, DataContractJsonSerializer>();
-        private XmlDictionaryReaderQuotas _readerQuotas = FormattingUtilities.CreateDefaultReaderQuotas();
-        private RequestHeaderMapping _requestHeaderMapping;
+        private readonly ConcurrentDictionary<Type, DataContractJsonSerializer> _dataContractSerializerCache = new ConcurrentDictionary<Type, DataContractJsonSerializer>();
+        private readonly XmlDictionaryReaderQuotas _readerQuotas = FormattingUtilities.CreateDefaultReaderQuotas();
+        private readonly RequestHeaderMapping _requestHeaderMapping;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="JsonMediaTypeFormatter"/> class.
@@ -216,17 +216,23 @@ public override object ReadFromStream(Type type, Stream readStream, Encoding eff
             {
                 DataContractJsonSerializer dataContractSerializer = GetDataContractSerializer(type);
 
-#if NETFX_CORE // JsonReaderWriterFactory is internal in netstandard1.3. Unfortunately, ignoring _readerQuotas.
-                // Force a preamble into the stream since DataContractJsonSerializer only supports auto-detecting
-                // encoding in netstandard1.3 ].
-                readStream = new ReadOnlyStreamWithEncodingPreamble(readStream, effectiveEncoding);
+                // JsonReaderWriterFactory is internal, CreateTextReader only supports auto-detecting the encoding
+                // and auto-detection fails in some cases for the NETFX_CORE project. In addition, DCS encodings are
+                // limited to UTF8, UTF16BE, and UTF16LE. Convert to UTF8 as we read.
+                Stream innerStream = string.Equals(effectiveEncoding.WebName, Utf8Encoding.WebName, StringComparison.OrdinalIgnoreCase) ?
+                    new NonClosingDelegatingStream(readStream) :
+                    new TranscodingStream(readStream, effectiveEncoding, Utf8Encoding, leaveOpen: true);
 
-                return dataContractSerializer.ReadObject(new NonClosingDelegatingStream(readStream));
-#else
-                using (XmlReader reader = JsonReaderWriterFactory.CreateJsonReader(new NonClosingDelegatingStream(readStream), effectiveEncoding, _readerQuotas, null))
+#if NETFX_CORE
+                using (innerStream)
                 {
-                    return dataContractSerializer.ReadObject(reader);
+                    // Unfortunately, we're ignoring _readerQuotas.
+                    return dataContractSerializer.ReadObject(innerStream);
                 }
+#else
+                // XmlReader will always dispose of innerStream when we dispose of the reader.
+                using var reader = JsonReaderWriterFactory.CreateJsonReader(innerStream, Utf8Encoding, _readerQuotas, onClose: null);
+                return dataContractSerializer.ReadObject(reader);
 #endif
             }
             else
@@ -277,14 +283,6 @@ public override void WriteToStream(Type type, object value, Stream writeStream,
 
             if (UseDataContractJsonSerializer)
             {
-#if NETFX_CORE // DataContractJsonSerializer writes only UTF8 in netstandard1.3. Later versions of (now public)
-               // JsonReaderWriterFactory can compensate.
-                if (!string.Equals(Encoding.UTF8.WebName, effectiveEncoding.WebName, StringComparison.OrdinalIgnoreCase))
-                {
-                    throw new NotSupportedException("!!! To be added !!!");
-                }
-#endif
-
                 if (MediaTypeFormatter.TryGetDelegatingTypeForIQueryableGenericOrSame(ref type))
                 {
                     if (value != null)
@@ -293,23 +291,39 @@ public override void WriteToStream(Type type, object value, Stream writeStream,
                     }
                 }
 
-                DataContractJsonSerializer dataContractSerializer = GetDataContractSerializer(type);
-
-#if NETFX_CORE // JsonReaderWriterFactory is internal in netstandard1.3.
-                dataContractSerializer.WriteObject(writeStream, value);
-#else
-                using (XmlWriter writer = JsonReaderWriterFactory.CreateJsonWriter(writeStream, effectiveEncoding, ownsStream: false))
+                WritePreamble(writeStream, effectiveEncoding);
+                if (string.Equals(effectiveEncoding.WebName, Utf8Encoding.WebName, StringComparison.OrdinalIgnoreCase))
                 {
-                    dataContractSerializer.WriteObject(writer, value);
+                    WriteObject(writeStream, type, value);
+                }
+                else
+                {
+                    // JsonReaderWriterFactory is internal and DataContractJsonSerializer only writes UTF8 for the
+                    // NETFX_CORE project. In addition, DCS encodings are limited to UTF8, UTF16BE, and UTF16LE.
+                    // Convert to UTF8 as we write.
+                    using var innerStream = new TranscodingStream(writeStream, effectiveEncoding, Utf8Encoding, leaveOpen: true);
+                    WriteObject(innerStream, type, value);
                 }
-#endif
             }
             else
             {
                 base.WriteToStream(type, value, writeStream, effectiveEncoding);
             }
         }
 
+        private void WriteObject(Stream stream, Type type, object value)
+        {
+            DataContractJsonSerializer dataContractSerializer = GetDataContractSerializer(type);
+
+            // Do not dispose of the stream. WriteToStream handles that where it's needed.
+#if NETFX_CORE
+            dataContractSerializer.WriteObject(stream, value);
+#else
+            using XmlWriter writer = JsonReaderWriterFactory.CreateJsonWriter(stream, Utf8Encoding, ownsStream: false);
+            dataContractSerializer.WriteObject(writer, value);
+#endif
+        }
+
         [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Catch all is around an extensibile method")]
         private DataContractJsonSerializer CreateDataContractSerializer(Type type, bool throwOnError)
         {

src/System.Net.Http.Formatting/Formatting/MediaTypeFormatter.cs

@@ -20,6 +20,8 @@ namespace System.Net.Http.Formatting
     /// </summary>
     public abstract class MediaTypeFormatter
     {
+        private protected static readonly Encoding Utf8Encoding =
+            new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
         private const int DefaultMinHttpCollectionKeys = 1;
         private const int DefaultMaxHttpCollectionKeys = 1000; // same default as ASPNET
         private const string IWellKnownComparerTypeName = "System.IWellKnownStringEqualityComparer, mscorlib, Version=4.0.0.0, PublicKeyToken=b77a5c561934e089";
@@ -499,6 +501,15 @@ public static object GetDefaultValueForType(Type type)
             return null;
         }
 
+        private protected static void WritePreamble(Stream stream, Encoding encoding)
+        {
+            byte[] bytes = encoding.GetPreamble();
+            if (bytes.Length > 0)
+            {
+                stream.Write(bytes, 0, bytes.Length);
+            }
+        }
+
         /// <summary>
         /// Collection class that validates it contains only <see cref="MediaTypeHeaderValue"/> instances
         /// that are not null and not media ranges.

src/System.Net.Http.Formatting/Formatting/XmlMediaTypeFormatter.cs

@@ -337,12 +337,14 @@ protected internal virtual XmlReader CreateXmlReader(Stream readStream, HttpCont
         {
             // Get the character encoding for the content
             Encoding effectiveEncoding = SelectCharacterEncoding(content == null ? null : content.Headers);
-#if NETFX_CORE
-            // Force a preamble into the stream, since CreateTextReader in WinRT only supports auto-detecting encoding.
-            return XmlDictionaryReader.CreateTextReader(new ReadOnlyStreamWithEncodingPreamble(readStream, effectiveEncoding), _readerQuotas);
-#else
-            return XmlDictionaryReader.CreateTextReader(new NonClosingDelegatingStream(readStream), effectiveEncoding, _readerQuotas, null);
-#endif
+
+            // DCS encodings are limited to UTF8, UTF16BE, and UTF16LE. Convert to UTF8 as we read.
+            Stream innerStream = string.Equals(effectiveEncoding.WebName, Utf8Encoding.WebName, StringComparison.OrdinalIgnoreCase) ?
+                new NonClosingDelegatingStream(readStream) :
+                new TranscodingStream(readStream, effectiveEncoding, Utf8Encoding, leaveOpen: true);
+
+            // XmlReader will always dispose of innerStream when caller disposes of the reader.
+            return XmlDictionaryReader.CreateTextReader(innerStream, Utf8Encoding, _readerQuotas, onClose: null);
         }
 
         /// <inheritdoc/>
@@ -433,9 +435,20 @@ protected internal virtual object GetSerializer(Type type, object value, HttpCon
         protected internal virtual XmlWriter CreateXmlWriter(Stream writeStream, HttpContent content)
         {
             Encoding effectiveEncoding = SelectCharacterEncoding(content != null ? content.Headers : null);
+            WritePreamble(writeStream, effectiveEncoding);
+
+            // DCS encodings are limited to UTF8, UTF16BE, and UTF16LE. Convert to UTF8 as we read.
+            Stream innerStream = string.Equals(effectiveEncoding.WebName, Utf8Encoding.WebName, StringComparison.OrdinalIgnoreCase) ?
+                writeStream :
+                new TranscodingStream(writeStream, effectiveEncoding, Utf8Encoding, leaveOpen: true);
+
             XmlWriterSettings writerSettings = WriterSettings.Clone();
-            writerSettings.Encoding = effectiveEncoding;
-            return XmlWriter.Create(writeStream, writerSettings);
+            writerSettings.Encoding = Utf8Encoding;
+
+            // Have XmlWriter dispose of innerStream when caller disposes of the writer if using a TranscodingStream.
+            writerSettings.CloseOutput = writeStream != innerStream;
+
+            return XmlWriter.Create(innerStream, writerSettings);
         }
 
         /// <summary>

src/System.Net.Http.Formatting/Internal/ReadOnlyStreamWithEncodingPreamble.cs

@@ -218,12 +218,6 @@ public void UseDataContractJsonSerializer_True_Indent_Throws()
 
         public override Task ReadFromStreamAsync_UsesCorrectCharacterEncoding(string content, string encoding, bool isDefaultEncoding)
         {
-            if (!isDefaultEncoding)
-            {
-                // XmlDictionaryReader/Writer only supports utf-8 and 16
-                return TaskHelpers.Completed();
-            }
-
             // Arrange
             DataContractJsonMediaTypeFormatter formatter = new DataContractJsonMediaTypeFormatter();
             string formattedContent = "\"" + content + "\"";
@@ -236,14 +230,6 @@ public override Task ReadFromStreamAsync_UsesCorrectCharacterEncoding(string con
 
         public override Task WriteToStreamAsync_UsesCorrectCharacterEncoding(string content, string encoding, bool isDefaultEncoding)
         {
-            // DataContractJsonSerializer does not honor the value of byteOrderMark in the UnicodeEncoding ctor.
-            // It doesn't include the BOM when byteOrderMark is set to true.
-            if (!isDefaultEncoding || encoding != "utf-8")
-            {
-                // XmlDictionaryReader/Writer only supports utf-8 and 16
-                return TaskHelpers.Completed();
-            }
-
             // Arrange
             DataContractJsonMediaTypeFormatter formatter = new DataContractJsonMediaTypeFormatter();
             string formattedContent = "\"" + content + "\"";