fix utf8 error recovery for stream decoding
authorKubo Kovac <kuko@fb.com>
Mon, 22 May 2017 16:47:27 +0000 (17:47 +0100)
committerJakub Kovac <kuko@localhost.localdomain>
Wed, 5 Jul 2017 12:47:04 +0000 (14:47 +0200)
cbits/cbits.c
tests/Tests/Properties.hs

index 029d7e8..9f46a41 100644 (file)
@@ -207,10 +207,8 @@ _hs_text_decode_utf8_state(uint16_t *const dest, size_t *destoff,
                            uint32_t *codepoint0, uint32_t *state0)
 {
   uint8_t const *ret = _hs_text_decode_utf8_int(dest, destoff, src, srcend,
-                                               codepoint0, state0);
-  if (*state0 == UTF8_REJECT)
-    ret -=1;
-  return ret;
+                                               codepoint0, state0);
+  return *src;
 }
 
 /*
index d490438..04156d9 100644 (file)
@@ -193,6 +193,16 @@ t_decode_with_error3 =
 t_decode_with_error4 =
   E.decodeUtf8With (\_ _ -> Just 'x') (B.pack [0xF0, 97, 97, 97]) === "xaaa"
 
+t_decode_with_error2' =
+  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97]) of
+    E.Some x _ _ -> x === "xa"
+t_decode_with_error3' =
+  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97]) of
+    E.Some x _ _ -> x === "xaa"
+t_decode_with_error4' =
+  case E.streamDecodeUtf8With (\_ _ -> Just 'x') (B.pack [0xC2, 97, 97, 97]) of
+    E.Some x _ _ -> x === "xaaa"
+
 s_Eq s            = (s==)    `eq` ((S.streamList s==) . S.streamList)
     where _types = s :: String
 sf_Eq p s =
@@ -972,7 +982,10 @@ tests =
       testGroup "error recovery" [
         testProperty "t_decode_with_error2" t_decode_with_error2,
         testProperty "t_decode_with_error3" t_decode_with_error3,
-        testProperty "t_decode_with_error4" t_decode_with_error4
+        testProperty "t_decode_with_error4" t_decode_with_error4,
+        testProperty "t_decode_with_error2'" t_decode_with_error2',
+        testProperty "t_decode_with_error3'" t_decode_with_error3',
+        testProperty "t_decode_with_error4'" t_decode_with_error4'
       ]
     ],