From d9c61dd60ec484909f70b7a916ada3a93af94b60 Mon Sep 17 00:00:00 2001
From: Erik Larsson <mechie@users.sourceforge.net>
Date: Fri, 8 Apr 2016 05:39:48 +0200
Subject: [PATCH 1/2] unistr.c: Enable encoding broken UTF-16 into broken
UTF-8, A.K.A. WTF-8.
Windows filenames may contain invalid UTF-16 sequences (specifically
broken surrogate pairs), which cannot be converted to UTF-8 if we do
strict conversion.
This patch enables encoding broken UTF-16 into similarly broken UTF-8 by
encoding any surrogate character that don't have a match into a separate
3-byte UTF-8 sequence.
This is "sort of" valid UTF-8, but not valid Unicode since the code
points used for surrogate pair encoding are not supposed to occur in a
valid Unicode string... but on the other hand the source UTF-16 data is
also broken, so we aren't really making things any worse.
This format is sometimes referred to as WTF-8 (Wobbly Translation
Format, 8-bit encoding) and is a common solution to represent broken
UTF-16 as UTF-8.
It is a lossless round-trip conversion, i.e converting from broken
UTF-16 to "WTF-8" and back to UTF-16 yields the same broken UTF-16
sequence. Because of this property it enables accessing these files
by filename through ntfs-3g and the ntfsprogs (e.g. ls -la works as
expected).
To disable this behaviour you can pass the preprocessor/compiler flag
'-DALLOW_BROKEN_SURROGATES=0' when building ntfs-3g.
---
libntfs-3g/unistr.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 65 insertions(+), 2 deletions(-)
diff --git a/libntfs-3g/unistr.c b/libntfs-3g/unistr.c
index 7f278cd..71802aa 100644
--- a/libntfs-3g/unistr.c
+++ b/libntfs-3g/unistr.c
@@ -61,6 +61,11 @@
#define NOREVBOM 0 /* JPA rejecting U+FFFE and U+FFFF, open to debate */
+#ifndef ALLOW_BROKEN_SURROGATES
+/* Erik allowing broken UTF-16 surrogate pairs by default, open to debate. */
+#define ALLOW_BROKEN_SURROGATES 1
+#endif /* !defined(ALLOW_BROKEN_SURROGATES) */
+
/*
* IMPORTANT
* =========
@@ -462,8 +467,22 @@ static int utf16_to_utf8_size(const ntfschar *ins, const int ins_len, int outs_l
if ((c >= 0xdc00) && (c < 0xe000)) {
surrog = FALSE;
count += 4;
- } else
+ } else {
+#if ALLOW_BROKEN_SURROGATES
+ /* The first UTF-16 unit of a surrogate pair has
+ * a value between 0xd800 and 0xdc00. It can be
+ * encoded as an individual UTF-8 sequence if we
+ * cannot combine it with the next UTF-16 unit
+ * unit as a surrogate pair. */
+ surrog = FALSE;
+ count += 3;
+
+ --i;
+ continue;
+#else
goto fail;
+#endif /* ALLOW_BROKEN_SURROGATES */
+ }
} else
if (c < 0x80)
count++;
@@ -473,6 +492,10 @@ static int utf16_to_utf8_size(const ntfschar *ins, const int ins_len, int outs_l
count += 3;
else if (c < 0xdc00)
surrog = TRUE;
+#if ALLOW_BROKEN_SURROGATES
+ else if (c < 0xe000)
+ count += 3;
+#endif /* ALLOW_BROKEN_SURROGATES */
#if NOREVBOM
else if ((c >= 0xe000) && (c < 0xfffe))
#else
@@ -487,7 +510,11 @@ static int utf16_to_utf8_size(const ntfschar *ins, const int ins_len, int outs_l
}
}
if (surrog)
+#if ALLOW_BROKEN_SURROGATES
+ count += 3; /* ending with a single surrogate */
+#else
goto fail;
+#endif /* ALLOW_BROKEN_SURROGATES */
ret = count;
out:
@@ -548,8 +575,24 @@ static int ntfs_utf16_to_utf8(const ntfschar *ins, const int ins_len,
*t++ = 0x80 + ((c >> 6) & 15) + ((halfpair & 3) << 4);
*t++ = 0x80 + (c & 63);
halfpair = 0;
- } else
+ } else {
+#if ALLOW_BROKEN_SURROGATES
+ /* The first UTF-16 unit of a surrogate pair has
+ * a value between 0xd800 and 0xdc00. It can be
+ * encoded as an individual UTF-8 sequence if we
+ * cannot combine it with the next UTF-16 unit
+ * unit as a surrogate pair. */
+ *t++ = 0xe0 | (halfpair >> 12);
+ *t++ = 0x80 | ((halfpair >> 6) & 0x3f);
+ *t++ = 0x80 | (halfpair & 0x3f);
+ halfpair = 0;
+
+ --i;
+ continue;
+#else
goto fail;
+#endif /* ALLOW_BROKEN_SURROGATES */
+ }
} else if (c < 0x80) {
*t++ = c;
} else {
@@ -562,6 +605,13 @@ static int ntfs_utf16_to_utf8(const ntfschar *ins, const int ins_len,
*t++ = 0x80 | (c & 0x3f);
} else if (c < 0xdc00)
halfpair = c;
+#if ALLOW_BROKEN_SURROGATES
+ else if (c < 0xe000) {
+ *t++ = 0xe0 | (c >> 12);
+ *t++ = 0x80 | ((c >> 6) & 0x3f);
+ *t++ = 0x80 | (c & 0x3f);
+ }
+#endif /* ALLOW_BROKEN_SURROGATES */
else if (c >= 0xe000) {
*t++ = 0xe0 | (c >> 12);
*t++ = 0x80 | ((c >> 6) & 0x3f);
@@ -570,6 +620,13 @@ static int ntfs_utf16_to_utf8(const ntfschar *ins, const int ins_len,
goto fail;
}
}
+#if ALLOW_BROKEN_SURROGATES
+ if (halfpair) { /* ending with a single surrogate */
+ *t++ = 0xe0 | (halfpair >> 12);
+ *t++ = 0x80 | ((halfpair >> 6) & 0x3f);
+ *t++ = 0x80 | (halfpair & 0x3f);
+ }
+#endif /* ALLOW_BROKEN_SURROGATES */
*t = '\0';
#if defined(__APPLE__) || defined(__DARWIN__)
@@ -693,10 +750,16 @@ static int utf8_to_unicode(u32 *wc, const char *s)
/* Check valid ranges */
#if NOREVBOM
if (((*wc >= 0x800) && (*wc <= 0xD7FF))
+#if ALLOW_BROKEN_SURROGATES
+ || ((*wc >= 0xD800) && (*wc <= 0xDFFF))
+#endif /* ALLOW_BROKEN_SURROGATES */
|| ((*wc >= 0xe000) && (*wc <= 0xFFFD)))
return 3;
#else
if (((*wc >= 0x800) && (*wc <= 0xD7FF))
+#if ALLOW_BROKEN_SURROGATES
+ || ((*wc >= 0xD800) && (*wc <= 0xDFFF))
+#endif /* ALLOW_BROKEN_SURROGATES */
|| ((*wc >= 0xe000) && (*wc <= 0xFFFF)))
return 3;
#endif
--
2.10.2