d3767b
From e87ab5b991b08092a7e07af82b3ec822a8604151 Mon Sep 17 00:00:00 2001
d3767b
From: Ondrej Oprala <ooprala@redhat.com>
d3767b
Date: Wed, 5 Aug 2015 09:15:09 +0200
d3767b
Subject: [PATCH] expand,unexpand: add multibyte support
d3767b
MIME-Version: 1.0
d3767b
Content-Type: text/plain; charset=UTF-8
d3767b
Content-Transfer-Encoding: 8bit
d3767b
d3767b
* NEWS: Mention the changes.
d3767b
* bootstrap.conf: Add mbfile to the list of modules.
d3767b
* configure.ac: Properly initialize mbfile.
d3767b
* src/expand.c (expand): Iterate over multibyte characters properly.
d3767b
* src/unexpand.c (unexpand): Iterate over multibyte characters
d3767b
properly.
d3767b
* tests/local.mk: Add new tests.
d3767b
* tests/{expand,unexpand}/mb.sh: New tests.
d3767b
d3767b
Co-authored-by: Pádraig Brady <pbrady@redhat.com>
d3767b
---
d3767b
 bootstrap.conf       |   1 +
d3767b
 configure.ac         |   2 +
d3767b
 lib/mbfile.c         |   3 +
d3767b
 lib/mbfile.h         | 255 +++++++++++++++++++++++++++++++++++++++++++++++++++
d3767b
 m4/mbfile.m4         |  14 +++
d3767b
 src/expand.c         |  43 +++++----
d3767b
 src/unexpand.c       |  54 +++++++----
d3767b
 tests/expand/mb.sh   |  98 ++++++++++++++++++++
d3767b
 tests/local.mk       |   2 +
d3767b
 tests/unexpand/mb.sh |  97 ++++++++++++++++++++
d3767b
 10 files changed, 535 insertions(+), 34 deletions(-)
d3767b
 create mode 100644 lib/mbfile.c
d3767b
 create mode 100644 lib/mbfile.h
d3767b
 create mode 100644 m4/mbfile.m4
d3767b
 create mode 100755 tests/expand/mb.sh
d3767b
 create mode 100755 tests/unexpand/mb.sh
d3767b
d3767b
diff --git a/bootstrap.conf b/bootstrap.conf
d3767b
index 8a0ff31..a1c78b2 100644
d3767b
--- a/bootstrap.conf
d3767b
+++ b/bootstrap.conf
d3767b
@@ -152,6 +152,7 @@ gnulib_modules="
d3767b
   maintainer-makefile
d3767b
   malloc-gnu
d3767b
   manywarnings
d3767b
+  mbfile
d3767b
   mbrlen
d3767b
   mbrtowc
d3767b
   mbsalign
d3767b
diff --git a/configure.ac b/configure.ac
d3767b
index 1e74b36..24c9725 100644
d3767b
--- a/configure.ac
d3767b
+++ b/configure.ac
d3767b
@@ -427,6 +427,8 @@ fi
d3767b
 # I'm leaving it here for now.  This whole thing needs to be modernized...
d3767b
 gl_WINSIZE_IN_PTEM
d3767b
 
d3767b
+gl_MBFILE
d3767b
+
d3767b
 gl_HEADER_TIOCGWINSZ_IN_TERMIOS_H
d3767b
 
d3767b
 if test $gl_cv_sys_tiocgwinsz_needs_termios_h = no && \
d3767b
diff --git a/lib/mbfile.c b/lib/mbfile.c
d3767b
new file mode 100644
d3767b
index 0000000..b0a468e
d3767b
--- /dev/null
d3767b
+++ b/lib/mbfile.c
d3767b
@@ -0,0 +1,3 @@
d3767b
+#include <config.h>
d3767b
+#define MBFILE_INLINE _GL_EXTERN_INLINE
d3767b
+#include "mbfile.h"
d3767b
diff --git a/lib/mbfile.h b/lib/mbfile.h
d3767b
new file mode 100644
d3767b
index 0000000..11f1b12
d3767b
--- /dev/null
d3767b
+++ b/lib/mbfile.h
d3767b
@@ -0,0 +1,255 @@
d3767b
+/* Multibyte character I/O: macros for multi-byte encodings.
d3767b
+   Copyright (C) 2001, 2005, 2009-2015 Free Software Foundation, Inc.
d3767b
+
d3767b
+   This program is free software: you can redistribute it and/or modify
d3767b
+   it under the terms of the GNU General Public License as published by
d3767b
+   the Free Software Foundation; either version 3 of the License, or
d3767b
+   (at your option) any later version.
d3767b
+
d3767b
+   This program is distributed in the hope that it will be useful,
d3767b
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
d3767b
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
d3767b
+   GNU General Public License for more details.
d3767b
+
d3767b
+   You should have received a copy of the GNU General Public License
d3767b
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
d3767b
+
d3767b
+/* Written by Mitsuru Chinen <mchinen@yamato.ibm.com>
d3767b
+   and Bruno Haible <bruno@clisp.org>.  */
d3767b
+
d3767b
+/* The macros in this file implement multi-byte character input from a
d3767b
+   stream.
d3767b
+
d3767b
+   mb_file_t
d3767b
+     is the type for multibyte character input stream, usable for variable
d3767b
+     declarations.
d3767b
+
d3767b
+   mbf_char_t
d3767b
+     is the type for multibyte character or EOF, usable for variable
d3767b
+     declarations.
d3767b
+
d3767b
+   mbf_init (mbf, stream)
d3767b
+     initializes the MB_FILE for reading from stream.
d3767b
+
d3767b
+   mbf_getc (mbc, mbf)
d3767b
+     reads the next multibyte character from mbf and stores it in mbc.
d3767b
+
d3767b
+   mb_iseof (mbc)
d3767b
+     returns true if mbc represents the EOF value.
d3767b
+
d3767b
+   Here are the function prototypes of the macros.
d3767b
+
d3767b
+   extern void          mbf_init (mb_file_t mbf, FILE *stream);
d3767b
+   extern void          mbf_getc (mbf_char_t mbc, mb_file_t mbf);
d3767b
+   extern bool          mb_iseof (const mbf_char_t mbc);
d3767b
+ */
d3767b
+
d3767b
+#ifndef _MBFILE_H
d3767b
+#define _MBFILE_H 1
d3767b
+
d3767b
+#include <assert.h>
d3767b
+#include <stdbool.h>
d3767b
+#include <stdio.h>
d3767b
+#include <string.h>
d3767b
+
d3767b
+/* Tru64 with Desktop Toolkit C has a bug: <stdio.h> must be included before
d3767b
+   <wchar.h>.
d3767b
+   BSD/OS 4.1 has a bug: <stdio.h> and <time.h> must be included before
d3767b
+   <wchar.h>.  */
d3767b
+#include <stdio.h>
d3767b
+#include <time.h>
d3767b
+#include <wchar.h>
d3767b
+
d3767b
+#include "mbchar.h"
d3767b
+
d3767b
+#ifndef _GL_INLINE_HEADER_BEGIN
d3767b
+ #error "Please include config.h first."
d3767b
+#endif
d3767b
+_GL_INLINE_HEADER_BEGIN
d3767b
+#ifndef MBFILE_INLINE
d3767b
+# define MBFILE_INLINE _GL_INLINE
d3767b
+#endif
d3767b
+
d3767b
+struct mbfile_multi {
d3767b
+  FILE *fp;
d3767b
+  bool eof_seen;
d3767b
+  bool have_pushback;
d3767b
+  mbstate_t state;
d3767b
+  unsigned int bufcount;
d3767b
+  char buf[MBCHAR_BUF_SIZE];
d3767b
+  struct mbchar pushback;
d3767b
+};
d3767b
+
d3767b
+MBFILE_INLINE void
d3767b
+mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf)
d3767b
+{
d3767b
+  size_t bytes;
d3767b
+
d3767b
+  /* If EOF has already been seen, don't use getc.  This matters if
d3767b
+     mbf->fp is connected to an interactive tty.  */
d3767b
+  if (mbf->eof_seen)
d3767b
+    goto eof;
d3767b
+
d3767b
+  /* Return character pushed back, if there is one.  */
d3767b
+  if (mbf->have_pushback)
d3767b
+    {
d3767b
+      mb_copy (mbc, &mbf->pushback);
d3767b
+      mbf->have_pushback = false;
d3767b
+      return;
d3767b
+    }
d3767b
+
d3767b
+  /* Before using mbrtowc, we need at least one byte.  */
d3767b
+  if (mbf->bufcount == 0)
d3767b
+    {
d3767b
+      int c = getc (mbf->fp);
d3767b
+      if (c == EOF)
d3767b
+        {
d3767b
+          mbf->eof_seen = true;
d3767b
+          goto eof;
d3767b
+        }
d3767b
+      mbf->buf[0] = (unsigned char) c;
d3767b
+      mbf->bufcount++;
d3767b
+    }
d3767b
+
d3767b
+  /* Handle most ASCII characters quickly, without calling mbrtowc().  */
d3767b
+  if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0]))
d3767b
+    {
d3767b
+      /* These characters are part of the basic character set.  ISO C 99
d3767b
+         guarantees that their wide character code is identical to their
d3767b
+         char code.  */
d3767b
+      mbc->wc = mbc->buf[0] = mbf->buf[0];
d3767b
+      mbc->wc_valid = true;
d3767b
+      mbc->ptr = &mbc->buf[0];
d3767b
+      mbc->bytes = 1;
d3767b
+      mbf->bufcount = 0;
d3767b
+      return;
d3767b
+    }
d3767b
+
d3767b
+  /* Use mbrtowc on an increasing number of bytes.  Read only as many bytes
d3767b
+     from mbf->fp as needed.  This is needed to give reasonable interactive
d3767b
+     behaviour when mbf->fp is connected to an interactive tty.  */
d3767b
+  for (;;)
d3767b
+    {
d3767b
+      /* We don't know whether the 'mbrtowc' function updates the state when
d3767b
+         it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or
d3767b
+         not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour.  We
d3767b
+         don't have an autoconf test for this, yet.
d3767b
+         The new behaviour would allow us to feed the bytes one by one into
d3767b
+         mbrtowc.  But the old behaviour forces us to feed all bytes since
d3767b
+         the end of the last character into mbrtowc.  Since we want to retry
d3767b
+         with more bytes when mbrtowc returns -2, we must backup the state
d3767b
+         before calling mbrtowc, because implementations with the new
d3767b
+         behaviour will clobber it.  */
d3767b
+      mbstate_t backup_state = mbf->state;
d3767b
+
d3767b
+      bytes = mbrtowc (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state);
d3767b
+
d3767b
+      if (bytes == (size_t) -1)
d3767b
+        {
d3767b
+          /* An invalid multibyte sequence was encountered.  */
d3767b
+          /* Return a single byte.  */
d3767b
+          bytes = 1;
d3767b
+          mbc->wc_valid = false;
d3767b
+          break;
d3767b
+        }
d3767b
+      else if (bytes == (size_t) -2)
d3767b
+        {
d3767b
+          /* An incomplete multibyte character.  */
d3767b
+          mbf->state = backup_state;
d3767b
+          if (mbf->bufcount == MBCHAR_BUF_SIZE)
d3767b
+            {
d3767b
+              /* An overlong incomplete multibyte sequence was encountered.  */
d3767b
+              /* Return a single byte.  */
d3767b
+              bytes = 1;
d3767b
+              mbc->wc_valid = false;
d3767b
+              break;
d3767b
+            }
d3767b
+          else
d3767b
+            {
d3767b
+              /* Read one more byte and retry mbrtowc.  */
d3767b
+              int c = getc (mbf->fp);
d3767b
+              if (c == EOF)
d3767b
+                {
d3767b
+                  /* An incomplete multibyte character at the end.  */
d3767b
+                  mbf->eof_seen = true;
d3767b
+                  bytes = mbf->bufcount;
d3767b
+                  mbc->wc_valid = false;
d3767b
+                  break;
d3767b
+                }
d3767b
+              mbf->buf[mbf->bufcount] = (unsigned char) c;
d3767b
+              mbf->bufcount++;
d3767b
+            }
d3767b
+        }
d3767b
+      else
d3767b
+        {
d3767b
+          if (bytes == 0)
d3767b
+            {
d3767b
+              /* A null wide character was encountered.  */
d3767b
+              bytes = 1;
d3767b
+              assert (mbf->buf[0] == '\0');
d3767b
+              assert (mbc->wc == 0);
d3767b
+            }
d3767b
+          mbc->wc_valid = true;
d3767b
+          break;
d3767b
+        }
d3767b
+    }
d3767b
+
d3767b
+  /* Return the multibyte sequence mbf->buf[0..bytes-1].  */
d3767b
+  mbc->ptr = &mbc->buf[0];
d3767b
+  memcpy (&mbc->buf[0], &mbf->buf[0], bytes);
d3767b
+  mbc->bytes = bytes;
d3767b
+
d3767b
+  mbf->bufcount -= bytes;
d3767b
+  if (mbf->bufcount > 0)
d3767b
+    {
d3767b
+      /* It's not worth calling memmove() for so few bytes.  */
d3767b
+      unsigned int count = mbf->bufcount;
d3767b
+      char *p = &mbf->buf[0];
d3767b
+
d3767b
+      do
d3767b
+        {
d3767b
+          *p = *(p + bytes);
d3767b
+          p++;
d3767b
+        }
d3767b
+      while (--count > 0);
d3767b
+    }
d3767b
+  return;
d3767b
+
d3767b
+eof:
d3767b
+  /* An mbchar_t with bytes == 0 is used to indicate EOF.  */
d3767b
+  mbc->ptr = NULL;
d3767b
+  mbc->bytes = 0;
d3767b
+  mbc->wc_valid = false;
d3767b
+  return;
d3767b
+}
d3767b
+
d3767b
+MBFILE_INLINE void
d3767b
+mbfile_multi_ungetc (const struct mbchar *mbc, struct mbfile_multi *mbf)
d3767b
+{
d3767b
+  mb_copy (&mbf->pushback, mbc);
d3767b
+  mbf->have_pushback = true;
d3767b
+}
d3767b
+
d3767b
+typedef struct mbfile_multi mb_file_t;
d3767b
+
d3767b
+typedef mbchar_t mbf_char_t;
d3767b
+
d3767b
+#define mbf_init(mbf, stream)                                           \
d3767b
+  ((mbf).fp = (stream),                                                 \
d3767b
+   (mbf).eof_seen = false,                                              \
d3767b
+   (mbf).have_pushback = false,                                         \
d3767b
+   memset (&(mbf).state, '\0', sizeof (mbstate_t)),                     \
d3767b
+   (mbf).bufcount = 0)
d3767b
+
d3767b
+#define mbf_getc(mbc, mbf) mbfile_multi_getc (&(mbc), &(mbf))
d3767b
+
d3767b
+#define mbf_ungetc(mbc, mbf) mbfile_multi_ungetc (&(mbc), &(mbf))
d3767b
+
d3767b
+#define mb_iseof(mbc) ((mbc).bytes == 0)
d3767b
+
d3767b
+#ifndef _GL_INLINE_HEADER_BEGIN
d3767b
+ #error "Please include config.h first."
d3767b
+#endif
d3767b
+_GL_INLINE_HEADER_BEGIN
d3767b
+
d3767b
+#endif /* _MBFILE_H */
d3767b
diff --git a/m4/mbfile.m4 b/m4/mbfile.m4
d3767b
new file mode 100644
d3767b
index 0000000..8589902
d3767b
--- /dev/null
d3767b
+++ b/m4/mbfile.m4
d3767b
@@ -0,0 +1,14 @@
d3767b
+# mbfile.m4 serial 7
d3767b
+dnl Copyright (C) 2005, 2008-2015 Free Software Foundation, Inc.
d3767b
+dnl This file is free software; the Free Software Foundation
d3767b
+dnl gives unlimited permission to copy and/or distribute it,
d3767b
+dnl with or without modifications, as long as this notice is preserved.
d3767b
+
d3767b
+dnl autoconf tests required for use of mbfile.h
d3767b
+dnl From Bruno Haible.
d3767b
+
d3767b
+AC_DEFUN([gl_MBFILE],
d3767b
+[
d3767b
+  AC_REQUIRE([AC_TYPE_MBSTATE_T])
d3767b
+  :
d3767b
+])
d3767b
diff --git a/src/expand.c b/src/expand.c
d3767b
index 9fa2e10..380e020 100644
d3767b
--- a/src/expand.c
d3767b
+++ b/src/expand.c
d3767b
@@ -37,6 +37,9 @@
d3767b
 #include <stdio.h>
d3767b
 #include <getopt.h>
d3767b
 #include <sys/types.h>
d3767b
+
d3767b
+#include <mbfile.h>
d3767b
+
d3767b
 #include "system.h"
d3767b
 #include "die.h"
d3767b
 #include "xstrndup.h"
d3767b
@@ -100,19 +103,19 @@ expand (void)
d3767b
 {
d3767b
   /* Input stream.  */
d3767b
   FILE *fp = next_file (NULL);
d3767b
+  mb_file_t mbf;
d3767b
+  mbf_char_t c;
d3767b
 
d3767b
   if (!fp)
d3767b
     return;
d3767b
 
d3767b
+  mbf_init (mbf, fp);
d3767b
+
d3767b
   while (true)
d3767b
     {
d3767b
-      /* Input character, or EOF.  */
d3767b
-      int c;
d3767b
-
d3767b
       /* If true, perform translations.  */
d3767b
       bool convert = true;
d3767b
 
d3767b
-
d3767b
       /* The following variables have valid values only when CONVERT
d3767b
          is true:  */
d3767b
 
d3767b
@@ -122,17 +125,23 @@ expand (void)
d3767b
       /* Index in TAB_LIST of next tab stop to examine.  */
d3767b
       size_t tab_index = 0;
d3767b
 
d3767b
-
d3767b
       /* Convert a line of text.  */
d3767b
 
d3767b
       do
d3767b
         {
d3767b
-          while ((c = getc (fp)) < 0 && (fp = next_file (fp)))
d3767b
-            continue;
d3767b
+          do {
d3767b
+            mbf_getc (c, mbf);
d3767b
+            if (mb_iseof (c))
d3767b
+              {
d3767b
+                mbf_init (mbf, fp = next_file (fp));
d3767b
+                continue;
d3767b
+              }
d3767b
+            }
d3767b
+          while (false);
d3767b
 
d3767b
           if (convert)
d3767b
             {
d3767b
-              if (c == '\t')
d3767b
+              if (mb_iseq (c, '\t'))
d3767b
                 {
d3767b
                   /* Column the next input tab stop is on.  */
d3767b
                   uintmax_t next_tab_column;
d3767b
@@ -151,32 +160,34 @@ expand (void)
d3767b
                     if (putchar (' ') < 0)
d3767b
                       die (EXIT_FAILURE, errno, _("write error"));
d3767b
 
d3767b
-                  c = ' ';
d3767b
+                  mb_setascii (&c, ' ');
d3767b
                 }
d3767b
-              else if (c == '\b')
d3767b
+              else if (mb_iseq (c, '\b'))
d3767b
                 {
d3767b
                   /* Go back one column, and force recalculation of the
d3767b
                      next tab stop.  */
d3767b
                   column -= !!column;
d3767b
                   tab_index -= !!tab_index;
d3767b
                 }
d3767b
-              else
d3767b
+              /* A leading control character could make us trip over.  */
d3767b
+              else if (!mb_iscntrl (c))
d3767b
                 {
d3767b
-                  column++;
d3767b
+                  column += mb_width (c);
d3767b
                   if (!column)
d3767b
                     die (EXIT_FAILURE, 0, _("input line is too long"));
d3767b
                 }
d3767b
 
d3767b
-              convert &= convert_entire_line || !! isblank (c);
d3767b
+              convert &= convert_entire_line || mb_isblank (c);
d3767b
             }
d3767b
 
d3767b
-          if (c < 0)
d3767b
+          if (mb_iseof (c))
d3767b
             return;
d3767b
 
d3767b
-          if (putchar (c) < 0)
d3767b
+          mb_putc (c, stdout);
d3767b
+          if (ferror (stdout))
d3767b
             die (EXIT_FAILURE, errno, _("write error"));
d3767b
         }
d3767b
-      while (c != '\n');
d3767b
+      while (!mb_iseq (c, '\n'));
d3767b
     }
d3767b
 }
d3767b
 
d3767b
diff --git a/src/unexpand.c b/src/unexpand.c
d3767b
index 7801274..569a7ee 100644
d3767b
--- a/src/unexpand.c
d3767b
+++ b/src/unexpand.c
d3767b
@@ -38,6 +38,9 @@
d3767b
 #include <stdio.h>
d3767b
 #include <getopt.h>
d3767b
 #include <sys/types.h>
d3767b
+
d3767b
+#include <mbfile.h>
d3767b
+
d3767b
 #include "system.h"
d3767b
 #include "die.h"
d3767b
 #include "xstrndup.h"
d3767b
@@ -107,11 +110,12 @@ unexpand (void)
d3767b
 {
d3767b
   /* Input stream.  */
d3767b
   FILE *fp = next_file (NULL);
d3767b
+  mb_file_t mbf;
d3767b
 
d3767b
   /* The array of pending blanks.  In non-POSIX locales, blanks can
d3767b
      include characters other than spaces, so the blanks must be
d3767b
      stored, not merely counted.  */
d3767b
-  char *pending_blank;
d3767b
+  mbf_char_t *pending_blank;
d3767b
 
d3767b
   if (!fp)
d3767b
     return;
d3767b
@@ -119,12 +123,14 @@ unexpand (void)
d3767b
   /* The worst case is a non-blank character, then one blank, then a
d3767b
      tab stop, then MAX_COLUMN_WIDTH - 1 blanks, then a non-blank; so
d3767b
      allocate MAX_COLUMN_WIDTH bytes to store the blanks.  */
d3767b
-  pending_blank = xmalloc (max_column_width);
d3767b
+  pending_blank = xmalloc (max_column_width * sizeof (mbf_char_t));
d3767b
+
d3767b
+  mbf_init (mbf, fp);
d3767b
 
d3767b
   while (true)
d3767b
     {
d3767b
       /* Input character, or EOF.  */
d3767b
-      int c;
d3767b
+      mbf_char_t c;
d3767b
 
d3767b
       /* If true, perform translations.  */
d3767b
       bool convert = true;
d3767b
@@ -158,12 +164,19 @@ unexpand (void)
d3767b
 
d3767b
       do
d3767b
         {
d3767b
-          while ((c = getc (fp)) < 0 && (fp = next_file (fp)))
d3767b
-            continue;
d3767b
+          do {
d3767b
+            mbf_getc (c, mbf);
d3767b
+            if (mb_iseof (c))
d3767b
+              {
d3767b
+                mbf_init (mbf, fp = next_file (fp));
d3767b
+                continue;
d3767b
+              }
d3767b
+            }
d3767b
+          while (false);
d3767b
 
d3767b
           if (convert)
d3767b
             {
d3767b
-              bool blank = !! isblank (c);
d3767b
+              bool blank = mb_isblank (c);
d3767b
 
d3767b
               if (blank)
d3767b
                 {
d3767b
@@ -180,16 +193,16 @@ unexpand (void)
d3767b
                       if (next_tab_column < column)
d3767b
                         die (EXIT_FAILURE, 0, _("input line is too long"));
d3767b
 
d3767b
-                      if (c == '\t')
d3767b
+                      if (mb_iseq (c, '\t'))
d3767b
                         {
d3767b
                           column = next_tab_column;
d3767b
 
d3767b
                           if (pending)
d3767b
-                            pending_blank[0] = '\t';
d3767b
+                            mb_setascii (&pending_blank[0], '\t');
d3767b
                         }
d3767b
                       else
d3767b
                         {
d3767b
-                          column++;
d3767b
+                          column += mb_width (c);
d3767b
 
d3767b
                           if (! (prev_blank && column == next_tab_column))
d3767b
                             {
d3767b
@@ -197,13 +210,14 @@ unexpand (void)
d3767b
                                  will be replaced by tabs.  */
d3767b
                               if (column == next_tab_column)
d3767b
                                 one_blank_before_tab_stop = true;
d3767b
-                              pending_blank[pending++] = c;
d3767b
+                              mb_copy (&pending_blank[pending++], &c);
d3767b
                               prev_blank = true;
d3767b
                               continue;
d3767b
                             }
d3767b
 
d3767b
                           /* Replace the pending blanks by a tab or two.  */
d3767b
-                          pending_blank[0] = c = '\t';
d3767b
+                          mb_setascii (&c, '\t');
d3767b
+                          mb_setascii (&pending_blank[0], '\t');
d3767b
                         }
d3767b
 
d3767b
                       /* Discard pending blanks, unless it was a single
d3767b
@@ -211,7 +225,7 @@ unexpand (void)
d3767b
                       pending = one_blank_before_tab_stop;
d3767b
                     }
d3767b
                 }
d3767b
-              else if (c == '\b')
d3767b
+              else if (mb_iseq (c, '\b'))
d3767b
                 {
d3767b
                   /* Go back one column, and force recalculation of the
d3767b
                      next tab stop.  */
f5d167
@@ -221,16 +235,20 @@ unexpand (void)
d3767b
                 }
d3767b
               else
d3767b
                 {
d3767b
-                  column++;
f5d167
-                  if (!column)
f5d167
+                  const uintmax_t orig_column = column;
d3767b
+                  column += mb_width (c);
f5d167
+                  if (column < orig_column)
d3767b
                     die (EXIT_FAILURE, 0, _("input line is too long"));
d3767b
                 }
f5d167
 
d3767b
               if (pending)
d3767b
                 {
d3767b
                   if (pending > 1 && one_blank_before_tab_stop)
d3767b
-                    pending_blank[0] = '\t';
d3767b
-                  if (fwrite (pending_blank, 1, pending, stdout) != pending)
d3767b
+                    mb_setascii (&pending_blank[0], '\t');
d3767b
+
d3767b
+                  for (int n = 0; n < pending; ++n)
d3767b
+                    mb_putc (pending_blank[n], stdout);
d3767b
+                  if (ferror (stdout))
d3767b
                     die (EXIT_FAILURE, errno, _("write error"));
d3767b
                   pending = 0;
d3767b
                   one_blank_before_tab_stop = false;
f5d167
@@ -240,16 +258,17 @@ unexpand (void)
d3767b
               convert &= convert_entire_line || blank;
d3767b
             }
d3767b
 
d3767b
-          if (c < 0)
d3767b
+          if (mb_iseof (c))
d3767b
             {
d3767b
               free (pending_blank);
d3767b
               return;
d3767b
             }
d3767b
 
d3767b
-          if (putchar (c) < 0)
d3767b
+          mb_putc (c, stdout);
d3767b
+          if (ferror (stdout))
d3767b
             die (EXIT_FAILURE, errno, _("write error"));
d3767b
         }
d3767b
-      while (c != '\n');
d3767b
+      while (!mb_iseq (c, '\n'));
d3767b
     }
d3767b
 }
d3767b
 
d3767b
diff --git a/tests/expand/mb.sh b/tests/expand/mb.sh
d3767b
new file mode 100755
d3767b
index 0000000..7971e18
d3767b
--- /dev/null
d3767b
+++ b/tests/expand/mb.sh
d3767b
@@ -0,0 +1,98 @@
d3767b
+#!/bin/sh
d3767b
+
d3767b
+# Copyright (C) 2012-2015 Free Software Foundation, Inc.
d3767b
+
d3767b
+# This program is free software: you can redistribute it and/or modify
d3767b
+# it under the terms of the GNU General Public License as published by
d3767b
+# the Free Software Foundation, either version 3 of the License, or
d3767b
+# (at your option) any later version.
d3767b
+
d3767b
+# This program is distributed in the hope that it will be useful,
d3767b
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
d3767b
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
d3767b
+# GNU General Public License for more details.
d3767b
+
d3767b
+# You should have received a copy of the GNU General Public License
d3767b
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
d3767b
+
d3767b
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
d3767b
+print_ver_ expand
d3767b
+
d3767b
+export LC_ALL=en_US.UTF-8
d3767b
+
d3767b
+#input containing multibyte characters
d3767b
+cat <<\EOF > in || framework_failure_
d3767b
+1234567812345678123456781
d3767b
+.       .       .       .
d3767b
+a	b	c	d
d3767b
+.       .       .       .
d3767b
+ä	ö	ü	ß
d3767b
+.       .       .       .
d3767b
+EOF
d3767b
+env printf '   äöü\t.    öüä.   \tä xx\n' >> in || framework_failure_
d3767b
+
d3767b
+cat <<\EOF > exp || framework_failure_
d3767b
+1234567812345678123456781
d3767b
+.       .       .       .
d3767b
+a       b       c       d
d3767b
+.       .       .       .
d3767b
+ä       ö       ü       ß
d3767b
+.       .       .       .
d3767b
+   äöü  .    öüä.       ä xx
d3767b
+EOF
d3767b
+
d3767b
+expand < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+#test characters with display widths != 1
d3767b
+env printf '12345678
d3767b
+e\t|ascii(1)
d3767b
+\u00E9\t|composed(1)
d3767b
+e\u0301\t|decomposed(1)
d3767b
+\u3000\t|ideo-space(2)
d3767b
+\uFF0D\t|full-hypen(2)
d3767b
+' > in || framework_failure_
d3767b
+
d3767b
+env printf '12345678
d3767b
+e       |ascii(1)
d3767b
+\u00E9       |composed(1)
d3767b
+e\u0301       |decomposed(1)
d3767b
+\u3000      |ideo-space(2)
d3767b
+\uFF0D      |full-hypen(2)
d3767b
+' > exp || framework_failure_
d3767b
+
d3767b
+expand < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+#shouldn't fail with "input line too long"
d3767b
+#when a line starts with a control character
d3767b
+env printf '\n' > in || framework_failure_
d3767b
+
d3767b
+expand < in > out || fail=1
d3767b
+compare in out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+#non-Unicode characters interspersed between Unicode ones
d3767b
+env printf '12345678
d3767b
+\t\xFF|
d3767b
+\xFF\t|
d3767b
+\t\xFFä|
d3767b
+ä\xFF\t|
d3767b
+\tä\xFF|
d3767b
+\xFF\tä|
d3767b
+äbcdef\xFF\t|
d3767b
+' > in || framework_failure_
d3767b
+
d3767b
+env printf '12345678
d3767b
+        \xFF|
d3767b
+\xFF       |
d3767b
+        \xFFä|
d3767b
+ä\xFF      |
d3767b
+        ä\xFF|
d3767b
+\xFF       ä|
d3767b
+äbcdef\xFF |
d3767b
+' > exp || framework_failure_
d3767b
+
d3767b
+expand < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+exit $fail
d3767b
diff --git a/tests/local.mk b/tests/local.mk
d3767b
index 192f776..8053397 100644
d3767b
--- a/tests/local.mk
d3767b
+++ b/tests/local.mk
d3767b
@@ -544,6 +544,7 @@ all_tests =					\
d3767b
   tests/du/threshold.sh				\
d3767b
   tests/du/trailing-slash.sh			\
d3767b
   tests/du/two-args.sh				\
d3767b
+  tests/expand/mb.sh				\
d3767b
   tests/id/gnu-zero-uids.sh			\
d3767b
   tests/id/no-context.sh			\
d3767b
   tests/id/context.sh				\
d3767b
@@ -684,6 +685,7 @@ all_tests =					\
d3767b
   tests/touch/read-only.sh			\
d3767b
   tests/touch/relative.sh			\
d3767b
   tests/touch/trailing-slash.sh			\
d3767b
+  tests/unexpand/mb.sh				\
d3767b
   $(all_root_tests)
d3767b
 
d3767b
 # See tests/factor/create-test.sh.
d3767b
diff --git a/tests/unexpand/mb.sh b/tests/unexpand/mb.sh
d3767b
new file mode 100755
d3767b
index 0000000..60d4c1a
d3767b
--- /dev/null
d3767b
+++ b/tests/unexpand/mb.sh
d3767b
@@ -0,0 +1,97 @@
d3767b
+#!/bin/sh
d3767b
+
d3767b
+# Copyright (C) 2012-2015 Free Software Foundation, Inc.
d3767b
+
d3767b
+# This program is free software: you can redistribute it and/or modify
d3767b
+# it under the terms of the GNU General Public License as published by
d3767b
+# the Free Software Foundation, either version 3 of the License, or
d3767b
+# (at your option) any later version.
d3767b
+
d3767b
+# This program is distributed in the hope that it will be useful,
d3767b
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
d3767b
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
d3767b
+# GNU General Public License for more details.
d3767b
+
d3767b
+# You should have received a copy of the GNU General Public License
d3767b
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
d3767b
+
d3767b
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
d3767b
+print_ver_ unexpand
d3767b
+
d3767b
+export LC_ALL=en_US.UTF-8
d3767b
+
d3767b
+#input containing multibyte characters
d3767b
+cat > in <<\EOF
d3767b
+1234567812345678123456781
d3767b
+.       .       .       .
d3767b
+a       b       c       d
d3767b
+.       .       .       .
d3767b
+ä       ö       ü       ß
d3767b
+.       .       .       .
d3767b
+   äöü  .    öüä.       ä xx
d3767b
+EOF
d3767b
+
d3767b
+cat > exp <<\EOF
d3767b
+1234567812345678123456781
d3767b
+.	.	.	.
d3767b
+a	b	c	d
d3767b
+.	.	.	.
d3767b
+ä	ö	ü	ß
d3767b
+.	.	.	.
d3767b
+   äöü	.    öüä.	ä xx
d3767b
+EOF
d3767b
+
d3767b
+unexpand -a < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+#test characters with a display width larger than 1
d3767b
+
d3767b
+env printf '12345678
d3767b
+e       |ascii(1)
d3767b
+\u00E9       |composed(1)
d3767b
+e\u0301       |decomposed(1)
d3767b
+\u3000      |ideo-space(2)
d3767b
+\uFF0D      |full-hypen(2)
d3767b
+' > in || framework_failure_
d3767b
+
d3767b
+env printf '12345678
d3767b
+e\t|ascii(1)
d3767b
+\u00E9\t|composed(1)
d3767b
+e\u0301\t|decomposed(1)
d3767b
+\u3000\t|ideo-space(2)
d3767b
+\uFF0D\t|full-hypen(2)
d3767b
+' > exp || framework_failure_
d3767b
+
d3767b
+unexpand -a < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+#test input where a blank of width > 1 is not being substituted
d3767b
+in="$(LC_ALL=en_US.UTF-8 printf ' \u3000  ö       ü       ß')"
d3767b
+exp='    ö	     ü	     ß'
d3767b
+
d3767b
+unexpand -a < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
+
d3767b
+#non-Unicode characters interspersed between Unicode ones
d3767b
+env printf '12345678
d3767b
+        \xFF|
d3767b
+\xFF       |
d3767b
+        \xFFä|
d3767b
+ä\xFF      |
d3767b
+        ä\xFF|
d3767b
+\xFF       ä|
d3767b
+äbcdef\xFF |
d3767b
+' > in || framework_failure_
d3767b
+
d3767b
+env printf '12345678
d3767b
+\t\xFF|
d3767b
+\xFF\t|
d3767b
+\t\xFFä|
d3767b
+ä\xFF\t|
d3767b
+\tä\xFF|
d3767b
+\xFF\tä|
d3767b
+äbcdef\xFF\t|
d3767b
+' > exp || framework_failure_
d3767b
+
d3767b
+unexpand -a < in > out || fail=1
d3767b
+compare exp out > /dev/null 2>&1 || fail=1
d3767b
-- 
d3767b
2.7.4
d3767b