bk_text and bk_info both need to know the on-screen width of

characters in order to wrap and align them properly. Therefore, they
should be using wcwidth(). So here are a couple of wrappers on
wcwidth(), one which filters out the Unicode characters not
representable in the target charset, and one which converts _from_ a
charset to Unicode before calling wcwidth(). bk_text and bk_info
should now align correctly even in the face of unsupported
characters and Japanese.

[originally from svn r4116]

6 files changed, 200 insertions, 19 deletions

diff --git a/Makefile b/Makefile
index 97debc8..719e9b4 100644
--- a/Makefile
+++ b/Makefile
@@ -111,7 +111,7 @@ include $(LIBCHARSET_SRCDIR)Makefile
 MODULES := main malloc ustring error help licence version misc tree234
 MODULES += input keywords contents index style biblio
 MODULES += bk_text bk_xhtml bk_whlp bk_man bk_info bk_paper bk_ps bk_pdf
-MODULES += winhelp psdata
+MODULES += winhelp psdata wcwidth
 
 OBJECTS := $(addsuffix .o,$(MODULES)) $(LIBCHARSET_OBJS)
 DEPS := $(addsuffix .d,$(MODULES))
diff --git a/bk_info.c b/bk_info.c
index f7efbf5..d513587 100644
--- a/bk_info.c
+++ b/bk_info.c
@@ -25,17 +25,6 @@
  *  - might be helpful to diagnose duplicate node names!
  */
 
-/*
- * FIXME:
- * 
- *  - alignment in the index is broken when a non-representable
- *    character appears with no alternative. More generally, I
- *    fear, this is the fault of the info_rdadd* functions failing
- *    to return correct width figures in this circumstance (so it
- *    will affect list paragraph prefixes and paragraph wrapping as
- *    well).
- */
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
@@ -767,7 +756,7 @@ static int info_width_internal(word *words, int xrefs, int charset) {
 		    attraux(words->aux) == attr_Always ? 0 : 1)
 		 : 0) +
 		(cvt_ok(charset, words->text) || !words->alt ?
-		 ustrlen(words->text) :
+		 ustrwid(words->text, charset) :
 		 info_width_internal_list(words->alt, xrefs, charset)));
 
       case word_WhiteSpace:
@@ -790,7 +779,7 @@ static int info_width_internal(word *words, int xrefs, int charset) {
       case word_LowerXref:
 	if (xrefs && words->private_data) {
 	    /* "*Note " plus "::" comes to 8 characters */
-	    return 8 + strlen(((node *)words->private_data)->name);
+	    return 8 + strwid(((node *)words->private_data)->name, charset);
 	}
 	break;
     }
@@ -995,9 +984,11 @@ static int info_rdadds(info_data *d, wchar_t const *wcs)
 
     if (wcs) {
 	char buf[256];
-	int len, origlen, ret;
+	int len, width, ret;
+
+	width = ustrwid(wcs, d->charset);
 
-	origlen = len = ustrlen(wcs);
+	len = ustrlen(wcs);
 	while (len > 0) {
 	    int prevlen = len;
 
@@ -1012,7 +1003,7 @@ static int info_rdadds(info_data *d, wchar_t const *wcs)
 	    }
 	}
 
-	return origlen;
+	return width;
     } else
 	return 0;
 }
diff --git a/bk_text.c b/bk_text.c
index 373de74..8fa7340 100644
--- a/bk_text.c
+++ b/bk_text.c
@@ -480,7 +480,7 @@ static int text_width(void *ctx, word *text) {
 		    attraux(text->aux) == attr_Always ? 0 : 1)
 		 : 0) +
 		(cvt_ok(charset, text->text) || !text->alt ?
-		 ustrlen(text->text) :
+		 ustrwid(text->text, charset) :
 		 text_width_list(ctx, text->alt)));
 
       case word_WhiteSpace:
diff --git a/halibut.h b/halibut.h
index 69953b6..26f2e4f 100644
--- a/halibut.h
+++ b/halibut.h
@@ -293,6 +293,12 @@ wchar_t *ustrftime(wchar_t *fmt, struct tm *timespec);
 int cvt_ok(int charset, const wchar_t *s);
 
 /*
+ * wcwidth.c
+ */
+int strwid(char const *s, int charset);
+int ustrwid(wchar_t const *s, int charset);
+
+/*
  * help.c
  */
 void help(void);
diff --git a/main.c b/main.c
index 61b37f7..18d644c 100644
--- a/main.c
+++ b/main.c
@@ -263,7 +263,7 @@ int main(int argc, char **argv) {
 	in.pushback = NULL;
 	in.reportcols = reportcols;
 	in.stack = NULL;
-	in.defcharset = locale_charset();
+	in.defcharset = charset_from_locale();
 
 	idx = make_index();
 
diff --git a/wcwidth.c b/wcwidth.c
new file mode 100644
index 0000000..269359f
--- /dev/null
+++ b/wcwidth.c
@@ -0,0 +1,184 @@
+/*
+ * A wrapper around wcswidth() to have it ignore characters which
+ * are not represented in a particular output charset.
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include "halibut.h"
+
+#ifndef HAS_WCSWIDTH
+
+/*
+ * Markus Kuhn's public domain wcwidth() implementation, cribbed
+ * from
+ * 
+ *   http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
+ */
+
+#define wcswidth mk_wcswidth
+
+struct interval {
+    int first;
+    int last;
+};
+
+/* auxiliary function for binary search in interval table */
+static int bisearch(wchar_t ucs, const struct interval *table, int max) {
+    int min = 0;
+    int mid;
+
+    if (ucs < table[0].first || ucs > table[max].last)
+	return 0;
+    while (max >= min) {
+	mid = (min + max) / 2;
+	if (ucs > table[mid].last)
+	    min = mid + 1;
+	else if (ucs < table[mid].first)
+	    max = mid - 1;
+	else
+	    return 1;
+    }
+
+    return 0;
+}
+
+int mk_wcwidth(wchar_t ucs)
+{
+    /* sorted list of non-overlapping intervals of non-spacing characters */
+    /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
+    static const struct interval combining[] = {
+	{ 0x0300, 0x0357 }, { 0x035D, 0x036F }, { 0x0483, 0x0486 },
+	{ 0x0488, 0x0489 }, { 0x0591, 0x05A1 }, { 0x05A3, 0x05B9 },
+    { 0x05BB, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 },
+    { 0x05C4, 0x05C4 }, { 0x0600, 0x0603 }, { 0x0610, 0x0615 },
+    { 0x064B, 0x0658 }, { 0x0670, 0x0670 }, { 0x06D6, 0x06E4 },
+    { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, { 0x070F, 0x070F },
+    { 0x0711, 0x0711 }, { 0x0730, 0x074A }, { 0x07A6, 0x07B0 },
+    { 0x0901, 0x0902 }, { 0x093C, 0x093C }, { 0x0941, 0x0948 },
+    { 0x094D, 0x094D }, { 0x0951, 0x0954 }, { 0x0962, 0x0963 },
+    { 0x0981, 0x0981 }, { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 },
+    { 0x09CD, 0x09CD }, { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 },
+    { 0x0A3C, 0x0A3C }, { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 },
+    { 0x0A4B, 0x0A4D }, { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 },
+    { 0x0ABC, 0x0ABC }, { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 },
+    { 0x0ACD, 0x0ACD }, { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 },
+    { 0x0B3C, 0x0B3C }, { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 },
+    { 0x0B4D, 0x0B4D }, { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 },
+    { 0x0BC0, 0x0BC0 }, { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 },
+    { 0x0C46, 0x0C48 }, { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 },
+    { 0x0CBC, 0x0CBC }, { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 },
+    { 0x0CCC, 0x0CCD }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D },
+    { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 },
+    { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E },
+    { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC },
+    { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 },
+    { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E },
+    { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 },
+    { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 },
+    { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 },
+    { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x1712, 0x1714 },
+    { 0x1732, 0x1734 }, { 0x1752, 0x1753 }, { 0x1772, 0x1773 },
+    { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD }, { 0x17C6, 0x17C6 },
+    { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD }, { 0x180B, 0x180D },
+    { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 }, { 0x1927, 0x1928 },
+    { 0x1932, 0x1932 }, { 0x1939, 0x193B }, { 0x200B, 0x200F },
+    { 0x202A, 0x202E }, { 0x2060, 0x2063 }, { 0x206A, 0x206F },
+    { 0x20D0, 0x20EA }, { 0x302A, 0x302F }, { 0x3099, 0x309A },
+    { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F }, { 0xFE20, 0xFE23 },
+    { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB }, { 0x1D167, 0x1D169 },
+    { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD },
+    { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F }, { 0xE0100, 0xE01EF }
+  };
+
+  /* test for 8-bit control characters */
+  if (ucs == 0)
+    return 0;
+  if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0))
+    return -1;
+
+  /* binary search in table of non-spacing characters */
+  if (bisearch(ucs, combining,
+	       sizeof(combining) / sizeof(struct interval) - 1))
+    return 0;
+
+  /* if we arrive here, ucs is not a combining or C0/C1 control character */
+
+  return 1 + 
+    (ucs >= 0x1100 &&
+     (ucs <= 0x115f ||                    /* Hangul Jamo init. consonants */
+      ucs == 0x2329 || ucs == 0x232a ||
+      (ucs >= 0x2e80 && ucs <= 0xa4cf &&
+       ucs != 0x303f) ||                  /* CJK ... Yi */
+      (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
+      (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */
+      (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
+      (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */
+      (ucs >= 0xffe0 && ucs <= 0xffe6) ||
+      (ucs >= 0x20000 && ucs <= 0x2fffd) ||
+      (ucs >= 0x30000 && ucs <= 0x3fffd)));
+}
+
+
+int mk_wcswidth(const wchar_t *pwcs, size_t n)
+{
+  int w, width = 0;
+
+  for (;*pwcs && n-- > 0; pwcs++)
+    if ((w = mk_wcwidth(*pwcs)) < 0)
+      return -1;
+    else
+      width += w;
+
+  return width;
+}
+
+#endif
+
+int ustrwid(wchar_t const *s, int charset)
+{
+    char buf[256];
+    int wid, len = ustrlen(s);
+    charset_state state = CHARSET_INIT_STATE;
+
+    wid = 0;
+
+    while (len > 0) {
+	int err, ret;
+	wchar_t const *s_orig;
+
+	err = 0;
+	s_orig = s;
+	ret = charset_from_unicode(&s, &len, buf, lenof(buf),
+				   charset, &state, &err);
+	wid += wcswidth(s_orig, s - s_orig);
+	if (err) {
+	    assert(len > 0 && *s);
+	    s++;
+	    len--;
+	}
+    }
+
+    return wid;
+}
+
+int strwid(char const *s, int charset)
+{
+    wchar_t buf[256];
+    int wid, len = strlen(s);
+    charset_state state = CHARSET_INIT_STATE;
+
+    wid = 0;
+
+    while (len > 0) {
+	int ret;
+
+	ret = charset_to_unicode(&s, &len, buf, lenof(buf),
+				 charset, &state, NULL, 0);
+
+	if (ret > 0)
+	    wid += wcswidth(buf, ret);
+    }
+
+    return wid;
+}