From 18141cd30a71da4299845a4122a390bb190de37a Mon Sep 17 00:00:00 2001
From: Bertrand Garrigues <bertrand.garrigues@laposte.net>
Date: Tue, 4 Jul 2017 23:40:46 +0200
Subject: [PATCH] Use libchardet for preconv

fix
---
 configure.ac                      | 20 +++++----
 m4/groff.m4                       | 36 ++++++++++++++++
 src/preproc/preconv/preconv.1.man | 10 ++++-
 src/preproc/preconv/preconv.am    |  3 +-
 src/preproc/preconv/preconv.cpp   | 88 +++++++++++++++++++++++++++++++++++++--
 5 files changed, 142 insertions(+), 15 deletions(-)

diff --git a/configure.ac b/configure.ac
index 05585159..c76de2f0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -87,6 +87,8 @@ AC_PROG_INSTALL
 AC_PROG_LN_S
 AC_PROG_MKDIR_P
 GROFF_PROG_XPMTOPPM
+PKG_PROG_PKG_CONFIG
+GROFF_UCHARDET
 
 # use a dummy substitution if no csh hack is necessary to avoid errors
 # with non-GNU sed programs
@@ -219,27 +221,29 @@ AC_OUTPUT
 echo "
 ${PACKAGE_NAME} version ${PACKAGE_VERSION}
 ----------------------------------------------------------------------
- Prefix            : ${prefix}
- Compiler          : ${CC} ${CFLAGS} ${CPPFLAGS}"
+ Prefix                        : ${prefix}
+ Compiler                      : ${CC} ${CFLAGS} ${CPPFLAGS}"
 if test "x$groff_no_x" = "xyes"; then
 echo "\
- X11 support       : no"
+ X11 support                   : no"
 else
 echo "\
- X11 support       : yes
- X11 resources dir : $appresdir"
+ X11 support                   : yes
+ X11 resources dir             : $appresdir"
 fi
 if test "x$doc" = x; then
 echo "\
- Doc build         : no"
+ Doc build                     : no"
 else
 echo "\
- Doc build         : ${doc} "
+ Doc build                     : ${doc} "
 fi
 echo "\
- URW fonts for pdf : $groff_have_urw_fonts"
+ URW fonts for pdf             : $groff_have_urw_fonts
+ Use libucharset.so for preconv: $groff_have_uchardet"
 echo "\
 ----------------------------------------------------------------------"
 
 GROFF_APPRESDIR_CHECK
 GROFF_URW_FONTS_CHECK
+GROFF_UCHARDET_CHECK
diff --git a/m4/groff.m4 b/m4/groff.m4
index d1727cdd..47d1d462 100644
--- a/m4/groff.m4
+++ b/m4/groff.m4
@@ -1621,3 +1621,39 @@ AC_DEFUN([GROFF_BASH],
      BASH_PROG=/bin/sh
   fi
   AC_SUBST([BASH_PROG])])
+
+# Looking for libuchardet.so, used by preconv.
+AC_DEFUN([GROFF_UCHARDET],
+  [AC_ARG_WITH([uchardet],
+               AS_HELP_STRING([--with-uchardet],
+                              [Build `preconv' with libuchardet.so for file \
+                               encoding automatic detection [=auto|no|yes]]))
+   AS_IF([test "x$with_uchardet" != "xno"],
+         [PKG_CHECK_MODULES([UCHARDET],
+                            [uchardet >= 0.0.1],
+                            [AC_DEFINE([HAVE_UCHARDET],
+                                       [1],
+                                       [libuchardet.so availability])
+                             groff_have_uchardet=yes],
+                            [if test "x$with_uchardet" = "xyes"; then
+                               AC_MSG_FAILURE([Could not found libuchardet.so])
+                             else
+                               AC_MSG_WARN([libuchardet.so not found, preconv \
+                                            might not work properly])
+                             fi
+                             groff_have_uchardet=no])],
+          [groff_have_uchardet=no]
+          )])
+
+# Warning if libuchardet.so was not found
+AC_DEFUN([GROFF_UCHARDET_CHECK],
+  [if test "x$groff_have_uchardet" = "xno" -a "x$with_uchardet" != "xno"; then
+  AC_MSG_WARN([
+  libuchardet.so was not found, preprocessor `preconv', which uses it to detect
+  the input file encoding, might not work properly (to check how and in which
+  order `preconv' tries to determine the file encoding see its man page).
+  ])
+  fi
+  ])
+
+
diff --git a/src/preproc/preconv/preconv.1.man b/src/preproc/preconv/preconv.1.man
index ce93d93b..5be0e697 100644
--- a/src/preproc/preconv/preconv.1.man
+++ b/src/preproc/preconv/preconv.1.man
@@ -142,13 +142,19 @@ Otherwise, check whether the input starts with a
 If found, use it.
 .
 .IP 3.
-Finally, check whether there is a known
+Otherwise, check whether there is a known
 .I coding tag
 (see below) in either the first or second input line.
 .
 If found, use it.
 .
-.IP 4.
+.IP 4
+Finally, if
+.BR libuchardet.so
+(an encoding detector library available on most major distributions)
+is available on the system, use it to try to detect the encoding of the file.
+.
+.IP 5.
 If everything fails, use a default encoding as given with option
 .BR \-D ,
 by the current locale, or \[oq]latin1\[cq] if the locale is set to
diff --git a/src/preproc/preconv/preconv.am b/src/preproc/preconv/preconv.am
index 942eb0a3..c873592a 100644
--- a/src/preproc/preconv/preconv.am
+++ b/src/preproc/preconv/preconv.am
@@ -19,8 +19,9 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 bin_PROGRAMS += preconv
-preconv_LDADD = libgroff.a $(LIBM) $(LIBICONV)
+preconv_LDADD = libgroff.a $(LIBM) $(LIBICONV) $(UCHARDET_LIBS)
 preconv_SOURCES = src/preproc/preconv/preconv.cpp
+preconv_CPPFLAGS = $(AM_CPPFLAGS) $(UCHARDET_CFLAGS)
 man1_MANS += src/preproc/preconv/preconv.1
 EXTRA_DIST += src/preproc/preconv/preconv.1.man
 
diff --git a/src/preproc/preconv/preconv.cpp b/src/preproc/preconv/preconv.cpp
index 2819b5e4..b7241b0b 100644
--- a/src/preproc/preconv/preconv.cpp
+++ b/src/preproc/preconv/preconv.cpp
@@ -1,5 +1,5 @@
 // -*- C++ -*-
-/* Copyright (C) 2005-2014  Free Software Foundation, Inc.
+/* Copyright (C) 2005-2017  Free Software Foundation, Inc.
      Written by Werner Lemberg (wl@gnu.org)
 
 This file is part of groff.
@@ -22,6 +22,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. */
 #include <assert.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <sys/stat.h>
+#ifdef HAVE_UCHARDET
+#include <uchardet/uchardet.h>
+#endif
 #include "errarg.h"
 #include "error.h"
 #include "localcharset.h"
@@ -997,6 +1001,67 @@ check_coding_tag(FILE *fp, string &data)
   return NULL;
 }
 
+char *
+detect_file_encoding(FILE *fp)
+{
+#ifdef HAVE_UCHARDET
+  uchardet_t ud;
+  struct stat stat_buf;
+  size_t len, read_bytes;
+  char *data;
+  int res, current_position;
+  const char *charset;
+  char *ret;
+
+  current_position = ftell(fp);
+  /* due to BOM and tag detection we are not at the begining of the file */
+  rewind(fp);
+  if (fstat(fileno(fp), &stat_buf) != 0) {
+    fprintf(stderr, "fstat: %s\n", strerror(errno));
+    return NULL;
+  }
+  len = stat_buf.st_size;
+  if (debug_flag)
+    fprintf(stderr, "  len: %zu\n", len);  
+  if (len == 0)
+    return NULL;
+  data = (char *)calloc(len, 1);
+  read_bytes = fread(data, 1, len, fp);
+  if (read_bytes == 0) {
+    fprintf(stderr, "fread: %s\n", strerror(errno));
+    return NULL;
+  }
+  /* We rewind back to the original position */
+  if (fseek(fp, current_position, SEEK_SET) != 0) {
+    fprintf(stderr, "Fatal error: fseek: %s\n", strerror(errno));
+    return NULL;
+  }
+  ud = uchardet_new();
+  res = uchardet_handle_data(ud, data, len);
+  if (res != 0) {
+    fprintf(stderr, "uchardet_handle_data: %d\n", res);
+    uchardet_delete(ud);
+    return NULL;
+  }
+  if (debug_flag)
+    fprintf(stderr, "  uchardet read: %zu bytes\n", read_bytes);
+  uchardet_data_end(ud);
+  charset = uchardet_get_charset(ud);
+  if (debug_flag)
+    fprintf(stderr, "  charset: %s\n", charset);
+  if (charset) {
+    ret = (char *)calloc(strlen(charset) + 1, 1);
+    strcpy(ret, charset);
+  }
+  uchardet_delete(ud);
+  free(data);
+
+  return ret;
+#else /* not HAVE_UCHARDET */
+  return NULL;
+#endif /* not HAVE_UCHARDET */
+}
+
 // ---------------------------------------------------------
 // Handle an input file.  If filename is `-' handle stdin.
 //
@@ -1025,6 +1090,7 @@ do_file(const char *filename)
   const char *BOM_encoding = get_BOM(fp, BOM, data);
   // Determine the encoding.
   char *encoding;
+  int must_free_encoding = 0;
   if (user_encoding[0]) {
     if (debug_flag) {
       fprintf(stderr, "  user-specified encoding `%s', "
@@ -1046,8 +1112,15 @@ do_file(const char *filename)
     char *file_encoding = check_coding_tag(fp, data);
     if (!file_encoding) {
       if (debug_flag)
-	fprintf(stderr, "  no file encoding\n");
-      file_encoding = default_encoding;
+	fprintf(stderr, "  no encoding tag\n");
+      file_encoding = detect_file_encoding(fp);
+      if (!file_encoding) {
+        if (debug_flag)
+          fprintf(stderr, "  could not detect encoding with uchardet\n");
+        file_encoding = default_encoding;
+      }
+      else
+        must_free_encoding = 1;
     }
     else
       if (debug_flag)
@@ -1056,6 +1129,8 @@ do_file(const char *filename)
   }
   strncpy(encoding_string, encoding, MAX_VAR_LEN - 1);
   encoding_string[MAX_VAR_LEN - 1] = 0;
+  if (must_free_encoding)
+    free(encoding);
   encoding = encoding_string;
   // Translate from MIME & Emacs encoding names to locale encoding names.
   encoding = emacs2mime(encoding_string);
@@ -1143,13 +1218,18 @@ main(int argc, char **argv)
 			    "dD:e:hrv", long_options, NULL)) != EOF)
     switch (opt) {
     case 'v':
-      printf("GNU preconv (groff) version %s %s iconv support\n",
+      printf("GNU preconv (groff) version %s %s iconv support and %s uchardet support\n",
 	     Version_string,
 #ifdef HAVE_ICONV
 	     "with"
 #else
 	     "without"
 #endif /* HAVE_ICONV */
+#ifdef HAVE_UCHARDET
+             "with"
+#else
+             "without"
+#endif /* HAVE_UCHARDET */
 	    );
       exit(0);
       break;
-- 
2.13.2

