# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1600442132 -7200
#      Fri Sep 18 17:15:32 2020 +0200
# Node ID b1be934e4cd3d5f11ae2804f66bb5c0d4f73c174
# Parent  1807a5762cba84d24f1da56b5795ea0cec750bd7
Add function "dir_encoding" to set a file encoding per directory (bug #49685).

* corefcn/load-path.h (load_path::read_dir_config): Add new function.
* corefcn/load-path.cc (load_path::read_dir_config): Add new function.
  (load_path::add): Call read_dir_config on adding a new directory to the load
  path.
* corefcn/input.h (input_system::dir_encoding, input_system::set_dir_encoding):
  Add new functions. Add private unordered_map "m_dir_encoding".
* corefcn/input.cc (input_system::dir_encoding, input_system::set_dir_encoding):
  Add new functions.
  (base_reader, file_reader, input_reader): Add new constructors that accept an
  encoding (use mfile_encoding by default). Add private property "m_encoding".
  (load_path_dir): Static function to get the part of the directory that would
   be added to the load path.
  (Fdir_encoding): New function.
* corefcn/interpreter.cc (interpreter::chdir): Call load_path::read_dir_config.
* parse-tree/lex.h (lexer): Add new constructor that accepts an encoding.
* parse-tree/parse.h (parser): Add new constructor that accepts an encoding.
* parse-tree/oct-parse.yy (parse_fcn_file): Pass dir_encoding to parser.
* doc/interpreter/func.txi: Add docstring of function "dir_encoding" to manual.

diff -r 1807a5762cba -r b1be934e4cd3 doc/interpreter/func.txi
--- a/doc/interpreter/func.txi	Wed Oct 07 09:38:48 2020 +0200
+++ b/doc/interpreter/func.txi	Fri Sep 18 17:15:32 2020 +0200
@@ -924,6 +924,8 @@
 
 @DOCSTRING(dir_in_loadpath)
 
+@DOCSTRING(dir_encoding)
+
 @node Subfunctions
 @subsection Subfunctions
 
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/corefcn/input.cc
--- a/libinterp/corefcn/input.cc	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/corefcn/input.cc	Fri Sep 18 17:15:32 2020 +0200
@@ -541,6 +541,92 @@
     return retval;
   }
 
+  // Get part of the directory that would be added to the load path
+  static std::string load_path_dir (const std::string& dir)
+  {
+    std::string lp_dir = dir;
+
+    // strip trailing filesep
+    size_t ipos = lp_dir.find_last_not_of (sys::file_ops::dir_sep_chars ());
+    if (ipos != std::string::npos)
+      lp_dir = lp_dir.erase (ipos+1);
+
+    // strip trailing private folder
+    ipos = lp_dir.find_last_of (sys::file_ops::dir_sep_chars ());
+    if (ipos != std::string::npos
+        && lp_dir.substr (ipos+1).compare ("private") == 0)
+      {
+        lp_dir = lp_dir.erase (ipos);
+        ipos = lp_dir.find_last_of (sys::file_ops::dir_sep_chars ());
+      }
+
+    // strip trailing @class folder
+    if (ipos != std::string::npos && lp_dir[ipos+1] == '@')
+      {
+        lp_dir = lp_dir.erase (ipos);
+        ipos = lp_dir.find_last_of (sys::file_ops::dir_sep_chars ());
+      }
+
+    // strip (nested) +namespace folders
+    while (ipos != std::string::npos && lp_dir[ipos+1] == '+')
+      {
+        lp_dir = lp_dir.erase (ipos);
+        ipos = lp_dir.find_last_of (sys::file_ops::dir_sep_chars ());
+      }
+
+    return lp_dir;
+  }
+
+  std::string input_system::dir_encoding (const std::string& dir)
+  {
+    std::string enc = m_mfile_encoding;
+
+    auto enc_it = m_dir_encoding.find (load_path_dir (dir));
+    if (enc_it != m_dir_encoding.end ())
+      enc = enc_it->second;
+
+    return enc;
+  }
+
+  void input_system::set_dir_encoding (const std::string& dir,
+                                       std::string& enc)
+  {
+    // use lower case
+    std::transform (enc.begin (), enc.end (), enc.begin (), ::tolower);
+
+    if (enc.compare ("delete") == 0)
+      {
+        // Remove path from map
+        m_dir_encoding.erase (load_path_dir (dir));
+        return;
+      }
+    else if (enc.compare ("utf-8"))
+      {
+        // Check for valid encoding name.
+        // FIXME: This will probably not happen very often and opening the
+        //        encoder doesn't take long.
+        //        Should we cache working encoding identifiers anyway?
+        void *codec
+          = octave_iconv_open_wrapper (enc.c_str (), "utf-8");
+
+        if (codec == reinterpret_cast<void *> (-1))
+          {
+            if (errno == EINVAL)
+              error ("dir_encoding: conversion from encoding '%s' "
+                     "not supported", enc.c_str ());
+            else
+              error ("dir_encoding: error %d opening encoding '%s'.",
+                     errno, enc.c_str ());
+          }
+        else
+          octave_iconv_close_wrapper (codec);
+      }
+
+    m_dir_encoding[load_path_dir (dir)] = enc;
+
+    return;
+  }
+
   bool input_system::yes_or_no (const std::string& prompt)
   {
     std::string prompt_string = prompt + "(yes or no) ";
@@ -808,7 +894,14 @@
   public:
 
     file_reader (interpreter& interp, FILE *f_arg)
-      : base_reader (interp), m_file (f_arg) { }
+      : base_reader (interp), m_file (f_arg)
+    {
+      octave::input_system& input_sys = interp.get_input_system ();
+      m_encoding = input_sys.mfile_encoding ();
+    }
+
+    file_reader (interpreter& interp, FILE *f_arg, const std::string& enc)
+      : base_reader (interp), m_file (f_arg), m_encoding (enc) { }
 
     std::string get_input (const std::string& prompt, bool& eof);
 
@@ -820,6 +913,8 @@
 
     FILE *m_file;
 
+    std::string m_encoding;
+
     static const std::string s_in_src;
   };
 
@@ -853,6 +948,10 @@
     : m_rep (new file_reader (interp, file))
   { }
 
+  input_reader::input_reader (interpreter& interp, FILE *file, const std::string& enc)
+    : m_rep (new file_reader (interp, file, enc))
+  { }
+
   input_reader::input_reader (interpreter& interp, const std::string& str)
     : m_rep (new eval_string_reader (interp, str))
   { }
@@ -882,9 +981,15 @@
 
     std::string src_str = octave_fgets (m_file, eof);
 
-    input_system& input_sys = m_interpreter.get_input_system ();
+    std::string mfile_encoding;
 
-    std::string mfile_encoding = input_sys.mfile_encoding ();
+    if (m_encoding.empty ())
+      {
+        input_system& input_sys = m_interpreter.get_input_system ();
+        mfile_encoding = input_sys.mfile_encoding ();
+      }
+    else
+      mfile_encoding = m_encoding;
 
     std::string encoding;
     if (mfile_encoding.compare ("system") == 0)
@@ -1402,3 +1507,70 @@
 
   return input_sys.mfile_encoding (args, nargout);
 }
+
+DEFMETHOD (dir_encoding, interp, args, nargout,
+           doc: /* -*- texinfo -*-
+@deftypefn {}  {@var{current_encoding} =} dir_encoding (@var{dir})
+@deftypefnx {} {@var{prev_encoding} =} dir_encoding (@var{dir}, @var{encoding})
+@deftypefnx {} {} dir_encoding (@dots{})
+Set and query the @var{encoding} that is used for reading m-files in @var{dir}.
+
+That encoding overrides the (globally set) m-file encoding.
+
+The string @var{DIR} must match the form how the directory would appear in the
+load path.
+
+The @var{encoding} must be a valid encoding identifier or @code{"delete"}.  In
+the latter case, the (globally set) m-file encoding will be used for the given
+@var{dir}.
+
+The currently or previously used encoding is returned in @var{current_encoding}
+or @var{prev_encoding}, respectively.  The output argument must be explicitly
+requested.
+
+The directory encoding is automatically read from the file @file{.oct_config}
+when a new path is added to the load path (for example with @code{addpath}).
+To set the encoding for all files in the same folder, that file must contain
+a line starting with @code{"encoding="} followed by the encoding identifier.
+
+For example to set the file encoding for all files in the same folder to
+ISO 8859-1 (Latin-1), create a file @file{.oct_config} with the following
+content:
+
+@example
+encoding=iso8859-1
+@end example
+
+If the file encoding is changed after the files have already been parsed, the
+files have to be parsed again for that change to take effect.  That can be done
+with the command @code{clear all}.
+
+@seealso{addpath, path}
+@end deftypefn */)
+{
+  int nargin = args.length ();
+
+  if (nargin < 1 || nargin > 2)
+    print_usage ();
+
+  std::string dir
+    = args(0).xstring_value ("dir_encoding: DIR must be a string");
+
+  octave_value retval;
+
+  octave::input_system& input_sys = interp.get_input_system ();
+
+  if (nargout > 0)
+    retval = input_sys.dir_encoding (dir);
+
+  if (nargin > 1)
+    {
+      std::string encoding
+        = args(1).xstring_value ("dir_encoding: ENCODING must be a string");
+
+      input_sys.set_dir_encoding (dir, encoding);
+    }
+
+  return ovl (retval);
+
+}
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/corefcn/input.h
--- a/libinterp/corefcn/input.h	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/corefcn/input.h	Fri Sep 18 17:15:32 2020 +0200
@@ -34,6 +34,7 @@
 
 #include <memory>
 #include <string>
+#include <unordered_map>
 
 #include "hook-fcn.h"
 #include "oct-time.h"
@@ -147,6 +148,10 @@
 
     void set_mfile_encoding (const std::string& s) { m_mfile_encoding = s; }
 
+    std::string dir_encoding (const std::string& dir);
+
+    void set_dir_encoding (const std::string& dir, std::string& enc);
+
     bool yes_or_no (const std::string& prompt);
 
     std::string interactive_input (const std::string& s, bool& eof);
@@ -184,6 +189,9 @@
     // Codepage which is used to read .m files
     std::string m_mfile_encoding;
 
+    // map of directories -> used mfile encoding
+    std::unordered_map<std::string, std::string> m_dir_encoding;
+
     // If we are in debugging mode, this is the last command entered,
     // so that we can repeat the previous command if the user just
     // types RET.
@@ -241,6 +249,8 @@
 
     input_reader (interpreter& interp, FILE *file);
 
+    input_reader (interpreter& interp, FILE *file, const std::string& enc);
+
     input_reader (interpreter& interp, const std::string& str);
 
     input_reader (const input_reader& ir) = default;
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/corefcn/interpreter.cc
--- a/libinterp/corefcn/interpreter.cc	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/corefcn/interpreter.cc	Fri Sep 18 17:15:32 2020 +0200
@@ -1392,6 +1392,7 @@
     // FIXME: should these actions be handled as a list of functions
     // to call so users can add their own chdir handlers?
 
+    m_load_path.read_dir_config (".");
     m_load_path.update ();
 
     m_event_manager.directory_changed (sys::env::get_current_directory ());
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/corefcn/load-path.cc
--- a/libinterp/corefcn/load-path.cc	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/corefcn/load-path.cc	Fri Sep 18 17:15:32 2020 +0200
@@ -28,6 +28,7 @@
 #endif
 
 #include <algorithm>
+#include <cctype>
 
 #include "dir-ops.h"
 #include "file-ops.h"
@@ -1084,6 +1085,8 @@
           {
             if (fs.is_dir ())
               {
+                read_dir_config (dir);
+
                 dir_info di (dir);
 
                 if (at_end)
@@ -1134,6 +1137,78 @@
       }
   }
 
+  void
+  load_path::read_dir_config (const std::string& dir) const
+  {
+    // read file with directory configuration
+    std::string conf_file = dir + sys::file_ops::dir_sep_str ()
+                            + ".oct_config";
+
+    FILE* cfile = sys::fopen (conf_file, "rb");
+
+    if (! cfile)
+      {
+        // reset directory encoding
+        input_system& input_sys
+          = __get_input_system__ ("load_path::read_dir_config");
+
+        std::string enc_val = "delete";
+        input_sys.set_dir_encoding (dir, enc_val);
+        return;
+      }
+
+    unwind_action close_file ([cfile] (void) { fclose (cfile); });
+
+    // find line with character encoding and read it
+    bool eof = false;
+    const std::string enc_prop = "encoding";
+    while (! eof)
+      {
+        std::string conf_str = octave_fgets (cfile, eof);
+
+        // delete any preceeding whitespace
+        auto it = std::find_if_not (conf_str.begin (), conf_str.end (),
+                                    [] (unsigned char c)
+                                    { return std::isblank (c); });
+        conf_str.erase (conf_str.begin (), it);
+
+        // match identifier
+        if (conf_str.compare (0, enc_prop.size (), enc_prop) == 0)
+          {
+            // skip delimiter characters
+            size_t pos = conf_str.find_first_not_of (" \t=:",
+                                                     enc_prop.size ());
+            if (pos == std::string::npos)
+              continue;
+
+            std::string enc_val = conf_str.substr (pos);
+
+            // take alphanumeric and '-' characters
+            it = std::find_if_not (enc_val.begin (), enc_val.end (),
+                                   [] (unsigned char c)
+                                   { return std::isalnum (c) || c == '-'; });
+            enc_val.erase(it, enc_val.end ());
+
+            if (enc_val.empty ())
+              continue;
+
+            // set encoding for this directory in input system
+            input_system& input_sys
+              = __get_input_system__ ("load_path::read_dir_config");
+            input_sys.set_dir_encoding (dir, enc_val);
+            return;
+          }
+      }
+
+    // reset directory encoding
+    input_system& input_sys
+      = __get_input_system__ ("load_path::read_dir_config");
+
+    std::string enc_val = "delete";
+    input_sys.set_dir_encoding (dir, enc_val);
+
+  }
+
   bool
   load_path::is_package (const std::string& name) const
   {
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/corefcn/load-path.h
--- a/libinterp/corefcn/load-path.h	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/corefcn/load-path.h	Fri Sep 18 17:15:32 2020 +0200
@@ -191,6 +191,8 @@
       remove_hook = f;
     }
 
+    void read_dir_config (const std::string& dir) const;
+
     void execute_pkg_add (const std::string& dir);
     void execute_pkg_del (const std::string& dir);
 
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/parse-tree/lex.h
--- a/libinterp/parse-tree/lex.h	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/parse-tree/lex.h	Fri Sep 18 17:15:32 2020 +0200
@@ -774,6 +774,10 @@
       : base_lexer (interp), m_reader (interp, file), m_initial_input (true)
     { }
 
+    lexer (FILE *file, interpreter& interp, const std::string& encoding)
+      : base_lexer (interp), m_reader (interp, file, encoding), m_initial_input (true)
+    { }
+
     lexer (const std::string& eval_string, interpreter& interp)
       : base_lexer (interp), m_reader (interp, eval_string),
         m_initial_input (true)
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/parse-tree/oct-parse.yy
--- a/libinterp/parse-tree/oct-parse.yy	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/parse-tree/oct-parse.yy	Fri Sep 18 17:15:32 2020 +0200
@@ -4998,7 +4998,9 @@
 
     unwind_action act ([=] (void) { ::fclose (ffile); });
 
-    parser parser (ffile, interp);
+    // get the encoding for this folder
+    octave::input_system& input_sys = interp.get_input_system ();
+    parser parser (ffile, interp, input_sys.dir_encoding (dir_name));
 
     parser.m_curr_class_name = dispatch_type;
     parser.m_curr_package_name = package_name;
diff -r 1807a5762cba -r b1be934e4cd3 libinterp/parse-tree/parse.h
--- a/libinterp/parse-tree/parse.h	Wed Oct 07 09:38:48 2020 +0200
+++ b/libinterp/parse-tree/parse.h	Fri Sep 18 17:15:32 2020 +0200
@@ -610,6 +610,10 @@
       : base_parser (*(new lexer (file, interp)))
     { }
 
+    parser (FILE *file, interpreter& interp, std::string encoding)
+      : base_parser (*(new lexer (file, interp, encoding)))
+    { }
+
     parser (const std::string& eval_string, interpreter& interp)
       : base_parser (*(new lexer (eval_string, interp)))
     { }
