# HG changeset patch
# User Andrew Janke <andrew@apjanke.net>
# Date 1547958405 18000
#      Sat Jan 19 23:26:45 2019 -0500
# Branch forge-pkg-list-2
# Node ID 903c954a0ce153cb125f112775c9e2c99f65b410
# Parent  570170dc46e5b7eaef0f2357e5aec5ec8d6b02af
pkg -forge: use new package list CSV file

diff --git a/scripts/pkg/pkg.m b/scripts/pkg/pkg.m
--- a/scripts/pkg/pkg.m
+++ b/scripts/pkg/pkg.m
@@ -161,20 +161,26 @@
 ## and those which were installed by the system administrator.
 ##
 ## @example
 ## [user_packages, system_packages] = pkg ("list")
 ## @end example
 ##
 ## The @qcode{"-forge"} option lists packages available at the Octave Forge
 ## repository.  This requires an internet connection and the cURL library.
+##
+## If the argouts are captured, the first argout is a cellst list of package
+## names. The second argout is a struct with fields Name, CurrentVersion, and
+## possibly others.
+##
 ## For example:
 ##
 ## @example
 ## oct_forge_pkgs = pkg ("list", "-forge")
+## [names, info] = pkg ("list", "-forge")
 ## @end example
 ##
 ## @item describe
 ## Show a short description of installed packages.  With the option
 ## @qcode{"-verbose"} also list functions provided by the package.  For
 ## example,
 ##
 ## @example
@@ -372,17 +378,17 @@ function [local_packages, global_package
     error ("pkg: '-forge' can only be used with install or list");
   endif
 
   ## Take action
   switch (action)
     case "list"
       if (octave_forge)
         if (nargout)
-          local_packages = list_forge_packages ();
+          [local_packages, global_packages] = list_forge_packages ();
         else
           list_forge_packages ();
         endif
       else
         if (nargout == 1)
           local_packages = installed_packages (local_list, global_list, files);
         elseif (nargout > 1)
           [local_packages, global_packages] = installed_packages (local_list,
diff --git a/scripts/pkg/private/forge_packages_base_url.m b/scripts/pkg/private/forge_packages_base_url.m
new file mode 100644
--- /dev/null
+++ b/scripts/pkg/private/forge_packages_base_url.m
@@ -0,0 +1,18 @@
+function out = forge_packages_base_url ()
+  persistent last_override is_initialized
+  if isempty (is_initialized)
+    is_initialized = true;
+    last_override = [];
+  end
+
+  out = "https://packages.octave.org";
+  % Backdoor for local testing
+  if isappdata (0, "forge.site_url")
+    out = getappdata (0, "forge.site_url");
+    if ~isequal (out, last_override)
+      fprintf ("Using alternate Octave Forge URL: %s\n", out);
+      last_override = out;
+    end
+  endif
+
+endfunction
diff --git a/scripts/pkg/private/get_forge_pkg.m b/scripts/pkg/private/get_forge_pkg.m
--- a/scripts/pkg/private/get_forge_pkg.m
+++ b/scripts/pkg/private/get_forge_pkg.m
@@ -25,64 +25,43 @@
 ## to download the file.
 ## @end deftypefn
 
 function [ver, url] = get_forge_pkg (name)
 
   ## Verify that name is valid.
   if (! (ischar (name) && rows (name) == 1 && ndims (name) == 2))
     error ("get_forge_pkg: package NAME must be a string");
-  elseif (! all (isalnum (name) | name == "-" | name == "." | name == "_"))
+  elseif ! regexp (name, '^[\s._-]+$')
     error ("get_forge_pkg: invalid package NAME: %s", name);
   endif
 
   name = tolower (name);
 
-  ## Try to download package's index page.
-  [html, succ] = urlread (sprintf ("https://packages.octave.org/%s/index.html",
-                                   name));
-  if (succ)
-    ## Remove blanks for simpler matching.
-    html(isspace(html)) = [];
-    ## Good.  Let's grep for the version.
-    pat = "<tdclass=""package_table"">PackageVersion:</td><td>([\\d.]*)</td>";
-    t = regexp (html, pat, "tokens");
-    if (isempty (t) || isempty (t{1}))
-      error ("get_forge_pkg: could not read version number from package's page");
-    else
-      ver = t{1}{1};
-      if (nargout > 1)
-        ## Build download string.
-        pkg_file = sprintf ("%s-%s.tar.gz", name, ver);
-        url = ["https://packages.octave.org/download/" pkg_file];
-        ## Verify that the package string exists on the page.
-        if (isempty (strfind (html, pkg_file)))
-          warning ("get_forge_pkg: download URL not verified");
-        endif
-      endif
-    endif
-  else
-    ## Try get the list of all packages.
-    [html, succ] = urlread ("https://packages.octave.org/list_packages.php");
-    if (! succ)
-      error ("get_forge_pkg: could not read URL, please verify internet connection");
-    endif
-    t = strsplit (html);
-    if (any (strcmp (t, name)))
-      error ("get_forge_pkg: package NAME exists, but index page not available");
+  [pkgnames, pkgsinfo] = list_forge_packages ();
+
+  [tf, loc] = ismember (name, pkgnames);
+  if ~tf
+    ## Check against other package names
+    if (any (strcmp (pkgnames, name)))
+      error ("get_forge_pkg: package ""%s"" exists, but index page not available", ...
+        name);
     endif
     ## Try a simplistic method to determine similar names.
     function d = fdist (x)
-      len1 = length (name);
-      len2 = length (x);
-      if (len1 <= len2)
-        d = sum (abs (name(1:len1) - x(1:len1))) + sum (x(len1+1:end));
-      else
-        d = sum (abs (name(1:len2) - x(1:len2))) + sum (name(len2+1:end));
-      endif
+      min_len = min (length (name), length (x)); 
+      d = sum (abs (name(1:min_len) - x(1:min_len))) + sum (x(min_len+1:end));
     endfunction
     dist = cellfun ("fdist", t);
     [~, i] = min (dist);
     error ("get_forge_pkg: package not found: ""%s"".  Maybe you meant ""%s?""",
-           name, t{i});
+           name, t{i});    
+  endif
+
+  ver = pkgsinfo.CurrentVersion{loc};
+  if (nargout > 1)
+    ## Build download string.
+    base_url = forge_packages_base_url ();
+    pkg_file = sprintf ("%s-%s.tar.gz", name, ver);
+    url = [base_url "/download/" pkg_file];
   endif
 
 endfunction
diff --git a/scripts/pkg/private/list_forge_packages.m b/scripts/pkg/private/list_forge_packages.m
--- a/scripts/pkg/private/list_forge_packages.m
+++ b/scripts/pkg/private/list_forge_packages.m
@@ -17,31 +17,55 @@
 ## along with Octave; see the file COPYING.  If not, see
 ## <https://www.gnu.org/licenses/>.
 
 ## -*- texinfo -*-
 ## @deftypefn {} {@var{list} =} list_forge_packages ()
 ## Undocumented internal function.
 ## @end deftypefn
 
-function list = list_forge_packages ()
-
-  [list, succ] = urlread ("https://packages.octave.org/list_packages.php");
-  if (! succ)
-    error ("pkg: could not read URL, please verify internet connection");
+function [names, info] = list_forge_packages ()
+  
+  base_url =  forge_packages_base_url ();
+  package_list_url = [base_url "/package_list.csv"];
+  [pkg_list_txt, ok] = urlread (package_list_url);
+  if (! ok)
+    error ("pkg: could not read URL, please verify internet connection. URL: %s", ...
+      package_list_url);
   endif
 
-  list = ostrsplit (list, " \n\t", true);
-
+  c = parse_csv_text (pkg_list_txt);
+  header = c(1,1:end);
+  data = c(2:end,:);
+  info = struct;
+  for i = 1:numel(header)
+    info.(header{i}) = c(:,i);
+  endfor
+  names = info.Name;
+  
   if (nargout == 0)
     page_screen_output (false, "local");
     puts ("Octave Forge provides these packages:\n");
-    for i = 1:length (list)
-      try
-        ver = get_forge_pkg (list{i});
-      catch
-        ver = "unknown";
-      end_try_catch
-      printf ("  %s %s\n", list{i}, ver);
+    for i = 1:length (names)
+      printf (" %s %s\n", info.Name{i}, info.CurrentVersion{i});
     endfor
   endif
 
 endfunction
+
+% A hacky, minimal CSV text reader, since csvread() only handles numerics.
+% Does not handle quotes or embedded newlines.
+function out = parse_csv_text (str)
+  str = regexprep (str, '[\r\n]+$', '');
+  lines = regexp (str, '\r?\n', "split");
+  line_data = {};
+  n_cols = 0;
+  for i = 1:numel (lines)
+    line = lines{i};
+    c = regexp (line, '\s*,\s*', "split");
+    line_data{end+1} = c;
+    n_cols = max (n_cols, numel (c));
+  endfor
+  for i = 1:numel (line_data)
+    line_data{i}(end+1:n_cols) = {""};
+  endfor
+  out = cat (1, line_data{:});
+endfunction
