## Copyright (C) 2014-2022 Philip Nienhuis
##
## This program is free software; you can redistribute it and/or modify it
## under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see .
## -*- texinfo -*-
## @deftypefn {Function File} [@var{data}, @var{datinfo}] = dbfread (@var{fname})
## @deftypefnx {Function File} [@var{data}, @var{datinfo}] = dbfread (@var{fname}, @var{recs})
## @deftypefnx {Function File} [@var{data}, @var{datinfo}] = dbfread (@var{fname}, @var{recs}, @var{cols})
## @deftypefnx {Function File} [@var{data}, @var{datinfo}] = dbfread (@var{fname}, @var{recs}, @var{cols}, @var{re})
## Read contents of a dbase (dbf) file, provisionally dbase III+, IV or V.
##
## @itemize
## @item
## @var{fname} should be the name of a valid dbase file; the file extension
## isn't required.
##
## @item
## @var{recs} can be an integer or logical array containing record numbers or
## record indicators for those records that need to be returned. If omitted,
## all records are read. Indices supplied in @var{recs} can be specified in
## any order, but the returned data are sorted in order of records in the file.
##
## @item
## @var{cols} can be a logical, integer, cellstr or character array indicating
## from which file columns the data should be returned. If a numeric array is
## supplied, it is considered to be like a logical array if the maximum entry
## value equals 1. Character arrays should have column names stacked in the
## vertical (first) dimension. @var{cols} entries can be supplied in any
## order, yet the returned data column order matches that of the columns order
## in the dbase file. Requested column names that don't exist in the .dbf
## file are simply ignored. For dbase files containing multiple columns with
## the same name, specify a numeric or logical array to select columns to be
## returned. If @var{cols} is omitted or empty, data from all file columns
## are returned.
##
## @item
## If a value of 1 or true is entered for @var{re}, dbfread also tries to
## return data from erased records. No guarantee can be given for these data
## to be correct or consistent! If omitted, erased records are skipped. The
## default value of @var{re} is false.
##
## @item
## Return value @var{data} is a N+1 x M cellstr array where the uppermost row
## contains the column names and the rest of the rows comprise the record data.
##
## @item
## Optional return argument @var{datinfo} is a struct array containing various
## information of the dbase file and record build-up.
## @end itemize
##
## Arguments @var{recs} and @var{cols} need not be as long as the number of
## records and columns in the file, resp.; dbfread will stop reading data if
## any of @var{recs} or @var{cols} (if supplied) is exhausted.
##
## Sometimes dbase files contain records indicated as being erased. The data
## in such records is silently skipped, unless the @var{re} flag is set
## and/or @var{recs} is supplied and erased records happen to be present in the
## requested record numbers.
##
## Examples:
##
## @example
## A = dbfread ("file.dbf");
## (returns all data in file.dbf in array A)
## @end example
##
##
## @example
## [A, B] = dbfread ("file.dbf", [], ["colB"; "colF"]);
## (returns all data in columns named "colB" and "colF" from
## file.dbf in array A and information on the database
## build-up in struct B)
## @end example
##
## @example
## A = dbfread ("file.dbf", [0 1 0 0 1 0 0]);
## -or-
## A = dbfread ("file.dbf", [2 5]);
## (returns data from record numbers 2 and 5 in
## file.dbf in array A)
## @end example
##
## @example
## A = dbfread ("file", [0 1 0 0 1 0]);
## (returns data from record numbers 2 and 5 in
## file.dbf in array A)
## @end example
##
## @example
## [~, B] = dbfread ("file.dbf", 0);
## (to returns info on column names and number of
## records, plus more info)
## @end example
##
## @example
## [A] = dbfread ("file", [], @{"Header1", "Col5"@});
## (returns data from columns with names (headers)
## Header1 and Col5, resp.)
## @end example
##
## @seealso{xlsread}
## @end deftypefn
## Author: Philip Nienhuis
## Created: 2014-11-03
## References:
## http://ulisse.elettra.trieste.it/services/doc/dbase/DBFstruct.htm
## http://www.dbf2002.com/dbf-file-format.html
## http://www.dbase.com/KnowledgeBase/int/db7_file_fmt.htm
function [data, datinfo] = dbfread (fname, recs=[], cols=[], rd_erased="")
## Check file name
if (ischar (fname))
[~, fnm, ext] = fileparts (fname);
if (isempty (ext))
fname = [fname ".dbf"];
endif
else
error ("dbfread: file name expected for arg # 1.\n");
endif
## Check recs arg. If needed turn into indices
if (! isempty (recs))
if (! (isnumeric (recs) || islogical (recs)))
error ("dbfread: numeric or logical array expected for arg # 2\n");
elseif (isnumeric (recs))
if (any (recs < 0))
error ("dbfread: illegal record selection indices\n");
elseif (min (recs) == 0 && max (recs) < 2)
recs = find (recs);
if (isempty (recs))
recs = -1;
endif
endif
elseif (islogical (recs))
recs = find (recs);
endif
recs = sort (recs);
endif
## Check cols arg. If needed turn into indices
if (! isempty (cols))
if (! (isnumeric (cols) || ischar (cols) || iscellstr (cols) || islogical (cols)))
error ("dbfread: numeric, cellstr, logical, or character array expected for arg # 3\n");
elseif (isnumeric (cols))
if (any (cols < 0))
error ("dbfread: illegal column selection indices\n");
elseif (max (cols) < 2)
## Input as 0 or 1 ("logicals")
cols = find (cols);
if (isempty (cols))
cols = -1;
endif
endif
elseif (islogical (cols))
cols = find (cols);
elseif (ischar (cols))
cols = cellstr (cols);
endif
endif
## Check rd_erased arg.
if (! isempty (rd_erased) && ! (islogical (rd_erased) || isnumeric (rd_erased)))
error ("dbfread: numeric or logical value expected for arg # 4\n");
endif
## Open file
fid = fopen (fname, "r", "ieee-le");
if (fid < 0)
error ("dbfread: file %s couldn't be opened.\n", fname);
endif
## Rewind, just to be sure
fseek (fid, 0, "bof");
## First check proper type
fbyte = uint8 (fread (fid, 1, "uint8"));
## Provisional type check.
## 3 = dbase III+ w/o memos
## 83 = dbase III+ w memos
## ...
vsn = bitand (fbyte, 7);
if (! ismember (vsn, [3, 4, 5]))
error ("dbfread: unsupported file type, only dbase III[+], IV & V supported.\n");
endif
## Memos present for fbyte == 83, and bits 3 and/or 7 set (1-based bit pos.)
hasmemo = (fbyte == 83) || (bitand (fbyte, 8)) || (bitand (fbyte, uint8 (128)));
## Start reading header
lasty = fread (fid, 1, "uint8") + 1900; ## Last dbf update
lastm = fread (fid, 1, "uint8"); ## month
lastd = fread (fid, 1, "uint8"); ## day
nrecs = fread (fid, 1, "uint32");
lhdr = fread (fid, 1, "uint16");
recl = fread (fid, 1, "uint16");
## Field descriptors
nfields = 0;
fseek (fid, 32, "bof");
fdesc = fread (fid, 32, "char=>char")';
do
++nfields;
## Get fields into struct
dbf(nfields).fldnam = deblank (fdesc(1:11)); ## Field name
dbf(nfields).fldtyp = fdesc(12); ## Field type
## Skip field dspl.
dbf(nfields).fldlng = int32 (fdesc(17)); ## Field length
dbf(nfields).flddec = int32 (fdesc(18)); ## Decimal places
dbf(nfields).fldflg = int8 (fdesc(19)); ## Flags
## Get next field descriptors
fdesc = fread (fid, 32, "char=>char")';
until (ftell (fid) >= lhdr)
## Seek to position after header terminator byte
fseek (fid, lhdr, "bof");
## Read rest of data. Skip if no records need be read. Turn into char array
if (isempty (recs))
txt = fread (fid, [recl, nrecs], "char=>char")';
else
txt = fread (fid, [recl, recs(end)], "char=>char")';
endif
## .dbf file is no longer needed
fclose (fid);
## Preallocate upper data row
data = cell (1, numel (dbf));
data(1, :) = {dbf.fldnam};
## If required, select requested records. Beware; truncate indices > nrecs
if (any (recs > nrecs))
recs (find (recs > nrecs)) = [];
## Check if we still have a selection...
if (isempty (recs))
## No more, signal this to below code
recs = -1;
endif
endif
if (! isempty (recs) && ! isempty (txt))
if (any (recs < 0))
## No data returned. Explore erased records anyway
wipedrec = txt(:, 1)' == "*";
txt = "";
recs = 0;
scol = numel (dbf);
else
txt = txt(recs, :);
## Preallocate data cell array for selected records
data = [data; cell(numel (recs), numel (dbf))];
endif
else
## Preallocate data cell array for all records
data = [data; cell(nrecs, numel (dbf))];
endif
if (! isempty (txt))
## There's something to read ;-) First, read memo file, if any
if (hasmemo)
## Also open accompanying .dbt file
fjd = fopen ([fnm ".dbt"], "r");
if (fjd < 0)
warning ("dbfread: associated memo file (%s) couldn't be opened.\n", ...
[fnm ".dbt"]);
printf ("(dbfread: skipping memo fields)\n");
memos = {};
else
fseek (fjd,0, "bof");
## Read memo fields
memos = fread (fjd, Inf, "char=>char")';
fclose (fjd);
## Pimp memos: replace all non-alphanumeric chars by space
memos(find (int8(memos) > 122)) = char(32);
memos(find (int8(memos) < 32)) = char(32);
switch fbyte
case {83, 131} ## Dbase III[+]
## Make it into a Nx512 char array. Pad unit length = multiple of 512
pad = ceil (length (memos) / 512) * 512 - length (memos);
memos = [memos repmat(int8 (32), 1, pad)];
memos = cellstr (reshape (memos, 512, [])');
otherwise
## FIXME: Dbase V, VII to follow
endswitch
endif
endif
txtp = sum ([dbf.fldlng]) + 1;
## Init output array column pointer
scol = 0;
for ii=numel (dbf) : -1 : 1
## First process selection if arg. cols was given...
if (! isempty (cols))
try
## try-catch, as cols array < nr. of cols is allowed.
## Switch dependent of cols input arg. type
switch class (cols)
case "cell"
getcol = ismember (dbf(ii).fldnam, cols);
case "double"
getcol = ! isempty (find (cols == ii));
otherwise
endswitch
catch
getcol = 0;
end_try_catch
else
## No cols arg. was given, so we'll read this column anyway
getcol = 1;
endif
if (getcol)
## Read column # ii
fld = txt(:, txtp-dbf(ii).fldlng+1 : txtp);
switch (dbf(ii).fldtyp)
case {"B", "G"}
## Block number into .dbt file, other than memo
data(2:end, ii) = num2cell (str2double (fld));
case "C"
## Text
data(2:end, ii) = cellstr (fld);
case "D"
## Date
dlf = cellstr (fld);
## Catch empty date fields. Put ridiculous value in
dlf(find (strcmp (cellstr (fld), "00000000"))) = "99991231";
dlf(find (cellfun ("isempty", dlf))) = "99991231";
dlf = datenum (dlf, "yyyymmdd");
## Reset temp values for empty dates
dlf (dlf >= 3652425) = 0;
data(2:end, ii) = num2cell (dlf);
case "L"
## Logical / boolean
data(2:end, ii) = false;
data(regexpi (fld, "[yt]")+1, ii) = true;
case {"F", "N"}
## Numeric
data(2:end, ii) = num2cell (str2double (fld));
case "M"
## Memo field pointer into .dbt file
if (! isempty (memos))
switch (fbyte)
case {83, 131} ## Dbase III[+]
idx = str2double (fld);
idx (find (isnan (idx))) = 1;
data(2:end, ii) = memos(idx);
otherwise
## FIXME other .dbf file versions to be implemented...
endswitch
endif
otherwise
endswitch
else
## Remove data column header & associated column from output array
data(:, ii) = [];
endif
## Next column of data from .dbf file
txtp -= dbf(ii).fldlng;
endfor
## Only now check for erased records, to avoid false positives below
wipedrec = txt(:, 1)' == "*";
if (! rd_erased)
## No erased records in user-supplied record selection => no worries
data (find (wiperec), :) = [];
elseif (sum (wipedrec) && ! isempty (recs))
## User did request erased record # => warn
warning ("(dbfread: %d erased records read\n", wipedrec);
endif
else
wipedrec = [];
scol = 0;
endif
if (isempty (data))
data = {};
endif
if (nargout <= 1)
## Only data requested
datinfo = [];
else
## Also (or only) dbf info struct requested
if (isempty (recs))
## Infer nr. of records from file
recs = size (data, 1) - 1;
endif
datinfo.type = fbyte;
datinfo.date = datenum (lasty, lastm, lastd);
datinfo.erasedrec = find (wipedrec);
datinfo.nrec = nrecs;
datinfo.srecs = recs;
datinfo.recl = recl;
datinfo.ncols = numel (dbf);
datinfo.data = dbf;
endif
endfunction