From 95aa52eff208bba4e28cd280a45e28ca1598bb95 Mon Sep 17 00:00:00 2001
From: Tomas Mudrunka <tomas.mudrunka@gmail.com>
Date: Fri, 23 Jan 2015 01:03:16 +0100
Subject: [PATCH] Added --level-ext parameter to limit depth of crawling
 external sites

---
 src/init.c    |  2 ++
 src/main.c    |  4 ++++
 src/options.h |  1 +
 src/recur.c   | 51 +++++++++++++++++++++++++++++++++++----------------
 4 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/src/init.c b/src/init.c
index 8c035df..34c7641 100644
--- a/src/init.c
+++ b/src/init.c
@@ -256,6 +256,7 @@ static const struct {
   { "randomwait",       &opt.random_wait,       cmd_boolean },
   { "readtimeout",      &opt.read_timeout,      cmd_time },
   { "reclevel",         &opt.reclevel,          cmd_number_inf },
+  { "reclevel_span",    &opt.reclevel_span,     cmd_number_inf },
   { "recursive",        NULL,                   cmd_spec_recursive },
   { "referer",          &opt.referer,           cmd_string },
   { "regextype",        &opt.regex_type,        cmd_spec_regex_type },
@@ -350,6 +351,7 @@ defaults (void)
   opt.verbose = -1;
   opt.ntry = 20;
   opt.reclevel = 5;
+  opt.reclevel_span = 1;
   opt.add_hostdir = true;
   opt.netrc = true;
   opt.ftp_glob = true;
diff --git a/src/main.c b/src/main.c
index 7675455..57acd75 100644
--- a/src/main.c
+++ b/src/main.c
@@ -242,6 +242,7 @@ static struct cmdline_option option_data[] =
     { "iri", 0, OPT_BOOLEAN, "iri", -1 },
     { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
     { "level", 'l', OPT_VALUE, "reclevel", -1 },
+    { "level-ext", 0, OPT_VALUE, "reclevel_span", -1 },
     { "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
     { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
     { "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
@@ -739,6 +740,8 @@ Recursive download:\n"),
     N_("\
   -l,  --level=NUMBER              maximum recursion depth (inf or 0 for infinite)\n"),
     N_("\
+  --level-ext=NUMBER              maximum recursion depth (inf or 0 for infinite) out of site (when using -H)\n"),
+    N_("\
        --delete-after              delete files locally after downloading them\n"),
     N_("\
   -k,  --convert-links             make links in downloaded HTML or CSS point to\n\
@@ -1233,6 +1236,7 @@ main (int argc, char **argv)
    *  this no-op, we explicitly set opt.debug to false and hence none of the
    *  Debug output messages will be printed.
    */
+
 #ifndef ENABLE_DEBUG
   if (opt.debug)
     {
diff --git a/src/options.h b/src/options.h
index ffb04a9..7de2148 100644
--- a/src/options.h
+++ b/src/options.h
@@ -47,6 +47,7 @@ struct options
   bool no_parent;               /* Restrict access to the parent
                                    directory.  */
   int reclevel;                 /* Maximum level of recursion */
+  int reclevel_span;            /* Maximum level of recursion out of site */
   bool dirstruct;               /* Do we build the directory structure
                                    as we go along? */
   bool no_dirstruct;            /* Do we hate dirstruct? */
diff --git a/src/recur.c b/src/recur.c
index b6b9dc6..12d7551 100644
--- a/src/recur.c
+++ b/src/recur.c
@@ -57,6 +57,7 @@ struct queue_element {
   const char *url;              /* the URL to download */
   const char *referer;          /* the referring document */
   int depth;                    /* the depth */
+  int depth_span;                    /* the depth of host spaning */
   bool html_allowed;            /* whether the document is allowed to
                                    be treated as HTML. */
   struct iri *iri;                /* sXXXav */
@@ -94,7 +95,7 @@ url_queue_delete (struct url_queue *queue)
 
 static void
 url_enqueue (struct url_queue *queue, struct iri *i,
-             const char *url, const char *referer, int depth,
+             const char *url, const char *referer, int depth, int depth_span,
              bool html_allowed, bool css_allowed)
 {
   struct queue_element *qel = xnew (struct queue_element);
@@ -102,6 +103,7 @@ url_enqueue (struct url_queue *queue, struct iri *i,
   qel->url = url;
   qel->referer = referer;
   qel->depth = depth;
+  qel->depth_span = depth_span;
   qel->html_allowed = html_allowed;
   qel->css_allowed = css_allowed;
   qel->next = NULL;
@@ -131,7 +133,7 @@ url_enqueue (struct url_queue *queue, struct iri *i,
 
 static bool
 url_dequeue (struct url_queue *queue, struct iri **i,
-             const char **url, const char **referer, int *depth,
+             const char **url, const char **referer, int *depth, int *depth_span,
              bool *html_allowed, bool *css_allowed)
 {
   struct queue_element *qel = queue->head;
@@ -147,6 +149,7 @@ url_dequeue (struct url_queue *queue, struct iri **i,
   *url = qel->url;
   *referer = qel->referer;
   *depth = qel->depth;
+  *depth_span = qel->depth_span;
   *html_allowed = qel->html_allowed;
   *css_allowed = qel->css_allowed;
 
@@ -181,9 +184,9 @@ static int blacklist_contains (struct hash_table *blacklist, const char *url)
   return ret;
 }
 
-static bool download_child_p (const struct urlpos *, struct url *, int,
+static bool download_child_p (const struct urlpos *, struct url *, int, int,
                               struct url *, struct hash_table *, struct iri *);
-static bool descend_redirect_p (const char *, struct url *, int,
+static bool descend_redirect_p (const char *, struct url *, int, int,
                                 struct url *, struct hash_table *, struct iri *);
 
 
@@ -239,7 +242,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
   /* Enqueue the starting URL.  Use start_url_parsed->url rather than
      just URL so we enqueue the canonical form of the URL.  */
-  url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, true,
+  url_enqueue (queue, i, xstrdup (start_url_parsed->url), NULL, 0, 0, true,
                false);
   blacklist_add (blacklist, start_url_parsed->url);
 
@@ -248,6 +251,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
       bool descend = false;
       char *url, *referer, *file = NULL;
       int depth;
+      int depth_span;
       bool html_allowed, css_allowed;
       bool is_css = false;
       bool dash_p_leaf_HTML = false;
@@ -261,7 +265,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
 
       if (!url_dequeue (queue, (struct iri **) &i,
                         (const char **)&url, (const char **)&referer,
-                        &depth, &html_allowed, &css_allowed))
+                        &depth, &depth_span, &html_allowed, &css_allowed))
         break;
 
       /* ...and download it.  Note that this download is in most cases
@@ -326,7 +330,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                  want to follow it.  */
               if (descend)
                 {
-                  if (!descend_redirect_p (redirected, url_parsed, depth,
+                  if (!descend_redirect_p (redirected, url_parsed, depth, depth_span,
                                            start_url_parsed, blacklist, i))
                     descend = false;
                   else
@@ -352,10 +356,18 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
         }
 
       if (descend
-          && depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION)
+          && (
+              (depth >= opt.reclevel && opt.reclevel != INFINITE_RECURSION)
+              || (depth_span >= opt.reclevel_span && opt.reclevel_span != INFINITE_RECURSION)
+             )
+         )
         {
           if (opt.page_requisites
-              && (depth == opt.reclevel || depth == opt.reclevel + 1))
+              && (
+                   (depth == opt.reclevel || depth == opt.reclevel + 1)
+                   || (depth_span == opt.reclevel_span || depth_span == opt.reclevel_span + 1)
+                 )
+             )
             {
               /* When -p is specified, we are allowed to exceed the
                  maximum depth, but only for the "inline" links,
@@ -413,13 +425,20 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
                     continue;
                   if (dash_p_leaf_HTML && !child->link_inline_p)
                     continue;
-                  if (download_child_p (child, url_parsed, depth, start_url_parsed,
+                  if (download_child_p (child, url_parsed, depth, depth_span, start_url_parsed,
                                         blacklist, i))
                     {
                       ci = iri_new ();
                       set_uri_encoding (ci, i->content_encoding, false);
+
+                      /* Increment depth of span if we are spaning or spaned in past */
+                      int depth_span_inc = 0;
+                      if(depth_span > 0) depth_span_inc = 1;
+                      if (0 != strcasecmp (url_parsed->host, child->url->host))
+                        depth_span_inc = 1;
+
                       url_enqueue (queue, ci, xstrdup (child->url->url),
-                                   xstrdup (referer_url), depth + 1,
+                                   xstrdup (referer_url), depth + 1, depth_span + depth_span_inc,
                                    child->link_expect_html,
                                    child->link_expect_css);
                       /* We blacklist the URL we have enqueued, because we
@@ -470,11 +489,11 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
      now.  */
   {
     char *d1, *d2;
-    int d3;
+    int d3, d7;
     bool d4, d5;
     struct iri *d6;
     while (url_dequeue (queue, (struct iri **)&d6,
-                        (const char **)&d1, (const char **)&d2, &d3, &d4, &d5))
+                        (const char **)&d1, (const char **)&d2, &d3, &d7, &d4, &d5))
       {
         iri_free (d6);
         xfree (d1);
@@ -502,7 +521,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
    will help if those URLs are encountered many times.  */
 
 static bool
-download_child_p (const struct urlpos *upos, struct url *parent, int depth,
+download_child_p (const struct urlpos *upos, struct url *parent, int depth, int depth_span,
                   struct url *start_url_parsed, struct hash_table *blacklist,
                   struct iri *iri)
 {
@@ -714,7 +733,7 @@ download_child_p (const struct urlpos *upos, struct url *parent, int depth,
    it is merely a simple-minded wrapper around download_child_p.  */
 
 static bool
-descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
+descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth, int depth_span,
                     struct url *start_url_parsed, struct hash_table *blacklist,
                     struct iri *iri)
 {
@@ -730,7 +749,7 @@ descend_redirect_p (const char *redirected, struct url *orig_parsed, int depth,
   upos = xnew0 (struct urlpos);
   upos->url = new_parsed;
 
-  success = download_child_p (upos, orig_parsed, depth,
+  success = download_child_p (upos, orig_parsed, depth, depth_span,
                               start_url_parsed, blacklist, iri);
 
   if (success)
-- 
2.2.2

