# HG changeset patch # User dtd@skybuilders.com # Date 1227484217 18000 # Node ID 8702fee342285107cc62f33b1dd6c2cc3868a8b3 # Parent 5a0a7c8b8b8b6f07c47ca17043dca5113e90a2f0 Heuristic mechanism to refuse to parse files that do not appear to be HTML. diff -r 5a0a7c8b8b8b -r 8702fee34228 src/ChangeLog --- a/src/ChangeLog Fri Nov 07 22:06:38 2008 +0100 +++ b/src/ChangeLog Sun Nov 23 18:50:17 2008 -0500 @@ -1,3 +1,10 @@ +2008-11-23 Derek T. Doyle + + * html-url.c (get_urls_html): Heuristic mechanism to refuse to + parse files that do not appear to be HTML by checking for the + existence of some recognized tag within the first 1k. Feature + listed as bug #20653. + 2008-11-05 Micah Cowan * ftp.c (print_length): Should print humanized "size remaining" diff -r 5a0a7c8b8b8b -r 8702fee34228 src/html-url.c --- a/src/html-url.c Fri Nov 07 22:06:38 2008 +0100 +++ b/src/html-url.c Sun Nov 23 18:50:17 2008 -0500 @@ -622,6 +622,15 @@ struct file_memory *fm; struct map_context ctx; int flags; + static char fk[1024]; + const char *doctype = "length))); + + /* Get the first K of the file for analysis */ + strncpy(fk, fm->content, 1024); + tag_recognized = strstr(fk, doctype); + if (tag_recognized == NULL) + tag_recognized = strstr(fk, html_lc); + if (tag_recognized == NULL) + tag_recognized = strstr(fk, html_uc); + if (tag_recognized == NULL) + tag_recognized = strstr(fk, head_lc); + if (tag_recognized == NULL) + tag_recognized = strstr(fk, head_uc); + if (tag_recognized == NULL) + tag_recognized = strstr(fk, body_lc); + if (tag_recognized == NULL) + tag_recognized = strstr(fk, body_uc); ctx.text = fm->content; ctx.head = ctx.tail = NULL; @@ -653,9 +678,12 @@ if (opt.strict_comments) flags |= MHT_STRICT_COMMENTS; - /* the NULL here used to be interesting_tags */ - map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags, + if (tag_recognized) + { + /* the NULL here used to be interesting_tags */ + map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags, NULL, interesting_attributes); + } DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow)); if (meta_disallow_follow)