# HG changeset patch # User dtd@skybuilders.com # Date 1229292350 18000 # Node ID d80dbcc7fcff9c9afde7d8c1b194ab4f393a3e21 # Parent 5a0a7c8b8b8b6f07c47ca17043dca5113e90a2f0 Heuristic mechanism to refuse to parse files that do not appear to be HTML. diff -r 5a0a7c8b8b8b -r d80dbcc7fcff src/ChangeLog --- a/src/ChangeLog Fri Nov 07 22:06:38 2008 +0100 +++ b/src/ChangeLog Sun Dec 14 17:05:50 2008 -0500 @@ -1,3 +1,10 @@ +2008-11-30 Derek T. Doyle + + * html-url.c (get_urls_html): Heuristic mechanism to refuse to + parse files that do not appear to be HTML by checking for the + existence of some recognized tag within the first 1k. Feature + listed as bug #20653. + 2008-11-05 Micah Cowan * ftp.c (print_length): Should print humanized "size remaining" diff -r 5a0a7c8b8b8b -r d80dbcc7fcff src/html-url.c --- a/src/html-url.c Fri Nov 07 22:06:38 2008 +0100 +++ b/src/html-url.c Sun Dec 14 17:05:50 2008 -0500 @@ -173,6 +173,29 @@ static struct hash_table *interesting_tags; static struct hash_table *interesting_attributes; + +/* tag_recognizers documents which tags identify a file as + recognizably HTML. It is used by get_urls_html. */ +static const char *tag_recognizers[] = { + "", + "/>" +}; static void init_interesting (void) @@ -622,6 +645,13 @@ struct file_memory *fm; struct map_context ctx; int flags; + size_t fmc_size; + size_t fk_size; + char fk[1025]; /* (up to) the first k of the file */ + char tr[32]; /* the tag to be recognized */ + char *tag_recognized; /* whether the tag was recognized */ + size_t i; + size_t j; /* Load the file. */ fm = read_file (file); @@ -631,6 +661,37 @@ return NULL; } DEBUGP (("Loaded %s (size %s).\n", file, number_to_static_string (fm->length))); + + /* Get (up to) the first k of the file and lower case it for analysis. */ + fmc_size = strlen(fm->content); + if (fmc_size > 1024) + fk_size = 1024; + else + fk_size = fmc_size; + strncpy(fk, fm->content, fk_size); + fk[fk_size + 1] = '\0'; + + for (i = 0; i < fk_size; i++) + fk[i] = c_tolower (fk[i]); + + /* Iterate tags to be recognized and possible suffixes. */ + for (i = 0; i < countof (tag_recognizers); i++) + { + for (j = 0; j < countof (tag_suffixes); j++) + { + strcpy(tr, tag_recognizers[i]); + strcat (tr, tag_suffixes[j]); + if (tag_recognized == NULL) + /* Recognize for tag concatenated with suffix. */ + tag_recognized = strstr (fk, tr); + else + { + /* exit inner and outer loop */ + i = countof (tag_recognizers); + break; + } + } + } ctx.text = fm->content; ctx.head = ctx.tail = NULL; @@ -653,9 +714,12 @@ if (opt.strict_comments) flags |= MHT_STRICT_COMMENTS; - /* the NULL here used to be interesting_tags */ - map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags, + if (tag_recognized) + { + /* the NULL here used to be interesting_tags */ + map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags, NULL, interesting_attributes); + } DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow)); if (meta_disallow_follow)