set_time_limit(3600);
// Don't let PHP timeout
$SEED_URL = "http://www.schrenk.com";
// First URL spider downloads
$MAX_PENETRATION = 1;
// Set spider penetration depth
$FETCH_DELAY = 1;
// Wait one second between page fetches
$ALLOW_OFFISTE = true;
// Don't allow spider to roam from the SEED_URL's domain
$spider_array = array();
# Get links from $SEED_URL
echo "Harvesting Seed URL \n";
$temp_link_array = harvest_links($SEED_URL);
$spider_array = archive_links($spider_array, 0, $temp_link_array);
# Spider links in remaining penetration levels
for ($penetration_level = 1; $penetration_level <= $MAX_PENETRATION; $penetration_level++) {
$previous_level = $penetration_level - 1;
for ($xx = 0; $xx < count($spider_array[$previous_level]); $xx++) {
unset($temp_link_array);
$temp_link_array = harvest_links($spider_array[$previous_level][$xx]);
echo "Level={$penetration_level}, xx={$xx} of " . count($spider_array[$previous_level]) . " <br>\n";
$spider_array = archive_links($spider_array, $penetration_level, $temp_link_array);
}
}
# Download images from pages referenced in $spider_array
for ($penetration_level = 1; $penetration_level <= $MAX_PENETRATION; $penetration_level++) {
for ($xx = 0; $xx < count($spider_array[$previous_level]); $xx++) {
download_images_for_page($spider_array[$previous_level][$xx]);
}
}
THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR
PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS,
TRADEMARKS OR OTHER RIGHTS.
COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF
ANY USE OF THE SOFTWARE OR DOCUMENTATION.
The name and trademarks of copyright holders may NOT be used in advertising or publicity pertaining to the
software without specific, written prior permission. Title to copyright in this software and any associated
documentation will at all times remain with copyright holders.
Copyright 2007, Michael Schrenk
THIS SCRIPT IS FOR DEMONSTRATION PURPOSES ONLY!
It is not suitable for any use other than demonstrating
the concepts presented in Webbots, Spiders and Screen Scrapers.
########################################################################
*/
?>
<?php
##################################################################
# Example: Download All Images referenced in a web page
#-----------------------------------------------------------------
include "../util/LIB_download_images.php";
$target = "http://www.nasa.gov/mission_pages/viking/index.html";
download_images_for_page($target);
