
Sitemap: https://archive.org/sitemap/sitemap.xml

##############################################
#
# Welcome to the Archive!
#
##############################################
# Please crawl our files.
# We appreciate if you can crawl responsibly.
# Stay open!
##############################################


# slow down the ask jeeves crawler which was hitting our SE a little too fast
# via collection pages.   --Feb2008 tracey--
User-agent: Teoma
Disallow: /control/
Disallow: /report/


User-agent: *
Disallow: /control/
Disallow: /report/
Disallow: /details/goldenbull2007john/
Disallow: /stream/goldenbull2007john/
Disallow: /download/goldenbull2007john/
Disallow: /14/items/goldenbull2007john/goldenbull2007john_djvu.txt
