#
# $ClearChain: www/data/robots.txt,v 1.2 2004/02/06 02:30:47 benjsc Exp $
#
# This file aids in providing web crawling software with restrictions on
# the content they should index
#
User-agent: *
Disallow: /mailman/
Disallow: /pipermail/
Disallow: /~benjsc/temp
Crawl-delay: 0.5

#
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
#
User-agent: wget
Disallow: /

#
# Wiki Requests
#

# Don't index non article wiki pages
User-agent: *
Disallow: /blog/wp-admin/