User-agent: *
Disallow: /cgi-bin
Disallow: /cgi-local
User-Agent: Twiceler
Disallow: /
User-Agent: Charlotte
Disallow: / 
User-agent: Slurp
Crawl-delay: 1