# Novacaster default robots.txt

User-agent: *
Disallow: /admin
Disallow: /ads.pl
Disallow: /anon
Disallow: /apply
Disallow: /asearc
Disallow: /attach
Disallow: /capab
Disallow: /cart
Disallow: /categ
Disallow: /db
Disallow: /desig
Disallow: /direct
Disallow: /edit
Disallow: /email
Disallow: /file
Disallow: /forgot
Disallow: /groups
Disallow: /keywor
Disallow: /log
Disallow: /lost
Disallow: /mail
Disallow: /master
Disallow: /navaid
Disallow: /new
Disallow: /poll
Disallow: /price
Disallow: /prog
Disallow: /reply
Disallow: /roam
Disallow: /schema
Disallow: /search
Disallow: /shop
Disallow: /showap
Disallow: /showjo
Disallow: /showpu
Disallow: /stat
Disallow: /sub
Disallow: /themes
Disallow: /tofrie
Disallow: /cgi
Disallow: /blacklist
Disallow: /*rss=1
Disallow: /*?PDA=1
Disallow: /*.php
# Yahoo-specific extension
Crawl-delay: 5

User-Agent: Googlebot
Disallow: /admin
Disallow: /ads.pl
Disallow: /anon
Disallow: /apply
Disallow: /asearc
Allow: /attach
Disallow: /capab
Disallow: /cart
Disallow: /categ
Disallow: /db
Disallow: /desig
Disallow: /direct
Disallow: /edit
Disallow: /email
Disallow: /file
Disallow: /forgot
Disallow: /groups
Disallow: /keywor
Disallow: /log
Disallow: /lost
Disallow: /mail
Disallow: /master
Disallow: /navaid
Disallow: /new
Disallow: /poll
Disallow: /price
Disallow: /prog
Disallow: /reply
Disallow: /roam
Disallow: /schema
Disallow: /search
Disallow: /shop
Disallow: /showap
Disallow: /showjo
Disallow: /showpu
Disallow: /stat
Disallow: /sub
Disallow: /themes
Disallow: /tofrie
Disallow: /cgi
Disallow: /blacklist
Disallow: /*rss=1
Disallow: /*?PDA=1
Disallow: /*.php

# Bots that claim to respect robots.txt, but haven't in the past
User-agent: twiceler
Disallow: /
User-agent: convera
Disallow: /