User-agent: *
Disallow: /cgi-bin/
Disallow: *.pdf

User-agent: dotbot
crawl-delay:30

user-agent: Baiduspider
crawl-delay:30