User-Agent: Googlebot
Disallow: /tmp/
Disallow: /test/

User-Agent: Googlebot-News
Disallow: /tmp/
Disallow: /test/

user-agent: AppleNewsBot
Disallow: /tmp/
Disallow: /test/

user-agent: bingbot
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 20

user-agent: msnbot
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 20

User-agent: Mediapartners-Google*
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 20

User-agent: Slurp
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 20

User-agent: Teoma
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 60

User-agent: Yandex
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 60

user-agent: Urbanspoonbot
Disallow: /tmp/
Disallow: /test/

User-agent: Twitterbot
Disallow: /tmp/
Disallow: /test/

User-agent: Crowsnest
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 20

User-agent: 008
Disallow: /tmp/
Disallow: /test/
Crawl-delay: 120

User-agent: Googlebot-Image
Disallow: /

User-agent: TinEye
Disallow: /

User-agent: *
Disallow: /

Sitemap: http://ny.curbed.com/sitemap.xml
Sitemap: http://ny.curbed.com/news-sitemap.xml