User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /phpBB2/
Disallow: /spanischlernen/tutor.htm
Disallow: /links/
Disallow: /wp-content/
Noindex: /wp-content/