# robots.txt for all robots
# AJK 2010-04-15 // 2011-03-28
# Generated file. do not edit.

Sitemap: https://library.wur.nl/sitemap.xml

# 110130 BJH: allow local linkchecker (case 4441)
User-agent: LinkChecker
Disallow:

User-agent: Googlebot
Disallow: /test		# please let us test them ourselves...
Disallow: /cgi-bin	# we do database access there...
Disallow: /wiki
Disallow: /metiswiki
Disallow: /local	# that means: local to our site!
Disallow: /usage	# our usage statistics
Disallow: /atc          # agro telematica centrum test
Disallow: /nlinet       # internal information
Disallow: /ctb/files    # ctb decisions on the admission of pesticides
Disallow: /vogin	# temporary
Disallow: /foodbib	# temporary
Disallow: /OpenURL	# Please stay away from our Open URL resolver
Disallow: /sfxmenu	# Please stay away from our Open URL resolver
Disallow: /sfx		# Please stay away from our Open URL resolver
Disallow: /sfx_local	# Please stay away from our Open URL resolver
Disallow: /link         # disallow link resolver
Disallow: /WebQuery/cab #Don't crawl the thesaurus
Disallow: /xsl
Disallow: /ns
Disallow: /java/
#Disallow: /WebQuery*&	# no searches

User-agent: gsa-crawler
Disallow: /test		# please let us test them ourselves...
Disallow: /cgi-bin	# we do database access there...
Disallow: /wiki
Disallow: /metiswiki
Disallow: /local	# that means: local to our site!
Disallow: /usage	# our usage statistics
Disallow: /atc          # agro telematica centrum test
Disallow: /nlinet       # internal information
Disallow: /ctb/files    # ctb decisions on the admission of pesticides
Disallow: /vogin	# temporary
Disallow: /foodbib	# temporary
Disallow: /OpenURL	# Please stay away from our Open URL resolver
Disallow: /sfxmenu	# Please stay away from our Open URL resolver
Disallow: /sfx		# Please stay away from our Open URL resolver
Disallow: /sfx_local	# Please stay away from our Open URL resolver
Disallow: /link         # disallow link resolver
Disallow: /WebQuery/cab #Don't crawl the thesaurus
Disallow: /xsl
Disallow: /ns
Disallow: /java

User-agent: *
Disallow: /test		# please let us test them ourselves...
Disallow: /cgi-bin	# we do database access there...
Disallow: /wiki
Disallow: /metiswiki
Disallow: /local	# that means: local to our site!
Disallow: /usage	# our usage statistics
Disallow: /atc          # agro telematica centrum test
Disallow: /nlinet       # internal information
Disallow: /ctb/files    # ctb decisions on the admission of pesticides
Disallow: /vogin	# temporary
Disallow: /foodbib	# temporary
Disallow: /OpenURL	# Please stay away from our Open URL resolver
Disallow: /sfxmenu	# Please stay away from our Open URL resolver
Disallow: /sfx		# Please stay away from our Open URL resolver
Disallow: /sfx_local	# Please stay away from our Open URL resolver
Disallow: /link         # disallow link resolver
Disallow: /WebQuery     # 210223 BJH: weer enabled, zie nog steeds crawlers die /WebQuery crawlen
Disallow: /xsl
Disallow: /ns
Disallow: /java
# 181129 BJH: add crawl delay (for Yandex crawlers)
Crawl-delay: 5