#
User-agent: *
Crawl-delay: 240
Request-rate: 1/5
Visit-time: 0200-0645
# list folders robots are not allowed to index
Disallow: http://www.veterans.gc.ca/eng/department/policy
Disallow: http://www.veterans.gc.ca/fra/ministere/politique
Disallow: http://veterans.gc.ca/eng/department/policy
Disallow: http://veterans.gc.ca/fra/ministere/politique
Disallow: http://www.veterans.gc.ca/eng/benefits
Disallow: http://www.veterans.gc.ca/fra/benefits
Disallow: http://veterans.gc.ca/eng/benefits
Disallow: http://veterans.gc.ca/fra/benefits
Disallow: http://www.veterans.gc.ca/fra/avantages
Disallow: http://veterans.gc.ca/fra/avantages
Disallow: http://198.103.134.2
Disallow: http://vac.gc.ca
Disallow: https://vac.gc.ca
Disallow: https://acc.gc.ca
Disallow: http://www.acc-vac.gc.ca
Disallow: http://acc.gc.ca
Disallow: http://veterans.scott-stewart.net
Disallow: http://veterans.scott-stewart.net/crawler.general.asp
Disallow: http://www.baidu.com/search/spider.html
Disallow: http://video.vac-acc.gc.ca
Disallow: /components/
Disallow: /config/
Disallow: /controllers/
Disallow: /xml/
Disallow: /admin/
Disallow: /CFIDE/
Disallow: /files/
Disallow: /js/
Disallow: /javascripts/
Disallow: /css/
Disallow: /video/
Disallow: /content/collections/VC/
Disallow: *.css
Disallow: *.jpg
Disallow: *.gif
Disallow: *.jpeg
Disallow: *.png
Disallow: *.zip
Disallow: *.log
Disallow: *.js
Disallow: Application.cfm
Disallow: Application.cfc
Disallow: *eventid=1716
Disallow: http://www.veterans.gc.ca/fra/collections/livres/livre-sgm/ww2/ww2223')
User-agent: Baiduspider/2.0
Disallow: /