# $Id: robots.txt,v 1.10 2009/09/16 11:05:53 eoin Exp $ # Disallow everyone from certain areas. User-agent: * Disallow: /admin Disallow: /accounts Disallow: /calendar Disallow: /includes Disallow: /intranet Disallow: /lists Disallow: /mantis Disallow: /oldsite Disallow: /rss Disallow: /search Disallow: /Templates Disallow: /cases Disallow: /casenum Disallow: /lists/2002/new Disallow: /lists/2002/decided Disallow: /lists/2003/new Disallow: /lists/2003/decided Disallow: /lists/2004/new Disallow: /lists/2004/decided Disallow: /lists/2005/new Disallow: /lists/2005/decided Disallow: /lists/2006/new Disallow: /lists/2006/decided Disallow: /lists/2007/new Disallow: /lists/2007/decided Disallow: /lists/2007/sid Disallow: /lists/2008/new Disallow: /lists/2008/decided Disallow: /lists/2008/sid Disallow: /lists/2009/new Disallow: /lists/2009/decided Disallow: /lists/2009/sid Disallow: /lists/2010/new Disallow: /lists/2010/decided Disallow: /lists/2010/sid Disallow: /lists/2011/new Disallow: /lists/2011/decided Disallow: /lists/2011/sid Disallow: /lists/2012/new Disallow: /lists/2012/decided Disallow: /lists/2012/sid Disallow: /lists/2013/new Disallow: /lists/2013/decided Disallow: /lists/2013/sid Disallow: /lists/2014/new Disallow: /lists/2014/decided Disallow: /lists/2014/sid Disallow: /lists/2015/new Disallow: /lists/2015/decided Disallow: /lists/2015/sid Disallow: /lists/2016/new Disallow: /lists/2016/decided Disallow: /lists/2016/sid Disallow: /lists/2017/new Disallow: /lists/2017/decided Disallow: /lists/2017/sid Disallow: /lists/2018/new Disallow: /lists/2018/decided Disallow: /lists/2018/sid Disallow: /DCT Disallow: /ORD Disallow: /REP Disallow: /dct Disallow: /ord Disallow: /rep Disallow: /documents Disallow: /worddocuments Disallow: /wordpress User-agent: gsa-crawler Allow: /lists/2010/eis/ Allow: /lists/2009/eis/ Allow: /lists/2008/eis/ Allow: /lists/2007/eis/ Allow: /lists/2006/eis/ Allow: /lists/referrals.htm Allow: /lists/oralhearings.htm # The following allows /home/pleanala/bin/mirror_entire_site.sh to get /documents User-agent: "Microsoft Internet Explorer" Allow: /documents User-agent: wget Allow: /documents Disallow: /admin Disallow: /accounts Disallow: /calendar Disallow: /includes Disallow: /intranet Disallow: /lists Disallow: /mantis Disallow: /oldsite Disallow: /rss Disallow: /search Disallow: /Templates Disallow: /cases Disallow: /lists/2002/new Disallow: /lists/2002/decided Disallow: /lists/2003/new Disallow: /lists/2003/decided Disallow: /lists/2004/new Disallow: /lists/2004/decided Disallow: /lists/2005/new Disallow: /lists/2005/decided Disallow: /lists/2006/new Disallow: /lists/2006/decided Disallow: /lists/2007/new Disallow: /lists/2007/decided Disallow: /lists/2007/sid Disallow: /lists/2008/new Disallow: /lists/2008/decided Disallow: /lists/2008/sid Disallow: /lists/2009/new Disallow: /lists/2009/decided Disallow: /lists/2009/sid Disallow: /lists/2010/new Disallow: /lists/2010/decided Disallow: /lists/2010/sid Disallow: /lists/2011/new Disallow: /lists/2011/decided Disallow: /lists/2011/sid Disallow: /lists/2012/new Disallow: /lists/2012/decided Disallow: /lists/2012/sid Disallow: /lists/2013/new Disallow: /lists/2013/decided Disallow: /lists/2013/sid Disallow: /lists/2014/new Disallow: /lists/2014/decided Disallow: /lists/2014/sid Disallow: /lists/2015/new Disallow: /lists/2015/decided Disallow: /lists/2015/sid Disallow: /lists/2016/new Disallow: /lists/2016/decided Disallow: /lists/2016/sid Disallow: /lists/2017/new Disallow: /lists/2017/decided Disallow: /lists/2017/sid Disallow: /lists/2018/new Disallow: /lists/2018/decided Disallow: /lists/2018/sid Disallow: /DCT Disallow: /ORD Disallow: /REP Disallow: /dct Disallow: /ord Disallow: /rep Disallow: /documents Disallow: /worddocuments Disallow: /wordpress