# /robots.txt file for http://www.micronix.com.au/ # To allow webcrawler everywhere #User-agent: webcrawler #Disallow: # to prevent lycra crawling anywhere #User-agent: lycra #Disallow: / # allow all agents everywhere except any tmp or logs directory User-agent: * Disallow: /tmp Disallow: /logs