# $Id: robots.txt,v 1.2 1995/10/30 04:15:40 fielding Exp $ # robots.txt for http://www.c4systm.com/ # see for an explanation. Sitemap: http://www.knownews.net/sitemap.php?do=showsitemap&sm=sitemap.xml.gz User-agent: MOMspider # The Multi-Owner Maintenance Spider Disallow: /Admin # Administration Disallow: /AIcons # AIcons directory Disallow: /bin # Script files Disallow: /cgi # Script files Disallow: /logs # Log files User-agent: * # All other spiders should avoid Disallow: /Admin # Administration Disallow: /AIcons # AIcons directory Disallow: /NEWS # Usenet test Disallow: /Test # Testing Disallow: /bin # Script files Disallow: /cgi # Script files Disallow: /about # Obsolete Disallow: /logs # Log files Disallow: /maintain # Maintenance Disallow: /old # Old pages Disallow: /samples # Sample pages Disallow: /schemes # Schemes files Disallow: /tmp # Temporary files Disallow: /Clari Disallow: /News Disallow: /News.old Disallow: /claripic Disallow: /include Disallow: /old