User-agent: Slurp
Disallow:
Crawl-delay: 5

User-agent: Yahoo! Slurp
Disallow:
Crawl-delay: 5

User-agent: IRLbot 
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: TeragramWebcrawler
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: nogoop-HttpClient
Disallow: /


User-agent: Scirus
Disallow: /

User-agent: Speedy Spider
Disallow: /

User-agent: PicoSearch
Disallow: /

User-agent: Plucker
Disallow: /

User-agent: DISCo Pump
Disallow: /

User-agent: Gulliver
Disallow: /

User-agent: vspider
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: Teleport Pro
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: pamuk
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebCapture
Disallow: /

User-agent: Mass Downloader
Disallow: /

User-agent: WebCopy
Disallow: /

User-agent: AWV0.8d
Disallow: /

User-agent: Crescent Internet ToolPak
Disallow: /

User-agent: JOC Web Spider
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: SiteSucker
Disallow: /

User-agent: Webdup
Disallow: /

User-agent: Scooter
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Python
Disallow: /

User-agent: Franklin Locator
Disallow: /

User-agent: CK-SillyDog
Disallow: /

User-agent: PocketHTTP
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: Xaldon WebSpider
Disallow: /

User-agent: WebCapture
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: sun4u
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: Sisi
Disallow: /
Disallow: /

User-agent: lmspider
Disallow: /

User-agent: NP
Disallow: /

User-agent: Robi
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: Webster
Disallow: /

User-agent: Zeus
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: MJ12bot
Disallow: /

User-Agent: Charlotte
Disallow: /

User-agent: dotbot
Disallow: / 

user-agent: AhrefsBot
disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: msnbot
Disallow: /admin
Crawl-delay: 2

User-agent: *
Disallow: /admin

Sitemap: https://ctext.org/siteindex.txt


