User-agent: *
Crawl-delay: 1

# Directories
Disallow: /test
Disallow: /test/
Disallow: /tmp
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /_*
Disallow: /_*/
Disallow: /adv/
Disallow: /captcha/
Disallow: /css/
Disallow: /download/
Disallow: /epoll/
Disallow: /gallery/
Disallow: /i/
Disallow: /inc/
Disallow: /inquirer/
Disallow: /js/
Disallow: /promo/
Disallow: /QR_cod/
Disallow: /registration/
Disallow: /response/
Disallow: /Scripts/
Disallow: /stat/
Disallow: /terms-of-use/
Disallow: /upload/
Disallow: /newsletter/

# Files
Disallow: /_*.*
Disallow: *.txt
Disallow: *.db
Disallow: *.doc
Disallow: *.docx
Disallow: *.xls
Disallow: *.xlsx
Disallow: *.htaccess
Disallow: *.htpasswd
Disallow: *.ppk
Disallow: /encode.pl
Disallow: /search.pl
Disallow: /chmod.php
Disallow: /info.php

User-agent: Yandex
Allow: /
sitemap: https://oncology.ru/sitemap.xml

# ЛУЧШЕ ЗАКРЫТЬ САЙТ ОТ ИНДЕКСАЦИИ ОТ БОТОВ. ОНИ ПОТРЕБЛЯЮТ МНОГО РЕСУРСОВ СЕРВЕРА, РАБОТАЯ ИСКЛЮЧИТЕЛЬНО НА СЕБЯ И СВОИХ КЛИЕНТОВ.

  User-agent: OLEcrawler
  Disallow: /

  User-agent: MJ12bot
  Disallow: /

  User-agent: wotbox
  Disallow: /

  User-agent: AhrefsBot
  Disallow: /

  User-agent: BLEXBot
  Disallow: /

  User-agent: BUbiNG
  Disallow: /

  User-agent: Vedma
  Disallow: /

  User-agent: SemrushBot
  Disallow: /

  User-agent: SemrushBot-SA
  Disallow: /

  User-agent: Riddler
  Disallow: /


Host: http://oncology.ru/
Sitemap: http://oncology.ru/sitemap.xml