User-Agent: *
Disallow: /xmlrpc.php
Disallow: /cgi-bin
Disallow: /wp-admin
Disallow: /wp-includes
Disallow: /wp-content/plugins
Disallow: /wp-content/cache
Disallow: /tag/*
Disallow: /author/*
Disallow: *?*
Allow: /news/*.css?*
Host: https://novostivl.ru

Crawl-delay: 5
Sitemap: https://novostivl.ru/export/rss_atom_feed.xml

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /