# robots for https://www.rdforum.org
#
# 1. Catch-all / Everything else (*)
# 2.  Explicitly Allowed Spiders, Named to be Apparent
# 3. Sitemaps
# 4. Directives not globally supported
#
#####################################
# 1. Prohibited Crawlers Catch All;
#     Grey bots that will obey
#     robots.txt
#####################################

User-agent: *
User-agent: Adsbot
User-agent: Alexa
User-agent: alexa site audit
User-agent: Alexabot
User-agent: AspiegelBot
User-agent: CCBot
User-agent: DataForSeoBot
User-agent: DotBot
User-agent: gsa-crawler
User-agent: heritrix
User-agent: ia_archiver
User-agent: InfoTigerBot
User-agent: magpie-crawler
User-agent: Majestic
User-agent: Majestic12
User-agent: MauiBot
User-agent: MJ12bot
User-agent: MojeekBot
User-agent: nutch
User-agent: PetalBot
User-agent: Seekport
User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: XoviBot

Disallow: /$
Disallow: /
Disallow: /*

Allow: /robots.txt
Allow: /robots.txt$

################################
# 2. Explicitly Allowed Crawlers
################################

User-agent: 360Spider
User-agent: 360Spider-Image
User-agent: 360Spider-Video
User-agent: AdIdxBot
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
User-agent: AhrefsBot
User-Agent: AhrefsSiteAudit
User-agent: APIs-Google
User-agent: AppEngine-Google
User-agent: Apple-PubSub
User-agent: Applebot
User-agent: AppleNewsBot
User-agent: aSpiegelBot
User-agent: BaiduSpider
User-agent: BaiduSpider-Ads
User-agent: BaiduSpider-CPro
User-agent: BaiduSpider-Honeycomb
User-agent: Baiduspider-Image
User-agent: BaiduSpider-Mobile
User-agent: Baiduspider-News
User-agent: BaiduSpider-Render
User-agent: BaiduSpider-Video
User-agent: Bingbot
User-agent: BingPreview
User-agent: BublupBot
User-agent: CCBot
User-agent: Cliqzbot
User-agent: coccoc
User-agent: coccocbot-image
User-agent: coccocbot-web
User-agent: Daumoa
User-agent: Dazoobot
User-agent: DeuSu
User-agent: Discordbot
User-agent: DuckDuckBot
User-agent: DuckDuckGo-Favicons-Bot
User-agent: DuplexWeb-Google
User-agent: EuripBot
User-agent: Exabot
User-agent: Exploratodo
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Feedfetcher-Google
User-agent: Feedly
User-agent: Findxbot
User-agent: Google Favicon
User-agent: Google-Adwords-Instant
User-agent: Google-Read-Aloud
User-agent: google-speakr
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: HaoSouSpider
User-agent: ichiro
User-agent: istellabot
User-agent: JikeSpider
User-agent: librabot
User-agent: LinkedInBot
User-agent: loader.io
User-agent: Lycos
User-agent: Mail.RU`
User-agent: Mail.RU_Bot
User-agent: Mediapartners-Google
User-agent: MSNBot
User-agent: MSNBot-Media
User-agent: MSNBot-NewsBlogs
User-agent: MSNBot-UDiscovery
User-agent: Naver
User-agent: Naverbot
User-agent: Neevabot
User-agent: OnPageBot
User-agent: OrangeBot
User-agent: Pinterest
User-agent: Plukkie
User-agent: Qwantify
User-agent: Railgun
User-agent: Rambler
User-agent: redditbot
User-agent: RyteBot
User-agent: SeznamBot
User-agent: Slack-ImgProxy
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: Slurp
User-agent: Sogou
User-agent: Sogou blog
User-agent: Sogou head spider
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou spider2
User-agent: Sogou web spider
User-agent: Sogou-Test-Spider
User-agent: Sosospider
User-agent: SputnikBot
User-agent: Teoma
User-agent: Twitterbot
User-agent: WhatsApp
User-agent: wotbox
User-agent: YacyBot
User-agent: YaDirectFetcher
User-agent: Yahoo-Blogs
User-agent: Yahoo-MMCrawler
User-agent: Yandex
User-agent: YandexAccessibilityBot
User-agent: YandexBlogs
User-agent: YandexBot
User-agent: YandexCalendar
User-agent: YandexDirect
User-agent: YandexDirectDyn
User-agent: YandexFavicons
User-agent: YandexImageResizer
User-agent: YandexImages
User-agent: YandexMarket
User-agent: YandexMedia
User-agent: YandexMetrika
User-agent: YandexMobileBot
User-agent: YandexNews
User-agent: YandexPagechecker
User-agent: YandexScreenshotBot
User-agent: YandexSearchShop
User-agent: YandexSitelinks
User-agent: YandexVerticals
User-agent: YandexVertis
User-agent: YandexVideo
User-agent: YandexVideoParser
User-agent: YandexWebmaster
User-agent: Yeti
User-agent: YioopBot
User-agent: YisouSpider
User-agent: yoozBot
User-agent: YoudaoBot

Disallow:
Allow: /$
Allow: /
Allow: /*
Disallow: *.env/*
Disallow: *.env/$
Disallow: *.env$
Disallow: */.env/*
Disallow: */.env/$
Disallow: */.env$
Disallow: */.git/*
Disallow: */.git/$
Disallow: */.git$
Disallow: */.svn/*
Disallow: */.svn/$
Disallow: */.svn$
Disallow: */*.env/*
Disallow: */CVS/*
Disallow: */CVS/$
Disallow: */CVS$
Disallow: /?__proto__[98765]=testv&__proto__.98765=testv&constructor[prototype][98765]=testv
Disallow: /*__proto__*
Disallow: /*.cfg$
Disallow: /*.conf$
Disallow: /*.config$
Disallow: /*.csv$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.ppt$
Disallow: /*.pptx$
Disallow: /*.txt$
Disallow: /*.xls$
Disallow: /*.xlsx$
Disallow: /*/cdn.jsdelivr.net
Disallow: /*/http*
Disallow: /*/mc.yandex.ru
Disallow: /*/player.vimeo.com
Disallow: /*/secure.gravatar.com
Disallow: /*/www.*
Disallow: /*/www.youtube.com
Disallow: /account-confirmation/*
Disallow: /account/*
Disallow: /admin.php?*
Disallow: /admin.php$
Disallow: /admin/*
Disallow: /administrator/*
Disallow: /cdn.jsdelivr.net
Disallow: /conversations/
Disallow: /conversations/*
Disallow: /conversations$
Disallow: /data/avatars/*
Disallow: /forums/*/post-thread
Disallow: /javascript%3A*
Disallow: /login/*
Disallow: /lost-password/*
Disallow: /mc.yandex.ru
Disallow: /membernotes/edit
Disallow: /members/*/ban
Disallow: /members/*/multiple-account-list*
Disallow: /misc/*
Disallow: /online/*
Disallow: /online/$
Disallow: /out/*
Disallow: /posts/*/edit
Disallow: /posts/*/report
Disallow: /proxy.php$
Disallow: /robomail/*
Disallow: /robomail/$
Disallow: /robomail$
Disallow: /search/*
Disallow: /secure.gravatar.com
Disallow: /threads/*/poll/*
Disallow: /threads/*/poll/$
Disallow: /threads/*/poll/view
Disallow: /threads/*/poll/vote
Disallow: /threads/*/poll$
Disallow: /threads/*/unread
Disallow: /whats-new/posts/*/$
Disallow: /warnings/*
Disallow: /www.youtube.com
Disallow: /xmlrpc.php*

Allow: /ads.txt$
Allow: /google*.html
Allow: /naver*.html
Allow: /seznam-wmt-*.txt

Allow: /.well-known/*
Allow: /humans.txt$
Allow: /robots.txt$
Allow: /sitemap-*.xml$
Allow: /sitemap.xml$

###########################
# 3. Sitemaps
###########################

Sitemap: https://www.rdforum.org/sitemap.xml

###########################
# 4. Directives not globally supported
###########################

Crawl-delay: 1
Clean-param: -indexing /
Clean-param: _openstat /
Clean-param: assigned_others&assigned_self /
Clean-param: content_type&content_id&creator_id /
Clean-param: date&debug&_debug&direction /
Clean-param: f&fbclid&filter&from /
Clean-param: gclid /
Clean-param: hash&highlight /
Clean-param: jump_from_id /
Clean-param: key /
Clean-param: last_days&location /
Clean-param: no_date_limit /
Clean-param: order /
Clean-param: page&perpage&prefix_id /
Clean-param: report_data_id /members/
Clean-param: s&sa&sid&skip&sort /
Clean-param: status_not[0]&status_not[1]&status_not[2]&status_not[3]&status_not[4] /tickets/
Clean-param: t&tid&title&type /
Clean-param: unassigned /tickets/
Clean-param: utm_source&utm_medium&utm_term&utm_content&utm_campaign /
Clean-param: ved /
Clean-param: yclid /
Clean-param: __proto__[98765] /
Clean-param: constructor[prototype][98765] /
Host: https://www.rdforum.org
Request-rate: 1/1s