Sitemap: https://app.shufti.jp/static/sitemap.xml

User-Agent: *
Disallow: /test.php
# Exclude all robots access to ajax actions #13742
Disallow: /*/ajax_*

# http://www.baidu.jp/spider/
User-Agent: Baiduspider
Disallow: /
User-agent: BaiduImagespider
Disallow: /
User-Agent: BaiduMobaider
Disallow: /
# http://help.soso.com/webspider.htm
User-Agent: Sosospider
Disallow: /
# http://www.entireweb.com/about/search_tech/speedy_spider/
User-Agent: Speedy
Disallow: /
# http://help.naver.com/robots/
User-Agent: Yeti
Disallow: /
# http://www.dotnetdotcom.org/
User-Agent: DotBot
Disallow: /
# http://www.metadatalabs.com/mlbot
User-agent: MLBot
Disallow: /
# http://www.alexa.com/site/help/webmasters
User-Agent: ia_archiver
Disallow: /
# http://www.sitebot.org/robot/
User-Agent: SiteBot/0.1
Disallow: /
# http://www.majestic12.co.uk/bot.php
User-Agent: MJ12bot
Disallow: /
# http://www.youdao.com/help/webmaster/spider/
User-Agent: YoudaoBot
Disallow: /
# http://morfeo.centrum.cz/bot
User-Agent: holmes
Disallow: /
# http://help.soso.com/webspider.htm
User-Agent: Sosospider
Disallow: /
# http://www.yodao.com/help/webmaster/spider/
User-Agent: YodaoBot
Disallow: /
# http://yandex.com/bots
User-Agent: YandexBot
Disallow: /
User-Agent: YandexMedia
Disallow: /
#
User-Agent: LexxeBot
Disallow: /
#
User-Agent: Ezooms
Disallow: /
# http://www.nerdbynature.net/bot
User-Agent: NerdByNature.Bot
Disallow: /
# http://www.turnitin.com/robot/crawlerinfo.html
User-Agent: TurnitinBot
Disallow: /
# http://sindice.com/developers/bot
User-agent: SindiceBot
Disallow: /
# http://www.warebay.com/bot.html
User-agent: WBSearchBot
Disallow: /