#####################################
# List of WHITELIST ALLOW user-agents

# CPM: Adsense banner and contexual bots
User-agent: Mediapartners-Google* # support.google.com/adsense/bin/answer.py?hl=en&answer=99376
User-agent: Mediapartners-Google # Adsense contextual targeting bot
User-agent: msnbot-media/1.0 # MSN contextual targeting bot
Disallow: /PDF/
Disallow: /pdftemp/
Disallow: /admin/
Disallow: /newsletter/
Disallow: /htmlheader.asp
Allow: /

# PPC: Adwords & AdCenter landing page bots
User-agent: Adsbot-Google # AdwordsPPC tinyurl.com/list-of-Google-Crawlers
User-agent: AdsBot-Google-Mobile # AdwordsPPC for mobile campaigns
User-agent: Adidxbot # BingPPC (aka MSNPTC) www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0
User-agent: Yahoo!-AdCrawler # YahooPPC tinyurl.com/About-Yahoo-Adcrawler
Disallow: /PDF/
Disallow: /pdftemp/
Disallow: /admin/
Disallow: /newsletter/
Disallow: /htmlheader.asp
Allow: /

# SEO: Organic Crawlers
User-agent: Googlebot # support.google.com/webmasters/bin/answer.py?hl=en&answer=1061943
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Googlebot-Image
User-agent: gsa-crawler # Google-Appliance tinyurl.com/google-sitesearch-appliance
User-agent: Bingbot # aka MSNBOT www.bing.com/bingbot.htm
User-agent: MSNBot # old version of bingbot still used for multimedia and feeds crawls
User-agent: BingPreview # bing.com/community/site_blogs/b/webmaster/archive/2012/10/26/page-snapshots-in-bing-windows-8-app-to-bring-new-crawl-traffic-to-sites.aspx
User-agent: Slurp # aka Yahoo! Slurp - now part of Bingbot
Crawl-delay: 20 # Reduce Server-Load using 20sec crawl delay 
Disallow: /PDF/
Disallow: /pdftemp/
Disallow: /admin/
Disallow: /newsletter/
Disallow: /htmlheader.asp
Allow: /

#####################################
# Other user agents you might want to include:
# ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider
# gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot
#####################################

# Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: *
User-agent: SiteCon # www.nccgroup.com/en/our-services/website-performance/website-performance-monitoring/
User-agent: GomezAgent # gomeznetworks.com/help/gpn/MySettings/Last_Mile_PP_Test_Settings.htm
User-agent: GomezAgent 1.0
User-agent: GomezAgent 2.0
User-agent: GomezAgent 3.0
User-agent: YottaaMonitor
User-agent: FunWebProducts
Disallow: /


Sitemap: http://www.boma.be/sitemap.xml