user-agent: *
Sitemap: https://www.dotcms.com/api/vtl/sitemap
Allow: /api/*
Disallow: /docs/webdav/autopub/*
Disallow: /webdav/*
Disallow: /html/*
Disallow: /landing-pages/*
Disallow: /*/javadocs/*
Disallow: /javadocs/*
Disallow: /rensai/*
Disallow: /docs/1.*
Disallow: /docs/2.*
Disallow: /docs/3.*
Disallow: /docs/4.*
Disallow: /docs/5.*
Disallow: /servlets/*
Disallow: /download/nightly-builds*
Disallow: /accelerators/*
Disallow: /shop/*
Disallow: /doc/dotmarketing/*
Disallow: /docs/com/dotmarketing/*
Disallow: /browse/*
Disallow: /devwiki/*

User-agent: msnbot
Crawl-delay: 120

User-agent: Slurp
Crawl-delay: 4