# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/core.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/companies_details.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/companies_browse.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/jobs_details.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/jobs_browse.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/salaries_details.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/salaries_browse.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/skills_details.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/skills_browse.xml.gz
SITEMAP: https://hired-user-uploads.s3.amazonaws.com/sitemaps/uncrawled_botify_urls.xml.gz

User-Agent: *
Disallow: /company/invite/*
Disallow: *set_locale=*
Disallow: /m/*
Disallow: /x/*
Disallow: /employers/welcome-center
Disallow: /m/gracehopper2017?utf8=
Disallow: /m/dc2?utf8=
Disallow: /m/teleport?utf8=
Disallow: /m/devjobs?utf8=
Disallow: /m/pandora?utf8=
Disallow: /m/generalassembly?utf8=
Disallow: /m/wework?utf8=
Disallow: /m/scotchio?utf8=
Disallow: /m/sitepoint?utf8=
Disallow: /m/codepen1?utf8=
Disallow: /m/codepen2?utf8=
Disallow: /m/codepen5?utf8=
Disallow: /m/codepen?utf8=
Disallow: /m/pastebin?utf8=
Disallow: /m/slugbay?utf8=
Disallow: /m/ziprecruiter?utf8=
Disallow: /m/codepen3?utf8=
Disallow: /m/codepen6?utf8=
Disallow: /jobs?utf8=
Disallow: /startup-jobs?utf8=
Disallow: /tech-jobs?utf8=
Disallow: /m/codepen4?utf8=
Disallow: /m/codepen7?utf8=
Disallow: /m/ruby-jobs?utf8=
Disallow: /m/javascriptdc?utf8=
Disallow: /m/quora?utf8=
Disallow: /m/zoe?utf8=
Disallow: /m/tradecraftsales?utf8=
Disallow: /m/growthxsf?utf8=
Disallow: /m/tradecraftgrowth?utf8=
Disallow: /m/tradecraftdesign?utf8=
Disallow: /m/nodedc?utf8=
Disallow: /m/insidesalesbootcamp?utf8=
Disallow: /m/abejqzkxmd?utf8=
Disallow: /abejqzkxmd