#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css
Allow: /core/*.js
Allow: /core/*.gif
Allow: /core/*.GIF
Allow: /core/*.jpg
Allow: /core/*.JPG
Allow: /core/*.jpeg
Allow: /core/*.JPEG
Allow: /core/*.png
Allow: /core/*.PNG
Allow: /core/*.svg
Allow: /core/*.SVG
Allow: /profiles/*.css
Allow: /profiles/*.js
Allow: /profiles/*.gif
Allow: /profiles/*.GIF
Allow: /profiles/*.jpg
Allow: /profiles/*.JPG
Allow: /profiles/*.jpeg
Allow: /profiles/*.JPEG
Allow: /profiles/*.png
Allow: /profiles/*.PNG
Allow: /profiles/*.svg
Allow: /profiles/*.SVG
Allow: /sites/*.gif
Allow: /sites/*.GIF
Allow: /sites/*.jpg
Allow: /sites/*.JPG
Allow: /sites/*.jpeg
Allow: /sites/*.JPEG
Allow: /sites/*.png
Allow: /sites/*.svg
Allow: /sites/*.SVG
Allow: /sites/*.css
Allow: /sites/*.js
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /search
Disallow: /user/*
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips/
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/
Disallow: /taxonomy/
Disallow: /dashboard/
Disallow: /desktop/
Disallow: /mobile/
Disallow: /billing/
Disallow: /member/
Disallow: /members
Disallow: /event/*
Disallow: /modal_forms/
Disallow: /company/built$
Disallow: /*search_api_fulltext*
Disallow: /salaries/entry-step1*
Disallow: /form/claim-company?*
Disallow: /blog/timeline_event.php*
Disallow: /views/ajax?*
Disallow: /jobs*?search=
Disallow: /jobs*%
Disallow: /login*
Disallow: /onboarding*
Disallow: /jobs/office-remote*
Disallow: /jobs*?trending*
Disallow: /company*?page=
Disallow: /company/*api.builtin.com
Disallow: /blogs*?page=
Disallow: /jobs*?f*
Disallow: /jobs/workspace*
Disallow: /jobs/vendor*
Disallow: /jobs/office*
Disallow: /companies/location/
Disallow: /jobs/*size
Disallow: *?page=
Disallow: *?region_id=
Allow: *sitemap.xml?page=
Disallow: /apply/
Disallow: /featureflag
Disallow: /form/
Disallow: /External/Challenge*
Disallow: /Signup
Disallow: /connect/authorize*
Disallow: *?event_src=
Disallow: *?elementor_library
Disallow: *?state=
Disallow: /salaries/get/
Disallow: *challenge-platform
Disallow: /signup
Disallow: /onboarding
Allow: /jobs*?page=1$
Allow: /jobs*?page=2$
Allow: /jobs*?page=3$
Disallow: /jobs*?page=
Disallow: /jobs*?companyId=
Disallow: /jobs*chicago
Disallow: /auth/
Disallow: /cdn-cgi/rum
Allow: /articles*?page=

# ----------------------------
# LLM Bots (OpenAI + Others)
# ----------------------------

User-agent: OAI-SearchBot
Disallow: / 
Allow: /$
Allow: /companies*
Allow: /company/*
Allow: /jobs*
Allow: /job/*
Allow: /salaries*
Allow: /awards*
Allow: /articles*
Allow: /artificial-intelligence*
Allow: /blockchain*
Allow: /company-culture*
Allow: /data-science*
Allow: /diversity-inclusion*
Allow: /hardware*
Allow: /learn*
Allow: /machine-learning*
Allow: /recruiting*
Allow: /robotics*
Allow: /software-engineering-perspectives*
Allow: /tech-topics*
Allow: /tags*

User-agent: GPTBot
Disallow: / 
Allow: /$
Allow: /companies*
Allow: /company/*
Allow: /jobs*
Allow: /job/*
Allow: /salaries*
Allow: /awards*
Allow: /articles*
Allow: /artificial-intelligence*
Allow: /blockchain*
Allow: /company-culture*
Allow: /data-science*
Allow: /diversity-inclusion*
Allow: /hardware*
Allow: /learn*
Allow: /machine-learning*
Allow: /recruiting*
Allow: /robotics*
Allow: /software-engineering-perspectives*
Allow: /tech-topics*
Allow: /tags*

User-agent: ChatGPT-User
Disallow: / 
Allow: /$
Allow: /companies*
Allow: /company/*
Allow: /jobs*
Allow: /job/*
Allow: /salaries*
Allow: /awards*
Allow: /articles*
Allow: /artificial-intelligence*
Allow: /blockchain*
Allow: /company-culture*
Allow: /data-science*
Allow: /diversity-inclusion*
Allow: /hardware*
Allow: /learn*
Allow: /machine-learning*
Allow: /recruiting*
Allow: /robotics*
Allow: /software-engineering-perspectives*
Allow: /tech-topics*
Allow: /tags*

User-agent: PerplexityBot
Disallow: / 
Allow: /$
Allow: /companies*
Allow: /company/*
Allow: /jobs*
Allow: /job/*
Allow: /salaries*
Allow: /awards*
Allow: /articles*
Allow: /artificial-intelligence*
Allow: /blockchain*
Allow: /company-culture*
Allow: /data-science*
Allow: /diversity-inclusion*
Allow: /hardware*
Allow: /learn*
Allow: /machine-learning*
Allow: /recruiting*
Allow: /robotics*
Allow: /software-engineering-perspectives*
Allow: /tech-topics*
Allow: /tags*