# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html # --- Non-standard robots --- User-agent: * Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /profiles/*.svg Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /modules/*.svg Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: /themes/*.svg # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/add/ Disallow: /search/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/reply/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/add/ Disallow: /?q=search/ Disallow: /?q=user/password/ Disallow: /?q=user/register/ Disallow: /?q=user/login/ # Paths (clean URLs no trailing slash) Disallow: /admin Disallow: /comment/reply Disallow: /contact Disallow: /logout Disallow: /node Disallow: /search Disallow: /user/register Disallow: /user/password Disallow: /user/login # Paths (no clean URLs no trailing slash) Disallow: /?q=admin Disallow: /?q=comment/reply Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node Disallow: /?q=search Disallow: /?q=user/password Disallow: /?q=user/register Disallow: /?q=user/login # Block from R&D site Disallow: /research # --- Google --- User-agent: Googlebot sitemap: https://www.fs.usda.gov/sitemap.xml sitemap: https://www.fs.usda.gov/research/sitemap.xml # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /modules/*.svg Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: /themes/*.svg Allow: /research/core/*.css$ Allow: /research/core/*.css? Allow: /research/core/*.js$ Allow: /research/core/*.js? Allow: /research/core/*.gif Allow: /research/core/*.jpg Allow: /research/core/*.jpeg Allow: /research/core/*.png Allow: /research/core/*.svg Allow: /research/modules/*.css$ Allow: /research/modules/*.css? Allow: /research/modules/*.js$ Allow: /research/modules/*.js? Allow: /research/modules/*.gif Allow: /research/modules/*.jpg Allow: /research/modules/*.jpeg Allow: /research/modules/*.png Allow: /research/modules/*.svg Allow: /research/themes/*.css$ Allow: /research/themes/*.css? Allow: /research/themes/*.js$ Allow: /research/themes/*.js? Allow: /research/themes/*.gif Allow: /research/themes/*.jpg Allow: /research/themes/*.jpeg Allow: /research/themes/*.png Allow: /research/themes/*.svg # Block Queries (Facets, Pagination, etc) Disallow: /research/*?* # Directories Disallow: /core/ Disallow: /modules/ Disallow: /themes/ Disallow: /profiles/ Disallow: /research/core/ Disallow: /research/modules/ Disallow: /research/themes/ Disallow: /research/profiles/ # Files Disallow: /README.txt Disallow: /web.config Disallow: /research/README.txt Disallow: /research/web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips Disallow: /contact/ Disallow: /logout/ Disallow: /node/ Disallow: /search/ Disallow: /user/ Disallow: /media/oembed Disallow: /*/media/oembed Disallow: /research/admin/ Disallow: /research/comment/ Disallow: /research/contact/ Disallow: /research/logout/ Disallow: /research/node/ Disallow: /research/search/ Disallow: /research/*/search/ Disallow: /research/treesearch/$ Disallow: /research/*/treesearch/$ Disallow: /research/user/ Disallow: /research/*/login/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/ Disallow: /?q=search/ Disallow: /?q=user/ Disallow: /index.php/media/oembed Disallow: /index.php/*/media/oembed Disallow: /research/?q=admin/ Disallow: /research/?q=comment/ Disallow: /research/?q=contact/ Disallow: /research/?q=logout/ Disallow: /research/?q=node/ Disallow: /research/?q=search/ Disallow: /research/?q=user/ # Paths (clean URLs) – No trailing slash Disallow: /admin Disallow: /comment/reply Disallow: /contact Disallow: /logout Disallow: /node Disallow: /search Disallow: /user Disallow: /research/admin Disallow: /research/comment Disallow: /research/contact Disallow: /research/logout Disallow: /research/node Disallow: /research/search Disallow: /research/*/search Disallow: /research/treesearch$ Disallow: /research/*/treesearch$ Disallow: /research/user Disallow: /research/*/login # Paths (no clean URLs) – No trailing slash Disallow: /?q=admin Disallow: /?q=comment Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node Disallow: /?q=search Disallow: /?q=user Disallow: /research/?q=admin Disallow: /research/?q=comment Disallow: /research/?q=contact Disallow: /research/?q=logout Disallow: /research/?q=node Disallow: /research/?q=search Disallow: /research/?q=user # --- Bing --- User-agent: Bingbot sitemap: https://www.fs.usda.gov/sitemap.xml sitemap: https://www.fs.usda.gov/research/sitemap.xml # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /modules/*.svg Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: /themes/*.svg Allow: /research/core/*.css$ Allow: /research/core/*.css? Allow: /research/core/*.js$ Allow: /research/core/*.js? Allow: /research/core/*.gif Allow: /research/core/*.jpg Allow: /research/core/*.jpeg Allow: /research/core/*.png Allow: /research/core/*.svg Allow: /research/modules/*.css$ Allow: /research/modules/*.css? Allow: /research/modules/*.js$ Allow: /research/modules/*.js? Allow: /research/modules/*.gif Allow: /research/modules/*.jpg Allow: /research/modules/*.jpeg Allow: /research/modules/*.png Allow: /research/modules/*.svg Allow: /research/themes/*.css$ Allow: /research/themes/*.css? Allow: /research/themes/*.js$ Allow: /research/themes/*.js? Allow: /research/themes/*.gif Allow: /research/themes/*.jpg Allow: /research/themes/*.jpeg Allow: /research/themes/*.png Allow: /research/themes/*.svg # Block Queries (Facets, Pagination, etc) Disallow: /research/*?* # Directories Disallow: /core/ Disallow: /modules/ Disallow: /themes/ Disallow: /profiles/ Disallow: /research/core/ Disallow: /research/modules/ Disallow: /research/themes/ Disallow: /research/profiles/ # Files Disallow: /README.txt Disallow: /web.config Disallow: /research/README.txt Disallow: /research/web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/ Disallow: /search/ Disallow: /user/ Disallow: /research/admin/ Disallow: /research/comment/ Disallow: /research/contact/ Disallow: /research/logout/ Disallow: /research/node/ Disallow: /research/search/ Disallow: /research/*/search/ Disallow: /research/treesearch/$ Disallow: /research/*/treesearch/$ Disallow: /research/user/ Disallow: /research/*/login/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/ Disallow: /?q=search/ Disallow: /?q=user/ Disallow: /research/?q=admin/ Disallow: /research/?q=comment/ Disallow: /research/?q=contact/ Disallow: /research/?q=logout/ Disallow: /research/?q=node/ Disallow: /research/?q=search/ Disallow: /research/?q=user/ # Paths (clean URLs) – No trailing slash Disallow: /admin Disallow: /comment/reply Disallow: /contact Disallow: /logout Disallow: /node Disallow: /search Disallow: /user Disallow: /research/admin Disallow: /research/comment Disallow: /research/contact Disallow: /research/logout Disallow: /research/node Disallow: /research/search Disallow: /research/*/search Disallow: /research/treesearch$ Disallow: /research/*/treesearch$ Disallow: /research/user Disallow: /research/*/login # Paths (no clean URLs) – No trailing slash Disallow: /?q=admin Disallow: /?q=comment Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node Disallow: /?q=search Disallow: /?q=user Disallow: /research/?q=admin Disallow: /research/?q=comment Disallow: /research/?q=contact Disallow: /research/?q=logout Disallow: /research/?q=node Disallow: /research/?q=search Disallow: /research/?q=user # --- Yahoo --- User-agent: Slurp sitemap: https://www.fs.usda.gov/sitemap.xml sitemap: https://www.fs.usda.gov/research/sitemap.xml # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /modules/*.svg Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: /themes/*.svg Allow: /research/core/*.css$ Allow: /research/core/*.css? Allow: /research/core/*.js$ Allow: /research/core/*.js? Allow: /research/core/*.gif Allow: /research/core/*.jpg Allow: /research/core/*.jpeg Allow: /research/core/*.png Allow: /research/core/*.svg Allow: /research/modules/*.css$ Allow: /research/modules/*.css? Allow: /research/modules/*.js$ Allow: /research/modules/*.js? Allow: /research/modules/*.gif Allow: /research/modules/*.jpg Allow: /research/modules/*.jpeg Allow: /research/modules/*.png Allow: /research/modules/*.svg Allow: /research/themes/*.css$ Allow: /research/themes/*.css? Allow: /research/themes/*.js$ Allow: /research/themes/*.js? Allow: /research/themes/*.gif Allow: /research/themes/*.jpg Allow: /research/themes/*.jpeg Allow: /research/themes/*.png Allow: /research/themes/*.svg # Block Queries (Facets, Pagination, etc) Disallow: /research/*?* # Directories Disallow: /core/ Disallow: /modules/ Disallow: /themes/ Disallow: /profiles/ Disallow: /research/core/ Disallow: /research/modules/ Disallow: /research/themes/ Disallow: /research/profiles/ # Files Disallow: /README.txt Disallow: /web.config Disallow: /research/README.txt Disallow: /research/web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/ Disallow: /search/ Disallow: /user/ Disallow: /research/admin/ Disallow: /research/comment/ Disallow: /research/contact/ Disallow: /research/logout/ Disallow: /research/node/ Disallow: /research/search/ Disallow: /research/*/search/ Disallow: /research/treesearch/$ Disallow: /research/*/treesearch/$ Disallow: /research/user/ Disallow: /research/*/login/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/ Disallow: /?q=search/ Disallow: /?q=user/ Disallow: /research/?q=admin/ Disallow: /research/?q=comment/ Disallow: /research/?q=contact/ Disallow: /research/?q=logout/ Disallow: /research/?q=node/ Disallow: /research/?q=search/ Disallow: /research/?q=user/ # Paths (clean URLs) – No trailing slash Disallow: /admin Disallow: /comment/reply Disallow: /contact Disallow: /logout Disallow: /node Disallow: /search Disallow: /user Disallow: /research/admin Disallow: /research/comment Disallow: /research/contact Disallow: /research/logout Disallow: /research/node Disallow: /research/search Disallow: /research/*/search Disallow: /research/treesearch$ Disallow: /research/*/treesearch$ Disallow: /research/user Disallow: /research/*/login # Paths (no clean URLs) – No trailing slash Disallow: /?q=admin Disallow: /?q=comment Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node Disallow: /?q=search Disallow: /?q=user Disallow: /research/?q=admin Disallow: /research/?q=comment Disallow: /research/?q=contact Disallow: /research/?q=logout Disallow: /research/?q=node Disallow: /research/?q=search Disallow: /research/?q=user # --- DuckDuckGo --- User-agent: DuckDuckBot sitemap: https://www.fs.usda.gov/sitemap.xml sitemap: https://www.fs.usda.gov/research/sitemap.xml # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /modules/*.svg Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: /themes/*.svg Allow: /research/core/*.css$ Allow: /research/core/*.css? Allow: /research/core/*.js$ Allow: /research/core/*.js? Allow: /research/core/*.gif Allow: /research/core/*.jpg Allow: /research/core/*.jpeg Allow: /research/core/*.png Allow: /research/core/*.svg Allow: /research/modules/*.css$ Allow: /research/modules/*.css? Allow: /research/modules/*.js$ Allow: /research/modules/*.js? Allow: /research/modules/*.gif Allow: /research/modules/*.jpg Allow: /research/modules/*.jpeg Allow: /research/modules/*.png Allow: /research/modules/*.svg Allow: /research/themes/*.css$ Allow: /research/themes/*.css? Allow: /research/themes/*.js$ Allow: /research/themes/*.js? Allow: /research/themes/*.gif Allow: /research/themes/*.jpg Allow: /research/themes/*.jpeg Allow: /research/themes/*.png Allow: /research/themes/*.svg # Block Queries (Facets, Pagination, etc) Disallow: /research/*?* # Directories Disallow: /core/ Disallow: /modules/ Disallow: /themes/ Disallow: /profiles/ Disallow: /research/core/ Disallow: /research/modules/ Disallow: /research/themes/ Disallow: /research/profiles/ # Files Disallow: /README.txt Disallow: /web.config Disallow: /research/README.txt Disallow: /research/web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/ Disallow: /search/ Disallow: /user/ Disallow: /research/admin/ Disallow: /research/comment/ Disallow: /research/contact/ Disallow: /research/logout/ Disallow: /research/node/ Disallow: /research/search/ Disallow: /research/*/search/ Disallow: /research/treesearch/$ Disallow: /research/*/treesearch/$ Disallow: /research/user/ Disallow: /research/*/login/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/ Disallow: /?q=search/ Disallow: /?q=user/ Disallow: /research/?q=admin/ Disallow: /research/?q=comment/ Disallow: /research/?q=contact/ Disallow: /research/?q=logout/ Disallow: /research/?q=node/ Disallow: /research/?q=search/ Disallow: /research/?q=user/ # Paths (clean URLs) – No trailing slash Disallow: /admin Disallow: /comment/reply Disallow: /contact Disallow: /logout Disallow: /node Disallow: /search Disallow: /user Disallow: /research/admin Disallow: /research/comment Disallow: /research/contact Disallow: /research/logout Disallow: /research/node Disallow: /research/search Disallow: /research/*/search Disallow: /research/treesearch$ Disallow: /research/*/treesearch$ Disallow: /research/user Disallow: /research/*/login # Paths (no clean URLs) – No trailing slash Disallow: /?q=admin Disallow: /?q=comment Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node Disallow: /?q=search Disallow: /?q=user Disallow: /research/?q=admin Disallow: /research/?q=comment Disallow: /research/?q=contact Disallow: /research/?q=logout Disallow: /research/?q=node Disallow: /research/?q=search Disallow: /research/?q=user # --- SiteImprove --- User-agent: SiteimproveBot sitemap: https://www.fs.usda.gov/sitemap.xml sitemap: https://www.fs.usda.gov/research/sitemap.xml # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /modules/*.svg Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png Allow: /themes/*.svg Allow: /research/core/*.css$ Allow: /research/core/*.css? Allow: /research/core/*.js$ Allow: /research/core/*.js? Allow: /research/core/*.gif Allow: /research/core/*.jpg Allow: /research/core/*.jpeg Allow: /research/core/*.png Allow: /research/core/*.svg Allow: /research/modules/*.css$ Allow: /research/modules/*.css? Allow: /research/modules/*.js$ Allow: /research/modules/*.js? Allow: /research/modules/*.gif Allow: /research/modules/*.jpg Allow: /research/modules/*.jpeg Allow: /research/modules/*.png Allow: /research/modules/*.svg Allow: /research/themes/*.css$ Allow: /research/themes/*.css? Allow: /research/themes/*.js$ Allow: /research/themes/*.js? Allow: /research/themes/*.gif Allow: /research/themes/*.jpg Allow: /research/themes/*.jpeg Allow: /research/themes/*.png Allow: /research/themes/*.svg # Block Queries (Facets, Pagination, etc) Disallow: /research/*?* # Directories Disallow: /core/ Disallow: /modules/ Disallow: /themes/ Disallow: /profiles/ Disallow: /research/core/ Disallow: /research/modules/ Disallow: /research/themes/ Disallow: /research/profiles/ # Files Disallow: /README.txt Disallow: /web.config Disallow: /research/README.txt Disallow: /research/web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/ Disallow: /search/ Disallow: /user/ Disallow: /research/admin/ Disallow: /research/comment/ Disallow: /research/contact/ Disallow: /research/logout/ Disallow: /research/node/ Disallow: /research/search/ Disallow: /research/*/search/ Disallow: /research/treesearch/$ Disallow: /research/*/treesearch/$ Disallow: /research/user/ Disallow: /research/*/login/ # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/ Disallow: /?q=search/ Disallow: /?q=user/ Disallow: /research/?q=admin/ Disallow: /research/?q=comment/ Disallow: /research/?q=contact/ Disallow: /research/?q=logout/ Disallow: /research/?q=node/ Disallow: /research/?q=search/ Disallow: /research/?q=user/ # Paths (clean URLs) – No trailing slash Disallow: /admin Disallow: /comment/reply Disallow: /contact Disallow: /logout Disallow: /node Disallow: /search Disallow: /user Disallow: /research/admin Disallow: /research/comment Disallow: /research/contact Disallow: /research/logout Disallow: /research/node Disallow: /research/search Disallow: /research/*/search Disallow: /research/treesearch$ Disallow: /research/*/treesearch$ Disallow: /research/user Disallow: /research/*/login # Paths (no clean URLs) – No trailing slash Disallow: /?q=admin Disallow: /?q=comment Disallow: /?q=contact Disallow: /?q=logout Disallow: /?q=node Disallow: /?q=search Disallow: /?q=user Disallow: /research/?q=admin Disallow: /research/?q=comment Disallow: /research/?q=contact Disallow: /research/?q=logout Disallow: /research/?q=node Disallow: /research/?q=search Disallow: /research/?q=user