# Lexiconn recommended for M2 plus the M2.1 defaults
# https://www.lexiconn.com/blog/2016/05/an-updated-robots-txt-file-for-magento-2-stores/

# Generic Crawlers are allowed
User-agent: *

# Directories
Disallow: /app/
Disallow: /bin/
Disallow: /catalog/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /review/
Disallow: /scripts/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
 
# Paths (clean URLs)
Disallow: /index.php/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
Disallow: /contacts/
Disallow: /customer/
Disallow: /customize/
Disallow: /newsletter/
Disallow: /sendfriend/
Disallow: /wishlist/
 
# Files
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /CONTRIBUTING.md
Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html
Disallow: /COPYING.txt
Disallow: /Gruntfile.js
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /nginx.conf.sample
Disallow: /package.json
Disallow: /php.ini.sample
Disallow: /RELEASE_NOTES.txt
 
# Do not index pages that are sorted or filtered.
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=
 
# Do not index session ID
Disallow: /*.php$
Disallow: /*?SID=
 
# Do not index anything else with a query param
Disallow: /*?

# CVS, SVN directory and dump files
Disallow: /*.CVS
Disallow: /*.Zip$
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$

# Dive Gear Express site specific edits below

# Do not crawl aliases(with prefix that ends in hypen) or test-products category
Disallow: /*alias-
Disallow: /bcs/alias-tank-straps
Disallow: /hardware/alias-backplates
Disallow: /gas-tanks/alias-drysuit-inflation
Disallow: /gas-tanks/valves/alias-xss-valve-din-7-8-14-unf-threads-w-vindicator
Disallow: /rebreathers/alias-rebreather-books
Disallow: /tools/dpvs/alias-submerge-minnus-dpv
Disallow: /test-category/alias-test-subcategory
Disallow: /test-products

# Do not crawl some Knowledge Base content
Disallow: /library/tek-tips/
Disallow: /library/product-tips/
Disallow: /library/company/

# Sitemap of CMS, Category, Product pages auto-generated by CUE XML Sitemap Plus Generator 
Sitemap: https://www.divegearexpress.com/sitemap_index_index.xml

# Sitemap of Library Article pages generated manually
Sitemap: https://www.divegearexpress.com/sitemap_index_articles.xml