# If the Joomla site is installed within a folder such as at # e.g. www.example.com/joomla/ the robots.txt file MUST be # moved to the site root at e.g. www.example.com/robots.txt # AND the joomla folder name MUST be prefixed to the disallowed # path, e.g. the Disallow rule for the /administrator/ folder # MUST be changed to read Disallow: /joomla/administrator/ # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/orig.html # # For syntax checking, see: # http://tool.motoricerca.info/robots-checker.phtml User-Agent: Googlebot Allow: /*.js* Allow: /*.css* Allow: /*.png* Allow: /*.jpg* Allow: /*.gif* Disallow: /administrator/ Disallow: /bin/ Disallow: /cache/ Disallow: /cli/ Disallow: /includes/ Disallow: /installation/ Disallow: /language/ Disallow: /layouts/ Disallow: /libraries/ Disallow: /logs/ Disallow: /out/ Disallow: /banners/ Disallow: /BI/ Disallow: /article-tree/ Disallow: /business-tree/ Disallow: /*/component/ Disallow: /fi/komponentti/* Disallow: /pt/componente/* Disallow: /no/komponent/* Disallow: /nl/bestanddeel/* Disallow: /de/Komponente/* Disallow: /pl/składnik/* Disallow: /sv/komponent/* Disallow: /it/componente/* Disallow: /es/componente/* Disallow: /da/komponent/* Disallow: /is/hluti/* Disallow: /fr/composant/* Disallow: */com_docmanpaypal/* Disallow: *com_joomlatools* Disallow: /joomlaspeedtest/ Disallow: /example/ Disallow: /exampleAMP/ Disallow: /tmp/ #Disabled all tags - throw 404 so that they get deindexed Disallow: */tag/* Disallow: /tags.html* Disallow: /tags* Disallow: /log-in.html Disallow: */log-in/* Disallow: */contact-us.html* Disallow: /forums/ Disallow: /forum/ Disallow: /osdownloads.html #new downloads link Disallow: */download.php* Disallow: /index.php?option=com_users* Disallow: *jos_change_template* Disallow: /*?task=view #Disallow: *view=* Disallow: /*?format=html Disallow: /index.php?option=com_banners* Disallow: /index.php?option=com_acymailing* Disallow: /index.php?subid=&option=com_acymailing* Disallow: /acymailing/ Disallow: */file.html # extra stuff being blocked due to the below #Disallow: */file #Disallow: /file/ Disallow: /index.php?option=com_ninjarsssyndicator #Disabled feed links - we therefore don't need to block them #Disallow: */feed/* #The below was breaking some images from being crawled Disallow: */page-*.html Disallow: */page-* Disallow: */search* Disallow: *com_search* #Remove thin download / demo pages Disallow: /*/joomla-25-and-joomla-3-plugins* Disallow: /*/joomla-25-and-joomla-3-modules* Disallow: /*/html-templates* Disallow: /*/psd-templates* Disallow: /*/extension-demos/* #Disallow: /*/wordpress-plugins/* Disallow: /*/downloads* Disallow: /*/joomla/joomla-extension-demos/* Disallow: /*/j/extension-demos* #Ezoic JS pages getting index Disallow: */porpoiseant/* Disallow: */tardisrocinante/* Disallow: */detroitchicago/* Disallow: */humix/* #Bunch of contact us pages and other pages got crawled - remove Disallow: *contact-us.html* Disallow: *_escaped_fragment_* Disallow: *ccomment-comment* #Googlebot crawling Cookies Policy Notification Pages #Added as parameter Disallow: *cpnb_method* #Ezoic expand article link Disallow: *expand_article* User-Agent: * Allow: /*.js* Allow: /*.css* Allow: /*.png* Allow: /*.jpg* Allow: /*.gif* Disallow: /administrator/ Disallow: /bin/ Disallow: /cache/ Disallow: /cli/ Disallow: /includes/ Disallow: /installation/ Disallow: /language/ Disallow: /layouts/ Disallow: /libraries/ Disallow: /logs/ Disallow: /out/ Disallow: /banners/ Disallow: /BI/ Disallow: /article-tree/ Disallow: /business-tree/ Disallow: /*/component/ Disallow: /fi/komponentti/* Disallow: /pt/componente/* Disallow: /no/komponent/* Disallow: /nl/bestanddeel/* Disallow: /de/Komponente/* Disallow: /pl/składnik/* Disallow: /sv/komponent/* Disallow: /it/componente/* Disallow: /es/componente/* Disallow: /da/komponent/* Disallow: /fr/composant/* Disallow: /is/hluti/* Disallow: */com_docmanpaypal/* Disallow: *com_joomlatools* Disallow: /joomlaspeedtest/ Disallow: /example/ Disallow: /exampleAMP/ Disallow: /tmp/ #Disabled all tags - throw 404 so that they get deindexed Disallow: */tag/* Disallow: /tags.html* Disallow: /tags* Disallow: /log-in.html Disallow: */log-in/* Disallow: */contact-us.html* Disallow: /forums/ Disallow: /forum/ Disallow: /osdownloads.html #new downloads link Disallow: */download.php* Disallow: /index.php?option=com_users* Disallow: *jos_change_template* Disallow: /*?task=view #Disallow: *view=* Disallow: /*?format=html Disallow: /index.php?option=com_banners* Disallow: /index.php?option=com_acymailing* Disallow: /index.php?subid=&option=com_acymailing* Disallow: /acymailing/ Disallow: */file.html # extra stuff being blocked due to the below #Disallow: */file #Disallow: /file/ Disallow: /index.php?option=com_ninjarsssyndicator #Disabled feed links - we therefore don't need to block them #Disallow: */feed/* #The below was breaking some images from being crawled Disallow: */page-*.html Disallow: */page-* Disallow: */search* Disallow: *com_search* #Remove thin download / demo pages Disallow: /*/joomla-25-and-joomla-3-plugins* Disallow: /*/joomla-25-and-joomla-3-modules* Disallow: /*/html-templates* Disallow: /*/psd-templates* Disallow: /*/extension-demos/* #Disallow: /*/wordpress-plugins/* Disallow: /*/downloads* Disallow: /*/joomla/joomla-extension-demos/* Disallow: /*/j/extension-demos* #Bunch of contact us pages got crawled - remove Disallow: *contact-us.html* Disallow: *_escaped_fragment_* Disallow: *ccomment-comment* #Googlebot crawling Cookies Policy Notification Pages #Added as parameter Disallow: *cpnb_method* #Ezoic expand article link Disallow: *expand_article* #Ezoic JS pages getting index Disallow: */porpoiseant/* Disallow: */tardisrocinante/* Disallow: */detroitchicago/* Disallow: */humix/* Sitemap: https://www.collectiveray.com/sitemap.xml