Problems with robots.txt and canonical url

Code: Select all

User-agent: *

Disallow: /admin
Disallow: /cache
Disallow: /catalog
Disallow: /download
Disallow: /export
Disallow: /system
Disallow: /index.php?route=account
Disallow: /index.php?route=account/account
Disallow: /index.php?route=checkout/cart
Disallow: /index.php?route=checkout/shipping
Disallow: /index.php?route=common/home
Disallow: /index.php?route=product/product/captcha
Disallow: /index.php?route=product/product*&manufacturer_id=
Disallow: /index.php?route=product/search
Disallow: /index.php?route=information/contact/captcha
Disallow: /index.php?route=information/information&information_id=
Disallow: /index.php?route=account/login
Disallow: /*&limit
Disallow: /*?limit
Disallow: /*?sort
Disallow: /*&sort
Disallow: /*?order
Disallow: /*&order
Disallow: /*?price
Disallow: /*&price
Disallow: /*?brand_tabletpc
Disallow: /*&brand_tabletpc
Disallow: /*?color_default
Disallow: /*&color_default
Disallow: /*?filter_tag
Disallow: /*&filter_tag
Disallow: /*?mode
Disallow: /*&mode
Disallow: /*?cat
Disallow: /*&cat
Disallow: /*?dir
Disallow: /*&dir
Disallow: /*?color
Disallow: /*&color
Disallow: /*?product_id
Disallow: /*&product_id
Disallow: /*?minprice
Disallow: /*&minprice
Disallow: /*?maxprice
Disallow: /*&maxprice
Disallow: /*?route=checkout/
Disallow: /*?route=account/
Disallow: /*?route=product/search
Disallow: /*?page=1
Disallow: /*&create=1
Disallow: /?route=information/contact
Disallow: /*?route=affiliate/
Disallow: /*?keyword
Disallow: /*?av
Disallow: /admin/
Disallow: /system/
Disallow: /catalog/
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /pages/thankyou.html

Sitemap: https://www.xxx.com/index.php?route=extension/feed/google_sitemap

Code: Select all

RewriteOptions inherit
Options +SymLinksIfOwnerMatch

# Prevent Directory listing 
Options -Indexes

# Prevent Direct Access to files
<FilesMatch "(?i)((^\.|\.txt|\.tpl|\.ini|\.log|(?<!robots)\.txt))">
    Order deny,allow
    Deny from all
    SetEnvIfNoCase User-Agent "Googlebot" goodbot
    Allow from env=goodbot 
</FilesMatch> 

<Files .htaccess>
    order allow,deny
    deny from all
</Files>

# disable etags
FileETag none

# SEO URL Settings
RewriteEngine On

# force www
RewriteCond %{HTTPS_HOST} ^www\.my-site\.com$
RewriteRule ^/?$ "https\:\/\/my-site\.com\/" [R=301,L]

# If your opencart installation does not run on the main web folder,
# make sure you folder it does run in ie. / becomes /shop/

RewriteBase /

# force https if not present, and remove www. if used in domain name
#RewriteCond %{HTTP_HOST} !^www\.
#RewriteRule ^(.*)$ http://www.%{HTTP_HOST}/$1 [R=301,L]

RewriteCond %{HTTP_HOST} ^my-site\.com [NC]
RewriteCond %{SERVER_PORT} 80
RewriteRule ^(.*)$ https://www.my-site.com/$1 [R,L]

# force HTTPS / SSL
#RewriteCond %{HTTPS} off
#RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [R,L]




# SEO: if it's not a real file or directory, and not certain images and auxiliary files, feed it to index.php
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_URI} !.*\.(ico|gif|jpg|jpeg|png|js|css)
RewriteCond %{REQUEST_URI} !^/[0-9]+\..+\.cpaneldcv$
RewriteCond %{REQUEST_URI} !^/\.well-known/acme-challenge/.+$
RewriteCond %{REQUEST_URI} !^/\.well-known/pki-validation/[A-F0-9]{32}\.txt(?:\ Comodo\ DCV)?$
RewriteRule ^([^?]*) index.php?_route_=$1 [L,QSA]

# block libwww-perl-based user agents (?)
RewriteCond %{HTTP_USER_AGENT} libwww-perl.* 
RewriteCond %{REQUEST_URI} !^/[0-9]+\..+\.cpaneldcv$
RewriteCond %{REQUEST_URI} !^/\.well-known/acme-challenge/.+$
RewriteCond %{REQUEST_URI} !^/\.well-known/pki-validation/[A-F0-9]{32}\.txt(?:\ Comodo\ DCV)?$
RewriteRule .* ? [F,L]

# Enable Gzip Compression
AddOutputFilterByType DEFLATE text/text
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/xml
AddOutputFilterByType DEFLATE text/css
AddOutputFilterByType DEFLATE application/xml
AddOutputFilterByType DEFLATE application/xhtml+xml
AddOutputFilterByType DEFLATE application/rss+xml
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript

# BEGIN Expire headers
<IfModule mod_expires.c>
  ExpiresActive On

  # Images
  ExpiresByType image/jpeg "access plus 1 year"
  ExpiresByType image/gif "access plus 1 year"
  ExpiresByType image/png "access plus 1 year"
  ExpiresByType image/webp "access plus 1 year"
  ExpiresByType image/svg+xml "access plus 1 year"
  ExpiresByType image/x-icon "access plus 1 year"

  # Video
  ExpiresByType video/mp4 "access plus 1 year"
  ExpiresByType video/mpeg "access plus 1 year"

  # CSS, JavaScript
  ExpiresByType text/css "access plus 1 month"
  ExpiresByType text/javascript "access plus 1 month"
  ExpiresByType application/javascript "access plus 1 month"

  # Others
  ExpiresByType application/pdf "access plus 1 month"
  ExpiresByType application/x-shockwave-flash "access plus 1 month"
</IfModule>
# END Expire headers

# Remove Browser Bugs
BrowserMatch ^Mozilla/4 gzip-only-text/html
BrowserMatch ^Mozilla/4\.0[678] no-gzip
BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
Header append Vary User-Agent

<IfModule mod_security.c>
 SecFilterEngine On
 SecFilterScanPOST On
</IfModule>

# Icon cross origin fix
<FilesMatch "\.(ttf|ttc|otf|eot|woff|font.css|css)$">
<IfModule mod_headers.c>
Header set Access-Control-Allow-Origin "*"
</IfModule>
</FilesMatch>

### Additional Settings that may need to be enabled for some servers
### Uncomment the commands by removing the # sign in front of it.
### If you get an "Internal Server Error 500" after enabling any of the following settings, restore the # as this means your host doesn't allow that.

# 1. If your cart only allows you to add one item at a time, it is possible register_globals is on. This may work to disable it:
#php_flag register_globals off

# 2. If your cart has magic quotes enabled, This may work to disable it:
#php_flag magic_quotes_gpc Off

# 3. Set max upload file size. Most hosts will limit this and not allow it to be overridden but you can try
#php_value upload_max_filesize 999M

# 4. set max post size. uncomment this line if you have a lot of product options or are getting errors where forms are not saving all fields
#php_value post_max_size 999M

# 5. set max time script can take. uncomment this line if you have a lot of product options or are getting errors where forms are not saving all fields
#php_value max_execution_time 200

# 6. set max time for input to be recieved. Uncomment this line if you have a lot of product options or are getting errors where forms are not saving all fields
#php_value max_input_time 200

# 7. disable open_basedir limitations
#RewriteCond %{HTTP_HOST} ^my-site\.com$ [OR]
#RewriteCond %{HTTP_HOST} ^www\.my-site\.com$
RewriteRule ^my-site\-dokumenti\/pogoji\ uporabe\-my-site\-2017\.pdf$ "https\:\/\/www\.my-site\.com\/my-site\-dokumenti\/pogoji\-uporabe\-my-site\-2018\.pdf" [R=301,L]
#RewriteCond %{HTTP_HOST} ^my-site\.com$ [OR]
#RewriteCond %{HTTP_HOST} ^www\.my-site\.com$
RewriteRule ^my-site\-dokumenti\/pogoji\ uporabe\-my-site\.pdf$ "https\:\/\/my-site\.com\/my-site\-dokumenti\/pogoji\-uporabe\-my-site\-2018\.pdf" [R=301,L]
#RewriteCond %{HTTP_HOST} ^my-site\.com$ [OR]
#RewriteCond %{HTTP_HOST} ^www\.my-site\.com$
RewriteRule ^darila\ ob\ rojstvu\/darilna\-odeja\-darilo\-ob\-rojstvu\.html$ "https\:\/\/my-site\.com\/darila\-ob\-rojstvu\/darilna\-odeja\-darilo\-ob\-rojstvu\.html" [R=301,L]
#RewriteCond %{HTTP_HOST} ^my-site\.com$ [OR]
#RewriteCond %{HTTP_HOST} ^www\.my-site\.com$
RewriteRule ^darila\ ob\ rojstvu\/darilna\-odeja\-darilo\-ob\-rojstvu\.html$ "https\:\/\/my-site\.com\/darila\-ob\-rojstvu\/darilna\-odeja\-darilo\-ob\-rojstvu\.html" [R=301,L]
RewriteCond %{HTTP_HOST} ^yy\.xx\.com$ [OR]
RewriteCond %{HTTP_HOST} ^www\.yy\.xx\.com$
RewriteRule ^/?$ "https\:\/\/xx\.com\/blabla" [R=301,L]