[SOLVED] crawlers/bots on the internet request random urls on my store

Code: Select all

# reject known hackers and spammers
Order Deny,Allow
Deny from 37.187.90.226

Code: Select all

# BLOCK UNNECESSARY REQUEST METHODS
RewriteCond %{REQUEST_METHOD} ^(CONNECT|DEBUG|DELETE|MOVE|PUT|TRACE|TRACK) [NC]
RewriteRule .* - [F,L]

ServerSignature Off
RewriteCond %{QUERY_STRING} (javascript:).*(\;) [NC,OR]
RewriteCond %{QUERY_STRING} (<|%3C).*script.*(>|%3) [NC,OR]
RewriteCond %{QUERY_STRING} (\;|\'|\"|%22).*(union|UNION|select|SELECT|insert|INSERT|drop|DROP|update|UPDATE|md5|MD5|benchmark) [NC,OR]
RewriteCond %{QUERY_STRING} (base64_encode|localhost|mosconfig) [NC,OR]
RewriteCond %{QUERY_STRING} (boot\.ini|echo.*kae|etc/passwd|passwd|eval|$_POST) [NC,OR]
RewriteCond %{QUERY_STRING} (GLOBALS|REQUEST)(=|\[|%) [NC]
RewriteRule ^(.*)$ blacklist.php [L]

Code: Select all

error_log('BLACKLISTING:' . $_SERVER['REMOTE_ADDR'] . ' ' . $_SERVER['REQUEST_URI']);
header($_SERVER['SERVER_PROTOCOL'] . " 404 Not Found", true);
exit();

Code: Select all

function valid_request () {
	$invalid		= "(\(\))"; // lets not look for quotes. [good]bots use them constantly. looking for () since technically parenthesis arent valid
	$period 		= "(\\002e|%2e|%252e|%c0%2e|\.)";
	$slash 			= "(\\2215|%2f|%252f|%5c|%255c|%c0%2f|%c0%af|\/|\\\)"; // http://security.stackexchange.com/questions/48879/why-does-directory-traversal-attack-c0af-work
	$routes 		= "(etc|dev|irj)" . $slash . "(passwds?|group|null|portal)|allow_url_include|auto_prepend_file|route_*=http";
	$filetypes 		= $period . "+(sql|db|sqlite|log|ini|cgi|bak|rc|apk|pkg|deb|rpm|exe|msi|bak|old|cache|lock|autoload|gitignore|ht(access|passwds?)|cpanel_config|history|zip|bz2|tar|(t)?gz)";
	$cgis 			= "cgi(-|_){0,1}(bin(-sdb)?|mod|sys)?";
	$phps 			= "(changelog|version|license|command|xmlrpc|admin-ajax|wsdl|tmp|shell|stats|echo|(my)?sql|sample|modx|load-config|cron|wp-(up|tmp|sitemaps|sitemap(s)?|signup|settings|" . $period . "?config(uration|-sample|bak)?))" . $period . "php";
	$doors 			= "(" . $cgis . $slash . "(common" . $period . "(cgi|php))|manager" . $slash . "html|stssys" . $period . "htm|((mysql|phpmy|db|my)admin|pma|sqlitemanager|sqlite|websql)" . $slash . "|(jmx|web)-console|bitrix|invoker|muieblackcat|w00tw00t|websql|xampp|cfide|wordpress|wp-admin|hnap1|tmunblock|soapcaller|zabbix|elfinder)";
	$sqls 			= "((un)?hex\(|name_const\(|char\(|a=0)";
	$nulls 			= "(%00|%2500)";
	$truth 			= "(.{1,4})=\1"; // catch OR always-true (1=1) clauses via sql inject - not used atm, its too broad and may capture search=chowder (ch=ch) for example
	$regex 			= "/$invalid|$period{1,2}$slash|$routes|$filetypes|$phps|$doors|$sqls|$nulls/i";
	$regex_ignore 	= "/captcha=0/i";
	$results 		= '';
	$matches 		= array();
	$str 			= $_SERVER['SERVER_NAME'] . $_SERVER['REQUEST_URI'];
	$has_agent 		= isset($_SERVER['HTTP_USER_AGENT']);
	$user_agent 	= ($has_agent ? $_SERVER['HTTP_USER_AGENT'] : 'no user agent');

	if (preg_match_all($regex, preg_replace($regex_ignore, '', $str), $matches) || !$has_agent) {
		if ($matches[0]) {
			$matches[0] = array_unique($matches[0]);
			foreach ($matches[0] as $match) {
				$results .= $match . ' ';
			}
		}
		if (!$has_agent) $results .= ' | No User Agent';
		return $results;
	}
	return true;
}

Code: Select all

/* VALIDATE URL */
$valid_request = $this->valid_request();
if ($valid_request !== true) {
	error_log('Invalid Request from IP: '.$_SERVER['REMOTE_ADDR']);
	error_log('Reason: '.$valid_request);
	error_log('URI: '.$_SERVER['REQUEST_URI']);
	if (isset($_SERVER['HTTP_USER_AGENT'])) {
		error_log('USER-AGENT: '.$_SERVER['HTTP_USER_AGENT']);
	} else {
		error_log('USER-AGENT: None');
	}
	header($_SERVER['SERVER_PROTOCOL'] . " 404 Not found", true);
	exit;
}

Code: Select all

RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_URI} !.*\.(ico|gif|jpg|jpeg|png|js|css)
RewriteRule ^([^?]*) index.php?_route_=$1 [L,QSA]

Code: Select all

RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_URI} !.*\.(env|php|xml|ashx|cfg|dat|ico|cur|txt|mp3|webp|svg|ttf|eot|woff|woff2|gif|jpg|JPG|jpeg|JPEG|png|js|cfg|css|pdf|zip|env|tar|sql|gz|tar|exe|rar|arj|cab|iso|rpm|tbz|tgz|old|bak|backup|dump|db|7z|asp|aspx|exp|html|htm)$
RewriteRule ^([^?]*) index.php?_route_=$1 [L,QSA]

Code: Select all

	RewriteCond %{REQUEST_URI} !.*\..*$
	RewriteCond %{REQUEST_FILENAME} !-f
	RewriteCond %{REQUEST_FILENAME} !-d
	RewriteRule ^([^?]*) index.php [L,QSA]