3 # httpd-guardian - detect DoS attacks by monitoring requests
4 # Apache Security, http://www.apachesecurity.net
5 # Copyright (C) 2005 Ivan Ristic <ivanr@webkreator.com>
7 # $Id: httpd-guardian,v 1.6 2005/12/04 11:30:35 ivanr Exp $
9 # This program is free software; you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, version 2.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23 # This script is designed to monitor all web server requests through
24 # the piped logging mechanism. It keeps track of the number of requests
25 # sent from each IP address. Request speed is calculated at 1 minute and
26 # 5 minute intervals. Once a threshold is reached, httpd-guardian can
27 # either emit a warning or execute a script to block the IP address.
29 # Error message will be sent to stderr, which means they will end up
30 # in the Apache error log.
32 # Usage (in httpd.conf)
33 # ---------------------
35 # Without mod_security, Apache 1.x:
37 # LogFormat "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %{UNIQUE_ID}e \"-\" %T 0 \"%{modsec_message}i\" 0" guardian
38 # CustomLog "|/path/to/httpd-guardian" guardian
40 # or without mod_security, Apache 2.x:
42 # LogFormat "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %{UNIQUE_ID}e \"-\" %T %D \"%{modsec_message}i\" 0" guardian
43 # CustomLog "|/path/to/httpd-guardian" guardian
45 # or with mod_security (better):
47 # SecGuardianLog "|/path/to/httpd-guardian"
49 # NOTE: In order for this script to be effective it must be able to
50 # see all requests coming to the web server. This will not happen
51 # if you are using per-virtual host logging. In such cases either
52 # use the ModSecurity 1.9 SecGuardianLog directive (which was designed
53 # for this very purpose).
59 # 1) First you need to make sure you have Spread running on the machine
60 # where you intend to run httpd-guardian on.
62 # 2) Then uncomment line "use Spread;" in this script, and change
65 # 3) The default port for Spread is 3333. Change it if you want to
66 # and then start httpd-guardian. We will be looking for messages
67 # in the Spread group called "httpd-guardian".
69 # TODO Add support to ignore certain log entries based on a
70 # regex applied script_name.
72 # TODO Warn about session hijacking.
74 # TODO Track ip addresses, sessions, and individual users.
76 # TODO Detect status code anomalies.
78 # TODO Track accesses to specific pages.
80 # TODO Open proxy detection.
82 # TODO Check IP addresses with blacklists (e.g.
83 # http://www.spamhaus.org/XBL/).
85 # TODO Is there a point to keep per-vhost state?
87 # TODO Enhance the script to tail a log file - useful for test
88 # runs, in preparation for deployment.
90 # TODO Can we track connections as Apache creates and destroys them?
92 # TODO Command-line option to support multiple log formats. E.g. common,
93 # combined, vcombined, guardian.
95 # TODO Command-line option not to save state
104 # -- Configuration----------------------------------------------------------
107 my $SPREAD_CLIENT_NAME = "httpd-guardian";
108 my $SPREAD_DAEMON = "3333";
109 my $SPREAD_GROUP_NAME = "httpd-guardian";
110 my $SPREAD_TIMEOUT = 1;
112 # If defined, execute this command when a threshold is reached
113 # block the IP address for one hour.
114 # $PROTECT_EXEC = "/sbin/blacklist block %s 3600";
115 # $PROTECT_EXEC = "/sbin/samtool -block -ip %s -dur 3600 snortsam.example.com";
119 my $PROTECT_EXEC = "/usr/bin/logger Possible DoS Attack from %s";
121 # Max. speed allowed, in requests per
122 # second, measured over an 1-minute period
123 #my $THRESHOLD_1MIN = 2; # 120 requests in a minute
126 my $THRESHOLD_1MIN = 0.01;
128 # Max. speed allowed, in requests per
129 # second, measured over a 5-minute period
130 my $THRESHOLD_5MIN = 1; # 360 requests in 5 minutes
132 # If defined, httpd-guardian will make a copy
133 # of the data it receives from Apache
137 # Remove IP address data after a 10-minute inactivity
138 my $STALE_INTERVAL = 400;
140 # Where to save state (at this point only useful
141 # for monitoring what the script does)
142 my $SAVE_STATE_FILE = "/tmp/httpd-guardian.state";
144 # How often to save state (in seconds).
145 my $SAVE_STATE_INTERVAL = 10;
150 # -----------------------------------------------------------------
167 # -- log parsing regular expression
170 # 127.0.0.1 192.168.2.11 - - [05/Jul/2005:16:56:54 +0100]
171 # "GET /favicon.ico HTTP/1.1" 404 285 "-"
172 # "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.8) Gecko/20050511 Firefox/1.0.4"
175 my $logline_regex = "";
178 $logline_regex .= "^(\\S+)";
179 # remote host, remote username, local username
180 $logline_regex .= "\\ (\\S+)\\ (\\S+)\\ (\\S+)";
181 # date, time, and gmt offset
182 $logline_regex .= "\\ \\[([^:]+):(\\d+:\\d+:\\d+)\\ ([^\\]]+)\\]";
183 # request method + request uri + protocol (as one field)
184 $logline_regex .= "\\ \"(.*)\"";
186 $logline_regex .= "\\ (\\d+)\\ (\\S+)";
187 # referer, user_agent
188 $logline_regex .= "\\ \"(.*)\"\\ \"(.*)\"";
189 # uniqueid, session, duration, duration_msec
190 $logline_regex .= "\\ (\\S+)\\ \"(.*)\"\\ (\\d+)\\ (\\d+)";
191 # modsec_message, modsec_rating
192 $logline_regex .= "\\ \"(.*)\"\\ (\\d+)";
194 # the rest (always keep this part of the regex)
195 $logline_regex .= "(.*)\$";
197 my $therequest_regex = "(\\S+)\\ (.*?)\\ (\\S+)";
200 my %ipaddresses = ();
209 $request{"invalid"} = 0;
211 my @parsed_logline = /$logline_regex/x;
212 if (@parsed_logline == 0) {
217 $request{"hostname"},
218 $request{"remote_ip"},
219 $request{"remote_username"},
220 $request{"username"},
223 $request{"gmt_offset"},
224 $request{"the_request"},
226 $request{"bytes_out"},
228 $request{"user_agent"},
229 $request{"unique_id"},
230 $request{"session_id"},
231 $request{"duration"},
232 $request{"duration_msec"},
233 $request{"modsec_message"},
234 $request{"modsec_rating"},
240 print "hostname = " . $request{"hostname"} . "\n";
241 print "remote_ip = " . $request{"remote_ip"} . "\n";
242 print "remote_username = " . $request{"remote_username"} . "\n";
243 print "username = " . $request{"username"} . "\n";
244 print "date = " . $request{"date"} . "\n";
245 print "time = " . $request{"time"} . "\n";
246 print "gmt_offset = " . $request{"gmt_offset"} . "\n";
247 print "the_request = " . $request{"the_request"} . "\n";
248 print "status = " . $request{"status"} . "\n";
249 print "bytes_out = " . $request{"bytes_out"} . "\n";
250 print "referer = " . $request{"referer"} . "\n";
251 print "user_agent = " . $request{"user_agent"} . "\n";
252 print "unique_id = " . $request{"unique_id"} . "\n";
253 print "session_id = " . $request{"session_id"} . "\n";
254 print "duration = " . $request{"duration"} . "\n";
255 print "duration_msec = " . $request{"duration_msec"} . "\n";
256 print "modsec_message = " . $request{"modsec_message"} . "\n";
257 print "modsec_rating = " . $request{"modsec_rating"} . "\n";
261 # parse the request line
262 $_ = $request{"the_request"};
263 my @parsed_therequest = /$therequest_regex/x;
264 if (@parsed_therequest == 0) {
265 $request{"invalid"} = "1";
266 $request{"request_method"} = "";
267 $request{"request_uri"} = "";
268 $request{"protocol"} = "";
271 $request{"request_method"},
272 $request{"request_uri"},
274 ) = @parsed_therequest;
277 if ($request{"bytes_out"} eq "-") {
278 $request{"bytes_out"} = 0;
281 # print "date=" . $request{"date"} . "\n";
283 $request{"time_mday"},
284 $request{"time_mon"},
285 $request{"time_year"}
286 ) = ( $request{"date"} =~ m/^(\d+)\/(\S+)\/(\d+)/x );
288 # print "time=" . $request{"time"} . "\n";
290 $request{"time_hour"},
291 $request{"time_min"},
293 ) = ( $request{"time"} =~ m/(\d+):(\d+):(\d+)/x );
295 $request{"time_mon"} = $months{$request{"time_mon"}};
297 $request{"time_epoch"} = timelocal(
298 $request{"time_sec"},
299 $request{"time_min"},
300 $request{"time_hour"},
301 $request{"time_mday"},
302 $request{"time_mon"},
303 $request{"time_year"}
308 my $offset = index($request{"request_uri"}, "?");
310 $request{"script_name"} = substr($request{"request_uri"}, 0, $offset);
311 $request{"query_string"} = substr($request{"request_uri"}, $offset + 1);
313 $request{"script_name"} = $request{"request_uri"};
314 $request{"query_string"} = "";
317 $request{"request_uri"} =~ s/\%([A-Fa-f0-9]{2})/pack('C', hex($1))/seg;
318 $request{"query_string"} =~ s/\%([A-Fa-f0-9]{2})/pack('C', hex($1))/seg;
323 sub update_ip_address() {
324 my $ipd = $ipaddresses{$request{"remote_ip"}};
325 if (defined($$ipd{"counter"})) {
326 $$ipd{"counter"} = $$ipd{"counter"} + 1;
329 print STDERR "httpd-guardian: Incrementing counter for " . $request{"remote_ip"} . " (" . $$ipd{"counter"} . ")\n";
334 # check the 1 min counter
335 if ($current_time - $$ipd{"time_1min"} > 60) {
337 my $speed = ($$ipd{"counter"} - $$ipd{"counter_1min"}) / ($current_time - $$ipd{"time_1min"});
338 if ($speed > $THRESHOLD_1MIN) {
339 print STDERR "httpd-guardian: IP address " . $ipaddresses{$request{"remote_ip"}} . " reached the 1 min threshold (speed = $speed req/sec, threshold = $THRESHOLD_1MIN req/sec)\n";
343 # reset the 1 min counter
344 $$ipd{"time_1min"} = $current_time;
345 $$ipd{"counter_1min"} = $$ipd{"counter"};
348 # check the 5 min counter
349 if ($current_time - $$ipd{"time_5min"} > 360) {
351 my $speed = ($$ipd{"counter"} - $$ipd{"counter_5min"}) / ($current_time - $$ipd{"time_5min"});
352 if ($speed > $THRESHOLD_5MIN) {
353 print STDERR "httpd-guardian: IP address " . $request{"remote_ip"} . " reached the 5 min threshold (speed = $speed req/sec, threshold = $THRESHOLD_5MIN req/sec)\n";
357 # reset the 5 min counter
358 $$ipd{"time_5min"} = $current_time;
359 $$ipd{"counter_5min"} = $$ipd{"counter"};
362 if (($exec == 1)&&(defined($PROTECT_EXEC))) {
363 my $cmd = sprintf($PROTECT_EXEC, $request{"remote_ip"});
364 print STDERR "httpd-guardian: Executing: $cmd\n";
369 # start tracking this email address
372 $ipd{"counter_1min"} = 1;
373 $ipd{"time_1min"} = $current_time;
374 $ipd{"counter_5min"} = 1;
375 $ipd{"time_5min"} = $current_time;
376 $ipaddresses{$request{"remote_ip"}} = \%ipd;
380 sub process_log_line {
384 sub remove_stale_data {
385 while(my($key, $value) = each(%ipaddresses)) {
386 if ($current_time - $$value{"time_1min"} > $STALE_INTERVAL) {
388 print STDERR "httpd-guardian: Removing key $key\n";
390 delete($ipaddresses{$key});
396 if (!defined($SAVE_STATE_FILE)) {
400 if (!defined($last_state_save)) {
401 $last_state_save = 0;
404 if ($current_time - $last_state_save > $SAVE_STATE_INTERVAL) {
405 open(FILE, ">$SAVE_STATE_FILE") || die("Failed to save state to $SAVE_STATE_FILE");
406 print FILE "# $current_time\n";
407 print FILE "# IP Address\x09Counter\x09\x091min (time)\x095min (time)\n";
408 while(my($key, $value) = each(%ipaddresses)) {
409 print FILE ("$key" . "\x09" . $$value{"counter"} . "\x09\x09" . $$value{"counter_1min"} . " (" . $$value{"time_1min"} . ")\x09" . $$value{"counter_5min"} . " (" . $$value{"time_5min"} . ")\n");
412 $last_state_save = $current_time;
416 # load state from $SAVE_STATE_FILE, store the data into $ipaddresses
418 return unless ( defined $SAVE_STATE_FILE );
419 return unless ( -e $SAVE_STATE_FILE && -r $SAVE_STATE_FILE );
420 open my $fd, "<", $SAVE_STATE_FILE
421 or die "cannot open state file for reading : $SAVE_STATE_FILE : $!";
425 #--------------------------------------------------
427 # # IP Address Counter 1min (time) 5min (time)
428 # 211.19.48.12 396 396 (1133599679) 395 (1133599379)
429 #--------------------------------------------------
430 my ($addr, $counter, $time1, $time5) = split /\t+/, $_; # TAB
431 my ($counter_1min, $time_1min) = split /\s+/, $time1;
432 my ($counter_5min, $time_5min) = split /\s+/, $time5;
433 $ipaddresses{$addr} = {
435 counter_1min => $counter_1min,
436 time_1min => chop_brace($time_1min),
437 counter_5min => $counter_5min,
438 time_5min => chop_brace($time_5min),
444 # return strings between braces
451 my $line = shift(@_);
453 if (defined($COPY_LOG)) {
454 print COPY_LOG_FD $line;
458 print STDERR "httpd-guardian: Received: $line";
461 %request = parse_logline($line);
462 if (!defined($request{0})) {
463 # TODO verify IP address is in correct format
465 # extract the time from the log line, to allow the
466 # script to be used for batch processing too
467 $current_time = $request{"time_epoch"};
473 print STDERR "Failed to parse line: " . $line;
477 # -----------------------------------
480 if (defined($COPY_LOG)) {
481 open(COPY_LOG_FD, ">>$COPY_LOG") || die("Failed to open $COPY_LOG for writing");
482 # enable autoflush on the file descriptor
483 $| = 1, select $_ for select COPY_LOG_FD;
490 $args{"spread_name"} = $SPREAD_DAEMON;
491 $args{"private_name"} = $SPREAD_CLIENT_NAME;
493 my($mbox, $privategroup) = Spread::connect(\%args);
494 if (!defined($mbox)) {
495 die "Failed to connect to Spread daemon: $sperrno\n";
498 Spread::join($mbox, $SPREAD_GROUP_NAME);
501 my($st, $s, $g, $mt, $e, $msg);
502 while(($st, $s, $g, $mt, $e, $msg) = Spread::receive($mbox, $SPREAD_TIMEOUT)) {
503 if ((defined($st))&&($st == 2)&&(defined($msg))) {
504 process_line($msg . "\n");
515 if (defined($COPY_LOG)) {