Fixed a small problem with cc extraction.

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9])\x00?[3-9](\x00?[0-9]){3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9]|$)/ &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			local redacted_cc = gsub(ccp, /[0-9]/, redaction_char);

			# we've got a match
			local parts = split_all(data, cc_regex);
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches
					local cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, ccp);

			# Trim the data
			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_redacted_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9])\x00?[3-9](\x00?[0-9]){3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9]|$)/ &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			# we've got a match
			local parts = split_all(data, cc_regex);
			local cc_match = "";
			local redacted_cc = gsub(ccp, /[0-9]/, redaction_char);
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches
					cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, cc_match);

			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : cc_match),
			                   $data=(redact_log ? redacted_data : data)];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}


Create ja3s.bro

Create ja3s.bro

First iteration of JA3S, JA3 for the Server Hello packet.

# This Bro script appends JA3S (JA3 Server) to ssl.log
# Version 1.0 (August 2018)
# This builds a fingerprint for the SSL Server Hello packet based on SSL/TLS version, cipher picked, and extensions used.
# Designed to be used in conjunction with JA3 to fingerprint SSL communication between clients and servers.
#
# Authors: John B. Althouse (jalthouse@salesforce.com) Jeff Atkinson (jatkinson@salesforce.com)
# Copyright (c) 2018, salesforce.com, inc.
# All rights reserved.
# Licensed under the BSD 3-Clause license.
# For full license text, see LICENSE.txt file in the repo root  or https://opensource.org/licenses/BSD-3-Clause
#

module JA3_Server;

export {
redef enum Log::ID += { LOG };
}

type JA3Sstorage: record {
       server_version:      count &default=0 &log;
       server_cipher:      count &default=0 &log;
       server_extensions:   string &default="" &log;
};

redef record connection += {
       ja3sfp: JA3Sstorage &optional;
};

redef record SSL::Info += {
  ja3s:            string &optional &log;
# LOG FIELD VALUES #
#  ja3s_version:  string &optional &log;
#  ja3s_cipher:  string &optional &log;
#  ja3s_extensions: string &optional &log;
};

const sep = "-";
event bro_init() {
    Log::create_stream(JA3_Server::LOG,[$columns=JA3Sstorage, $path="ja3sfp"]);
}

event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
{
if ( ! c?$ja3sfp )
    c$ja3sfp=JA3Sstorage();
    if ( is_orig == F ) {
        if ( c$ja3sfp$server_extensions == "" ) {
            c$ja3sfp$server_extensions = cat(code);
        }
        else {
            c$ja3sfp$server_extensions = string_cat(c$ja3sfp$server_extensions, sep,cat(code));
        }
    }
}

event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=1
{
    if ( !c?$ja3sfp )
    c$ja3sfp=JA3Sstorage();
    c$ja3sfp$server_version = version;
    c$ja3sfp$server_cipher = cipher;
    local sep2 = ",";
    local ja3s_string = string_cat(cat(c$ja3sfp$server_version),sep2,cat(c$ja3sfp$server_cipher),sep2,c$ja3sfp$server_extensions);
    local ja3sfp_1 = md5_hash(ja3s_string);
    c$ssl$ja3s = ja3sfp_1;

# LOG FIELD VALUES #
#c$ssl$ja3s_version = cat(c$ja3sfp$server_version);
#c$ssl$ja3s_cipher = cat(c$ja3sfp$server_cipher);
#c$ssl$ja3s_extensions = c$ja3sfp$server_extensions;
#
# FOR DEBUGGING #
#print "JA3S: "+ja3sfp_1+" Fingerprint String: "+ja3s_string;

}

error: pathspec 'bro/ja3s.bro' did not match any file(s) known to git


Added another analyzer for identifying file types of http transfers.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

module HTTP;

export {
	const http_magic_log = open_log_file("http-identified-files") &raw_output &redef;

	# Base the libmagic analysis on at least this many bytes.
	const magic_content_limit = 1024 &redef;

	const watched_mime_types: set[string] = {
		"application/x-dosexec",      # Windows and DOS executables
		"application/x-executable",   # *NIX executable binary
	} &redef;

	const watched_descriptions =
		/PHP script text/ &redef;

	# URLs included here are not logged and notices are not thrown.
	# Take care when defining regexes to not be overly broad.
	const ignored_urls = /^http:\/\/www\.download\.windowsupdate\.com\// &redef;

	redef enum Notice += {
		# This notice is thrown when the file extension doesn't match the file contents
		HTTP_IncorrectFileType,
	};

	# Create regexes that *should* in be in the urls for specifics mime types.
	# Notices are thrown if the pattern doesn't match the url for the file type.
	const mime_types_extensions: table[string] of pattern = {
		["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/,
	} &redef;
}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( is_orig ) # We are only watching for server responses
		return;

	local s = lookup_http_request_stream(c);
	local msg = get_http_message(s, is_orig);

@ifndef	( content_truncation_limit )
	# This is only done if http-body.bro is not loaded.
	msg$data_length = msg$data_length + length;
@endif

	# For the time being, we'll just use the data from the first packet.
	# Don't continue until we have enough data
	#if ( msg$data_length < magic_content_limit )
	#	return;

	# Right now, only try this for the first chunk of data
	if ( msg$data_length > length )
		return;

	local abstract = sub_bytes(data, 1, magic_content_limit);
	local magic_mime = identify_data(abstract, T);
	local magic_descr = identify_data(abstract, F);

	if ( (magic_mime in watched_mime_types ||
	      watched_descriptions in magic_descr) &&
	     s$first_pending_request in s$requests )
		{
		local r = s$requests[s$first_pending_request];
		local host = (s$next_request$host=="") ? fmt("%s", c$id$resp_h) : s$next_request$host;
		local url = fmt("http://%s%s", host, r$URI);

		event file_transferred(c, abstract, magic_descr, magic_mime);

		if ( ignored_urls in url )
			return;

		local file_type: string = "";
		if ( magic_mime in watched_mime_types )
			file_type = magic_mime;
		else
			file_type = magic_descr;

		print http_magic_log, cat_sep("\t", "\\N", network_time(), s$id, c$id$orig_h, fmt("%d", c$id$orig_p), c$id$resp_h, fmt("%d", c$id$resp_p), file_type, r$method, url);

		if ( (magic_mime in mime_types_extensions && mime_types_extensions[magic_mime] !in url) ||
		     (magic_descr in mime_types_extensions && mime_types_extensions[magic_descr] !in url) )
			{
			local message = fmt("%s %s %s", file_type, r$method, url);
			NOTICE([$note=HTTP_IncorrectFileType,
			        $msg=message,
			        $conn=c,
			        $method=r$method,
			        $URL=url]);
			}
		}
	}

error: pathspec 'http-identified-files.bro' did not match any file(s) known to git


stats get written once per worker.. need to fix

for now, just don't synchronize

#output something like this
#http_metrics total=343243 inbound=102313 outbound=3423432 exe_download=23

@load global-ext
@load http-ext

export {
    global http_metrics: table[string] of count &default=0; #&synchronized;
    global http_metrics_interval = +60sec;
    const http_metrics_log = open_log_file("http-ext-metrics");
}

event http_write_stats()
    {
    if (http_metrics["total"]!=0)
        {
        print http_metrics_log, fmt("http_metrics time=%.6f total=%d inbound=%d outbound=%d video_download=%d youtube_watches=%d",
            network_time(),
            http_metrics["total"],
            http_metrics["inbound"],
            http_metrics["outbound"],
            http_metrics["video_download"],
            http_metrics["youtube_watches"]
            );
        clear_table(http_metrics);
        }
    schedule http_metrics_interval { http_write_stats() };
    }

event bro_init()
    {
    set_buf(http_metrics_log, F);
    schedule http_metrics_interval { http_write_stats() };
    }

event http_ext(id: conn_id, si: http_ext_session_info) &priority=-10
    {
    ++http_metrics["total"];
    if(is_local_addr(id$orig_h))
        ++http_metrics["outbound"];
    else
        ++http_metrics["inbound"];

    if (/\.(avi|flv|mp4|mpg)/ in si$uri)
        ++http_metrics["video_download"];
    if (/watch\?v=/ in si$uri && /youtube/ in si$host)
        ++http_metrics["youtube_watches"];

    }

#output something like this
#http_metrics total=343243 inbound=102313 outbound=3423432 exe_download=23

@load global-ext
@load http-ext

export {
    global http_metrics: table[string] of count &default=0 &synchronized;
    global http_metrics_interval = +60sec;
    const http_metrics_log = open_log_file("http-ext-metrics");
}

event http_write_stats()
    {
    if (http_metrics["total"]!=0)
        {
        print http_metrics_log, fmt("http_metrics time=%.6f total=%d inbound=%d outbound=%d video_download=%d youtube_watches=%d",
            network_time(),
            http_metrics["total"],
            http_metrics["inbound"],
            http_metrics["outbound"],
            http_metrics["video_download"],
            http_metrics["youtube_watches"]
            );
        clear_table(http_metrics);
        }
    schedule http_metrics_interval { http_write_stats() };
    }

event bro_init()
    {
    set_buf(http_metrics_log, F);
    schedule http_metrics_interval { http_write_stats() };
    }

event http_ext(id: conn_id, si: http_ext_session_info) &priority=-10
    {
    ++http_metrics["total"];
    if(is_local_addr(id$orig_h))
        ++http_metrics["outbound"];
    else
        ++http_metrics["inbound"];

    if (/\.(avi|flv|mp4|mpg)/ in si$uri)
        ++http_metrics["video_download"];
    if (/watch\?v=/ in si$uri && /youtube/ in si$host)
        ++http_metrics["youtube_watches"];

    }


Create __load__.bro

Create __load__.bro

error: pathspec 'bro-scripts/intel-extensions/__load__.bro' did not match any file(s) known to git


check for presence of mime type first

check for presence of mime type first

@load base/frameworks/metrics

redef enum Metrics::ID += {
    HTTP_MIME_METRICS,
};

event bro_init()
{
    Metrics::add_filter(HTTP_MIME_METRICS,
                [$name="all",
                 $break_interval=3600secs
                ]);
}

event HTTP::log_http(rec: HTTP::Info)
{
    if(Site::is_local_addr(rec$id$resp_h) && rec?$mime_type) {
        Metrics::add_data(HTTP_MIME_METRICS, [$str=rec$mime_type], rec$response_body_len);
    }
}

@load base/frameworks/metrics

redef enum Metrics::ID += {
    HTTP_MIME_METRICS,
};

event bro_init()
{
    Metrics::add_filter(HTTP_MIME_METRICS,
                [$name="all",
                 $break_interval=3600secs
                ]);
}

event HTTP::log_http(rec: HTTP::Info)
{
    if(Site::is_local_addr(rec$id$resp_h)) {
        Metrics::add_data(HTTP_MIME_METRICS, [$str=rec$mime_type], rec$response_body_len);
    }
}


don't block the same address more than once at the same time

function notice_exec_ipblocker(n: notice_info, a: NoticeAction): NoticeAction
{
    local cmd = fmt("lckdo /tmp/bro_ipblocker_%s /usr/local/bin/bro_ipblocker_block", n$id$orig_h);
    execute_with_notice(cmd, n);
    return NOTICE_ALARM_ALWAYS;
}

function notice_exec_ipblocker(n: notice_info, a: NoticeAction): NoticeAction
{
    execute_with_notice("/usr/local/bin/bro_ipblocker_block", n);
    return NOTICE_ALARM_ALWAYS;
}


Fix from Vern.

Fix from Vern.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}


Fixed an issue with logging of user-agent headers.  Thanks Justin. Also, *actually* fixed the indentation issue from last commit.

@load global-ext
@load http-request
@load http-entity

type http_ext_session_info: record {
	start_time: time;
	method: string &default="";
	host: string &default="";
	uri: string &default="";
	url: string &default="";
	referrer: string &default="";
	user_agent: string &default="";
	proxied_for: string &default="";

	force_log: bool &default=F; # This will force the request to be logged (if doing any logging)
	force_log_client_body: bool &default=F; # This will force the client body to be logged.
	force_log_reasons: set[string]; # Reasons why the forced logging happened.

	# This is internal state tracking.
	full: bool &default=F;
	new_user_agent: bool &default=F;
};

function default_http_ext_session_info(): http_ext_session_info
	{
	local x: http_ext_session_info;
	local tmp: set[string] = set();
	x$start_time=network_time();
	x$force_log_reasons=tmp;
	return x;
	}

type http_ext_activity_count: record {
	sql_injections: track_count;
	sql_injection_probes: track_count;
	suspicious_posts: track_count;
};

function default_http_ext_activity_count(a:addr):http_ext_activity_count
	{
	local x: http_ext_activity_count;
	return x;
	}

# Define the generic http_ext events that can be handled from other scripts
global http_ext: event(id: conn_id, si: http_ext_session_info);

module HTTP;

# Uncomment the following lines if you have these data sets available and would
# like to use them.
#@load malware_com_br_block_list-data
#@load zeus-data
#@load malwaredomainlist-data

export {
	redef enum Notice += {
		HTTP_Suspicious,
		HTTP_SQL_Injection_Attack,
		HTTP_SQL_Injection_Heavy_Probing,

		HTTP_Malware_com_br_Block_List,
		HTTP_Zeus_Communication,
		HTTP_MalwareDomainList_Communication,
	};

	# This is the regular expression that is used to match URI based SQL injections
	const sql_injection_regex =
		  /[\?&][^[:blank:]\|]+?=[\-0-9%]+([[:blank:]]|\/\*.*?\*\/)*['"]?([[:blank:]]|\/\*.*?\*\/|\)?;)+([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])[^a-zA-Z&]/
		| /[\?&][^[:blank:]\|]+?=[\-0-9%]+([[:blank:]]|\/\*.*?\*\/)*['"]?([[:blank:]]|\/\*.*?\*\/|\)?;)+([oO][rR]|[aA][nN][dD])([[:blank:]]|\/\*.*?\*\/)+['"]?[^a-zA-Z&]+?=/
		| /[\?&][^[:blank:]]+?=[\-0-9%]*([[:blank:]]|\/\*.*?\*\/)*['"]([[:blank:]]|\/\*.*?\*\/)*(\-|\+|\|\|)([[:blank:]]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/
		| /[\?&][^[:blank:]\|]+?=([[:blank:]]|\/\*.*?\*\/)*['"]([[:blank:]]|\/\*.*?\*\/|;)*([oO][rR]|[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT]|\()[^a-zA-Z&]/
		| /[\?&][^[:blank:]]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/ &redef;

	# SQL injection probes are considered to be:
	#  1. A single single-quote at the end of a URL with no other single-quotes.
	#  2. URLs with one single quote at the end of a normal GET value.
	const sql_injection_probe_regex =
		  /^[^\']+\'$/
		| /^[^\']+\'&[^\']*$/ &redef;

	# Define which hosts user-agents you'd like to track.
	const track_user_agents_for = LocalHosts &redef;

	# If there is something creating large number of strange user-agent,
	# you can filter those out with this pattern.
	const ignored_user_agents = /DONT_MATCH_ANYTHING/ &redef;

	# HTTP post data contents that appear suspicious.
	#  This is usually spammy forum postings and the like.
	const suspicious_http_posts =
		/[vV][iI][aA][gG][rR][aA]/ |
		/[tT][rR][aA][mM][aA][dD][oO][lL]/ |
		/[cC][iI][aA][lL][iI][sS]/ |
		/[sS][oO][mM][aA]/ |
		/[hH][yY][dD][rR][oO][cC][oO][dD][oO][nN][eE]/ |
		/[cC][aA][nN][aA][dD][iI][aA][nN].{0,15}[pP][hH][aA][rR][mM][aA][cC][yY]/ |
		/[rR][iI][nN][gG].?[tT][oO][nN][eE]/ |
		/[pP][eE][nN][iI][sS]/ |
		/[oO][nN][lL][iI][nN][eE].?[cC][aA][sS][iI][nN][oO]/ |
		/[rR][eE][mM][oO][rR][tT][gG][aA][gG][eE][sS]/ |
		# more than 4 bbcode style links in a POST is deemed suspicious
		/(url=http:.*){4}/ |
		# more that 4 html links is also suspicious
		/(a.href(=|%3[dD]).*){4}/ &redef;

	const http_forwarded_headers = {
		"HTTP_FORWARDED",
		"FORWARDED",
		"HTTP_X_FORWARDED_FOR",
		"X_FORWARDED_FOR",
		"HTTP_X_FORWARDED_FROM",
		"X_FORWARDED_FROM",
		"HTTP_CLIENT_IP",
		"CLIENT_IP",
		"HTTP_FROM",
		"FROM",
		"HTTP_VIA",
		"VIA",
		"HTTP_XROXY_CONNECTION",
		"XROXY_CONNECTION",
		"HTTP_PROXY_CONNECTION",
		"PROXY_CONNECTION",
	} &redef;

	global conn_info: table[conn_id] of http_ext_session_info
		&read_expire=5mins
		&redef;

	global activity_counters: table[addr] of http_ext_activity_count
		&create_expire=1day
		&synchronized
		&default=default_http_ext_activity_count
		&redef;

	# You can inspect this during runtime from other modules to see what
	# user-agents a host has used.
	global known_user_agents: table[addr] of set[string]
		&create_expire=3hrs
		&synchronized
		&default=addr_empty_string_set
		&redef;
}

# This is called from signatures (theoretically)
function log_post(state: signature_state, data: string): bool
	{
	# Log the post data when it becomes available.
	if ( state$conn$id in conn_info )
		{
		conn_info[state$conn$id]$force_log_client_body = T;
		add conn_info[state$conn$id]$force_log_reasons[fmt("matched_signature_%s",state$id)];
		}

	# We'll always allow the signature to fire
	return T;
	}

event http_request(c: connection, method: string, original_URI: string,
	               unescaped_URI: string, version: string)
	{
	if ( c$id !in conn_info )
		{
		local x = default_http_ext_session_info();
		conn_info[c$id] = x;
		}

	local sess_ext = conn_info[c$id];
	sess_ext$method = method;
	sess_ext$uri = unescaped_URI;
	}

event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=5
	{
	if ( !is_orig )
		return;

	local id = c$id;
	if ( id !in conn_info )
		return;

	local sess_ext = conn_info[id];
	sess_ext$url = fmt("http://%s%s", sess_ext$host, sess_ext$uri);

	# Detect and log SQL injection attempts in their own log file
	if ( sql_injection_regex in sess_ext$uri )
		{
		sess_ext$force_log=T;
		add sess_ext$force_log_reasons["sql_injection"];

		++(activity_counters[id$orig_h]$sql_injections$n);

		if ( default_check_threshold(activity_counters[id$orig_h]$sql_injections) )
			{
			NOTICE([$note=HTTP_SQL_Injection_Attack,
			        $msg=fmt("SQL injection attack (n=%d): %s -> %s",
			                 activity_counters[id$orig_h]$sql_injections$n,
			                 numeric_id_string(id), sess_ext$url),
			        $conn=c,
			        $n=activity_counters[id$orig_h]$sql_injections$n]);
			}
		}
	if ( sql_injection_probe_regex in sess_ext$uri )
		{
		sess_ext$force_log=T;
		add sess_ext$force_log_reasons["sql_injection_probe"];
		++(activity_counters[id$orig_h]$sql_injection_probes$n);

		if ( default_check_threshold(activity_counters[c$id$orig_h]$sql_injection_probes) )
			{
			NOTICE([$note=HTTP_SQL_Injection_Heavy_Probing,
			        $msg=fmt("Heavy probing from %s", id$orig_h),
			        $n=activity_counters[c$id$orig_h]$sql_injection_probes$n,
			        $conn=c]);
			}
		}

@ifdef ( MalwareComBr_BlockList )
	if ( MalwareComBr_BlockList in sess_ext$url )
		{
		sess_ext$force_log=T;
		sess_ext$force_log_client_body=T;
		add sess_ext$force_log_reasons["malware_com_br"];
		local malware_com_br_msg = fmt("%s %s %s", sess_ext$method, sess_ext$url, sess_ext$referrer);
		NOTICE([$note=HTTP_Malware_com_br_Block_List, $msg=malware_com_br_msg, $conn=c]);
		}
@endif

@ifdef ( ZeusDomains )
	if ( sess_ext$host in ZeusDomains )
		{
		sess_ext$force_log=T;
		sess_ext$force_log_client_body=T;
		add sess_ext$force_log_reasons["zeustracker"];
		local zeus_msg = fmt("%s communicated with likely Zeus controller at %s", c$id$orig_h, sess_ext$host);
		NOTICE([$note=HTTP_Zeus_Communication, $msg=zeus_msg, $sub=sess_ext$url, $conn=c]);
		}
@endif

@ifdef ( MalwareDomainList )
	if ( sess_ext$url in MalwareDomainList )
		{
		sess_ext$force_log=T;
		sess_ext$force_log_client_body=T;
		add sess_ext$force_log_reasons["malwaredomainlist"];
		local mdl_msg = fmt("%s communicated with malwaredomainlist.com URL at %s", c$id$orig_h, sess_ext$url);
		NOTICE([$note=HTTP_MalwareDomainList_Communication, $msg=mdl_msg, $sub=MalwareDomainList[sess_ext$url], $conn=c]);
		}
@endif

	event http_ext(id, sess_ext);

	# No data from the reply is supported yet, so it's ok to delete here.
	delete conn_info[c$id];
	}

event http_header(c: connection, is_orig: bool, name: string, value: string)
	{
	if ( !is_orig ) return;

	if ( c$id !in conn_info )
		conn_info[c$id] = default_http_ext_session_info();

	local ci = conn_info[c$id];

	if ( name == "REFERER" )
		ci$referrer = value;

	else if ( name == "HOST" )
		ci$host = value;

	else if ( name == "USER-AGENT" )
		{
		ci$user_agent = value;

		if ( ignored_user_agents in value )
			return;

		if ( addr_matches_hosts(c$id$orig_h, track_user_agents_for) &&
			 value !in known_user_agents[c$id$orig_h] )
			{
			if ( c$id$orig_h !in known_user_agents )
				known_user_agents[c$id$orig_h] = set();
			add known_user_agents[c$id$orig_h][value];
			ci$new_user_agent = T;
			}
		}

	else if ( name in http_forwarded_headers )
		{
		if ( ci$proxied_for == "" )
			ci$proxied_for = fmt("(%s::%s)", name, value);
		else
			ci$proxied_for = fmt("%s, (%s::%s)", ci$proxied_for, name, value);
		}
	}

@load global-ext
@load http-request
@load http-entity

type http_ext_session_info: record {
	start_time: time;
	method: string &default="";
	host: string &default="";
	uri: string &default="";
	url: string &default="";
	referrer: string &default="";
	user_agent: string &default="";
	proxied_for: string &default="";

	force_log: bool &default=F; # This will force the request to be logged (if doing any logging)
	force_log_client_body: bool &default=F; # This will force the client body to be logged.
	force_log_reasons: set[string]; # Reasons why the forced logging happened.

	# This is internal state tracking.
	full: bool &default=F;
	new_user_agent: bool &default=F;
};

function default_http_ext_session_info(): http_ext_session_info
	{
	local x: http_ext_session_info;
	local tmp: set[string] = set();
	x$start_time=network_time();
	x$force_log_reasons=tmp;
	return x;
	}

type http_ext_activity_count: record {
	sql_injections: track_count;
	sql_injection_probes: track_count;
	suspicious_posts: track_count;
};

function default_http_ext_activity_count(a:addr):http_ext_activity_count
	{
	local x: http_ext_activity_count;
	return x;
	}

# Define the generic http_ext events that can be handled from other scripts
global http_ext: event(id: conn_id, si: http_ext_session_info);

module HTTP;

# Uncomment the following lines if you have these data sets available and would
# like to use them.
#@load malware_com_br_block_list-data
#@load zeus-data
#@load malwaredomainlist-data

export {
	redef enum Notice += {
		HTTP_Suspicious,
		HTTP_SQL_Injection_Attack,
		HTTP_SQL_Injection_Heavy_Probing,

		HTTP_Malware_com_br_Block_List,
		HTTP_Zeus_Communication,
		HTTP_MalwareDomainList_Communication,
	};

	# This is the regular expression that is used to match URI based SQL injections
	const sql_injection_regex =
		  /[\?&][^[:blank:]\|]+?=[\-0-9%]+([[:blank:]]|\/\*.*?\*\/)*['"]?([[:blank:]]|\/\*.*?\*\/|\)?;)+([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])[^a-zA-Z&]/
		| /[\?&][^[:blank:]\|]+?=[\-0-9%]+([[:blank:]]|\/\*.*?\*\/)*['"]?([[:blank:]]|\/\*.*?\*\/|\)?;)+([oO][rR]|[aA][nN][dD])([[:blank:]]|\/\*.*?\*\/)+['"]?[^a-zA-Z&]+?=/
		| /[\?&][^[:blank:]]+?=[\-0-9%]*([[:blank:]]|\/\*.*?\*\/)*['"]([[:blank:]]|\/\*.*?\*\/)*(\-|\+|\|\|)([[:blank:]]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/
		| /[\?&][^[:blank:]\|]+?=([[:blank:]]|\/\*.*?\*\/)*['"]([[:blank:]]|\/\*.*?\*\/|;)*([oO][rR]|[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT]|\()[^a-zA-Z&]/
		| /[\?&][^[:blank:]]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/ &redef;

	# SQL injection probes are considered to be:
	#  1. A single single-quote at the end of a URL with no other single-quotes.
	#  2. URLs with one single quote at the end of a normal GET value.
	const sql_injection_probe_regex =
		  /^[^\']+\'$/
		| /^[^\']+\'&[^\']*$/ &redef;

	# Define which hosts user-agents you'd like to track.
	const track_user_agents_for = LocalHosts &redef;

	# If there is something creating large number of strange user-agent,
	# you can filter those out with this pattern.
	const ignored_user_agents = /DONT_MATCH_ANYTHING/ &redef;

	# HTTP post data contents that appear suspicious.
	#  This is usually spammy forum postings and the like.
	const suspicious_http_posts =
		/[vV][iI][aA][gG][rR][aA]/ |
		/[tT][rR][aA][mM][aA][dD][oO][lL]/ |
		/[cC][iI][aA][lL][iI][sS]/ |
		/[sS][oO][mM][aA]/ |
		/[hH][yY][dD][rR][oO][cC][oO][dD][oO][nN][eE]/ |
		/[cC][aA][nN][aA][dD][iI][aA][nN].{0,15}[pP][hH][aA][rR][mM][aA][cC][yY]/ |
		/[rR][iI][nN][gG].?[tT][oO][nN][eE]/ |
		/[pP][eE][nN][iI][sS]/ |
		/[oO][nN][lL][iI][nN][eE].?[cC][aA][sS][iI][nN][oO]/ |
		/[rR][eE][mM][oO][rR][tT][gG][aA][gG][eE][sS]/ |
		# more than 4 bbcode style links in a POST is deemed suspicious
		/(url=http:.*){4}/ |
		# more that 4 html links is also suspicious
		/(a.href(=|%3[dD]).*){4}/ &redef;

		const http_forwarded_headers = {
			"HTTP_FORWARDED",
			"FORWARDED",
			"HTTP_X_FORWARDED_FOR",
			"X_FORWARDED_FOR",
			"HTTP_X_FORWARDED_FROM",
			"X_FORWARDED_FROM",
			"HTTP_CLIENT_IP",
			"CLIENT_IP",
			"HTTP_FROM",
			"FROM",
			"HTTP_VIA",
			"VIA",
			"HTTP_XROXY_CONNECTION",
			"XROXY_CONNECTION",
			"HTTP_PROXY_CONNECTION",
			"PROXY_CONNECTION",
		} &redef;

	global conn_info: table[conn_id] of http_ext_session_info
		&read_expire=5mins
		&redef;

	global activity_counters: table[addr] of http_ext_activity_count
		&create_expire=1day
		&synchronized
		&default=default_http_ext_activity_count
		&redef;

	# You can inspect this during runtime from other modules to see what
	# user-agents a host has used.
	global known_user_agents: table[addr] of set[string]
		&create_expire=3hrs
		&synchronized
		&default=addr_empty_string_set
		&redef;
}

# This is called from signatures (theoretically)
function log_post(state: signature_state, data: string): bool
	{
	# Log the post data when it becomes available.
	if ( state$conn$id in conn_info )
		{
		conn_info[state$conn$id]$force_log_client_body = T;
		add conn_info[state$conn$id]$force_log_reasons[fmt("matched_signature_%s",state$id)];
		}

	# We'll always allow the signature to fire
	return T;
	}

event http_request(c: connection, method: string, original_URI: string,
	               unescaped_URI: string, version: string)
	{
	if ( c$id !in conn_info )
		{
		local x = default_http_ext_session_info();
		conn_info[c$id] = x;
		}

	local sess_ext = conn_info[c$id];
	sess_ext$method = method;
	sess_ext$uri = unescaped_URI;
	}

event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=5
	{
	if ( !is_orig )
		return;

	local id = c$id;
	if ( id !in conn_info )
		return;

	local sess_ext = conn_info[id];
	sess_ext$url = fmt("http://%s%s", sess_ext$host, sess_ext$uri);

	# Detect and log SQL injection attempts in their own log file
	if ( sql_injection_regex in sess_ext$uri )
		{
		sess_ext$force_log=T;
		add sess_ext$force_log_reasons["sql_injection"];

		++(activity_counters[id$orig_h]$sql_injections$n);

		if ( default_check_threshold(activity_counters[id$orig_h]$sql_injections) )
			{
			NOTICE([$note=HTTP_SQL_Injection_Attack,
			        $msg=fmt("SQL injection attack (n=%d): %s -> %s",
			                 activity_counters[id$orig_h]$sql_injections$n,
			                 numeric_id_string(id), sess_ext$url),
			        $conn=c,
			        $n=activity_counters[id$orig_h]$sql_injections$n]);
			}
		}
	if ( sql_injection_probe_regex in sess_ext$uri )
		{
		sess_ext$force_log=T;
		add sess_ext$force_log_reasons["sql_injection_probe"];
		++(activity_counters[id$orig_h]$sql_injection_probes$n);

		if ( default_check_threshold(activity_counters[c$id$orig_h]$sql_injection_probes) )
			{
			NOTICE([$note=HTTP_SQL_Injection_Heavy_Probing,
			        $msg=fmt("Heavy probing from %s", id$orig_h),
			        $n=activity_counters[c$id$orig_h]$sql_injection_probes$n,
			        $conn=c]);
			}
		}

@ifdef ( MalwareComBr_BlockList )
	if ( MalwareComBr_BlockList in sess_ext$url )
		{
		sess_ext$force_log=T;
		sess_ext$force_log_client_body=T;
		add sess_ext$force_log_reasons["malware_com_br"];
		local malware_com_br_msg = fmt("%s %s %s", sess_ext$method, sess_ext$url, sess_ext$referrer);
		NOTICE([$note=HTTP_Malware_com_br_Block_List, $msg=malware_com_br_msg, $conn=c]);
		}
@endif

@ifdef ( ZeusDomains )
	if ( sess_ext$host in ZeusDomains )
		{
		sess_ext$force_log=T;
		sess_ext$force_log_client_body=T;
		add sess_ext$force_log_reasons["zeustracker"];
		local zeus_msg = fmt("%s communicated with likely Zeus controller at %s", c$id$orig_h, sess_ext$host);
		NOTICE([$note=HTTP_Zeus_Communication, $msg=zeus_msg, $sub=sess_ext$url, $conn=c]);
		}
@endif

@ifdef ( MalwareDomainList )
	if ( sess_ext$url in MalwareDomainList )
		{
		sess_ext$force_log=T;
		sess_ext$force_log_client_body=T;
		add sess_ext$force_log_reasons["malwaredomainlist"];
		local mdl_msg = fmt("%s communicated with malwaredomainlist.com URL at %s", c$id$orig_h, sess_ext$url);
		NOTICE([$note=HTTP_MalwareDomainList_Communication, $msg=mdl_msg, $sub=MalwareDomainList[sess_ext$url], $conn=c]);
		}
@endif

	event http_ext(id, sess_ext);

	# No data from the reply is supported yet, so it's ok to delete here.
	delete conn_info[c$id];
	}

event http_header(c: connection, is_orig: bool, name: string, value: string)
	{
	if ( !is_orig ) return;

	if ( c$id !in conn_info )
		conn_info[c$id] = default_http_ext_session_info();

	local ci = conn_info[c$id];

	if ( name == "REFERER" )
		ci$referrer = value;

	else if ( name == "HOST" )
		ci$host = value;

	else if ( name == "USER-AGENT" )
		{
		ci$user_agent = value;

		if ( ignored_user_agents in value )
			return;

		if ( addr_matches_hosts(c$id$orig_h, track_user_agents_for) ||
			 value in known_user_agents[c$id$orig_h] )
			{
			if ( c$id$orig_h !in known_user_agents )
				{
				known_user_agents[c$id$orig_h] = set();
				ci$new_user_agent = T;
				}
			add known_user_agents[c$id$orig_h][value];
			}
		}

	else if ( name in http_forwarded_headers )
		{
		if ( ci$proxied_for == "" )
			ci$proxied_for = fmt("(%s::%s)", name, value);
		else
			ci$proxied_for = fmt("%s, (%s::%s)", ci$proxied_for, name, value);
		}
	}


Add Bro script to ingest events.

Add Bro script to ingest events.

The script simply sends all events to VAST.

@load base/frameworks/communication

Communication::nodes["vast"] = [$host = 127.0.0.1,
                                $p = 42000/tcp,
                                $events = /^.*$/,
                                #$retry=10secs,
                                $connect=T];

error: pathspec 'test/bro/ingest.bro' did not match any file(s) known to git


Revert "Removes kafka from main config"

Revert "Removes kafka from main config"

# Copyright (C) 2016, Missouri Cyber Team
# All Rights Reserved
# See the file "LICENSE" in the main distribution directory for details

module ROCK;

export {
  const sensor_id = "sensor001-001" &redef;
}

# Load integration with Snort on ROCK
@load ./frameworks/files/unified2-integration

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Configure Kafka output
# Bro Kafka Output (plugin must be loaded!)
@load Kafka/KafkaWriter/logs-to-kafka
redef KafkaLogger::topic_name = "bro_raw";
redef KafkaLogger::sensor_name = ROCK::sensor_id;

# Add GeoIP info to conn log
@load ./misc/conn-add-geoip

# Add worker information to conn log
@load ./misc/conn-add-worker

# Copyright (C) 2016, Missouri Cyber Team
# All Rights Reserved
# See the file "LICENSE" in the main distribution directory for details

module ROCK;

export {
  const sensor_id = "sensor001-001" &redef;
}

# Load integration with Snort on ROCK
@load ./frameworks/files/unified2-integration

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add GeoIP info to conn log
@load ./misc/conn-add-geoip

# Add worker information to conn log
@load ./misc/conn-add-worker


Added logging extension transform.

Added logging extension transform.

# Copyright (C) 2016, Missouri Cyber Team
# All Rights Reserved
# See the file "LICENSE" in the main distribution directory for details

module ROCK;

export {
  const sensor_id = "sensor001-001" &redef;
}

# Load integration with Snort on ROCK
@load ./frameworks/files/unified2-integration

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add GeoIP info to conn log
@load ./misc/conn-add-geoip

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension


remove RDP namespace, it isn't needed and was breaking signatures

# $Id$
@load notice
global rdp_connection: event(c: connection);

@load signatures
redef signature_files += "rdp.sig";
redef signature_actions += { ["dpd_rdp"] = SIG_IGNORE };

global rdp_ports = {
	3389/tcp
};
redef capture_filters += { ["rdp"] = "tcp and port 3389"};

event signature_match(state: signature_state, msg: string, data: string)
{
    if (state$id == "dpd_rdp"){
		add state$conn$service["RDP"];
        event rdp_connection(state$conn);
    }
}

# $Id$
@load notice
global rdp_connection: event(c: connection);

module RDP;

@load signatures
redef signature_files += "rdp.sig";
redef signature_actions += { ["dpd_rdp"] = SIG_IGNORE };

global rdp_ports = {
	3389/tcp
};
redef capture_filters += { ["rdp"] = "tcp and port 3389"};

event signature_match(state: signature_state, msg: string, data: string)
{
    if (state$id == "dpd_rdp"){
		add state$conn$service["RDP"];
        event rdp_connection(state$conn);
    }
}


notice action filters for sending email to a subnet administrator

@load my-notice-action-filter
global subnet_admins: table[subnet] of string &redef;

function notice_email_subnet_admins(n: notice_info, a: NoticeAction): NoticeAction
{
    local id = n$conn$id;
    local host = is_local_addr(id$orig_h) ? id$orig_h : id$resp_h;
    local admin = "";

    if(host !in subnet_admins)
        admin = mail_dest;
    else
        admin = subnet_admins[host];
    email_notice_to(n, admin);
    event notice_alarm(n, NOTICE_EMAIL);
    return NOTICE_FILE;
}

function notice_email_subnet_admins_then_tally(n: notice_info, a: NoticeAction): NoticeAction
{
    a = notice_email_then_tally(n, a);
    if(a == NOTICE_EMAIL){
        a = notice_email_subnet_admins(n, a);
    }
    return a;
}

error: pathspec 'subnet-admins.bro' did not match any file(s) known to git


alert on more things

alert on more things

@load http-ext
@load ipblocker

module HTTP;

export {
    redef enum Notice += {
        HTTP_IncorrectFileTypeBadHost
    };

    const bad_exec_domains =
        /co\.cc/
      | /cz\.cc/
        &redef;

    const bad_exec_urls =
        /php.adv=/
      | /http:\/\/[0-9]{1,3}\.[0-9]{1,3}.*\/index\.php\?[^=]+=[^=]+$/ #try to match http://1.2.3.4/index.php?foo=bar
        &redef;

    const bad_user_agents =
        /Java\/1/
        &redef;

}

redef notice_action_filters += {
    [HTTP_IncorrectFileTypeBadHost] = notice_exec_ipblocker_dest,
};

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(is_local_addr(id$resp_h))
        return;
    if(! ("identified-files" in si$tags && si$mime_type == "application/x-dosexec"))
        return;

    if( (bad_exec_domains in si$host || bad_exec_urls in si$url)
      ||(/\.exe/ !in si$url && bad_user_agents in si$user_agent)) {
        NOTICE([$note=HTTP_IncorrectFileTypeBadHost,
                $id=id,
                $msg=fmt("EXE Downloaded from bad host %s %s %s", id$orig_h, id$resp_h, si$url),
                $sub="http-ext"
            ]);
        }
}

@load http-ext
@load ipblocker

module HTTP;

export {
    redef enum Notice += {
        HTTP_IncorrectFileTypeBadHost
    };
}

redef notice_action_filters += {
    [HTTP_IncorrectFileTypeBadHost] = notice_exec_ipblocker_dest,
};

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(is_local_addr(id$resp_h))
        return;
    if(! ("identified-files" in si$tags && si$mime_type == "application/x-dosexec"))
        return;

    if(/co.cc/ in si$host) {
        NOTICE([$note=HTTP_IncorrectFileTypeBadHost,
                $id=id,
                $msg=fmt("EXE Downloaded from bad host %s %s %s", id$orig_h, id$resp_h, si$url),
                $sub="http-ext"
            ]);
        }
}


Fix case typo for hassh

Fix case typo for hassh

Fixes #22

# Copyright (C) 2016-2018 RockNSM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module ROCK;

export {
  const sensor_id = gethostname() &redef;
}

#=== Bro built-ins ===================================

# Enable VLAN Logging
@load policy/protocols/conn/vlan-logging

# Log MAC addresses
@load policy/protocols/conn/mac-logging

# Log (All) Client and Server HTTP Headers
@load policy/protocols/http/header-names

#== ROCK specific scripts ============================
# Add empty Intel framework database
@load ./frameworks/intel

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension

# Log all orig and resp cert hashes in ssl log
@load ./protocols/ssl/ssl-add-cert-hash

# Enable pop3 logging
@load ./protocols/pop3

# Notice on all recently created certs
# @load ./protocols/ssl/new-certs

# Generate log of all unique DNS queries with answers
# @load ./protocols/dns/known_domains

# Generate log of all URLs seen in an SMTP body
# @load ./protocols/smtp/smtp-url

# Generate log of local systems using unencrypted protocols
# @load ./frameworks/compliance/detect-insecure-protos

#== 3rd Party Scripts =================================
# Add Salesforce's JA3 SSL fingerprinting
@load ./misc/ja3

# Add Salesforce's HASSH SSH fingerprinting
@load ./misc/hassh

### Sensor specific scripts ######################

# Configure AF_PACKET, if in use
@load ./plugins/afpacket

# Copyright (C) 2016-2018 RockNSM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module ROCK;

export {
  const sensor_id = gethostname() &redef;
}

#=== Bro built-ins ===================================

# Enable VLAN Logging
@load policy/protocols/conn/vlan-logging

# Log MAC addresses
@load policy/protocols/conn/mac-logging

# Log (All) Client and Server HTTP Headers
@load policy/protocols/http/header-names

#== ROCK specific scripts ============================
# Add empty Intel framework database
@load ./frameworks/intel

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension

# Log all orig and resp cert hashes in ssl log
@load ./protocols/ssl/ssl-add-cert-hash

# Enable pop3 logging
@load ./protocols/pop3

# Notice on all recently created certs
# @load ./protocols/ssl/new-certs

# Generate log of all unique DNS queries with answers
# @load ./protocols/dns/known_domains

# Generate log of all URLs seen in an SMTP body
# @load ./protocols/smtp/smtp-url

# Generate log of local systems using unencrypted protocols
# @load ./frameworks/compliance/detect-insecure-protos

#== 3rd Party Scripts =================================
# Add Salesforce's JA3 SSL fingerprinting
@load ./misc/ja3

# Add Salesforce's HASSH SSH fingerprinting
@load ./misc/HASSH

### Sensor specific scripts ######################

# Configure AF_PACKET, if in use
@load ./plugins/afpacket


Update detect-rogue-dns.bro

Update detect-rogue-dns.bro

Updated year

# Detects a Hurricane Panda tactic of using Hurricane Electric to resolve commonly accessed websites
# Alerts when a domain in the Alexa top 500 is resolved via Hurricane Electric and/or when a host connects to an IP in the DNS response
# CrowdStrike 2015
# josh.liburdi@crowdstrike.com

@load base/protocols/dns
@load base/frameworks/notice
@load base/frameworks/input

module CrowdStrike::Hurricane_Panda;

export {
  redef enum Notice::Type += {
    HE_Request,
    HE_Successful_Conn
  };
}

# File generated by scrape-alexa.py
const alexa_file = "./alexa_domains.txt" &redef;
# Table to store list of domains in file above
global alexa_table: set[string];
# Record for domains in file above
type alexa_idx: record {
  alexa: string;
};

const he_nets: set[subnet] = {
  216.218.130.2,
  216.66.1.2,
  216.66.80.18,
  216.218.132.2,
  216.218.131.2
} &redef;

# Pattern used to identify subdomains
const subdomains =  /^d?ns[0-9]*\./    |
                    /^smtp[0-9]*\./    |
                    /^mail[0-9]*\./    |
                    /^pop[0-9]*\./     |
                    /^imap[0-9]*\./    |
                    /^www[0-9]*\./     |
                    /^ftp[0-9]*\./     |
                    /^img[0-9]*\./     |
                    /^images?[0-9]*\./ |
                    /^search[0-9]*\./  |
                    /^nginx[0-9]*\./ &redef;

# Container to store IP address answers from Hurricane Electric queries
# Each entry expires 5 minutes after the last time it was written
global he_answers: set[addr] &write_expire=5min;

# Pulls data from scrape-alexa.py output (alexa_file)
event bro_init()
{
Input::add_table([$source=alexa_file,
                  $name="alexa_domains",
                  $idx=alexa_idx,
                  $destination=alexa_table,
                  $mode=Input::REREAD]);
}

event DNS::log_dns(rec: DNS::Info)
{
# Do not process the event if no query exists or if the query is not being resolved by Hurricane Electric
if ( ! rec?$query ) return;
if ( rec$id$resp_h !in he_nets ) return;

# If necessary, clean the query so that it can be found in the list of Alexa domains
local query = rec$query;
if ( subdomains in query )
  query = sub(rec$query,subdomains,"");

# Check if the query is in the list of Alexa domains
# If it is, then this activity is suspicious and should be investigated
if ( query in alexa_table )
  {
  # Prepare the sub-message for the notice
  # Include the domain queried in the sub-message
  local sub_msg = fmt("Query: %s",query);

  # If the query was answered, include the answers in the sub-message
  if ( rec?$answers )
    {
    sub_msg += fmt(" %s: ", rec$total_answers > 1 ? "Answers":"Answer");

    for ( ans in rec$answers )
      {
      # If an answers value is an IP address, store it for later processing
      if ( is_valid_ip(rec$answers[ans]) == T )
        add he_answers[to_addr(rec$answers[ans])];
      sub_msg += fmt("%s, ", rec$answers[ans]);
      }

    # Clean the sub-message
    sub_msg = cut_tail(sub_msg,2);
    }

  # Generate the notice
  # Includes the connection flow, host intiating the lookup, domain queried, and query answers (if available)
  NOTICE([$note=HE_Request,
          $msg=fmt("%s made a suspicious DNS lookup to Hurricane Electric", rec$id$orig_h),
          $sub=sub_msg,
          $id=rec$id,
          $uid=rec$uid,
          $identifier=cat(rec$id$orig_h,rec$query)]);
  }
}

event connection_state_remove(c: connection)
{
# Check if a host connected to an IP address seen in an answer from Hurricane Electric
if ( c$id$resp_h in he_answers )
  NOTICE([$note=HE_Successful_Conn,
          $conn=c,
          $msg=fmt("%s connected to a suspicious server resolved by Hurricane Electric", c$id$orig_h),
          $identifier=cat(c$id$orig_h,c$id$resp_h)]);
}

# Detects a Hurricane Panda tactic of using Hurricane Electric to resolve commonly accessed websites
# Alerts when a domain in the Alexa top 500 is resolved via Hurricane Electric and/or when a host connects to an IP in the DNS response
# CrowdStrike 2014
# josh.liburdi@crowdstrike.com

@load base/protocols/dns
@load base/frameworks/notice
@load base/frameworks/input

module CrowdStrike::Hurricane_Panda;

export {
  redef enum Notice::Type += {
    HE_Request,
    HE_Successful_Conn
  };
}

# File generated by scrape-alexa.py
const alexa_file = "./alexa_domains.txt" &redef;
# Table to store list of domains in file above
global alexa_table: set[string];
# Record for domains in file above
type alexa_idx: record {
  alexa: string;
};

const he_nets: set[subnet] = {
  216.218.130.2,
  216.66.1.2,
  216.66.80.18,
  216.218.132.2,
  216.218.131.2
} &redef;

# Pattern used to identify subdomains
const subdomains =  /^d?ns[0-9]*\./    |
                    /^smtp[0-9]*\./    |
                    /^mail[0-9]*\./    |
                    /^pop[0-9]*\./     |
                    /^imap[0-9]*\./    |
                    /^www[0-9]*\./     |
                    /^ftp[0-9]*\./     |
                    /^img[0-9]*\./     |
                    /^images?[0-9]*\./ |
                    /^search[0-9]*\./  |
                    /^nginx[0-9]*\./ &redef;

# Container to store IP address answers from Hurricane Electric queries
# Each entry expires 5 minutes after the last time it was written
global he_answers: set[addr] &write_expire=5min;

# Pulls data from scrape-alexa.py output (alexa_file)
event bro_init()
{
Input::add_table([$source=alexa_file,
                  $name="alexa_domains",
                  $idx=alexa_idx,
                  $destination=alexa_table,
                  $mode=Input::REREAD]);
}

event DNS::log_dns(rec: DNS::Info)
{
# Do not process the event if no query exists or if the query is not being resolved by Hurricane Electric
if ( ! rec?$query ) return;
if ( rec$id$resp_h !in he_nets ) return;

# If necessary, clean the query so that it can be found in the list of Alexa domains
local query = rec$query;
if ( subdomains in query )
  query = sub(rec$query,subdomains,"");

# Check if the query is in the list of Alexa domains
# If it is, then this activity is suspicious and should be investigated
if ( query in alexa_table )
  {
  # Prepare the sub-message for the notice
  # Include the domain queried in the sub-message
  local sub_msg = fmt("Query: %s",query);

  # If the query was answered, include the answers in the sub-message
  if ( rec?$answers )
    {
    sub_msg += fmt(" %s: ", rec$total_answers > 1 ? "Answers":"Answer");

    for ( ans in rec$answers )
      {
      # If an answers value is an IP address, store it for later processing
      if ( is_valid_ip(rec$answers[ans]) == T )
        add he_answers[to_addr(rec$answers[ans])];
      sub_msg += fmt("%s, ", rec$answers[ans]);
      }

    # Clean the sub-message
    sub_msg = cut_tail(sub_msg,2);
    }

  # Generate the notice
  # Includes the connection flow, host intiating the lookup, domain queried, and query answers (if available)
  NOTICE([$note=HE_Request,
          $msg=fmt("%s made a suspicious DNS lookup to Hurricane Electric", rec$id$orig_h),
          $sub=sub_msg,
          $id=rec$id,
          $uid=rec$uid,
          $identifier=cat(rec$id$orig_h,rec$query)]);
  }
}

event connection_state_remove(c: connection)
{
# Check if a host connected to an IP address seen in an answer from Hurricane Electric
if ( c$id$resp_h in he_answers )
  NOTICE([$note=HE_Successful_Conn,
          $conn=c,
          $msg=fmt("%s connected to a suspicious server resolved by Hurricane Electric", c$id$orig_h),
          $identifier=cat(c$id$orig_h,c$id$resp_h)]);
}


removing a print out

removing a print out

Former-commit-id: c5c329659507683fca545726be63726edfe8953d

##! Extract exe, pdf, jar and cab files.

global ext_map: table[string] of string = {
    ["application/x-dosexec"] = "exe",
    ["application/pdf"] = "pdf",
    ["application/zip"] = "zip",
    ["application/jar"] = "jar",
    ["application/vnd.ms-cab-compressed"] = "cab",
    ["text/plain"] = "txt",
    ["image/gif"] = "gif",
    ["image/jpeg"] = "jpg",
    ["image/png"] = "png",
    ["text/html"] = "html",
    ["application/vnd.ms-fontobject"] = "ms_font",
    ["application/x-shockwave-flash"] = "swf"
} &default ="unknown";

event file_new(f: fa_file)
    {
    local ext = ext_map[f$mime_type];
    if (ext != "txt" && ext != "html" && ext != "ms_font" && ext != "jpg" && ext != "gif" && ext != "png")
        {
        local fname = fmt("%s-%s.%s", f$source, f$id, ext);
        Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
        }
    }

##! Extract exe, pdf, jar and cab files.

global ext_map: table[string] of string = {
    ["application/x-dosexec"] = "exe",
    ["application/pdf"] = "pdf",
    ["application/zip"] = "zip",
    ["application/jar"] = "jar",
    ["application/vnd.ms-cab-compressed"] = "cab",
    ["text/plain"] = "txt",
    ["image/gif"] = "gif",
    ["image/jpeg"] = "jpg",
    ["image/png"] = "png",
    ["text/html"] = "html",
    ["application/vnd.ms-fontobject"] = "ms_font",
    ["application/x-shockwave-flash"] = "swf"
} &default ="unknown";

event file_new(f: fa_file)
    {
    local ext = ext_map[f$mime_type];
    print f$mime_type, ext;
    if (ext != "txt" && ext != "html" && ext != "ms_font" && ext != "jpg" && ext != "gif" && ext != "png")
        {
        local fname = fmt("%s-%s.%s", f$source, f$id, ext);
        Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
        }
    }


watched_countries is defined under the SSH module

@load global-ext
@load ssh
@load notice

module SSH;

export {
	const ssh_ext_log = open_log_file("ssh-ext") &raw_output;

	const password_guesses_limit = 30 &redef;
	const authentication_data_size = 5500 &redef;
	const guessing_timeout = 30 mins;

	# Keeps count of how many rejections a host has had
	global password_rejections: table[addr] of count &default=0 &write_expire=guessing_timeout;
	# Keeps track of hosts identified as guessing passwords
	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr;

	type ssh_versions: record {
		client: string &default="";
		server: string &default="";
	};

	# Only monitor SSH connections for up to 15 minutes
	global active_ssh_conns: table[conn_id] of ssh_versions &create_expire=15mins;

	# If you want to lookup and log geoip data in the event of a failed login.
	const log_geodata_on_failure = F &redef;

	# The set of countries for which you'd like to throw notices upon successful login
	#   requires Bro compiled with libGeoIP support
	const watched_countries: set[string] = {"RO"} &redef;

	# Strange/bad host names to originate successful SSH logins
	const strange_hostnames =
			/^d?ns[0-9]*\./ |
			/^smtp[0-9]*\./ |
			/^mail[0-9]*\./ |
			/^pop[0-9]*\./  |
			/^imap[0-9]*\./ |
			/^www[0-9]*\./  |
			/^ftp[0-9]*\./  &redef;

	# This is a table with orig subnet as the key, and subnet as the value.
	const ignore_guessers: table[subnet] of subnet &redef;

	redef enum Notice += {
		SSH_Login,
		SSH_PasswordGuessing,
		SSH_LoginByPasswordGuesser,
		SSH_Login_From_Strange_Hostname,
		SSH_Bytecount_Inconsistency,
	};
}

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a successful ssh login happens from or to a watched country
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_Login && n$sub in SSH::watched_countries); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a password guesser logs in successfully anywhere
#	# To avoid false positives, setting the lower bound for notification to 50 bad password attempts.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_LoginByPasswordGuesser && n$n > 50); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a local host is password guessing.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_PasswordGuessing &&
#		          is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

# Don't stop processing SSH connections in the default ssh policy script
redef skip_processing_after_handshake = F;

event check_ssh_connection(c: connection, done: bool)
	{
	# If this is no longer a known SSH connection, just return.
	if ( c$id !in active_ssh_conns )
		return;

	# If this is still a live connection and the byte count has not
	# crossed the threshold, just return and let the resheduled check happen later.
	if ( !done && c$resp$size < authentication_data_size )
		return;

	# Make sure the server has sent back more than 50 bytes to filter out
	# hosts that are just port scanning.  Nothing is ever logged if the server
	# doesn't send back at least 50 bytes.
	if (c$resp$size < 50)
		return;

	local versions = active_ssh_conns[c$id];
	local status = "failure";
	local direction = is_local_addr(c$id$orig_h) ? "to" : "from";
	local location: geo_location;
	# Need to give these values defaults in bro.init.
	location$country_code=""; location$region=""; location$city=""; location$latitude=0.0; location$longitude=0.0;

	if ( done && c$resp$size < authentication_data_size )
		{
		# presumed failure

		# Track the number of rejections
		if ( !(c$id$orig_h in ignore_guessers &&
		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
			password_rejections[c$id$orig_h] += 1;

		if ( password_rejections[c$id$orig_h] > password_guesses_limit &&
		     c$id$orig_h !in password_guessers )
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_PasswordGuessing,
			        $conn=c,
			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]),
			        $n=password_rejections[c$id$orig_h]]);
			}
		}
	# TODO: This is to work around a quasi-bug in Bro which occasionally
	#       causes the byte count to be oversized.
	else if (c$resp$size < 20000000)
		{
		# presumed successful login
		status = "success";

		if ( password_rejections[c$id$orig_h] > password_guesses_limit &&
		     c$id$orig_h !in password_guessers)
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_LoginByPasswordGuesser,
			        $conn=c,
			        $n=password_rejections[c$id$orig_h],
			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h])]);
			}

		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
		              direction, location$country_code, location$region, location$city,
		              location$latitude, location$longitude,
		              numeric_id_string(c$id), c$resp$size);
		# TODO: rewrite the message once a location variable can be put in notices
		NOTICE([$note=SSH_Login,
		        $conn=c,
		        $msg=message,
		        $sub=location$country_code]);

		# Check to see if this login came from a weird hostname (nameserver, mail server, etc.)
		when( local hostname = lookup_addr(c$id$orig_h) )
			{
			if ( strange_hostnames in hostname )
				{
				NOTICE([$note=SSH_Login_From_Strange_Hostname,
				        $conn=c,
				        $msg=fmt("Strange login from %s", hostname),
				        $sub=hostname]);
				}
			}
		}
	else if (c$resp$size >= 20000000)
		{
		NOTICE([$note=SSH_Bytecount_Inconsistency,
		        $conn=c,
		        $msg="During byte counting in extended SSH analysis, an overly large value was seen.",
		        $sub=fmt("%d",c$resp$size)]);
		}

	if ( (log_geodata_on_failure && status == "failure") ||
	     status == "success" )
		{
		location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
		}

	print ssh_ext_log, cat_sep("\t", "\\N", c$start_time,
				c$id$orig_h, fmt("%d", c$id$orig_p),
				c$id$resp_h, fmt("%d", c$id$resp_p),
				status, direction,
				location$country_code, location$region,
				versions$client, versions$server,
				c$resp$size);

	delete active_ssh_conns[c$id];
	# Stop watching this connection, we don't care about it anymore.
	skip_further_processing(c$id);
	set_record_packets(c$id, F);
	}

event connection_state_remove(c: connection)
	{
	event check_ssh_connection(c, T);
	}

event ssh_watcher(c: connection)
	{
	local id = c$id;
	# don't go any further if this connection is gone already!
	if ( !connection_exists(id) )
		{
		delete active_ssh_conns[id];
		return;
		}

	event check_ssh_connection(c, F);
	if ( c$id in active_ssh_conns )
		schedule +2mins { ssh_watcher(c) };
	}

event ssh_client_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$client = version;
	}

event ssh_server_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$server = version;
	}

event protocol_confirmation(c: connection, atype: count, aid: count)
	{
	if ( atype == ANALYZER_SSH )
		{
		local tmp: ssh_versions;
		active_ssh_conns[c$id]=tmp;
		schedule +2mins { ssh_watcher(c) };
		}
	}

@load global-ext
@load ssh
@load notice

module SSH;

export {
	const ssh_ext_log = open_log_file("ssh-ext") &raw_output;

	const password_guesses_limit = 30 &redef;
	const authentication_data_size = 5500 &redef;
	const guessing_timeout = 30 mins;

	# Keeps count of how many rejections a host has had
	global password_rejections: table[addr] of count &default=0 &write_expire=guessing_timeout;
	# Keeps track of hosts identified as guessing passwords
	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr;

	type ssh_versions: record {
		client: string &default="";
		server: string &default="";
	};

	# Only monitor SSH connections for up to 15 minutes
	global active_ssh_conns: table[conn_id] of ssh_versions &create_expire=15mins;

	# If you want to lookup and log geoip data in the event of a failed login.
	const log_geodata_on_failure = F &redef;

	# The set of countries for which you'd like to throw notices upon successful login
	#   requires Bro compiled with libGeoIP support
	const watched_countries: set[string] = {"RO"} &redef;

	# Strange/bad host names to originate successful SSH logins
	const strange_hostnames =
			/^d?ns[0-9]*\./ |
			/^smtp[0-9]*\./ |
			/^mail[0-9]*\./ |
			/^pop[0-9]*\./  |
			/^imap[0-9]*\./ |
			/^www[0-9]*\./  |
			/^ftp[0-9]*\./  &redef;

	# This is a table with orig subnet as the key, and subnet as the value.
	const ignore_guessers: table[subnet] of subnet &redef;

	redef enum Notice += {
		SSH_Login,
		SSH_PasswordGuessing,
		SSH_LoginByPasswordGuesser,
		SSH_Login_From_Strange_Hostname,
		SSH_Bytecount_Inconsistency,
	};
}

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a successful ssh login happens from or to a watched country
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_Login && n$sub in watched_countries); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a password guesser logs in successfully anywhere
#	# To avoid false positives, setting the lower bound for notification to 50 bad password attempts.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_LoginByPasswordGuesser && n$n > 50); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a local host is password guessing.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_PasswordGuessing &&
#		          is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

# Don't stop processing SSH connections in the default ssh policy script
redef skip_processing_after_handshake = F;

event check_ssh_connection(c: connection, done: bool)
	{
	# If this is no longer a known SSH connection, just return.
	if ( c$id !in active_ssh_conns )
		return;

	# If this is still a live connection and the byte count has not
	# crossed the threshold, just return and let the resheduled check happen later.
	if ( !done && c$resp$size < authentication_data_size )
		return;

	# Make sure the server has sent back more than 50 bytes to filter out
	# hosts that are just port scanning.  Nothing is ever logged if the server
	# doesn't send back at least 50 bytes.
	if (c$resp$size < 50)
		return;

	local versions = active_ssh_conns[c$id];
	local status = "failure";
	local direction = is_local_addr(c$id$orig_h) ? "to" : "from";
	local location: geo_location;
	# Need to give these values defaults in bro.init.
	location$country_code=""; location$region=""; location$city=""; location$latitude=0.0; location$longitude=0.0;

	if ( done && c$resp$size < authentication_data_size )
		{
		# presumed failure

		# Track the number of rejections
		if ( !(c$id$orig_h in ignore_guessers &&
		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
			password_rejections[c$id$orig_h] += 1;

		if ( password_rejections[c$id$orig_h] > password_guesses_limit &&
		     c$id$orig_h !in password_guessers )
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_PasswordGuessing,
			        $conn=c,
			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]),
			        $n=password_rejections[c$id$orig_h]]);
			}
		}
	# TODO: This is to work around a quasi-bug in Bro which occasionally
	#       causes the byte count to be oversized.
	else if (c$resp$size < 20000000)
		{
		# presumed successful login
		status = "success";

		if ( password_rejections[c$id$orig_h] > password_guesses_limit &&
		     c$id$orig_h !in password_guessers)
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_LoginByPasswordGuesser,
			        $conn=c,
			        $n=password_rejections[c$id$orig_h],
			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h])]);
			}

		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
		              direction, location$country_code, location$region, location$city,
		              location$latitude, location$longitude,
		              numeric_id_string(c$id), c$resp$size);
		# TODO: rewrite the message once a location variable can be put in notices
		NOTICE([$note=SSH_Login,
		        $conn=c,
		        $msg=message,
		        $sub=location$country_code]);

		# Check to see if this login came from a weird hostname (nameserver, mail server, etc.)
		when( local hostname = lookup_addr(c$id$orig_h) )
			{
			if ( strange_hostnames in hostname )
				{
				NOTICE([$note=SSH_Login_From_Strange_Hostname,
				        $conn=c,
				        $msg=fmt("Strange login from %s", hostname),
				        $sub=hostname]);
				}
			}
		}
	else if (c$resp$size >= 20000000)
		{
		NOTICE([$note=SSH_Bytecount_Inconsistency,
		        $conn=c,
		        $msg="During byte counting in extended SSH analysis, an overly large value was seen.",
		        $sub=fmt("%d",c$resp$size)]);
		}

	if ( (log_geodata_on_failure && status == "failure") ||
	     status == "success" )
		{
		location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
		}

	print ssh_ext_log, cat_sep("\t", "\\N", c$start_time,
				c$id$orig_h, fmt("%d", c$id$orig_p),
				c$id$resp_h, fmt("%d", c$id$resp_p),
				status, direction,
				location$country_code, location$region,
				versions$client, versions$server,
				c$resp$size);

	delete active_ssh_conns[c$id];
	# Stop watching this connection, we don't care about it anymore.
	skip_further_processing(c$id);
	set_record_packets(c$id, F);
	}

event connection_state_remove(c: connection)
	{
	event check_ssh_connection(c, T);
	}

event ssh_watcher(c: connection)
	{
	local id = c$id;
	# don't go any further if this connection is gone already!
	if ( !connection_exists(id) )
		{
		delete active_ssh_conns[id];
		return;
		}

	event check_ssh_connection(c, F);
	if ( c$id in active_ssh_conns )
		schedule +2mins { ssh_watcher(c) };
	}

event ssh_client_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$client = version;
	}

event ssh_server_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$server = version;
	}

event protocol_confirmation(c: connection, atype: count, aid: count)
	{
	if ( atype == ANALYZER_SSH )
		{
		local tmp: ssh_versions;
		active_ssh_conns[c$id]=tmp;
		schedule +2mins { ssh_watcher(c) };
		}
	}