Fixed default bahavior and comment for log redaction.

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /^[3-9]{4}([ -\.]?\x00?[0-9]{4}){3}$/ &redef;

	const cc_separators = /\.(.*\.){3}/ |
	                      /\-(.*\-){3}/ |
	                      /[:blank:](.*[:blank:]){3}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	local parts = str_split(gsub(val, /[^0-9]/, ""), vector(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15));
	for ( i in parts )
		{
		odd = !odd;
		local digit = to_count(parts[i]);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			# we've got a match
			local parts = split_all(data, cc_regex);
			local cc_match = "";
			local redacted_cc = "";
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches and save one back for
					# finding it's location.
					cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					redacted_cc = parts[i];
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, cc_match);

			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : cc_match),
			                   $data=(redact_log ? redacted_data : data)];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-10secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-10secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by defaultIf you want to see the credit card numbers in
	## the log, redef this value to T.
	## Notices are automatically and unchangeably redacted.
	const redact_log = F &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /^[3-9]{4}([ -\.]?\x00?[0-9]{4}){3}$/ &redef;

	const cc_separators = /\.(.*\.){3}/ |
	                      /\-(.*\-){3}/ |
	                      /[:blank:](.*[:blank:]){3}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	local parts = str_split(gsub(val, /[^0-9]/, ""), vector(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15));
	for ( i in parts )
		{
		odd = !odd;
		local digit = to_count(parts[i]);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			# we've got a match
			local parts = split_all(data, cc_regex);
			local cc_match = "";
			local redacted_cc = "";
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches and save one back for
					# finding it's location.
					cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					redacted_cc = parts[i];
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, cc_match);

			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : cc_match),
			                   $data=(redact_log ? redacted_data : data)];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-10secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-10secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}


Added beemaster::conninfo_to_alertinfo

Added beemaster::conninfo_to_alertinfo

module beemaster;

@load base/utils/addrs

export {
  type AlertInfo: record {
    timestamp: time;
    incident_type: string;
    protocol: string;
    source_ip: string;
    source_port: count;
    destination_ip: string;
    destination_port: count;
  };
}

function conninfo_to_alertinfo(input: Conn::Info) : AlertInfo {
    local conn = input$id;
    return AlertInfo($timestamp = input$ts, $incident_type = "foo", $protocol = "bar",
        $source_ip = addr_to_uri(conn$orig_h), $source_port = port_to_count(conn$orig_p),
        $destination_ip = addr_to_uri(conn$resp_h), $destination_port = port_to_count(conn$resp_p));
}

module beemaster;

export {
  type AlertInfo: record {
    timestamp: time;
    incident_type: string;
    protocol: string;
    source_ip: string;
    source_port: port;
    destination_ip: addr;
    destination_port: port;
  };
}


log sql injection requests

log sql injection requests

event bro_init()
{
    Log::add_filter(HTTP::LOG, [$name = "http-sqli",
                                $path = "http_sqli",
                                $pred(rec: HTTP::Info) = { return HTTP::URI_SQLI in rec$tags ; }
                                ]);
}

error: pathspec 'log-http-sqli.bro' did not match any file(s) known to git


dio_mysql_log.bro hinzugefügt

dio_mysql_log.bro hinzugefügt

module Dio_mysql;

export {
  redef enum Log::ID += { LOG };

  type Info: record {
    ts: time &log;
    id: string &log;
    local_ip: addr &log;
    local_port: port &log;
    remote_ip: addr &log;
    remote_port: port &log;
    transport: string &log;
    args: string $log;
  };
}

event bro_init() &priority=5 {
  Log::create_stream(Dio_mysql::LOG, [$columns=Info, $path="Dionaea_MySQL"]);
}

error: pathspec 'custom_scripts/dio_mysql_log.bro' did not match any file(s) known to git


terminate connection updated for bro 2.1

module TerminateConnection;

export {
	redef enum Notice::Type += {
		TerminatingConnection,	# connection will be terminated
		TerminatingConnectionIgnored,	# connection terminated disabled
	};

	# Whether we're allowed (and/or are capable) to terminate connections
	# using "rst".
	const activate_terminate_connection = F &redef;

	# Terminate the given connection.
	global terminate_connection: function(c: connection);

}

function full_id_string(c: connection): string
    {
    local id = c$id;
    return fmt("%s:%s -> %s:%s", id$orig_h, id$orig_p, id$resp_h, id$resp_p);
    }

function terminate_connection(c: connection)
	{
	local id = c$id;

	if ( activate_terminate_connection )
		{
		local local_init = Site::is_local_addr(id$orig_h);

		local term_cmd = fmt("rst %s -n 32 -d 20 %s %d %d %s %d %d",
					local_init ? "-R" : "",
					id$orig_h, id$orig_p, get_orig_seq(id),
					id$resp_h, id$resp_p, get_resp_seq(id));

		if ( reading_live_traffic() )
			system(term_cmd);
		else
			NOTICE([$note=TerminatingConnection, $conn=c,
				$msg=term_cmd, $sub="first termination command"]);

		term_cmd = fmt("rst %s -r 2 -n 4 -s 512 -d 20 %s %d %d %s %d %d",
				local_init ? "-R" : "",
				id$orig_h, id$orig_p, get_orig_seq(id),
				id$resp_h, id$resp_p, get_resp_seq(id));

		if ( reading_live_traffic() )
			system(term_cmd);
		else
			NOTICE([$note=TerminatingConnection, $conn=c,
				$msg=term_cmd, $sub="second termination command"]);

		NOTICE([$note=TerminatingConnection, $conn=c,
			$msg=fmt("terminating %s", full_id_string(c))]);
		}

	else
		NOTICE([$note=TerminatingConnectionIgnored, $conn=c,
			$msg=fmt("ignoring request to terminate %s",
					full_id_string(c))]);
	}

# $Id$

@load site
@load notice

# Ugly: we need the following from conn.bro, but we can't soundly load
# it because it in turn loads us.
global full_id_string: function(c: connection): string;

module TerminateConnection;

export {
	redef enum Notice += {
		TerminatingConnection,	# connection will be terminated
		TerminatingConnectionIgnored,	# connection terminated disabled
	};

	# Whether we're allowed (and/or are capable) to terminate connections
	# using "rst".
	const activate_terminate_connection = F &redef;

	# Terminate the given connection.
	global terminate_connection: function(c: connection);

}

function terminate_connection(c: connection)
	{
	local id = c$id;

	if ( activate_terminate_connection )
		{
		local local_init = is_local_addr(id$orig_h);

		local term_cmd = fmt("rst %s -n 32 -d 20 %s %d %d %s %d %d",
					local_init ? "-R" : "",
					id$orig_h, id$orig_p, get_orig_seq(id),
					id$resp_h, id$resp_p, get_resp_seq(id));

		if ( reading_live_traffic() )
			system(term_cmd);
		else
			NOTICE([$note=TerminatingConnection, $conn=c,
				$msg=term_cmd, $sub="first termination command"]);

		term_cmd = fmt("rst %s -r 2 -n 4 -s 512 -d 20 %s %d %d %s %d %d",
				local_init ? "-R" : "",
				id$orig_h, id$orig_p, get_orig_seq(id),
				id$resp_h, id$resp_p, get_resp_seq(id));

		if ( reading_live_traffic() )
			system(term_cmd);
		else
			NOTICE([$note=TerminatingConnection, $conn=c,
				$msg=term_cmd, $sub="second termination command"]);

		NOTICE([$note=TerminatingConnection, $conn=c,
			$msg=fmt("terminating %s", full_id_string(c))]);
		}

	else
		NOTICE([$note=TerminatingConnectionIgnored, $conn=c,
			$msg=fmt("ignoring request to terminate %s",
					full_id_string(c))]);
	}


Modified scripts for flexible log selection. - Added "include_logs" and "exclude_logs" that allows for default   to include all, or if explicitly include logs or exclude logs.

module KafkaLogger;

export {
	# redefine this in your script to identify the logs
	# that should be sent to Kafka. By default, all will be sent.
	# for example:
	#
	# redef KafkaLogger::include_logs = set(HTTP::LOG, Conn::Log, DNS::LOG);
	#
	# that will send the HTTP, Conn, and DNS logs up to Kafka.
	#
	const include_logs: set[Log::ID] &redef;
	# redefine this in your script to identify the logs
	# that should be excluded from sending to Kafka. By default, all
	# will be sent.
	# for example:
	#
	# redef KafkaLogger::include_logs = set(HTTP::LOG, Conn::Log, DNS::LOG);
	#
	# that will send the HTTP, Conn, and DNS logs up to Kafka.
	#
	const exclude_logs: set[Log::ID] &redef;

}

event bro_init() &priority=-5
{

	for (stream_id in Log::active_streams)
	{
	    if (|include_logs| > 0 && stream_id !in include_logs){
	        next;
	    }
			if ( stream_id in exclude_logs ) {
					next;
			}
	    # note: the filter name is different for each log, to cause Bro to instantiate
	    # a new Writer instance for each log. The bro folks might want me
	    # to do this with a single writer instance, but the mechanics of
	    # modifying the field names and adding fields made it quite complicated,
	    # so I opted to make one log writer per bro log going to Kafka.
	    # this means there will be multiple connections from bro to the kafka
	    # server, one per log file.
	    local streamString = fmt("%s", stream_id);
	    local pathname = fmt("%s", KafkaLogger::log_names[streamString]);
	    local filter: Log::Filter = [$name = fmt("kafka-%s",stream_id),
	    							 $writer = Log::WRITER_KAFKAWRITER,
	    							 $path = pathname
	    							];
	    Log::add_filter(stream_id, filter);

	}

}

module KafkaLogger;

export {
	# redefine this in your script to identify the logs
	# that should be sent up to bro.
	# for example:
	#
	# redef KafkaLogger::logs_to_send = set(HTTP::LOG, Conn::Log, DNS::LOG);
	#
	# that will send the HTTP, Conn, and DNS logs up to Kafka.
	#
	const logs_to_send: set[Log::ID] &redef;
}

event bro_init() &priority=-5
{

	for (stream_id in Log::active_streams)
	{
	    if (stream_id !in logs_to_send){
	        next;
	    }
	    # note: the filter name is different for each log, to cause Bro to instantiate
	    # a new Writer instance for each log. The bro folks might want me
	    # to do this with a single writer instance, but the mechanics of
	    # modifying the field names and adding fields made it quite complicated,
	    # so I opted to make one log writer per bro log going to Kafka.
	    # this means there will be multiple connections from bro to the kafka
	    # server, one per log file.
	    local streamString = fmt("%s", stream_id);
	    local pathname = fmt("%s", KafkaLogger::log_names[streamString]);
	    local filter: Log::Filter = [$name = fmt("kafka-%s",stream_id),
	    							 $writer = Log::WRITER_KAFKAWRITER,
	    							 $path = pathname
	    							];
	    Log::add_filter(stream_id, filter);

	}

}


Added some new DNSBLs to the smtp-ext script.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp_ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
	};

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    ExternalHosts - only capture the path until an internal host is found.
	#    InternalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=10secs;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

function find_address_in_smtp_header(header: string): string
{
	local text_ip = "";
	local parts: string_array;
	if ( /\[.*\]/ in header )
		parts = split(header, /[\[\]]/);
	else if ( /\(.*\)/ in header )
		parts = split(header, /[\(\)]/);

	if (|parts| > 1)
		{
		if ( |parts| > 3 && parts[4] == ip_addr_regex )
			text_ip = parts[4];
		else if ( parts[2] == ip_addr_regex )
			text_ip = parts[2];
		}
	return text_ip;
}

# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in smtp_sessions )
		return;

	if ( /^[^[:blank:]]*?:[[:blank:]]/ in data && id !in smtp_received_finished )
		delete in_received_from_headers[id];
	if ( /^Received:[[:blank:]]/ in data && id !in smtp_received_finished )
		add in_received_from_headers[id];

	local session = smtp_sessions[id];

	if ( session$in_header &&              # headers are currently being analyzed
	     id in in_received_from_headers && # currently seeing received from headers
	     id !in smtp_received_finished &&  # we don't want to stop seeing this message yet
	     /[\[\(]/ in data )                # the line might contain an ip address
		{
		local text_ip = find_address_in_smtp_header(data);

		# check for valid-ish ip - some mtas are weird.
		if ( is_valid_ip(text_ip) )
			{
			local ip = to_addr(text_ip);

			# I don't care if mail bounces around on localhost
			if ( ip == 127.0.0.1 ) return;

			if ( ip_matches_hosts(ip, mail_path_capture) ||
			     ip in private_address_space )
				{
				if (smtp_forward_paths[id] == "")
					smtp_forward_paths[id] = fmt("%s :: %s -> %s", ip, id$orig_h, id$resp_h);
				else
					smtp_forward_paths[id] = fmt("%s -> %s", ip, smtp_forward_paths[id]);
				}
			else
				{
				if (smtp_forward_paths[id] == "")
					smtp_forward_paths[id] = fmt("... %s :: %s -> %s", ip, id$orig_h, id$resp_h);
				else
					smtp_forward_paths[id] = fmt("... %s -> %s", ip, smtp_forward_paths[id]);

				add smtp_received_finished[id];
				}
			}
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		{
		add smtp_received_finished[id];
		}
	}

function end_smtp_extended_logging(id: conn_id)
	{
	if ( id !in conn_info )
		return;
	local conn_log = conn_info[id];

	print smtp_ext_log, cat_sep("\t", "\\N", network_time(),
	                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
	                            conn_log$helo, conn_log$msg_id, conn_log$in_reply_to,
	                            conn_log$mailfrom, fmt_str_set(conn_log$rcptto, /[\"\'<>]|([[:blank:]].*$)/),
	                            conn_log$date, conn_log$from, conn_log$reply_to, fmt_str_set(conn_log$to, /[\"\']/),
	                            conn_log$last_reply, smtp_forward_paths[id]);
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( code >= 400 && id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# If a local MTA receives a message from a remote host telling it that it's on a block list, raise a notice.
		if ( smtp_bl_error_messages in msg && is_local_addr(c$id$orig_h) )
			{
			local note = SMTP_BL_Error_Message;

			# Determine if the originator's IP address is in the message.
			# TODO: make this work with IPv6
			local msg_parts = split_all(msg, /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/);
			local text_ip = "";
			if ( |msg_parts| > 2 )
				text_ip = msg_parts[2];
			if ( is_valid_ip(text_ip) && to_addr(text_ip) == c$id$orig_h )
				note = SMTP_BL_Blocked_Host;

			NOTICE([$note=note,
			        $conn=c,
			        $msg=fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h),
			        $sub=fmt("Remote host said: %s", msg)]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id in smtp_sessions )
		{
		if ( id !in conn_info )
			conn_info[id] = default_session_info();
		local conn_log = conn_info[id];

		if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
			conn_log$helo = arg;

		if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
			add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

		if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
			{
			local partially_done = split1(arg, /:[[:blank:]]*/)[2];
			conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
			}
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;
	if ( id !in conn_info )
	  	return;

	local conn_log = conn_info[id];
	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:[[:blank:]]/ in data )
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:[[:blank:]]/ in data )
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[dD][aA][tT][eE]:[[:blank:]]/ in data )
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[fF][rR][oO][mM]:[[:blank:]]/ in data )
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[tT][oO]:[[:blank:]]/ in data )
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];

	if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:[[:blank:]]/ in data )
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp_ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
	};

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    ExternalHosts - only capture the path until an internal host is found.
	#    InternalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=10secs;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

function find_address_in_smtp_header(header: string): string
{
	local text_ip = "";
	local parts: string_array;
	if ( /\[.*\]/ in header )
		parts = split(header, /[\[\]]/);
	else if ( /\(.*\)/ in header )
		parts = split(header, /[\(\)]/);

	if (|parts| > 1)
		{
		if ( |parts| > 3 && parts[4] == ip_addr_regex )
			text_ip = parts[4];
		else if ( parts[2] == ip_addr_regex )
			text_ip = parts[2];
		}
	return text_ip;
}

# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in smtp_sessions )
		return;

	if ( /^[^[:blank:]]*?:[[:blank:]]/ in data && id !in smtp_received_finished )
		delete in_received_from_headers[id];
	if ( /^Received:[[:blank:]]/ in data && id !in smtp_received_finished )
		add in_received_from_headers[id];

	local session = smtp_sessions[id];

	if ( session$in_header &&              # headers are currently being analyzed
	     id in in_received_from_headers && # currently seeing received from headers
	     id !in smtp_received_finished &&  # we don't want to stop seeing this message yet
	     /[\[\(]/ in data )                # the line might contain an ip address
		{
		local text_ip = find_address_in_smtp_header(data);

		# check for valid-ish ip - some mtas are weird.
		if ( is_valid_ip(text_ip) )
			{
			local ip = to_addr(text_ip);

			# I don't care if mail bounces around on localhost
			if ( ip == 127.0.0.1 ) return;

			if ( ip_matches_hosts(ip, mail_path_capture) ||
			     ip in private_address_space )
				{
				if (smtp_forward_paths[id] == "")
					smtp_forward_paths[id] = fmt("%s :: %s -> %s", ip, id$orig_h, id$resp_h);
				else
					smtp_forward_paths[id] = fmt("%s -> %s", ip, smtp_forward_paths[id]);
				}
			else
				{
				if (smtp_forward_paths[id] == "")
					smtp_forward_paths[id] = fmt("... %s :: %s -> %s", ip, id$orig_h, id$resp_h);
				else
					smtp_forward_paths[id] = fmt("... %s -> %s", ip, smtp_forward_paths[id]);

				add smtp_received_finished[id];
				}
			}
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		{
		add smtp_received_finished[id];
		}
	}

function end_smtp_extended_logging(id: conn_id)
	{
	if ( id !in conn_info )
		return;
	local conn_log = conn_info[id];

	print smtp_ext_log, cat_sep("\t", "\\N", network_time(),
	                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
	                            conn_log$helo, conn_log$msg_id, conn_log$in_reply_to,
	                            conn_log$mailfrom, fmt_str_set(conn_log$rcptto, /[\"\'<>]|([[:blank:]].*$)/),
	                            conn_log$date, conn_log$from, conn_log$reply_to, fmt_str_set(conn_log$to, /[\"\']/),
	                            conn_log$last_reply, smtp_forward_paths[id]);
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( code >= 400 && id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# If a local MTA receives a message from a remote host telling it that it's on a block list, raise a notice.
		if ( smtp_bl_error_messages in msg && is_local_addr(c$id$orig_h) )
			{
			local note = SMTP_BL_Error_Message;

			# Determine if the originator's IP address is in the message.
			# TODO: make this work with IPv6
			local msg_parts = split_all(msg, /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/);
			local text_ip = "";
			if ( |msg_parts| > 2 )
				text_ip = msg_parts[2];
			if ( is_valid_ip(text_ip) && to_addr(text_ip) == c$id$orig_h )
				note = SMTP_BL_Blocked_Host;

			NOTICE([$note=note,
			        $conn=c,
			        $msg=fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h),
			        $sub=fmt("Remote host said: %s", msg)]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id in smtp_sessions )
		{
		if ( id !in conn_info )
			conn_info[id] = default_session_info();
		local conn_log = conn_info[id];

		if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
			conn_log$helo = arg;

		if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
			add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

		if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
			{
			local partially_done = split1(arg, /:[[:blank:]]*/)[2];
			conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
			}
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;
	if ( id !in conn_info )
	  	return;

	local conn_log = conn_info[id];
	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:[[:blank:]]/ in data )
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:[[:blank:]]/ in data )
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[dD][aA][tT][eE]:[[:blank:]]/ in data )
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[fF][rR][oO][mM]:[[:blank:]]/ in data )
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[tT][oO]:[[:blank:]]/ in data )
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];

	if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:[[:blank:]]/ in data )
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}


Huge changes to smtp-ext (additions, fixes, .and code refactoring)

New notice:
  SMTP_Suspicious_Origination - Thrown when mail appears to originate from
     a network or country where unwanted mail typically originates for your site.

New data logged:
  x-originating-ip
  files transferred in the email

New smtp block lists:
  mail-abuse.com
  bbl.barracudacentral.com

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

# This is a locally defined event.  Handle it with a priority greater than
# negative 5 if you want to dig into the conn_info record before it's deleted.
function end_smtp_extended_logging(id: conn_id)
	{
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s). Subject: %s", loc$country_code, ip, conn_log$subject),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s). Subject: %s", loc$country_code, ip, conn_log$subject),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(id);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
	};

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=10secs;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local text_ip = "";
	local parts: string_array;
	if ( /\[.*\]/ in header )
		parts = split(header, /[\[\]]/);
	else if ( /\(.*\)/ in header )
		parts = split(header, /[\(\)]/);

	if (|parts| > 1)
		{
		if ( |parts| > 3 && parts[4] == ip_addr_regex )
			text_ip = parts[4];
		else if ( parts[2] == ip_addr_regex )
			text_ip = parts[2];
		}
	return text_ip;
}

# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in smtp_sessions )
		return;

	if ( /^[^[:blank:]]*?:[[:blank:]]/ in data && id !in smtp_received_finished )
		delete in_received_from_headers[id];
	if ( /^Received:[[:blank:]]/ in data && id !in smtp_received_finished )
		add in_received_from_headers[id];

	local session = smtp_sessions[id];

	if ( session$in_header &&              # headers are currently being analyzed
	     id in in_received_from_headers && # currently seeing received from headers
	     id !in smtp_received_finished &&  # we don't want to stop seeing this message yet
	     /[\[\(]/ in data )                # the line might contain an ip address
		{
		local text_ip = find_address_in_smtp_header(data);

		# check for valid-ish ip - some mtas are weird.
		if ( is_valid_ip(text_ip) )
			{
			local ip = to_addr(text_ip);

			# I don't care if mail bounces around on localhost
			if ( ip == 127.0.0.1 ) return;

			if ( addr_matches_hosts(ip, mail_path_capture) ||
			     ip in private_address_space )
				{
				if (smtp_forward_paths[id] == "")
					smtp_forward_paths[id] = fmt("%s :: %s -> %s", ip, id$orig_h, id$resp_h);
				else
					smtp_forward_paths[id] = fmt("%s -> %s", ip, smtp_forward_paths[id]);
				}
			else
				{
				if (smtp_forward_paths[id] == "")
					smtp_forward_paths[id] = fmt("... %s :: %s -> %s", ip, id$orig_h, id$resp_h);
				else
					smtp_forward_paths[id] = fmt("... %s -> %s", ip, smtp_forward_paths[id]);

				add smtp_received_finished[id];
				}
			}
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		{
		add smtp_received_finished[id];
		}
	}

function end_smtp_extended_logging(id: conn_id)
	{
	if ( id !in conn_info )
		return;
	local conn_log = conn_info[id];

	print smtp_ext_log, cat_sep("\t", "\\N", network_time(),
	                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
	                            conn_log$helo, conn_log$msg_id, conn_log$in_reply_to,
	                            conn_log$mailfrom, fmt_str_set(conn_log$rcptto, /[\"\'<>]|([[:blank:]].*$)/),
	                            conn_log$date, conn_log$from, conn_log$reply_to, fmt_str_set(conn_log$to, /[\"\']/),
	                            conn_log$last_reply, smtp_forward_paths[id]);
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( code >= 400 && id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local msg_parts = split_all(msg, ip_addr_regex);
			local text_ip = "";
			if ( |msg_parts| > 2 )
				text_ip = msg_parts[2];
			if ( is_valid_ip(text_ip) && to_addr(text_ip) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id in smtp_sessions )
		{
		if ( id !in conn_info )
			conn_info[id] = default_session_info();
		local conn_log = conn_info[id];

		if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
			conn_log$helo = arg;

		if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
			add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

		if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
			{
			local partially_done = split1(arg, /:[[:blank:]]*/)[2];
			conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
			}
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;
	if ( id !in conn_info )
	  	return;

	local conn_log = conn_info[id];
	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:[[:blank:]]/ in data )
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:[[:blank:]]/ in data )
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[dD][aA][tT][eE]:[[:blank:]]/ in data )
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[fF][rR][oO][mM]:[[:blank:]]/ in data )
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];

	if ( /^[tT][oO]:[[:blank:]]/ in data )
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];

	if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:[[:blank:]]/ in data )
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}


Added Bro example

Added Bro example

##! Add countries for the originator and responder of a connection
##! to the connection logs.

module Conn;

export {
	redef record Conn::Info += {
		## Country code for the originator of the connection based
		## on a GeoIP lookup.
		orig_cc: string &optional &log;
		## Country code for the responser of the connection based
		## on a GeoIP lookup.
		resp_cc: string &optional &log;
	};
}

event connection_state_remove(c: connection)
	{
	local orig_loc = lookup_location(c$id$orig_h);
	if ( orig_loc?$country_code )
		c$conn$orig_cc = orig_loc$country_code;

	local resp_loc = lookup_location(c$id$resp_h);
	if ( resp_loc?$country_code )
		c$conn$resp_cc = resp_loc$country_code;
	}

TODO add a nice demo!
Try to keep it short!


rsyslog-invalid-pri.bro: set facility less than 128

# Generates a notice and e-mail if syslog messages contain a PRI greater than 191 but less than 1016
#                                                                        fac: (23 but less than 127)
# Vulnerability: http://www.rsyslog.com/remote-syslog-pri-vulnerability/
#
#       To use:
#               1. Add script to configuration local.bro: $ echo '@load rsyslog-invalid-pri.bro' >> $BROPREFIX/share/bro/site/local.bro
#               2. Copy script to $BROPREFIX/share/bro/site
#               3. $ broctl check && broctl install && broctl restart
# Testing:
# mausezahn -t syslog severity=7,facility=24,host=mausezahn -P "Invalid PRI message test" -B 10.1.1.100
#
# Facility: Divide the PRI number by 8.
# 191/8 = 23
#
# Severity:  multiply facility by 8 and subtract from PRI
# 191 - (23 * 8 ) = 7
#
#   PRI = Facility 23 and Priority (7)

# Possible combinations:
# for pri in {1..1016}; do FAC="$((pri/8))"; echo -e -n "PRI:$pri Fac:$FAC Sev:$((pri - ($FAC * 8 )))\n"; done

@load base/frameworks/notice

export {
        redef enum Notice::Type += {
                SYSLOG::Invalid_PRI
        };

        # List your NTP servers here
        const syslog_servers: set[addr] = {
	10.1.1.100
        } &redef;

}

redef Notice::emailed_types += {
                SYSLOG::Invalid_PRI
};

event syslog_message(c: connection, facility: count, severity: count, msg: string)
        {

        if ( c$id$resp_h !in syslog_servers )
                return;

	local pri = (facility * 8) + severity;

        if (facility > 23 && facility < 128)
                {
                NOTICE([$note=SYSLOG::Invalid_PRI,
		$msg=fmt("Syslog message with invalid PRI of %d", pri),
                $conn=c,
                $identifier=cat(c$id$orig_h,c$id$resp_h),
                $suppress_for=1day]);
                }
        }

# Generates a notice and e-mail if syslog messages contain a PRI greater than 191 but less than 1016
#                                                                        fac: (23 but less than 127)
# Vulnerability: http://www.rsyslog.com/remote-syslog-pri-vulnerability/
#
#       To use:
#               1. Add script to configuration local.bro: $ echo '@load rsyslog-invalid-pri.bro' >> $BROPREFIX/share/bro/site/local.bro
#               2. Copy script to $BROPREFIX/share/bro/site
#               3. $ broctl check && broctl install && broctl restart
# Testing:
# mausezahn -t syslog severity=7,facility=24,host=mausezahn -P "Invalid PRI message test" -B 10.1.1.100
#
# Facility: Divide the PRI number by 8.
# 191/8 = 23
#
# Severity:  multiply facility by 8 and subtract from PRI
# 191 - (23 * 8 ) = 7
#
#   PRI = Facility 23 and Priority (7)

# Possible combinations:
# for pri in {1..1016}; do FAC="$((pri/8))"; echo -e -n "PRI:$pri Fac:$FAC Sev:$((pri - ($FAC * 8 )))\n"; done

@load base/frameworks/notice

export {
        redef enum Notice::Type += {
                SYSLOG::Invalid_PRI
        };

        # List your NTP servers here
        const syslog_servers: set[addr] = {
	10.1.1.100
        } &redef;

}

redef Notice::emailed_types += {
                SYSLOG::Invalid_PRI
};

event syslog_message(c: connection, facility: count, severity: count, msg: string)
        {

        if ( c$id$resp_h !in syslog_servers )
                return;

	local pri = (facility * 8) + severity;

        if (facility > 23 && facility < 127)
                {
                NOTICE([$note=SYSLOG::Invalid_PRI,
		$msg=fmt("Syslog message with invalid PRI of %d", pri),
                $conn=c,
                $identifier=cat(c$id$orig_h,c$id$resp_h),
                $suppress_for=1day]);
                }
        }


Allow use of tcpdump-like BPF

Allow use of tcpdump-like BPF

Add the bro config which enables the tcpdump-style filter options, which vortex also supports, to be added to the command-line

@load base/frameworks/packet-filter
@load conn-contents.bro


correct the notice msg, the direction in the wording is backwards

# For this script to work, you must define a local_mail table
# listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };

@ifdef ( local_mail )

@load conn-id
@load smtp

module SMTP;

type smtp_counter: record {
 rejects: count;
 total: count;
 connects: set[conn_id];
 rej_conns: set[conn_id];
};

export {
 # The idea for this is that if a host makes more than spam_threshold
 # smtp connections per hour of which at least spam_percent of those are
 # rejected and the host is not a known mail sending host, then it's likely
 # sending spam or viruses.
 #
 const spam_threshold = 300 &redef;
 const spam_percent = 30 &redef;

 # These are smtp status codes that are considered "rejected".
 const smtp_reject_codes: set[count] = {
   501, # Bad sender address syntax
   550, # Requested action not taken: mailbox unavailable
   551, # User not local; please try <forward-path>
   553, # Requested action not taken: mailbox name not allowed
   554, # Transaction failed
 };

 redef enum Notice += {
   SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
   SMTP_PossibleInternalSpam, # Internal host seems to be spamming
   SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
 };

 global smtp_status_comparison: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
		             msg: string, cont_resp: bool)
{
 if ( c$id$orig_h !in smtp_status_comparison )
 {
   local bar: set[conn_id];
   local blarg: set[conn_id];
   smtp_status_comparison[c$id$orig_h] = [$rejects=0, $total=0, $connects=bar, $rej_conns=blarg];
 }

 # Set the smtp_counter to the local var "foo"
 local foo = smtp_status_comparison[c$id$orig_h];

 if ( code in smtp_reject_codes &&
           c$id !in foo$rej_conns )
 {
   ++foo$rejects;
   local session = smtp_sessions[c$id];
   add foo$rej_conns[c$id];
 }

 if ( c$id !in foo$connects )
 {
   ++foo$total;
   add foo$connects[c$id];
 }

 local host = c$id$orig_h;
 if ( foo$total >= spam_threshold ) {
   local percent = (foo$rejects*100) / foo$total;
   if ( percent >= spam_percent ) {
     if ( host in local_mail )
       NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("a high percentage of mail from %s is being rejected", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else if ( is_local_addr(host) )
       NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else
       NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
   }
 }
}

@endif

# For this script to work, you must define a local_mail table
# listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };

@ifdef ( local_mail )

@load conn-id
@load smtp

module SMTP;

type smtp_counter: record {
 rejects: count;
 total: count;
 connects: set[conn_id];
 rej_conns: set[conn_id];
};

export {
 # The idea for this is that if a host makes more than spam_threshold
 # smtp connections per hour of which at least spam_percent of those are
 # rejected and the host is not a known mail sending host, then it's likely
 # sending spam or viruses.
 #
 const spam_threshold = 300 &redef;
 const spam_percent = 30 &redef;

 # These are smtp status codes that are considered "rejected".
 const smtp_reject_codes: set[count] = {
   501, # Bad sender address syntax
   550, # Requested action not taken: mailbox unavailable
   551, # User not local; please try <forward-path>
   553, # Requested action not taken: mailbox name not allowed
   554, # Transaction failed
 };

 redef enum Notice += {
   SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
   SMTP_PossibleInternalSpam, # Internal host seems to be spamming
   SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
 };

 global smtp_status_comparison: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
		             msg: string, cont_resp: bool)
{
 if ( c$id$orig_h !in smtp_status_comparison )
 {
   local bar: set[conn_id];
   local blarg: set[conn_id];
   smtp_status_comparison[c$id$orig_h] = [$rejects=0, $total=0, $connects=bar, $rej_conns=blarg];
 }

 # Set the smtp_counter to the local var "foo"
 local foo = smtp_status_comparison[c$id$orig_h];

 if ( code in smtp_reject_codes &&
           c$id !in foo$rej_conns )
 {
   ++foo$rejects;
   local session = smtp_sessions[c$id];
   add foo$rej_conns[c$id];
 }

 if ( c$id !in foo$connects )
 {
   ++foo$total;
   add foo$connects[c$id];
 }

 local host = c$id$orig_h;
 if ( foo$total >= spam_threshold ) {
   local percent = (foo$rejects*100) / foo$total;
   if ( percent >= spam_percent ) {
     if ( host in local_mail )
       NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("%s is rejecting a high percentage of mail", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else if ( is_local_addr(host) )
       NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else
       NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
   }
 }
}

@endif


Removed debugging code

Removed debugging code

@load base/frameworks/files

module SMBx;

global smbports = { 139/tcp, 445/tcp };
redef capture_filters = { ["smb"] = "port 445" };

redef record connection += {
	smbx: smb2_session &optional;
};

redef record Files::Info += {
	path: string &optional &log;
};

event bro_init() &priority=5
{
    Analyzer::register_for_ports(Analyzer::ANALYZER_SMBX, smbports);
}

event smb2_pre_file_transfer(c: connection, h: smb2_header, f: smb2_fileinfo)
{
	if (!c?$smbx)
	{
		local t : table[string] of smb2_fileinfo &read_expire=5mins;
		local s: smb2_session = smb2_session();
		s$files = t;
		c$smbx = s;
	}

	c$smbx$files[f$id] = f;
}

event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
{
	if (!c?$smbx || f$source != "SMBX")
		return;

	if (f$id !in c$smbx$files)
		return;

	local f2 = c$smbx$files[f$id];
	f$info$filename = f2$name;
	f$info$path = f2$path;
}

@load base/frameworks/files

module SMBx;

global smbports = { 139/tcp, 445/tcp };
redef capture_filters = { ["smb"] = "port 445" };

redef record connection += {
	smbx: smb2_session &optional;
};

redef record Files::Info += {
	path: string &optional &log;
};

event bro_init() &priority=5
{
    Analyzer::register_for_ports(Analyzer::ANALYZER_SMBX, smbports);
}

event smb2_pre_file_transfer(c: connection, h: smb2_header, f: smb2_fileinfo)
{
	print fmt("Hi!!");
	if (!c?$smbx)
	{
		local t : table[string] of smb2_fileinfo &read_expire=5mins;
		local s: smb2_session = smb2_session();
		s$files = t;
		c$smbx = s;
	}

	c$smbx$files[f$id] = f;
}

event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
{
	print fmt("Here %s %s", f$source, c?$smbx);
	if (!c?$smbx || f$source != "SMBX")
		return;

	print "And Here";

	if (f$id !in c$smbx$files)
		return;

	print "And And Here";
	local f2 = c$smbx$files[f$id];
	f$info$filename = f2$name;
	f$info$path = f2$path;
}


refactor

refactor

module DNS;

export {
    redef record Info += {
        is_external: bool &default=F;
        ## Country code of external DNS server
        cc: string &log &optional;
        ## hostname of external DNS server
        #resp_h_hostname: string &log &optional;
    };

    const ignore_external: set[subnet] = {
        8.8.8.8/32, #google dns
        8.8.4.4/32, #google dns
        208.67.220.123/32, #opendns
        208.67.222.222/32, #opendns
        208.67.220.220/32, #opendns
    } &redef;

    redef enum Notice::Type += {
        ## Indicates that a user is using an external DNS server
        EXTERNAL_DNS,

        ## Indicates that a user is using an external DNS server in
        ## a foreign country.
        EXTERNAL_FOREIGN_DNS,
    };

    redef Notice::type_suppression_intervals += {
        [EXTERNAL_DNS] = 12hr,
        [EXTERNAL_FOREIGN_DNS] = 12hr,
    };

    const local_countries: set[string] = {
        "US",
    } &redef;

}

event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
{
    local rec = c$dns;
    if(!rec$RD)
        return;

    local orig_h = rec$id$orig_h;
    local resp_h = rec$id$resp_h;

    #is this an outbound query
    if(!Site::is_local_addr(orig_h) || Site::is_local_addr(resp_h))
        return;

    if(orig_h in ignore_external || resp_h in ignore_external)
        return;

    rec$is_external=T;

    local loc = lookup_location(resp_h);

    rec$cc = "";
    if(loc?$country_code){
        rec$cc = loc$country_code;
    }

    when (local hostname = lookup_addr(resp_h)) {
        #Doesn't work for the log, but we can use it here :-(
        #rec$resp_h_hostname = hostname;

        local note = EXTERNAL_DNS;
        local nmsg = fmt("An external DNS server is in use %s(%s)", resp_h, hostname);

        if(rec$cc !in local_countries){
            note = EXTERNAL_FOREIGN_DNS;
            nmsg = fmt("An external foreign(%s) DNS server is in use %s(%s).", rec$cc, resp_h, hostname);
        }
        local ident = fmt("%s-%s", orig_h, resp_h);
        NOTICE([$note=note,
                $msg=nmsg,
                $sub=fmt("%s", resp_h),
                $identifier=ident,
                $conn=c]);

    }

}

event bro_init()
{
    Log::add_filter(DNS::LOG, [$name = "external-dns",
                                $path = "external_dns",
                                $exclude = set("uid", "proto", "trans_id","qclass", "qclass_name", "qtype", "rcode",
                                "QR","AA","TC","RD","RA","Z","answers","TTLs"),
                                $pred(rec: DNS::Info) = {

        return rec$is_external==T;
            return F;
    } ]);
}

module DNS;

export {
    redef record Info += {
        ## Country code of external DNS server
        cc: string &log &optional;
        ## hostname of external DNS server
        #resp_h_hostname: string &log &optional;
    };

    const ignore_external: set[subnet] = {
        8.8.8.8/32, #google dns
        8.8.4.4/32, #google dns
        208.67.220.123/32, #opendns
        208.67.222.222/32, #opendns
        208.67.220.220/32, #opendns
    } &redef;

}

event bro_init()
{
    Log::add_filter(DNS::LOG, [$name = "external-dns",
                                $path = "external_dns",
                                $exclude = set("uid", "proto", "trans_id","qclass", "qclass_name", "qtype", "rcode",
                                "QR","AA","TC","RD","RA","Z","answers","TTLs"),
                                $pred(rec: DNS::Info) = {
        if(!rec$RD)
            return F;

        local orig_h = rec$id$orig_h;
        local resp_h = rec$id$resp_h;

        #is this an outbound query
        if(!Site::is_local_addr(orig_h) || Site::is_local_addr(resp_h))
            return F;

        if(orig_h in ignore_external || resp_h in ignore_external)
            return F;

        local loc = lookup_location(resp_h);

        rec$cc = "";
        if(loc?$country_code){
            rec$cc = loc$country_code;
        }
        return T;

    } ]);
}


Fixed a problem with accurately recording the received headers.

@load global-ext
@load smtp

type smtp_ext_session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	first_received: string &default="";
	second_received: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: set[string];
	path: string &default="";
	is_webmail: bool &default=F; # This is not being set yet.
	agent: string &default="";

	# These are used during processing and are likely less useful.
	current_header: string &default="";
	in_received_from_headers: bool &default=F;
	received_finished: bool &default=F;
};

# Define the generic smtp-ext event that can be handled from other scripts
global smtp_ext: event(id: conn_id, si: smtp_ext_session_info);

module SMTP;

export {
	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Direction to capture the full "Received from" path. (from the Hosts enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /spamcop\.net\/bl/
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /rbl\.knology\.net\//
	  | /intercept\.datapacket\.net\// &redef;
}

function default_smtp_ext_session_info(): smtp_ext_session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	local tmp3: set[string] = set();
	return [$rcptto=tmp, $to=tmp2, $files=tmp3];
	}

# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of smtp_ext_session_info &read_expire=4mins;

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	# Throw the event for other scripts to handle
	event smtp_ext(id, conn_log);

	delete conn_info[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_smtp_ext_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_smtp_ext_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			add conn_info[id]$files[data];
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$current_header == "received" )
			conn_log$first_received += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		else if ( conn_log$current_header == "agent" )
			conn_log$agent += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}

	else if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		{
		conn_log$second_received = conn_log$first_received;
		conn_log$first_received = split1(data, /:[[:blank:]]*/)[2];
		# Fill in the second value in case there is only one hop in the message.
		if ( conn_log$second_received == "" )
			conn_log$second_received = conn_log$first_received;

		conn_log$current_header = "received";
		}

	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	else if ( /^[xX]-[mM][aA][iI][lL][eE][rR]:[[:blank:]]/ |
	          /^[uU][sS][eE][rR]-[aA][gG][eE][nN][tT]:[[:blank:]]/ in data )
		{
		conn_log$agent = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "agent";
		}

	}

# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header )
		return;

	local conn_log = conn_info[id];

	# If we've decided that we're done watching the received headers, we're done.
	if ( conn_log$received_finished )
		return;

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		conn_log$in_received_from_headers = T;
	else if ( /^[[:blank:]]/ !in data )
		conn_log$in_received_from_headers = F;

	if ( conn_log$in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !resp_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			conn_log$received_finished=T;
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && !conn_log$received_finished )
		conn_log$received_finished=T;
	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

@load global-ext
@load smtp

type smtp_ext_session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	first_received: string &default="";
	second_received: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: set[string];
	path: string &default="";
	is_webmail: bool &default=F; # This is not being set yet.
	agent: string &default="";

	# These are used during processing and are likely less useful.
	current_header: string &default="";
	in_received_from_headers: bool &default=F;
	received_finished: bool &default=F;
};

# Define the generic smtp-ext event that can be handled from other scripts
global smtp_ext: event(id: conn_id, si: smtp_ext_session_info);

module SMTP;

export {
	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Direction to capture the full "Received from" path. (from the Hosts enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /spamcop\.net\/bl/
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /rbl\.knology\.net\//
	  | /intercept\.datapacket\.net\// &redef;
}

function default_smtp_ext_session_info(): smtp_ext_session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	local tmp3: set[string] = set();
	return [$rcptto=tmp, $to=tmp2, $files=tmp3];
	}

# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of smtp_ext_session_info &read_expire=4mins;

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	# Throw the event for other scripts to handle
	event smtp_ext(id, conn_log);

	delete conn_info[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_smtp_ext_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_smtp_ext_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			add conn_info[id]$files[data];
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		else if ( conn_log$current_header == "agent" )
			conn_log$agent += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}

	else if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		{
		conn_log$second_received = conn_log$first_received;
                conn_log$first_received = split1(data, /:[[:blank:]]*/)[2];
                conn_log$current_header = "received";

		# Fill in the second value in case there is only one hop in the message.
		if ( conn_log$second_received == "" )
			{
			conn_log$second_received = conn_log$first_received;
			}
		}

	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	else if ( /^[xX]-[mM][aA][iI][lL][eE][rR]:[[:blank:]]/ |
	          /^[uU][sS][eE][rR]-[aA][gG][eE][nN][tT]:[[:blank:]]/ in data )
		{
		conn_log$agent = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "agent";
		}

	}

# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header )
		return;

	local conn_log = conn_info[id];

	# If we've decided that we're done watching the received headers, we're done.
	if ( conn_log$received_finished )
		return;

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		conn_log$in_received_from_headers = T;
	else if ( /^[[:blank:]]/ !in data )
		conn_log$in_received_from_headers = F;

	if ( conn_log$in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !resp_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			conn_log$received_finished=T;
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && !conn_log$received_finished )
		conn_log$received_finished=T;
	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}