Small fix leftover from debugging.

Small fix leftover from debugging.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

# This is a locally defined event.  Handle it with a priority greater than
# negative 5 if you want to dig into the conn_info record before it's deleted.
function end_smtp_extended_logging(id: conn_id)
	{
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(id);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

# This is a locally defined event.  Handle it with a priority greater than
# negative 5 if you want to dig into the conn_info record before it's deleted.
function end_smtp_extended_logging(id: conn_id)
	{
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s). Subject: %s", loc$country_code, ip, conn_log$subject),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(id);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}


New script for watching for correct headers and order of those headers.  This is very much in testing, do not run in production!

@load http
@load http-request
@load http-ext

module HTTP;

type browser_header_info: record {
	name: string &default="";
	user_agent_regex: pattern;
	required_headers: vector of string;
	headers: vector of string;
	rev_headers: table[string] of int;
};

export {
	# Domains where header order is frequently messed up for various reasons.
	const ignore_header_order_at = /\.facebook\.com$/ |
	                               /\.fbcdn\.net$/ |
	                               /\.apmebf\.com$/ |
	                               /\.qq\.com$/ |
	                               /\.yahoo\.com$/ |
	                               /\.mozilla\.com$/ |
	                               /\.google\.com$/ &redef;

	# This is a set of local proxies (proxies frequently rewrite headers)
	const local_http_proxies: set[addr] &redef;
}

const BROWSER_HEADERS: table[string] of browser_header_info = {
	["IE6"] = record($name = "IE6",
	                 $user_agent_regex = /Mozilla\/.*compatible; MSIE 6/ |
	                                     #/^iTunes\/.*Windows/ | /^Microsoft-CryptoAPI\// |
	                                     /Windows-Update-Agent/,
	                 $required_headers = vector("ACCEPT", "USER-AGENT", "CONNECTION"),
	                 $headers = vector("ACCEPT", "REFERER", "ACCEPT-LANGUAGE", "ACCEPT-ENCODING", "USER-AGENT", "CONNECTION"),
	                 $rev_headers = table([""]=0)),

	["IE7"] = record($name = "IE7",
	                 $user_agent_regex = /Mozilla\/.*compatible; MSIE 7/,
	                 $required_headers = vector("ACCEPT", "UA-CPU", "USER-AGENT", "CONNECTION"),
	                 $headers = vector("ACCEPT", "REFERER", "ACCEPT-LANGUAGE", "UA-CPU", "ACCEPT-ENCODING", "ACCEPT-CHARSET", "IF-MODIFIED-SINCE", "IF-NONE-MATCH", "USER-AGENT", "CONNECTION", "KEEP-ALIVE"),
	                 $rev_headers = table([""]=0)),

	["IE8"] = record($name = "IE8",
	                 $user_agent_regex = /Mozilla\/.*MSIE 8/ |
	                                     /Mozilla\/.*compatible; MSIE 7.*Trident\/4\.0/,
	                 $required_headers = vector("ACCEPT", "USER-AGENT", "UA-CPU",  "HOST", "CONNECTION"),
	                 $headers = vector("ACCEPT", "REFERER", "ACCEPT-LANGUAGE", "USER-AGENT", "UA-CPU", "ACCEPT-ENCODING", "HOST", "CONNECTION", "COOKIE"),
	                 $rev_headers = table([""]=0)),

	["MSOffice"] = record($name = "MSOffice",
	                 $user_agent_regex = /MSOffice/,
	                 $required_headers = vector("ACCEPT", "USER-AGENT", "UA-CPU", "CONNECTION"),
	                 $headers = vector("ACCEPT", "REFERER", "ACCEPT-LANGUAGE", "USER-AGENT", "UA-CPU", "ACCEPT-ENCODING", "CONNECTION", "COOKIE"),
	                 $rev_headers = table([""]=0)),

	["FIREFOX"] = record($name = "FIREFOX",
	                     $user_agent_regex = /Gecko\/.*(Firefox|Thunderbird|Netscape)\// |
	                                         /^mozbar [0-9\.]* xpi/,
	                     $required_headers = vector("USER-AGENT", "ACCEPT", "ACCEPT-LANGUAGE", "ACCEPT-CHARSET", "CONNECTION"),
	                     $headers = vector("HOST", "USER-AGENT", "ACCEPT", "ACCEPT-LANGUAGE", "ACCEPT-ENCODING", "ACCEPT-CHARSET", "CONTENT-TYPE", "REFERER", "CONTENT-LENGTH", "COOKIE", "RANGE", "CONNECTION"),
	                     $rev_headers = table([""]=0)),

	["WEBKIT_OSX_<=312"] = record($name="WEBKIT_OSX_<=312",
	                    $user_agent_regex = /(PPC|Intel) Mac OS X;.*Safari\//,
	                    $required_headers = vector("HOST", "CONNECTION", "USER-AGENT", "ACCEPT", "ACCEPT-LANGUAGE"),
	                    $headers = vector("HOST", "CONNECTION", "REFERER", "USER-AGENT", "IF-MODIFIED-SINCE", "ACCEPT", "ACCEPT-ENCODING", "ACCEPT-LANGUAGE", "COOKIE"),
	                    $rev_headers = table([""]=0)),

	["WEBKIT_OSX_PPC"] = record($name = "WEBKIT_OSX_PPC",
	                    $user_agent_regex = /PPC Mac OS X.*AppleWebKit\/.*(Safari\/)?/,
	                    $required_headers = vector("HOST", "CONNECTION", "USER-AGENT", "ACCEPT", "ACCEPT-LANGUAGE"),
	                    $headers = vector("HOST", "CONNECTION", "REFERER", "USER-AGENT", "ACCEPT", "ACCEPT-ENCODING", "ACCEPT-LANGUAGE"),
	                    $rev_headers = table([""]=0)),

	# ACCEPT was removed as a header because it is put in two different locations at different times.
	["WEBKIT_OSX_10.4"] = record($name = "WEBKIT_OSX_10.4",
	                    $user_agent_regex = /^AppleSyndication/ |
	                                        /Mac OS X.*AppleWebKit\/.*(Safari\/)?/,
	                    $required_headers = vector("ACCEPT-LANGUAGE", "ACCEPT-ENCODING", "USER-AGENT", "CONNECTION"),
	                    $headers = vector("ACCEPT-LANGUAGE", "ACCEPT-ENCODING", "COOKIE", "REFERER", "USER-AGENT", "CONNECTION"),
	                    $rev_headers = table([""]=0)),

	["WEBKIT_OSX_10.5"] = record($name = "WEBKIT_OSX_10.5",
	                    $user_agent_regex = /^Apple-PubSub/ |
	                                        /CFNetwork\/.*Darwin\// |
	                                        /(Windows|Mac OS X|iPhone OS).*AppleWebKit\/.*(Safari\/)?/,
	                    $required_headers = vector("USER-AGENT", "ACCEPT", "ACCEPT-LANGUAGE", "CONNECTION"),
	                    $headers = vector("USER-AGENT", "REFERER", "ACCEPT", "ACCEPT-LANGUAGE", "COOKIE", "CONNECTION"),
	                    $rev_headers = table([""]=0)),

	["CHROME_<4.0"] = record($name = "CHROME_<4.0",
	                    $user_agent_regex = /Chrome\/.*Safari\//,
	                    $required_headers = vector("USER-AGENT", "ACCEPT-LANGUAGE", "ACCEPT-CHARSET", "HOST", "CONNECTION"),
	                    $headers = vector("USER-AGENT", "REFERER", "CONTENT-LENGTH", "CONTENT-TYPE", "ACCEPT", "RANGE", "COOKIE", "ACCEPT-LANGUAGE", "ACCEPT-CHARSET", "HOST", "CONNECTION"),
	                    $rev_headers = table([""]=0)),

	["CHROME_>=4.0"] = record($name = "CHROME_>=4.0",
	                    $user_agent_regex = /Chrome\/.*Safari\//,
	                    $required_headers = vector("HOST", "CONNECTION", "USER-AGENT", "ACCEPT", "ACCEPT-ENCODING", "ACCEPT-LANGUAGE", "ACCEPT-CHARSET"),
	                    $headers = vector("HOST", "CONNECTION", "USER-AGENT", "REFERER", "CONTENT-LENGTH", "CONTENT-TYPE", "ACCEPT", "RANGE", "ACCEPT-ENCODING", "COOKIE", "ACCEPT-LANGUAGE", "ACCEPT-CHARSET"),
	                    $rev_headers = table([""]=0)),

	["FLASH"] = record($name = "FLASH",
	                   $user_agent_regex = /blah... nothing matches/,
	                   $required_headers = vector("ACCEPT", "ACCEPT-LANGUAGE", "REFERER", "X-FLASH-VERSION", "ACCEPT-ENCODING", "USER-AGENT", "COOKIE", "CONNECTION"),
	                   $headers = vector("ACCEPT", "ACCEPT-LANGUAGE", "REFERER", "X-FLASH-VERSION", "ACCEPT-ENCODING", "USER-AGENT", "COOKIE", "CONNECTION", "HOST"),
	                   $rev_headers = table([""]=0)),
};

# Generate all of the reverse header tables.
event bro_init()
	{
	for ( browser_name in BROWSER_HEADERS )
		{
		local browser = BROWSER_HEADERS[browser_name];
		delete browser$rev_headers[""];
		for ( i in browser$headers )
			{
			browser$rev_headers[browser$headers[i]] = i;
			}
		}
	}

const ordered_headers: set[string] = {
	"HOST",
	"USER-AGENT",
	"ACCEPT",
	"ACCEPT-LANGUAGE",
	"ACCEPT-ENCODING",
	"ACCEPT-CHARSET",
	"KEEP-ALIVE",
	"CONNECTION",
	"CONTENT-TYPE",
	"REFERER",
	"CONTENT-LENGTH",
	"COOKIE",
	"RANGE",
	"UA-CPU",
	"X-FLASH-VERSION",
};

type header_tracker: record {
	ua_identified: set[string];
	identified: set[string];
	broken: set[string];
	possibles: table[string] of count;
};

global tracking_headers: table[conn_id] of header_tracker &read_expire=30secs;
global recently_examined: set[addr] &create_expire=30secs &redef;

event http_header(c: connection, is_orig: bool, name: string, value: string)
	{
	if ( !is_orig ||
	     name !in ordered_headers )
		return;

	if ( !is_local_addr(c$id$orig_h) ||
	     #c$id$orig_h in recently_examined ||
	     c$id$orig_h in local_http_proxies )
		return;

	local header = name;
	if ( c$id !in tracking_headers )
		{
		tracking_headers[c$id] = [$ua_identified=set(""),
		                          $identified=set(""),
		                          $broken=set(""),
		                          $possibles=table(["IE6"]=0, ["IE7"]=0, ["IE8"]=0, ["MSOffice"]=0, ["FIREFOX"]=0, ["WEBKIT_OSX_PPC"]=0, ["WEBKIT_OSX_10.4"]=0, ["WEBKIT_OSX_10.5"]=0, ["CHROME_<4.0"]=0, ["CHROME_>=4.0"]=0, ["FLASH"]=0)];
		# FIXME: this is a hack because set("") above needed an empty element for some reason.
		delete tracking_headers[c$id]$identified[""];
		delete tracking_headers[c$id]$ua_identified[""];
		delete tracking_headers[c$id]$broken[""];
		}

	local ht = tracking_headers[c$id];

	#print fmt("CHECKING HEADER: %s", header);
	for ( browser_name in BROWSER_HEADERS )
		{
		if ( header == "USER-AGENT" )
			{
			if ( BROWSER_HEADERS[browser_name]$user_agent_regex in value )
				add ht$ua_identified[browser_name];
			}

		if ( browser_name !in ht$identified )
			{
			local browser = BROWSER_HEADERS[browser_name];

			if ( browser_name in ht$possibles &&
				 header in browser$rev_headers )
				{
				local possible_browser = ht$possibles[browser_name]; # count
				local browser_rev_headers = browser$rev_headers; # table[string] of int
				local h_position = browser_rev_headers[header]; # count
				local req_headers = browser$required_headers; # vector of string
				local next_required_header = req_headers[possible_browser+1]; # string
				local current_header_val = req_headers[h_position]; # string
				#print fmt("for browser: %s :: checking header: %s :: req position: %d :: next required: %s :: len of req headers: %d", browser_name, header, ht$possibles[browser_name], req_headers[ht$possibles[browser_name]+1], |req_headers|);

				if ( next_required_header == header )
					{
					++ht$possibles[browser_name];
					}

				else if ( possible_browser == 0 || possible_browser == |req_headers| ||
				          (browser_rev_headers[req_headers[possible_browser]] < h_position &&
				           h_position < browser_rev_headers[next_required_header]) )
					{
					#print fmt("%s is an optional header for %s (but it is in the correct position).", header, browser_name);
					}
				else
					{
					delete ht$possibles[browser_name];
					}

				# Have we found a browser yet?
				if ( browser_name in ht$possibles &&
					 ht$possibles[browser_name] == |req_headers| )
					{
					add ht$identified[browser_name];
					}
				}
			}
		}
	}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=-10
	{
	if ( id in tracking_headers &&
	     id$orig_h !in local_http_proxies &&
	     si$proxied_for == "" )
		{
		add recently_examined[id$orig_h];

		if ( ignore_header_order_at in si$host )
			{
			#print "we're going to ignore this entire request.";
			return;
			}

		local is_matched = F;
		#if ( |tracking_headers[id]$identified| > 0 )
		#	{
 		#	is_matched = F;
		#	for ( b in tracking_headers[id]$identified )
		#		{
		#		if ( BROWSER_HEADERS[b]$user_agent_regex in si$user_agent )
		#			{
		#			is_matched = T;
		#			}
		#		}
		#	if ( !is_matched )
		#		{
		#		#print fmt("Headers look like %s, but User-Agent doesn't match.", fmt_str_set(tracking_headers[id]$identified, /blah/));
		#		print cat_sep("\t", "\\N",
		#		              si$start_time,
		#		              id$orig_h, port_to_count(id$orig_p),
		#		              id$resp_h, port_to_count(id$resp_p),
		#		              fmt_str_set(si$force_log_reasons, /DONTMATCH/),
		#		              si$method, si$url, si$referrer,
		#		              si$user_agent, si$proxied_for);
		#		}
		#	}

		# Do this in case the User-Agent is known, but the headers don't match it.
		is_matched=F;
		for ( b in tracking_headers[id]$ua_identified )
			{
			if ( b in tracking_headers[id]$identified )
				{
				is_matched=T;
				}
			}
		if ( |tracking_headers[id]$ua_identified| > 0 && !is_matched )
			{
			print fmt("User-Agent looks like %s, but headers look like %s.", fmt_str_set(tracking_headers[id]$ua_identified, /blah/), fmt_str_set(tracking_headers[id]$identified, /blah/));
			print cat_sep(" :: ", "\\N",
			              si$start_time,
			              id$orig_h, port_to_count(id$orig_p),
			              id$resp_h, port_to_count(id$resp_p),
			              fmt_str_set(si$force_log_reasons, /DONTMATCH/),
			              si$method, si$url, si$referrer,
			              si$user_agent, si$proxied_for);

			}
		}
	delete tracking_headers[id];
	}

error: pathspec 'testing/http-watch-header-order.bro' did not match any file(s) known to git


Fixed an issue with the smtp_ext event.  It has finally been tested.

@load smtp
@load global-ext

type smtp_ext_session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /spamcop\.net\/bl/
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /rbl\.knology\.net\//
	  | /intercept\.datapacket\.net\// &redef;
}

# Define the generic smtp-ext event that can be handled from other scripts
global smtp_ext: event(id: conn_id, cl: smtp_ext_session_info);

function default_smtp_ext_session_info(): smtp_ext_session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of smtp_ext_session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);

	event smtp_ext(id, conn_log);

	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_smtp_ext_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_smtp_ext_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /spamcop\.net\/bl/
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /rbl\.knology\.net\//
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

# Define the generic smtp-ext event that can be handled from other scripts
global smtp_ext: event(id: conn_id, cl: session_info);

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);

	event smtp_ext(id, conn_log);

	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}


update to use new logging framework

update to use new logging framework

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logging = Outbound &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", logging, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !orig_matches_direction(c$id$resp_h, logging) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	const log = open_log_file("ssl-known-certs") &raw_output &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const logged_hosts: Hosts = LocalHosts &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !resp_matches_hosts(c$id$resp_h, logged_hosts) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}


bug fix: bro file extraction, add: bro owasp beginnings

Former-commit-id: 97415935f543cbe81ebae23c2bf2c7a0d449e1a0

##! Extract exe, pdf, jar and cab files.

global ext_map: table[string] of string = {
    ["application/x-dosexec"] = "exe",
    ["application/pdf"] = "pdf",
    ["application/zip"] = "zip",
    ["application/jar"] = "jar",
    ["application/vnd.ms-cab-compressed"] = "cab",
    ["text/plain"] = "txt",
    ["image/gif"] = "gif",
    ["image/jpeg"] = "jpg",
    ["image/png"] = "png",
    ["text/html"] = "html",
    ["application/vnd.ms-fontobject"] = "ms_font",
    ["application/x-shockwave-flash"] = "swf"
} &default ="unknown";

event file_new(f: fa_file)
    {
    if (!f?$mime_type) return;
    local ext = ext_map[f$mime_type];
    if (ext != "txt" && ext != "html" && ext != "ms_font" && ext != "jpg" && ext != "gif" && ext != "png")
        {
        local fname = fmt("%s-%s.%s", f$source, f$id, ext);
        Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
        }
    }

##! Extract exe, pdf, jar and cab files.

global ext_map: table[string] of string = {
    ["application/x-dosexec"] = "exe",
    ["application/pdf"] = "pdf",
    ["application/zip"] = "zip",
    ["application/jar"] = "jar",
    ["application/vnd.ms-cab-compressed"] = "cab",
    ["text/plain"] = "txt",
    ["image/gif"] = "gif",
    ["image/jpeg"] = "jpg",
    ["image/png"] = "png",
    ["text/html"] = "html",
    ["application/vnd.ms-fontobject"] = "ms_font",
    ["application/x-shockwave-flash"] = "swf"
} &default ="unknown";

event file_new(f: fa_file)
    {
    local ext = ext_map[f$mime_type];
    if (ext != "txt" && ext != "html" && ext != "ms_font" && ext != "jpg" && ext != "gif" && ext != "png")
        {
        local fname = fmt("%s-%s.%s", f$source, f$id, ext);
        Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
        }
    }


Removed a comment

Removed a comment

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

# This is a locally defined event.  Handle it with a priority greater than
# negative 5 if you want to dig into the conn_info record before it's deleted.
function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}


don't bother to keep track of recipients

# For this script to work, you must define a local_mail table
# listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };

@ifdef ( local_mail )

@load conn-id
@load smtp

module SMTP;

type smtp_counter: record {
 rejects: count;
 total: count;
 connects: set[conn_id];
 rej_conns: set[conn_id];
};

export {
 # The idea for this is that if a host makes more than spam_threshold
 # smtp connections per hour of which at least spam_percent of those are
 # rejected and the host is not a known mail sending host, then it's likely
 # sending spam or viruses.
 #
 const spam_threshold = 300 &redef;
 const spam_percent = 30 &redef;

 # These are smtp status codes that are considered "rejected".
 const smtp_reject_codes: set[count] = {
   501, # Bad sender address syntax
   550, # Requested action not taken: mailbox unavailable
   551, # User not local; please try <forward-path>
   553, # Requested action not taken: mailbox name not allowed
   554, # Transaction failed
 };

 redef enum Notice += {
   SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
   SMTP_PossibleInternalSpam, # Internal host seems to be spamming
   SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
 };

 global smtp_status_comparison: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
		             msg: string, cont_resp: bool)
{
 if ( c$id$orig_h !in smtp_status_comparison )
 {
   local bar: set[conn_id];
   local blarg: set[conn_id];
   smtp_status_comparison[c$id$orig_h] = [$rejects=0, $total=0, $connects=bar, $rej_conns=blarg];
 }

 # Set the smtp_counter to the local var "foo"
 local foo = smtp_status_comparison[c$id$orig_h];

 if ( code in smtp_reject_codes &&
           c$id !in foo$rej_conns )
 {
   ++foo$rejects;
   local session = smtp_sessions[c$id];
   add foo$rej_conns[c$id];
 }

 if ( c$id !in foo$connects )
 {
   ++foo$total;
   add foo$connects[c$id];
 }

 local host = c$id$orig_h;
 if ( foo$total >= spam_threshold ) {
   local percent = (foo$rejects*100) / foo$total;
   if ( percent >= spam_percent ) {
     if ( host in local_mail )
       NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("%s is rejecting a high percentage of mail", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else if ( is_local_addr(host) )
       NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else
       NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
   }
 }
}

@endif

# For this script to work, you must define a local_mail table
# listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };

@ifdef ( local_mail )

@load conn-id
@load smtp

module SMTP;

type smtp_counter: record {
 rejects: count;
 total: count;
 connects: set[conn_id];
 rej_conns: set[conn_id];
 recipients: string;
};

export {
 # The idea for this is that if a host makes more than spam_threshold
 # smtp connections per hour of which at least spam_percent of those are
 # rejected and the host is not a known mail sending host, then it's likely
 # sending spam or viruses.
 #
 const spam_threshold = 300 &redef;
 const spam_percent = 30 &redef;

 # These are smtp status codes that are considered "rejected".
 const smtp_reject_codes: set[count] = {
   501, # Bad sender address syntax
   550, # Requested action not taken: mailbox unavailable
   551, # User not local; please try <forward-path>
   553, # Requested action not taken: mailbox name not allowed
   554, # Transaction failed
 };

 redef enum Notice += {
   SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
   SMTP_PossibleInternalSpam, # Internal host seems to be spamming
   SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
 };

 global smtp_status_comparison: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
		             msg: string, cont_resp: bool)
{
 if ( c$id$orig_h !in smtp_status_comparison )
 {
   local bar: set[conn_id];
   local blarg: set[conn_id];
   smtp_status_comparison[c$id$orig_h] = [$rejects=0, $total=0, $connects=bar, $rej_conns=blarg, $recipients=""];
 }

 # Set the smtp_counter to the local var "foo"
 local foo = smtp_status_comparison[c$id$orig_h];

 if ( code in smtp_reject_codes &&
           c$id !in foo$rej_conns )
 {
   ++foo$rejects;
   local session = smtp_sessions[c$id];
   if( foo$recipients == "" )
   {
     foo$recipients = session$recipients;
   } else if (session$recipients != "") {
     foo$recipients = cat(foo$recipients, ",", session$recipients);
   }
   add foo$rej_conns[c$id];
 }

 if ( c$id !in foo$connects )
 {
   ++foo$total;
   add foo$connects[c$id];
 }

 local host = c$id$orig_h;
 if ( foo$total >= spam_threshold ) {
   local percent = (foo$rejects*100) / foo$total;
   if ( percent >= spam_percent ) {
     if ( host in local_mail )
       NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("%s is rejecting a high percentage of mail", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else if ( is_local_addr(host) )
       NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent mailto: %s", foo$total, percent, foo$recipients), $conn=c]);
     else
       NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
   }
 }
}

@endif


fix regex

fix regex

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)albany\.edu($|:)/ &redef;
}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=10
{
    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];
    }
}

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)albany\.edu$/ &redef;
}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=10
{
    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];
    }
}