Further restriction to not allow hyphens on either side of candidate SSN.

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9\-])\x00?[3-9](\x00?[0-9]){3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9\-]|$)/ &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			local redacted_cc = gsub(ccp, /[0-9]/, redaction_char);

			# we've got a match
			local parts = split_all(data, cc_regex);
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches
					local cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, ccp);

			# Trim the data
			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_redacted_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9])\x00?[3-9](\x00?[0-9]){3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9]|$)/ &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			local redacted_cc = gsub(ccp, /[0-9]/, redaction_char);

			# we've got a match
			local parts = split_all(data, cc_regex);
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches
					local cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, ccp);

			# Trim the data
			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_redacted_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}


Fix an occasional crash and remove a print statement.

 - There was a stray print.
 - Removed a case of modifying a vector while
   iterating over it by taking a copy.  A user was
   seeing rare crashes.

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9\-])\x00?[3-9](\x00?[0-9]){2,3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9\-]|$)/ &redef;

	## Configure this to `F` if you'd like to stop enforcing that
	## credit cards use an internal digit separator.
	const use_cc_separators = T &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = F;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local found_cnt = 0;

	local ccps = find_all(data, cc_regex);
	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");

		if ( (!use_cc_separators || cc_separators in ccp) && luhn_check(ccp) )
			{
			++found_cnt;

			# we've got a match
			local cc_parts = split_string_all(data, cc_regex);
			# take a copy to avoid modifying the vector while iterating.
			for ( i in copy(ccp_parts) )
				{
				if ( i % 2 == 1 )
					{
					# Redact all matches
					local cc_match = cc_parts[i];
					cc_parts[i] = gsub(cc_parts[i], /[0-9]/, "X");
					}
				}
			local redacted_data = join_string_vec(cc_parts, "");

			# Trim the data
			local begin = 0;
			local cc_location = strstr(data, ccp);
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? gsub(ccp, /[0-9]/, "X") : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			}

		}
	if ( found_cnt > 0 )
		{
		NOTICE([$note=CreditCardExposure::Found,
		        $conn=c,
		        $msg=fmt("Found at least %d credit card number%s", found_cnt, found_cnt > 1 ? "s" : ""),
		        $sub=trimmed_redacted_data,
		        $identifier=cat(c$id$orig_h,c$id$resp_h)]);
		return T;
		}
	else
		{
		return F;
		}
	}

# This is used if the signature based technique is in use
#function validate_credit_card_match(state: signature_state, data: string): bool
#	{
#	# TODO: Don't handle HTTP data this way.
#	if ( /^GET/ in data )
#		return F;
#
#	return check_cards(state$conn, data);
#	}

event CreditCardExposure::stream_data(f: fa_file, data: string)
	{
	local c: connection;
	for ( id in f$conns )
		{
		c = f$conns[id];
		break;
		}
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event file_new(f: fa_file)
	{
	if ( f$source == "HTTP" || f$source == "SMTP" )
		{
		Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT,
		                    [$stream_event=CreditCardExposure::stream_data]);
		}
	}

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9\-])\x00?[3-9](\x00?[0-9]){2,3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9\-]|$)/ &redef;

	## Configure this to `F` if you'd like to stop enforcing that
	## credit cards use an internal digit separator.
	const use_cc_separators = T &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = F;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local found_cnt = 0;

	local ccps = find_all(data, cc_regex);
	print ccps;
	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");

		if ( (!use_cc_separators || cc_separators in ccp) && luhn_check(ccp) )
			{
			++found_cnt;

			# we've got a match
			local cc_parts = split_string_all(data, cc_regex);
			for ( i in cc_parts )
				{
				if ( i % 2 == 1 )
					{
					# Redact all matches
					local cc_match = cc_parts[i];
					cc_parts[i] = gsub(cc_parts[i], /[0-9]/, "X");
					}
				}
			local redacted_data = join_string_vec(cc_parts, "");

			# Trim the data
			local begin = 0;
			local cc_location = strstr(data, ccp);
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? gsub(ccp, /[0-9]/, "X") : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			}

		}
	if ( found_cnt > 0 )
		{
		NOTICE([$note=CreditCardExposure::Found,
		        $conn=c,
		        $msg=fmt("Found at least %d credit card number%s", found_cnt, found_cnt > 1 ? "s" : ""),
		        $sub=trimmed_redacted_data,
		        $identifier=cat(c$id$orig_h,c$id$resp_h)]);
		return T;
		}
	else
		{
		return F;
		}
	}

# This is used if the signature based technique is in use
#function validate_credit_card_match(state: signature_state, data: string): bool
#	{
#	# TODO: Don't handle HTTP data this way.
#	if ( /^GET/ in data )
#		return F;
#
#	return check_cards(state$conn, data);
#	}

event CreditCardExposure::stream_data(f: fa_file, data: string)
	{
	local c: connection;
	for ( id in f$conns )
		{
		c = f$conns[id];
		break;
		}
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event file_new(f: fa_file)
	{
	if ( f$source == "HTTP" || f$source == "SMTP" )
		{
		Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT,
		                    [$stream_event=CreditCardExposure::stream_data]);
		}
	}


don't bother to keep track of recipients


Create smtp-subject.bro

Create smtp-subject.bro

# Intel framework support for SMTP subjects
# CrowdStrike 2014
# josh.liburdi@crowdstrike.com

@load base/protocols/smtp
@load base/frameworks/intel
@load policy/frameworks/intel/seen/where-locations

event smtp_request(c: connection, is_orig: bool, command: string, arg: string)
{
if ( c$smtp?$subject )
  Intel::seen([$indicator=c$smtp$subject,
               $indicator_type=Intel::EMAIL_SUBJECT,
               $conn=c,
               $where=SMTP::IN_SUBJECT]);
}

error: pathspec 'bro-scripts/intel-extensions/seen/smtp-subject.bro' did not match any file(s) known to git


Added spamcop.

Added spamcop.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /spamcop\.net\/bl/
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}


update to use new function name

update to use new function name

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logged_hosts: Hosts = LocalHosts &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", All, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !addr_matches_hosts(c$id$resp_h, logged_hosts) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logged_hosts: Hosts = LocalHosts &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", All, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !resp_matches_hosts(c$id$resp_h, logged_hosts) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}


Create __load__.bro

Create __load__.bro

error: pathspec 'bro-scripts/extensions/dns/__load__.bro' did not match any file(s) known to git


connection hostnames logging

connection hostnames logging

add a script that adds the hostnames to conn objects and creates a
conn_hostnames log.  Should possible be two scripts :)

@load base/protocols/conn
@load base/protocols/http
@load base/protocols/ssl
@load base/utils/site

module Conn;

event connection_established(c: connection)
{
    Conn::set_conn(c, F);
}

redef record Conn::Info += {
    resp_hostname: string &optional &log;
};

event http_header (c: connection, is_orig: bool, name: string, value: string)
{
    if(name == "HOST") {
        c$conn$resp_hostname = value;
        print "set hostname", value;
        flush_all();
    }
}

event ssl_established(c: connection)
{
    if(c?$ssl && c$ssl?$server_name) {
        c$conn$resp_hostname = c$ssl$server_name;
        print "set hostname", c$ssl$server_name;
        flush_all();
    }
}

event bro_init()
{
    Log::add_filter(Conn::LOG, [$name = "conn-hostnames",
                                $path = "conn_hostnames",
                                $pred(rec: Conn::Info) = {
        return (rec?$resp_hostname);
    }]);
}

error: pathspec 'conn-hostnames.bro' did not match any file(s) known to git


fix load.bro with new filenames

fix load.bro with new filenames

@load ./master_log.bro
@load ./bro_master.bro
@load ./dio_access.bro
@load ./dio_download_complete.bro
@load ./dio_download_offer.bro
@load ./dio_ftp.bro
@load ./dio_mysql_command.bro
@load ./dio_mysql_login.bro
@load ./dio_smb_bind.bro
@load ./dio_smb_request.bro

@load ./master_log.bro
@load ./bro_master.bro
@load ./dio_access.bro
@load ./dio_download_complete.bro
@load ./dio_download_offer.bro
@load ./dio_ftp.bro
@load ./dio_mysql.bro
@load ./dio_smb_bind.bro
@load ./dio_smb_request.bro


Added the known-services script.

Added the known-services script.

This script will currently log all discovered TCP services that are allowing a full handshake.  You can optionally choose to log remote hosts, local hosts or all hosts services.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load global-ext

module KnownServices;

export {
	const services_log = open_log_file("known-services") &raw_output &redef;

	global established_conns: set[addr, port] &create_expire=1day &redef;
	global known_services: set[addr, port] &create_expire=1day &synchronized &persistent;

	# The hosts whose services should be logged.
	const logged_hosts: Hosts = LocalHosts &redef;

}

event connection_established(c: connection)
	{
	local id = c$id;
	if ( [id$resp_h, id$resp_p] !in established_conns &&
	     ip_matches_hosts(id$resp_h, logged_hosts) )
		add established_conns[id$resp_h, id$resp_p];
	}

event known_services_done(c: connection)
	{
	local id = c$id;
	if ( [id$resp_h, id$resp_p] !in known_services &&
	     [id$resp_h, id$resp_p] in established_conns &&
	     "ftp-data" !in c$service )
		{
		add known_services[id$resp_h, id$resp_p];
		print services_log, cat_sep("\t", "\\N", id$resp_h, fmt("%d", id$resp_p), fmt_str_set(c$service, /\-/));
		}
	}

event connection_state_remove(c: connection)
	{
	event known_services_done(c);
	}

error: pathspec 'known-services.bro' did not match any file(s) known to git


Added the Sophos blacklist.

Added the Sophos blacklist.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /spamhaus\.org\//
	  | /sophos\.com\/security\//
	  | /cbl\.abuseat\.org\//
	  | /sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 1 )
		return ips[2];
	if ( |ips| > 0 )
		return ips[1];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;

	if ( id !in conn_info ||
		 id !in smtp_sessions ||
		 !smtp_sessions[id]$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !smtp_sessions[id]$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}


Small tweaks to try avoiding a problem.

Small tweaks to try avoiding a problem.

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9\-])\x00?[3-9](\x00?[0-9]){3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9\-]|$)/ &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			# we've got a match
			local cc_parts = split_all(data, cc_regex);
			for ( i in cc_parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches
					local cc_match = cc_parts[i];
					cc_parts[i] = gsub(cc_parts[i], /[0-9]/, redaction_char);
					}
				}
			local redacted_data = join_string_array("", cc_parts);

			# Trim the data
			local begin = 0;
			local cc_location = strstr(data, ccp);
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_redacted_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? gsub(ccp, /[0-9]/, redaction_char) : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}

@load ./bin-list

module CreditCardExposure;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		Found
	};

	type Info: record {
		## When the SSN was seen.
		ts:   time    &log;
		## Unique ID for the connection.
		uid:  string  &log;
		## Connection details.
		id:   conn_id &log;
		## Credit card number that was discovered.
		cc:   string  &log &optional;
		## Bank Indentification number information
		bank: Bank    &log &optional;
		## Data that was received when the credit card was discovered.
		data: string  &log;
	};

	## Logs are redacted by default.  If you want to see the credit card
	## numbers in the log, redef this value to F.
	## Notices are automatically and unchangeably redacted.
	const redact_log = T &redef;

	## The character used for redaction to replace all numbers.
	const redaction_char = "X" &redef;

	## The number of bytes around the discovered credit card number that is used
	## as a summary in notices.
	const summary_length = 200 &redef;

	const cc_regex = /(^|[^0-9\-])\x00?[3-9](\x00?[0-9]){3}([[:blank:]\-\.]?\x00?[0-9]{4}){3}([^0-9\-]|$)/ &redef;

	const cc_separators = /\.([0-9]*\.){2}/ |
	                      /\-([0-9]*\-){2}/ |
	                      /[:blank:]([0-9]*[:blank:]){2}/ &redef;
}

const luhn_vector = vector(0,2,4,6,8,1,3,5,7,9);
function luhn_check(val: string): bool
	{
	local sum = 0;
	local odd = T;
	for ( char in gsub(val, /[^0-9]/, "") )
		{
		odd = !odd;
		local digit = to_count(char);
		sum += (odd ? digit : luhn_vector[digit]);
		}
	return sum % 10 == 0;
	}

event bro_init() &priority=5
	{
	Log::create_stream(CreditCardExposure::LOG, [$columns=Info]);
	}

function check_cards(c: connection, data: string): bool
	{
	local ccps = find_all(data, cc_regex);

	for ( ccp in ccps )
		{
		# Remove non digit characters from the beginning and end of string.
		ccp = sub(ccp, /^[^0-9]*/, "");
		ccp = sub(ccp, /[^0-9]*$/, "");
		# Remove any null bytes.
		ccp = gsub(ccp, /\x00/, "");
		if ( cc_separators in ccp && luhn_check(ccp) )
			{
			local redacted_cc = gsub(ccp, /[0-9]/, redaction_char);

			# we've got a match
			local parts = split_all(data, cc_regex);
			for ( i in parts )
				{
				if ( i % 2 == 0 )
					{
					# Redact all matches
					local cc_match = parts[i];
					parts[i] = gsub(parts[i], /[0-9]/, redaction_char);
					}
				}

			local redacted_data = join_string_array("", parts);
			local cc_location = strstr(data, ccp);

			# Trim the data
			local begin = 0;
			if ( cc_location > (summary_length/2) )
				begin = cc_location - (summary_length/2);

			local byte_count = summary_length;
			if ( begin + summary_length > |redacted_data| )
				byte_count = |redacted_data| - begin;

			local trimmed_redacted_data = sub_bytes(redacted_data, begin, byte_count);

			NOTICE([$note=Found,
			        $conn=c,
			        $msg=fmt("Redacted excerpt of disclosed credit card session: %s", trimmed_redacted_data),
			        $identity=cat(c$id$orig_h,c$id$resp_h)]);

			local log: Info = [$ts=network_time(),
			                   $uid=c$uid, $id=c$id,
			                   $cc=(redact_log ? redacted_cc : ccp),
			                   $data=(redact_log ? trimmed_redacted_data : sub_bytes(data, begin, byte_count))];

			local bin_number = to_count(sub_bytes(gsub(ccp, /[^0-9]/, ""), 1, 6));
			if ( bin_number in bin_list )
				log$bank = bin_list[bin_number];

			Log::write(CreditCardExposure::LOG, log);
			return T;
			}
		}
	return F;
	}

event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

event mime_segment_data(c: connection, length: count, data: string)
	{
	if ( c$start_time > network_time()-20secs )
		check_cards(c, data);
	}

# This is used if the signature based technique is in use
function validate_credit_card_match(state: signature_state, data: string): bool
	{
	# TODO: Don't handle HTTP data this way.
	if ( /^GET/ in data )
		return F;

	return check_cards(state$conn, data);
	}