Work on IP address support functions.

Work on IP address support functions.

Reworked is_valid_ip function to be more accurate.
ip_valid_ip has now started to support IPv6 addresses.
New function for extracting all discovered IP addresses from strings.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

function numeric_id_string(id: conn_id): string
	{
	return fmt("%s:%d > %s:%d",
	           id$orig_h, id$orig_p,
	           id$resp_h, id$resp_p);
	}

function fmt_str_set(input: string_set, strip: pattern): string
	{
	local output = "{";
	local tmp = "";
	local len = length(input);
	local i = 1;

	for ( item in input )
		{
		tmp = fmt("%s", gsub(item, strip, ""));
		if ( len != i )
			tmp = fmt("%s, ", tmp);
		i = i+1;
		output = fmt("%s%s", output, tmp);
		}
	return fmt("%s}", output);
	}

# Regular expressions for matching IP addresses in strings.
const ipv4_addr_regex = /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/;
const ipv6_8hex_regex = /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/;
const ipv6_compressed_hex_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/;
const ipv6_hex4dec_regex = /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;
const ipv6_compressed_hex4dec_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;

# These are only commented out until this bug is fixed:
#    http://www.bro-ids.org/wiki/index.php/Known_Issues#Bug_with_OR-ing_together_pattern_variables
#const ipv6_addr_regex = ipv6_8hex_regex |
#                        ipv6_compressed_hex_regex |
#                        ipv6_hex4dec_regex |
#                        ipv6_compressed_hex4dec_regex;
#const ip_addr_regex = ipv4_addr_regex | ipv6_addr_regex;

const ipv6_addr_regex =
    /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/ |
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/ | # IPv6 Compressed Hex
    /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec

const ip_addr_regex =
    /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/ |
    /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/ |
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/ | # IPv6 Compressed Hex
    /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec

function is_valid_ip(ip_str: string): bool
	{
	if ( ip_str == ipv4_addr_regex )
		{
		local octets = split(ip_str, /\./);
		if ( |octets| > 4 )
			return F;

		local num=0;
		for ( i in octets )
			{
			num = to_count(octets[i]);
			if ( num < 0 || 255 < num )
				return F;
			}
		return T;
		}
	else if ( ip_str == ipv6_addr_regex )
		{
		# TODO: make this work correctly.
		return T;
		}
	return F;
	}

# This output a string_array of ip addresses extracted from a string.
# given: "this is 1.1.1.1 a test 2.2.2.2 string with ip addresses 3.3.3.3"
# outputs: { [1] = 1.1.1.1, [2] = 2.2.2.2, [3] = 3.3.3.3 }
function find_ip_addresses(input: string): string_array
	{
	local parts = split_all(input, ip_addr_regex);
	local output: string_array;

	for ( i in parts )
		{
		if ( i % 2 == 0 && is_valid_ip(parts[i]) )
			output[i/2] = parts[i];
		}
	return output;
	}

# Some enums for deciding what and when to log.
type Direction: enum { Inbound, Outbound, BiDirectional };
type Hosts: enum { LocalHosts, RemoteHosts, AllHosts };

function orig_h_matches_direction(ip: addr, d: Direction): bool
	{
	return ( (d == Outbound && is_local_addr(ip)) ||
	         (d == Inbound && !is_local_addr(ip)) ||
	         d == BiDirectional );
	}
function conn_matches_direction(id: conn_id, d: Direction): bool
	{
	return orig_h_matches_direction(id$orig_h, d);
	}
function addr_matches_hosts(ip: addr, d: Hosts): bool
	{
	return ( (d == LocalHosts && is_local_addr(ip)) ||
	         (d == RemoteHosts && !is_local_addr(ip)) ||
	         d == AllHosts );
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

function numeric_id_string(id: conn_id): string
	{
	return fmt("%s:%d > %s:%d",
	           id$orig_h, id$orig_p,
	           id$resp_h, id$resp_p);
	}

function fmt_str_set(input: string_set, strip: pattern): string
	{
	local output = "{";
	local tmp = "";
	local len = length(input);
	local i = 1;

	for ( item in input )
		{
		tmp = fmt("%s", gsub(item, strip, ""));
		if ( len != i )
			tmp = fmt("%s, ", tmp);
		i = i+1;
		output = fmt("%s%s", output, tmp);
		}
	return fmt("%s}", output);
	}

# TODO: include IPv6 regexes
const ip_addr_regex = /^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$/;
function is_valid_ip(ip_str: string): bool
	{
	return (ip_str == ip_addr_regex);
	}

# Some enums for deciding what and when to log.
type Direction: enum { Inbound, Outbound, BiDirectional };
type Hosts: enum { LocalHosts, RemoteHosts, AllHosts };

function orig_h_matches_direction(ip: addr, d: Direction): bool
	{
	return ( (d == Outbound && is_local_addr(ip)) ||
	         (d == Inbound && !is_local_addr(ip)) ||
	         d == BiDirectional );
	}
function conn_matches_direction(id: conn_id, d: Direction): bool
	{
	return orig_h_matches_direction(id$orig_h, d);
	}
function addr_matches_hosts(ip: addr, d: Hosts): bool
	{
	return ( (d == LocalHosts && is_local_addr(ip)) ||
	         (d == RemoteHosts && !is_local_addr(ip)) ||
	         d == AllHosts );
	}


filter mdns log in bro

filter mdns log in bro

##! Local site policy. Customize as appropriate.
##!
##! This file will not be overwritten when upgrading or reinstalling!

redef ignore_checksums = T;
redef SSL::disable_analyzer_after_detection = F;

#@load site/sqlite.bro

# This script logs which scripts were loaded during each run.
@load misc/loaded-scripts

# Apply the default tuning scripts for common tuning settings.
@load tuning/defaults
@load tuning/json-logs

# Load the scan detection script.
@load misc/scan

# Log some information about web applications being used by users
# on your network.
@load misc/app-stats

# Detect traceroute being run on the network.
@load misc/detect-traceroute

# Generate notices when vulnerable versions of software are discovered.
# The default is to only monitor software found in the address space defined
# as "local".  Refer to the software framework's documentation for more
# information.
@load frameworks/software/vulnerable

# Detect software changing (e.g. attacker installing hacked SSHD).
@load frameworks/software/version-changes
@load frameworks/software/windows-version-detection

# This adds signatures to detect cleartext forward and reverse windows shells.
@load-sigs frameworks/signatures/detect-windows-shells

# Load all of the scripts that detect software in various protocols.
@load protocols/ftp/software
@load protocols/smtp/software
@load protocols/ssh/software
@load protocols/http/software
# The detect-webapps script could possibly cause performance trouble when
# running on live traffic.  Enable it cautiously.
#@load protocols/http/detect-webapps

# This script detects DNS results pointing toward your Site::local_nets
# where the name is not part of your local DNS zone and is being hosted
# externally.  Requires that the Site::local_zones variable is defined.
@load protocols/dns/detect-external-names

# Script to detect various activity in FTP sessions.
@load protocols/ftp/detect

# Scripts that do asset tracking.
@load protocols/conn/known-hosts
@load protocols/conn/known-services
@load protocols/ssl/known-certs

# This script enables SSL/TLS certificate validation.
@load protocols/ssl/validate-certs

# This script prevents the logging of SSL CA certificates in x509.log
@load protocols/ssl/log-hostcerts-only

# Uncomment the following line to check each SSL certificate hash against the ICSI
# certificate notary service; see http://notary.icsi.berkeley.edu .
# @load protocols/ssl/notary

# If you have libGeoIP support built in, do some geographic detections and
# logging for SSH traffic.
@load protocols/ssh/geo-data
# Detect hosts doing SSH bruteforce attacks.
@load protocols/ssh/detect-bruteforcing
# Detect logins using "interesting" hostnames.
@load protocols/ssh/interesting-hostnames

# Detect SQL injection attacks.
@load protocols/http/detect-sqli

#### Network File Handling ####

# Enable MD5 and SHA1 hashing for all files.
@load frameworks/files/hash-all-files

# Detect SHA1 sums in Team Cymru's Malware Hash Registry.
@load frameworks/files/detect-MHR

# Uncomment the following line to enable detection of the heartbleed attack. Enabling
# this might impact performance a bit.
@load policy/protocols/ssl/heartbleed

# enable link-layer address information to connection logs
#@load policy/protocols/conn/mac-logging

redef restrict_filters += [["not-mdns"] = "not port 5353"];

redef SSL::disable_analyzer_after_detection = F;
redef Communication::listen_interface = 127.0.0.1;

@load base/protocols/dhcp

##! Local site policy. Customize as appropriate.
##!
##! This file will not be overwritten when upgrading or reinstalling!

redef ignore_checksums = T;
redef SSL::disable_analyzer_after_detection = F;

#@load site/sqlite.bro

# This script logs which scripts were loaded during each run.
@load misc/loaded-scripts

# Apply the default tuning scripts for common tuning settings.
@load tuning/defaults
@load tuning/json-logs

# Load the scan detection script.
@load misc/scan

# Log some information about web applications being used by users
# on your network.
@load misc/app-stats

# Detect traceroute being run on the network.
@load misc/detect-traceroute

# Generate notices when vulnerable versions of software are discovered.
# The default is to only monitor software found in the address space defined
# as "local".  Refer to the software framework's documentation for more
# information.
@load frameworks/software/vulnerable

# Detect software changing (e.g. attacker installing hacked SSHD).
@load frameworks/software/version-changes
@load frameworks/software/windows-version-detection

# This adds signatures to detect cleartext forward and reverse windows shells.
@load-sigs frameworks/signatures/detect-windows-shells

# Load all of the scripts that detect software in various protocols.
@load protocols/ftp/software
@load protocols/smtp/software
@load protocols/ssh/software
@load protocols/http/software
# The detect-webapps script could possibly cause performance trouble when
# running on live traffic.  Enable it cautiously.
#@load protocols/http/detect-webapps

# This script detects DNS results pointing toward your Site::local_nets
# where the name is not part of your local DNS zone and is being hosted
# externally.  Requires that the Site::local_zones variable is defined.
@load protocols/dns/detect-external-names

# Script to detect various activity in FTP sessions.
@load protocols/ftp/detect

# Scripts that do asset tracking.
@load protocols/conn/known-hosts
@load protocols/conn/known-services
@load protocols/ssl/known-certs

# This script enables SSL/TLS certificate validation.
@load protocols/ssl/validate-certs

# This script prevents the logging of SSL CA certificates in x509.log
@load protocols/ssl/log-hostcerts-only

# Uncomment the following line to check each SSL certificate hash against the ICSI
# certificate notary service; see http://notary.icsi.berkeley.edu .
# @load protocols/ssl/notary

# If you have libGeoIP support built in, do some geographic detections and
# logging for SSH traffic.
@load protocols/ssh/geo-data
# Detect hosts doing SSH bruteforce attacks.
@load protocols/ssh/detect-bruteforcing
# Detect logins using "interesting" hostnames.
@load protocols/ssh/interesting-hostnames

# Detect SQL injection attacks.
@load protocols/http/detect-sqli

#### Network File Handling ####

# Enable MD5 and SHA1 hashing for all files.
@load frameworks/files/hash-all-files

# Detect SHA1 sums in Team Cymru's Malware Hash Registry.
@load frameworks/files/detect-MHR

# Uncomment the following line to enable detection of the heartbleed attack. Enabling
# this might impact performance a bit.
@load policy/protocols/ssl/heartbleed

# enable link-layer address information to connection logs
#@load policy/protocols/conn/mac-logging

redef SSL::disable_analyzer_after_detection = F;
redef Communication::listen_interface = 127.0.0.1;

@load base/protocols/dhcp


initial policy to log http requests to hosts not on your domain

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)albany\.edu$/ &redef;
}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=10
{
    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];
    }
}

error: pathspec 'http-ext-external-names.bro' did not match any file(s) known to git


Short circuit in the simple case of being interested in everything.  This avoids running is_local_addr *a lot* of times.

function numeric_id_string(id: conn_id): string
	{
	return fmt("%s:%d > %s:%d",
	           id$orig_h, id$orig_p,
	           id$resp_h, id$resp_p);
	}

function fmt_addr_set(input: addr_set): string
	{
	local output = "";
	local tmp = "";
	local len = length(input);
	local i = 1;

	for ( item in input )
		{
		tmp = fmt("%s", item);
		if ( len != i )
			tmp = fmt("%s ", tmp);
		i = i+1;
		output = fmt("%s%s", output, tmp);
		}
	return fmt("%s", output);
	}

function fmt_str_set(input: string_set, strip: pattern): string
	{
	local len = length(input);
	if ( len == 0 )
		return "{}";

	local output = "{";
	local tmp = "";
	local i = 1;

	for ( item in input )
		{
		tmp = fmt("%s", gsub(item, strip, ""));
		if ( len != i )
			tmp = fmt("%s, ", tmp);
		i = i+1;
		output = fmt("%s%s", output, tmp);
		}
	return fmt("%s}", output);
	}

# Regular expressions for matching IP addresses in strings.
const ipv4_addr_regex = /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/;
const ipv6_8hex_regex = /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/;
const ipv6_compressed_hex_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/;
const ipv6_hex4dec_regex = /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;
const ipv6_compressed_hex4dec_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;

# These are only commented out until this bug is fixed:
#    http://www.bro-ids.org/wiki/index.php/Known_Issues#Bug_with_OR-ing_together_pattern_variables
#const ipv6_addr_regex = ipv6_8hex_regex |
#                        ipv6_compressed_hex_regex |
#                        ipv6_hex4dec_regex |
#                        ipv6_compressed_hex4dec_regex;
#const ip_addr_regex = ipv4_addr_regex | ipv6_addr_regex;

const ipv6_addr_regex =
    /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/ |
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/ | # IPv6 Compressed Hex
    /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec

const ip_addr_regex =
    /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/ |
    /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/ |
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/ | # IPv6 Compressed Hex
    /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec

function is_valid_ip(ip_str: string): bool
	{
	if ( ip_str == ipv4_addr_regex )
		{
		local octets = split(ip_str, /\./);
		if ( |octets| != 4 )
			return F;

		local num=0;
		for ( i in octets )
			{
			num = to_count(octets[i]);
			if ( num < 0 || 255 < num )
				return F;
			}
		return T;
		}
	else if ( ip_str == ipv6_addr_regex )
		{
		# TODO: make this work correctly.
		return T;
		}
	return F;
	}

# This output a string_array of ip addresses extracted from a string.
# given: "this is 1.1.1.1 a test 2.2.2.2 string with ip addresses 3.3.3.3"
# outputs: { [1] = 1.1.1.1, [2] = 2.2.2.2, [3] = 3.3.3.3 }
function find_ip_addresses(input: string): string_array
	{
	local parts = split_all(input, ip_addr_regex);
	local output: string_array;

	for ( i in parts )
		{
		if ( i % 2 == 0 && is_valid_ip(parts[i]) )
			output[i/2] = parts[i];
		}
	return output;
	}

# Some enums for deciding what and when to log.
type Direction: enum { Inbound, Outbound, All };
type Hosts: enum { LocalHosts, RemoteHosts, AllHosts };

function orig_matches_direction(ip: addr, d: Direction): bool
	{
	if ( d == None ) return F;

	return ( d == All ||
	         (d == Outbound && is_local_addr(ip)) ||
	         (d == Inbound && !is_local_addr(ip)) );
	}

function resp_matches_direction(ip: addr, d: Direction): bool
	{
	if ( d == None ) return F;

	return ( d == All ||
	         (d == Inbound && is_local_addr(ip)) ||
	         (d == Outbound && !is_local_addr(ip)) );
	}

function conn_matches_direction(id: conn_id, d: Direction): bool
	{
	if ( d == None ) return F;

	return orig_matches_direction(id$orig_h, d);
	}

function resp_matches_hosts(ip: addr, d: Hosts): bool
	{
	return ( d == AllHosts ||
	         (d == LocalHosts && is_local_addr(ip)) ||
	         (d == RemoteHosts && !is_local_addr(ip)) );
	}

function numeric_id_string(id: conn_id): string
	{
	return fmt("%s:%d > %s:%d",
	           id$orig_h, id$orig_p,
	           id$resp_h, id$resp_p);
	}

function fmt_addr_set(input: addr_set): string
	{
	local output = "";
	local tmp = "";
	local len = length(input);
	local i = 1;

	for ( item in input )
		{
		tmp = fmt("%s", item);
		if ( len != i )
			tmp = fmt("%s ", tmp);
		i = i+1;
		output = fmt("%s%s", output, tmp);
		}
	return fmt("%s", output);
	}

function fmt_str_set(input: string_set, strip: pattern): string
	{
	local len = length(input);
	if ( len == 0 )
		return "{}";

	local output = "{";
	local tmp = "";
	local i = 1;

	for ( item in input )
		{
		tmp = fmt("%s", gsub(item, strip, ""));
		if ( len != i )
			tmp = fmt("%s, ", tmp);
		i = i+1;
		output = fmt("%s%s", output, tmp);
		}
	return fmt("%s}", output);
	}

# Regular expressions for matching IP addresses in strings.
const ipv4_addr_regex = /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/;
const ipv6_8hex_regex = /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/;
const ipv6_compressed_hex_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/;
const ipv6_hex4dec_regex = /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;
const ipv6_compressed_hex4dec_regex = /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;

# These are only commented out until this bug is fixed:
#    http://www.bro-ids.org/wiki/index.php/Known_Issues#Bug_with_OR-ing_together_pattern_variables
#const ipv6_addr_regex = ipv6_8hex_regex |
#                        ipv6_compressed_hex_regex |
#                        ipv6_hex4dec_regex |
#                        ipv6_compressed_hex4dec_regex;
#const ip_addr_regex = ipv4_addr_regex | ipv6_addr_regex;

const ipv6_addr_regex =
    /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/ |
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/ | # IPv6 Compressed Hex
    /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec

const ip_addr_regex =
    /[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}/ |
    /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/ |
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)/ | # IPv6 Compressed Hex
    /(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/ | # 6Hex4Dec
    /(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/; # CompressedHex4Dec

function is_valid_ip(ip_str: string): bool
	{
	if ( ip_str == ipv4_addr_regex )
		{
		local octets = split(ip_str, /\./);
		if ( |octets| != 4 )
			return F;

		local num=0;
		for ( i in octets )
			{
			num = to_count(octets[i]);
			if ( num < 0 || 255 < num )
				return F;
			}
		return T;
		}
	else if ( ip_str == ipv6_addr_regex )
		{
		# TODO: make this work correctly.
		return T;
		}
	return F;
	}

# This output a string_array of ip addresses extracted from a string.
# given: "this is 1.1.1.1 a test 2.2.2.2 string with ip addresses 3.3.3.3"
# outputs: { [1] = 1.1.1.1, [2] = 2.2.2.2, [3] = 3.3.3.3 }
function find_ip_addresses(input: string): string_array
	{
	local parts = split_all(input, ip_addr_regex);
	local output: string_array;

	for ( i in parts )
		{
		if ( i % 2 == 0 && is_valid_ip(parts[i]) )
			output[i/2] = parts[i];
		}
	return output;
	}

# Some enums for deciding what and when to log.
type Direction: enum { Inbound, Outbound, All };
type Hosts: enum { LocalHosts, RemoteHosts, AllHosts };

function orig_matches_direction(ip: addr, d: Direction): bool
	{
	if ( d == None ) return F;

	return ( (d == Outbound && is_local_addr(ip)) ||
	         (d == Inbound && !is_local_addr(ip)) ||
	         d == All );
	}

function resp_matches_direction(ip: addr, d: Direction): bool
	{
	if ( d == None ) return F;

	return ( (d == Inbound && is_local_addr(ip)) ||
	         (d == Outbound && !is_local_addr(ip)) ||
	         d == All );
	}

function conn_matches_direction(id: conn_id, d: Direction): bool
	{
	if ( d == None ) return F;

	return orig_matches_direction(id$orig_h, d);
	}

function resp_matches_hosts(ip: addr, d: Hosts): bool
	{
	return ( (d == LocalHosts && is_local_addr(ip)) ||
	         (d == RemoteHosts && !is_local_addr(ip)) ||
	         d == AllHosts );
	}


Forgot to include a connection with the notices.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

# This is a locally defined event.  Handle it with a priority greater than
# negative 5 if you want to dig into the conn_info record before it's deleted.
function end_smtp_extended_logging(c: connection)
	{
	local id = c$id;
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
				    $conn=c]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject),
					$conn=c]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(c);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c);
	}

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load smtp
@load global-ext

module SMTP;

export {
	global smtp_ext_log = open_log_file("smtp-ext") &raw_output &redef;

	redef enum Notice += {
		# Thrown when a local host receives a reply mentioning an smtp block list
		SMTP_BL_Error_Message,
		# Thrown when the local address is seen in the block list error message
		SMTP_BL_Blocked_Host,
		# When mail seems to originate from a suspicious location
		SMTP_Suspicious_Origination,
	};

	# Uncomment this next line or define it in your own file to totally
	# disable inspection into the smtp received from headers.
	#   Disabling supressses the suspicious_origination notice when it's
	#   tracked through the received from headers.
	const smtp_capture_mail_path = 1;

	# Direction to capture the full "Received from" path. (from the Direction enum)
	#    RemoteHosts - only capture the path until an internal host is found.
	#    LocalHosts - only capture the path until the external host is discovered.
	#    AllHosts - capture the entire path.
	const mail_path_capture: Hosts = LocalHosts &redef;

	# Places where it's suspicious for mail to originate from.
	#  this requires all-capital letter, two character country codes (e.x. US)
	const suspicious_origination_countries: set[string] = {} &redef;
	const suspicious_origination_networks: set[subnet] = {} &redef;

	# This matches content in SMTP error messages that indicate some
	# block list doesn't like the connection/mail.
	const smtp_bl_error_messages =
	    /www\.spamhaus\.org\//
	  | /cbl\.abuseat\.org\//
	  | /www\.sorbs\.net\//
	  | /bsn\.borderware\.com\//
	  | /mail-abuse\.com\//
	  | /bbl\.barracudacentral\.com\//
	  | /psbl\.surriel\.com\//
	  | /antispam\.imp\.ch\//
	  | /www\.dyndns\.com\/.*spam/
	  | /intercept\.datapacket\.net\// &redef;
}

type session_info: record {
	msg_id: string &default="";
	in_reply_to: string &default="";
	helo: string &default="";
	mailfrom: string &default="";
	rcptto: set[string];
	date: string &default="";
	from: string &default="";
	to: set[string];
	reply_to: string &default="";
	subject: string &default="";
	x_originating_ip: string &default="";
	received_from_originating_ip: string &default="";
	last_reply: string &default=""; # last message the server sent to the client
	files: string &default="";
	path: string &default="";
	current_header: string &default="";
};

function default_session_info(): session_info
	{
	local tmp: set[string] = set();
	local tmp2: set[string] = set();
	return [$rcptto=tmp, $to=tmp2];
	}
# TODO: setting a default function doesn't seem to be working correctly here.
global conn_info: table[conn_id] of session_info &read_expire=4mins;

global in_received_from_headers: set[conn_id] &create_expire = 2min;
global smtp_received_finished: set[conn_id] &create_expire = 2min;
global smtp_forward_paths: table[conn_id] of string &create_expire = 2min &default = "";

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a local host is on an SMTP watch list
#	[$pred(n: notice_info) =
#		{ return (n$note == SMTP::SMTP_BL_Blocked_Host && is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

function find_address_in_smtp_header(header: string): string
{
	local ips = find_ip_addresses(header);

	if ( |ips| > 0 )
		return ips[1];
	if ( |ips| > 1 )
		return ips[2];
	return "";
}

@ifdef( smtp_capture_mail_path )
# This event handler builds the "Received From" path by reading the
# headers in the mail
event smtp_data(c: connection, is_orig: bool, data: string)
	{
	local id = c$id;
	local session = smtp_sessions[id];

	if ( id !in conn_info ||
		 !session$in_header ||
		 id in smtp_received_finished)
		return;

	local conn_log = conn_info[id];

	if ( /^[rR][eE][cC][eE][iI][vV][eE][dD]:/ in data )
		add in_received_from_headers[id];
	else if ( /^[[:blank:]]/ !in data )
		delete in_received_from_headers[id];

	if ( id in in_received_from_headers ) # currently seeing received from headers
		{
		local text_ip = find_address_in_smtp_header(data);

		if ( text_ip == "" )
			return;

		local ip = to_addr(text_ip);

		# I don't care if mail bounces around on localhost
		if ( ip == 127.0.0.1 ) return;

		# This overwrites each time.
		conn_log$received_from_originating_ip = text_ip;

		local ellipsis = "";
		if ( !addr_matches_hosts(ip, mail_path_capture) &&
		     ip !in private_address_space )
			{
			ellipsis = "... ";
			add smtp_received_finished[id];
			}

		if (conn_log$path == "")
			conn_log$path = fmt("%s%s -> %s -> %s", ellipsis, ip, id$orig_h, id$resp_h);
		else
			conn_log$path = fmt("%s%s -> %s", ellipsis, ip, conn_log$path);
		}
	else if ( !session$in_header && id !in smtp_received_finished )
		add smtp_received_finished[id];
	}
@endif

# This is a locally defined event.  Handle it with a priority greater than
# negative 5 if you want to dig into the conn_info record before it's deleted.
function end_smtp_extended_logging(id: conn_id)
	{
	local conn_log = conn_info[id];

	local loc: geo_location;
	local ip: addr;
	if ( conn_log$x_originating_ip != "" )
		{
		ip = to_addr(conn_log$x_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$received_from_originating_ip != "" &&
	     conn_log$received_from_originating_ip != conn_log$x_originating_ip )
		{
		ip = to_addr(conn_log$received_from_originating_ip);
		loc = lookup_location(ip);

		if ( loc$country_code in suspicious_origination_countries ||
			 ip in suspicious_origination_networks )
			{
			NOTICE([$note=SMTP_Suspicious_Origination,
				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
				    $sub=fmt("Subject: %s", conn_log$subject)]);
			}
		}

	if ( conn_log$mailfrom != "" )
		print smtp_ext_log, cat_sep("\t", "\\N",
		                            network_time(),
		                            id$orig_h, fmt("%d", id$orig_p), id$resp_h, fmt("%d", id$resp_p),
		                            conn_log$helo,
		                            conn_log$msg_id,
		                            conn_log$in_reply_to,
		                            conn_log$mailfrom,
		                            fmt_str_set(conn_log$rcptto, /["'<>]|([[:blank:]].*$)/),
		                            conn_log$date,
		                            conn_log$from,
		                            conn_log$reply_to,
		                            fmt_str_set(conn_log$to, /["']/),
		                            gsub(conn_log$files, /["']/, ""),
		                            conn_log$last_reply,
		                            conn_log$x_originating_ip,
		                            conn_log$path);
	delete conn_info[id];
	delete smtp_received_finished[id];
	}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	local id = c$id;
	# This continually overwrites, but we want the last reply, so this actually works fine.
	if ( (code != 421 && code >= 400) &&
	     id in conn_info )
		{
		conn_info[id]$last_reply = fmt("%d %s", code, msg);

		# Raise a notice when an SMTP error about a block list is discovered.
		if ( smtp_bl_error_messages in msg )
			{
			local note = SMTP_BL_Error_Message;
			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);

			# Determine if the originator's IP address is in the message.
			local ips = find_ip_addresses(msg);
			local text_ip = "";
			if ( |ips| > 0 && to_addr(ips[1]) == c$id$orig_h )
				{
				note = SMTP_BL_Blocked_Host;
				message = fmt("%s is on an SMTP block list", c$id$orig_h);
				}

			NOTICE([$note=note,
			        $conn=c,
			        $msg=message,
			        $sub=msg]);
			}
		}
	}

event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=-5
	{
	local id = c$id;
	if ( id !in smtp_sessions )
		return;

	# In case this is not the first message in a session
	if ( ((/^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg) ) &&
	     id in conn_info )
		{
		local tmp_helo = conn_info[id]$helo;
		end_smtp_extended_logging(id);
		conn_info[id] = default_session_info();
		conn_info[id]$helo = tmp_helo;
		}

	if ( id !in conn_info )
		conn_info[id] = default_session_info();
	local conn_log = conn_info[id];

	if ( /^([hH]|[eE]){2}[lL][oO]/ in command )
		conn_log$helo = arg;

	if ( /^[rR][cC][pP][tT]/ in command && /^[tT][oO]:/ in arg )
		add conn_log$rcptto[split1(arg, /:[[:blank:]]*/)[2]];

	if ( /^[mM][aA][iI][lL]/ in command && /^[fF][rR][oO][mM]:/ in arg )
		{
		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
		conn_log$mailfrom = split1(partially_done, /[[:blank:]]/)[1];
		}
	}

event smtp_data(c: connection, is_orig: bool, data: string) &priority=-5
	{
	local id = c$id;

	if ( id !in conn_info )
	return;

	if ( !smtp_sessions[id]$in_header )
		{
		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
			{
			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
			if ( conn_info[id]$files == "" )
				conn_info[id]$files = data;
			else
				conn_info[id]$files += fmt(", %s", data);
			}
		return;
		}

	local conn_log = conn_info[id];
	# This is to fully construct headers that will tend to wrap around.
	if ( /^[[:blank:]]/ in data )
		{
		data = sub(data, /^[[:blank:]]/, "");
		if ( conn_log$current_header == "message-id" )
			conn_log$msg_id += data;
		else if ( conn_log$in_reply_to == "in-reply-to" )
			conn_log$in_reply_to += data;
		else if ( conn_log$current_header == "subject" )
			conn_log$subject += data;
		else if ( conn_log$current_header == "from" )
			conn_log$from += data;
		else if ( conn_log$current_header == "reply-to" )
			conn_log$reply_to += data;
		return;
		}
	conn_log$current_header = "";

	if ( /^[mM][eE][sS][sS][aA][gG][eE]-[iI][dD]:/ in data )
		{
		conn_log$msg_id = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "message-id";
		}
	else if ( /^[iI][nN]-[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$in_reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "in-reply-to";
		}

	else if ( /^[dD][aA][tT][eE]:/ in data )
		{
		conn_log$date = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "date";
		}

	else if ( /^[fF][rR][oO][mM]:/ in data )
		{
		conn_log$from = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "from";
		}

	else if ( /^[tT][oO]:/ in data )
		{
		add conn_log$to[split1(data, /:[[:blank:]]*/)[2]];
		conn_log$current_header = "to";
		}

	else if ( /^[rR][eE][pP][lL][yY]-[tT][oO]:/ in data )
		{
		conn_log$reply_to = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "reply-to";
		}

	else if ( /^[sS][uU][bB][jJ][eE][cC][tT]:/ in data )
		{
		conn_log$subject = split1(data, /:[[:blank:]]*/)[2];
		conn_log$current_header = "subject";
		}

	else if ( /^[xX]-[oO][rR][iI][gG][iI][nN][aA][tT][iI][nN][gG]-[iI][pP]:/ in data )
		{
		conn_log$x_originating_ip = find_ip_addresses(data)[1];
		conn_log$current_header = "x-originating-ip";
		}

	}

event connection_finished(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}

event connection_state_remove(c: connection) &priority=5
	{
	if ( c$id in conn_info )
		end_smtp_extended_logging(c$id);
	}


subscribe master to dionaea as fallback

subscribe master to dionaea as fallback

@load ./dio_log.bro
@load ./bro_log.bro
@load ./dio_mysql_log.bro
const broker_port: port = 9999/tcp &redef;
redef exit_only_after_terminate = T;
redef Broker::endpoint_name = "bro_receiver";
global dionaea_access: event(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string);
global dionaea_mysql: event(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string);
global get_protocol: function(proto_str: string) : transport_proto;
global log_bro: function(msg: string);
global slaves: table[string] of count;
global connectors: opaque of Broker::Handle;
global add_to_balance: function(peer_name: string);
global remove_from_balance: function(peer_name: string);
global rebalance_all: function();

event bro_init() {
    log_bro("bro_receiver.bro: bro_init()");
    Broker::enable([$auto_publish=T]);

    Broker::listen(broker_port, "0.0.0.0");

    Broker::subscribe_to_events("bro/forwarder");
    Broker::subscribe_to_events("honeypot/dionaea");

    ## create a distributed datastore for the connector to link against:
    connectors = Broker::create_master("connectors");

    log_bro("bro_receiver.bro: bro_init() done");
}
event bro_done() {
    log_bro("bro_receiver.bro: bro_done()");
}
event dionaea_access(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string) {
    local sport: port = count_to_port(src_port, get_protocol(transport));
    local dport: port = count_to_port(dst_port, get_protocol(transport));
    print fmt("dionaea_access: timestamp=%s, dst_ip=%s, dst_port=%s, src_hostname=%s, src_ip=%s, src_port=%s, transport=%s, protocol=%s, connector_id=%s", timestamp, dst_ip, dst_port, src_hostname, src_ip, src_port, transport, protocol, connector_id);
    local rec: Dio_access::Info = [$ts=timestamp, $dst_ip=dst_ip, $dst_port=dport, $src_hostname=src_hostname, $src_ip=src_ip, $src_port=sport, $transport=transport, $protocol=protocol, $connector_id=connector_id];

    Log::write(Dio_access::LOG, rec);
}
event dionaea_mysql(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string) {
    local lport: port = count_to_port(local_port, get_protocol(transport));
    local rport: port = count_to_port(remote_port, get_protocol(transport));
    print fmt("dionaea_mysql: timestamp=%s, id=%s, local_ip=%s, local_port=%s, remote_ip=%s, remote_port=%s, transport=%s, args=%s, connector_id=%s", timestamp, id, local_ip, local_port, remote_ip, remote_port, transport, args, connector_id);
    local rec: Dio_mysql::Info = [$ts=timestamp, $id=id, $local_ip=local_ip, $local_port=lport, $remote_ip=remote_ip, $remote_port=rport, $transport=transport, $args=args, $connector_id=connector_id];
    Log::write(Dio_mysql::LOG, rec);
}
event Broker::incoming_connection_established(peer_name: string) {
    print "Incoming connection established " + peer_name;
    log_bro("Incoming connection established " + peer_name);
    add_to_balance(peer_name);
}
event Broker::incoming_connection_broken(peer_name: string) {
    print "Incoming connection broken for " + peer_name;
    log_bro("Incoming connection broken for " + peer_name);
    remove_from_balance(peer_name);
}
function get_protocol(proto_str: string) : transport_proto {
    # https://www.bro.org/sphinx/scripts/base/init-bare.bro.html#type-transport_proto
    if (proto_str == "tcp") {
        return tcp;
    }
    if (proto_str == "udp") {
        return udp;
    }
    if (proto_str == "icmp") {
        return icmp;
    }
    return unknown_transport;
}

function log_bro(msg:string) {
    local rec: Brolog::Info = [$msg=msg];
    Log::write(Brolog::LOG, rec);
}

function add_to_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        slaves[peer_name] = 0;

        print "Registered new slave ", peer_name;
        log_bro("Registered new slave " + peer_name);

        rebalance_all();
    }
    if(/beemaster-connector-/ in peer_name) {
        local min_count_conn = 100000;
        local best_slave = "";

        for(slave in slaves) {
            local count_conn = slaves[slave];
            if (count_conn < min_count_conn) {
                best_slave = slave;
                min_count_conn = count_conn;
            }
        }
        # add the connector, regardless if any slave is ready. Thus, at least a reference is stored, that will get rebalanced once a new slave registers.
        Broker::insert(connectors, Broker::data(peer_name), Broker::data(best_slave));
        if (best_slave != "") {
            ++slaves[best_slave];
            print "Registered connector", peer_name, "and balanced to", best_slave;
            log_bro("Registered connector " + peer_name + " and balanced to " + best_slave);
        }
        else {
            print "Could not balance connector", peer_name, "because no slaves are ready";
            log_bro("Could not balance connector " + peer_name + " because no slaves are ready");
        }

    }
}

function remove_from_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        delete slaves[peer_name];

        print "Unregistered old slave ", peer_name;
        log_bro("Unregistered old slave " + peer_name + " ...");

        rebalance_all();
    }
    if(/beemaster-connector-/ in peer_name) {
        when(local cs = Broker::lookup(connectors, Broker::data(peer_name))) {
            local connected_slave = Broker::refine_to_string(cs$result);
            Broker::erase(connectors, Broker::data(peer_name));
            if (connected_slave == "") {
                # connector was registered, but no slave was there to handle it. If it now goes away, OK!
                print "Unregistered old connector", peer_name, "no connected slave found";
                log_bro("Unregistered old connector " + peer_name + " no connected slave found");
                return;
            }
            local count_conn = slaves[connected_slave];
            if (count_conn > 0) {
                slaves[connected_slave] = count_conn - 1;
                print "Unregistered old connector", peer_name, "from slave", connected_slave;
                log_bro("Unregistered old connector " + peer_name + " from slave " + connected_slave);
            }
        }
        timeout 100msec {
            print "Timeout unregistering connector", peer_name;
            log_bro("Timeout unregistering connector " + peer_name);
        }
    }
}

function rebalance_all() {
    local total_slaves = |slaves|;
    local slave_vector: vector of string;
    slave_vector = vector();
    local i = 0;
    for (slave in slaves) {
        slave_vector[i] = slave;
        ++i;
    }
    i = 0;
    when (local keys = Broker::keys(connectors)) {
        local connector_vector: vector of string;
        connector_vector = vector();
        local total_connectors = Broker::vector_size(keys$result);
        while (i < total_connectors) {
            connector_vector[i] = Broker::refine_to_string(Broker::vector_lookup(keys$result, i));
            ++i;
        }
        if (total_slaves == 0) {
            print "No registered slaves found, invalidating all connectors";
            log_bro("No registered slaves found, invalidating all connectors");
            local j = 0;
            while (j < i) {
                local connector = connector_vector[j];
                Broker::insert(connectors, Broker::data(connector), Broker::data(""));
                ++j;
            }
            return; # break out.
        }
        while (total_slaves > 0 && total_connectors > 0) {
            local balance_amount = total_connectors / total_slaves;
            if (total_connectors % total_slaves > 0) {
                balance_amount = total_connectors / total_slaves + 1;
            }
            --total_slaves;
            local balanced_to = slave_vector[total_slaves];
            slaves[balanced_to] = balance_amount;
            local balance_index = total_connectors - balance_amount; # do this once, do this here!
            while (total_connectors > balance_index) {
                --total_connectors;
                local rebalanced_conn = connector_vector[total_connectors];
                Broker::insert(connectors, Broker::data(rebalanced_conn), Broker::data(balanced_to));
                print "Rebalanced connector", rebalanced_conn, "to slave", balanced_to;
                log_bro("Rebalanced connector " + rebalanced_conn + " to slave " + balanced_to);
            }
        }
    }
    timeout 100msec {
        log_bro("ERROR: Unable to query keys in 'connectors-data-store' within 100ms, timeout");
    }
}

@load ./dio_log.bro
@load ./bro_log.bro
@load ./dio_mysql_log.bro
const broker_port: port = 9999/tcp &redef;
redef exit_only_after_terminate = T;
redef Broker::endpoint_name = "bro_receiver";
global dionaea_access: event(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string);
global dionaea_mysql: event(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string);
global get_protocol: function(proto_str: string) : transport_proto;
global log_bro: function(msg: string);
global slaves: table[string] of count;
global connectors: opaque of Broker::Handle;
global add_to_balance: function(peer_name: string);
global remove_from_balance: function(peer_name: string);
global rebalance_all: function();

event bro_init() {
    log_bro("bro_receiver.bro: bro_init()");
    Broker::enable([$auto_publish=T]);

    Broker::listen(broker_port, "0.0.0.0");

    Broker::subscribe_to_events("bro/forwarder");

    ## create a distributed datastore for the connector to link against:
    connectors = Broker::create_master("connectors");

    log_bro("bro_receiver.bro: bro_init() done");
}
event bro_done() {
    log_bro("bro_receiver.bro: bro_done()");
}
event dionaea_access(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string) {
    local sport: port = count_to_port(src_port, get_protocol(transport));
    local dport: port = count_to_port(dst_port, get_protocol(transport));
    print fmt("dionaea_access: timestamp=%s, dst_ip=%s, dst_port=%s, src_hostname=%s, src_ip=%s, src_port=%s, transport=%s, protocol=%s, connector_id=%s", timestamp, dst_ip, dst_port, src_hostname, src_ip, src_port, transport, protocol, connector_id);
    local rec: Dio_access::Info = [$ts=timestamp, $dst_ip=dst_ip, $dst_port=dport, $src_hostname=src_hostname, $src_ip=src_ip, $src_port=sport, $transport=transport, $protocol=protocol, $connector_id=connector_id];

    Log::write(Dio_access::LOG, rec);
}
event dionaea_mysql(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string) {
    local lport: port = count_to_port(local_port, get_protocol(transport));
    local rport: port = count_to_port(remote_port, get_protocol(transport));
    print fmt("dionaea_mysql: timestamp=%s, id=%s, local_ip=%s, local_port=%s, remote_ip=%s, remote_port=%s, transport=%s, args=%s, connector_id=%s", timestamp, id, local_ip, local_port, remote_ip, remote_port, transport, args, connector_id);
    local rec: Dio_mysql::Info = [$ts=timestamp, $id=id, $local_ip=local_ip, $local_port=lport, $remote_ip=remote_ip, $remote_port=rport, $transport=transport, $args=args, $connector_id=connector_id];
    Log::write(Dio_mysql::LOG, rec);
}
event Broker::incoming_connection_established(peer_name: string) {
    print "Incoming connection established " + peer_name;
    log_bro("Incoming connection established " + peer_name);
    add_to_balance(peer_name);
}
event Broker::incoming_connection_broken(peer_name: string) {
    print "Incoming connection broken for " + peer_name;
    log_bro("Incoming connection broken for " + peer_name);
    remove_from_balance(peer_name);
}
function get_protocol(proto_str: string) : transport_proto {
    # https://www.bro.org/sphinx/scripts/base/init-bare.bro.html#type-transport_proto
    if (proto_str == "tcp") {
        return tcp;
    }
    if (proto_str == "udp") {
        return udp;
    }
    if (proto_str == "icmp") {
        return icmp;
    }
    return unknown_transport;
}

function log_bro(msg:string) {
    local rec: Brolog::Info = [$msg=msg];
    Log::write(Brolog::LOG, rec);
}

function add_to_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        slaves[peer_name] = 0;

        print "Registered new slave ", peer_name;
        log_bro("Registered new slave " + peer_name);

        rebalance_all();
    }
    if(/beemaster-connector-/ in peer_name) {
        local min_count_conn = 100000;
        local best_slave = "";

        for(slave in slaves) {
            local count_conn = slaves[slave];
            if (count_conn < min_count_conn) {
                best_slave = slave;
                min_count_conn = count_conn;
            }
        }
        # add the connector, regardless if any slave is ready. Thus, at least a reference is stored, that will get rebalanced once a new slave registers.
        Broker::insert(connectors, Broker::data(peer_name), Broker::data(best_slave));
        if (best_slave != "") {
            ++slaves[best_slave];
            print "Registered connector", peer_name, "and balanced to", best_slave;
            log_bro("Registered connector " + peer_name + " and balanced to " + best_slave);
        }
        else {
            print "Could not balance connector", peer_name, "because no slaves are ready";
            log_bro("Could not balance connector " + peer_name + " because no slaves are ready");
        }

    }
}

function remove_from_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        delete slaves[peer_name];

        print "Unregistered old slave ", peer_name;
        log_bro("Unregistered old slave " + peer_name + " ...");

        rebalance_all();
    }
    if(/beemaster-connector-/ in peer_name) {
        when(local cs = Broker::lookup(connectors, Broker::data(peer_name))) {
            local connected_slave = Broker::refine_to_string(cs$result);
            Broker::erase(connectors, Broker::data(peer_name));
            if (connected_slave == "") {
                # connector was registered, but no slave was there to handle it. If it now goes away, OK!
                print "Unregistered old connector", peer_name, "no connected slave found";
                log_bro("Unregistered old connector " + peer_name + " no connected slave found");
                return;
            }
            local count_conn = slaves[connected_slave];
            if (count_conn > 0) {
                slaves[connected_slave] = count_conn - 1;
                print "Unregistered old connector", peer_name, "from slave", connected_slave;
                log_bro("Unregistered old connector " + peer_name + " from slave " + connected_slave);
            }
        }
        timeout 100msec {
            print "Timeout unregistering connector", peer_name;
            log_bro("Timeout unregistering connector " + peer_name);
        }
    }
}

function rebalance_all() {
    local total_slaves = |slaves|;
    local slave_vector: vector of string;
    slave_vector = vector();
    local i = 0;
    for (slave in slaves) {
        slave_vector[i] = slave;
        ++i;
    }
    i = 0;
    when (local keys = Broker::keys(connectors)) {
        local connector_vector: vector of string;
        connector_vector = vector();
        local total_connectors = Broker::vector_size(keys$result);
        while (i < total_connectors) {
            connector_vector[i] = Broker::refine_to_string(Broker::vector_lookup(keys$result, i));
            ++i;
        }
        if (total_slaves == 0) {
            print "No registered slaves found, invalidating all connectors";
            log_bro("No registered slaves found, invalidating all connectors");
            local j = 0;
            while (j < i) {
                local connector = connector_vector[j];
                Broker::insert(connectors, Broker::data(connector), Broker::data(""));
                ++j;
            }
            return; # break out.
        }
        while (total_slaves > 0 && total_connectors > 0) {
            local balance_amount = total_connectors / total_slaves;
            if (total_connectors % total_slaves > 0) {
                balance_amount = total_connectors / total_slaves + 1;
            }
            --total_slaves;
            local balanced_to = slave_vector[total_slaves];
            slaves[balanced_to] = balance_amount;
            local balance_index = total_connectors - balance_amount; # do this once, do this here!
            while (total_connectors > balance_index) {
                --total_connectors;
                local rebalanced_conn = connector_vector[total_connectors];
                Broker::insert(connectors, Broker::data(rebalanced_conn), Broker::data(balanced_to));
                print "Rebalanced connector", rebalanced_conn, "to slave", balanced_to;
                log_bro("Rebalanced connector " + rebalanced_conn + " to slave " + balanced_to);
            }
        }
    }
    timeout 100msec {
        log_bro("ERROR: Unable to query keys in 'connectors-data-store' within 100ms, timeout");
    }
}


correct host variable

correct host variable

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logging = Outbound &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", logging, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !orig_matches_direction(c$id$resp_h, logging) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logging = Outbound &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", logging, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !orig_matches_direction(c$id$resp_h, logging) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", info$c$id$orig_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}


tweak expire time, synchronize variables

@load global-ext
@load ssh-ext
@load subnet-helper
@load notice

module SSH;

export {
    global ssh_attacked: table[addr] of addr_set &create_expire=30mins &synchronized;# default isn't working &default=function(a:addr):addr_set { print a;return set();};
    global libssh_scanners: set[addr] &create_expire=10mins &synchronized;
    const subnet_threshold = 3 &redef;

    redef enum Notice += {
        SSH_Libssh_Scanner,
    };
}

function notice_exec_test(n: notice_info, a: NoticeAction): NoticeAction
{
    #execute_with_notice("/usr/local/bin/bro_ipblocker_block", n);
    execute_with_notice("/tmp/test", n);
    return NOTICE_ALARM_ALWAYS;
}

redef notice_action_filters += {
    [SSH_Libssh_Scanner] = notice_exec_test,
};

event ssh_ext(id: conn_id, si: ssh_ext_session_info) &priority=-10
{
    if(is_local_addr(id$orig_h)  ||
       /libssh/ !in si$client    ||
       si$status == "success")
        return;

    local subnets = add_attack(ssh_attacked, id$orig_h, id$resp_h);
    print fmt("%s scanned %d subnets", id$orig_h, subnets);

    if(subnets >= subnet_threshold && id$orig_h !in libssh_scanners){
        add libssh_scanners[id$orig_h];

        NOTICE([$note=SSH_Libssh_Scanner,
                $id=id,
                $msg=fmt("SSH libssh scanning. %s scanned %d subnets", id$orig_h, subnets),
                $sub="ssh-ext",
                $n=subnets]);
    }

}

@load global-ext
@load ssh-ext
@load subnet-helper
@load notice

module SSH;

export {
    global ssh_attacked: table[addr] of addr_set &create_expire=30mins;# default isn't working &default=function(a:addr):addr_set { print a;return set();};
    global libssh_scanners: set[addr] &read_expire=1hr;
    const subnet_threshold = 3 &redef;

    redef enum Notice += {
        SSH_Libssh_Scanner,
    };
}

function notice_exec_test(n: notice_info, a: NoticeAction): NoticeAction
{
    #execute_with_notice("/usr/local/bin/bro_ipblocker_block", n);
    execute_with_notice("/tmp/test", n);
    return NOTICE_ALARM_ALWAYS;
}

redef notice_action_filters += {
    [SSH_Libssh_Scanner] = notice_exec_test,
};

event ssh_ext(id: conn_id, si: ssh_ext_session_info) &priority=-10
{
    if(is_local_addr(id$orig_h)  ||
       /libssh/ !in si$client    ||
       si$status == "success")
        return;

    local subnets = add_attack(ssh_attacked, id$orig_h, id$resp_h);
    print fmt("%s scanned %d subnets", id$orig_h, subnets);

    if(subnets >= subnet_threshold && id$orig_h !in libssh_scanners){
        add libssh_scanners[id$orig_h];

        NOTICE([$note=SSH_Libssh_Scanner,
                $id=id,
                $msg=fmt("SSH libssh scanning. %s scanned %d subnets", id$orig_h, subnets),
                $sub="ssh-ext",
                $n=subnets]);
    }

}


small bug in logging.smtp-ext.bro

small bug in logging.smtp-ext.bro

@load global-ext
@load smtp-ext

module SMTP;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;

	# Which mail transactions to log.
	# Choices are: Inbound, Outbound, All
	const logging = All &redef;
}

event bro_init()
	{
	LOG::create_logs("smtp-ext", logging, split_log_file, T);
	LOG::define_header("smtp-ext", cat_sep("\t", "",
	                                      "ts",
	                                      "orig_h", "orig_p",
	                                      "resp_h", "resp_p",
	                                      "helo", "message-id", "in-reply-to",
	                                      "mailfrom", "rcptto",
	                                      "date", "from", "reply_to", "to",
	                                      "files", "last_reply", "x-originating-ip",
	                                      "path", "is_webmail", "agent"));
	}

event smtp_ext(id: conn_id, si: smtp_ext_session_info)
	{
	if ( si$mailfrom != "" )
		{
		local log = LOG::get_file("smtp-ext", id$resp_h, F);
		print log, cat_sep("\t", "\\N",
		                   network_time(),
		                   id$orig_h, port_to_count(id$orig_p), id$resp_h, port_to_count(id$resp_p),
		                   si$helo,
		                   si$msg_id,
		                   si$in_reply_to,
		                   si$mailfrom,
		                   fmt_str_set(si$rcptto, /["'<>]|([[:blank:]].*$)/),
		                   si$date,
		                   si$from,
		                   si$reply_to,
		                   fmt_str_set(si$to, /["']/),
		                   fmt_str_set(si$files, /["']/),
		                   si$last_reply,
		                   si$x_originating_ip,
		                   si$path,
		                   si$is_webmail,
		                   si$agent);
		}

	}

@load global-ext
@load smtp-ext

module SMTP;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;

	# Which mail transactions to log.
	# Choices are: Inbound, Outbound, All
	const logging = All &redef;
}

event bro_init()
	{
	LOG::create_logs("smtp-ext", logging, split_log_file, T);
	LOG::define_header("smtp-ext", cat_sep("\t", "",
	                                      "ts",
	                                      "orig_h", "orig_p",
	                                      "resp_h", "resp_p",
	                                      "helo", "message-id", "in-reply-to",
	                                      "mailfrom", "rcptto",
	                                      "date", "from", "reply_to", "to",
	                                      "files", "last_reply", "x-originating-ip",
	                                      "path", "is_webmail", "agent"));
	}

event smtp_ext(id: conn_id, si: smtp_ext_session_info)
	{
	if ( si$mailfrom != "" )
		local log = LOG::get_file("smtp-ext", id$resp_h, F);
		print log, cat_sep("\t", "\\N",
		                   network_time(),
		                   id$orig_h, port_to_count(id$orig_p), id$resp_h, port_to_count(id$resp_p),
		                   si$helo,
		                   si$msg_id,
		                   si$in_reply_to,
		                   si$mailfrom,
		                   fmt_str_set(si$rcptto, /["'<>]|([[:blank:]].*$)/),
		                   si$date,
		                   si$from,
		                   si$reply_to,
		                   fmt_str_set(si$to, /["']/),
		                   fmt_str_set(si$files, /["']/),
		                   si$last_reply,
		                   si$x_originating_ip,
		                   si$path,
		                   si$is_webmail,
		                   si$agent);

	}


Create expand-query.bro

Create expand-query.bro

Init of expand-query.bro.

# Creates a vector from a DNS query
# CrowdStrike 2015
# josh.liburdi@crowdstrike.com
# @jshlbrd

@load base/protocols/dns
@load base/protocols/http

module DNS;

redef record DNS::Info += {
	query_vec:        vector of string  &log &optional;
	query_vec_size:   count             &log &optional;
};

event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
{
c$dns$query_vec = HTTP::extract_keys(query,/\./);
c$dns$query_vec_size = |c$dns$query_vec|;
}

error: pathspec 'bro-scripts/extensions/dns/expand-query.bro' did not match any file(s) known to git


Small comment change.

Small comment change.

@load ssh-ext

module SSH;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;
	# Which SSH logins to record.
	# Choices are: Inbound, Outbound, All
	const logging = All &redef;
}

event bro_init()
	{
	LOG::create_logs("ssh-ext", logging, split_log_file);
	}

event ssh_ext(id: conn_id, si: ssh_ext_session_info)
	{
	local log = LOG::choose("ssh-ext", id$resp_h);

	print log, cat_sep("\t", "\\N",
	                   si$start_time,
	                   id$orig_h, fmt("%d", id$orig_p),
	                   id$resp_h, fmt("%d", id$resp_p),
	                   si$status, si$direction,
	                   si$location$country_code, si$location$region,
	                   si$client, si$server,
	                   si$resp_size);
	}

@load ssh-ext

module SSH;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;
	# Which mail transactions to log.
	# Choices are: Inbound, Outbound, All
	const logging = All &redef;
}

event bro_init()
	{
	LOG::create_logs("ssh-ext", logging, split_log_file);
	}

event ssh_ext(id: conn_id, si: ssh_ext_session_info)
	{
	local log = LOG::choose("ssh-ext", id$resp_h);

	print log, cat_sep("\t", "\\N",
	                   si$start_time,
	                   id$orig_h, fmt("%d", id$orig_p),
	                   id$resp_h, fmt("%d", id$resp_p),
	                   si$status, si$direction,
	                   si$location$country_code, si$location$region,
	                   si$client, si$server,
	                   si$resp_size);
	}


Using complex structs with multiple enums with bro results in the following
error:

  error: identifier or enumerator value in enumerated type definition already exists

#
# @TEST-EXEC: bro ./dtest.evt %INPUT
#
# @TEST-KNOWN-FAILURE: enum convert problem. Comment line 16 to toggle error.

# @TEST-START-FILE dtest.evt

grammar dtest.pac2;

protocol analyzer pac2::dtest over UDP:
    parse with dtest::Message,
    port 47808/udp;

on dtest::Message -> event dtest_message(self.func);

on dtest::Message -> event dtest_result (self.sub.result);

# @TEST-END-FILE
# @TEST-START-FILE dtest.pac2

module dtest;

type FUNCS = enum {
 A, B, C, D, E, F
};

type RESULT = enum {
 A, B, C, D, E, F
};

export type Message = unit {
  func: uint8 &convert=FUNCS($$);
  sub: SubMessage;
};

type SubMessage = unit {
  result: uint8 &convert=RESULT($$);
};

# @TEST-END-FILE

error: pathspec 'bro/tests/pac2/double-types.bro' did not match any file(s) known to git


fix notice suppression

fix notice suppression

module DNS;

export {
    redef record Info += {
        is_external: bool &default=F;
        ## Country code of external DNS server
        cc: string &log &optional;
        ## hostname of external DNS server
        #resp_h_hostname: string &log &optional;
    };

    const ignore_external: set[subnet] = {
        8.8.8.8/32, #google dns
        8.8.4.4/32, #google dns
        208.67.220.123/32, #opendns
        208.67.222.222/32, #opendns
        208.67.220.220/32, #opendns
    } &redef;

    redef enum Notice::Type += {
        ## Indicates that a user is using an external DNS server
        EXTERNAL_DNS,

        ## Indicates that a user is using an external DNS server in
        ## a foreign country.
        EXTERNAL_FOREIGN_DNS,
    };

    const local_countries: set[string] = {
        "US",
    } &redef;

}

event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
{
    local rec = c$dns;
    if(!rec$RD)
        return;

    local orig_h = rec$id$orig_h;
    local resp_h = rec$id$resp_h;

    #is this an outbound query
    if(!Site::is_local_addr(orig_h) || Site::is_local_addr(resp_h))
        return;

    if(orig_h in ignore_external || resp_h in ignore_external)
        return;

    rec$is_external=T;

    local loc = lookup_location(resp_h);

    rec$cc = "";
    if(loc?$country_code){
        rec$cc = loc$country_code;
    }

    when (local hostname = lookup_addr(resp_h)) {
        #Doesn't work for the log, but we can use it here :-(
        #rec$resp_h_hostname = hostname;

        local note = EXTERNAL_DNS;
        local nmsg = fmt("An external DNS server is in use %s(%s)", resp_h, hostname);

        if(rec$cc !in local_countries){
            note = EXTERNAL_FOREIGN_DNS;
            nmsg = fmt("An external foreign(%s) DNS server is in use %s(%s).", rec$cc, resp_h, hostname);
        }
        local ident = fmt("%s-%s", orig_h, resp_h);
        NOTICE([$note=note,
                $msg=nmsg,
                $sub=hostname,
                $identifier=ident,
                $remote_location=loc,
                $suppress_for=1day,
                $conn=c]);
    }
}

event bro_init()
{
    Log::add_filter(DNS::LOG, [$name = "external-dns",
                                $path = "external_dns",
                                $exclude = set("uid", "proto", "trans_id","qclass", "qclass_name", "qtype", "rcode",
                                "QR","AA","TC","RD","RA","Z","answers","TTLs"),
                                $pred(rec: DNS::Info) = {
        return rec$is_external;
    } ]);
}

module DNS;

export {
    redef record Info += {
        is_external: bool &default=F;
        ## Country code of external DNS server
        cc: string &log &optional;
        ## hostname of external DNS server
        #resp_h_hostname: string &log &optional;
    };

    const ignore_external: set[subnet] = {
        8.8.8.8/32, #google dns
        8.8.4.4/32, #google dns
        208.67.220.123/32, #opendns
        208.67.222.222/32, #opendns
        208.67.220.220/32, #opendns
    } &redef;

    redef enum Notice::Type += {
        ## Indicates that a user is using an external DNS server
        EXTERNAL_DNS,

        ## Indicates that a user is using an external DNS server in
        ## a foreign country.
        EXTERNAL_FOREIGN_DNS,
    };

    redef Notice::type_suppression_intervals += {
        [EXTERNAL_DNS] = 12hr,
        [EXTERNAL_FOREIGN_DNS] = 12hr,
    };

    const local_countries: set[string] = {
        "US",
    } &redef;

}

event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
{
    local rec = c$dns;
    if(!rec$RD)
        return;

    local orig_h = rec$id$orig_h;
    local resp_h = rec$id$resp_h;

    #is this an outbound query
    if(!Site::is_local_addr(orig_h) || Site::is_local_addr(resp_h))
        return;

    if(orig_h in ignore_external || resp_h in ignore_external)
        return;

    rec$is_external=T;

    local loc = lookup_location(resp_h);

    rec$cc = "";
    if(loc?$country_code){
        rec$cc = loc$country_code;
    }

    when (local hostname = lookup_addr(resp_h)) {
        #Doesn't work for the log, but we can use it here :-(
        #rec$resp_h_hostname = hostname;

        local note = EXTERNAL_DNS;
        local nmsg = fmt("An external DNS server is in use %s(%s)", resp_h, hostname);

        if(rec$cc !in local_countries){
            note = EXTERNAL_FOREIGN_DNS;
            nmsg = fmt("An external foreign(%s) DNS server is in use %s(%s).", rec$cc, resp_h, hostname);
        }
        local ident = fmt("%s-%s", orig_h, resp_h);
        NOTICE([$note=note,
                $msg=nmsg,
                $sub=hostname,
                $identifier=ident,
                $remote_location=loc,
                $conn=c]);
    }
}

event bro_init()
{
    Log::add_filter(DNS::LOG, [$name = "external-dns",
                                $path = "external_dns",
                                $exclude = set("uid", "proto", "trans_id","qclass", "qclass_name", "qtype", "rcode",
                                "QR","AA","TC","RD","RA","Z","answers","TTLs"),
                                $pred(rec: DNS::Info) = {
        return rec$is_external;
    } ]);
}


fix Zeek SMB module loading

fix Zeek SMB module loading

SMB location has changed

# Copyright (C) 2016-2018 RockNSM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module ROCK;

export {
  const sensor_id = gethostname() &redef;
}

#=== Bro built-ins ===================================

# Collect on SMB protocol
@load base/protocols/smb

# Enable VLAN Logging
@load policy/protocols/conn/vlan-logging

# Log MAC addresses
@load policy/protocols/conn/mac-logging

# Log (All) Client and Server HTTP Headers
@load policy/protocols/http/header-names.bro

#== ROCK specific scripts ============================
# Add empty Intel framework database
@load ./frameworks/intel

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension

#== 3rd Party Scripts =================================
# Add Salesforce's JA3 SSL fingerprinting
@load ./misc/ja3

### Sensor specific scripts ######################

# Configure AF_PACKET, if in use
@load ./plugins/afpacket

# Copyright (C) 2016-2018 RockNSM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module ROCK;

export {
  const sensor_id = gethostname() &redef;
}

#=== Bro built-ins ===================================

# Collect on SMB protocol
@load policy/protocols/smb

# Enable VLAN Logging
@load policy/protocols/conn/vlan-logging

# Log MAC addresses
@load policy/protocols/conn/mac-logging

# Log (All) Client and Server HTTP Headers
@load policy/protocols/http/header-names.bro

#== ROCK specific scripts ============================
# Add empty Intel framework database
@load ./frameworks/intel

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension

#== 3rd Party Scripts =================================
# Add Salesforce's JA3 SSL fingerprinting
@load ./misc/ja3

### Sensor specific scripts ######################

# Configure AF_PACKET, if in use
@load ./plugins/afpacket