Bro module to extract HTTP Headers

Bro module to extract HTTP Headers

module HTTP;

# Original version credit to Mike Sconzo modified from https://github.com/ClickSecurity/data_hacking/blob/master/browser_fingerprinting/bro_scripts/http-headers.bro
# Modified to also keep track of the Bro session ID for easy cross reference of suspicious entries with other Bro logs

export {
    type header_info_record: record
    {
        ## Header name and value.
        name:         string;
        value:        string;
    };

    redef record Info += { header_info_vector: vector of header_info_record &optional; };
}

module HTTPHeaders;

export
{
    redef enum Log::ID += { LOG };

    type Info: record
    {
        ts:                 time &log;
        uid:                string &log;
        origin:             string &log;
        identifier:         string &log;
        header_events_kv:   string &log;
        # header_events_json: string &log;
    };

    type header_info_record_type: record
    {
        ## Header name and value.
        name:         string;
        value:        string;
    };

    ## A type alias for a vector of header_info_records.
    type header_info_vector_type: vector of header_info_record_type;

}

redef record connection += {
    header_info_vector:        header_info_vector_type  &optional;
};

event bro_init()
{
    Log::create_stream(HTTPHeaders::LOG, [$columns=Info]);
}

# These events just init the vector, the whole http object gets wiped w/each request/reply
event http_request(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string)
        {
        c$http$header_info_vector = vector();
        }

event http_reply(c: connection, version: string, code: count, reason: string)
        {
        c$http$header_info_vector = vector();
        }

function sanitize_string(s: string): string
{
    local sanitized: string;
    local escape_chars: string;
    # escape_chars = "\\\/\b\f\n\r\t\"\:\,";
    escape_chars = "\"";

    sanitized = s;
    sanitized = to_string_literal(s);
    sanitized = subst_string(sanitized, "\\x", "\\u00"); # Replacing any hex escapes
    # sanitized = string_escape(sanitized, escape_chars);

    return sanitized;
}

function vector_to_kv_string(info_vector: header_info_vector_type): string
{
    local kv_string: string;
    local key: string;
    local value: string;

    kv_string = "";
    for ( i in info_vector )
        {
            key = sanitize_string(info_vector[i]$name);
            value = sanitize_string(info_vector[i]$value);

            # Stubbing out Cookie for now
            if (key == "COOKIE") { value = "-COOKIE-"; }

            kv_string += "\"" + key + "\"" + ":" + "\"" + value + "\",";
        }

    # Remove the last comma
    return cut_tail(kv_string, 1);
}

function vector_to_json_string(info_vector: header_info_vector_type): string
{
    local json_string: string;
    local key: string;
    local value: string;

    json_string = "[";
    for ( i in info_vector )
        {
            key = sanitize_string(info_vector[i]$name);
            value = sanitize_string(info_vector[i]$value);

            # Stubbing out Cookie for now
            if (key == "COOKIE") { value = "-COOKIE-"; }

            json_string += "{\"" + key + "\"" + ":" + "\"" + value + "\"},";
        }

    # Remove the last comma and add the closing bracket
    return cut_tail(json_string, 1) + "]";
}

event http_header(c: connection, is_orig: bool, name: string, value: string)
{
    local header_record: HTTP::header_info_record;
    local vector_size: int;

    if ( ! c?$http || ! c$http?$header_info_vector )
        return;

    # Add this http header info to the vector
    header_record$name = name;
    header_record$value = value;

    # Get current size of vector and add the record to the end
    c$http$header_info_vector[|c$http$header_info_vector|] = header_record;
}

event http_all_headers(c: connection, is_orig: bool, hlist: mime_header_list)
{
    local my_log: Info;
    local origin: string;
    local identifier: string;
    # local event_json_string: string;
    local event_kv_string: string;

    # Is the header from a client request or server response
    if ( is_orig )
        origin = "client";
    else
        origin = "server";

    # If we don't have a header_info_vector than punt
    if ( ! c?$http || ! c$http?$header_info_vector )
        return;

    print c$http$header_info_vector;

    # At this point our c$header_info_vector should contain all the
    # name/value pairs associated with the header, so will turn the
    # vector into a JSON string and add it to our log file.
    # event_json_string = vector_to_json_string(c$http$header_info_vector);
    event_kv_string = vector_to_kv_string(c$http$header_info_vector);

    # Okay now set the user agent field
    identifier = "NOTPRESENT";
    for ( i in hlist )
    {
        if ( origin == "client" )
            if ( hlist[i]$name == "USER-AGENT" ){identifier = hlist[i]$value;}
        if ( origin == "server" )
            if ( hlist[i]$name == "SERVER" ){identifier = hlist[i]$value;}
    }

    # Now add all the info and the event list to the log
    my_log = [$ts=c$start_time,
        $uid=c$uid,
        $origin=fmt("%s", origin),
        $identifier=fmt("%s", identifier),
        $header_events_kv=fmt("%s", event_kv_string)];

        # $header_events_kv=fmt("%s", event_kv_string),
        # $header_events_json=fmt("%s", event_json_string)];

    Log::write(HTTPHeaders::LOG, my_log);
}

module HTTP;

# Original version credit to Mike Sconzo modified from https://github.com/ClickSecurity/data_hacking/blob/master/browser_fingerprinting/bro_scripts/http-headers.bro

export {
    type header_info_record: record
    {
        ## Header name and value.
        name:         string;
        value:        string;
    };

    redef record Info += { header_info_vector: vector of header_info_record &optional; };
}

module HTTPHeaders;

export
{
    redef enum Log::ID += { LOG };

    type Info: record
    {
        ts:                 time &log;
        uid:                string &log;
        origin:             string &log;
        identifier:         string &log;
        header_events_kv:   string &log;
        # header_events_json: string &log;
    };

    type header_info_record_type: record
    {
        ## Header name and value.
        name:         string;
        value:        string;
    };

    ## A type alias for a vector of header_info_records.
    type header_info_vector_type: vector of header_info_record_type;

}

redef record connection += {
    header_info_vector:        header_info_vector_type  &optional;
};

event bro_init()
{
    Log::create_stream(HTTPHeaders::LOG, [$columns=Info]);
}

# These events just init the vector, the whole http object gets wiped w/each request/reply
event http_request(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string)
        {
        c$http$header_info_vector = vector();
        }

event http_reply(c: connection, version: string, code: count, reason: string)
        {
        c$http$header_info_vector = vector();
        }

function sanitize_string(s: string): string
{
    local sanitized: string;
    local escape_chars: string;
    # escape_chars = "\\\/\b\f\n\r\t\"\:\,";
    escape_chars = "\"";

    sanitized = s;
    sanitized = to_string_literal(s);
    sanitized = subst_string(sanitized, "\\x", "\\u00"); # Replacing any hex escapes
    # sanitized = string_escape(sanitized, escape_chars);

    return sanitized;
}

function vector_to_kv_string(info_vector: header_info_vector_type): string
{
    local kv_string: string;
    local key: string;
    local value: string;

    kv_string = "";
    for ( i in info_vector )
        {
            key = sanitize_string(info_vector[i]$name);
            value = sanitize_string(info_vector[i]$value);

            # Stubbing out Cookie for now
            if (key == "COOKIE") { value = "-COOKIE-"; }

            kv_string += "\"" + key + "\"" + ":" + "\"" + value + "\",";
        }

    # Remove the last comma
    return cut_tail(kv_string, 1);
}

function vector_to_json_string(info_vector: header_info_vector_type): string
{
    local json_string: string;
    local key: string;
    local value: string;

    json_string = "[";
    for ( i in info_vector )
        {
            key = sanitize_string(info_vector[i]$name);
            value = sanitize_string(info_vector[i]$value);

            # Stubbing out Cookie for now
            if (key == "COOKIE") { value = "-COOKIE-"; }

            json_string += "{\"" + key + "\"" + ":" + "\"" + value + "\"},";
        }

    # Remove the last comma and add the closing bracket
    return cut_tail(json_string, 1) + "]";
}

event http_header(c: connection, is_orig: bool, name: string, value: string)
{
    local header_record: HTTP::header_info_record;
    local vector_size: int;

    if ( ! c?$http || ! c$http?$header_info_vector )
        return;

    # Add this http header info to the vector
    header_record$name = name;
    header_record$value = value;

    # Get current size of vector and add the record to the end
    c$http$header_info_vector[|c$http$header_info_vector|] = header_record;
}

event http_all_headers(c: connection, is_orig: bool, hlist: mime_header_list)
{
    local my_log: Info;
    local origin: string;
    local identifier: string;
    # local event_json_string: string;
    local event_kv_string: string;

    # Is the header from a client request or server response
    if ( is_orig )
        origin = "client";
    else
        origin = "server";

    # If we don't have a header_info_vector than punt
    if ( ! c?$http || ! c$http?$header_info_vector )
        return;

    print c$http$header_info_vector;

    # At this point our c$header_info_vector should contain all the
    # name/value pairs associated with the header, so will turn the
    # vector into a JSON string and add it to our log file.
    # event_json_string = vector_to_json_string(c$http$header_info_vector);
    event_kv_string = vector_to_kv_string(c$http$header_info_vector);

    # Okay now set the user agent field
    identifier = "NOTPRESENT";
    for ( i in hlist )
    {
        if ( origin == "client" )
            if ( hlist[i]$name == "USER-AGENT" ){identifier = hlist[i]$value;}
        if ( origin == "server" )
            if ( hlist[i]$name == "SERVER" ){identifier = hlist[i]$value;}
    }

    # Now add all the info and the event list to the log
    my_log = [$ts=c$start_time,
        $uid=c$uid,
        $origin=fmt("%s", origin),
        $identifier=fmt("%s", identifier),
        $header_events_kv=fmt("%s", event_kv_string)];

        # $header_events_kv=fmt("%s", event_kv_string),
        # $header_events_json=fmt("%s", event_json_string)];

    Log::write(HTTPHeaders::LOG, my_log);
}


Adding the SMTP rejected connections counting.

# For this script to work, you must define a local_mail table
# listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };

@ifdef ( local_mail )

@load conn-id
@load smtp

module SMTP;

type smtp_counter: record {
 rejects: count;
 total: count;
 connects: set[conn_id];
 rej_conns: set[conn_id];
 recipients: string;
};

export {
 # The idea for this is that if a host makes more than spam_threshold
 # smtp connections per hour of which at least spam_percent of those are
 # rejected and the host is not a known mail sending host, then it's likely
 # sending spam or viruses.
 #
 const spam_threshold = 300 &redef;
 const spam_percent = 30 &redef;

 # These are smtp status codes that are considered "rejected".
 const smtp_reject_codes: set[count] = {
   501, # Bad sender address syntax
   550, # Requested action not taken: mailbox unavailable
   551, # User not local; please try <forward-path>
   553, # Requested action not taken: mailbox name not allowed
   554, # Transaction failed
 };

 redef enum Notice += {
   SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
   SMTP_PossibleInternalSpam, # Internal host seems to be spamming
   SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
 };

 global smtp_status_comparison: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
		             msg: string, cont_resp: bool)
{
 if ( c$id$orig_h !in smtp_status_comparison )
 {
   local bar: set[conn_id];
   local blarg: set[conn_id];
   smtp_status_comparison[c$id$orig_h] = [$rejects=0, $total=0, $connects=bar, $rej_conns=blarg, $recipients=""];
 }

 # Set the smtp_counter to the local var "foo"
 local foo = smtp_status_comparison[c$id$orig_h];

 if ( code in smtp_reject_codes &&
           c$id !in foo$rej_conns )
 {
   ++foo$rejects;
   local session = smtp_sessions[c$id];
   if( foo$recipients == "" )
   {
     foo$recipients = session$recipients;
   } else if (session$recipients != "") {
     foo$recipients = cat(foo$recipients, ",", session$recipients);
   }
   add foo$rej_conns[c$id];
 }

 if ( c$id !in foo$connects )
 {
   ++foo$total;
   add foo$connects[c$id];
 }

 local host = c$id$orig_h;
 if ( foo$total >= spam_threshold ) {
   local percent = (foo$rejects*100) / foo$total;
   if ( percent >= spam_percent ) {
     if ( host in local_mail )
       NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("%s is rejecting a high percentage of mail", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else if ( is_local_addr(host) )
       NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent mailto: %s", foo$total, percent, foo$recipients), $conn=c]);
     else
       NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
   }
 }
}

@endif

error: pathspec 'smtp-ext-count-rejects.bro' did not match any file(s) known to git


updating the scan_cmd to also include the python keyword--this addresses edge cases where the shell wasn't handling the shebang properly

# knifehands
# take extracted files and submit to FSF

event file_state_remove(f: fa_file)
    {
        if ( f$info?$extracted )
        {
               # invoke the FSF-CLIENT and add the source metadata of ROCK01 (sensorID), we're suppressing the returned report
               # becuase we don't need that
               local script_path = cat(@DIR, "/fsf-client/fsf_client.py");
	       local scan_cmd = fmt("python %s --suppress-report --archive none --source %s %s%s", script_path, ROCK::sensor_id, FileExtract::prefix, f$info$extracted);
               system(scan_cmd);
         }
}

# knifehands
# take extracted files and submit to FSF

event file_state_remove(f: fa_file)
    {
        if ( f$info?$extracted )
        {
               # invoke the FSF-CLIENT and add the source metadata of ROCK01 (sensorID), we're suppressing the returned report
               # becuase we don't need that
               local script_path = cat(@DIR, "/fsf-client/fsf_client.py");
               local scan_cmd = fmt("%s --suppress-report --archive none --source %s %s/%s", script_path, ROCK::sensor_id, FileExtract::prefix, f$info$extracted);
               system(scan_cmd);
         }
}


include auxiliary information in the ssh notices for client version and country code

@load global-ext
@load ssh
@load notice

type ssh_ext_session_info: record {
	start_time: time;
	client: string &default="";
	server: string &default="";
	location: geo_location;
	status: string &default="";
	direction: string &default="";
	resp_size: count &default=0;
};

# Define the generic ssh-ext event that can be handled from other scripts
global ssh_ext: event(id: conn_id, si: ssh_ext_session_info);

module SSH;

export {
	const password_guesses_limit = 30 &redef;
	const authentication_data_size = 5500 &redef;
	const guessing_timeout = 30 mins;

	redef enum Notice += {
		SSH_Login,
		SSH_PasswordGuessing,
		SSH_LoginByPasswordGuesser,
		SSH_Login_From_Strange_Hostname,
		SSH_Bytecount_Inconsistency,
	};

	# Keeps count of how many rejections a host has had
	global password_rejections: table[addr] of track_count
		&default=default_track_count
		&write_expire=guessing_timeout;

	# Keeps track of hosts identified as guessing passwords
	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr;

	# The list of active SSH connections and the associated session info.
	global active_ssh_conns: table[conn_id] of ssh_ext_session_info &read_expire=2mins;

	# If you want to lookup and log geoip data in the event of a failed login.
	const log_geodata_on_failure = F &redef;

	# The set of countries for which you'd like to throw notices upon successful login
	#   requires Bro compiled with libGeoIP support
	const watched_countries: set[string] = {"RO"} &redef;

	# Strange/bad host names to originate successful SSH logins
	const strange_hostnames =
			/^d?ns[0-9]*\./ |
			/^smtp[0-9]*\./ |
			/^mail[0-9]*\./ |
			/^pop[0-9]*\./  |
			/^imap[0-9]*\./ |
			/^www[0-9]*\./  |
			/^ftp[0-9]*\./  &redef;

	# This is a table with orig subnet as the key, and subnet as the value.
	const ignore_guessers: table[subnet] of subnet &redef;
}

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a successful ssh login happens from or to a watched country
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_Login && n$aux["country"] in SSH::watched_countries); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a successful ssh login happens from a strange client version
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_Login && /lib/ in n$aux["client"]); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a password guesser logs in successfully anywhere
#	# To avoid false positives, setting the lower bound for notification to 50 bad password attempts.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_LoginByPasswordGuesser && n$n > 50); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a local host is password guessing.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_PasswordGuessing &&
#		          is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

# Don't stop processing SSH connections in the default ssh policy script
redef skip_processing_after_handshake = F;

event check_ssh_connection(c: connection, done: bool)
	{
	# If this is no longer a known SSH connection, just return.
	if ( c$id !in active_ssh_conns )
		return;

	# If this is still a live connection and the byte count has not
	# crossed the threshold, just return and let the resheduled check happen later.
	if ( !done && c$resp$size < authentication_data_size )
		return;

	# Make sure the server has sent back more than 50 bytes to filter out
	# hosts that are just port scanning.  Nothing is ever logged if the server
	# doesn't send back at least 50 bytes.
	if (c$resp$size < 50)
		return;

	local ssh_conn = active_ssh_conns[c$id];
	local status = "failure";
	local direction = is_local_addr(c$id$orig_h) ? "to" : "from";
	local location: geo_location;
	# Need to give these values defaults in bro.init.
	location$country_code=""; location$region=""; location$city=""; location$latitude=0.0; location$longitude=0.0;

	if ( done && c$resp$size < authentication_data_size )
		{
		# presumed failure
		if ( log_geodata_on_failure )
			location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);

		if ( c$id$orig_h !in password_rejections )
			password_rejections[c$id$orig_h] = default_track_count(c$id$orig_h);

		# Track the number of rejections
		if ( !(c$id$orig_h in ignore_guessers &&
		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
			++password_rejections[c$id$orig_h]$n;

		if ( default_check_threshold(password_rejections[c$id$orig_h]) )
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_PasswordGuessing,
			        $conn=c,
			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n),
			        $aux=table(["client"]=ssh_conn$client, ["country"]=location$country_code),
			        $n=password_rejections[c$id$orig_h]$n]);
			}
		}
	# TODO: This is to work around a quasi-bug in Bro which occasionally
	#       causes the byte count to be oversized.
	else if (c$resp$size < 20000000)
		{
		# presumed successful login
		status = "success";
		location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);

		if ( password_rejections[c$id$orig_h]$n > password_guesses_limit &&
		     c$id$orig_h !in password_guessers)
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_LoginByPasswordGuesser,
			        $conn=c,
			        $n=password_rejections[c$id$orig_h]$n,
			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
			        $aux=table(["client"]=ssh_conn$client, ["country"]=location$country_code),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]);
			}

		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
		              direction, location$country_code, location$region, location$city,
		              location$latitude, location$longitude,
		              numeric_id_string(c$id), c$resp$size);
		# TODO: rewrite the message once a location variable can be put in notices
		NOTICE([$note=SSH_Login,
		        $conn=c,
		        $msg=message,
		        $aux=table(["client"]=ssh_conn$client, ["country"]=location$country_code)]);

		# Check to see if this login came from a weird hostname (nameserver, mail server, etc.)
		when( local hostname = lookup_addr(c$id$orig_h) )
			{
			if ( strange_hostnames in hostname )
				{
				NOTICE([$note=SSH_Login_From_Strange_Hostname,
				        $conn=c,
				        $msg=fmt("Strange login from %s", hostname),
				        $aux=table(["client"]=ssh_conn$client, ["country"]=location$country_code),
				        $sub=hostname]);
				}
			}
		}
	else if (c$resp$size >= 200000000)
		{
		NOTICE([$note=SSH_Bytecount_Inconsistency,
		        $conn=c,
		        $msg="During byte counting in extended SSH analysis, an overly large value was seen.",
		        $sub=fmt("%d",c$resp$size)]);
		}

	ssh_conn$start_time = c$start_time;
	ssh_conn$location = location;
	ssh_conn$status = status;
	ssh_conn$direction = direction;
	ssh_conn$resp_size = c$resp$size;

	event ssh_ext(c$id, ssh_conn);

	delete active_ssh_conns[c$id];
	# Stop watching this connection, we don't care about it anymore.
	skip_further_processing(c$id);
	set_record_packets(c$id, F);
	}

event connection_state_remove(c: connection)
	{
	event check_ssh_connection(c, T);
	}

event ssh_watcher(c: connection)
	{
	local id = c$id;
	# don't go any further if this connection is gone already!
	if ( !connection_exists(id) )
		{
		delete active_ssh_conns[id];
		return;
		}

	event check_ssh_connection(c, F);
	if ( c$id in active_ssh_conns )
		schedule +15secs { ssh_watcher(c) };
	}

event ssh_client_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$client = version;
	}

event ssh_server_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$server = version;
	}

event protocol_confirmation(c: connection, atype: count, aid: count)
	{
	if ( atype == ANALYZER_SSH )
		{
		local tmp: ssh_ext_session_info;
		active_ssh_conns[c$id]=tmp;
		schedule +15secs { ssh_watcher(c) };
		}
	}

@load global-ext
@load ssh
@load notice

type ssh_ext_session_info: record {
	start_time: time;
	client: string &default="";
	server: string &default="";
	location: geo_location;
	status: string &default="";
	direction: string &default="";
	resp_size: count &default=0;
};

# Define the generic ssh-ext event that can be handled from other scripts
global ssh_ext: event(id: conn_id, si: ssh_ext_session_info);

module SSH;

export {
	const password_guesses_limit = 30 &redef;
	const authentication_data_size = 5500 &redef;
	const guessing_timeout = 30 mins;

	redef enum Notice += {
		SSH_Login,
		SSH_PasswordGuessing,
		SSH_LoginByPasswordGuesser,
		SSH_Login_From_Strange_Hostname,
		SSH_Bytecount_Inconsistency,
	};

	# Keeps count of how many rejections a host has had
	global password_rejections: table[addr] of track_count
		&default=default_track_count
		&write_expire=guessing_timeout;

	# Keeps track of hosts identified as guessing passwords
	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr;

	# The list of active SSH connections and the associated session info.
	global active_ssh_conns: table[conn_id] of ssh_ext_session_info &read_expire=2mins;

	# If you want to lookup and log geoip data in the event of a failed login.
	const log_geodata_on_failure = F &redef;

	# The set of countries for which you'd like to throw notices upon successful login
	#   requires Bro compiled with libGeoIP support
	const watched_countries: set[string] = {"RO"} &redef;

	# Strange/bad host names to originate successful SSH logins
	const strange_hostnames =
			/^d?ns[0-9]*\./ |
			/^smtp[0-9]*\./ |
			/^mail[0-9]*\./ |
			/^pop[0-9]*\./  |
			/^imap[0-9]*\./ |
			/^www[0-9]*\./  |
			/^ftp[0-9]*\./  &redef;

	# This is a table with orig subnet as the key, and subnet as the value.
	const ignore_guessers: table[subnet] of subnet &redef;
}

# Examples for how to handle notices from this script.
#     (define these in a local script)...
#redef notice_policy += {
#	# Send email if a successful ssh login happens from or to a watched country
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_Login && n$sub in SSH::watched_countries); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a password guesser logs in successfully anywhere
#	# To avoid false positives, setting the lower bound for notification to 50 bad password attempts.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_LoginByPasswordGuesser && n$n > 50); },
#	 $result = NOTICE_EMAIL],
#
#	# Send email if a local host is password guessing.
#	[$pred(n: notice_info) =
#		{ return (n$note == SSH::SSH_PasswordGuessing &&
#		          is_local_addr(n$conn$id$orig_h)); },
#	 $result = NOTICE_EMAIL],
#};

# Don't stop processing SSH connections in the default ssh policy script
redef skip_processing_after_handshake = F;

event check_ssh_connection(c: connection, done: bool)
	{
	# If this is no longer a known SSH connection, just return.
	if ( c$id !in active_ssh_conns )
		return;

	# If this is still a live connection and the byte count has not
	# crossed the threshold, just return and let the resheduled check happen later.
	if ( !done && c$resp$size < authentication_data_size )
		return;

	# Make sure the server has sent back more than 50 bytes to filter out
	# hosts that are just port scanning.  Nothing is ever logged if the server
	# doesn't send back at least 50 bytes.
	if (c$resp$size < 50)
		return;

	local ssh_conn = active_ssh_conns[c$id];
	local status = "failure";
	local direction = is_local_addr(c$id$orig_h) ? "to" : "from";
	local location: geo_location;
	# Need to give these values defaults in bro.init.
	location$country_code=""; location$region=""; location$city=""; location$latitude=0.0; location$longitude=0.0;

	if ( done && c$resp$size < authentication_data_size )
		{
		# presumed failure
		if ( log_geodata_on_failure )
			location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);

		if ( c$id$orig_h !in password_rejections )
			password_rejections[c$id$orig_h] = default_track_count(c$id$orig_h);

		# Track the number of rejections
		if ( !(c$id$orig_h in ignore_guessers &&
		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
			++password_rejections[c$id$orig_h]$n;

		if ( default_check_threshold(password_rejections[c$id$orig_h]) )
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_PasswordGuessing,
			        $conn=c,
			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n),
			        $n=password_rejections[c$id$orig_h]$n]);
			}
		}
	# TODO: This is to work around a quasi-bug in Bro which occasionally
	#       causes the byte count to be oversized.
	else if (c$resp$size < 20000000)
		{
		# presumed successful login
		status = "success";
		location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);

		if ( password_rejections[c$id$orig_h]$n > password_guesses_limit &&
		     c$id$orig_h !in password_guessers)
			{
			add password_guessers[c$id$orig_h];
			NOTICE([$note=SSH_LoginByPasswordGuesser,
			        $conn=c,
			        $n=password_rejections[c$id$orig_h]$n,
			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]);
			}

		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
		              direction, location$country_code, location$region, location$city,
		              location$latitude, location$longitude,
		              numeric_id_string(c$id), c$resp$size);
		# TODO: rewrite the message once a location variable can be put in notices
		NOTICE([$note=SSH_Login,
		        $conn=c,
		        $msg=message,
		        $sub=location$country_code]);

		# Check to see if this login came from a weird hostname (nameserver, mail server, etc.)
		when( local hostname = lookup_addr(c$id$orig_h) )
			{
			if ( strange_hostnames in hostname )
				{
				NOTICE([$note=SSH_Login_From_Strange_Hostname,
				        $conn=c,
				        $msg=fmt("Strange login from %s", hostname),
				        $sub=hostname]);
				}
			}
		}
	else if (c$resp$size >= 200000000)
		{
		NOTICE([$note=SSH_Bytecount_Inconsistency,
		        $conn=c,
		        $msg="During byte counting in extended SSH analysis, an overly large value was seen.",
		        $sub=fmt("%d",c$resp$size)]);
		}

	ssh_conn$start_time = c$start_time;
	ssh_conn$location = location;
	ssh_conn$status = status;
	ssh_conn$direction = direction;
	ssh_conn$resp_size = c$resp$size;

	event ssh_ext(c$id, ssh_conn);

	delete active_ssh_conns[c$id];
	# Stop watching this connection, we don't care about it anymore.
	skip_further_processing(c$id);
	set_record_packets(c$id, F);
	}

event connection_state_remove(c: connection)
	{
	event check_ssh_connection(c, T);
	}

event ssh_watcher(c: connection)
	{
	local id = c$id;
	# don't go any further if this connection is gone already!
	if ( !connection_exists(id) )
		{
		delete active_ssh_conns[id];
		return;
		}

	event check_ssh_connection(c, F);
	if ( c$id in active_ssh_conns )
		schedule +15secs { ssh_watcher(c) };
	}

event ssh_client_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$client = version;
	}

event ssh_server_version(c: connection, version: string)
	{
	if ( c$id in active_ssh_conns )
		active_ssh_conns[c$id]$server = version;
	}

event protocol_confirmation(c: connection, atype: count, aid: count)
	{
	if ( atype == ANALYZER_SSH )
		{
		local tmp: ssh_ext_session_info;
		active_ssh_conns[c$id]=tmp;
		schedule +15secs { ssh_watcher(c) };
		}
	}


add load.php to regex

add load.php to regex

@load http-ext
@load ipblocker

module HTTP;

export {
    redef enum Notice += {
        HTTP_IncorrectFileTypeBadHost
    };

    const bad_exec_domains =
        /co\.cc/
      | /cx\.cc/
      | /cz\.cc/
        &redef;

    const bad_exec_urls =
        /php.adv=/
      | /http:\/\/[0-9]{1,3}\.[0-9]{1,3}.*\/index\.php\?[^=]+=[^=]+$/ #try to match http://1.2.3.4/index.php?foo=bar
      | /load.php/
        &redef;

    const bad_user_agents =
        /Java\/1/
        &redef;

}

redef notice_action_filters += {
    [HTTP_IncorrectFileTypeBadHost] = notice_exec_ipblocker_dest,
};

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(is_local_addr(id$resp_h))
        return;
    if(! ("identified-files" in si$tags && si$mime_type == "application/x-dosexec"))
        return;

    if( (bad_exec_domains in si$host || bad_exec_urls in si$url)
      ||(/\.exe/ !in si$url && bad_user_agents in si$user_agent)) {
        NOTICE([$note=HTTP_IncorrectFileTypeBadHost,
                $id=id,
                $msg=fmt("EXE Downloaded from bad host %s %s %s", id$orig_h, id$resp_h, si$url),
                $sub="http-ext"
            ]);
        }
}

@load http-ext
@load ipblocker

module HTTP;

export {
    redef enum Notice += {
        HTTP_IncorrectFileTypeBadHost
    };

    const bad_exec_domains =
        /co\.cc/
      | /cx\.cc/
      | /cz\.cc/
        &redef;

    const bad_exec_urls =
        /php.adv=/
      | /http:\/\/[0-9]{1,3}\.[0-9]{1,3}.*\/index\.php\?[^=]+=[^=]+$/ #try to match http://1.2.3.4/index.php?foo=bar
        &redef;

    const bad_user_agents =
        /Java\/1/
        &redef;

}

redef notice_action_filters += {
    [HTTP_IncorrectFileTypeBadHost] = notice_exec_ipblocker_dest,
};

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(is_local_addr(id$resp_h))
        return;
    if(! ("identified-files" in si$tags && si$mime_type == "application/x-dosexec"))
        return;

    if( (bad_exec_domains in si$host || bad_exec_urls in si$url)
      ||(/\.exe/ !in si$url && bad_user_agents in si$user_agent)) {
        NOTICE([$note=HTTP_IncorrectFileTypeBadHost,
                $id=id,
                $msg=fmt("EXE Downloaded from bad host %s %s %s", id$orig_h, id$resp_h, si$url),
                $sub="http-ext"
            ]);
        }
}


maintain a list of known external sites

maintain a list of known external sites

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)example\.com($|:)/ &redef;

    # in site policy, redef HTTP::ignore_external_addr += {1.2.3.4};
    #                 redef HTTP::ignore_external_host += {"www.foo.com"};

    global ignore_external_addr: set[addr]   &redef;
    global ignore_external_host: set[string] &redef;

    #maybe this should be two tables for mapping addr -> string and string -> addr?
    global external_names: set[addr, string] &create_expire=1day &synchronized &persistent;
}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(id$resp_h in ignore_external_addr || si$host in ignore_external_host)
        return;

    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];

        if([id$resp_h, si$host] !in external_names)
            add external_names[id$resp_h, si$host];
    }
}

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)example\.com($|:)/ &redef;

    # in site policy, redef HTTP::ignore_external_addr += {1.2.3.4};
    #                 redef HTTP::ignore_external_host += {"www.foo.com"};

    global ignore_external_addr: set[addr]   &redef;
    global ignore_external_host: set[string] &redef;
}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(id$resp_h in ignore_external_addr || si$host in ignore_external_host)
        return;

    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];
    }
}


apparently I do need a priority

apparently I do need a priority

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)albany\.edu($|:)/ &redef;
}

event http_ext(id: conn_id, si: http_ext_session_info) &priority=1
{
    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];
    }
}

@load http-ext

module HTTP;

export {
    const local_domains = /(^|\.)albany\.edu($|:)/ &redef;
}

event http_ext(id: conn_id, si: http_ext_session_info)
{
    if(is_local_addr(id$resp_h) && local_domains !in si$host) {
        si$force_log = T;
        add si$force_log_reasons["external_name"];
    }
}


WIP: start rebalance feature

WIP: start rebalance feature

@load ./dio_log.bro
@load ./bro_log.bro
@load ./dio_mysql_log.bro
const broker_port: port = 9999/tcp &redef;
redef exit_only_after_terminate = T;
redef Broker::endpoint_name = "bro_receiver";
global dionaea_access: event(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string);
global dionaea_mysql: event(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string);
global get_protocol: function(proto_str: string) : transport_proto;
global log_bro: function(msg: string);
global slaves: table[string] of count;
global connectors: opaque of Broker::Handle;
global add_to_balance: function(peer_name: string);
global remove_from_balance: function(peer_name: string);

event bro_init() {
    log_bro("bro_receiver.bro: bro_init()");
    Broker::enable([$auto_publish=T]);

    Broker::listen(broker_port, "0.0.0.0");

    Broker::subscribe_to_events("bro/forwarder");

    ## create a distributed datastore for the connector to link against:
    connectors = Broker::create_master("connectors");

    log_bro("bro_receiver.bro: bro_init() done");
}
event bro_done() {
    log_bro("bro_receiver.bro: bro_done()");
}
event dionaea_access(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string) {
    log_bro("Bro Master log access event!");
    local sport: port = count_to_port(src_port, get_protocol(transport));
    local dport: port = count_to_port(dst_port, get_protocol(transport));
    print fmt("dionaea_access: timestamp=%s, dst_ip=%s, dst_port=%s, src_hostname=%s, src_ip=%s, src_port=%s, transport=%s, protocol=%s, connector_id=%s", timestamp, dst_ip, dst_port, src_hostname, src_ip, src_port, transport, protocol, connector_id);
    local rec: Dio_access::Info = [$ts=timestamp, $dst_ip=dst_ip, $dst_port=dport, $src_hostname=src_hostname, $src_ip=src_ip, $src_port=sport, $transport=transport, $protocol=protocol, $connector_id=connector_id];

    Log::write(Dio_access::LOG, rec);
}
event dionaea_mysql(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string) {
    log_bro("Bro Master log mysql event!");
    local lport: port = count_to_port(local_port, get_protocol(transport));
    local rport: port = count_to_port(remote_port, get_protocol(transport));
    print fmt("dionaea_mysql: timestamp=%s, id=%s, local_ip=%s, local_port=%s, remote_ip=%s, remote_port=%s, transport=%s, args=%s, connector_id=%s", timestamp, id, local_ip, local_port, remote_ip, remote_port, transport, args, connector_id);
    local rec: Dio_mysql::Info = [$ts=timestamp, $id=id, $local_ip=local_ip, $local_port=lport, $remote_ip=remote_ip, $remote_port=rport, $transport=transport, $args=args, $connector_id=connector_id];
    Log::write(Dio_mysql::LOG, rec);
}
event Broker::incoming_connection_established(peer_name: string) {
    local inc: string = "Incoming connection extablished " + peer_name;
    log_bro(inc);
    add_to_balance(peer_name);
}
event Broker::incoming_connection_broken(peer_name: string) {
    local inc: string = "Incoming connection broken for " + peer_name;
    log_bro(inc);
    remove_from_balance(peer_name);
}
function get_protocol(proto_str: string) : transport_proto {
    # https://www.bro.org/sphinx/scripts/base/init-bare.bro.html#type-transport_proto
    if (proto_str == "tcp") {
        return tcp;
    }
    if (proto_str == "udp") {
        return udp;
    }
    if (proto_str == "icmp") {
        return icmp;
    }
    return unknown_transport;
}

function log_bro(msg:string) {
    local rec: Brolog::Info = [$msg=msg];
    Log::write(Brolog::LOG, rec);
}

function add_to_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        log_bro("Registering new slave " + peer_name);
        slaves[peer_name] = 0;
        local total_slaves = |slaves|;
        local total_connectors = 0;
        local slave_table = vector of string;
        local i = 0;
        for (slave in slaves) {
            total_connectors += slaves[slave];
            slave_table[i] = slave;
            ++i;
        }
        i = 0;
        when (local keys = Broker::keys(connectors)) {
            local connector_table = vector of string;
            while (i < total_connectors) {
                connector_table[i] = Broker::vector_lookup(keys$result, i);
            }

            print "Starting add slave to balance:";
            print "total_slaves", total_slaves, "total_connectors", total_connectors;
            while (total_slaves > 0) {
                local balance_amount = balance_amount = total_connectors;
                if (total_connectors % total_slaves > 0) {
                    balance_amount = total_connectors / total_slaves + 1;
                }
                --total_slaves;
                total_connectors -= balance_amount;
                print "total_slaves", total_slaves, "total_connectors", total_connectors;
                # TODO: set balanceamount as value for one slave
                # TODO: set for all those connectors that one slave
            }

        }
        timeout 100msec {
            print "aww";
        }

    }
    if(/beemaster-connector-/ in peer_name) {
        log_bro("Registering new connector " + peer_name);
        local min_count_conn = 100000;
        local best_slave = "";

        local size = |slaves|;

        if (size <= 0) {
            log_bro("No slaves are registered, cannot register connector " + peer_name);
            print "should return";
            return;
        }
        for(slave in slaves) {
            local count_conn = slaves[slave];
            if (count_conn < min_count_conn) {
                best_slave = slave;
                min_count_conn = count_conn;
            }
        }
        if (best_slave != "") {
            Broker::insert(connectors, Broker::data(peer_name), Broker::data(best_slave));
            ++slaves[best_slave];
            print "Registered connector", peer_name, "and balanced to", best_slave;
            log_bro("Registered connector " + peer_name + " and balanced to " + best_slave);
        }
        else {
            print "Could not register connector ", peer_name;
            log_bro("Could not register connector " + peer_name);
        }

    }
}

function remove_from_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        log_bro("Unregistering old slave " + peer_name);
        delete slaves[peer_name];
        # TODO rebalance
    }
    if(/beemaster-connector-/ in peer_name) {
        when(local cs = Broker::lookup(connectors, Broker::data(peer_name))) {
            local connected_slave = Broker::refine_to_string(cs$result);
            local count_conn = slaves[connected_slave];
            if (count_conn > 0) {
                slaves[connected_slave] = count_conn - 1;
            }
            Broker::erase(connectors, Broker::data(peer_name));
            print "Unregistered old connector", peer_name, "leaving slave table", slaves;
            log_bro("Unregistered old connector " + peer_name);
        }
        timeout 100msec {
            print "Timeout unregistering connector", peer_name;
            log_bro("Timeout unregistering connector " + peer_name);
        }
    }
}

@load ./dio_log.bro
@load ./bro_log.bro
@load ./dio_mysql_log.bro
const broker_port: port = 9999/tcp &redef;
redef exit_only_after_terminate = T;
redef Broker::endpoint_name = "bro_receiver";
global dionaea_access: event(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string);
global dionaea_mysql: event(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string);
global get_protocol: function(proto_str: string) : transport_proto;
global log_bro: function(msg: string);
global slaves: table[string] of count;
global connectors: opaque of Broker::Handle;
global add_to_balance: function(peer_name: string);
global remove_from_balance: function(peer_name: string);

event bro_init() {
    log_bro("bro_receiver.bro: bro_init()");
    Broker::enable([$auto_publish=T]);

    Broker::listen(broker_port, "0.0.0.0");

    Broker::subscribe_to_events("bro/forwarder");

    ## create a distributed datastore for the connector to link against:
    connectors = Broker::create_master("connectors");

    log_bro("bro_receiver.bro: bro_init() done");
}
event bro_done() {
    log_bro("bro_receiver.bro: bro_done()");
}
event dionaea_access(timestamp: time, dst_ip: addr, dst_port: count, src_hostname: string, src_ip: addr, src_port: count, transport: string, protocol: string, connector_id: string) {
    log_bro("Bro Master log access event!");
    local sport: port = count_to_port(src_port, get_protocol(transport));
    local dport: port = count_to_port(dst_port, get_protocol(transport));
    print fmt("dionaea_access: timestamp=%s, dst_ip=%s, dst_port=%s, src_hostname=%s, src_ip=%s, src_port=%s, transport=%s, protocol=%s, connector_id=%s", timestamp, dst_ip, dst_port, src_hostname, src_ip, src_port, transport, protocol, connector_id);
    local rec: Dio_access::Info = [$ts=timestamp, $dst_ip=dst_ip, $dst_port=dport, $src_hostname=src_hostname, $src_ip=src_ip, $src_port=sport, $transport=transport, $protocol=protocol, $connector_id=connector_id];

    Log::write(Dio_access::LOG, rec);
}
event dionaea_mysql(timestamp: time, id: string, local_ip: addr, local_port: count, remote_ip: addr, remote_port: count, transport: string, args: string, connector_id: string) {
    log_bro("Bro Master log mysql event!");
    local lport: port = count_to_port(local_port, get_protocol(transport));
    local rport: port = count_to_port(remote_port, get_protocol(transport));
    print fmt("dionaea_mysql: timestamp=%s, id=%s, local_ip=%s, local_port=%s, remote_ip=%s, remote_port=%s, transport=%s, args=%s, connector_id=%s", timestamp, id, local_ip, local_port, remote_ip, remote_port, transport, args, connector_id);
    local rec: Dio_mysql::Info = [$ts=timestamp, $id=id, $local_ip=local_ip, $local_port=lport, $remote_ip=remote_ip, $remote_port=rport, $transport=transport, $args=args, $connector_id=connector_id];
    Log::write(Dio_mysql::LOG, rec);
}
event Broker::incoming_connection_established(peer_name: string) {
    local inc: string = "Incoming connection extablished " + peer_name;
    log_bro(inc);
    add_to_balance(peer_name);
}
event Broker::incoming_connection_broken(peer_name: string) {
    local inc: string = "Incoming connection broken for " + peer_name;
    log_bro(inc);
    remove_from_balance(peer_name);
}
function get_protocol(proto_str: string) : transport_proto {
    # https://www.bro.org/sphinx/scripts/base/init-bare.bro.html#type-transport_proto
    if (proto_str == "tcp") {
        return tcp;
    }
    if (proto_str == "udp") {
        return udp;
    }
    if (proto_str == "icmp") {
        return icmp;
    }
    return unknown_transport;
}

function log_bro(msg:string) {
    local rec: Brolog::Info = [$msg=msg];
    Log::write(Brolog::LOG, rec);
}

function add_to_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        log_bro("Registering new slave " + peer_name);
        slaves[peer_name] = 0;
        # TODO realance clients to this new slave
    }
    if(/beemaster-connector-/ in peer_name) {
        log_bro("Registering new connector " + peer_name);
        local min_count_conn = 100000;
        local best_slave = "";

        local size = |slaves|;

        if (size <= 0) {
            log_bro("No slaves are registered, cannot register connector " + peer_name);
            print "should return";
            return;
        }

        for(slave in slaves) {
            local count_conn = slaves[slave];

            if (count_conn < min_count_conn) {
                best_slave = slave;
                min_count_conn = count_conn;
            }
        }
        if (best_slave != "") {
            Broker::insert(connectors, Broker::data(peer_name), Broker::data(best_slave));
            ++slaves[best_slave];
            print "Registered connector", peer_name, "and balanced to", best_slave;
            log_bro("Registered connector " + peer_name + " and balanced to " + best_slave);
        }
        else {
            print "Could not register connector ", peer_name;
            log_bro("Could not register connector " + peer_name);
        }

    }
}

function remove_from_balance(peer_name: string) {
    if(/bro-slave-/ in peer_name) {
        log_bro("Unregistering old slave " + peer_name);
        delete slaves[peer_name];
        # TODO rebalance
    }
    if(/beemaster-connector-/ in peer_name) {
        when(local cs = Broker::lookup(connectors, Broker::data(peer_name))) {
            local connected_slave = Broker::refine_to_string(cs$result);
            local count_conn = slaves[connected_slave];
            if (count_conn > 0) {
                slaves[connected_slave] = count_conn - 1;
            }
            Broker::erase(connectors, Broker::data(peer_name));
            print "Unregistered old connector", peer_name, "leaving slave table", slaves;
            log_bro("Unregistered old connector " + peer_name);
        }
        timeout 100msec {
            print "Timeout unregistering connector", peer_name;
            log_bro("Timeout unregistering connector " + peer_name);
        }
    }
}


include the subject in the SMTP_PossiblePWPhishReply notice

@load global-ext
@load smtp-ext

module PHISH;

global smtp_password_conns: set[conn_id] &read_expire=2mins;

export {
    redef enum Notice += {
        SMTP_PossiblePWPhish,
        SMTP_PossiblePWPhishReply,
    };
    global phishing_counter: table[string] of count &default=0 &create_expire=1hr &synchronized;
    global phishing_reply_tos: set[string] &synchronized &redef;
    global phishing_ignore_froms: set[string] &redef;
    global phishing_threshold = 50;

    const phish_keywords =
          /[pP][aA][sS][sS][wW][oO][rR][dD]/
        | /[uU][sS][eE][rR].?[nN][aA][mM][eE]/
        | /[nN][eE][tT][iI][dD]/ &redef;
}

event bro_init()
{
    LOG::create_logs("password-mail", All, F, T);
    LOG::define_header("password-mail", cat_sep("\t", "",
                                          "ts",
                                          "orig_h", "orig_p",
                                          "resp_h", "resp_p",
                                          "helo", "message-id", "in-reply-to",
                                          "mailfrom", "rcptto",
                                          "date", "from", "reply_to", "to", "subject",
                                          "files", "last_reply", "x-originating-ip",
                                          "path", "is_webmail", "agent"));
}

event bro_done()
{
    print "Counter";
    print phishing_counter;
    print "bad reply-tos";
    print phishing_reply_tos;
}

event smtp_data(c: connection, is_orig: bool, data: string)
{
    if(is_local_addr(c$id$orig_h))
        return;
    # look for 'password'
    if(phish_keywords in data)
        add smtp_password_conns[c$id];
}

event smtp_ext(id: conn_id, si: smtp_ext_session_info)
{
    if(is_local_addr(id$orig_h)) {
        for (to in si$rcptto){
            if(to in phishing_reply_tos){
                NOTICE([$note=SMTP_PossiblePWPhishReply,
                        $msg=fmt("%s replied to %s - %s", si$mailfrom, to, si$subject),
                        $sub=si$mailfrom
                      ]);
            }
        }
    } else {
        if (id !in smtp_password_conns)
            return;
        if(si$mailfrom in phishing_ignore_froms)
            return;
        if(++(phishing_counter[si$mailfrom]) > phishing_threshold){
            local to_add ="";
            if(si$reply_to != "")
                to_add = si$reply_to;
            else
                to_add = si$mailfrom;
            if(to_add !in phishing_reply_tos){
                add phishing_reply_tos[to_add];
                NOTICE([$note=SMTP_PossiblePWPhish,
                        $msg=fmt("%s(%s) may be phishing - %s", si$mailfrom, si$reply_to, si$subject),
                        $sub=si$mailfrom
                      ]);
            }
        }

        local log = LOG::get_file_by_id("password-mail", id, F);
        print log, cat_sep("\t", "\\N",
                    network_time(),
                    id$orig_h, port_to_count(id$orig_p), id$resp_h, port_to_count(id$resp_p),
                    si$helo,
                    si$msg_id,
                    si$in_reply_to,
                    si$mailfrom,
                    fmt_str_set(si$rcptto, /["'<>]|([[:blank:]].*$)/),
                    si$date,
                    si$from,
                    si$reply_to,
                    fmt_str_set(si$to, /["']/),
                    si$subject,
                    fmt_str_set(si$files, /["']/),
                    si$last_reply,
                    si$x_originating_ip,
                    si$path,
                    si$is_webmail,
                    si$agent);
    }

}

@load global-ext
@load smtp-ext

module PHISH;

global smtp_password_conns: set[conn_id] &read_expire=2mins;

export {
    redef enum Notice += {
        SMTP_PossiblePWPhish,
        SMTP_PossiblePWPhishReply,
    };
    global phishing_counter: table[string] of count &default=0 &create_expire=1hr &synchronized;
    global phishing_reply_tos: set[string] &synchronized &redef;
    global phishing_ignore_froms: set[string] &redef;
    global phishing_threshold = 50;

    const phish_keywords =
          /[pP][aA][sS][sS][wW][oO][rR][dD]/
        | /[uU][sS][eE][rR].?[nN][aA][mM][eE]/
        | /[nN][eE][tT][iI][dD]/ &redef;
}

event bro_init()
{
    LOG::create_logs("password-mail", All, F, T);
    LOG::define_header("password-mail", cat_sep("\t", "",
                                          "ts",
                                          "orig_h", "orig_p",
                                          "resp_h", "resp_p",
                                          "helo", "message-id", "in-reply-to",
                                          "mailfrom", "rcptto",
                                          "date", "from", "reply_to", "to", "subject",
                                          "files", "last_reply", "x-originating-ip",
                                          "path", "is_webmail", "agent"));
}

event bro_done()
{
    print "Counter";
    print phishing_counter;
    print "bad reply-tos";
    print phishing_reply_tos;
}

event smtp_data(c: connection, is_orig: bool, data: string)
{
    if(is_local_addr(c$id$orig_h))
        return;
    # look for 'password'
    if(phish_keywords in data)
        add smtp_password_conns[c$id];
}

event smtp_ext(id: conn_id, si: smtp_ext_session_info)
{
    if(is_local_addr(id$orig_h)) {
        for (to in si$rcptto){
            if(to in phishing_reply_tos){
                NOTICE([$note=SMTP_PossiblePWPhishReply,
                        $msg=fmt("%s replied to %s", si$mailfrom, to),
                        $sub=si$mailfrom
                      ]);
            }
        }
    } else {
        if (id !in smtp_password_conns)
            return;
        if(si$mailfrom in phishing_ignore_froms)
            return;
        if(++(phishing_counter[si$mailfrom]) > phishing_threshold){
            local to_add ="";
            if(si$reply_to != "")
                to_add = si$reply_to;
            else
                to_add = si$mailfrom;
            if(to_add !in phishing_reply_tos){
                add phishing_reply_tos[to_add];
                NOTICE([$note=SMTP_PossiblePWPhish,
                        $msg=fmt("%s(%s) may be phishing - %s", si$mailfrom, si$reply_to, si$subject),
                        $sub=si$mailfrom
                      ]);
            }
        }

        local log = LOG::get_file_by_id("password-mail", id, F);
        print log, cat_sep("\t", "\\N",
                    network_time(),
                    id$orig_h, port_to_count(id$orig_p), id$resp_h, port_to_count(id$resp_p),
                    si$helo,
                    si$msg_id,
                    si$in_reply_to,
                    si$mailfrom,
                    fmt_str_set(si$rcptto, /["'<>]|([[:blank:]].*$)/),
                    si$date,
                    si$from,
                    si$reply_to,
                    fmt_str_set(si$to, /["']/),
                    si$subject,
                    fmt_str_set(si$files, /["']/),
                    si$last_reply,
                    si$x_originating_ip,
                    si$path,
                    si$is_webmail,
                    si$agent);
    }

}


Update detect-rogue-dns.bro

Update detect-rogue-dns.bro

Moved alexa_file to export

# Detects a Hurricane Panda tactic of using Hurricane Electric to resolve commonly accessed websites
# Alerts when a domain in the Alexa top 500 is resolved via Hurricane Electric and/or when a host connects to an IP in the DNS response
# CrowdStrike 2015
# josh.liburdi@crowdstrike.com

@load base/protocols/dns
@load base/frameworks/notice
@load base/frameworks/input

module CrowdStrike::Hurricane_Panda;

export {
  redef enum Notice::Type += {
    HE_Request,
    HE_Successful_Conn
  };

  # File generated by scrape-alexa.py
  const alexa_file = "./alexa_domains.txt" &redef;
}

# Table to store list of domains in file above
global alexa_table: set[string];
# Record for domains in file above
type alexa_idx: record {
  alexa: string;
};

const he_nets: set[subnet] = {
  216.218.130.2,
  216.66.1.2,
  216.66.80.18,
  216.218.132.2,
  216.218.131.2
} &redef;

# Pattern used to identify subdomains
const subdomains =  /^d?ns[0-9]*\./    |
                    /^smtp[0-9]*\./    |
                    /^mail[0-9]*\./    |
                    /^pop[0-9]*\./     |
                    /^imap[0-9]*\./    |
                    /^www[0-9]*\./     |
                    /^ftp[0-9]*\./     |
                    /^img[0-9]*\./     |
                    /^images?[0-9]*\./ |
                    /^search[0-9]*\./  |
                    /^nginx[0-9]*\./ &redef;

# Container to store IP address answers from Hurricane Electric queries
# Each entry expires 5 minutes after the last time it was written
global he_answers: set[addr] &write_expire=5min;

# Pulls data from scrape-alexa.py output (alexa_file)
event bro_init()
{
Input::add_table([$source=alexa_file,
                  $name="alexa_domains",
                  $idx=alexa_idx,
                  $destination=alexa_table,
                  $mode=Input::REREAD]);
}

event DNS::log_dns(rec: DNS::Info)
{
# Do not process the event if no query exists or if the query is not being resolved by Hurricane Electric
if ( ! rec?$query ) return;
if ( rec$id$resp_h !in he_nets ) return;

# If necessary, clean the query so that it can be found in the list of Alexa domains
local query = rec$query;
if ( subdomains in query )
  query = sub(rec$query,subdomains,"");

# Check if the query is in the list of Alexa domains
# If it is, then this activity is suspicious and should be investigated
if ( query in alexa_table )
  {
  # Prepare the sub-message for the notice
  # Include the domain queried in the sub-message
  local sub_msg = fmt("Query: %s",query);

  # If the query was answered, include the answers in the sub-message
  if ( rec?$answers )
    {
    sub_msg += fmt(" %s: ", rec$total_answers > 1 ? "Answers":"Answer");

    for ( ans in rec$answers )
      {
      # If an answers value is an IP address, store it for later processing
      if ( is_valid_ip(rec$answers[ans]) == T )
        add he_answers[to_addr(rec$answers[ans])];
      sub_msg += fmt("%s, ", rec$answers[ans]);
      }

    # Clean the sub-message
    sub_msg = cut_tail(sub_msg,2);
    }

  # Generate the notice
  # Includes the connection flow, host intiating the lookup, domain queried, and query answers (if available)
  NOTICE([$note=HE_Request,
          $msg=fmt("%s made a suspicious DNS lookup to Hurricane Electric", rec$id$orig_h),
          $sub=sub_msg,
          $id=rec$id,
          $uid=rec$uid,
          $identifier=cat(rec$id$orig_h,rec$query)]);
  }
}

event connection_state_remove(c: connection)
{
# Check if a host connected to an IP address seen in an answer from Hurricane Electric
if ( c$id$resp_h in he_answers )
  NOTICE([$note=HE_Successful_Conn,
          $conn=c,
          $msg=fmt("%s connected to a suspicious server resolved by Hurricane Electric", c$id$orig_h),
          $identifier=cat(c$id$orig_h,c$id$resp_h)]);
}

# Detects a Hurricane Panda tactic of using Hurricane Electric to resolve commonly accessed websites
# Alerts when a domain in the Alexa top 500 is resolved via Hurricane Electric and/or when a host connects to an IP in the DNS response
# CrowdStrike 2015
# josh.liburdi@crowdstrike.com

@load base/protocols/dns
@load base/frameworks/notice
@load base/frameworks/input

module CrowdStrike::Hurricane_Panda;

export {
  redef enum Notice::Type += {
    HE_Request,
    HE_Successful_Conn
  };
}

# File generated by scrape-alexa.py
const alexa_file = "./alexa_domains.txt" &redef;
# Table to store list of domains in file above
global alexa_table: set[string];
# Record for domains in file above
type alexa_idx: record {
  alexa: string;
};

const he_nets: set[subnet] = {
  216.218.130.2,
  216.66.1.2,
  216.66.80.18,
  216.218.132.2,
  216.218.131.2
} &redef;

# Pattern used to identify subdomains
const subdomains =  /^d?ns[0-9]*\./    |
                    /^smtp[0-9]*\./    |
                    /^mail[0-9]*\./    |
                    /^pop[0-9]*\./     |
                    /^imap[0-9]*\./    |
                    /^www[0-9]*\./     |
                    /^ftp[0-9]*\./     |
                    /^img[0-9]*\./     |
                    /^images?[0-9]*\./ |
                    /^search[0-9]*\./  |
                    /^nginx[0-9]*\./ &redef;

# Container to store IP address answers from Hurricane Electric queries
# Each entry expires 5 minutes after the last time it was written
global he_answers: set[addr] &write_expire=5min;

# Pulls data from scrape-alexa.py output (alexa_file)
event bro_init()
{
Input::add_table([$source=alexa_file,
                  $name="alexa_domains",
                  $idx=alexa_idx,
                  $destination=alexa_table,
                  $mode=Input::REREAD]);
}

event DNS::log_dns(rec: DNS::Info)
{
# Do not process the event if no query exists or if the query is not being resolved by Hurricane Electric
if ( ! rec?$query ) return;
if ( rec$id$resp_h !in he_nets ) return;

# If necessary, clean the query so that it can be found in the list of Alexa domains
local query = rec$query;
if ( subdomains in query )
  query = sub(rec$query,subdomains,"");

# Check if the query is in the list of Alexa domains
# If it is, then this activity is suspicious and should be investigated
if ( query in alexa_table )
  {
  # Prepare the sub-message for the notice
  # Include the domain queried in the sub-message
  local sub_msg = fmt("Query: %s",query);

  # If the query was answered, include the answers in the sub-message
  if ( rec?$answers )
    {
    sub_msg += fmt(" %s: ", rec$total_answers > 1 ? "Answers":"Answer");

    for ( ans in rec$answers )
      {
      # If an answers value is an IP address, store it for later processing
      if ( is_valid_ip(rec$answers[ans]) == T )
        add he_answers[to_addr(rec$answers[ans])];
      sub_msg += fmt("%s, ", rec$answers[ans]);
      }

    # Clean the sub-message
    sub_msg = cut_tail(sub_msg,2);
    }

  # Generate the notice
  # Includes the connection flow, host intiating the lookup, domain queried, and query answers (if available)
  NOTICE([$note=HE_Request,
          $msg=fmt("%s made a suspicious DNS lookup to Hurricane Electric", rec$id$orig_h),
          $sub=sub_msg,
          $id=rec$id,
          $uid=rec$uid,
          $identifier=cat(rec$id$orig_h,rec$query)]);
  }
}

event connection_state_remove(c: connection)
{
# Check if a host connected to an IP address seen in an answer from Hurricane Electric
if ( c$id$resp_h in he_answers )
  NOTICE([$note=HE_Successful_Conn,
          $conn=c,
          $msg=fmt("%s connected to a suspicious server resolved by Hurricane Electric", c$id$orig_h),
          $identifier=cat(c$id$orig_h,c$id$resp_h)]);
}


Create detect-rfd.bro

Create detect-rfd.bro

# Detect reflected file download attacks as described in
# http://packetstorm.foofus.com/papers/presentations/eu-14-Hafif-Reflected-File-Download-A-New-Web-Attack-Vector.pdf
# CrowdStrike 2015
# josh.liburdi@crowdstrike.com

@load base/frameworks/notice
@load base/protocols/http

module CrowdStrike;

export {
  redef enum Notice::Type += {
    Reflected_File_Download
  };

  redef enum HTTP::Tags += {
    RFD
  };
}

const rfd_content_type: set[string] = {
  "application/json",
  "application/x-javascript",
  "application/javascript",
  "application/notexist",
  "text/json",
  "text/x-javascript",
  "text/plain",
  "text/notexist",
  "application/xml",
  "text/xml",
  "text/html"
} &redef;

const rfd_pattern =   /\"\|\|/ |
                      /\"\<\</ |
                      /\"\>\>/ |
                      /[^?]*\.bat(\;|$)/ |
                      /[^?]*\.cmd(\;|$)/ |
                      /[^?]*[Ss][Ee][Tt][Uu][Pp][:alnum:]*?\.[:alpha:]{3,4}(\;|$)/ |
                      /[^?]*[Ii][nn][Ss][Tt][Aa][Ll][Ll][:alnum:]*?\.[:alpha:]{3,4}(\;|$)/ |
                      /[^?]*[Uu][Pp][Dd][Aa][Tt][Ee][:alnum:]*?\.[:alpha:]{3,4}(\;|$)/ |
                      /[^?]*[Uu][Nn][In][Ss][Tt][:alnum:]*?\.[:alpha:]{3,4}(\;|$)/ &redef;

# Perform a pattern match for reflected file downloads.
# If found, add HTTP tag and generate a notice.
function rfd_do_notice(c: connection)
{
if ( rfd_pattern in c$http$uri )
  {
  add c$http$tags[RFD];
  NOTICE([$note=Reflected_File_Download,
          $msg="A reflected file download was attempted",
          $sub=c$http$uri,
          $id=c$id,
          $identifier=cat(c$id$orig_h,c$id$resp_h,c$http$uri)]);
  }
}

# Check for reflected file downloads in HTTP transactions that have a
# CONTENT-TYPE header. Valid RFD content types are identified anywhere
# in the CONTENT-TYPE header value.
event http_header(c: connection, is_orig: bool, name: string, value: string)
{
if ( is_orig || ! c$http?$uri || c$http$uri == "" ) return;

if ( name == "CONTENT-TYPE" )
  for ( rct in rfd_content_type )
    if ( rct in value )
      rfd_do_notice(c);
}

# Check for reflected file downloads in HTTP transactions that do not have a
# CONTENT-TYPE header.
event http_all_headers(c: connection, is_orig: bool, hlist: mime_header_list)
{
if ( is_orig || ! c$http?$uri || c$http$uri == "" ) return;

local headers: set[string];

for ( h in hlist )
  {
  local name = hlist[h]$name;
  add headers[name];
  }

if ( "CONTENT-TYPE" !in headers )
  rfd_do_notice(c);
else return;
}

error: pathspec 'bro-scripts/rfd/detect-rfd.bro' did not match any file(s) known to git


fixed log path

fixed log path

module Acu_result;

export{
  redef enum Log::ID += { LOG };
  redef LogAscii::empty_field = "EMPTY";

  type Info: record {
    ts: time &log;
    attack: string &log;
  };
}

event bro_init() &priority=5 {
  Log::create_stream(Acu_result::LOG, [$columns=Info, $path="acu_result"]);
}

module Acu_result;

export{
  redef enum Log::ID += { LOG };
  redef LogAscii::empty_field = "EMPTY";

  type Info: record {
    ts: time &log;
    attack: string &log;
  };
}

event bro_init() &priority=5 {
  Log::create_stream(Acu_result::LOG, [$columns=Info, $path="master"]);
}


Mostly just style and naming changes here.

# For the SMTP_StrangeRejectBehavior notice to work, you must define a
# local_mail table listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };
@load smtp

module SMTP;

type smtp_counter: record {
	rejects: count;
	total: count;
};

export {
	# The idea for this is that if a host makes more than spam_threshold
	# smtp connections per hour of which at least spam_percent of those are
	# rejected and the host is not a known mail sending host, then it's likely
	# sending spam or viruses.
	#
	const spam_threshold = 300 &redef;
	const spam_percent = 30 &redef;

	# These are smtp status codes that are considered "rejected".
	const smtp_reject_codes: set[count] = {
		501, # Bad sender address syntax
		550, # Requested action not taken: mailbox unavailable
		551, # User not local; please try <forward-path>
		553, # Requested action not taken: mailbox name not allowed
		554, # Transaction failed
	};

	redef enum Notice += {
		SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
		SMTP_PossibleInternalSpam, # Internal host seems to be spamming
		SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
	};

	# This variable keeps track of the number of rejected and accepted
	# RCPT TO's a host is doing and receiving per hour.
	global reject_counter: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                 msg: string, cont_resp: bool)
	{
	if ( c$id$orig_h !in reject_counter )
		reject_counter[c$id$orig_h] = [$rejects=0, $total=0];

	# Set the smtp_counter to the local var "foo"
	local foo = reject_counter[c$id$orig_h];

	if ( code in smtp_reject_codes)
		++foo$rejects;

	if ( /^([hH]|[eE]){2}[lL][oO]/ in cmd )
		++foo$total;

	local host = c$id$orig_h;
	if ( foo$total >= spam_threshold )
		{
		local percent = (foo$rejects*100) / foo$total;
		if ( percent >= spam_percent )
			{
@ifdef ( local_mail )
			if ( host in local_mail )
				NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("a high percentage of mail from %s is being rejected", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
			else
@endif
			if ( is_local_addr(host) )
				NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
			else
				NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
			}
		}
	}

# For this script to work, you must define a local_mail table
# listing all of your known mail sending hosts.
# i.e.  const local_mail: set[subnet] = { 1.2.3.4/32 };

@ifdef ( local_mail )

@load conn-id
@load smtp

module SMTP;

type smtp_counter: record {
 rejects: count;
 total: count;
};

export {
 # The idea for this is that if a host makes more than spam_threshold
 # smtp connections per hour of which at least spam_percent of those are
 # rejected and the host is not a known mail sending host, then it's likely
 # sending spam or viruses.
 #
 const spam_threshold = 300 &redef;
 const spam_percent = 30 &redef;

 # These are smtp status codes that are considered "rejected".
 const smtp_reject_codes: set[count] = {
   501, # Bad sender address syntax
   550, # Requested action not taken: mailbox unavailable
   551, # User not local; please try <forward-path>
   553, # Requested action not taken: mailbox name not allowed
   554, # Transaction failed
 };

 redef enum Notice += {
   SMTP_PossibleSpam, # Host sending mail *to* internal hosts is suspicious
   SMTP_PossibleInternalSpam, # Internal host seems to be spamming
   SMTP_StrangeRejectBehavior, # Local mail server is getting high numbers of rejects
 };

 global smtp_status_comparison: table[addr] of smtp_counter &create_expire=1hr &redef;
}

event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
		             msg: string, cont_resp: bool)
{
 if ( c$id$orig_h !in smtp_status_comparison )
 {
   smtp_status_comparison[c$id$orig_h] = [$rejects=0, $total=0];
 }

 # Set the smtp_counter to the local var "foo"
 local foo = smtp_status_comparison[c$id$orig_h];

 if ( code in smtp_reject_codes)
   ++foo$rejects;

 if ( /^([hH]|[eE]){2}[lL][oO]/ in cmd )
     ++foo$total;

 local host = c$id$orig_h;
 if ( foo$total >= spam_threshold ) {
   local percent = (foo$rejects*100) / foo$total;
   if ( percent >= spam_percent ) {
     if ( host in local_mail )
       NOTICE([$note=SMTP_StrangeRejectBehavior, $msg=fmt("a high percentage of mail from %s is being rejected", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else if ( is_local_addr(host) )
       NOTICE([$note=SMTP_PossibleInternalSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
     else
       NOTICE([$note=SMTP_PossibleSpam, $msg=fmt("%s appears to be spamming", host), $sub=fmt("sent: %d rejected: %d percent", foo$total, percent), $conn=c]);
   }
 }
}

@endif


Fix performance and behavior issues.

Fix performance and behavior issues.

The connpolling timeout wasn't being set correctly to end polling
for connections that lived long enough to go beyond the defined "long"
intervals.

There was a state tracking table for the currently watched offset that
wasn't being cleaned up.  This was fixed by becoming a field in the
connection.

@load base/protocols/conn
@load base/utils/time

# This is probably not so great to reach into the Conn namespace..
module Conn;

export {
function set_conn_log_data_hack(c: connection)
	{
	Conn::set_conn(c, T);
	}
}

# Now onto the actual code for this script...

module LongConnection;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		## Notice for when a long connection is found.
		## The `sub` field in the notice represents the number
		## of seconds the connection has currently been alive.
		LongConnection::found
	};

	## Aliasing vector of interval values as
	## "Durations"
	type Durations: vector of interval;

	## The default duration that you are locally
	## considering a connection to be "long".
	const default_durations = Durations(10min, 30min, 1hr, 12hr, 24hrs, 3days) &redef;

	## These are special cases for particular hosts or subnets
	## that you may want to watch for longer or shorter
	## durations than the default.
	const special_cases: table[subnet] of Durations = {} &redef;
}

redef record connection += {
	## Offset of the currently watched connection duration by the long-connections script.
	long_conn_offset: count &default=0;
};

event bro_init() &priority=5
	{
	Log::create_stream(LOG, [$columns=Conn::Info, $path="conn_long"]);
	}

function get_durations(c: connection): Durations
	{
	local check_it: Durations;
	if ( c$id$orig_h in special_cases )
		check_it = special_cases[c$id$orig_h];
	else if ( c$id$resp_h in special_cases )
		check_it = special_cases[c$id$resp_h];
	else
		check_it = default_durations;

	return check_it;
	}

function long_callback(c: connection, cnt: count): interval
	{
	local check_it = get_durations(c);

	if ( c$long_conn_offset < |check_it| && c$duration >= check_it[c$long_conn_offset] )
		{
		Conn::set_conn_log_data_hack(c);
		Log::write(LongConnection::LOG, c$conn);

		local message = fmt("%s -> %s:%s remained alive for longer than %s",
		                    c$id$orig_h, c$id$resp_h, c$id$resp_p, duration_to_mins_secs(c$duration));
		NOTICE([$note=LongConnection::found,
		        $msg=message,
		        $sub=fmt("%.2f", c$duration),
		        $conn=c]);

		++c$long_conn_offset;
		}

	# Keep watching if there are potentially more thresholds.
	if ( c$long_conn_offset < |check_it| )
		return check_it[c$long_conn_offset];
	else
		return -1sec;
	}

event connection_established(c: connection)
	{
	local check = get_durations(c);
	if ( |check| > 0 )
		{
		ConnPolling::watch(c, long_callback, 1, check[0]);
		}
	}

@load base/protocols/conn
@load base/utils/time

# This is probably not so great to reach into the Conn namespace..
module Conn;

export {
function set_conn_log_data_hack(c: connection)
	{
	Conn::set_conn(c, T);
	}
}

# Now onto the actual code for this script...

module LongConnection;

export {
	redef enum Log::ID += { LOG };

	redef enum Notice::Type += {
		## Notice for when a long connection is found.
		## The `sub` field in the notice represents the number
		## of seconds the connection has currently been alive.
		LongConnection::found
	};

	## Aliasing vector of interval values as
	## "Durations"
	type Durations: vector of interval;

	## The default duration that you are locally
	## considering a connection to be "long".
	const default_durations = Durations(10min, 30min, 1hr, 12hr, 24hrs, 3days) &redef;

	## These are special cases for particular hosts or subnets
	## that you may want to watch for longer or shorter
	## durations than the default.
	const special_cases: table[subnet] of Durations = {} &redef;
}

# The yield value on this is the current offset into the
# Durations value.
global tracking_conns: table[string] of count &default=0;

event bro_init() &priority=5
	{
	Log::create_stream(LOG, [$columns=Conn::Info, $path="conn_long"]);
	}

function get_durations(c: connection): Durations
	{
	local check_it: Durations;
	if ( c$id$orig_h in special_cases )
		check_it = special_cases[c$id$orig_h];
	else if ( c$id$resp_h in special_cases )
		check_it = special_cases[c$id$resp_h];
	else
		check_it = default_durations;

	return check_it;
	}

function long_callback(c: connection, cnt: count): interval
	{
	local check_it = get_durations(c);
	local offset = tracking_conns[c$uid];

	if ( offset < |check_it| && c$duration >= check_it[offset] )
		{
		Conn::set_conn_log_data_hack(c);
		Log::write(LongConnection::LOG, c$conn);

		local message = fmt("%s -> %s:%s remained alive for longer than %s",
		                    c$id$orig_h, c$id$resp_h, c$id$resp_p, duration_to_mins_secs(c$duration));
		NOTICE([$note=LongConnection::found,
		        $msg=message,
		        $sub=fmt("%.2f", c$duration),
		        $conn=c]);

		++tracking_conns[c$uid];
		# We're only bumping the local offset value
		# here so we can use it below and don't need
		# to do the hash table lookup again.
		++offset;
		}

	# Keep watching if there are potentially more thresholds.
	if ( offset < |check_it| )
		return check_it[offset];
	else
		return 0secs;
	}

event connection_established(c: connection)
	{
	local check = get_durations(c);
	if ( |check| > 0 )
		{
		ConnPolling::watch(c, long_callback, 1, check[0]);
		}
	}


global-ext needed to load site for some things to work.

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load site
@load functions-ext

const private_address_space: set[subnet] = {10.0.0.0/8, 192.168.0.0/16, 127.0.0.0/8, 172.16.0.0/12};

# Copyright 2008 Seth Hall <hall.692@osu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that: (1) source code distributions
# retain the above copyright notice and this paragraph in its entirety, (2)
# distributions including binary code include the above copyright notice and
# this paragraph in its entirety in the documentation or other materials
# provided with the distribution, and (3) all advertising materials mentioning
# features or use of this software display the following acknowledgement:
# ``This product includes software developed by the University of California,
# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of
# the University nor the names of its contributors may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

@load functions-ext

const private_address_space: set[subnet] = {10.0.0.0/8, 192.168.0.0/16, 127.0.0.0/8, 172.16.0.0/12};


Update rock.bro

Update rock.bro

logging http headers allows for a few things:
- domain fronting detection
- http evasion sort of things with double headers.
- greater visibility into potential software/programs
- additional indicator possibilites
- additional hunting possibilities looking for abnormal/rare headers

# Copyright (C) 2016-2018 RockNSM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module ROCK;

export {
  const sensor_id = gethostname() &redef;
}

#=== Bro built-ins ===================================

# Collect on SMB protocol
@load policy/protocols/smb

# Enable VLAN Logging
@load policy/protocols/conn/vlan-logging

# Log MAC addresses
@load policy/protocols/conn/mac-logging

# Log (All) Client and Server HTTP Headers
@load policy/protocols/http/header-names.bro

#== ROCK specific scripts ============================
# Add empty Intel framework database
@load ./frameworks/intel

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension

#== 3rd Party Scripts =================================
# Add Salesforce's JA3 SSL fingerprinting
@load ./misc/ja3

### Sensor specific scripts ######################

# Configure AF_PACKET, if in use
@load ./plugins/afpacket

# Copyright (C) 2016-2018 RockNSM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module ROCK;

export {
  const sensor_id = gethostname() &redef;
}

#=== Bro built-ins ===================================

# Collect on SMB protocol
@load policy/protocols/smb

# Enable VLAN Logging
@load policy/protocols/conn/vlan-logging

# Log MAC addresses
@load policy/protocols/conn/mac-logging

#== ROCK specific scripts ============================
# Add empty Intel framework database
@load ./frameworks/intel

# Load integration with FSF
@load ./frameworks/files/extract2fsf

# Load file extraction
@load ./frameworks/files/extraction
redef FileExtract::prefix = "/data/bro/logs/extract_files/";
redef FileExtract::default_limit = 1048576000;

# Add sensor and log meta information to each log
@load ./frameworks/logging/extension

#== 3rd Party Scripts =================================
# Add Salesforce's JA3 SSL fingerprinting
@load ./misc/ja3

### Sensor specific scripts ######################

# Configure AF_PACKET, if in use
@load ./plugins/afpacket


Oops.  ssl-ext only logs localhosts by default now.

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;

	# Which SSH logins to record.
	# Choices are: LocalHosts, RemoteHosts, AllHosts, NoHosts
	const logging = LocalHosts &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;
}

event bro_init()
	{
	LOG::create_logs("ssl-ext", logging, split_log_file, T);
	LOG::define_header("ssl-ext", cat_sep("\t", "\\N",
	                                      "ts",
	                                      "host", "port",
	                                      "cert_subject"));
	}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	if ( is_server )
		event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !addr_matches_hosts(c$id$resp_h, logging) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];

		local log = LOG::get_file_by_addr("ssl-ext", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N",
		                   network_time(),
		                   c$id$resp_h, port_to_count(c$id$resp_p),
		                   cert$subject);
		}
	}

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;

	# Which SSH logins to record.
	# Choices are: LocalHosts, RemoteHosts, AllHosts, NoHosts
	const logging = AllHosts &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;
}

event bro_init()
	{
	LOG::create_logs("ssl-ext", logging, split_log_file, T);
	LOG::define_header("ssl-ext", cat_sep("\t", "\\N",
	                                      "ts",
	                                      "host", "port",
	                                      "cert_subject"));
	}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	if ( is_server )
		event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !addr_matches_hosts(c$id$resp_h, logging) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];

		local log = LOG::get_file_by_addr("ssl-ext", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N",
		                   network_time(),
		                   c$id$resp_h, port_to_count(c$id$resp_p),
		                   cert$subject);
		}
	}


Create __load__.bro

Create __load__.bro

error: pathspec 'bro-scripts/tor/policy/__load__.bro' did not match any file(s) known to git


load global-ext from ssh-ext-logging

load global-ext from ssh-ext-logging

@load global-ext
@load ssh-ext

module SSH;

export {
	# If set to T, this will split inbound and outbound transactions
	# into separate files.  F merges everything into a single file.
	const split_log_file = F &redef;
	# Which SSH logins to record.
	# Choices are: Inbound, Outbound, All
	const logging = All &redef;
}

event bro_init()
	{
	LOG::create_logs("ssh-ext", logging, split_log_file, T);
	}

event ssh_ext(id: conn_id, si: ssh_ext_session_info)
	{
	local log = LOG::choose("ssh-ext", id$resp_h);

	print log, cat_sep("\t", "\\N",
	                   si$start_time,
	                   id$orig_h, fmt("%d", id$orig_p),
	                   id$resp_h, fmt("%d", id$resp_p),
	                   si$status, si$direction,
	                   si$location$country_code, si$location$region,
	                   si$client, si$server,
	                   si$resp_size);
	}


fix logged hosts configuration

fix logged hosts configuration

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logged_hosts: Hosts = LocalHosts &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", All, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !resp_matches_hosts(c$id$resp_h, logged_hosts) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}

@load global-ext
@load ssl

module SSL_KnownCerts;

export {
	const local_domains = /(^|\.)(osu|ohio-state)\.edu$/ |
	                      /(^|\.)akamai(tech)?\.net$/ &redef;

	# The list of all detected certs.  This prevents over-logging.
	global certs: set[addr, port, string] &create_expire=1day &synchronized;

	# The hosts that should be logged.
	const split_log_file = F &redef;
	const logging = Outbound &redef;

	redef enum Notice += {
		# Raised when a non-local name is found in the CN of a SSL cert served
		# up from a local system.
		SSLExternalName,
		};
}

event bro_init()
{
    LOG::create_logs("ssl-known-certs", logging, split_log_file, T);
    LOG::define_header("ssl-known-certs", cat_sep("\t", "\\N", "resp_h", "resp_p", "subject"));
}

event ssl_certificate(c: connection, cert: X509, is_server: bool)
	{
	# The ssl analyzer doesn't do this yet, so let's do it here.
	#add c$service["SSL"];
	event protocol_confirmation(c, ANALYZER_SSL, 0);

	if ( !orig_matches_direction(c$id$resp_h, logging) )
		return;

	lookup_ssl_conn(c, "ssl_certificate", T);
	local conn = ssl_connections[c$id];
	if ( [c$id$resp_h, c$id$resp_p, cert$subject] !in certs )
		{
		add certs[c$id$resp_h, c$id$resp_p, cert$subject];
		local log = LOG::get_file("ssl-known-certs", c$id$resp_h, F);
		print log, cat_sep("\t", "\\N", c$id$resp_h, fmt("%d", c$id$resp_p), cert$subject);

	    if(is_local_addr(c$id$resp_h))
	        {
	        local parts = split_all(cert$subject, /CN=[^\/]+/);
	        if (|parts| == 3)
	            {
	            local cn = parts[2];
	            if (local_domains !in cn)
	                {
	                NOTICE([$note=SSLExternalName,
	                        $msg=fmt("SSL Cert for %s returned from an internal host - %s", cn, c$id$resp_h),
	                        $conn=c]);
	                }
	            }
	        }
	    }
	}


Listen on localhost only.

Listen on localhost only.

@load base/frameworks/communication

Communication::nodes["vast"] = [$host = 127.0.0.1,
                                $p = 42000/tcp,
                                $events = /^.*$/,
                                #$retry=10secs,
                                $connect=T];

@load base/frameworks/communication

Communication::nodes["vast"] = [$host = 192.150.187.38,
                                $p = 42000/tcp,
                                $events = /^.*$/,
                                #$retry=10secs,
                                $connect=T];


add more opendns servers

add more opendns servers

module DNS;

export {
    redef record Info += {
        ## Country code of external DNS server
        cc: string &log &optional;
        ## hostname of external DNS server
        #resp_h_hostname: string &log &optional;
    };

    const ignore_external: set[subnet] = {
        8.8.8.8/32, #google dns
        8.8.4.4/32, #google dns
        208.67.220.123/32, #opendns
        208.67.222.222/32, #opendns
        208.67.220.220/32, #opendns
    } &redef;

}

event bro_init()
{
    Log::add_filter(DNS::LOG, [$name = "external-dns",
                                $path = "external_dns",
                                $exclude = set("uid", "proto", "trans_id","qclass", "qclass_name", "qtype", "rcode",
                                "QR","AA","TC","RD","RA","Z","answers","TTLs"),
                                $pred(rec: DNS::Info) = {
        if(!rec$RD)
            return F;

        local orig_h = rec$id$orig_h;
        local resp_h = rec$id$resp_h;

        #is this an outbound query
        if(!Site::is_local_addr(orig_h) || Site::is_local_addr(resp_h))
            return F;

        if(orig_h in ignore_external || resp_h in ignore_external)
            return F;

        local loc = lookup_location(resp_h);

        rec$cc = "";
        if(loc?$country_code){
            rec$cc = loc$country_code;
        }
        return T;

    } ]);
}

module DNS;

export {
    redef record Info += {
        ## Country code of external DNS server
        cc: string &log &optional;
        ## hostname of external DNS server
        #resp_h_hostname: string &log &optional;
    };

    const ignore_external: set[subnet] = {
        8.8.8.8/32, #google dns
        8.8.4.4/32, #google dns
        208.67.220.123/32, #opendns
        208.67.222.222/32, #opendns
    } &redef;

}

event bro_init()
{
    Log::add_filter(DNS::LOG, [$name = "external-dns",
                                $path = "external_dns",
                                $exclude = set("uid", "proto", "trans_id","qclass", "qclass_name", "qtype", "rcode",
                                "QR","AA","TC","RD","RA","Z","answers","TTLs"),
                                $pred(rec: DNS::Info) = {
        if(!rec$RD)
            return F;

        local orig_h = rec$id$orig_h;
        local resp_h = rec$id$resp_h;

        #is this an outbound query
        if(!Site::is_local_addr(orig_h) || Site::is_local_addr(resp_h))
            return F;

        if(orig_h in ignore_external || resp_h in ignore_external)
            return F;

        local loc = lookup_location(resp_h);

        rec$cc = "";
        if(loc?$country_code){
            rec$cc = loc$country_code;
        }
        return T;

    } ]);
}