Add support for designating destdir.

Add support for designating destdir.

BEGIN {
	version = "0.1.3"
	printf("link-parser, v%s\n", version)

	true = 1
	false = 0
	g_pat_srccommon = "\\/sw\\/vobs.+"
	g_sroot = ""
	inst_root = ""
	tar_root = ""

	# Colors
	RED="\033[0;31m"
	GRN="\033[0;32m"
	BLU="\033[0;34m"
	NC="\033[0m"
	BOL="\033[1m"

	count = 0

	if (arg ~ /ut/) {
		do_test();
		SKIP_END = true
		exit 0
	}

	if (arg ~ /pretend/) {
		pretend = true
	}

	if (arg ~ /dest=/) {
		destdir = gensub(/^.+dest=([^ \t;,]+)/, "\\1", "g", arg)
		if (system("test -d " destdir) != 0) {
			print "Directory [" destdir "] doesn't exist!"
			SKIP_END = true
			exit 0
		}
	}

	# Skip head lines
	getline
}

/^[^-]/ {
	link = $0
	# Remove the ending ' \'
	gsub(/[ \t]+\\$/, "", link)

	if (g_sroot == "") {
		printf("Grabbing sources and headers ...\n")
		g_sroot = get_sroot(link)
		"date +%Y%m%d-%H%M%S" | getline date
		inst_root = g_sroot"-"date
	}

	# ---------
	# Sources
	# ---------

	cmd_file = link".cmd"
	while ((getline line < cmd_file) > 0) {
		# abosolute path
		if (line ~ /[ \t]+-c[ \t]+\/.+$/)
			source = gensub(/^.+[ \t]+-c[ \t]+(.+)$/, "\\1", "g", line)
		else # relative path
			source = gensub(/^cd (.+);.+-c (.+)/, "\\1/\\2", "g", line)

		rel_install(source, g_sroot, inst_root)
	}
	close(cmd_file)

	# ---------
	# Headers
	# ---------

	gsub(/\.(obj|o|lib)$/, "", link)
	dep_file = link ".D"

	if ((getline < dep_file) < 0)
		next

	# Get path prefix
	cur_path = $1
	gsub(/^#/, "", cur_path)

	while ((getline < dep_file) > 0) {
		gsub(/:/, " ")
		for (i = 1; i <= NF; i++) {
			if ($i !~ /\.(h|hpp)$/)
				continue
			header = cur_path "/" $i
			rel_install(header, g_sroot, inst_root)
		}
	}
	close(dep_file)
}

END {
	if (SKIP_END)
		exit 0
	print "Total dumped files: " count
	if (arg ~ /tar/)
		make_tarball(arg)
}

function rel_install(source, rel_root, new_root) {
	count++
	if (pretend)
		return 0

	dir = dirname(source)
	base = basename(source)
	patt = "^.+\\/"rel_root"\\/(sw.+)"
	rel_path = gensub(patt, "\\1", "g", dir)

	if (destdir)
		new_root = destdir "/" new_root

	system("mkdir -p " new_root "/" rel_path)
	system("cp -L " source " " new_root "/" rel_path)
}

function make_tarball(flag) {
	tarball = inst_root ".tar.gz"
	printf("Generating source tarball %s%s%s%s ...\n", RED, BOL, tarball, NC)

	if (pretend || SKIP_END)
		exit 0

	"pwd" |getline oldpwd
	if (destdir)
		system("cd " destdir)

	system("tar zcf " tarball " " inst_root)
	if (flag ~ /purge/)
		system("rm -r "inst_root)

	if (destdir)
		system("cd " oldpwd)
}

function get_sroot(line) {
	if (line !~ g_pat_srccommon)
		return ""

	gsub(g_pat_srccommon, "", line);
	gsub(/.+\//, "", line);
	return line;
}

# dirname
function dirname(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)

	# Remove basename
	gsub(/[^\/]+$/, "", path)
	if (path == "") return "."

	# Remove the ending '/' again
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	return path
}

# basename
function basename(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	path = gensub(/^.+\/+(.+)$/, "\\1", "g", path)

	return path;
}

###### Test #########
function do_test() {
	printf("%s%s%s%s\n", BLU, BOL, "---- Test get_sroot ----", NC)
	test_get_sroot("/repo/lulinw/isam/sw/vobs/esam/build");
	test_get_sroot("/repo/lulinw/isam/sw/esam/build");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test dirname ----", NC)
	test_dirname("/home/abc");
	test_dirname("home///abc//");
	test_dirname("home///abc/de//");
	test_dirname("../abc//");
	test_dirname("abc");
	test_dirname("..");
	test_dirname("..///");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test basename ----", NC)
	test_basename("/home//abc")
	test_basename("///")
	test_basename("/")
	test_basename("..")
	test_basename("../abc//")
	test_basename("/../abc//")
	test_basename("/../abc/efg/")
}

function test_get_sroot(line) {
	if ((ret = get_sroot(line)) == "") {
		printf("Unexpected line [PAT=%s]:\n  %s%s%s\n",
				g_pat_srccommon, RED, line, NC)
	} else
		printf("Root of %s is\n  %s%s%s\n", line, GRN, get_sroot(line), NC)
}

function test_dirname(path) {
	printf("dirname(%s) is \n  %s%s%s\n", path, GRN, dirname(path), NC)
}

function test_basename(path) {
	printf("basename(%s) is \n %s%s%s\n", path, GRN, basename(path), NC)
}

#!@include "functions.awk"

BEGIN {
	version = "0.1.2"
	printf("link-parser, v%s\n", version)

	true = 1
	false = 0
	g_pat_srccommon = "\\/sw\\/vobs.+"
	g_sroot = ""
	inst_root = ""
	tar_root = ""

	# Colors
	RED="\033[0;31m"
	GRN="\033[0;32m"
	BLU="\033[0;34m"
	NC="\033[0m"
	BOL="\033[1m"

	if (arg ~ /test/) {
		do_test();
		SKIP_END = true
		exit 0
	}

	if (arg ~ /pretend/) {
		pretend = true
	}
	count = 0

	# Skip head lines
	getline
}

/^[^-]/ {
	link = $0
	# Remove the ending ' \'
	gsub(/[ \t]+\\$/, "", link)

	if (g_sroot == "") {
		printf("Grabbing sources and headers ...\n")
		g_sroot = get_sroot(link)
		"date +%Y%m%d%H-%M%S" | getline date
		inst_root = g_sroot"-"date
	}

	# ---------
	# Sources
	# ---------

	cmd_file = link".cmd"
	while ((getline line < cmd_file) > 0) {
		# abosolute path
		if (line ~ /[ \t]+-c[ \t]+\/.+$/)
			source = gensub(/^.+[ \t]+-c[ \t]+(.+)$/, "\\1", "g", line)
		else # relative path
			source = gensub(/^cd (.+);.+-c (.+)/, "\\1/\\2", "g", line)

		rel_install(source, g_sroot, inst_root)
	}
	close(cmd_file)

	# ---------
	# Headers
	# ---------

	gsub(/\.(obj|o|lib)$/, "", link)
	dep_file = link ".D"

	if ((getline < dep_file) < 0)
		next

	# Get path prefix
	cur_path = $1
	gsub(/^#/, "", cur_path)

	while ((getline < dep_file) > 0) {
		gsub(/:/, " ")
		for (i = 1; i <= NF; i++) {
			if ($i !~ /\.(h|hpp)$/)
				continue
			header = cur_path "/" $i
			rel_install(header, g_sroot, inst_root)
		}
	}
	close(dep_file)
}

END {
	print "Total dumped files: " count
	if (pretend)
		exit 0

	tarball = inst_root ".tar.gz"
	printf("Generating source tarball %s%s%s%s ...\n", RED, BOL, tarball, NC)
	system("tar zcf " tarball " " inst_root)
	system("rm -r "inst_root)
}

function rel_install(source, rel_root, new_root) {
	count++
	dir = dirname(source)
	base = basename(source)
	patt = "^.+\\/"rel_root"\\/(sw.+)"
	rel_path = gensub(patt, "\\1", "g", dir)

	if (pretend)
		return 0
	system("mkdir -p " new_root "/" rel_path)
	system("cp -L " source " " new_root "/" rel_path)
}

function get_sroot(line) {
	if (line !~ g_pat_srccommon)
		return ""

	gsub(g_pat_srccommon, "", line);
	gsub(/.+\//, "", line);
	return line;
}

# dirname
function dirname(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)

	# Remove basename
	gsub(/[^\/]+$/, "", path)
	if (path == "") return "."

	# Remove the ending '/' again
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	return path
}

# basename
function basename(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	path = gensub(/^.+\/+(.+)$/, "\\1", "g", path)

	return path;
}

###### Test #########
function do_test() {
	printf("%s%s%s%s\n", BLU, BOL, "---- Test get_sroot ----", NC)
	test_get_sroot("/repo/lulinw/isam/sw/vobs/esam/build");
	test_get_sroot("/repo/lulinw/isam/sw/esam/build");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test dirname ----", NC)
	test_dirname("/home/abc");
	test_dirname("home///abc//");
	test_dirname("home///abc/de//");
	test_dirname("../abc//");
	test_dirname("abc");
	test_dirname("..");
	test_dirname("..///");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test basename ----", NC)
	test_basename("/home//abc")
	test_basename("///")
	test_basename("/")
	test_basename("..")
	test_basename("../abc//")
	test_basename("/../abc//")
	test_basename("/../abc/efg/")
}

function test_get_sroot(line) {
	if ((ret = get_sroot(line)) == "") {
		printf("Unexpected line [PAT=%s]:\n  %s%s%s\n",
				g_pat_srccommon, RED, line, NC)
	} else
		printf("Root of %s is\n  %s%s%s\n", line, GRN, get_sroot(line), NC)
}

function test_dirname(path) {
	printf("dirname(%s) is \n  %s%s%s\n", path, GRN, dirname(path), NC)
}

function test_basename(path) {
	printf("basename(%s) is \n %s%s%s\n", path, GRN, basename(path), NC)
}


aux/ematch.awk: better character class support detection in UTF-8 locales

#! /usr/bin/awk -f
#
# Script for 'str ematch' for shells that cannot internally handle POSIX
# extended regular expressions (EREs).
#
# Converts POSIX EREs (with bounds) to traditional awk regular expressions
# (without bounds). On very old awk's without support for POSIX character
# classes in bracket expressions, it translates character classes to ASCII
# equivalents. It also disables awk-specific extensions while doing all that.
#
# This script can also be invoked independently as a command.
#
# --- begin license ---
# Copyright (c) 2019 Martijn Dekker <martijn@inlv.org>, Groningen, Netherlands
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# --- end license ---

BEGIN {
	if (ARGC != 3)
		errorout("usage: ematch.awk <string> <ERE>");
	detectclass();
	exit !match(ARGV[1], convertere(ARGV[2]));
}

function errorout(s, ere, i) {
	if (s) printf("str ematch: %s\n", s) | "cat >&2";
	if (ere) printf("%s\n", ere) | "cat >&2";
	if (i) printf(i>1 ? ("%")(i-1)("c^\n") : "^\n", " ") | "cat >&2";
	exit 2;
}

function mylocale() {
	if ("LC_ALL" in ENVIRON && ENVIRON["LC_ALL"] != "")
		return ENVIRON["LC_ALL"];
	else if ("LC_CTYPE" in ENVIRON && ENVIRON["LC_CTYPE"] != "")
		return ENVIRON["LC_CTYPE"];
	else if ("LANG" in ENVIRON && ENVIRON["LANG"] != "")
		return ENVIRON["LANG"];
	else
		return "C";
}

# Detect whether this awk supports character classes properly.
function detectclass() {
	# When a UTF-8 locale is active, onetrueawk (before 2019) only matches the first
	# character class in a bracket expression, even when matching simple ASCII characters.
	# Ref.: http://gnats.netbsd.org/54424
	hasclass = match("1", /[[:alpha:][:digit:]]/);

	# Due to a bug in the macOS C library, onetrueawk and mawk (which don't support UTF-8) yield both false positives
	# and false negatives when matching character classes against high-bit bytes in UTF-8 locales on macOS.
	# Ref.: https://github.com/onetrueawk/awk/issues/45
	if (hasclass && match(mylocale(), /[Uu][Tt][Ff]-?8/))
		hasclass = match("éïÑ", /^[[:alpha:]][[:alpha:]][[:alpha:]]$/);
}

# The terms 'branch', 'piece', 'atom' and 'bound' are used as defined
# in the manual page re_format(7) on BSD/macOS, or regex(7) on Linux.
# OpenBSD has the best version: https://man.openbsd.org/re_format.7

function convertere(ere, par, \
	eere, piece, atom, L, c, i, isave, j, b, m, n, d1, d2, havepunct)
{
	par += 0;  # initialise if not given

	L = length(ere);
	c = substr(ere, 1, 1);
	i = 1;
	while (i <= L) {
		#### Parse just enough ERE grammar to know one atom from the next.
		if (c == "(") {
			# Sub-ERE in parentheses
			atom = ("(")(convertere(substr(ere, ++i), par + 1))(")");
			i += (_Msh_eB_isave - 1);
		} else if (par && c == ")") {
			# End of sub-ERE in parentheses
			_Msh_eB_isave = i;
			return eere;
		} else if (c == "|" || c == "^" || c == "$") {
			# Branch separator, ^/$ anchor, or non-anchor special ^/$
			eere = (eere)(c);
			c = substr(ere, ++i, 1);
			continue;
		} else if (c == "\\" && i < L) {
			# Backslash-escaped character: remove backslash for non-special characters.
			# This disables the C-style backslash escapes that are specific to awk.
			c = substr(ere, ++i, 1);
			if (match(c, /[.[\\()*+?{|^$]/))
				atom = ("\\")(c);
			else
				atom = c;
		} else if (c == "[") {
			# Bracket expression
			havepunct = 0;	# if translating to ASCII, handle [:punct:] specially
			atom = "";
			isave = i;
			i++;
			if (substr(ere, i, 1) == "^") {
				i++;  # negator
				atom = (atom)("^");
			}
			if (substr(ere, i, 1) == "]") {
				i++;  # initial ']' represents itself
				atom = (atom)("]");
			}
			while (i <= L && (c = substr(ere, i, 1)) != "]") {
				if (match(d1 = substr(ere, i, 2), /\[[.=:]/)) {
					# [.collation.], [=equivalence=] or [:character:] class
					d2 = (substr(ere, i+1, 1))("]");
					i += 2;
					isave = i - 1;
					while (i < L && substr(ere, i, 2) != d2)
						i++;
					i++;
					j = substr(ere, isave, i - isave);
					if (!hasclass) {
						# No class/locale support: translate to traditional ASCII. Note that awk supports
						# C-style backslash codes even within bracket expressions, unlike POSIX EREs.
						if (j == ":alnum:" ) {
							j = "A-Za-z0-9";
						} else if (j == ":alpha:") {
							j = "A-Za-z";
						} else if (j == ":blank:") {
							j = " \\t";
						} else if (j == ":cntrl:") {
							j = "\\1-\\37\\177";
						} else if (j == ":digit:") {
							j = "0-9";
						} else if (j == ":graph:") {
							j = "\\41-\\176";
						} else if (j == ":lower:") {
							j = "a-z";
						} else if (j == ":print:") {
							j = "\\40-\\176";
						} else if (j == ":punct:") {
							havepunct++;
							j = "";
						} else if (j == ":space:") {
							# onetrueawk does not support "\v", so use "\13".
							# https://github.com/onetrueawk/awk/pull/44
							j = " \\t\\r\\n\\13\\f";
						} else if (j == ":upper:") {
							j = "A-Z";
						} else if (j == ":xdigit:") {
							j = "A-Fa-f0-9";
						} else if (match(j, /^[=.].*[=.]$/)) {
							# Translate collation and equivalence classes to literal
							# characters in parentheses. TODO: is this a sensible fallback?
							j = ("(")(substr(j, 2, length(j) - 2))(")");
							gsub(/\\/, "\\\\", j);
						} else {
							errorout("invalid character class", ere, isave);
						}
						atom = (atom)(j);
					} else {
						# class/locale support: keep it as is
						atom = (atom)("[")(j)("]");
					}
				} else if (c == "\\") {
					# In POSIX EREs, the backslash is not special at all within a bracket expression, but awk
					# parses C-style backslash escape codes in the entire ERE, including bracket expressions.
					# So to convert a POSIX bracket expression, we have to escape the backslash.
					atom = (atom)("\\\\");
				} else {
					# Other single character in bracket expression.
					atom = (atom)(c);
				}
				i++;
			}
			if (i > L)
				errorout("unterminated bracket expression", ere, isave);
			if (havepunct) {
				# Translate ASCII [:punct:] specially, making sure ']' is the first
				# character, and '-' the last, in the entire bracket expression.
				if (substr(atom, 1, 1) == "]") {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(substr(atom, 2));
				} else {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(atom);
				}
				if (substr(atom, length(atom), 1) != "-") {
					atom = (atom)("-");
				}
			}
			atom = ("[")(atom)("]");
		} else if (c == "*" || c == "+" || c == "?") {
			errorout("repetition operator not valid here", ere, i);
		} else {
			# Simple character
			atom = c;
		}

		#### Now that we've got an atom, parse repetitions to get a piece.
		c = substr(ere, ++i, 1);
		if (c == "*" || c == "+" || c == "?") {
			# Traditional repetiton operator.
			piece = (atom)(c);
			c = substr(ere, ++i, 1);
		} else if (c == "{" && match(substr(ere, i+1, 1), /[0123456789]/)) {
			# Interval expression, a.k.a. repetition expression, a.k.a. bound.
			# Expand it in terms of the repetition operators '*', '+' and '?'.
			piece = "";
			isave = i;
			b = 1;		# number of bound params
			m = ""; n = "";	# bound params
			while ( (c = substr(ere, ++i, 1)) != "}" && i <= L ) {
				if (c == ",")
					b++;
				else if (!match(c, /[0123456789]/))
					errorout("bound: bad number", ere, i);
				else if (b == 1)
					m = ( (m)(c) ) + 0; # concatenate, convert to int
				else if (b == 2)
					n = ( (n)(c) ) + 0;
			}
			if (i > L)
				errorout("unterminated bound", ere, isave);
			if (b > 2)
				errorout("bound: too many parameters", ere, isave);
			if (b == 1) {
				# {m}: exactly m occurrences
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
			} else if (n == "") {
				# {m,}: at least m occurrences
				if (m == 0) {
					piece = (atom)("*");
				} else {
					for (j = 1; j <= m; j++)
						piece = (piece)(atom);
					piece = (piece)("+");
				}
			} else {
				# {m,n}: any number of occurrences between m and n, inclusive
				if (n < m)
					errorout("bad bound: max < min", ere, isave);
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
				for (; j <= n; j++)
					piece = (piece)(atom)("?");
			}
			c = substr(ere, ++i, 1);
		} else {
			# No repetition.
			piece = atom;
		}
		eere = (eere)(piece);
	}
	# End of ERE.
	if (par)
		errorout("unbalanced parentheses", ere);
	return eere;
}

#! /usr/bin/awk -f
#
# Script for 'str ematch' for shells that cannot internally handle POSIX
# extended regular expressions (EREs).
#
# Converts POSIX EREs (with bounds) to traditional awk regular expressions
# (without bounds). On very old awk's without support for POSIX character
# classes in bracket expressions, it translates character classes to ASCII
# equivalents. It also disables awk-specific extensions while doing all that.
#
# This script can also be invoked independently as a command.
#
# --- begin license ---
# Copyright (c) 2019 Martijn Dekker <martijn@inlv.org>, Groningen, Netherlands
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# --- end license ---

BEGIN {
	if (ARGC != 3)
		errorout("usage: ematch.awk <string> <ERE>");
	hasclass = match("1", /[[:alpha:][:digit:]]/);
	exit !match(ARGV[1], convertere(ARGV[2]));
}

function errorout(s, ere, i) {
	if (s) printf("str ematch: %s\n", s) | "cat >&2";
	if (ere) printf("%s\n", ere) | "cat >&2";
	if (i) printf(i>1 ? ("%")(i-1)("c^\n") : "^\n", " ") | "cat >&2";
	exit 2;
}

# The terms 'branch', 'piece', 'atom' and 'bound' are used as defined
# in the manual page re_format(7) on BSD/macOS, or regex(7) on Linux.
# OpenBSD has the best version: https://man.openbsd.org/re_format.7

function convertere(ere, par, \
	eere, piece, atom, L, c, i, isave, j, b, m, n, d1, d2, havepunct)
{
	par += 0;  # initialise if not given

	L = length(ere);
	c = substr(ere, 1, 1);
	i = 1;
	while (i <= L) {
		#### Parse just enough ERE grammar to know one atom from the next.
		if (c == "(") {
			# Sub-ERE in parentheses
			atom = ("(")(convertere(substr(ere, ++i), par + 1))(")");
			i += (_Msh_eB_isave - 1);
		} else if (par && c == ")") {
			# End of sub-ERE in parentheses
			_Msh_eB_isave = i;
			return eere;
		} else if (c == "|" || c == "^" || c == "$") {
			# Branch separator, ^/$ anchor, or non-anchor special ^/$
			eere = (eere)(c);
			c = substr(ere, ++i, 1);
			continue;
		} else if (c == "\\" && i < L) {
			# Backslash-escaped character: remove backslash for non-special characters.
			# This disables the C-style backslash escapes that are specific to awk.
			c = substr(ere, ++i, 1);
			if (match(c, /[.[\\()*+?{|^$]/))
				atom = ("\\")(c);
			else
				atom = c;
		} else if (c == "[") {
			# Bracket expression
			havepunct = 0;	# if translating to ASCII, handle [:punct:] specially
			atom = "";
			isave = i;
			i++;
			if (substr(ere, i, 1) == "^") {
				i++;  # negator
				atom = (atom)("^");
			}
			if (substr(ere, i, 1) == "]") {
				i++;  # initial ']' represents itself
				atom = (atom)("]");
			}
			while (i <= L && (c = substr(ere, i, 1)) != "]") {
				if (match(d1 = substr(ere, i, 2), /\[[.=:]/)) {
					# [.collation.], [=equivalence=] or [:character:] class
					d2 = (substr(ere, i+1, 1))("]");
					i += 2;
					isave = i - 1;
					while (i < L && substr(ere, i, 2) != d2)
						i++;
					i++;
					j = substr(ere, isave, i - isave);
					if (!hasclass) {
						# No class/locale support: translate to traditional ASCII. Note that awk supports
						# C-style backslash codes even within bracket expressions, unlike POSIX EREs.
						if (j == ":alnum:" ) {
							j = "A-Za-z0-9";
						} else if (j == ":alpha:") {
							j = "A-Za-z";
						} else if (j == ":blank:") {
							j = " \\t";
						} else if (j == ":cntrl:") {
							j = "\\1-\\37\\177";
						} else if (j == ":digit:") {
							j = "0-9";
						} else if (j == ":graph:") {
							j = "\\41-\\176";
						} else if (j == ":lower:") {
							j = "a-z";
						} else if (j == ":print:") {
							j = "\\40-\\176";
						} else if (j == ":punct:") {
							havepunct++;
							j = "";
						} else if (j == ":space:") {
							# onetrueawk does not support "\v", so use "\13".
							# https://github.com/onetrueawk/awk/pull/44
							j = " \\t\\r\\n\\13\\f";
						} else if (j == ":upper:") {
							j = "A-Z";
						} else if (j == ":xdigit:") {
							j = "A-Fa-f0-9";
						} else if (match(j, /^[=.].*[=.]$/)) {
							# Translate collation and equivalence classes to literal
							# characters in parentheses. TODO: is this a sensible fallback?
							j = ("(")(substr(j, 2, length(j) - 2))(")");
							gsub(/\\/, "\\\\", j);
						} else {
							errorout("invalid character class", ere, isave);
						}
						atom = (atom)(j);
					} else {
						# class/locale support: keep it as is
						atom = (atom)("[")(j)("]");
					}
				} else if (c == "\\") {
					# In POSIX EREs, the backslash is not special at all within a bracket expression, but awk
					# parses C-style backslash escape codes in the entire ERE, including bracket expressions.
					# So to convert a POSIX bracket expression, we have to escape the backslash.
					atom = (atom)("\\\\");
				} else {
					# Other single character in bracket expression.
					atom = (atom)(c);
				}
				i++;
			}
			if (i > L)
				errorout("unterminated bracket expression", ere, isave);
			if (havepunct) {
				# Translate ASCII [:punct:] specially, making sure ']' is the first
				# character, and '-' the last, in the entire bracket expression.
				if (substr(atom, 1, 1) == "]") {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(substr(atom, 2));
				} else {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(atom);
				}
				if (substr(atom, length(atom), 1) != "-") {
					atom = (atom)("-");
				}
			}
			atom = ("[")(atom)("]");
		} else if (c == "*" || c == "+" || c == "?") {
			errorout("repetition operator not valid here", ere, i);
		} else {
			# Simple character
			atom = c;
		}

		#### Now that we've got an atom, parse repetitions to get a piece.
		c = substr(ere, ++i, 1);
		if (c == "*" || c == "+" || c == "?") {
			# Traditional repetiton operator.
			piece = (atom)(c);
			c = substr(ere, ++i, 1);
		} else if (c == "{" && match(substr(ere, i+1, 1), /[0123456789]/)) {
			# Interval expression, a.k.a. repetition expression, a.k.a. bound.
			# Expand it in terms of the repetition operators '*', '+' and '?'.
			piece = "";
			isave = i;
			b = 1;		# number of bound params
			m = ""; n = "";	# bound params
			while ( (c = substr(ere, ++i, 1)) != "}" && i <= L ) {
				if (c == ",")
					b++;
				else if (!match(c, /[0123456789]/))
					errorout("bound: bad number", ere, i);
				else if (b == 1)
					m = ( (m)(c) ) + 0; # concatenate, convert to int
				else if (b == 2)
					n = ( (n)(c) ) + 0;
			}
			if (i > L)
				errorout("unterminated bound", ere, isave);
			if (b > 2)
				errorout("bound: too many parameters", ere, isave);
			if (b == 1) {
				# {m}: exactly m occurrences
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
			} else if (n == "") {
				# {m,}: at least m occurrences
				if (m == 0) {
					piece = (atom)("*");
				} else {
					for (j = 1; j <= m; j++)
						piece = (piece)(atom);
					piece = (piece)("+");
				}
			} else {
				# {m,n}: any number of occurrences between m and n, inclusive
				if (n < m)
					errorout("bad bound: max < min", ere, isave);
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
				for (; j <= n; j++)
					piece = (piece)(atom)("?");
			}
			c = substr(ere, ++i, 1);
		} else {
			# No repetition.
			piece = atom;
		}
		eere = (eere)(piece);
	}
	# End of ERE.
	if (par)
		errorout("unbalanced parentheses", ere);
	return eere;
}


Bugfix: Print also the last element of array after sorting the array.

	modified:   code/awk/generate_source_list_from_make_output.awk

#!/usr/bin/awk -f
BEGIN{
    # This program generates source list to be used in `CMakeLists.txt`
    # or elsewhere using `foo.txt` produced by `make >foo.txt 2>&1` as input.

    array_index = 0;
    split("", dir_array);
    current_dir = "";
    new_filename = "";
    current_filename = "";
    entering_directory_regex = "Entering directory '";
    cpp_regex = "\\<[a-zA-Z0-9_-]+\\.cpp\\>";
    c_regex = "\\<[a-zA-Z0-9_-]+\\.c\\>";
} # BEGIN

function get_filename()
{
    # OK, there's a `.cpp` file on this line.
    cpp_filename = substr($0, RSTART, RLENGTH);
    new_filename = current_dir "/" cpp_filename;

    if (new_filename != current_filename && new_filename != "")
    {
        current_filename = new_filename;

        # find out if there's a matching source file.
        if (system("test -f " current_filename) == 0)
        {
            # print the filename.
            print current_filename;
        }

        # find out if there's a matching `*.h` header file too.
        header_filename = gensub(/(^.*\.)c(|pp)$/, "\\1h", "g", current_filename);

        if (system("test -f " header_filename) == 0)
        {
            # header exists, so print its filename too!
            print header_filename;
        }
    }
}
{
    entering_directory_match_index = match($0, entering_directory_regex);
    cpp_match_index = match($0, cpp_regex);
    c_match_index = match($0, c_regex);

    if (entering_directory_match_index > 0)
    {
        # OK, we are entering a new directory on this line.
        junk_part_length = entering_directory_match_index + length(entering_directory_regex);

        # read current dir.
        current_dir = substr($0, junk_part_length, length($0) - junk_part_length);

        # store current dir into array.
        dir_array[array_index++] = current_dir "/";
    }
    else if (cpp_match_index > 0)
    {
        match($0, cpp_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
    else if (c_match_index > 0)
    {
        match($0, c_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
}
END{
    asort(dir_array);

    print "";
    print "include_directories(";

    for (i in dir_array) print dir_array[i];
} # END

#!/usr/bin/awk -f
BEGIN{
    # This program generates source list to be used in `CMakeLists.txt`
    # or elsewhere using `foo.txt` produced by `make >foo.txt 2>&1` as input.

    array_index = 0;
    split("", dir_array);
    current_dir = "";
    new_filename = "";
    current_filename = "";
    entering_directory_regex = "Entering directory '";
    cpp_regex = "\\<[a-zA-Z0-9_-]+\\.cpp\\>";
    c_regex = "\\<[a-zA-Z0-9_-]+\\.c\\>";
} # BEGIN

function get_filename()
{
    # OK, there's a `.cpp` file on this line.
    cpp_filename = substr($0, RSTART, RLENGTH);
    new_filename = current_dir "/" cpp_filename;

    if (new_filename != current_filename && new_filename != "")
    {
        current_filename = new_filename;

        # find out if there's a matching source file.
        if (system("test -f " current_filename) == 0)
        {
            # print the filename.
            print current_filename;
        }

        # find out if there's a matching `*.h` header file too.
        header_filename = gensub(/(^.*\.)c(|pp)$/, "\\1h", "g", current_filename);

        if (system("test -f " header_filename) == 0)
        {
            # header exists, so print its filename too!
            print header_filename;
        }
    }
}
{
    entering_directory_match_index = match($0, entering_directory_regex);
    cpp_match_index = match($0, cpp_regex);
    c_match_index = match($0, c_regex);

    if (entering_directory_match_index > 0)
    {
        # OK, we are entering a new directory on this line.
        junk_part_length = entering_directory_match_index + length(entering_directory_regex);

        # read current dir.
        current_dir = substr($0, junk_part_length, length($0) - junk_part_length);

        # store current dir into array.
        dir_array[array_index++] = current_dir "/";
    }
    else if (cpp_match_index > 0)
    {
        match($0, cpp_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
    else if (c_match_index > 0)
    {
        match($0, c_regex); # update `RSTART` & `RLENGTH`.
        get_filename();
    }
}
END{
    n_dirs = array_index;
    asort(dir_array);

    print "";
    print "include_directories(";

    for (i = 0; i < n_dirs; i++)
    {
        print dir_array[i];
    }
} # END


minor cleanups

minor cleanups

#! /usr/bin/awk -f

# remove first and last chars
function trim(str){
	return substr(str,2,length(str)-2)
}

# remove final element in slashed path
function stripid(uri){
	l = match(uri,/.*\//)  # side effect is RLENGTH
	if(l<=0)
		return uri
	else
		return substr(uri,1,RLENGTH -1)
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	if(uri in prefix)
		return prefix[uri]
	else
		for(ex in exception){
			start = match(uri, ex)
			if(0 < start)
				return exception[substr(uri, start, RLENGTH)]
		}
		return simplify(uri)
}

# get the final (fragment identifier) portion of a slashed path
function final(uri){
	split(trim(uri),b,"/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# Monarchinitiave's dipper ingest files have cases
	# which escape the general rules
	# in everything
	exception["http://www.w3.org/2002/07"]="OWL_"
	# in go,mgi,wormbase
	exception["http://dx.doi.org"]="DOI"
	# in mgi
	exception["https://www.mousephenotype.org"]="IMPC_"
	# in panther
	exception["http://identifiers.org/wormbase"]="WormBase"
	# just until skolemized bnodes get in the curie map?
	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
}

# main loop
# parse and stash the curie file (first file)
(FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ {
	split($0,a,"'")
	prefix[stripid(a[4])]=a[2]
}
# process the ntriple file(s)  which are not the first file
### case when subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final($2)
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist[s " -> " o " [label=\"" p ]++
}
### case when the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final($2)
	### not a uri
	o = "LITERAL"
	edgelist[s " -> " o " [label=\"" p ]++
	nodelist[o " [shape=record];"]++
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist) print edge " (" edgelist[edge] ")\"];"
	for(node in nodelist) print node
	print "}"
}

#! /usr/bin/awk -f

# remove first and last chars
function trim(str){
	return substr(str,2,length(str)-2)
}

# remove final element in slashed path
function stripid(uri){
	l = match(uri,/.*\//)  # side effect is RLENGTH
	if(l<=0)
		return uri
	else
		return substr(uri,1,RLENGTH -1)
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

function contract(uri){
	if(uri in prefix)
		return prefix[uri]
	else
		for(ex in exception){
			start = match(uri, ex)
			if(0 < start)
				return exception[substr(uri, start, RLENGTH)]
		}
	# else
		return simplify(uri)
}

# get the final (identifier) portion of a slashed path
function final(uri){
	split(trim(uri),b,"/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return
}

BEGIN{
	# Monarchinitiave's dipper ingest files have cases
	# which escape the general rules
	# in everything
	exception["http://www.w3.org/2002/07"]="OWL_"
	# in go,mgi,wormbase
	exception["http://dx.doi.org"]="DOI"
	# in mgi
	exception["https://www.mousephenotype.org"]="IMPC_"
	# in panther
	exception["http://identifiers.org/wormbase"]="WormBase"
	# just until skolemized bnodes get in the curie map?
	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
}

# main loop
# parse and stash the curie file (first file)
(FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ {
	split($0,a,"'")
	prefix[stripid(a[4])]=a[2]
}
# process the ntriple file(s)  which are not the first file
### case when subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final($2)
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist[s " -> " o " [label=\"" p ]++
}
### case when the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final($2)
	### not a uri
	o = "LITERAL"
	edgelist[s " -> " o " [label=\"" p ]++
	nodelist[o " [shape=record];"]++
}

END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist){
		print edge " (" edgelist[edge] ")\"];"
	}
	for(node in nodelist){
		print node
	}
	print "}"
}


oops, this introduced nasty bugs, this one is better, uses more Wordnet data

BEGIN {#FS=":|\|"
#Noun
map["N"]="NN"
#Plural
map["p"]="NNS"
#Noun Phrase
map["h"]="" #delete this in FSA
#Verb (usu participle)
map["V"]="VBG"
#Verb (transitive)
map["t"]="VB"
#Verb (intransitive)
map["i"]="VB" #no difference here!
#Adjective
map["A"]="JJ"
#Adverb
map["v"]="RB"
#Conjunction
map["C"]="CC"
#Preposition
map["P"]="IN"
#Interjection
map["!"]="UH"
#Pronoun
map["r"]="PRP"
#Definite Article
map["D"]="DT"
#Indefinite Article
map["I"]="DT"
#Nominative
map["o"]="" # no such tags in Moby?
#Technical mark
map["\|"]=""	#delete this, unusable
map["\?"]=""	#as above
}
#additional rules: JJ & "er"_END_OF_WORD=JJR
#additional rules: JJ & "est"_END_OF_WORD=JJS
#NN & Uppercase = NNP
#NNS & Uppercase = NNPS
#verbs: case of 5 fields
#give V: gave | given | giving | gives
# VBP     VBD    VBN     VBG      VBZ
#in case of 4 fields: VBD=VBN.
#PRP$, WDT, WP, WP$, WRB only by enumeration
#MD by enumeration

#adjectives
/A:/{
gsub(/{.*}/,"")
gsub(/,/,"")
gsub(/\|/,"")
gsub(/[0-9]/,"")
mark=0
total=0
for (i=3;i<=NF;i++)
	{
	if ($i~/\?|</) mark++
	total++
	}
if (mark==total && "BEGIN"$1!~/BEGIN[A-Z]/) {
#dummy: this is just wrong tagging
	mark=0
	total=0
	} else {
if ($1"_END"!~/ly_END/) print $1 "\t" $1 "\tJJ"; else print $1 "\t" $1 "\tRB"
for (i=2;i<=NF;i++) {
	if ($i"_END"~/er_END/)
		{print $i "\t" $1 "\tJJR"
		JJR[$i]=$1}
	if ($i"_END"~/est_END/)
		{print $i "\t" $1 "\tJJS"
		JJS[$i]=$1
		}
	}
}
}

/N:|N\?:/ {if ($1!~/[A-Z][a-z]/) print $1 "\t" $1 "\tNN"; else print $1 "\t" $1 "\tNNP"
gsub(/,/,"")
gsub(/{.*}/,"")
gsub(/[0-9]\.[0-9]/,"")
gsub(/[0-9]+ /,"")
for (i=3;i<=NF;i++) {
	if ($i!~/[\?\~\!<]/ && "PFX"$i"SFX"!~/PFX([0-9]+|\|)SFX/)
	if ($i!~/[A-Z][a-z]/) print $i "\t" $1 "\tNNS"; else print $i "\t" $1 "\tNNPS"
}
}

/V:/ && !/be V:/ {
gsub(/,/,"")
gsub(/{.*}/,"")
gsub(/[0-9]\.[0-9]/,"")
gsub(/[0-9]+ /,"")
split($0,verb_fields,/\||:/)
#for (kk=1;kk<=10;kk++) {
#	print kk verb_fields[kk]
#	}
if (verb_fields[5]=="") {
	print $1 "\t" $1 "\tVB"
	print $1 "\t" $1 "\tVBP"
	split (verb_fields[2], VBD, " ")
	for (n in VBD) {
	if (VBD[n]!~/[\?\~\!<]/ && "PFX"VBD[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBD[n] "\t" $1 "\tVBD"
	if (VBD[n]!~/[\?\~\!<]/ && "PFX"VBD[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBD[n] "\t" $1 "\tVBN"
	}
	split (verb_fields[3], VBG, " ")
	for (n in VBG) {
	if (VBG[n]!~/[\?\~\!<]/ && "PFX"VBG[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBG[n] "\t" $1 "\tVBG"
	}
	split (verb_fields[4], VBZ, " ")
	for (n in VBZ) {
	if (VBZ[n]!~/[\?\~\!<]/ && "PFX"VBZ[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBZ[n] "\t" $1 "\tVBZ"
	}
	}
if (verb_fields[5]!="") {
	print $1 "\t" $1 "\tVB"
	print $1 "\t" $1 "\tVBP"
	split (verb_fields[2], VBD, " ")
	for (n in VBD) {
		if (VBD[n]!~/[\?\~\!<]/ && "PFX"VBD[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBD[n] "\t" $1 "\tVBD"
	}
	split (verb_fields[3], VBN, " ")
	for (n in VBN) {
		if (VBN[n]!~/[\?\~\!<]/ && "PFX"VBN[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBN[n] "\t" $1 "\tVBN"
	}
	split (verb_fields[4], VBG, " ")
	for (n in VBG) {
		if (VBG[n]!~/[\?\~\!<]/ && "PFX"VBG[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBG[n] "\t" $1 "\tVBG"
	}
	split (verb_fields[5], VBZ, " ")
	for (n in VBZ) {
		if (VBZ[n]!~/[\?\~\!<]/ && "PFX"VBZ[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBZ[n] "\t" $1 "\tVBZ"
	}
	}
}

"BEGIN"$2"END"~/BEGINCEND/ { print $1 "\t" $1 "\tCC"}
"BEGIN"$2"END"~/BEGINPEND/ { print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGIN\!END/ { print $1 "\t" $1 "\t"map[$2]}
#aaaa "BEGIN"$2"END"~/BEGINDEND/ { print $1 "\t" $1 "\t"map[$2]} #its as determiner!
"BEGIN"$2"END"~/BEGINIEND/ && !/ /{ print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGINrEND/ { print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGINvEND/ { print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGINAEND/ { if (JJR[$1]=="" && JJS[$1]=="") print $1 "\t" $1 "\t"map[$2]}

/\tvA/ {print $1 "\t" $1 "\t"map["v"]
	print $1 "\t" $1 "\t"map["A"]}

/\tPCv/ {print $1 "\t" $1 "\t"map["P"]
	print $1 "\t" $1 "\t"map["C"]
	print $1 "\t" $1 "\t"map["v"]}

/\tCv/ {print $1 "\t" $1 "\t"map["C"]
	print $1 "\t" $1 "\t"map["v"]}

BEGIN {#FS=":|\|"
#Noun
map["N"]="NN"
#Plural
map["p"]="NNS"
#Noun Phrase
map["h"]="" #delete this in FSA
#Verb (usu participle)
map["V"]="VBG"
#Verb (transitive)
map["t"]="VB"
#Verb (intransitive)
map["i"]="VB" #no difference here!
#Adjective
map["A"]="JJ"
#Adverb
map["v"]="RB"
#Conjunction
map["C"]="CC"
#Preposition
map["P"]="IN"
#Interjection
map["!"]="UH"
#Pronoun
map["r"]="PRP"
#Definite Article
map["D"]="DT"
#Indefinite Article
map["I"]="DT"
#Nominative
map["o"]="" # no such tags in Moby?
#Technical mark
map["\|"]=""	#delete this, unusable
map["\?"]=""	#as above
}
#additional rules: JJ & "er"_END_OF_WORD=JJR
#additional rules: JJ & "est"_END_OF_WORD=JJS
#NN & Uppercase = NNP
#NNS & Uppercase = NNPS
#verbs: case of 5 fields
#give V: gave | given | giving | gives
# VBP     VBD    VBN     VBG      VBZ
#in case of 4 fields: VBD=VBN.
#PRP$, WDT, WP, WP$, WRB only by enumeration
#MD by enumeration

#adjectives
/A:/{
gsub(/{.*}/,"")
gsub(/,/,"")
gsub(/\|/,"")
gsub(/[0-9]/,"")
mark=0
total=0
for (i=3;i<=NF;i++)
	{
	if ($i~/\?|</) mark++
	total++
	}
if (mark==total && "BEGIN"$1!~/BEGIN[A-Z]/) print $1 "\t" $1 "\tRB"; else {
if ($1"_END"!~/ly_END/) print $1 "\t" $1 "\tJJ"; else print $1 "\t" $1 "\tRB"
}
for (i=2;i<=NF;i++) {
	if ($i"_END"~/er_END/)
		{print $i "\t" $1 "\tJJR"
		JJR[$i]=$1}
	if ($i"_END"~/est_END/)
		{print $i "\t" $1 "\tJJS"
		JJS[$i]=$1
		}
	}
}

/N:|N\?:/ {if ($1!~/[A-Z][a-z]/) print $1 "\t" $1 "\tNN"; else print $1 "\t" $1 "\tNNP"
gsub(/,/,"")
gsub(/{.*}/,"")
gsub(/[0-9]\.[0-9]/,"")
gsub(/[0-9]+ /,"")
for (i=3;i<=NF;i++) {
	if ($i!~/[\?\~\!<]/ && "PFX"$i"SFX"!~/PFX([0-9]+|\|)SFX/)
	if ($i!~/[A-Z][a-z]/) print $i "\t" $1 "\tNNS"; else print $i "\t" $1 "\tNNPS"
}
}

/V:/ && !/be V:/ {
gsub(/,/,"")
gsub(/{.*}/,"")
gsub(/[0-9]\.[0-9]/,"")
gsub(/[0-9]+ /,"")
split($0,verb_fields,/\||:/)
#for (kk=1;kk<=10;kk++) {
#	print kk verb_fields[kk]
#	}
if (verb_fields[5]=="") {
	print $1 "\t" $1 "\tVB"
	print $1 "\t" $1 "\tVBP"
	split (verb_fields[2], VBD, " ")
	for (n in VBD) {
	if (VBD[n]!~/[\?\~\!<]/ && "PFX"VBD[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBD[n] "\t" $1 "\tVBD"
	if (VBD[n]!~/[\?\~\!<]/ && "PFX"VBD[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBD[n] "\t" $1 "\tVBN"
	}
	split (verb_fields[3], VBG, " ")
	for (n in VBG) {
	if (VBG[n]!~/[\?\~\!<]/ && "PFX"VBG[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBG[n] "\t" $1 "\tVBG"
	}
	split (verb_fields[4], VBZ, " ")
	for (n in VBZ) {
	if (VBZ[n]!~/[\?\~\!<]/ && "PFX"VBZ[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBZ[n] "\t" $1 "\tVBZ"
	}
	}
if (verb_fields[5]!="") {
	print $1 "\t" $1 "\tVB"
	print $1 "\t" $1 "\tVBP"
	split (verb_fields[2], VBD, " ")
	for (n in VBD) {
		if (VBD[n]!~/[\?\~\!<]/ && "PFX"VBD[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBD[n] "\t" $1 "\tVBD"
	}
	split (verb_fields[3], VBN, " ")
	for (n in VBN) {
		if (VBN[n]!~/[\?\~\!<]/ && "PFX"VBN[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBN[n] "\t" $1 "\tVBN"
	}
	split (verb_fields[4], VBG, " ")
	for (n in VBG) {
		if (VBG[n]!~/[\?\~\!<]/ && "PFX"VBG[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBG[n] "\t" $1 "\tVBG"
	}
	split (verb_fields[5], VBZ, " ")
	for (n in VBZ) {
		if (VBZ[n]!~/[\?\~\!<]/ && "PFX"VBZ[n]"SFX"!~/PFX([0-9]+|\|)SFX/) print VBZ[n] "\t" $1 "\tVBZ"
	}
	}
}

"BEGIN"$2"END"~/BEGINCEND/ { print $1 "\t" $1 "\tCC"}
"BEGIN"$2"END"~/BEGINPEND/ { print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGIN\!END/ { print $1 "\t" $1 "\t"map[$2]}
#aaaa "BEGIN"$2"END"~/BEGINDEND/ { print $1 "\t" $1 "\t"map[$2]} #its as determiner!
"BEGIN"$2"END"~/BEGINIEND/ && !/ /{ print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGINrEND/ { print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGINvEND/ { print $1 "\t" $1 "\t"map[$2]}
"BEGIN"$2"END"~/BEGINAEND/ { if (JJR[$1]=="" && JJS[$1]=="") print $1 "\t" $1 "\t"map[$2]}


Tolerate spaces in VMPARAM_vm_disk in libvirt awk script. #6132

Condor_submit no longer removes spaces around the commas in the vm_disk
attribute in the submit file. Update the awk script that the vm-gahp
calls by default to tolerate them.

#!/usr/bin/awk -f

BEGIN {
# So that the awk interpreter does not attempt to read the command
# line as a list of input files.
    ARGC = 1;

# Debugging output is requested
    if(ARGV[1] != "")
    {
	debug_file = ARGV[1];
    }
    else
    {
# By doing this, we ensure that we only need to test for debugging
# output once; if it is not requested, we just write all the debugging
# statements to /dev/null
	debug_file = "/dev/null";
    }
}

# The idea here is to collect the job attributes as they are passed to
# us by the VM GAHP.  The GAHP then outputs a blank line to indicate
# that the entire classad has been sent to us.
{
    gsub(/\"/, "")
    key = $1
    # Matching value should be $3-$NR
    $1 = ""
    $2 = ""
    sub(/^  /, "")
    attrs[key] = $0
    bootloader = ""
    root = ""
    initrd = ""
    kernel = ""
    kern_params = ""
    print "Received attribute: " key "=" attrs[key] >>debug_file
}

END {
    if(index(attrs["JobVMType"],"xen") != 0)
    {
	print "<domain type='xen'>";
	os_type = "linux"
	if(index(attrs["VMPARAM_Xen_Kernel"],"included") != 0)
	{
	    bootloader = attrs["VMPARAM_Xen_Bootloader"]
	}
	else
	{
	    kernel = attrs["VMPARAM_Xen_Kernel"]
	    root = attrs["VMPARAM_Xen_Root"]
	    initrd = attrs["VMPARAM_Xen_Initrd"]
            kern_params = attrs["VMPARAM_Xen_Kernel_Params"]
        }
    }
    else if(index(attrs["JobVMType"],"kvm") != 0)
    {
	print "<domain type='kvm'>" ;
        os_type = "hvm"
    }
    else
    {
	print "Unknown VM type: " index(attrs["JobVMType"], "kvm") >>debug_file;
	exit(-1);
    }
    print "<name>" attrs["VMPARAM_VM_NAME"] "</name>" ;

    if( attrs["JobVMMemory"] != "" && attrs["JobVMMemory"] != "undefined" ) {
    	print "<memory>" attrs["JobVMMemory"] * 1024 "</memory>" ;
    } else {
	    print "<memory>" attrs["RequestMemory"] * 1024 "</memory>" ;
    }

    if( attrs["JobVM_VCPUS"] != "" && attrs["JobVM_VCPUS"] != "undefined" ) {
	    print "<vcpu>" attrs["JobVM_VCPUS"] "</vcpu>" ;
    } else {
	    print "<vcpu>" attrs["RequestCpus"] "</vcpu>" ;
    }

    print "<os><type>" os_type "</type>" ;
    if(kernel != "")
    {
	print "<kernel>" kernel "</kernel>"
    }
    if(initrd != "")
    {
	print "<initrd>" initrd "</initrd>"
    }
    if(root != "")
    {
	print "<root>" root "</root>"
    }
    if(kern_params != "")
    {
	print "<cmdline>" kern_params "</cmdline>"
    }
    print "</os>" ;
    if(bootloader != "")
    {
	print "<bootloader>" bootloader "</bootloader>"
    }

    # Make sure guests destroy on power off
    print "<features><acpi/><apic/><pae/></features>" ;
    print "<on_poweroff>destroy</on_poweroff><on_reboot>restart</on_reboot><on_crash>restart</on_crash>" ;

    print "<devices>" ;
    print "<console type='pty'><source path='/dev/ptmx'/></console>" ;

    # Check VNC settings
    if(attrs["JobVMVNCConsole"] == "true")
    {
      print "<graphics type='vnc' port='-1' autoport='yes' keymap='en-us'/>" ;
    }

    # To see full input ad to script set D_FULLDEBUG
    if(attrs["JobVMNetworking"] == "true")
    {
	if(index(attrs["JobVMNetworkingType"],"nat") != 0)
	{
	    print "<interface type='network'><source network='default'/>" ;
            if(attrs["JobVM_MACADDR"] != "")
            {
		print"<mac address='" attrs["JobVM_MACADDR"] "'/>" ;
            }
	    print "</interface>" ;
	}
	else
	{
	    print "<interface type='bridge'>" ;
            if(attrs["JobVM_MACADDR"] != "")
            {
		print"<mac address='" attrs["JobVM_MACADDR"] "'/>" ;
            }
	    if(attrs["VMPARAM_Bridge_Interface"] != "")
	    {
		print "<source bridge='" attrs["VMPARAM_Bridge_Interface"] "'/>" ;
	    }
	    print "</interface>" ;
	}
    }

    n=split(attrs["VMPARAM_vm_Disk"], full_disk, " *, *");
    for ( i=1; i<=n; i++ )
    {
        # count is used to determine if format is passed.
        p = split(full_disk[i], disk_string, ":");

        if (index( disk_string[1], "iso") )
        {
            print "<disk type='file' device='cdrom'>";
        }
        else
        {
            print "<disk type='file'>" ;
        }

        if ( p == 4 )
        {
            # only an issue w/qemu
            if (attrs["JobVMType"] == "kvm")
            {
                print "<driver name='qemu' type='" disk_string[4] "'/>";
            }
        }

        if ( index( disk_string[1], "/" ) == 1 )
        {
            print "<source file='" disk_string[1] "'/>" ;
        }
        else
        {
            print "<source file='" attrs["VM_WORKING_DIR"] "/" disk_string[1] "'/>" ;
        }
        print "<target dev='" disk_string[2] "'/>" ;

        if ( disk_string[3] == "r" )
        {
            print "<readonly/>"
        }

        print "</disk>" ;
    }

     print "</devices></domain>"

    exit(0);
}

#!/usr/bin/awk -f

BEGIN {
# So that the awk interpreter does not attempt to read the command
# line as a list of input files.
    ARGC = 1;

# Debugging output is requested
    if(ARGV[1] != "")
    {
	debug_file = ARGV[1];
    }
    else
    {
# By doing this, we ensure that we only need to test for debugging
# output once; if it is not requested, we just write all the debugging
# statements to /dev/null
	debug_file = "/dev/null";
    }
}

# The idea here is to collect the job attributes as they are passed to
# us by the VM GAHP.  The GAHP then outputs a blank line to indicate
# that the entire classad has been sent to us.
{
    gsub(/\"/, "")
    key = $1
    # Matching value should be $3-$NR
    $1 = ""
    $2 = ""
    sub(/^  /, "")
    attrs[key] = $0
    bootloader = ""
    root = ""
    initrd = ""
    kernel = ""
    kern_params = ""
    print "Received attribute: " key "=" attrs[key] >>debug_file
}

END {
    if(index(attrs["JobVMType"],"xen") != 0)
    {
	print "<domain type='xen'>";
	os_type = "linux"
	if(index(attrs["VMPARAM_Xen_Kernel"],"included") != 0)
	{
	    bootloader = attrs["VMPARAM_Xen_Bootloader"]
	}
	else
	{
	    kernel = attrs["VMPARAM_Xen_Kernel"]
	    root = attrs["VMPARAM_Xen_Root"]
	    initrd = attrs["VMPARAM_Xen_Initrd"]
            kern_params = attrs["VMPARAM_Xen_Kernel_Params"]
        }
    }
    else if(index(attrs["JobVMType"],"kvm") != 0)
    {
	print "<domain type='kvm'>" ;
        os_type = "hvm"
    }
    else
    {
	print "Unknown VM type: " index(attrs["JobVMType"], "kvm") >>debug_file;
	exit(-1);
    }
    print "<name>" attrs["VMPARAM_VM_NAME"] "</name>" ;

    if( attrs["JobVMMemory"] != "" && attrs["JobVMMemory"] != "undefined" ) {
    	print "<memory>" attrs["JobVMMemory"] * 1024 "</memory>" ;
    } else {
	    print "<memory>" attrs["RequestMemory"] * 1024 "</memory>" ;
    }

    if( attrs["JobVM_VCPUS"] != "" && attrs["JobVM_VCPUS"] != "undefined" ) {
	    print "<vcpu>" attrs["JobVM_VCPUS"] "</vcpu>" ;
    } else {
	    print "<vcpu>" attrs["RequestCpus"] "</vcpu>" ;
    }

    print "<os><type>" os_type "</type>" ;
    if(kernel != "")
    {
	print "<kernel>" kernel "</kernel>"
    }
    if(initrd != "")
    {
	print "<initrd>" initrd "</initrd>"
    }
    if(root != "")
    {
	print "<root>" root "</root>"
    }
    if(kern_params != "")
    {
	print "<cmdline>" kern_params "</cmdline>"
    }
    print "</os>" ;
    if(bootloader != "")
    {
	print "<bootloader>" bootloader "</bootloader>"
    }

    # Make sure guests destroy on power off
    print "<features><acpi/><apic/><pae/></features>" ;
    print "<on_poweroff>destroy</on_poweroff><on_reboot>restart</on_reboot><on_crash>restart</on_crash>" ;

    print "<devices>" ;
    print "<console type='pty'><source path='/dev/ptmx'/></console>" ;

    # Check VNC settings
    if(attrs["JobVMVNCConsole"] == "true")
    {
      print "<graphics type='vnc' port='-1' autoport='yes' keymap='en-us'/>" ;
    }

    # To see full input ad to script set D_FULLDEBUG
    if(attrs["JobVMNetworking"] == "true")
    {
	if(index(attrs["JobVMNetworkingType"],"nat") != 0)
	{
	    print "<interface type='network'><source network='default'/>" ;
            if(attrs["JobVM_MACADDR"] != "")
            {
		print"<mac address='" attrs["JobVM_MACADDR"] "'/>" ;
            }
	    print "</interface>" ;
	}
	else
	{
	    print "<interface type='bridge'>" ;
            if(attrs["JobVM_MACADDR"] != "")
            {
		print"<mac address='" attrs["JobVM_MACADDR"] "'/>" ;
            }
	    if(attrs["VMPARAM_Bridge_Interface"] != "")
	    {
		print "<source bridge='" attrs["VMPARAM_Bridge_Interface"] "'/>" ;
	    }
	    print "</interface>" ;
	}
    }

    n=split(attrs["VMPARAM_vm_Disk"], full_disk, ",");
    for ( i=1; i<=n; i++ )
    {
        # count is used to determine if format is passed.
        p = split(full_disk[i], disk_string, ":");

        if (index( disk_string[1], "iso") )
        {
            print "<disk type='file' device='cdrom'>";
        }
        else
        {
            print "<disk type='file'>" ;
        }

        if ( p == 4 )
        {
            # only an issue w/qemu
            if (attrs["JobVMType"] == "kvm")
            {
                print "<driver name='qemu' type='" disk_string[4] "'/>";
            }
        }

        if ( index( disk_string[1], "/" ) == 1 )
        {
            print "<source file='" disk_string[1] "'/>" ;
        }
        else
        {
            print "<source file='" attrs["VM_WORKING_DIR"] "/" disk_string[1] "'/>" ;
        }
        print "<target dev='" disk_string[2] "'/>" ;

        if ( disk_string[3] == "r" )
        {
            print "<readonly/>"
        }

        print "</disk>" ;
    }

     print "</devices></domain>"

    exit(0);
}


Update permissions

Update permissions

#!/usr/bin/env awk -f
#
# Computes the corrected sample standard deviation.
#
# Usage: stdev
#
# Input:
#   - a column of numbers
#
# Output:
#   - corrected sample standard deviation
#
# Notes:
#   - Uses [Welford's method][1].
#
# [1]: https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm

BEGIN {
	delta = 0
	mean = 0
	M2 = 0
	N = 0
}
{
	N += 1
	delta = $1 - mean
	mean += delta / N
	M2 += delta * ($1 - mu)
}
END {
	if (N < 2) {
		print 0
	} else {
		print sqrt(M2 / (N-1))
	}
}


corrige un peut le output du script de détection des dépendances

#!/usr/bin/gawk

# To generate with sorted entries
# gawk -f regexes/generateDependenciesGraph.gawk  behaviors/behavior_action_*/src/behavior_*/*.py&& sort -u regexes/graphs/dependenceGraph.plantuml

function basename(file) {
    sub(".*/", "", file);
    sub(".py","", file);
    sub("_?((sm)|(SM))","", file);
    return file;
}
function writeF(str){

    sub("_?((sm)|(SM))","", str);
    return system("echo \""str"\">>regexes/graphs/dependenceGraph.plantuml")
}

BEGIN{
    f=""
    umlFileName="regexes/graphs/dependenceGraph";
    file=umlFileName".plantuml";
    system("echo \"\" >"file);
    # system("echo \"@startuml\n \" >"file);
}
{
    if(f != FILENAME){
        f=FILENAME;
        print "reading",f
        class = basename(FILENAME);
    #NeedTo Ignore case and remove "_"
#    writeF("node" class"\n")
    }
    if(match($0, /from .* import (.*)/,arr)&&!match($0, /Behavior/)){
        print "\t",arr[1];
        writeF(arr[1]" -> "class"");
    }
}
END{
#    system("sort -u "file" >"file);
#    system("echo '@startuml\n node SaraSay\n' | cat - "file" > temp && mv temp todo.txt")
#    system("sed -i '1s/^/\@startuml\n node SaraSay\n/' "file);
#    system("cat regexes/graphs/dependenceGraph.plantuml");
#    system("echo '@startuml\n node SaraSay' >/tmp/tempUml.txt");
#    system("cat regexes/graphs/dependenceGraph.plantuml >>/tmp/tempUml.txt");
#    system("awk 'BEGIN{print \\\"@startuml\n node SaraSay\\\"}{print}' "file" > "file);
#    system("cp /tmp/tempUml.txt "file)
    # writeF("@enduml");
    # system("sort "file" > "file);
    # system("plantuml -tsvg "file);
    # system("eog "umlFileName".svg");
}

#In command line (for now)
#gawk -f regexes/getDependenciesState.gawk  behaviors/behavior*/src/behavior*/*.py > regexes/outputAwkTest.txt

#For no-underscores (_)
#gawk -f regexes/getDependenciesState.gawk  behaviors/behavior*/src/behavior*/*.py| sed "s/_//" > regexes/outputSedTest.txt

#For use count of each
#gawk 'BEGIN{f=""}{if(match($0, /from .* import (.*)/,arr)){print "\t",arr[1];}}' behaviors/behavior*/src/behavior*/*.py|sort|uniq -c|sort -n > regexes/outputUniq.txt

#!/usr/bin/gawk

# To generate with sorted entries
# gawk -f regexes/generateDependenciesGraph.gawk  behaviors/behavior_action_*/src/behavior_*/*.py&& sort -u regexes/graphs/dependenceGraph.plantuml

function basename(file) {
    sub(".*/", "", file);
    sub(".py","", file);
    return file;
}
function writeF(str){
    return system("echo \""str"\">>regexes/graphs/dependenceGraph.plantuml")
}

BEGIN{
    f=""
    umlFileName="regexes/graphs/dependenceGraph";
    file=umlFileName".plantuml";
    system("echo \"\" >"file);
    # system("echo \"@startuml\n \" >"file);
}
{
    if(f != FILENAME){
        f=FILENAME;
        print "reading",f
        class = basename(FILENAME);
    #NeedTo Ignore case and remove "_"
#    writeF("node" class"\n")
    }
    if(match($0, /from .* import (.*)/,arr)&&!match($0, /Behavior/)){
        print "\t",arr[1];
        writeF(arr[1]" -> "class"\n");
    }
}
END{
#    system("sort -u "file" >"file);
#    system("echo '@startuml\n node SaraSay\n' | cat - "file" > temp && mv temp todo.txt")
#    system("sed -i '1s/^/\@startuml\n node SaraSay\n/' "file);
#    system("cat regexes/graphs/dependenceGraph.plantuml");
#    system("echo '@startuml\n node SaraSay' >/tmp/tempUml.txt");
#    system("cat regexes/graphs/dependenceGraph.plantuml >>/tmp/tempUml.txt");
#    system("awk 'BEGIN{print \\\"@startuml\n node SaraSay\\\"}{print}' "file" > "file);
#    system("cp /tmp/tempUml.txt "file)
    # writeF("@enduml");
    # system("sort "file" > "file);
    # system("plantuml -tsvg "file);
    # system("eog "umlFileName".svg");
}

#In command line (for now)
#gawk -f regexes/getDependenciesState.gawk  behaviors/behavior*/src/behavior*/*.py > regexes/outputAwkTest.txt

#For no-underscores (_)
#gawk -f regexes/getDependenciesState.gawk  behaviors/behavior*/src/behavior*/*.py| sed "s/_//" > regexes/outputSedTest.txt

#For use count of each
#gawk 'BEGIN{f=""}{if(match($0, /from .* import (.*)/,arr)){print "\t",arr[1];}}' behaviors/behavior*/src/behavior*/*.py|sort|uniq -c|sort -n > regexes/outputUniq.txt


fixes summary option not working

fixes summary option not working

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action == \"validate\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
}

$1 == "option" {
    options[$2]=$3;
}

$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]
    cat=params[3]
    field_index=header_index[field]
    test=""
    msg=""

    if (rule_type == "type") {
        type=params[2]
        if (type == "numeric"){
            test=field " && !is_numeric(" field ")"
            msg="Field " field " in \" FILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        test=field " == \"\""
        msg="Field " field " in \" FILENAME \" line \" NR \" is required"
    } else if (rule_type == "unique") {
        test="!is_unique(" field ")"
        msg="Field " field " in \" FILENAME \" line \" NR \" is a duplicate and should be unique"
    } else if (rule_type ~ /^min|max$/) {
        comparator=$2 == "max" ? ">" : "<"
        limit=params[2]
        term=$2 == "max" ? "less" : "greater"

        test=field " " comparator " " limit
        msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
    } else if (rule_type == "pattern") {
        pattern=params[2]

        test=field " != \"\" && " field " !~ " pattern
        msg=field " in line \" NR \" should match the following pattern " pattern " and was \" " field " \" "
    }

    if (!options["mode"]) options["mode"] = "text"
    if (!options["dcat"]) options["dcat"] = "na"
    if (!cat) cat = options["dcat"]
    handler_prefix="{ log_err(\"" cat "\"); "
    if (options["mode"] == "text") {
        err_handler= handler_prefix "print \"" msg "\" RS $0 RS; }"
    } else if (options["mode"] == "wrap") {
        acc="$" field_index
        err_handler= handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"
    } else if (options["mode"] == "append") {
        err_handler= handler_prefix "print $0 FS \"" msg "\" }"
    }

    if (rule_type !~ /^(none|colmap)$/) { print "action == \"validate\" && NR > 1 && " test " " err_handler " " }
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]

    if (rule_type ~ /^(min|max)$/ || (rule_type == "type" && params[2] == "numeric")) {
        types[field] = "numeric"
    } else if (rule_type != "colmap") {
        types[field] = "text"
    }

    if(rule_type == "colmap") mappings[field]=params[2]
}

END {
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in types) {
        type=types[field]
        field_index=header_index[field]
        dval= type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    gsub(\"Range\", \"rangeno\");"
    print "    print \"COPY " options["table"] " (\" $0 \") FROM stdin;\""
    print "}"

    print "action == \"insert\" && NR > 1 {"
    print "    gsub(\",\", \"\t\");"
    print "    print;"
    print "}"

    print "action == \"table\" && NR == 1 {"

    fields=""
    l=length(headers)
    for (i in headers) {
        c++
        field=headers[i]
        type=types[field]
        if (length(mappings) && mappings[field]) field=mappings[field]
        fields=fields field " " type
        if (c < l) fields=fields ","
    }

    if(options["pkey"]) {
        cols=options["pkey"]
        gsub(":", ",", cols)
        fields=fields ", PRIMARY KEY (" cols ") "
    }
    if(options["fkey"]) {
        split(options["fkey"], ref, " ")
        ftable=ref[1]
        fcols=ref[2]
        gsub(":", ",", fcols)
        fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
    }

    print "     print \"CREATE TABLE IF NOT EXISTS " options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }" RS

    print "# la fin"
    print "END {"
    print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action == \"validate\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
}

$1 == "option" {
    options[$2]=$3;
}

$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]
    cat=params[3]
    field_index=header_index[field]
    test=""
    msg=""

    if (rule_type == "type") {
        type=params[2]
        if (type == "numeric"){
            test=field " && !is_numeric(" field ")"
            msg="Field " field " in \" FILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        test=field " == \"\""
        msg="Field " field " in \" FILENAME \" line \" NR \" is required"
    } else if (rule_type == "unique") {
        test="!is_unique(" field ")"
        msg="Field " field " in \" FILENAME \" line \" NR \" is a duplicate and should be unique"
    } else if (rule_type ~ /^min|max$/) {
        comparator=$2 == "max" ? ">" : "<"
        limit=params[2]
        term=$2 == "max" ? "less" : "greater"

        test=field " " comparator " " limit
        msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
    } else if (rule_type == "pattern") {
        pattern=params[2]

        test=field " != \"\" && " field " !~ " pattern
        msg=field " in line \" NR \" should match the following pattern " pattern " and was \" " field " \" "
    }

    if (!options["mode"]) options["mode"] = "text"
    if (!options["dcat"]) options["dcat"] = "na"
    if (!cat) cat = options["dcat"]
    handler_prefix="{ log_err(\"" cat "\"); "
    if (options["mode"] == "text") {
        err_handler= handler_prefix "print \"" msg "\" RS $0 RS; }"
    } else if (options["mode"] == "wrap") {
        acc="$" field_index
        err_handler= handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"
    } else if (options["mode"] == "append") {
        err_handler= handler_prefix "print $0 FS \"" msg "\" }"
    }

    if (rule_type !~ /^(none|colmap)$/) { print "action == \"validate\" && NR > 1 && " test " " err_handler " " }
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]

    if (rule_type ~ /^(min|max)$/ || (rule_type == "type" && params[2] == "numeric")) {
        types[field] = "numeric"
    } else if (rule_type != "colmap") {
        types[field] = "text"
    }

    if(rule_type == "colmap") mappings[field]=params[2]
}

END {
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in types) {
        type=types[field]
        field_index=header_index[field]
        dval= type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    gsub(\"Range\", \"rangeno\");"
    print "    print \"COPY " options["table"] " (\" $0 \") FROM stdin;\""
    print "}"

    print "action == \"insert\" && NR > 1 {"
    print "    gsub(\",\", \"\t\");"
    print "    print;"
    print "}"

    print "action == \"table\" && NR == 1 {"

    fields=""
    l=length(headers)
    for (i in headers) {
        c++
        field=headers[i]
        type=types[field]
        if (length(mappings) && mappings[field]) field=mappings[field]
        fields=fields field " " type
        if (c < l) fields=fields ","
    }

    if(options["pkey"]) {
        cols=options["pkey"]
        gsub(":", ",", cols)
        fields=fields ", PRIMARY KEY (" cols ") "
    }
    if(options["fkey"]) {
        split(options["fkey"], ref, " ")
        ftable=ref[1]
        fcols=ref[2]
        gsub(":", ",", fcols)
        fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
    }

    print "     print \"CREATE TABLE IF NOT EXISTS " options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }" RS

    print "# la fin"
    print "END {"
    print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\") print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); "
    print "}"
}


(/^[0-9A-F]+\.\./): Fix regex for matching.

# SCRIPT.awk -- awk script to generate SCRIPT.tab
# Copyright (C) 2007
#   National Institute of Advanced Industrial Science and Technology (AIST)
#   Registration Number H15PRO112

# This file is part of the m17n database; a sub-part of the m17n
# library.

# The m17n library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.

# The m17n library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with the m17n library; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.

BEGIN {
  tohex["0"] = 1;
  tohex["1"] = 2;
  tohex["2"] = 3;
  tohex["3"] = 4;
  tohex["4"] = 5;
  tohex["5"] = 6;
  tohex["6"] = 7;
  tohex["7"] = 8;
  tohex["8"] = 9;
  tohex["9"] = 10;
  tohex["A"] = 11;
  tohex["B"] = 12;
  tohex["C"] = 13;
  tohex["D"] = 14;
  tohex["E"] = 15;
  tohex["F"] = 16;
  tohex["a"] = 11;
  tohex["b"] = 12;
  tohex["c"] = 13;
  tohex["d"] = 14;
  tohex["e"] = 15;
  tohex["f"] = 16;

  FS = "[ \t]*[;#][ \t]*";
  initialized = 0;
  charcount = 0;
  range_index = 0;
}

function decode_hex(str, idx) {
    n = 0;
    len = length(str);
    for (i = idx; i <= len; i++) {
	c = tohex[substr(str, i, 1)];
	if (c == 0)
	    break;
	n = n * 16 + c - 1;
    }
    return n;
}

function initialize() {
    first = -1;
    while (getline line < "UNIDATA/UnicodeData.txt" > 0) {
	if (line ~ /^[0-9A-F][0-9A-F]*/) {
	    last = decode_hex(line, 1);
	    if (first >= 0) {
		range[range_index++] = first;
		range[range_index++] = last;
		first = -1;
	    } else if (line ~ /First>/) {
		first = last;
	    } else {
		exists[last] = 1;
	    }
	}
    }
}

function char_exist_p(c) {
    if (exists[c] == 1)
	return 1;
    for (i = 0; i < range_index; i += 2)
	if (range[i] >= c && range[i + 1] <= c)
	    return 1;
    return 0;
}

function maybe_emit(ch1, ch2, this_script) {
    if (initialized == 0) {
	initialize();
	initialized = 1;
	print "# Ranges may contain non-existing character codes.";
	print "0x0-0x3FFFFF common";
	first = ch1;
	last = ch2;
	script = this_script;
    } else {
	if (script == this_script) {
	    for (j = last + 1; j < ch1; j++)
		if (char_exist_p(j))
		    break;
	    if (j == ch1) {
		last = ch2;
		return;
	    }
	}
	if (script != "Common") {
	    if (first == last)
		printf "0x%04X %s\n", first, tolower(script);
	    else
		printf "0x%04X-0x%04X %s\n", first, last, tolower(script);
	}
	first = ch1;
	last = ch2;
	script = this_script;
    }
}

/^[0-9A-F]+\.\./ {
    maybe_emit(decode_hex($0, 1), decode_hex($0, match($0, "\\.\\.") + 2), $2);
    next;
}

/^[0-9A-F]/ {
    ch = decode_hex($0, 1);
    maybe_emit(ch, ch, $2);
    next;
}

END {
    if (script != "Common") {
	if (first == last)
	    printf "0x%04X %s\n", first, tolower (script);
	else
	    printf "0x%04X-0x%04X %s\n", first, last, tolower (script);
    }
    while (getline < "SCRIPT.ext" > 0) {
	if ($0 ~ /^[0-9A-F][0-9A-F]*/) {
	    print;
	}
    }

}

# SCRIPT.awk -- awk script to generate SCRIPT.tab
# Copyright (C) 2007
#   National Institute of Advanced Industrial Science and Technology (AIST)
#   Registration Number H15PRO112

# This file is part of the m17n database; a sub-part of the m17n
# library.

# The m17n library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.

# The m17n library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with the m17n library; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.

BEGIN {
  tohex["0"] = 1;
  tohex["1"] = 2;
  tohex["2"] = 3;
  tohex["3"] = 4;
  tohex["4"] = 5;
  tohex["5"] = 6;
  tohex["6"] = 7;
  tohex["7"] = 8;
  tohex["8"] = 9;
  tohex["9"] = 10;
  tohex["A"] = 11;
  tohex["B"] = 12;
  tohex["C"] = 13;
  tohex["D"] = 14;
  tohex["E"] = 15;
  tohex["F"] = 16;
  tohex["a"] = 11;
  tohex["b"] = 12;
  tohex["c"] = 13;
  tohex["d"] = 14;
  tohex["e"] = 15;
  tohex["f"] = 16;

  FS = "[ \t]*[;#][ \t]*";
  initialized = 0;
  charcount = 0;
  range_index = 0;
}

function decode_hex(str, idx) {
    n = 0;
    len = length(str);
    for (i = idx; i <= len; i++) {
	c = tohex[substr(str, i, 1)];
	if (c == 0)
	    break;
	n = n * 16 + c - 1;
    }
    return n;
}

function initialize() {
    first = -1;
    while (getline line < "UNIDATA/UnicodeData.txt" > 0) {
	if (line ~ /^[0-9A-F][0-9A-F]*/) {
	    last = decode_hex(line, 1);
	    if (first >= 0) {
		range[range_index++] = first;
		range[range_index++] = last;
		first = -1;
	    } else if (line ~ /First>/) {
		first = last;
	    } else {
		exists[last] = 1;
	    }
	}
    }
}

function char_exist_p(c) {
    if (exists[c] == 1)
	return 1;
    for (i = 0; i < range_index; i += 2)
	if (range[i] >= c && range[i + 1] <= c)
	    return 1;
    return 0;
}

function maybe_emit(ch1, ch2, this_script) {
    if (initialized == 0) {
	initialize();
	initialized = 1;
	print "# Ranges may contain non-existing character codes.";
	print "0x0-0x3FFFFF common";
	first = ch1;
	last = ch2;
	script = this_script;
    } else {
	if (script == this_script) {
	    for (j = last + 1; j < ch1; j++)
		if (char_exist_p(j))
		    break;
	    if (j == ch1) {
		last = ch2;
		return;
	    }
	}
	if (script != "Common") {
	    if (first == last)
		printf "0x%04X %s\n", first, tolower(script);
	    else
		printf "0x%04X-0x%04X %s\n", first, last, tolower(script);
	}
	first = ch1;
	last = ch2;
	script = this_script;
    }
}

/^[0-9A-F]+\.\./ {
    maybe_emit(decode_hex($0, 1), decode_hex($0, match($0, "\.\.") + 2), $2);
    next;
}

/^[0-9A-F]/ {
    ch = decode_hex($0, 1);
    maybe_emit(ch, ch, $2);
    next;
}

END {
    if (script != "Common") {
	if (first == last)
	    printf "0x%04X %s\n", first, tolower (script);
	else
	    printf "0x%04X-0x%04X %s\n", first, last, tolower (script);
    }
    while (getline < "SCRIPT.ext" > 0) {
	if ($0 ~ /^[0-9A-F][0-9A-F]*/) {
	    print;
	}
    }

}


Removed empty line.

Removed empty line.

# Linux: https://elixir.bootlin.com/linux/master/source/include/uapi/linux/can.h#L56
#        https://elixir.bootlin.com/linux/master/source/include/uapi/linux/can/error.h
# PEAK CAN TRC File Format: https://www.peak-system.com/produktcd/Pdf/English/PEAK_CAN_TRC_File_Format.pdf
{
    if (1==NR)
    {
        startTimestamp = substr($1, 2, length($1) - 2)
        printf(";$FILEVERSION=1.3\n")
        printf(";$STARTTIME=%u.%.0f\n", mktime(strftime("%Y %m %d 0 0 0", startTimestamp)) / 86400 + 25569, \
                10000000000.0 / 86400.0 * (mktime(strftime("%Y %m %d %H %M %S", startTimestamp)) - mktime(strftime("%Y %m %d 0 0 0", startTimestamp))))
        printf(";\n; Start time: %s\n", startTimestamp)
        printf(";             %u\n", mktime(strftime("%Y %m %d %H %M %S", startTimestamp)))
        printf(";             %s\n;\n", strftime("%Y-%m-%dT%H:%M:%S%z (%Z)", startTimestamp))
        currentTimeOffset = 0.0
    }
    else
    {
        previousTimeOffset = currentTimeOffset
        currentTimestamp = substr($1, 2, length($1) - 2)
        currentTimeOffset = currentTimestamp - startTimestamp
        if (currentTimeOffset < 0)
        {
            startTimestamp = currentTimestamp + currentTimeOffset
            currentTimeOffset = previousTimeOffset
        }
    }
    if ("ERRORFRAME" == $NF)
    {
        printf ";"$0"\n;  "
        if ("20000001" == $3)
        {
            printf "TX timeout (by netdevice driver)\n"
            printf("%u) %.3f 1 Error 0008 - 4 00 00 00 00", NR, currentTimeOffset * 1000)
        }
        else if ("20000002" == $3)
        {
            printf "lost arbitration: "
            if ("00" != $5) printf("in bit number %u\n", $5)
            else            printf "unspecified\n"
            printf("%u) %.3f 1 Error 0008 - 4 00 00 00 00", NR, currentTimeOffset * 1000)
        }
        else if ("20000004" == $3)
        {
            printf "controller problems: "
            direction = 0
            errors = "0"
            if      ("00" == $6) { printf "unspecified\n" }
            else if ("01" == $6) { printf "RX buffer overflow\n"; errors = "40"; direction = 1 }
            else if ("02" == $6) { printf "TX buffer overflow\n"; errors = "40" }
            else if ("04" == $6) { printf "reached warning level for RX errors\n"; errors = "60"; direction = 1 }
            else if ("08" == $6) { printf "reached warning level for TX errors\n"; errors = "60" }
            else if ("10" == $6) { printf "reached error passive status RX\n"; errors = "80"; direction = 1 }
            else if ("20" == $6) { printf "reached error passive status TX\n"; errors = "80" }
            else if ("40" == $6) { printf "recovered to error active state\n" }
            if (1 == direction) printf("%u) %.3f 1 Error 0000 - 4 01 00 %s 00", NR, currentTimeOffset * 1000, errors)
            else                printf("%u) %.3f 1 Error 0000 - 4 00 00 00 %s", NR, currentTimeOffset * 1000, errors)
        }
        else if ("20000008" == $3)
        {
            printf "protocol violations: "
            type = "0008"
            position = $8
            if      ("00" == $7) { printf "unspecified" }
            else if ("01" == $7) { printf "single bit error"; type = "0001" }
            else if ("02" == $7) { printf "frame format error"; type = "0002" }
            else if ("04" == $7) { printf "bit stuffing error"; type = "0004" }
            else if ("08" == $7) { printf "unable to send dominant bit" }
            else if ("10" == $7) { printf "unable to send recessive bit " }
            else if ("20" == $7) { printf "bus overload" }
            else if ("40" == $7) { printf "active error announcement" }
            else if ("80" == $7) { printf "error occurred on transmission" }
            printf " @ "
            if      ("00" == $8) printf "unspecified\n"
            else if ("03" == $8) printf "start of frame\n"
            else if ("02" == $8) printf "ID bits 28 - 21 (SFF: 10 - 3)\n"
            else if ("06" == $8) printf "D bits 20 - 18 (SFF: 2 - 0)\n"
            else if ("04" == $8) printf "substitute RTR (SFF: RTR)\n"
            else if ("05" == $8) printf "identifier extension\n"
            else if ("07" == $8) printf "ID bits 17-13\n"
            else if ("0F" == $8) printf "ID bits 12-5\n"
            else if ("0E" == $8) printf "ID bits 4-0\n"
            else if ("0C" == $8) printf "RTR\n"
            else if ("0D" == $8) printf "reserved bit 1\n"
            else if ("09" == $8) printf "reserved bit 0\n"
            else if ("0B" == $8) printf "data length code\n"
            else if ("0A" == $8) printf "data section\n"
            else if ("08" == $8) printf "CRC sequence\n"
            else if ("18" == $8) printf "CRC delimiter\n"
            else if ("19" == $8) printf "ACK slot\n"
            else if ("1B" == $8) printf "ACK delimiter\n"
            else if ("1A" == $8) printf "end of frame\n"
            else if ("12" == $8) printf "intermission\n"
            printf("%u) %.3f 1 Error %s - 4 00 %s 00 00", NR, currentTimeOffset * 1000, type, position)
        }
        else if ("20000010" == $3)
        {
            printf "transceiver status: "
            if      ("00" == $9) printf "UNSPEC\n"
            else if ("04" == $9) printf "CANH_NO_WIRE\n"
            else if ("05" == $9) printf "CANH_SHORT_TO_BAT\n"
            else if ("06" == $9) printf "CANH_SHORT_TO_VCC\n"
            else if ("07" == $9) printf "CANH_SHORT_TO_GND\n"
            else if ("40" == $9) printf "CANL_NO_WIRE\n"
            else if ("50" == $9) printf "CANL_SHORT_TO_BAT\n"
            else if ("60" == $9) printf "CANL_SHORT_TO_VCC\n"
            else if ("70" == $9) printf "CANL_SHORT_TO_GND\n"
            else if ("80" == $9) printf "CANL_SHORT_TO_CANH\n"
            printf("%u) %.3f 1 Error 0008 - 4 00 1C 00 00", NR, currentTimeOffset * 1000)
        }
        else if ("20000020" == $3)
        {
            printf "received no ACK on transmission\n"
            printf("%u) %.3f 1 Error 0008 - 4 00 00 00 00", NR, currentTimeOffset * 1000)
        }
        else if ("20000040" == $3)
        {
            printf "bus off\n"
            printf("%u) %.3f 1 Error 0000 - 4 00 00 FF FF", NR, currentTimeOffset * 1000)
        }
        else if ("20000080" == $3)
        {
            printf "bus error\n"
            printf("%u) %.3f 1 Error 0008 - 4 00 00 00 00", NR, currentTimeOffset * 1000)
        }
        else if ("20000100" == $3)
        {
            printf "controller restarted\n"
            printf("%u) %.3f 1 Error 0000 - 4 00 00 00 00", NR, currentTimeOffset * 1000)
        }
    }
    else
    {
        printf("%u) %.3f 1 Rx %s - %s", NR, currentTimeOffset * 1000, $3, substr($4, 2, length($4) - 2))
        if (5<=NF)
        {
            for (i=5;i<=NF;i++)
            {
                printf " "$i
            }
        }
    }
    printf "\n"
}


Pretty formatting for repport

Pretty formatting for repport

#!/bin/awk
function getTime()
{
  cmd = "date +%s.%N";
  cmd | getline time;
  close(cmd);
  return time;
}

BEGIN {
  start = getTime();
}

{
  r[$0]++
}

END {
  exec = getTime() - start;
  for(i in r)
  {
    printf("%d\t%.2f\t%s\n", r[i], r[i]/exec, i)
  }
}

#!/bin/awk
function getTime()
{
  cmd = "date +%s.%N";
  cmd | getline time;
  close(cmd);
  return time;
}

BEGIN {
  start = getTime();
}

{
  r[$0]++
}

END {
  exec = getTime() - start;
  for(i in r)
  {
    print r[i]"\t"(r[i]/exec)"\t"i
  }
}


raw params need foaf and dct for provenance

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,machine_uri,whoami);
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n   sioc:account_of <%s>;\n.\n",machine_uri,whoami,whoami,person_uri);
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier <%s>;\n",person_uri,whoami);
   }
                                     print
                                     print ":dataset a void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("      dcterms:creator <%s%s\>;\n",machine_uri,whoami);
   }else if(length(person_uri)) {
                                    printf("      dcterms:creator \"%s\";\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator \"%s\";\n",whoami);
   }
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   } # END if(showConversionProcess > 0)
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS);
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
   }
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,machine_uri,whoami);
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n   sioc:account_of <%s>;\n.\n",machine_uri,whoami,whoami,person_uri);
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier <%s>;\n",person_uri,whoami);
   }
                                     print
                                     print ":dataset a void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("      dcterms:creator <%s%s\>;\n",machine_uri,whoami);
   }else if(length(person_uri)) {
                                    printf("      dcterms:creator \"%s\";\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator \"%s\";\n",whoami);
   }
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   } # END if(showConversionProcess > 0)
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS);
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}