aux/ematch.awk: add bug-compatibility with Busybox awk

The Busybox version of awk won't match backslash-escaped characters
in bracket expressions passed from variables, as awk should.

Ref.: https://bugs.busybox.net/show_bug.cgi?id=12531

To avoid breaking bug-compatibility with Solaris awk, we need a
conditional workaround, so add another test to detectclass() and
use the result if/when translating character classes to ASCII.

#! /usr/bin/awk -f
#
# Script for 'str ematch' for shells that cannot internally handle POSIX
# extended regular expressions (EREs).
#
# Converts POSIX EREs (with bounds) to traditional awk regular expressions
# (without bounds). On very old awk's without support for POSIX character
# classes in bracket expressions, it translates character classes to ASCII
# equivalents. It also disables awk-specific extensions while doing all that.
#
# This script can also be invoked independently as a command.
#
# --- begin license ---
# Copyright (c) 2019 Martijn Dekker <martijn@inlv.org>, Groningen, Netherlands
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# --- end license ---

BEGIN {
	if (ARGC != 3)
		errorout("usage: ematch.awk <string> <ERE>");
	detectclass();
	exit !match(ARGV[1], convertere(ARGV[2]));
}

function errorout(s, ere, i) {
	if (s) printf("str ematch: %s\n", s) | "cat >&2";
	if (ere) printf("%s\n", ere) | "cat >&2";
	if (i) printf(i>1 ? ("%")(i-1)("c^\n") : "^\n", " ") | "cat >&2";
	exit 2;
}

function mylocale() {
	if ("LC_ALL" in ENVIRON && ENVIRON["LC_ALL"] != "")
		return ENVIRON["LC_ALL"];
	else if ("LC_CTYPE" in ENVIRON && ENVIRON["LC_CTYPE"] != "")
		return ENVIRON["LC_CTYPE"];
	else if ("LANG" in ENVIRON && ENVIRON["LANG"] != "")
		return ENVIRON["LANG"];
	else
		return "C";
}

# Detect whether this awk supports character classes properly.
function detectclass() {
	# When a UTF-8 locale is active, onetrueawk (before 2019) only matches the first
	# character class in a bracket expression, even when matching simple ASCII characters.
	# Ref.: http://gnats.netbsd.org/54424
	hasclass = match("1", /[[:alpha:][:digit:]]/);

	# Due to a bug in the macOS C library, onetrueawk and mawk (which don't support UTF-8) yield both false positives
	# and false negatives when matching character classes against high-bit bytes in UTF-8 locales on macOS.
	# Ref.: https://github.com/onetrueawk/awk/issues/45
	if (hasclass && match(mylocale(), /[Uu][Tt][Ff]-?8/))
		hasclass = match("éïÑ", /^[[:alpha:]][[:alpha:]][[:alpha:]]$/);

	# Detect bug with bracket expressions on Busybox awk: it doesn't match backslashed chars.
	# Due to another bug, Solaris awk requires the backslash-escaping that Busybox awk won't
	# match, so we need a conditional workaround when translating character classes.
	# Ref.: https://bugs.busybox.net/show_bug.cgi?id=12531
	if (!hasclass)
		bug_brexpr = !match("\t", "^\\t$");
}

# The terms 'branch', 'piece', 'atom' and 'bound' are used as defined
# in the manual page re_format(7) on BSD/macOS, or regex(7) on Linux.
# OpenBSD has the best version: https://man.openbsd.org/re_format.7

function convertere(ere, par, \
	eere, piece, atom, L, c, i, isave, j, b, m, n, d1, d2, havepunct)
{
	par += 0;  # initialise if not given

	L = length(ere);
	c = substr(ere, 1, 1);
	i = 1;
	while (i <= L) {
		#### Parse just enough ERE grammar to know one atom from the next.
		if (c == "(") {
			# Sub-ERE in parentheses
			atom = ("(")(convertere(substr(ere, ++i), par + 1))(")");
			i += (_Msh_eB_isave - 1);
		} else if (par && c == ")") {
			# End of sub-ERE in parentheses
			_Msh_eB_isave = i;
			return eere;
		} else if (c == "|" || c == "^" || c == "$") {
			# Branch separator, ^/$ anchor, or non-anchor special ^/$
			eere = (eere)(c);
			c = substr(ere, ++i, 1);
			continue;
		} else if (c == "\\" && i < L) {
			# Backslash-escaped character: remove backslash for non-special characters.
			# This disables the C-style backslash escapes that are specific to awk.
			c = substr(ere, ++i, 1);
			if (match(c, /[.[\\()*+?{|^$]/))
				atom = ("\\")(c);
			else
				atom = c;
		} else if (c == "[") {
			# Bracket expression
			havepunct = 0;	# if translating to ASCII, handle [:punct:] specially
			atom = "";
			isave = i;
			i++;
			if (substr(ere, i, 1) == "^") {
				i++;  # negator
				atom = (atom)("^");
			}
			if (substr(ere, i, 1) == "]") {
				i++;  # initial ']' represents itself
				atom = (atom)("]");
			}
			while (i <= L && (c = substr(ere, i, 1)) != "]") {
				if (match(d1 = substr(ere, i, 2), /\[[.=:]/)) {
					# [.collation.], [=equivalence=] or [:character:] class
					d2 = (substr(ere, i+1, 1))("]");
					i += 2;
					isave = i - 1;
					while (i < L && substr(ere, i, 2) != d2)
						i++;
					i++;
					if (i > L)
						errorout("unterminated class", ere, isave);
					j = substr(ere, isave, i - isave);
					if (!hasclass) {
						# No class/locale support: translate to traditional ASCII. Note that awk supports
						# C-style backslash codes even within bracket expressions, unlike POSIX EREs.
						if (j == ":alnum:" ) {
							j = "A-Za-z0-9";
						} else if (j == ":alpha:") {
							j = "A-Za-z";
						} else if (j == ":blank:") {
							if (bug_brexpr) j = " \t"; else j = " \\t";
						} else if (j == ":cntrl:") {
							if (bug_brexpr) j = "\1-\37\177"; else j = "\\1-\\37\\177";
						} else if (j == ":digit:") {
							j = "0-9";
						} else if (j == ":graph:") {
							if (bug_brexpr) j = "\41-\176"; else j = "\\41-\\176";
						} else if (j == ":lower:") {
							j = "a-z";
						} else if (j == ":print:") {
							if (bug_brexpr) j = "\40-\176"; else j = "\\40-\\176";
						} else if (j == ":punct:") {
							havepunct++;
							j = "";
						} else if (j == ":space:") {
							# onetrueawk does not support "\v", so use "\13".
							# https://github.com/onetrueawk/awk/pull/44
							if (bug_brexpr) j = " \t\r\n\13\f"; else j = " \\t\\r\\n\\13\\f";
						} else if (j == ":upper:") {
							j = "A-Z";
						} else if (j == ":xdigit:") {
							j = "A-Fa-f0-9";
						} else if (match(j, /^[=.].*[=.]$/)) {
							# Ignore 1-character collation and equivalence classes.
							if (length(j) > 3)
								errorout("invalid collation character", ere, isave + 1);
							j = substr(j, 2, 1);
							if (j == "\\")
								j = "\\\\";
						} else {
							errorout("invalid character class", ere, isave);
						}
						atom = (atom)(j);
					} else {
						# class/locale support: keep it as is
						atom = (atom)("[")(j)("]");
					}
				} else if (c == "\\") {
					# In POSIX EREs, the backslash is not special at all within a bracket expression, but awk
					# parses C-style backslash escape codes in the entire ERE, including bracket expressions.
					# So to convert a POSIX bracket expression, we have to escape the backslash.
					atom = (atom)("\\\\");
				} else {
					# Other single character in bracket expression.
					atom = (atom)(c);
				}
				i++;
			}
			if (i > L)
				errorout("unterminated bracket expression", ere, isave);
			if (havepunct) {
				# Translate ASCII [:punct:] specially, making sure ']' is the first
				# character, and '-' the last, in the entire bracket expression.
				if (substr(atom, 1, 1) == "]") {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(substr(atom, 2));
				} else {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(atom);
				}
				if (substr(atom, length(atom), 1) != "-") {
					atom = (atom)("-");
				}
			}
			atom = ("[")(atom)("]");
		} else if (c == "*" || c == "+" || c == "?") {
			errorout("repetition operator not valid here", ere, i);
		} else {
			# Simple character
			atom = c;
		}

		#### Now that we've got an atom, parse repetitions to get a piece.
		c = substr(ere, ++i, 1);
		if (c == "*" || c == "+" || c == "?") {
			# Traditional repetition operator.
			piece = (atom)(c);
			c = substr(ere, ++i, 1);
		} else if (c == "{" && match(substr(ere, i+1, 1), /[0123456789]/)) {
			# Interval expression, a.k.a. repetition expression, a.k.a. bound.
			# Expand it in terms of the repetition operators '*', '+' and '?'.
			piece = "";
			isave = i;
			b = 1;		# number of bound params
			m = ""; n = "";	# bound params
			while ( (c = substr(ere, ++i, 1)) != "}" && i <= L ) {
				if (c == ",")
					b++;
				else if (!match(c, /[0123456789]/))
					errorout("bound: bad number", ere, i);
				else if (b == 1)
					m = ( (m)(c) ) + 0; # concatenate, convert to int
				else if (b == 2)
					n = ( (n)(c) ) + 0;
			}
			if (i > L)
				errorout("unterminated bound", ere, isave);
			if (b > 2)
				errorout("bound: too many parameters", ere, isave);
			if (b == 1) {
				# {m}: exactly m occurrences
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
			} else if (n == "") {
				# {m,}: at least m occurrences
				if (m == 0) {
					piece = (atom)("*");
				} else {
					for (j = 1; j <= m; j++)
						piece = (piece)(atom);
					piece = (piece)("+");
				}
			} else {
				# {m,n}: any number of occurrences between m and n, inclusive
				if (n < m)
					errorout("bad bound: max < min", ere, isave);
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
				for (; j <= n; j++)
					piece = (piece)(atom)("?");
			}
			c = substr(ere, ++i, 1);
		} else {
			# No repetition.
			piece = atom;
		}
		eere = (eere)(piece);
	}
	# End of ERE.
	if (par)
		errorout("unbalanced parentheses", ere);
	return eere;
}

#! /usr/bin/awk -f
#
# Script for 'str ematch' for shells that cannot internally handle POSIX
# extended regular expressions (EREs).
#
# Converts POSIX EREs (with bounds) to traditional awk regular expressions
# (without bounds). On very old awk's without support for POSIX character
# classes in bracket expressions, it translates character classes to ASCII
# equivalents. It also disables awk-specific extensions while doing all that.
#
# This script can also be invoked independently as a command.
#
# --- begin license ---
# Copyright (c) 2019 Martijn Dekker <martijn@inlv.org>, Groningen, Netherlands
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# --- end license ---

BEGIN {
	if (ARGC != 3)
		errorout("usage: ematch.awk <string> <ERE>");
	detectclass();
	exit !match(ARGV[1], convertere(ARGV[2]));
}

function errorout(s, ere, i) {
	if (s) printf("str ematch: %s\n", s) | "cat >&2";
	if (ere) printf("%s\n", ere) | "cat >&2";
	if (i) printf(i>1 ? ("%")(i-1)("c^\n") : "^\n", " ") | "cat >&2";
	exit 2;
}

function mylocale() {
	if ("LC_ALL" in ENVIRON && ENVIRON["LC_ALL"] != "")
		return ENVIRON["LC_ALL"];
	else if ("LC_CTYPE" in ENVIRON && ENVIRON["LC_CTYPE"] != "")
		return ENVIRON["LC_CTYPE"];
	else if ("LANG" in ENVIRON && ENVIRON["LANG"] != "")
		return ENVIRON["LANG"];
	else
		return "C";
}

# Detect whether this awk supports character classes properly.
function detectclass() {
	# When a UTF-8 locale is active, onetrueawk (before 2019) only matches the first
	# character class in a bracket expression, even when matching simple ASCII characters.
	# Ref.: http://gnats.netbsd.org/54424
	hasclass = match("1", /[[:alpha:][:digit:]]/);

	# Due to a bug in the macOS C library, onetrueawk and mawk (which don't support UTF-8) yield both false positives
	# and false negatives when matching character classes against high-bit bytes in UTF-8 locales on macOS.
	# Ref.: https://github.com/onetrueawk/awk/issues/45
	if (hasclass && match(mylocale(), /[Uu][Tt][Ff]-?8/))
		hasclass = match("éïÑ", /^[[:alpha:]][[:alpha:]][[:alpha:]]$/);
}

# The terms 'branch', 'piece', 'atom' and 'bound' are used as defined
# in the manual page re_format(7) on BSD/macOS, or regex(7) on Linux.
# OpenBSD has the best version: https://man.openbsd.org/re_format.7

function convertere(ere, par, \
	eere, piece, atom, L, c, i, isave, j, b, m, n, d1, d2, havepunct)
{
	par += 0;  # initialise if not given

	L = length(ere);
	c = substr(ere, 1, 1);
	i = 1;
	while (i <= L) {
		#### Parse just enough ERE grammar to know one atom from the next.
		if (c == "(") {
			# Sub-ERE in parentheses
			atom = ("(")(convertere(substr(ere, ++i), par + 1))(")");
			i += (_Msh_eB_isave - 1);
		} else if (par && c == ")") {
			# End of sub-ERE in parentheses
			_Msh_eB_isave = i;
			return eere;
		} else if (c == "|" || c == "^" || c == "$") {
			# Branch separator, ^/$ anchor, or non-anchor special ^/$
			eere = (eere)(c);
			c = substr(ere, ++i, 1);
			continue;
		} else if (c == "\\" && i < L) {
			# Backslash-escaped character: remove backslash for non-special characters.
			# This disables the C-style backslash escapes that are specific to awk.
			c = substr(ere, ++i, 1);
			if (match(c, /[.[\\()*+?{|^$]/))
				atom = ("\\")(c);
			else
				atom = c;
		} else if (c == "[") {
			# Bracket expression
			havepunct = 0;	# if translating to ASCII, handle [:punct:] specially
			atom = "";
			isave = i;
			i++;
			if (substr(ere, i, 1) == "^") {
				i++;  # negator
				atom = (atom)("^");
			}
			if (substr(ere, i, 1) == "]") {
				i++;  # initial ']' represents itself
				atom = (atom)("]");
			}
			while (i <= L && (c = substr(ere, i, 1)) != "]") {
				if (match(d1 = substr(ere, i, 2), /\[[.=:]/)) {
					# [.collation.], [=equivalence=] or [:character:] class
					d2 = (substr(ere, i+1, 1))("]");
					i += 2;
					isave = i - 1;
					while (i < L && substr(ere, i, 2) != d2)
						i++;
					i++;
					if (i > L)
						errorout("unterminated class", ere, isave);
					j = substr(ere, isave, i - isave);
					if (!hasclass) {
						# No class/locale support: translate to traditional ASCII. Note that awk supports
						# C-style backslash codes even within bracket expressions, unlike POSIX EREs.
						if (j == ":alnum:" ) {
							j = "A-Za-z0-9";
						} else if (j == ":alpha:") {
							j = "A-Za-z";
						} else if (j == ":blank:") {
							j = " \\t";
						} else if (j == ":cntrl:") {
							j = "\\1-\\37\\177";
						} else if (j == ":digit:") {
							j = "0-9";
						} else if (j == ":graph:") {
							j = "\\41-\\176";
						} else if (j == ":lower:") {
							j = "a-z";
						} else if (j == ":print:") {
							j = "\\40-\\176";
						} else if (j == ":punct:") {
							havepunct++;
							j = "";
						} else if (j == ":space:") {
							# onetrueawk does not support "\v", so use "\13".
							# https://github.com/onetrueawk/awk/pull/44
							j = " \\t\\r\\n\\13\\f";
						} else if (j == ":upper:") {
							j = "A-Z";
						} else if (j == ":xdigit:") {
							j = "A-Fa-f0-9";
						} else if (match(j, /^[=.].*[=.]$/)) {
							# Ignore 1-character collation and equivalence classes.
							if (length(j) > 3)
								errorout("invalid collation character", ere, isave + 1);
							j = substr(j, 2, 1);
							if (j == "\\")
								j = "\\\\";
						} else {
							errorout("invalid character class", ere, isave);
						}
						atom = (atom)(j);
					} else {
						# class/locale support: keep it as is
						atom = (atom)("[")(j)("]");
					}
				} else if (c == "\\") {
					# In POSIX EREs, the backslash is not special at all within a bracket expression, but awk
					# parses C-style backslash escape codes in the entire ERE, including bracket expressions.
					# So to convert a POSIX bracket expression, we have to escape the backslash.
					atom = (atom)("\\\\");
				} else {
					# Other single character in bracket expression.
					atom = (atom)(c);
				}
				i++;
			}
			if (i > L)
				errorout("unterminated bracket expression", ere, isave);
			if (havepunct) {
				# Translate ASCII [:punct:] specially, making sure ']' is the first
				# character, and '-' the last, in the entire bracket expression.
				if (substr(atom, 1, 1) == "]") {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(substr(atom, 2));
				} else {
					atom = ("][!\"#$%&'()*+,\\./:;<=>?@^_`{|}~")(atom);
				}
				if (substr(atom, length(atom), 1) != "-") {
					atom = (atom)("-");
				}
			}
			atom = ("[")(atom)("]");
		} else if (c == "*" || c == "+" || c == "?") {
			errorout("repetition operator not valid here", ere, i);
		} else {
			# Simple character
			atom = c;
		}

		#### Now that we've got an atom, parse repetitions to get a piece.
		c = substr(ere, ++i, 1);
		if (c == "*" || c == "+" || c == "?") {
			# Traditional repetition operator.
			piece = (atom)(c);
			c = substr(ere, ++i, 1);
		} else if (c == "{" && match(substr(ere, i+1, 1), /[0123456789]/)) {
			# Interval expression, a.k.a. repetition expression, a.k.a. bound.
			# Expand it in terms of the repetition operators '*', '+' and '?'.
			piece = "";
			isave = i;
			b = 1;		# number of bound params
			m = ""; n = "";	# bound params
			while ( (c = substr(ere, ++i, 1)) != "}" && i <= L ) {
				if (c == ",")
					b++;
				else if (!match(c, /[0123456789]/))
					errorout("bound: bad number", ere, i);
				else if (b == 1)
					m = ( (m)(c) ) + 0; # concatenate, convert to int
				else if (b == 2)
					n = ( (n)(c) ) + 0;
			}
			if (i > L)
				errorout("unterminated bound", ere, isave);
			if (b > 2)
				errorout("bound: too many parameters", ere, isave);
			if (b == 1) {
				# {m}: exactly m occurrences
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
			} else if (n == "") {
				# {m,}: at least m occurrences
				if (m == 0) {
					piece = (atom)("*");
				} else {
					for (j = 1; j <= m; j++)
						piece = (piece)(atom);
					piece = (piece)("+");
				}
			} else {
				# {m,n}: any number of occurrences between m and n, inclusive
				if (n < m)
					errorout("bad bound: max < min", ere, isave);
				for (j = 1; j <= m; j++)
					piece = (piece)(atom);
				for (; j <= n; j++)
					piece = (piece)(atom)("?");
			}
			c = substr(ere, ++i, 1);
		} else {
			# No repetition.
			piece = atom;
		}
		eere = (eere)(piece);
	}
	# End of ERE.
	if (par)
		errorout("unbalanced parentheses", ere);
	return eere;
}


Add TOLOWER to lexicon.awk

Add TOLOWER to lexicon.awk

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    for (i = 1; i <= split($3, words, ", "); i++) {
        if (TOLOWER) words[i] = tolower(words[i]);
        print words[i] | "sort --parallel=$(nproc) -S1G -us";
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    for (i = 1; i <= split($3, words, ", "); i++) {
        print words[i] | "sort --parallel=$(nproc) -S1G -us";
    }
}


Commons: add append()

Commons: add append()

#!/usr/bin/gawk -f

# Append an element into an array (zero-based).
function append(array, element) {
    array[length(array)] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Debug an array.
function da(array, formatString, sortedIn,
                ####
                i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

#!/usr/bin/gawk -f

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Debug an array.
function da(array, formatString, sortedIn,
                ####
                i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}


TestParser: test parseJson()

TestParser: test parseJson()

@include "include/Parser"

BEGIN {
    START_TEST("Parser.awk")

    T("tokenize()", 8)
    {
        delete tokens
        tokenize(tokens, "0")
        delete expected
        expected[0] = "0"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "3.14")
        delete expected
        expected[0] = "3.14"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "Atom")
        delete expected
        expected[0] = "Atom"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"foo bar\"")
        delete expected
        expected[0] = "\"foo bar\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"\\\"\"")
        delete expected
        expected[0] = "\"\\\"\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "(QUOTE A)")
        delete expected
        expected[0] = "("
        expected[1] = "QUOTE"
        expected[2] = "A"
        expected[3] = ")"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "1 + 2 * 3")
        delete expected
        expected[0] = "1"
        expected[1] = "+"
        expected[2] = "2"
        expected[3] = "*"
        expected[4] = "3"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "[42, \"hello, world\", [Foo, bar]]")
        delete expected
        expected[0] = "["
        expected[1] = "42"
        expected[2] = ","
        expected[3] = "\"hello, world\""
        expected[4] = ","
        expected[5] = "["
        expected[6] = "Foo"
        expected[7] = ","
        expected[8] = "bar"
        expected[9] = "]"
        expected[10] = "]"
        assertEqual(tokens, expected)
    }

    T("parseJson()", 5)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseJson(ast, tokens)
        expected[0] = "0"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "null")
        parseJson(ast, tokens)
        expected[0] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "[42]")
        parseJson(ast, tokens)
        expected[0 SUBSEP 0] = "42"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "[42, \"answer\", null]")
        parseJson(ast, tokens)
        expected[0 SUBSEP 0] = "42"
        expected[0 SUBSEP 1] = "\"answer\""
        expected[0 SUBSEP 2] = "null"
        assertEqual(ast, expected)

        delete tokens; delete ast; delete expected
        tokenize(tokens, "{\"answer\": [42], \"Answer\": null}")
        parseJson(ast, tokens)
        expected[0 SUBSEP "answer" SUBSEP 0] = 42
        expected[0 SUBSEP "Answer"] = "null"
        assertEqual(ast, expected)
    }

    T("parseList()", 1)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseList(ast, tokens)
        expected[NULLSTR][0] = "0"
        assertEqual(ast, expected)
    }

    END_TEST()
}

@include "include/Parser"

BEGIN {
    START_TEST("Parser.awk")

    T("tokenize()", 8)
    {
        delete tokens
        tokenize(tokens, "0")
        delete expected
        expected[0] = "0"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "3.14")
        delete expected
        expected[0] = "3.14"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "Atom")
        delete expected
        expected[0] = "Atom"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"foo bar\"")
        delete expected
        expected[0] = "\"foo bar\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "\"\\\"\"")
        delete expected
        expected[0] = "\"\\\"\""
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "(QUOTE A)")
        delete expected
        expected[0] = "("
        expected[1] = "QUOTE"
        expected[2] = "A"
        expected[3] = ")"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "1 + 2 * 3")
        delete expected
        expected[0] = "1"
        expected[1] = "+"
        expected[2] = "2"
        expected[3] = "*"
        expected[4] = "3"
        assertEqual(tokens, expected)

        delete tokens
        tokenize(tokens, "[42, \"hello, world\", [Foo, bar]]")
        delete expected
        expected[0] = "["
        expected[1] = "42"
        expected[2] = ","
        expected[3] = "\"hello, world\""
        expected[4] = ","
        expected[5] = "["
        expected[6] = "Foo"
        expected[7] = ","
        expected[8] = "bar"
        expected[9] = "]"
        expected[10] = "]"
        assertEqual(tokens, expected)
    }

    T("parseList()", 1)
    {
        delete tokens; delete ast; delete expected
        tokenize(tokens, "0")
        parseList(ast, tokens)
        expected[NULLSTR][0] = "0"
        assertEqual(ast, expected)
    }

    END_TEST()
}


include param to reduce visual clutter when counts have no meaning

#! /usr/bin/awk -f
# deltadot.awk <encumbant.gv> <candidate.gv>
# Use to illuminate the commonality and differences between dot files created
# by 'ntriples2dot.awk' but might work for other dot files
#
# To ignore counts
# deltadot.awk -v"NOCOUNTS=1" <encumbant.gv> <candidate.gv>
#
# expects two graphviz dot digraph files;
# by convention "old"  then "new"
# for color to mean;  orange lost in "new" and blue gained in "new"
# with simple node names ('C-lang' variable naming conventions)
# edge's unadorned predicate label
# concludes with a space then count in parens
#
# For example:
# NODE1 -> NODE2 [label="curie:term (42)"];
#
# filter for edges in each file associate the count with the edge
# if an edge is common between the files
# then change its "count" to the difference. (second minus first)
# otherwise leave count as is
# UNLESS there is a parameter NOCOUNTS not equal to 0
# in which case set all counts the same
# this is for use when the counts have no meaning
# such as with random samples

# associate the count with an edge
function parse(str, arr){
	split(str, a, "(")
	arr[a[1]] = substr(a[2], 1, index(a[2], ")")-1)
}

# strip metadata from filename
function de_path_ext(pth){
	split(pth, a, "/")
	return substr(a[length(a)], 1, index(a[length(a)], ".")-1)
}
BEGIN { NOCOUNTS=0}

# collect the edges
NR==FNR && / -> / {
	label1 = FILENAME;
	parse($0, edge1)
}
NR!=FNR && / -> / {
	label2 = FILENAME;
	parse($0, edge2)
}

END{
	for(e in edge1){
		if(!(e in edge2)) edges[e "(" edge1[e] ")\", color=\"orange\"];"];
		else {
			# TODO flag losses?
			if(NOCOUNTS)
			    diff = ""
			else
			    diff = edge2[e] - edge1[e];
			#if(diff < 0) diff = "<b>" diff "</b>";
			edges[e "(" diff ")\", color=\"black\"];"]
			delete edge2[e]
		}
	}
	for(e in edge2) edges[e "(" edge2[e] ")\", color=\"blue\"];"];

	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	print "label=\"" de_path_ext(label1) " <=> " de_path_ext(label2) "\""

	n = asorti(edges,output)
	for(i=1;i<=n;i++ ){print output[i]}
	print "\n}\n"
}

#! /usr/bin/awk -f
#deltadot.awk <encumbant.gv> <candidate.gv>
# Use to illuminate the commonality and differences between dot files created
# by 'ntriples2dot.awk' but might work for other dot files
#
#
# expects two graphviz dot digraph files by convention "old"  then "new"
# for color to mean;  orange lost in "new" and blue gained in "new"
# with simple node names ('C' variable naming conventions)
# edge's unadorned predicate label
# concludes with a space then count in parens
#
# For example:
# NODE1 -> NODE2 [label="curie:term (42)"];
#
# filter for edges in each file associate the count with the edge
# if an edge is common between the files
# then change its "count" to the difference. (second minus first)
# otherwise leave count as is

# associate the count with an edge
function parse(str, arr){
	split(str, a, "(")
	arr[a[1]] = substr(a[2], 1, index(a[2], ")")-1)
}

# strip metadata from filename
function de_path_ext(pth){
	split(pth, a, "/")
	return substr(a[length(a)], 1, index(a[length(a)], ".")-1)
}

# collect the edges
NR==FNR && / -> / {
	label1 = FILENAME;
	parse($0, edge1)
}
NR!=FNR && / -> / {
	label2 = FILENAME;
	parse($0, edge2)
}

END{
	for(e in edge1){
		if(!(e in edge2)) edges[e "(" edge1[e] ")\", color=\"orange\"];"];
		else {
			# TODO flag losses?
			diff =  edge2[e] - edge1[e];
			#if(diff < 0) diff = "<b>" diff "</b>";
			edges[e "(" diff ")\", color=\"black\"];"]
			delete edge2[e]
		}
	}
	for(e in edge2) edges[e "(" edge2[e] ")\", color=\"blue\"];"];

	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	print "label=\"" de_path_ext(label1) " <=> " de_path_ext(label2) "\""

	n = asorti(edges,output)
	for(i=1;i<=n;i++ ){print output[i]}
	print "\n}\n"
}


adding a newline at the end of the file, I'm getting an error otherwise

# posi.awk
# used by TKLSpell (posi.sh)
# get position from languagetool output
# input: languagetool output with -x
# output: first_pos second_pos text
#  after sort -n all errors sequentially
#
#  usage: python TextChecker.py -l hu -x tools/mytext.txt >tools/mytextout.txt
#         awk -f <tools/mytextout-txt >tools/mytextout2.txt
#         sort -n tools/mytextout.txt >tools/mytextout3.txt
#
# assuming, yourtext.txt is languagetool/tools
# to simplify usage, use: sh posi.sh yourtext
#
/from=/  { out = "";
          split($0, a, "from=\"");
			 split(a[2], b, "\"");
          split(b[3], c, "\"");
			 out = b[1] " " c[1];
          }
/<message>/ {split($0, aa, "<message>");
 				 split(aa[2], bb, "</message>");
             out = out " " bb[1];
				}
			{ if(bb[1] == ""){
			    split($0, aaa, "\">");
				 split(aaa[2], bbb, "</error>");
				 out = out " " bbb[1];
		     }
				print out;
			  bb[1] = "";
			}


Update dxfinfo.awk

Update dxfinfo.awk

#!/bin/gawk -f
#
# statistics about entities and layers in a DXF file
# Zoltan Siki siki.zoltan@epito.bme.hu
# usage: dxfinfo.awk your_file.dxf
#		or
#        gawk -f dxfinfo.awk your_file.dxf > statistics.txt
#        gawk -f dxfinfo.awk your_file.dxf | sort > statistics.txt

BEGIN { # initialize variables
	last = ""; entity = ""; layer = "";
}
/ENTITIES/,/ENDSEC/ { # process only the entities section
	if (last == "  0") {	# last row entity start code?
		entity = $0;
	} else if (last == "  8") {	# last row layer name code?
		layer = $0;
		dxf[layer,entity]++;	# increment entity count for layer, entity combination
	}
	last = $0;	# remember last input row
}
END {	# print out result
	for (i in dxf) {	# for each layer,entity pairs
		split(i, w, SUBSEP);	# separate layer and entity in index into w array
		print w[1], w[2], dxf[i];	# print layer, entity and count
	}
}

#!/bin/gawk -f
#
# statistics about entities and layers in a DXF file
# Zoltan Siki siki.zoltan@epito.bme.hu
# usage: dxfinfo.awk your_file.dxf
#		or
#        gawk -f dxfinfo.awk your_file.dxf > statistics.txt
#        gawk -f dxfinfo.awk your_file.dxf | sort > statistics.txt

BEGIN { # initialize variables
	last = ""; entity = ""; layer = "";
}
/ENTITIES/,/ENDSEC/ { # process only the entities section
	if (last == "  0") {	# last row entity start code?
		entity = $0;
	} else if (last == "  8") {	# last row layer name code?
		layer = $0;
		dxf[layer,entity]++;	# increment entity count for layer, entity combination
	}
	last = $0;	# remember last input row
}
END {	# print out result
	for (i in dxf) {	# for each layer,entity pairs
		split(i, w, SUBSEP);	# separate layer and entity in index into w array
		print w[1], w[2], dxf[i];	# print layer, entity anc count
	}
}


Update year for gln testing.

Update year for gln testing.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@39483 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either status is _not_ manifest or ending year is gt or eq to contract year
  current_year = 2014
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "American R")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "American S") && (substr(p[pn],1,2) <= "Bz")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Rz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "SA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either status is _not_ manifest or ending year is gt or eq to contract year
  current_year = 2013
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "American R")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "American S") && (substr(p[pn],1,2) <= "Bz")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Rz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "SA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}