Parser: fix parseJson() for empty objects

####################################################################
# Parser.awk                                                       #
####################################################################

# Tokenize a string and return a token list.
function tokenize(returnTokens, string,
                  delimiters,
                  newlines,
                  quotes,
                  escapeChars,
                  leftBlockComments,
                  rightBlockComments,
                  lineComments,
                  reservedOperators,
                  reservedPatterns,
                  ####
                  blockCommenting,
                  c,
                  currentToken,
                  escaping,
                  i,
                  lineCommenting,
                  p,
                  quoting,
                  r,
                  s,
                  tempGroup,
                  tempPattern,
                  tempString) {
    # Default parameters
    if (!delimiters[0]) {
        delimiters[0] = " "  # whitespace
        delimiters[1] = "\t" # horizontal tab
        delimiters[2] = "\v" # vertical tab
    }
    if (!newlines[0]) {
        newlines[0] = "\n" # line feed
        newlines[1] = "\r" # carriage return
    }
    if (!quotes[0]) {
        quotes[0] = "\"" # double quote
    }
    if (!escapeChars[0]) {
        escapeChars[0] = "\\" # backslash
    }
    if (!leftBlockComments[0]) {
        leftBlockComments[0] = "#|" # Lisp-style extended comment (open)
        leftBlockComments[1] = "/*" # C-style comment (open)
        leftBlockComments[2] = "(*" # ML-style comment (open)
    }
    if (!rightBlockComments[0]) {
        rightBlockComments[0] = "|#" # Lisp-style extended comment (close)
        rightBlockComments[1] = "*/" # C-style comment (close)
        rightBlockComments[2] = "*)" # ML-style comment (close)
    }
    if (!lineComments[0]) {
        lineComments[0] = ";"  # Lisp-style line comment
        lineComments[1] = "//" # C++-style line comment
        lineComments[2] = "#"  # hash comment
    }
    if (!reservedOperators[0]) {
        reservedOperators[0] = "(" #  left parenthesis
        reservedOperators[1] = ")" # right parenthesis
        reservedOperators[2] = "[" #  left bracket
        reservedOperators[3] = "]" # right bracket
        reservedOperators[4] = "{" #  left brace
        reservedOperators[5] = "}" # right brace
        reservedOperators[6] = "," # comma
    }
    if (!reservedPatterns[0]) {
        reservedPatterns[0] = "[+-]?((0|[1-9][0-9]*)|[.][0-9]*|(0|[1-9][0-9]*)[.][0-9]*)([Ee][+-]?[0-9]+)?" # numeric literal (scientific notation possible)
        reservedPatterns[1] = "[+-]?0[0-7]+([.][0-7]*)?" # numeric literal (octal)
        reservedPatterns[2] = "[+-]?0[Xx][0-9A-Fa-f]+([.][0-9A-Fa-f]*)?" # numeric literal (hexadecimal)
    }

    split(string, s, "")
    currentToken = ""
    quoting = escaping = blockCommenting = lineCommenting = 0
    p = 0
    i = 1
    while (i <= length(s)) {
        c = s[i]
        r = substr(string, i)

        if (blockCommenting) {
            if (tempString = startsWithAny(r, rightBlockComments))
                blockCommenting = 0 # block comment ends

            i++

        } else if (lineCommenting) {
            if (belongsTo(c, newlines))
                lineCommenting = 0 # line comment ends

            i++

        } else if (quoting) {
            currentToken = currentToken c

            if (escaping) {
                escaping = 0 # escape ends

            } else {
                if (belongsTo(c, quotes)) {
                    # Finish the current token
                    if (currentToken) {
                        returnTokens[p++] = currentToken
                        currentToken = ""
                    }

                    quoting = 0 # quotation ends

                } else if (belongsTo(c, escapeChars)) {
                    escaping = 1 # escape begins

                } else {
                    # Continue
                }
            }

            i++

        } else {
            if (belongsTo(c, delimiters) || belongsTo(c, newlines)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                i++

            } else if (belongsTo(c, quotes)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                }

                currentToken = c

                quoting = 1 # quotation begins

                i++

            } else if (tempString = startsWithAny(r, leftBlockComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                blockCommenting = 1 # block comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, lineComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                lineCommenting = 1 # line comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, reservedOperators)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                returnTokens[p++] = tempString

                i += length(tempString)

            } else if (tempPattern = matchesAny(r, reservedPatterns)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                match(r, "^" tempPattern, tempGroup)
                returnTokens[p++] = tempGroup[0]

                i += length(tempGroup[0])

            } else {
                # Continue with the current token
                currentToken = currentToken c

                i++
            }
        }
    }

    # Finish the last token
    if (currentToken)
        returnTokens[p++] = currentToken

}

# Parse a token list of JSON array and return an AST.
function parseJsonArray(returnAST, tokens,
                        leftBrackets,
                        rightBrackets,
                        separators,
                        ####
                        i, j, key, p, stack, token) {
    # Default parameters
    if (!leftBrackets[0]) {
        leftBrackets[0] = "(" # left parenthesis
        leftBrackets[1] = "[" # left bracket
        leftBrackets[2] = "{" # left brace
    }
    if (!rightBrackets[0]) {
        rightBrackets[0] = ")" # right parenthesis
        rightBrackets[1] = "]" # right bracket
        rightBrackets[2] = "}" # right brace
    }
    if (!separators[0]) {
        separators[0] = "," # comma
    }

    stack[p = 0] = 0
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, leftBrackets))
            stack[++p] = 0
        else if (belongsTo(token, rightBrackets))
            --p
        else if (belongsTo(token, separators))
            stack[p]++
        else {
            key = stack[0]
            for (j = 1; j <= p; j++)
                key = key SUBSEP stack[j]
            returnAST[key] = token
        }
    }
}

# JSON parser.
function parseJson(returnAST, tokens,
                   arrayStartTokens, arrayEndTokens,
                   objectStartTokens, objectEndTokens,
                   commas, colons,
                   ####
                   flag, i, j, key, name, p, stack, token) {
    # Default parameters
    if (!arrayStartTokens[0])  arrayStartTokens[0]  = "["
    if (!arrayEndTokens[0])    arrayEndTokens[0]    = "]"
    if (!objectStartTokens[0]) objectStartTokens[0] = "{"
    if (!objectEndTokens[0])   objectEndTokens[0]   = "}"
    if (!commas[0])            commas[0]            = ","
    if (!colons[0])            colons[0]            = ":"

    stack[p = 0] = 0
    flag = 0 # ready to read key
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, arrayStartTokens))
            stack[++p] = 0
        else if (belongsTo(token, objectStartTokens))
            stack[++p] = NULLSTR
        else if (belongsTo(token, objectEndTokens) ||
                 belongsTo(token, arrayEndTokens))
            --p
        else if (belongsTo(token, commas)) {
            if (isnum(stack[p])) # array
                stack[p]++ # increase index
            else # anyway
                flag = 0 # ready to read key
        } else if (belongsTo(token, colons)) {
            flag = 1 # ready to read value
        } else if (isnum(stack[p]) || flag) {
            # Read a value
            key = stack[0]
            for (j = 1; j <= p; j++)
                key = key SUBSEP stack[j]
            returnAST[key] = token
            flag = 0 # ready to read key
        } else {
            # Read a key
            stack[p] = unparameterize(token)
        }
    }
}

# S-expr parser.
function parseList(returnAST, tokens,
                   leftBrackets,
                   rightBrackets,
                   separators,
                   ####
                   i, j, key, p, stack, token) {
    # Default parameters
    if (!leftBrackets[0]) {
        leftBrackets[0] = "(" # left parenthesis
        leftBrackets[1] = "[" # left bracket
        leftBrackets[2] = "{" # left brace
    }
    if (!rightBrackets[0]) {
        rightBrackets[0] = ")" # right parenthesis
        rightBrackets[1] = "]" # right bracket
        rightBrackets[2] = "}" # right brace
    }
    if (!separators[0]) {
        separators[0] = "," # comma
    }

    stack[p = 0] = 0
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, leftBrackets)) {
            stack[++p] = 0
        } else if (belongsTo(token, rightBrackets)) {
            stack[--p]++
        } else if (belongsTo(token, separators)) {
            # skip
        } else {
            key = NULLSTR
            if (p > 0) {
                for (j = 0; j < p - 1; j++)
                    key = key SUBSEP stack[j]
                returnAST[key][stack[p - 1]] = NULLSTR
                key = key SUBSEP stack[p - 1]
            }
            returnAST[key][stack[p]] = token
            stack[p]++
        }
    }
}

####################################################################
# Parser.awk                                                       #
####################################################################

# Tokenize a string and return a token list.
function tokenize(returnTokens, string,
                  delimiters,
                  newlines,
                  quotes,
                  escapeChars,
                  leftBlockComments,
                  rightBlockComments,
                  lineComments,
                  reservedOperators,
                  reservedPatterns,
                  ####
                  blockCommenting,
                  c,
                  currentToken,
                  escaping,
                  i,
                  lineCommenting,
                  p,
                  quoting,
                  r,
                  s,
                  tempGroup,
                  tempPattern,
                  tempString) {
    # Default parameters
    if (!delimiters[0]) {
        delimiters[0] = " "  # whitespace
        delimiters[1] = "\t" # horizontal tab
        delimiters[2] = "\v" # vertical tab
    }
    if (!newlines[0]) {
        newlines[0] = "\n" # line feed
        newlines[1] = "\r" # carriage return
    }
    if (!quotes[0]) {
        quotes[0] = "\"" # double quote
    }
    if (!escapeChars[0]) {
        escapeChars[0] = "\\" # backslash
    }
    if (!leftBlockComments[0]) {
        leftBlockComments[0] = "#|" # Lisp-style extended comment (open)
        leftBlockComments[1] = "/*" # C-style comment (open)
        leftBlockComments[2] = "(*" # ML-style comment (open)
    }
    if (!rightBlockComments[0]) {
        rightBlockComments[0] = "|#" # Lisp-style extended comment (close)
        rightBlockComments[1] = "*/" # C-style comment (close)
        rightBlockComments[2] = "*)" # ML-style comment (close)
    }
    if (!lineComments[0]) {
        lineComments[0] = ";"  # Lisp-style line comment
        lineComments[1] = "//" # C++-style line comment
        lineComments[2] = "#"  # hash comment
    }
    if (!reservedOperators[0]) {
        reservedOperators[0] = "(" #  left parenthesis
        reservedOperators[1] = ")" # right parenthesis
        reservedOperators[2] = "[" #  left bracket
        reservedOperators[3] = "]" # right bracket
        reservedOperators[4] = "{" #  left brace
        reservedOperators[5] = "}" # right brace
        reservedOperators[6] = "," # comma
    }
    if (!reservedPatterns[0]) {
        reservedPatterns[0] = "[+-]?((0|[1-9][0-9]*)|[.][0-9]*|(0|[1-9][0-9]*)[.][0-9]*)([Ee][+-]?[0-9]+)?" # numeric literal (scientific notation possible)
        reservedPatterns[1] = "[+-]?0[0-7]+([.][0-7]*)?" # numeric literal (octal)
        reservedPatterns[2] = "[+-]?0[Xx][0-9A-Fa-f]+([.][0-9A-Fa-f]*)?" # numeric literal (hexadecimal)
    }

    split(string, s, "")
    currentToken = ""
    quoting = escaping = blockCommenting = lineCommenting = 0
    p = 0
    i = 1
    while (i <= length(s)) {
        c = s[i]
        r = substr(string, i)

        if (blockCommenting) {
            if (tempString = startsWithAny(r, rightBlockComments))
                blockCommenting = 0 # block comment ends

            i++

        } else if (lineCommenting) {
            if (belongsTo(c, newlines))
                lineCommenting = 0 # line comment ends

            i++

        } else if (quoting) {
            currentToken = currentToken c

            if (escaping) {
                escaping = 0 # escape ends

            } else {
                if (belongsTo(c, quotes)) {
                    # Finish the current token
                    if (currentToken) {
                        returnTokens[p++] = currentToken
                        currentToken = ""
                    }

                    quoting = 0 # quotation ends

                } else if (belongsTo(c, escapeChars)) {
                    escaping = 1 # escape begins

                } else {
                    # Continue
                }
            }

            i++

        } else {
            if (belongsTo(c, delimiters) || belongsTo(c, newlines)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                i++

            } else if (belongsTo(c, quotes)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                }

                currentToken = c

                quoting = 1 # quotation begins

                i++

            } else if (tempString = startsWithAny(r, leftBlockComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                blockCommenting = 1 # block comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, lineComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                lineCommenting = 1 # line comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, reservedOperators)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                returnTokens[p++] = tempString

                i += length(tempString)

            } else if (tempPattern = matchesAny(r, reservedPatterns)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                match(r, "^" tempPattern, tempGroup)
                returnTokens[p++] = tempGroup[0]

                i += length(tempGroup[0])

            } else {
                # Continue with the current token
                currentToken = currentToken c

                i++
            }
        }
    }

    # Finish the last token
    if (currentToken)
        returnTokens[p++] = currentToken

}

# Parse a token list of JSON array and return an AST.
function parseJsonArray(returnAST, tokens,
                        leftBrackets,
                        rightBrackets,
                        separators,
                        ####
                        i, j, key, p, stack, token) {
    # Default parameters
    if (!leftBrackets[0]) {
        leftBrackets[0] = "(" # left parenthesis
        leftBrackets[1] = "[" # left bracket
        leftBrackets[2] = "{" # left brace
    }
    if (!rightBrackets[0]) {
        rightBrackets[0] = ")" # right parenthesis
        rightBrackets[1] = "]" # right bracket
        rightBrackets[2] = "}" # right brace
    }
    if (!separators[0]) {
        separators[0] = "," # comma
    }

    stack[p = 0] = 0
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, leftBrackets))
            stack[++p] = 0
        else if (belongsTo(token, rightBrackets))
            --p
        else if (belongsTo(token, separators))
            stack[p]++
        else {
            key = stack[0]
            for (j = 1; j <= p; j++)
                key = key SUBSEP stack[j]
            returnAST[key] = token
        }
    }
}

# JSON parser.
function parseJson(returnAST, tokens,
                   arrayStartTokens, arrayEndTokens,
                   objectStartTokens, objectEndTokens,
                   commas, colons,
                   ####
                   flag, i, j, key, name, p, stack, token) {
    # Default parameters
    if (!arrayStartTokens[0])  arrayStartTokens[0]  = "["
    if (!arrayEndTokens[0])    arrayEndTokens[0]    = "]"
    if (!objectStartTokens[0]) objectStartTokens[0] = "{"
    if (!objectEndTokens[0])   objectEndTokens[0]   = "}"
    if (!commas[0])            commas[0]            = ","
    if (!colons[0])            colons[0]            = ":"

    stack[p = 0] = 0
    flag = 0 # ready to read key
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, arrayStartTokens))
            stack[++p] = 0
        else if (belongsTo(token, objectStartTokens))
            stack[++p] = NULLSTR
        else if (belongsTo(token, objectEndTokens) ||
                 belongsTo(token, arrayEndTokens))
            --p
        else if (belongsTo(token, commas)) {
            if (isnum(stack[p])) # array
                stack[p]++ # increase index
        } else if (belongsTo(token, colons)) {
            flag = 1 # ready to read value
        } else if (isnum(stack[p]) || flag) {
            # Read a value
            key = stack[0]
            for (j = 1; j <= p; j++)
                key = key SUBSEP stack[j]
            returnAST[key] = token
            flag = 0 # ready to read key
        } else {
            # Read a key
            stack[p] = unparameterize(token)
        }
    }
}

# S-expr parser.
function parseList(returnAST, tokens,
                   leftBrackets,
                   rightBrackets,
                   separators,
                   ####
                   i, j, key, p, stack, token) {
    # Default parameters
    if (!leftBrackets[0]) {
        leftBrackets[0] = "(" # left parenthesis
        leftBrackets[1] = "[" # left bracket
        leftBrackets[2] = "{" # left brace
    }
    if (!rightBrackets[0]) {
        rightBrackets[0] = ")" # right parenthesis
        rightBrackets[1] = "]" # right bracket
        rightBrackets[2] = "}" # right brace
    }
    if (!separators[0]) {
        separators[0] = "," # comma
    }

    stack[p = 0] = 0
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, leftBrackets)) {
            stack[++p] = 0
        } else if (belongsTo(token, rightBrackets)) {
            stack[--p]++
        } else if (belongsTo(token, separators)) {
            # skip
        } else {
            key = NULLSTR
            if (p > 0) {
                for (j = 0; j < p - 1; j++)
                    key = key SUBSEP stack[j]
                returnAST[key][stack[p - 1]] = NULLSTR
                key = key SUBSEP stack[p - 1]
            }
            returnAST[key][stack[p]] = token
            stack[p]++
        }
    }
}


devtools: fix directory filter in forbidden token check

checkpatches.sh current complains on a patch [1] adding
ALLOW_EXPERIMENTAL_API in an example while this check is for app, lib
and drivers directories:

Warning in examples/ethtool/ethtool-app/Makefile:
Using experimental build flag for in-tree compilation

The regexp on entering files concerned by this filter is incorrect.
In the [1] case, the file full name is matched against "app" rather than
"+++ b/app".

1: https://patchwork.dpdk.org/patch/83902/

Fixes: 7413e7f2aeb3 ("devtools: alert on new calls to exit from libs")
Cc: stable@dpdk.org

Signed-off-by: David Marchand <f28529695cfa9e3e1eb84e7072aa626af4abb18e@redhat.com>

# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2018 Arnon Warshavsky <arnon@qwilt.com>

# This awk script receives a list of expressions to monitor
# and a list of folders to search these expressions in
# - No search is done inside comments
# - Both additions and removals of the expressions are checked
#   A positive balance of additions fails the check

BEGIN {
	split(FOLDERS,deny_folders," ");
	split(EXPRESSIONS,deny_expr," ");
	in_file=0;
	in_comment=0;
	count=0;
	comment_start="/*"
	comment_end="*/"
}
# search for add/remove instances in current file
# state machine assumes the comments structure is enforced by
# checkpatches.pl
(in_file) {
	# comment start
	if (index($0,comment_start) > 0) {
		in_comment = 1
	}
	# non comment code
	if (in_comment == 0) {
		for (i in deny_expr) {
			forbidden_added = "^\\+.*" deny_expr[i];
			forbidden_removed="^-.*" deny_expr[i];
			current = expressions[deny_expr[i]]
			if ($0 ~ forbidden_added) {
				count = count + 1;
				expressions[deny_expr[i]] = current + 1
			}
			if ($0 ~ forbidden_removed) {
				count = count - 1;
				expressions[deny_expr[i]] = current - 1
			}
		}
	}
	# comment end
	if (index($0,comment_end) > 0) {
		in_comment = 0
	}
}
# switch to next file , check if the balance of add/remove
# of previous filehad new additions
($0 ~ "^\\+\\+\\+ b/") {
	in_file = 0;
	if (count > 0) {
		exit;
	}
	for (i in deny_folders) {
		re = "^\\+\\+\\+ b/" deny_folders[i];
		if ($0 ~ re) {
			in_file = 1
			last_file = $0
		}
	}
}
END {
	if (count > 0) {
		print "Warning in " substr(last_file,7) ":"
		print MESSAGE
		exit RET_ON_FAIL
	}
}

# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2018 Arnon Warshavsky <arnon@qwilt.com>

# This awk script receives a list of expressions to monitor
# and a list of folders to search these expressions in
# - No search is done inside comments
# - Both additions and removals of the expressions are checked
#   A positive balance of additions fails the check

BEGIN {
	split(FOLDERS,deny_folders," ");
	split(EXPRESSIONS,deny_expr," ");
	in_file=0;
	in_comment=0;
	count=0;
	comment_start="/*"
	comment_end="*/"
}
# search for add/remove instances in current file
# state machine assumes the comments structure is enforced by
# checkpatches.pl
(in_file) {
	# comment start
	if (index($0,comment_start) > 0) {
		in_comment = 1
	}
	# non comment code
	if (in_comment == 0) {
		for (i in deny_expr) {
			forbidden_added = "^\\+.*" deny_expr[i];
			forbidden_removed="^-.*" deny_expr[i];
			current = expressions[deny_expr[i]]
			if ($0 ~ forbidden_added) {
				count = count + 1;
				expressions[deny_expr[i]] = current + 1
			}
			if ($0 ~ forbidden_removed) {
				count = count - 1;
				expressions[deny_expr[i]] = current - 1
			}
		}
	}
	# comment end
	if (index($0,comment_end) > 0) {
		in_comment = 0
	}
}
# switch to next file , check if the balance of add/remove
# of previous filehad new additions
($0 ~ "^\\+\\+\\+ b/") {
	in_file = 0;
	if (count > 0) {
		exit;
	}
	for (i in deny_folders) {
		re = "^\\+\\+\\+ b/" deny_folders[i];
		if ($0 ~ deny_folders[i]) {
			in_file = 1
			last_file = $0
		}
	}
}
END {
	if (count > 0) {
		print "Warning in " substr(last_file,7) ":"
		print MESSAGE
		exit RET_ON_FAIL
	}
}


Remove unnecessary global data variables


Simplify toloka.awk

Simplify toloka.awk

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    print "INPUT:hyponym", "INPUT:hypernym", "INPUT:genitive", "GOLDEN:judgement", "HINT:text";
}
NR == 1 {
    for (i=1; i<=NF; i++) ix[$i] = i;
    next;
}
!$ix["found"] {
    next;
}
{
    hypernyms[$ix["hypernym"] OFS $ix["genitive"]];
}
$ix["hyponym"] != hyponym {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }

    hyponym = $ix["hyponym"];
    delete hypernyms;
}
END {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    print "INPUT:hyponym", "INPUT:hypernym", "INPUT:genitive", "GOLDEN:judgement", "HINT:text";
}
NR == 1 {
    for (i=1; i<=NF; i++) ix[$i] = i;
}
NR > 1 && $ix["found"] {
    hypernyms[$ix["hypernym"] OFS $ix["genitive"]];
}
NR > 1 && $ix["found"] && $ix["hyponym"] != hyponym {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }

    hyponym = $ix["hyponym"];
    delete hypernyms;
}
END {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }
}


Removed unused code

Removed unused code

BEGIN {
	types[1] = ""
	fields[1] = ""
	line_index = 1
	decl_index = 1

	# Java to Go Map
	j_to_g_map["jboolean"] = "bool"
	j_to_g_map["jbyte"] = "int8"
	j_to_g_map["jchar"] = "uint16"
	j_to_g_map["jshort"] = "int16"
	j_to_g_map["jint"] = "int32"
	j_to_g_map["jlong"] = "int64"
	j_to_g_map["jfloat"] = "float32"
	j_to_g_map["jdouble"] = "float64"
	j_to_g_map["jobject"] = "JObject"
}

function capitalize(s, count) {
	return toupper(substr(s, 1, count)) substr(s, count + 1)
}

function get_type(line) {
	match($0, /^[a-z]+/)
	return substr($0, RSTART, RLENGTH)
}

function get_field(line) {
	match($0, /[a-z];$/)
	return substr($0, RSTART, RLENGTH - 1)
}

function wrap(type, expr) {
	if (type != "jobject") {
		return "Go" capitalize(j_to_g_map[type], 1) "(" expr ")"
	}
	return "NewJObject(" expr ")"
}

function unwrap(type, expr) {
	if (type != "jobject") {
		return "Java" capitalize(substr(type, 2), 1) "(" expr ")"
	}
	return expr ".Peer()"
}

END {
	# Beginning Stub
	print "package main"
	print ""
	print "// #include \"jvalue.h\""
	print "import \"C\""

	for (i = 1; i <= length(types); i++) {
		print ""
		print "func " capitalize(types[i], 2) "ToJValue(" fields[i] " C." types[i] ") C.jvalue {"
		print "\treturn C._GoJni" capitalize(types[i], 2) "ToJValue(" fields[i] ")"
		print "}"
	}
	for (i = 1; i <= length(types); i++) {
		print ""
		print "func JValueTo" capitalize(types[i], 2) "(v C.jvalue) C." types[i] " {"
		print "\treturn C._GoJniJValueTo" capitalize(types[i], 2) "(v)"
		print "}"
	}
	for (i = 1; i <= length(types); i++) {
		print ""
		print "func JValueFrom" capitalize(j_to_g_map[types[i]], 1) "(" fields[i] " " j_to_g_map[types[i]] ") JValue {"
		print "\treturn JValue{" capitalize(types[i], 2) "ToJValue(" unwrap(types[i], fields[i]) ")}"
		print "}"
	}
	for (i = 1; i <= length(types); i++) {
		print ""
		print "func (value JValue) " capitalize(j_to_g_map[types[i]], 1) "() " j_to_g_map[types[i]] " {"
		print "\treturn " wrap(types[i], "JValueTo" capitalize(types[i], 2) "(value.peer)")
		print "}"
	}
}

/^typedef union jvalue {$/, /^} jvalue;$/ {
	sub(/^[\t ]+/, "", $0)
	if ($0 !~ /^j/) {
		next
	}

	# Type and Field
	types[decl_index] = get_type($0)
	fields[decl_index] = get_field($0)
	++decl_index
}

BEGIN {
	types[1] = ""
	fields[1] = ""
	line_index = 1
	decl_index = 1

	# Java to Go Map
	j_to_g_map["jboolean"] = "bool"
	j_to_g_map["jbyte"] = "int8"
	j_to_g_map["jchar"] = "uint16"
	j_to_g_map["jshort"] = "int16"
	j_to_g_map["jint"] = "int32"
	j_to_g_map["jlong"] = "int64"
	j_to_g_map["jfloat"] = "float32"
	j_to_g_map["jdouble"] = "float64"
	j_to_g_map["jobject"] = "JObject"

	# Go to Java Map
	for (k in j_to_g_map) {
		g_to_k_map[j_to_g_map[k]] = k
	}
}

function capitalize(s, count) {
	return toupper(substr(s, 1, count)) substr(s, count + 1)
}

function get_type(line) {
	match($0, /^[a-z]+/)
	return substr($0, RSTART, RLENGTH)
}

function get_field(line) {
	match($0, /[a-z];$/)
	return substr($0, RSTART, RLENGTH - 1)
}

function wrap(type, expr) {
	if (type != "jobject") {
		return "Go" capitalize(j_to_g_map[type], 1) "(" expr ")"
	}
	return "NewJObject(" expr ")"
}

function unwrap(type, expr) {
	if (type != "jobject") {
		return "Java" capitalize(substr(type, 2), 1) "(" expr ")"
	}
	return expr ".Peer()"
}

END {
	# Beginning Stub
	print "package main"
	print ""
	print "// #include \"jvalue.h\""
	print "import \"C\""

	for (i = 1; i <= length(types); i++) {
		print ""
		print "func " capitalize(types[i], 2) "ToJValue(" fields[i] " C." types[i] ") C.jvalue {"
		print "\treturn C._GoJni" capitalize(types[i], 2) "ToJValue(" fields[i] ")"
		print "}"
	}
	for (i = 1; i <= length(types); i++) {
		print ""
		print "func JValueTo" capitalize(types[i], 2) "(v C.jvalue) C." types[i] " {"
		print "\treturn C._GoJniJValueTo" capitalize(types[i], 2) "(v)"
		print "}"
	}
	for (i = 1; i <= length(types); i++) {
		print ""
		print "func JValueFrom" capitalize(j_to_g_map[types[i]], 1) "(" fields[i] " " j_to_g_map[types[i]] ") JValue {"
		print "\treturn JValue{" capitalize(types[i], 2) "ToJValue(" unwrap(types[i], fields[i]) ")}"
		print "}"
	}
	for (i = 1; i <= length(types); i++) {
		print ""
		print "func (value JValue) " capitalize(j_to_g_map[types[i]], 1) "() " j_to_g_map[types[i]] " {"
		print "\treturn " wrap(types[i], "JValueTo" capitalize(types[i], 2) "(value.peer)")
		print "}"
	}
}

/^typedef union jvalue {$/, /^} jvalue;$/ {
	sub(/^[\t ]+/, "", $0)
	if ($0 !~ /^j/) {
		next
	}

	# Type and Field
	types[decl_index] = get_type($0)
	fields[decl_index] = get_field($0)
	++decl_index
}


Add a layout for the french Apple black usb keyboard (fr.apple). With help from Jean-Marc Harang, thanks!

#! /usr/bin/awk -f
#	$OpenBSD: makemap.awk,v 1.4 2005/05/19 10:40:48 miod Exp $
#
# Copyright (c) 2005, Miodrag Vallat
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# This script attempts to convert, with minimal hacks and losses, the
# regular PS/2 keyboard (pckbd) layout tables into USB keyboard (ukbd)
# layout tables.
#

BEGIN {
	rcsid = "$OpenBSD: makemap.awk,v 1.4 2005/05/19 10:40:48 miod Exp $"
	ifdepth = 0
	ignore = 0
	declk = 0
	haskeys = 0
	kbfr = 0

	# PS/2 id -> UKBD conversion table, or "sanity lossage 102"
	# (101 is for GSC keyboards!)
	for (i = 0; i < 256; i++)
		conv[i] = -1

	conv[1] = 41
	conv[2] = 30
	conv[3] = 31
	conv[4] = 32
	conv[5] = 33
	conv[6] = 34
	conv[7] = 35
	conv[8] = 36
	conv[9] = 37
	conv[10] = 38
	conv[11] = 39
	conv[12] = 45
	conv[13] = 46
	conv[14] = 42
	conv[15] = 43
	conv[16] = 20
	conv[17] = 26
	conv[18] = 8
	conv[19] = 21
	conv[20] = 23
	conv[21] = 28
	conv[22] = 24
	conv[23] = 12
	conv[24] = 18
	conv[25] = 19
	conv[26] = 47
	conv[27] = 48
	conv[28] = 40
	conv[29] = 224
	conv[30] = 4
	conv[31] = 22
	conv[32] = 7
	conv[33] = 9
	conv[34] = 10
	conv[35] = 11
	conv[36] = 13
	conv[37] = 14
	conv[38] = 15
	conv[39] = 51
	conv[40] = 52
	conv[41] = 53
	conv[42] = 225
	conv[43] = 50
	conv[44] = 29
	conv[45] = 27
	conv[46] = 6
	conv[47] = 25
	conv[48] = 5
	conv[49] = 17
	conv[50] = 16
	conv[51] = 54
	conv[52] = 55
	conv[53] = 56
	conv[54] = 229
	conv[55] = 85
	conv[56] = 226
	conv[57] = 44
	conv[58] = 57
	conv[59] = 58
	conv[60] = 59
	conv[61] = 60
	conv[62] = 61
	conv[63] = 62
	conv[64] = 63
	conv[65] = 64
	conv[66] = 65
	conv[67] = 66
	conv[68] = 67
	conv[69] = 83
	conv[70] = 71
	conv[71] = 95
	conv[72] = 96
	conv[73] = 97
	conv[74] = 86
	conv[75] = 92
	conv[76] = 93
	conv[77] = 94
	conv[78] = 87
	conv[79] = 89
	conv[80] = 90M
	conv[81] = 91
	conv[82] = 98
	conv[83] = 99
	conv[86] = 100
	conv[87] = 68
	conv[88] = 69
	conv[112] = 135
	conv[115] = 136
	conv[121] = 137
	conv[123] = 138
	conv[125] = 139
	conv[127] = 72
	conv[156] = 88
	conv[157] = 228
	conv[170] = 70
	conv[181] = 84
	conv[184] = 230
	# 198 is #if 0 in the PS/2 map...
	conv[199] = 74
	conv[200] = 82
	conv[201] = 75
	conv[203] = 80
	conv[205] = 79
	conv[207] = 77
	conv[208] = 81
	conv[209] = 78
	conv[210] = 73
	conv[211] = 99
	conv[219] = 227
	conv[220] = 231
	conv[221] = 101
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)
	gsub("\\$", "", rcsid)

	printf("/*\t\$OpenBSD\$\t*/\n\n")
	printf("/*\n")
	printf(" * THIS FILE IS AUTOMAGICALLY GENERATED.  DO NOT EDIT.\n")
	printf(" *\n")
	printf(" * generated by:\n")
	printf(" *\t%s\n", rcsid)
	printf(" * generated from:\n")
	printf(" */\n")
	print VERSION

	next
}

#
# A very limited #if ... #endif parser. We only want to correctly detect
# ``#if 0'' constructs, so as not to process their contents. This is necessary
# since our output is out-of-order from our input.
#
# Note that this does NOT handle ``#ifdef notyet'' correctly - please only use
# ``#if 0'' constructs in the input.
#

/^#if/ {
	ignores[ifdepth] = ignore
	if ($2 == "0")
		ignore = 1
	else
		ignore = 0
	ifdepth++
	if (ignore)
		next
}
/^#endif/ {
	oldignore = ignore
	ifdepth--
	ignore = ignores[ifdepth]
	ignores[ifdepth] = 0
	if (oldignore)
		next
}

$1 == "#include" {
	if (ignore)
		next
	if ($2 == "<dev/pckbc/wskbdmap_mfii.h>")
		print "#include <dev/usb/usb_port.h>"
	else
		printf("#include %s\n", $2)

	next
}
$1 == "#define" || $1 == "#undef" {
	if (ignore)
		next
	print $0
	next
}

# Don't bother converting the DEC LK layout.
/declk\[/ {
	declk = 1
	next
}
/declk/ {
	next
}

/pckbd/ {
	gsub("pckbd", "ukbd", $0)
	mapname = $4
}

/KC/ {
	if (ignore)
		next

	if (declk)
		next

	haskeys = 1

	sidx = substr($1, 4, length($1) - 5)
	orig = int(sidx)
	id = conv[orig]

	# 183 is another Print Screen...
	if (orig == 183)
		next

	if (id == -1) {
		printf("/* initially KC(%d),", orig)
		for (f = 2; f <= NF; f++) {
			if ($f != "/*" && $f != "*/")
				printf("\t%s", $f)
		}
		printf("\t*/\n")
	} else {
		lines[id] = sprintf("    KC(%d),\t", id)
		#
		# This makes sure that the non-comment part of the output
		# ends up with a trailing comma. This is necessary since
		# the last line of an input block might not have a trailing
		# comma, but might not be the last line of an output block
		# due to sorting.
		#
		comma = 0
		for (f = 2; f <= NF; f++) {
			l = length($f)
			if ($f == "/*")
				comma++
			if (comma == 0 && substr($f, l) != ",") {
				lines[id] = sprintf("%s%s,", lines[id], $f)
				l++
			} else {
				lines[id] = sprintf("%s%s", lines[id], $f)
			}
			if (comma == 0 && f != NF) {
				if (l < 2 * 8)
					lines[id] = lines[id] "\t"
				if (l < 8)
					lines[id] = lines[id] "\t"
			}
			if ($f == "*/")
				comma--
		}
	}

	next
}
/};/ {
	if (ignore)
		next

	if (declk) {
		declk = 0
		next
	}

	if (haskeys) {
		# Duplicate 42 (backspace) as 76
		# XXX maybe not correct anymore?
		lines[76] = lines[42]
		sub("42", "76", lines[76])

		for (i = 0; i < 256; i++)
			if (lines[i]) {
				print lines[i]
				lines[i] = ""
			}

		haskeys = 0

		#
		# Apple black USB keyboards use a slightly different
		# layout. We define them here.
		#
		if (mapname == "ukbd_keydesc_fr[]") {
			print $0
			print "\nstatic const keysym_t ukbd_keydesc_fr_apple[] = {"
			print "    KC(35),\tKS_section,\tKS_6,"
			print "    KC(37),\tKS_exclam,\tKS_8,"
			print "    KC(46),\tKS_minus,\tKS_underscore,"
			print "    KC(48),\tKS_dollar,\tKS_asterisk,"
			print "    KC(50),\tKS_backslash,\tKS_sterling,"
			print "    KC(53),\tKS_at,\tKS_numbersign,"
			print "    KC(56),\tKS_equal,\tKS_plus,"
			print "    KC(103),\tKS_KP_Equal,"
		} else
		if (mapname == "ukbd_keydesc_pt[]") {
			print $0
			print "\nstatic const keysym_t ukbd_keydesc_pt_apple[] = {"
			print "/*  pos\t\tnormal\t\tshifted */"
			print "    KC(46),\tKS_plus,\tKS_asterisk,"
			print "    KC(47),\tKS_masculine,\tKS_ordfeminine,"
			print "    KC(50),\tKS_backslash,\tKS_bar,"
			print "    KC(52),\tKS_dead_tilde,\tKS_dead_circumflex"
		}
	}
}
/KB_FR/ {
	print $0
	if (kbfr++ == 0) {
		print "\tKBD_MAP(KB_FR | KB_APPLE,\tKB_FR,\tukbd_keydesc_fr_apple),"
	} else {
		print "\tKBD_MAP(KB_FR | KB_APPLE | KB_SWAPCTRLCAPS,\tKB_FR | KB_APPLE,"
		print "\t\tukbd_keydesc_swapctrlcaps),"
	}
	next
}
/KB_PT/ {
	print $0
	print "\tKBD_MAP(KB_PT | KB_APPLE,\tKB_PT,\tukbd_keydesc_pt_apple),"
	next
}
{
	if (ignore)
		next
	if (declk)
		next
	print $0
}

#! /usr/bin/awk -f
#	$OpenBSD: makemap.awk,v 1.3 2005/05/12 16:21:24 miod Exp $
#
# Copyright (c) 2005, Miodrag Vallat
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#
# This script attempts to convert, with minimal hacks and losses, the
# regular PS/2 keyboard (pckbd) layout tables into USB keyboard (ukbd)
# layout tables.
#

BEGIN {
	rcsid = "$OpenBSD: makemap.awk,v 1.3 2005/05/12 16:21:24 miod Exp $"
	ifdepth = 0
	ignore = 0
	declk = 0
	haskeys = 0

	# PS/2 id -> UKBD conversion table, or "sanity lossage 102"
	# (101 is for GSC keyboards!)
	for (i = 0; i < 256; i++)
		conv[i] = -1

	conv[1] = 41
	conv[2] = 30
	conv[3] = 31
	conv[4] = 32
	conv[5] = 33
	conv[6] = 34
	conv[7] = 35
	conv[8] = 36
	conv[9] = 37
	conv[10] = 38
	conv[11] = 39
	conv[12] = 45
	conv[13] = 46
	conv[14] = 42
	conv[15] = 43
	conv[16] = 20
	conv[17] = 26
	conv[18] = 8
	conv[19] = 21
	conv[20] = 23
	conv[21] = 28
	conv[22] = 24
	conv[23] = 12
	conv[24] = 18
	conv[25] = 19
	conv[26] = 47
	conv[27] = 48
	conv[28] = 40
	conv[29] = 224
	conv[30] = 4
	conv[31] = 22
	conv[32] = 7
	conv[33] = 9
	conv[34] = 10
	conv[35] = 11
	conv[36] = 13
	conv[37] = 14
	conv[38] = 15
	conv[39] = 51
	conv[40] = 52
	conv[41] = 53
	conv[42] = 225
	conv[43] = 50
	conv[44] = 29
	conv[45] = 27
	conv[46] = 6
	conv[47] = 25
	conv[48] = 5
	conv[49] = 17
	conv[50] = 16
	conv[51] = 54
	conv[52] = 55
	conv[53] = 56
	conv[54] = 229
	conv[55] = 85
	conv[56] = 226
	conv[57] = 44
	conv[58] = 57
	conv[59] = 58
	conv[60] = 59
	conv[61] = 60
	conv[62] = 61
	conv[63] = 62
	conv[64] = 63
	conv[65] = 64
	conv[66] = 65
	conv[67] = 66
	conv[68] = 67
	conv[69] = 83
	conv[70] = 71
	conv[71] = 95
	conv[72] = 96
	conv[73] = 97
	conv[74] = 86
	conv[75] = 92
	conv[76] = 93
	conv[77] = 94
	conv[78] = 87
	conv[79] = 89
	conv[80] = 90M
	conv[81] = 91
	conv[82] = 98
	conv[83] = 99
	conv[86] = 100
	conv[87] = 68
	conv[88] = 69
	conv[112] = 135
	conv[115] = 136
	conv[121] = 137
	conv[123] = 138
	conv[125] = 139
	conv[127] = 72
	conv[156] = 88
	conv[157] = 228
	conv[170] = 70
	conv[181] = 84
	conv[184] = 230
	# 198 is #if 0 in the PS/2 map...
	conv[199] = 74
	conv[200] = 82
	conv[201] = 75
	conv[203] = 80
	conv[205] = 79
	conv[207] = 77
	conv[208] = 81
	conv[209] = 78
	conv[210] = 73
	conv[211] = 99
	conv[219] = 227
	conv[220] = 231
	conv[221] = 101
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)
	gsub("\\$", "", rcsid)

	printf("/*\t\$OpenBSD\$\t*/\n\n")
	printf("/*\n")
	printf(" * THIS FILE IS AUTOMAGICALLY GENERATED.  DO NOT EDIT.\n")
	printf(" *\n")
	printf(" * generated by:\n")
	printf(" *\t%s\n", rcsid)
	printf(" * generated from:\n")
	printf(" */\n")
	print VERSION

	next
}

#
# A very limited #if ... #endif parser. We only want to correctly detect
# ``#if 0'' constructs, so as not to process their contents. This is necessary
# since our output is out-of-order from our input.
#
# Note that this does NOT handle ``#ifdef notyet'' correctly - please only use
# ``#if 0'' constructs in the input.
#

/^#if/ {
	ignores[ifdepth] = ignore
	if ($2 == "0")
		ignore = 1
	else
		ignore = 0
	ifdepth++
	if (ignore)
		next
}
/^#endif/ {
	oldignore = ignore
	ifdepth--
	ignore = ignores[ifdepth]
	ignores[ifdepth] = 0
	if (oldignore)
		next
}

$1 == "#include" {
	if (ignore)
		next
	if ($2 == "<dev/pckbc/wskbdmap_mfii.h>")
		print "#include <dev/usb/usb_port.h>"
	else
		printf("#include %s\n", $2)

	next
}
$1 == "#define" || $1 == "#undef" {
	if (ignore)
		next
	print $0
	next
}

# Don't bother converting the DEC LK layout.
/declk\[/ {
	declk = 1
	next
}
/declk/ {
	next
}

/pckbd/ {
	gsub("pckbd", "ukbd", $0)
	mapname = $4
}

/KC/ {
	if (ignore)
		next

	if (declk)
		next

	haskeys = 1

	sidx = substr($1, 4, length($1) - 5)
	orig = int(sidx)
	id = conv[orig]

	# 183 is another Print Screen...
	if (orig == 183)
		next

	if (id == -1) {
		printf("/* initially KC(%d),", orig)
		for (f = 2; f <= NF; f++) {
			if ($f != "/*" && $f != "*/")
				printf("\t%s", $f)
		}
		printf("\t*/\n")
	} else {
		lines[id] = sprintf("    KC(%d),\t", id)
		#
		# This makes sure that the non-comment part of the output
		# ends up with a trailing comma. This is necessary since
		# the last line of an input block might not have a trailing
		# comma, but might not be the last line of an output block
		# due to sorting.
		#
		comma = 0
		for (f = 2; f <= NF; f++) {
			l = length($f)
			if ($f == "/*")
				comma++
			if (comma == 0 && substr($f, l) != ",") {
				lines[id] = sprintf("%s%s,", lines[id], $f)
				l++
			} else {
				lines[id] = sprintf("%s%s", lines[id], $f)
			}
			if (comma == 0 && f != NF) {
				if (l < 2 * 8)
					lines[id] = lines[id] "\t"
				if (l < 8)
					lines[id] = lines[id] "\t"
			}
			if ($f == "*/")
				comma--
		}
	}

	next
}
/};/ {
	if (ignore)
		next

	if (declk) {
		declk = 0
		next
	}

	if (haskeys) {
		# Duplicate 42 (backspace) as 76
		# XXX maybe not correct anymore?
		lines[76] = lines[42]
		sub("42", "76", lines[76])

		for (i = 0; i < 256; i++)
			if (lines[i]) {
				print lines[i]
				lines[i] = ""
			}

		haskeys = 0

		#
		# Apple portuguese USB keyboards use a slightly different
		# layout. We define it here.
		#
		if (mapname == "ukbd_keydesc_pt[]") {
			print $0
			print "\nstatic const keysym_t ukbd_keydesc_pt_apple[] = {"
			print "/*  pos\t\tnormal\t\tshifted */"
			print "    KC(46),\tKS_plus,\tKS_asterisk,"
			print "    KC(47),\tKS_masculine,\tKS_ordfeminine,"
			print "    KC(50),\tKS_backslash,\tKS_bar,"
			print "    KC(52),\tKS_dead_tilde,\tKS_dead_circumflex"
		}

	}
}
/KB_PT/ {
	print $0
	print "\tKBD_MAP(KB_PT | KB_APPLE,\tKB_PT,\tukbd_keydesc_pt_apple),"
	next
}
{
	if (ignore)
		next
	if (declk)
		next
	print $0
}


seems to function...

seems to function...

#! /usr/bin/gawk -f
# distill_dot.awk <numbered.gv>  > <filtered_weighted.gv>
#

# I have dot files with edge lables that terminate with a number in parens `(nnn)`.
# the numbers may be simple counts (n > 0) or may be differences [nil neg zero pos ]
#
# I'm initally intrested in the case where the numbers are a difference
# and will filter out small differences and colorize/weight larger changes.
#
# To normalize, will need to read through first then output

BEGIN {hi = -2^PREC; lo = 2^PREC; THRESHOLD = 10}

# there is an unfortunate case where the label contains parens e.g.
# ZFIN -> LITERAL [label=<dc:title ! title (dce) (30783)>];

# AspGD -> GO [label=<RO:0002331 ! involved in (-12)>, color="pink"];

/.*label=<.*\(-?[0-9]+\).*;$/{

    n = split($0, a, /[()]/)
    # for(i in  a) print a[i]
    val = a[length(a)-1]
    sfx = a[length(a)]
    # print "Val=" val
    if(val > hi) hi = val
    if(val < lo) lo = val
    aval = (val < 0.0) ? -val : val
    if(aval > THRESHOLD){
        a[length(a)] = ""
        a[length(a)-1] = ""
        key=""
        for(x in a) key = key a[x]
        value[key] = val
        sufix[key] = substr(sfx,2)
    }
}

# not processed
!/.*label=<.*\(-?[0-9]+\).*;$/ {
    pass_on[pss++] = $0
}

END {
    SCALE =  (hi > -lo)? hi: -lo
    # Threshold could extended as a fraction of scale ...
    THRESHOLD =  SCALE / 200

#    if (SCALE < 1){
#       print "ERROR not to scale: " SCALE
#        SCALE = 1
#    }

    for(i = 0; i < pss; i++)  # not the last line
        if( (pass_on[i]!="") && (pass_on[i]!="}")) print pass_on[i]

    for(key in value)   {
        val = value[key]
        aval = (val < 0.0) ? -val : val
        if(aval >= THRESHOLD)
            print key ">, penwidth = \""(aval/SCALE*5)"\", weight = \"" aval "\"" sufix[key]
    }

    print "}"
}

#! /usr/bin/awk -f
# distill_dot.awk <numbered.gv>  > <filtered_weighted.gv>
#

# I have dot files with edge lables that terminate with a number in parens `(nnn)`.
# the numbers may be simple counts (n > 0) or may be differences [nil neg zero pos ]
#
# I'm initally intrested in the case where the numbers are a difference
# and will filter out small differences and colorize/weight larger changes.

BEGIN: {hi=-1000000; lo=1000000}

# there is an unfortunate case where the label cantains parens e.g.
# ZFIN -> LITERAL [label=<dc:title ! title (dce) (30783)>];
# will need to

/^[^ ]+ -> [^ ]+ \[label=<.*([^)]+>.*];$/ {

    #
}


Output whitespace cleanup.

Output whitespace cleanup.

function printUrlQueryString(queryString, j)
{
    split(queryString, pairs, "&")
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=")
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\t\t\"%s\": \"%s\"", k, urlDecode(urlDecode(v))
        if (j + 1 < len) {
            printf ","
        }
        printf "\n"
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url, i) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    logLineSeen = 0
    print "{";
}

END {
    printf "\n\t]\n"
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ")
    numFields = length(fields)
    printf "\t\"records\": [\n"
}

/^#/ {
    # ignore anything else starting with "#"
}

! /^#/ {
    if (logLineSeen == 1) {
        printf ",\n"
    }
    split($0, fieldsThisLine, " ")
    urlIndex = 0
    if (NF == numFields) {
        print "\t\t{"
        for(i = 1; i < NF; i++)
        {
            fieldName = fields[i]
            if (fieldName == "cs-uri-query") {
                urlIndex = i;
                continue;
            }
            fieldValue = fieldsThisLine[i]
            maybeDecoded = (fieldName == "cs-uri-query" ? urlDecode(fieldValue) : fieldValue)
            printf "\t\t\t\"%s\": \"%s\"", fieldName, fieldValue
            if (i + 1 < NF && urlIndex > 0) {
                printf ","
            }
            printf "\n"
        }
        if (urlIndex > 0) {
            printf "\t\t\t\"cs-url-query\": {\n"
            printUrlQueryString(fieldsThisLine[urlIndex])
            printf "\t\t\t}\n"
        }
        printf "\t\t}"
    }
    logLineSeen = 1;
}

function printUrlQueryString(queryString, j)
{
    split(queryString, pairs, "&")
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=")
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url, i) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    logLineSeen = 0
    print "{";
}

END {
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ")
    numFields = length(fields)
}

/^#/ {
    # ignore anything else starting with "#"
}

! /^#/ {
    if (logLineSeen == 1) {
        printf ",\n"
    }
    split($0, fieldsThisLine, " ")
    urlIndex = 0
    if (NF == numFields) {
        print "\t{"
        for(i = 1; i < NF; i++)
        {
            fieldName = fields[i]
            if (fieldName == "cs-uri-query") {
                urlIndex = i;
                continue;
            }
            fieldValue = fieldsThisLine[i]
            maybeDecoded = (fieldName == "cs-uri-query" ? urlDecode(fieldValue) : fieldValue)
            printf "\t\t\"%s\": \"%s\"", fieldName, fieldValue
            if (i + 1 < NF && urlIndex > 0) {
                printf ","
            }
            printf "\n"
        }
        if (urlIndex > 0) {
            printf "\t\t\"cs-url-query\": {\n"
            printUrlQueryString(fieldsThisLine[urlIndex])
            printf "}\n"
        }
        printf "\t}"
    }
    logLineSeen = 1;
}


ahrf.awk: Allow _ for paragraphs

ahrf.awk: Allow _ for paragraphs

#!/home/ypnose/Dev/AWK/nawk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0, ":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9_]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====/ {
	if (NF > 2 && $NF == "====") {
		gsub(/^[\t ]*====[\t ]*\n|\n[\t ]*====[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\*/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l != "") {
			syb = substr($l,2,1)
			if (syb == " ") {
				str = substr($l,3)
			} else {
				str = substr($l,2)
			}
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u, "\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}

#!/home/ypnose/Dev/AWK/nawk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0, ":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====/ {
	if (NF > 2 && $NF == "====") {
		gsub(/^[\t ]*====[\t ]*\n|\n[\t ]*====[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\*/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l != "") {
			syb = substr($l,2,1)
			if (syb == " ") {
				str = substr($l,3)
			} else {
				str = substr($l,2)
			}
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u, "\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}


ahrf.awk: Comment a dirty I wrote (don't remember why...)

#!/usr/bin/awk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0,":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9_]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====+/ {
	if (NF > 2 && $NF ~ /====+/) {
		gsub(/^[\t ]*====+[\t ]*\n|\n[\t ]*====+[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		#gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\* +/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l ~ /\* +/) {
			match($l,"\\* +")
			str = substr($l,RSTART+RLENGTH)
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u,"\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}

#!/usr/bin/awk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0,":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9_]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====+/ {
	if (NF > 2 && $NF ~ /====+/) {
		gsub(/^[\t ]*====+[\t ]*\n|\n[\t ]*====+[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\* +/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l ~ /\* +/) {
			match($l,"\\* +")
			str = substr($l,RSTART+RLENGTH)
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u,"\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}


optimization ;-)

optimization ;-)


Move out edge and node guts into macros.

#!/usr/bin/awk

# TODO:
# cat nfa.dot | dot -Tplain > /tmp/a.mp \
# && mpost -s 'outputformat="svg"' -s 'outputtemplate="%j-%c.%o"' /tmp/a.mp

# Only intended to consume Dot generated from libfsm's .dot output,
# and not graphviz diagrams in general.

function sname(s)
{
	if (s == "start") {
		return "S0";
	}

	return s;
}

# TODO: convert label from graphviz's html to a utf8
# This is either <xyz> or abc (with no quotes)
function slabel(s)
{
	# XXX: placeholder
	if (s == "<>") {
		return "";
	}
	if (s == "\"\"") {
		return "";
	}

# XXX: placeholder
gsub(/ε/, "e", s); # XXX: why not?
gsub(/"/, "", s)
gsub(/</, "", s)
gsub(/>/, "", s)

	return s;
}

BEGIN {

	# TODO: would be simpler to gvpr to rename start to S0
	# TODO: common stuff for multiple graphs
	print "pair S[];"
	print "pair q[];"
	print "path e;"

	print "ahlength := 10bp;"
	print "ahangle  := 45;"

	print "path S[].n;"

	# diameter of node
	print "numeric S[].diam;"

	# number of self-edges
	print "numeric S[].loops;"

#	print "secondarydef v projectedalong w =";
#	print "	if pair(v) and pair(w):";
#	print "		(v dotprod w) / (w dotprod w) * w";
#	print "	else:";
#	print "		errmessage \"arguments must be pairs\"";
#	print "	fi";
#	print "enddef;";

	# TODO: centralise to my equivalent of automata.mp, and include that

	print "vardef fsmnode(expr node, diam, final, lab) =";
	print "	draw fullcircle scaled diam shifted node withpen pencircle scaled 1bp;"
	print "	if final:"
	print "		draw fullcircle scaled (diam - 8bp) shifted node withpen pencircle scaled 1bp;"
	print "	fi;"
	print "	if lab <> \"\":"
	print "		label(lab, node);"
	print "	fi;"
	print "enddef;";

	print "vardef fsmloop(expr node, diam, p, q, lab, loops) ="
	print "	path pp; pp = node -- 0.5[p, q];"
	print "	draw p -- q withpen pencircle scaled 0.5bp withcolor green;"
	print "	draw 0.5[p, q] withpen pencircle scaled 0.8bp withcolor green;"
	print "	draw node -- 0.5[p, q] withpen pencircle scaled 0.8bp withcolor green;"

	# extend loop 1.25 * diameter
	# count number of self-loops for this node, and increment extension
	print "	l := length pp;"
	print "	pair m; m := (0.5 + 2 * mlog(loops + 1) / 256) * diam * unitvector(direction l of pp) shifted node;"
	# u3 = u1 projectedalong u2;
	print "	draw node -- m withpen pencircle scaled 0.5bp withcolor green;"

	print "	if lab <> \"\":"
	# TODO: top or bottom, depending on green line direction
	print "		label.top(lab, m);"
	print "	fi;"

	print "	path b; b := node .. q .. m .. p .. node;"
	print "	drawarrow b cutbefore q cutafter p withpen pencircle scaled 1bp;";

	print "enddef;";

	print "vardef fsmgvedge(expr tail, taildiam, head, headdiam, e, lab, lxy, loops) =";
	print "	draw e withpen pencircle scaled 0.25bp withcolor red;";

	# TODO: better explain this
	# TODO: explain reversed for q, else we'd find p again, where head == tail
	print "	path sn; sn := fullcircle scaled headdiam shifted head;" # outline
	print "	path st; st := fullcircle scaled taildiam shifted tail;" # outline
	print "	pair p; p = e intersectionpoint sn;\n";
	print "	pair q; q = (reverse e) intersectionpoint st;\n";

	print "	if head = tail:"
	print "		fsmloop(head, headdiam, p, q, lab, loops);"
	print "	else:"

	print "		path b; b = tail .. q .. p .. head;"

	# TODO: explain this. we permit a 2% lengthening threshold relative to graphviz's b-spline
	# this affects NFA especially
	print "		r := arclength b / arclength e;";
	print "		boolean g; g := (r > 1.02) or (r < 0.93);"; # XXX: or (arclength b > 6in);"; - no, wigglyness instead

	print "		if g:"
	print "			b := e;"
	print "		fi;"

	print "		if lab <> \"\":"
	print "			pair bl;";
	print "			pair ml; ml := .5[tail, head];"

	# TODO: new idea; draw right angle from midpoint, and use that to find intersection with b
	# this way it should always align for multiple edges, even if on a slant

	print "			draw lxy withpen pencircle scaled 5bp withcolor red;"
	print "			draw point (length b / 2) of b withpen pencircle scaled 4bp withcolor blue;"
#	print "			draw tail -- head withpen pencircle scaled 1bp withcolor green;"
#	print "			draw ml withpen pencircle scaled 5bp withcolor green;"

	print "			if g:"
	print "				bl := lxy;"
	print "			else:"
	print "				bl := point (length b / 2) of b;"
	print "			fi;"

	# TODO: plus label delta, distancing it from point. make a label function
	# TODO: maybe an "extend" function, to extend a path in its direction. use dotprod for that?

	# lft | rt | top | bot | ulft | urt | llft | lrt
	# labels ought to be below, if they're below the straight line from head-tail, or above otherwise
	# TODO: round label coordinates to grid? quantize rather

	# XXX: if using graphviz's point, label exactly on the spot
	print "			if g:";
	# TODO: still want to distance by +/- 4bp or so
	print "				label(lab, bl);"
	print "			else:"
	print "				if ypart bl < ypart ml:" # TODO: threshold for considering "below"
	print "					label.bot(lab, bl shifted (0, -3bp));"
	print "				else:"
	print "					label.top(lab, bl shifted (0, +3bp));"
	print "				fi;"
	print "			fi;"
	print "		fi;"

	print "	drawarrow b cutbefore q cutafter p withpen pencircle scaled 1bp;";

	print "	fi;"

	print "enddef;";
}

/^graph / {
	printf "beginfig(%s)\n", $2
}

# node S6 0.97222 0.41667 0.30556 0.30556 "" solid circle black lightgrey
# node name x y width height label style shape color fillcolor
/^node / {
	name   = sname($2);
	x      = $3;
	y      = $4;
	width  = $5;
	height = $6;
	label  = slabel($7);
	style  = $8;
	shape  = $9;
	# TODO: label delimited by <>, may contain spaces

	# TODO: quantise node positions
	printf "\t%s := (%fin, %fin);\n", name, x, y
	printf "\t%s.loops := 0;\n", name

# TODO: when plotting in graphviz, assume all nodes have the same width (not zero!). enough for "SXX" perhaps
# that should align them all on the same centre line. and pass fixedsize=true

	# TODO: explain this is diameter
	# TODO: store 8bp as a constant somewhere
# TODO: explain we take diameter from label presence and endstateness.
# ignore graphviz height and width, because it scales with label width and that looks bad for S1 vs S5
	diam = 0.3 + length(label) * 0.05;

	final = shape == "doublecircle";

	if (final) {
		printf "\t%s.diam = %fin + 8bp;\n", name, diam;
	} else {
		printf "\t%s.diam = %fin;\n", name, diam;
	}

	if ($2 != "start") {
		printf "\tfsmnode(%s, %s.diam, %s, \"%s\");\n", name, name, final ? "true" : "false", label
	}
}

# edge tail head n x1 y1 .. xn yn [label xl yl] style color
# edge S3 S6 10 3.5244 0.20119 3.2727 0.19014 2.7322 0.17358 2.2778 0.20833 1.873 0.23929 1.7755 0.28067 1.375 0.34722 1.3389 0.35322 1.3009 0.35974 1.2635 0.3663 <ε> 2.3264 0.3125 solid black
/^edge / {
	tail = sname($2);
	head = sname($3);
	n    = $4;

	v = split($0, a, " ");
	# TODO: assert n < v - whatever
	# TODO: assert n >= 4 (knot, control, control); mind trailing stuff... label etc

	# TODO: note we assume -Etailclip=false -Eheadclip=false -Edir=none
	# TODO: explain that requirement

	# things i wish the graphviz documentation said about -Tplain:
	# graphviz always outputs cubic b-splines
	# 3i+1 points total. format is { knot, control, control }* knot
	# bezier is ordered tail to head
	# first two points (knot, control) are ontop of each other, in the tail node
	# last two points (control, knot) are ontop of each other, in the head node
	# coordinates are in inches

	printf "\te :=\n";
	for (i = 1; i <= n - 1; i += 3) {
		# knot
		ox = a[(i + 0) * 2 + 3];
		oy = a[(i + 0) * 2 + 4];

		# control
		c1x = a[(i + 1) * 2 + 3];
		c1y = a[(i + 1) * 2 + 4];

		# control
		c2x = a[(i + 2) * 2 + 3];
		c2y = a[(i + 2) * 2 + 4];

		printf "\t(%fin, %fin) .. controls (%fin, %fin) and (%fin, %fin) ..\n", ox, oy, c1x, c1y, c2x, c2y;
	}
	# TODO: knot at end, which i haven't drawn
	# last point:
	{
		# knot
		ox = a[(n + 0) * 2 + 3];
		oy = a[(n + 0) * 2 + 4];
		printf "\t(%fin, %fin);\n", ox, oy;
	}

	# XXX: assuming no space in label
label = slabel(a[n * 2 + 5]);
lx    = (a[n * 2 + 6]);
ly    = (a[n * 2 + 7]);

	if (n * 2 == v - 4 - 2) {
		label = "";
	}

	printf "fsmgvedge(%s, %s.diam, %s, %s.diam, e, \"%s\", (%fin, %fin), %s.loops);", tail, tail, head, head, label, lx, ly, head;

	if (head == tail) {
		print "	%s.loops := %s.loops + 1;", head, head
	}
}

/^stop$/ {
	# XXX: ought to be per graph; stop is per file
	print "endfig;"
}

END {
	print "end."
}

#!/usr/bin/awk

# TODO:
# cat nfa.dot | dot -Tplain > /tmp/a.mp \
# && mpost -s 'outputformat="svg"' -s 'outputtemplate="%j-%c.%o"' /tmp/a.mp

# Only intended to consume Dot generated from libfsm's .dot output,
# and not graphviz diagrams in general.

function sname(s)
{
	if (s == "start") {
		return "S0";
	}

	return s;
}

# TODO: convert label from graphviz's html to a utf8
# This is either <xyz> or abc (with no quotes)
function slabel(s)
{
	# XXX: placeholder
	if (s == "<>") {
		return "";
	}
	if (s == "\"\"") {
		return "";
	}

# XXX: placeholder
gsub(/ε/, "e", s); # XXX: why not?
gsub(/"/, "", s)
gsub(/</, "", s)
gsub(/>/, "", s)

	return s;
}

BEGIN {

	# TODO: would be simpler to gvpr to rename start to S0
	# TODO: common stuff for multiple graphs
	print "pair S[];"
	print "pair q[];"
	print "path e;"

	print "ahlength := 10bp;"
	print "ahangle  := 45;"

	print "path S[].n;"

	# diameter of node
	print "numeric S[].diam;"

	# number of self-edges
	print "numeric S[].loops;"

#	print "secondarydef v projectedalong w =";
#	print "	if pair(v) and pair(w):";
#	print "		(v dotprod w) / (w dotprod w) * w";
#	print "	else:";
#	print "		errmessage \"arguments must be pairs\"";
#	print "	fi";
#	print "enddef;";

}

/^graph / {
	printf "beginfig(%s)\n", $2
}

# node S6 0.97222 0.41667 0.30556 0.30556 "" solid circle black lightgrey
# node name x y width height label style shape color fillcolor
/^node / {
	name   = sname($2);
	x      = $3;
	y      = $4;
	width  = $5;
	height = $6;
	label  = slabel($7);
	style  = $8;
	shape  = $9;
	# TODO: label delimited by <>, may contain spaces

	printf "\t%s.loops := 0;\n", name

	printf "\t%s := (%fin, %fin);\n", name, x, y

# TODO: when plotting in graphviz, assume all nodes have the same width (not zero!). enough for "SXX" perhaps
# that should align them all on the same centre line. and pass fixedsize=true

	# TODO: explain this is diameter
	# TODO: store 8bp as a constant somewhere
# TODO: explain we take diameter from label presence and endstateness.
# ignore graphviz height and width, because it scales with label width and that looks bad for S1 vs S5
	diam = 0.3 + length(label) * 0.05;

	if (shape == "doublecircle") {
		printf "\t%s.diam = %fin + 8bp;\n", name, diam;
	} else {
		printf "\t%s.diam = %fin;\n", name, diam;
	}

	if (shape == "doublecircle") {
# TODO: would rather && together paths to %s.n or somesuch
		printf "\tdraw fullcircle scaled (%s.diam - 8bp) shifted %s withpen pencircle scaled 1bp;\n", name, name
	}

	printf "\t%s.n := fullcircle scaled %s.diam shifted %s;\n", name, name, name

	if ($2 != "start") {
		printf "\tdraw %s.n withpen pencircle scaled 1bp;\n", name

		if (label != "") {
			printf "\tlabel(\"%s\", %s);\n", label, name
		}
	}
}

# edge tail head n x1 y1 .. xn yn [label xl yl] style color
# edge S3 S6 10 3.5244 0.20119 3.2727 0.19014 2.7322 0.17358 2.2778 0.20833 1.873 0.23929 1.7755 0.28067 1.375 0.34722 1.3389 0.35322 1.3009 0.35974 1.2635 0.3663 <ε> 2.3264 0.3125 solid black
/^edge / {
	tail = sname($2);
	head = sname($3);
	n    = $4;

	v = split($0, a, " ");
	# TODO: assert n < v - whatever
	# TODO: assert n >= 4 (knot, control, control); mind trailing stuff... label etc

	# TODO: note we assume -Etailclip=false -Eheadclip=false -Edir=none
	# TODO: explain that requirement

	# things i wish the graphviz documentation said about -Tplain:
	# graphviz always outputs cubic b-splines
	# 3i+1 points total. format is { knot, control, control }* knot
	# bezier is ordered tail to head
	# first two points (knot, control) are ontop of each other, in the tail node
	# last two points (control, knot) are ontop of each other, in the head node
	# coordinates are in inches

	printf "\te :=\n";
	for (i = 1; i <= n - 1; i += 3) {
		# knot
		ox = a[(i + 0) * 2 + 3];
		oy = a[(i + 0) * 2 + 4];

		# control
		c1x = a[(i + 1) * 2 + 3];
		c1y = a[(i + 1) * 2 + 4];

		# control
		c2x = a[(i + 2) * 2 + 3];
		c2y = a[(i + 2) * 2 + 4];

		printf "\t(%fin, %fin) .. controls (%fin, %fin) and (%fin, %fin) ..\n", ox, oy, c1x, c1y, c2x, c2y;
	}
	# TODO: knot at end, which i haven't drawn
	# last point:
	{
		# knot
		ox = a[(n + 0) * 2 + 3];
		oy = a[(n + 0) * 2 + 4];
		printf "\t(%fin, %fin);\n", ox, oy;
	}

	printf "\tdraw e withpen pencircle scaled 0.25bp withcolor red;\n";

	# TODO: better explain this
	# TODO: explain reversed for q, else we'd find p again, where head == tail
	printf "pair p; p = e intersectionpoint %s.n;\n", head;
	printf "pair q; q = (reverse e) intersectionpoint %s.n;\n", tail;

	# XXX: assuming no space in label
label = slabel(a[n * 2 + 5]);
lx    = (a[n * 2 + 6]);
ly    = (a[n * 2 + 7]);

	if (n * 2 == v - 4 - 2) {
		label = "";
	}

	if (head == tail) {
		printf "path pp; pp = %s -- 0.5[p, q];\n", head;
printf "draw p -- q withpen pencircle scaled 0.5bp withcolor green;\n";
printf "draw 0.5[p, q] withpen pencircle scaled 0.8bp withcolor green;\n";
printf "draw %s -- 0.5[p, q] withpen pencircle scaled 0.8bp withcolor green;\n", head;

		# extend loop 1.25 * diameter
		# count number of self-loops for this node, and increment extension
		printf "\t%s.loops := %s.loops + 1;", head, head
		print "l := length pp;"
		printf "\tpair m; m := (0.5 + 2 * mlog(%s.loops + 1) / 256) * %s.diam * unitvector(direction l of pp) shifted %s;\n", head, head, head
# u3 = u1 projectedalong u2;
		printf "\tdraw %s -- m withpen pencircle scaled 0.5bp withcolor green;\n", head

		printf "\tpath b; b = %s .. q .. m .. p .. %s;\n", tail, head;

		if (label != "") {
			# TODO: top or bottom, depending on green line direction
			printf "\tlabel.top(\"%s\", m);\n", slabel(label)
		}
	} else {
		printf "\tpath b; b = %s .. q .. p .. %s;\n", tail, head;

		# TODO: explain this. we permit a 2% lengthening threshold relative to graphviz's b-spline
		# this affects NFA especially
		print "\tr := arclength b / arclength e;\n";
		print "\tboolean g; g := (r > 1.02) or (r < 0.93);"; # XXX: or (arclength b > 6in);"; - no, wigglyness instead

		print "\tif g:";
		print "\t\tb := e;";
		print "\tfi;";

# if it wouldn't have been a straight line, go via the label point instead:
# XXX: this idea doesn't work, because i don't know which side of the line the label point would be on
#printf "\t\tb := %s .. q .. (%fin, %fin) .. p .. %s;\n", tail, lx, ly, head;

		if (label != "") {
			printf "\tpair bl;\n";
			printf "\tpair ml; ml := .5[%s, %s];\n", tail, head;

# TODO: new idea; draw right angle from midpoint, and use that to find intersection with b
# this way it should always align for multiple edges, even if on a slant

printf "draw (%fin, %fin) withpen pencircle scaled 5bp withcolor red;\n", lx, ly
printf "draw point (length b / 2) of b withpen pencircle scaled 4bp withcolor blue;\n"
#printf "draw %s -- %s withpen pencircle scaled 1bp withcolor green;\n", tail, head
#printf "draw ml withpen pencircle scaled 5bp withcolor green;\n"

			print "\tif g:";
				printf "\t\tbl := (%fin, %fin);", lx, ly;
			print "\telse:";
				printf "\t\tbl := point (length b / 2) of b;\n";
			print "\tfi;";

# TODO: plus label delta, distancing it from point. make a label function
# TODO: maybe an "extend" function, to extend a path in its direction. use dotprod for that?

			# lft | rt | top | bot | ulft | urt | llft | lrt
			# labels ought to be below, if they're below the straight line from head-tail, or above otherwise
			# TODO: round label coordinates to grid? quantize rather
# XXX: if using graphviz's point, label exactly on the spot
print "\tif g:";
# TODO: still want to distance by +/- 4bp or so
			printf "\t\tlabel(\"%s\", bl);\n", label
print "\telse:";
			printf "\tif ypart bl < ypart ml:\n" # TODO: threshold for considering "below"
			printf "\t\tlabel.bot(\"%s\", bl shifted (0, -3bp));\n", label
			printf "\telse:\n"
			printf "\t\tlabel.top(\"%s\", bl shifted (0, +3bp));\n", label
			printf "\tfi;\n"
print "\tfi;";
		}
	}

	printf "drawarrow b cutbefore q cutafter p withpen pencircle scaled 1bp;\n";
}

/^stop$/ {
	# XXX: ought to be per graph; stop is per file
	print "endfig;"
}

END {
	print "end."
}


(minor) fix symbol table gen script

(minor) fix symbol table gen script

#
# Converts the output of 'nm' into a C source containing an array of symbols
# that correspond to functions (t and T in terms of 'nm').
#
# Usage:
#   nm -n image | awk -f nm2c.awk
#
#   Date: Aug 28, 2012
# Author: Eldar Abusalimov
#

BEGIN {
	print "/* Auto-generated file. Do not edit. */";
	print "";
	print "#include <debug/symbol.h>";
	print "#include <stddef.h>";
	print "";
	print "const struct symbol __symbol_table[] = {";
}

/^[0-9a-fA-F]* [tT]/ {
	split($4,a,":");
	if (a[1])
		loc = sprintf("{ \"%s\", %d }", a[1], a[2])
	else
		loc = "{ 0 }"
	printf "\t{ (void *) 0x%s, \"%s\", %s },\n", $1, $3, loc;
}

END {
	print "};";
	print "const size_t __symbol_table_size =";
	print "\tsizeof(__symbol_table) / sizeof(__symbol_table[0]);";
	print "";
}

#
# Converts the output of 'nm' into a C source containing an array of symbols
# that correspond to functions (t and T in terms of 'nm').
#
# Usage:
#   nm -n image | awk -f nm2c.awk
#
#   Date: Aug 28, 2012
# Author: Eldar Abusalimov
#

BEGIN {
	print "/* Auto-generated file. Do not edit. */";
	print "";
	print "#include <debug/symbol.h>";
	print "#include <stddef.h>";
	print "";
	print "const struct symbol __symbol_table[] = {";
}

/^[0-9a-fA-F]* [tT]/ {
	split($4,a,":");
	if (a[1])
		loc = sprintf("{ \"%s\", %d }", a[1], a[2])
	else
		loc = sprintf("{ }", a[1], a[2])
	printf "\t{ (void *) 0x%s, \"%s\", %s },\n", $1, $3, loc;
}

END {
	print "};";
	print "const size_t __symbol_table_size =";
	print "\tsizeof(__symbol_table) / sizeof(__symbol_table[0]);";
	print "";
}


Fix from Aero.  Note that this file is depercated, but for the record...

#!/usr/bin/awk -f
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#

BEGIN {
    found = 0;
}

#
# "true" pc850 reports:
#    Host bridge: Intel Corp. 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (AGP disabled) (rev 3).
# upgraded pc600 reports:
#    Host bridge: Intel Corp. 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (rev 3).
#

/^[ ]+Host bridge: Intel Corp.*[0-9][0-9][0-9][0-9][0-9]BX.*\(AGP disabled\)/ {
    print "BX";
    found = 1;
    exit
}
/^[ ]+Host bridge: Intel Corp.*[0-9][0-9][0-9][0-9][0-9]BX/ {
    print "BX-AGP";
    found = 1;
    exit
}
/^[ ]+Host bridge: Intel Corp.*[0-9][0-9][0-9][0-9][0-9]GX/ {
    print "GX";
    found = 1;
    exit
}
/^[ ]+PCI bridge: Intel Corp.*HI_C Virtual PCI-to-PCI Bridge/ {
    print "HI_C";
    found = 1;
    exit
}
/^[ ]+PCI bridge: Intel Corp.*HI_B Virtual PCI-to-PCI Bridge/ {
    print "HI_B";
    found = 1;
    exit
}
END {
    if (found == 0) {
	print "??";
    }
}

#!/usr/bin/awk -f
#
# EMULAB-COPYRIGHT
# Copyright (c) 2000-2004 University of Utah and the Flux Group.
# All rights reserved.
#

BEGIN {
    found = 0;
}

#
# "true" pc850 reports:
#    Host bridge: Intel Corp. 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (AGP disabled) (rev 3).
# upgraded pc600 reports:
#    Host bridge: Intel Corp. 440BX/ZX/DX - 82443BX/ZX/DX Host bridge (rev 3).
#

/^[ ]+Host bridge: Intel Corp.*[0-9][0-9][0-9][0-9][0-9]BX.*\(AGP disabled\)/ {
    print "BX";
    found = 1;
    exit
}
/^[ ]+Host bridge: Intel Corp.*[0-9][0-9][0-9][0-9][0-9]BX/ {
    print "BX-AGP";
    found = 1;
    exit
}
/^[ ]+Host bridge: Intel Corp.*[0-9][0-9][0-9][0-9][0-9]GX/ {
    print "GX";
    found = 1;
    exit
}
/^[ ]+PCI bridge: Intel Corp.*HI_B Virtual PCI-to-PCI Bridge/ {
    print "HI_B";
    found = 1;
    exit
}
END {
    if (found == 0) {
	print "??";
    }
}


updated makemega addstats to go with latest stats;

#!/usr/bin/awk -f
##########
#The MIT License (MIT)
#
# Copyright (c) 2015 Aiden Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
##########
# Helper script for taking in multiple inter.txt and producing sums
# Juicer version 1.5
$1=="Sequenced"{
gsub(/,/,"",$4);
 tot=tot+$4;
}
$1=="Unmapped:"{
gsub(/,/,"",$2);
 unm=unm+$2
}
$1=="Normal"{
gsub(/,/,"",$3);
 norm=norm+$3}
$1=="Chimeric" && $2=="Paired:"{
gsub(/,/,"",$3);
 chim=chim+$3}
$1=="MAPQ" && $2=="0:"{
gsub(/,/,"",$3);
 mapq0=mapq0+$3}
$1=="Collisions:"{
gsub(/,/,"",$2);
 coll=coll+$2}
$3=="Collisions:"{
gsub(/,/,"",$4);
 lowc=lowc+$4}
$1=="Ligation" {
gsub(/,/,"",$4);
 lig=lig+$4}
$1=="Unique" {
gsub(/,/,"",$3);
 dedup=dedup+$3}
$1=="PCR"{
gsub(/,/,"",$3);
 dups=dups+$3}
$1=="Optical"{
gsub(/,/,"",$3);
 optdups=optdups+$3}
END{
  printf("Sequenced Read Pairs:  %'d\n Normal Paired: %'d (%0.2f%)\n Chimeric Paired: %'d (%0.2f%)\n Collisions: %'d (%0.2f%)\n Low MAPQ Collisions: %'d (%0.2f%)\n Unmapped: %'d (%0.2f%)\n MAPQ 0: %'d (%0.2f%)\n Ligation Motif Present: %'d (%0.2f%)\nAlignable (Normal+Chimeric Paired): %'d (%0.2f%)\n Unique Reads: %'d\n PCR Duplicates: %'d\n Optical Duplicates: %'d\n", tot, norm, norm*100/tot, chim, chim*100/tot, coll, coll*100/tot, lowc, lowc*100/tot, unm, unm*100/tot, mapq0, mapq0*100/tot, lig, lig*100/tot, norm+chim, (norm+chim)*100/tot, dedup, dups, optdups );
}

#!/usr/bin/awk -f
##########
#The MIT License (MIT)
#
# Copyright (c) 2015 Aiden Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
##########
# Helper script for taking in multiple inter.txt and producing sums
# Juicer version 1.5
$1=="Sequenced"{
gsub(/,/,"",$4);
 total=total+$4;
}
$1=="Unmapped:"{
gsub(/,/,"",$2);
 unmapped=unmapped+$2
}
$1=="Normal"{
gsub(/,/,"",$3);
 regular=regular+$3}
$1=="Chimeric" && $2=="Paired:"{
gsub(/,/,"",$3);
 normal=normal+$3}
$2=="Ambiguous:"{
gsub(/,/,"",$3);
 abnorm=abnorm+$3}
$1=="Alignable"{
gsub(/,/,"",$4);
 alignable=alignable+$4}
$1=="Unique" {
gsub(/,/,"",$3);
 dedup=dedup+$3}
$1=="PCR"{
gsub(/,/,"",$3);
 dups=dups+$3}
$1=="Optical"{
gsub(/,/,"",$3);
 optdups=optdups+$3}
END{
 printf("%s %'d\n", "Sequenced Read Pairs:", total);
 printf(" %s %'d (%0.2f%)\n", "Normal Paired:", regular, regular*100/total);
 printf(" %s %'d (%0.2f%)\n", "Chimeric Paired:", normal, normal*100/total);
 printf(" %s %'d (%0.2f%)\n", "Chimeric Ambiguous:", abnorm, abnorm*100/total);
 printf(" %s %'d (%0.2f%)\n", "Unmapped:", unmapped, unmapped*100/total);
 printf(" %s %'d (%0.2f%)\n", "Alignable (Normal+Chimeric Paired):", alignable, alignable*100/total);
 printf("%s %'d\n", "Unique Reads:", dedup);
 printf("%s %'d\n", "PCR Duplicates:", dups);
  printf("%s %'d\n", "Optical Duplicates:", optdups);
}


remove todo

remove todo

#
# Render a zodiac page
#

BEGIN {
  action = "none"
  ext = "none"
  main = "_main"
  helpers_loaded = "no"
  layout = ""
}

{
  split(FILENAME, parts, ".")
  ext = parts[length(parts)]
  if (FILENAME == "-") {
    action = "config"
  } else if (ext == "meta") {
    action = "meta"
  } else if (ext == "layout") {
    action = "layout"
  } else {
    action = "page"
    filter_ext = ext
  }
}

# Process lines from meta files
action == "meta" {
  split($0, kv, ": ")
  data[kv[1]] = kv[2]
  next
}

# Done processing meta
# Since data is loaded now, load the helpers
action != "meta" && helpers_loaded == "no" && helpers == "yes" {
  load_helpers()
  helpers_loaded = "yes"
}

# Process lines from the page
action == "page" {
  if (!contents[main]) {
    contents[main] = bind_data($0)

    # save the extension for this content type
    # to find the appropriate filter to render it
    filter_exts[main] = ext
  } else {
    contents[main] = contents[main] "\n" bind_data($0)
  }
  next
}

# Process lines from the layout
action == "layout" {

  # replace yield with rendered content
  if (match($0, /{{{yield}}}/)) {
    sub(/{{{yield}}}/, render_content(main))
  }

  if (layout == "") {
    layout = bind_data($0)
  } else {
    layout = layout "\n" bind_data($0)
  }
}

END {
  if (layout != "") {
    print layout
  } else {
    print render_content(main)
  }
}

function bind_data(txt,   tag, key) {
  if (match(txt, /{{([^}]*)}}/)) {
    tag = substr(txt, RSTART, RLENGTH)
    match(tag, /([[:alnum:]_]|[?]).*[^}]/)
    key = substr(tag, RSTART, RLENGTH)
    gsub(tag, data[key], txt)
    return bind_data(txt, data)
  } else {
    return txt
  }
}

function render_content(type,     ext_key, filter_ext, filter_cmd, txt) {
  ext_key = type "_ext"

  # Get the extension of the content type
  filter_ext = filter_exts[type]

  # Get the appropriate filter command for this extension
  filter_cmd = filter[filter_ext]

  # Get the text of the content for the given type
  txt = contents[type]

  if (filter_cmd != "none") {
    return run_filter(filter_cmd, txt)
  } else {
    return txt
  }
}

function run_filter(cmd, txt,   rand_date, tmpfile, rendered_txt, date_cmd, markdown_cmd, line) {
  date_cmd = "date +%Y%m%d%H%M%S"
  date_cmd | getline rand_date
  close(date_cmd)

  tmpfile = "/tmp/awk_render" rand_date
  filter_cmd = cmd " > " tmpfile

  # pipe content to filter.awk
  print txt | filter_cmd
  close(filter_cmd)

  # pull out the filtered page
  while((getline line < tmpfile) > 0) {
    rendered_txt = rendered_txt "\n" line
  }
  close(tmpfile)

  system("rm " tmpfile)

  return rendered_txt
}

#
# Render a zodiac page
#

BEGIN {
  action = "none"
  ext = "none"
  main = "_main"
  helpers_loaded = "no"
  layout = ""
}

{
  split(FILENAME, parts, ".")
  ext = parts[length(parts)]
  if (FILENAME == "-") {
    action = "config"
  } else if (ext == "meta") {
    action = "meta"
  } else if (ext == "layout") {
    action = "layout"
  } else {
    action = "page"
    filter_ext = ext
  }
}

# Process lines from meta files
action == "meta" {
  split($0, kv, ": ")
  data[kv[1]] = kv[2]
  next
}

# Done processing meta
# Since data is loaded now, load the helpers
action != "meta" && helpers_loaded == "no" && helpers == "yes" {
  load_helpers()
  helpers_loaded = "yes"
}

# Process lines from the page
action == "page" {
  if (!contents[main]) {
    contents[main] = bind_data($0)

    # save the extension for this content type
    # to find the appropriate filter to render it
    filter_exts[main] = ext
  } else {
    contents[main] = contents[main] "\n" bind_data($0)
  }
  next
}

# Process lines from the layout
action == "layout" {

  # replace yield with rendered content
  if (match($0, /{{{yield}}}/)) {
    sub(/{{{yield}}}/, render_content(main))
  }

  if (layout == "") {
    layout = bind_data($0)
  } else {
    layout = layout "\n" bind_data($0)
  }
}

END {
  if (layout != "") {
    print layout
  } else {
    print render_content(main)
  }
}

function bind_data(txt,   tag, key) {
  if (match(txt, /{{([^}]*)}}/)) {
    tag = substr(txt, RSTART, RLENGTH)
    match(tag, /([[:alnum:]_]|[?]).*[^}]/)
    key = substr(tag, RSTART, RLENGTH)
    gsub(tag, data[key], txt)
    return bind_data(txt, data)
  } else {
    return txt
  }
}

function render_content(type,     ext_key, filter_ext, filter_cmd, txt) {
  ext_key = type "_ext"

  # Get the extension of the content type
  filter_ext = filter_exts[type]

  # Get the appropriate filter command for this extension
  filter_cmd = filter[filter_ext]

  # Get the text of the content for the given type
  txt = contents[type]

  if (filter_cmd != "none") {
    return run_filter(filter_cmd, txt)
  } else {
    return txt
  }
}

function run_filter(cmd, txt,   rand_date, tmpfile, rendered_txt, date_cmd, markdown_cmd, line) {
  # TODO use mktemp instead
  date_cmd = "date +%Y%m%d%H%M%S"
  date_cmd | getline rand_date
  close(date_cmd)

  tmpfile = "/tmp/awk_render" rand_date
  filter_cmd = cmd " > " tmpfile

  # pipe content to filter.awk
  print txt | filter_cmd
  close(filter_cmd)

  # pull out the filtered page
  while((getline line < tmpfile) > 0) {
    rendered_txt = rendered_txt "\n" line
  }
  close(tmpfile)

  system("rm " tmpfile)

  return rendered_txt
}


Clarify that input must be sorted

Clarify that input must be sorted

#!/usr/bin/env awk -f
#
# Computes the median.
#
# Usage: median
#
# Input:
#   - a sorted column of numbers
#
# Output:
#   - median value

BEGIN {
	i = 0
}
{
	a[i++] = $1
}
END {
	j = i/2
	if ( i%2 == 1 ) {
		median = a[int(j)]
	} else {
		median = (a[j] + a[j-1])/2
	}
	print median
}

#!/usr/bin/env awk -f
#
# Computes the median.
#
# Usage: median
#
# Input:
#   - a column of numbers
#
# Output:
#   - median value

BEGIN {
	i = 0
}
{
	a[i++] = $1
}
END {
	j = i/2
	if ( i%2 == 1 ) {
		median = a[int(j)]
	} else {
		median = (a[j] + a[j-1])/2
	}
	print median
}


multiline whitespace fix: do not offer "PICK LINE" again after it has been run

# copy data to x11 selection
$0 != get("ctx.sel") {
    label("SELECT"); menu("@xsel " $0)
}
# copy data to x11 clipboard
{
    label("COPY"); menu("echo -n " Q($0) " | xclip -i -selection clipboard")
}

# bookmarks, usually hand picked lines from history
attr["from"] != "bookmarks" {
    label("BOOKMARK"); menu("@bookmark " $0)
}
attr["from"] == "bookmarks" {
    label("FORGET"); menu("@forget " $0)
}

# multiline features
func pick_line(s) {
    s = collapse(s)
    c = btick(nest(s, "menu"))
    runHook(nest(trim(c)))
}
# this is a hack to reclassify full line, see main.head.awk:
func merge_lines(s) {
    attr["filter_merge"] = 1
    attr["filter_merge_prefix"] = s
}
# todo: remove NR==1 check? allow mergeing late?
NR == 1 && RT {
    label("PICK LINE"); menu("@pick_line " $0)
    label("MERGE LINES"); menu("@merge_lines " $0)
}
# cancel multiline
RT || NR > 1 {
    label("CANCEL"); menu("@quit 1")
}

# debug
{
    label("DUMP"); menu("@dump")
    label("DUMP MENU"); menu("@dumpMenu")
}

# copy data to x11 selection
$0 != get("ctx.sel") {
    label("SELECT"); menu("@xsel " $0)
}
# copy data to x11 clipboard
{
    label("COPY"); menu("echo -n " Q($0) " | xclip -i -selection clipboard")
}

# bookmarks, usually hand picked lines from history
attr["from"] != "bookmarks" {
    label("BOOKMARK"); menu("@bookmark " $0)
}
attr["from"] == "bookmarks" {
    label("FORGET"); menu("@forget " $0)
}

# multiline features
func pick_line(s) {
    s = collapse(s)
    c = btick(nest(s, "menu"))
    runHook(nest(c))
}
# this is a hack to reclassify full line, see main.head.awk:
func merge_lines(s) {
    attr["filter_merge"] = 1
    attr["filter_merge_prefix"] = s
}
# todo: remove NR==1 check? allow mergeing late?
NR == 1 && RT {
    label("PICK LINE"); menu("@pick_line " $0)
    label("MERGE LINES"); menu("@merge_lines " $0)
}
# cancel multiline
RT || NR > 1 {
    label("CANCEL"); menu("@quit 1")
}

# debug
{
    label("DUMP"); menu("@dump")
    label("DUMP MENU"); menu("@dumpMenu")
}