Utils: fix curl() by parameterize(url)

Utils: fix curl() by parameterize(url)

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev" SUPERR
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev" SUPERR "| sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    }

    if (ENVIRON["TRANS_ENTRY"]) {
        command = Rlwrap " " ENVIRON["TRANS_ENTRY"] " "                 \
            parameterize("-no-rlwrap") # never fork rlwrap again!
    } else if (fileExists(EntryPoint)) {
        command = Rlwrap " " Gawk " -f " EntryPoint                     \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            command = command " " parameterize(ARGV[i])

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    }

    params = ""
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            params = params " " parameterize(ARGV[i], "\"")
    if (ENVIRON["TRANS_ENTRY"]) {
        el = "(progn (setq explicit-shell-file-name \"" ENVIRON["TRANS_ENTRY"] "\") " \
            "(setq explicit-" Command "-args '(\"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else if (fileExists(EntryPoint)) {
        el = "(progn (setq explicit-shell-file-name \"" Gawk "\") "     \
            "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    command = Emacs " --eval " parameterize(el)

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    } else {
        command = Curl " --location --silent " parameterize(url)
        content = NULLSTR
        while ((command |& getline line) > 0)
            content = (content ? content "\n" : NULLSTR) line
        return content
    }
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    # hexdump tricks:
    # (1) use -v (--no-squeezing)
    # (2) use "%u" (unsigned integers)
    command = "hexdump" " -v -e'1/1 \"%03u\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

# Base64 encode a string.
# NOTE: can only be ran once for each text! Build a cache.
function base64(text,    command, temp) {
    command = "base64"
    ("echo -n " parameterize(text) PIPE command) | getline temp
    return temp
}

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev" SUPERR
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev" SUPERR "| sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    }

    if (ENVIRON["TRANS_ENTRY"]) {
        command = Rlwrap " " ENVIRON["TRANS_ENTRY"] " "                 \
            parameterize("-no-rlwrap") # never fork rlwrap again!
    } else if (fileExists(EntryPoint)) {
        command = Rlwrap " " Gawk " -f " EntryPoint                     \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            command = command " " parameterize(ARGV[i])

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    }

    params = ""
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            params = params " " parameterize(ARGV[i], "\"")
    if (ENVIRON["TRANS_ENTRY"]) {
        el = "(progn (setq explicit-shell-file-name \"" ENVIRON["TRANS_ENTRY"] "\") " \
            "(setq explicit-" Command "-args '(\"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else if (fileExists(EntryPoint)) {
        el = "(progn (setq explicit-shell-file-name \"" Gawk "\") "     \
            "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    command = Emacs " --eval " parameterize(el)

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    } else {
        command = Curl " --location --silent " url
        content = NULLSTR
        while ((command |& getline line) > 0)
            content = (content ? content "\n" : NULLSTR) line
        return content
    }
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    # hexdump tricks:
    # (1) use -v (--no-squeezing)
    # (2) use "%u" (unsigned integers)
    command = "hexdump" " -v -e'1/1 \"%03u\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

# Base64 encode a string.
# NOTE: can only be ran once for each text! Build a cache.
function base64(text,    command, temp) {
    command = "base64"
    ("echo -n " parameterize(text) PIPE command) | getline temp
    return temp
}


mpers.awk: add support for DWARF files generated by clang

* mpers.awk: Define default_pointer_size.  Initialize byte_size
for DW_TAG_pointer_type to default_pointer_size.  Set parent
for DW_TAG_member only.  Handle DW_AT_count as an alternative
to DW_AT_upper_bound.

#!/bin/gawk
#
# Copyright (c) 2015 Elvira Khabirova <lineprinter0@gmail.com>
# Copyright (c) 2015-2016 Dmitry V. Levin <ldv@altlinux.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

function compare_indices(i1, v1, i2, v2) {
	c1 = strtonum(sprintf("%s", i1))
	c2 = strtonum(sprintf("%s", i2))
	if (c1 < c2)
		return -1
	return (c1 != c2)
}
function array_get(array_idx, array_member, array_return)
{
	array_return = array[array_idx][array_member]
	if ("" == array_return) {
		printf("%s: index [%s] without %s\n",
		       FILENAME, array_idx, array_member) > "/dev/stderr"
		exit 1
	}
	return array_return
}
function array_seq(array_idx)
{
	if ("seq" in array[array_idx])
		return array[array_idx]["seq"]
	index_seq++
	array[array_idx]["seq"] = index_seq
	return index_seq
}
function enter(array_idx)
{
	if (array_idx in called) {
		printf("%s: index loop detected:", FILENAME) > "/dev/stderr"
		for (item in called)
			printf(" %s", item) > "/dev/stderr"
		print "" > "/dev/stderr"
		exit 1
	}
	called[array_idx] = 1
}
function leave(array_idx, to_return)
{
	delete called[array_idx]
	return to_return
}
function what_is(what_idx, type_idx, special, item, \
		 location, prev_location, prev_returned_size)
{
	enter(what_idx)
	special = array_get(what_idx, "special")
	switch (special) {
	case "base_type":
		switch (array_get(what_idx, "encoding")) {
		case 5: # signed
			printf("int%s_t ",
			       8 * array_get(what_idx, "byte_size"))
			break
		case 7: # unsigned
			printf("uint%s_t ",
			       8 * array_get(what_idx, "byte_size"))
			break
		default: # float, signed/unsigned char
			printf("%s ", array_get(what_idx, "name"))
			break
		}
		returned_size = array_get(what_idx, "byte_size")
		break
	case "enumeration_type":
		type_idx = array_get(what_idx, "type")
		returned_size = array_get(what_idx, "byte_size")
		printf("uint%s_t ", 8 * returned_size)
		break
	case "pointer_type":
		printf("mpers_ptr_t ")
		returned_size = array_get(what_idx, "byte_size")
		break
	case "array_type":
		type_idx = array_get(what_idx, "type")
		what_is(type_idx)
		to_return = array[what_idx]["upper_bound"]
		if ("" == to_return)
			to_return = 0
		returned_size = to_return * returned_size
		return leave(what_idx, to_return)
		break
	case "structure_type":
		print "struct {"
		prev_location = 0
		location = 0
		returned_size = 0
		prev_returned_size = 0
		for (item in array) {
			if ("parent" in array[item] && \
				array_get(item, "parent") == what_idx) {
				location = array_get(item, "location")
				loc_diff = location - prev_location - \
					prev_returned_size
				if (loc_diff != 0) {
					printf("unsigned char mpers_%s_%s[%s];\n",
					       "filler", array_seq(item), loc_diff)
				}
				prev_location = location
				returned = what_is(item)
				prev_returned_size = returned_size
				printf("%s", array[item]["name"])
				if ("" != returned) {
					printf("[%s]", returned)
				}
				print ";"
			}
		}
		returned_size = array_get(what_idx, "byte_size")
		loc_diff = returned_size - prev_location - prev_returned_size
		if (loc_diff != 0) {
			printf("unsigned char mpers_%s_%s[%s];\n",
			       "end_filler", array_seq(item), loc_diff)
		}
		printf("} ATTRIBUTE_PACKED ")
		break
	case "union_type":
		print "union {"
		for (item in array) {
			if ("parent" in array[item] && \
				array_get(item, "parent") == what_idx) {
				returned = what_is(item)
				printf("%s", array_get(item, "name"))
				if ("" != returned) {
					printf("[%s]", returned)
				}
				print ";"
			}
		}
		printf("} ")
		returned_size = array_get(what_idx, "byte_size")
		break
	case "typedef":
		type_idx = array_get(what_idx, "type")
		return leave(what_idx, what_is(type_idx))
		break
	case "member":
		type_idx = array_get(what_idx, "type")
		return leave(what_idx, what_is(type_idx))
		break
	default:
		type_idx = array_get(what_idx, "type")
		what_is(type_idx)
		break
	}
	return leave(what_idx, "")
}
BEGIN {
	match(ARCH_FLAG, /[[:digit:]]+/, temparray)
	default_pointer_size = temparray[0] / 8
	print "#include <inttypes.h>"
}
/^<[[:xdigit:]]+>/ {
	match($0, /([[:alnum:]]+)><([[:alnum:]]+)/, matches)
	level = matches[1]
	idx = "0x" matches[2]
	array[idx]["idx"] = idx
	parent[level] = idx
}
/^DW_AT_data_member_location/ {
	if (!match($0, /\(DW_OP_plus_uconst:[[:space:]]+([[:digit:]]+)\)/, temparray))
		match($0, /([[:digit:]]+)/, temparray)
	array[idx]["location"] = temparray[1]
}
/^DW_AT_name/ {
	match($0, /:[[:space:]]+([[:alpha:]_][[:alnum:]_[:space:]]*)/, \
		temparray)
	array[idx]["name"] = temparray[1]
}
/^DW_AT_byte_size/ {
	match($0, /[[:digit:]]+/, temparray)
	array[idx]["byte_size"] = temparray[0]
}
/^DW_AT_encoding/ {
	match($0, /[[:digit:]]+/, temparray)
	array[idx]["encoding"] = temparray[0]
}
/^DW_AT_type/ {
	match($0, /:[[:space:]]+<(0x[[:xdigit:]]*)>$/, temparray)
	array[idx]["type"] = temparray[1]
}
/^DW_AT_upper_bound/ {
	match($0, /[[:digit:]]+/, temparray)
	array[parent[level-1]]["upper_bound"] = temparray[0] + 1
}
/^DW_AT_count/ {
	match($0, /[[:digit:]]+/, temparray)
	array[parent[level-1]]["upper_bound"] = temparray[0]
}
/^Abbrev Number:[^(]+\(DW_TAG_/ {
	if (match($0, /typedef|union_type|structure_type|pointer_type\
|enumeration_type|array_type|base_type|member/, temparray)) {
		array[idx]["special"] = temparray[0]
		if ("pointer_type" == temparray[0])
			array[idx]["byte_size"] = default_pointer_size
		if (level > 1 && "member" == temparray[0])
			array[idx]["parent"] = parent[level-1]
	}
}
END {
	PROCINFO["sorted_in"] = "compare_indices"
	for (item in array) {
		if (array[item]["special"] == "pointer_type") {
			print "typedef uint" \
				8 * array_get(item, "byte_size") "_t mpers_ptr_t;"
			break
		}
	}
	for (item in array) {
		if (array[item]["name"] == VAR_NAME) {
			type = array_get(item, "type")
			print "typedef"
			what_is(type)
			name = array_get(type, "name")
			print ARCH_FLAG "_" name ";"
			print "#define MPERS_" \
				ARCH_FLAG "_" name " " \
				ARCH_FLAG "_" name
			break
		}
	}
}

#!/bin/gawk
#
# Copyright (c) 2015 Elvira Khabirova <lineprinter0@gmail.com>
# Copyright (c) 2015 Dmitry V. Levin <ldv@altlinux.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

function compare_indices(i1, v1, i2, v2) {
	c1 = strtonum(sprintf("%s", i1))
	c2 = strtonum(sprintf("%s", i2))
	if (c1 < c2)
		return -1
	return (c1 != c2)
}
function array_get(array_idx, array_member, array_return)
{
	array_return = array[array_idx][array_member]
	if ("" == array_return) {
		printf("%s: index [%s] without %s\n",
		       FILENAME, array_idx, array_member) > "/dev/stderr"
		exit 1
	}
	return array_return
}
function array_seq(array_idx)
{
	if ("seq" in array[array_idx])
		return array[array_idx]["seq"]
	index_seq++
	array[array_idx]["seq"] = index_seq
	return index_seq
}
function enter(array_idx)
{
	if (array_idx in called) {
		printf("%s: index loop detected:", FILENAME) > "/dev/stderr"
		for (item in called)
			printf(" %s", item) > "/dev/stderr"
		print "" > "/dev/stderr"
		exit 1
	}
	called[array_idx] = 1
}
function leave(array_idx, to_return)
{
	delete called[array_idx]
	return to_return
}
function what_is(what_idx, type_idx, special, item, \
		 location, prev_location, prev_returned_size)
{
	enter(what_idx)
	special = array_get(what_idx, "special")
	switch (special) {
	case "base_type":
		switch (array_get(what_idx, "encoding")) {
		case 5: # signed
			printf("int%s_t ",
			       8 * array_get(what_idx, "byte_size"))
			break
		case 7: # unsigned
			printf("uint%s_t ",
			       8 * array_get(what_idx, "byte_size"))
			break
		default: # float, signed/unsigned char
			printf("%s ", array_get(what_idx, "name"))
			break
		}
		returned_size = array_get(what_idx, "byte_size")
		break
	case "enumeration_type":
		type_idx = array_get(what_idx, "type")
		returned_size = array_get(what_idx, "byte_size")
		printf("uint%s_t ", 8 * returned_size)
		break
	case "pointer_type":
		printf("mpers_ptr_t ")
		returned_size = array_get(what_idx, "byte_size")
		break
	case "array_type":
		type_idx = array_get(what_idx, "type")
		what_is(type_idx)
		to_return = array[what_idx]["upper_bound"]
		if ("" == to_return)
			to_return = 0
		returned_size = to_return * returned_size
		return leave(what_idx, to_return)
		break
	case "structure_type":
		print "struct {"
		prev_location = 0
		location = 0
		returned_size = 0
		prev_returned_size = 0
		for (item in array) {
			if ("parent" in array[item] && \
				array_get(item, "parent") == what_idx) {
				location = array_get(item, "location")
				loc_diff = location - prev_location - \
					prev_returned_size
				if (loc_diff != 0) {
					printf("unsigned char mpers_%s_%s[%s];\n",
					       "filler", array_seq(item), loc_diff)
				}
				prev_location = location
				returned = what_is(item)
				prev_returned_size = returned_size
				printf("%s", array[item]["name"])
				if ("" != returned) {
					printf("[%s]", returned)
				}
				print ";"
			}
		}
		returned_size = array_get(what_idx, "byte_size")
		loc_diff = returned_size - prev_location - prev_returned_size
		if (loc_diff != 0) {
			printf("unsigned char mpers_%s_%s[%s];\n",
			       "end_filler", array_seq(item), loc_diff)
		}
		printf("} ATTRIBUTE_PACKED ")
		break
	case "union_type":
		print "union {"
		for (item in array) {
			if ("parent" in array[item] && \
				array_get(item, "parent") == what_idx) {
				returned = what_is(item)
				printf("%s", array_get(item, "name"))
				if ("" != returned) {
					printf("[%s]", returned)
				}
				print ";"
			}
		}
		printf("} ")
		returned_size = array_get(what_idx, "byte_size")
		break
	case "typedef":
		type_idx = array_get(what_idx, "type")
		return leave(what_idx, what_is(type_idx))
		break
	case "member":
		type_idx = array_get(what_idx, "type")
		return leave(what_idx, what_is(type_idx))
		break
	default:
		type_idx = array_get(what_idx, "type")
		what_is(type_idx)
		break
	}
	return leave(what_idx, "")
}
BEGIN {
	print "#include <inttypes.h>"
}
/^<[[:xdigit:]]+>/ {
	match($0, /([[:alnum:]]+)><([[:alnum:]]+)/, matches)
	level = matches[1]
	idx = "0x" matches[2]
	array[idx]["idx"] = idx
	parent[level] = idx
	if (level > 1) {
		array[idx]["parent"] = parent[level-1]
	}
}
/^DW_AT_data_member_location/ {
	if (!match($0, /\(DW_OP_plus_uconst:[[:space:]]+([[:digit:]]+)\)/, temparray))
		match($0, /([[:digit:]]+)/, temparray)
	array[idx]["location"] = temparray[1]
}
/^DW_AT_name/ {
	match($0, /:[[:space:]]+([[:alpha:]_][[:alnum:]_[:space:]]*)/, \
		temparray)
	array[idx]["name"] = temparray[1]
}
/^DW_AT_byte_size/ {
	match($0, /[[:digit:]]+/, temparray)
	array[idx]["byte_size"] = temparray[0]
}
/^DW_AT_encoding/ {
	match($0, /[[:digit:]]+/, temparray)
	array[idx]["encoding"] = temparray[0]
}
/^DW_AT_type/ {
	match($0, /:[[:space:]]+<(0x[[:xdigit:]]*)>$/, temparray)
	array[idx]["type"] = temparray[1]
}
/^DW_AT_upper_bound/ {
	match($0, /[[:digit:]]+/, temparray)
	array[parent[level-1]]["upper_bound"] = temparray[0] + 1
}
/^Abbrev Number:[^(]+\(DW_TAG_/ {
	if (match($0, /typedef|union_type|structure_type|pointer_type\
|enumeration_type|array_type|base_type|member/, temparray)) {
		array[idx]["special"] = temparray[0]
	}
}
END {
	PROCINFO["sorted_in"] = "compare_indices"
	for (item in array) {
		if (array[item]["special"] == "pointer_type") {
			print "typedef uint" \
				8 * array_get(item, "byte_size") "_t mpers_ptr_t;"
			break
		}
	}
	for (item in array) {
		if (array[item]["name"] == VAR_NAME) {
			type = array_get(item, "type")
			print "typedef"
			what_is(type)
			name = array_get(type, "name")
			print ARCH_FLAG "_" name ";"
			print "#define MPERS_" \
				ARCH_FLAG "_" name " " \
				ARCH_FLAG "_" name
			break
		}
	}
}


Updated timeCalculator to work with the more complex prints.

#![$2] += $NF }
$0 ~ "^Time edu.berkeley.cs.amplab.carat." { a[$2] += $NF }
$0 ~ "^Time spark.timeseries." { a[$2] += $NF }

END {
        for (k in a){
                name=k
                gsub("edu.berkeley.cs.amplab.carat.", "", name)
                gsub("spark.timeseries.", "", name)
                print a[k], name
        }
}

#!/usr/bin/awk -f

$1 ~ "^Time edu.berkeley.cs." { a[$1] += $2 }
$1 ~ "^Time spark.timeseries." { a[$1] += $2 }

END {
        for (k in a){
                name=k
                gsub("Time edu.berkeley.cs.amplab.carat", "", name)
                gsub("Time spark.timeseries.", "", name)
                print a[k], name
        }
}


Clarify printlist.awk

Clarify printlist.awk

# Format and print episode information from spreadsheets

BEGIN {
    FS = "\t"
}

# No processing on header and other lines unrelated to shows
/=HYPERLINK/ {
    split ($1,str,"\"")
    title = str[4] "\n"
    #
    duration = ""
    if ($2 && $2 !~ /^00:00/) {
        split ($2,fld,":")
        if (fld[1] != "00" ) {
            sub (/^0/,"",fld[1])
            duration = fld[1] "h "
        }
        sub (/^0/,"",fld[2])
        duration = duration fld[2] "m\n"
    }
    description = $3 "\n"
    print title duration description
}

# Format and print episode information from spreadsheets

BEGIN {
    FS = "\t"
}

# No processing on header and other lines unrelated to shows
/=HYPERLINK/ {
    split ($1,str,"\"")
    description = str[4] "\n"
    #
    if ($2 && $2 !~ /^00:00/) {
        split ($2,fld,":")
        if (fld[1] != "00" ) {
            sub (/^0/,"",fld[1])
            description = description fld[1] "h "
        }
        sub (/^0/,"",fld[2])
        description = description fld[2] "m\n"
    }
    description = description $3 "\n"
    print description
}


Probably going to rewrite in Python.  Stupid AWK and its lack of... sane language features.

BEGIN {
    split("", vanilla_fg_response_times);
    split("", vanilla_bg_total_throughputs);
    split("", intnw_fg_response_time);
    split("", intnw_bg_total_throughputs);

    split("", vanilla_avg_fg_chunksizes);
    split("", vanilla_avg_bg_chunksizes);
    split("", intnw_avg_fg_chunksizes);
    split("", intnw_avg_bg_chunksizes);

    current_type = "";
    current_label = "";
}

/Results for run [0-9]+/ {
    run_num = $4;

    # initialize data structures
    split("", vanilla_fg_sender_nums);
    split("", vanilla_bg_sender_nums);
    split("", intnw_fg_sender_nums);
    split("", intnw_bg_sender_nums);

    total_send_time = 0;
    send_count = 0;
    sender_num = 0;
    sender_count = 0;
}

# Worker PID 12345 - intnw foreground sender results
/Worker PID [0-9]+ - .+ .+ sender results/ {
    pid = $3;
    type = $5; # 'intnw' or 'vanilla'
    label = $6; # 'foreground' or 'background'

    #if it's not the first time...
    if (current_type != "" && sender_num == 0 || sender_num == pid) {
        # add to running average - depends on whether I'm FG or BG
        # TODO: pick up here.  write down what figures we need
        #       and then figure out how to compute them.
        # XXX: the array management and other code is really ugly.
        #  Might be better overall to rewrite in Python;
        #  the simplicity of AWK's text parsing doesn't appear
        #  to be worth the headaches.
        if (current_type == "vanilla") {
            if (current_label == "foreground") {
                vanilla_fg_response_times[run_num] += (total_send_time / send_count);
            } else if (current_label == "background") {
                vanilla_bg_throughputs[run_num] += ((send_count * current_chunksize) / total_send_time);
            }
        } else if (current_type == "intnw") {
            if (current_label == "foreground") {
                intnw_fg_response_times[run_num] += (total_send_time / send_count);
            } else if (current_label == "background") {
                intnw_bg_throughputs[run_num] += ((send_count * current_chunksize) / total_send_time);
            }
        }
    }

    if (sender_num != pid) {
        if (current_type == "vanilla") {
            if (current_label == "foreground") {
                vanilla_fg_response_times[run_num] /= sender_count;
            } else if (current_label == "background") {
                vanilla_bg_response_times[run_num] /= sender_count;
            }
        } else if (current_type == "intnw") {
            if (current_label == "foreground") {
                intnw_fg_response_times[run_num] /= sender_count;
            } else if (current_label == "background") {
                intnw_bg_response_times[run_num] /= sender_count;
            }
        }

        sender_count = 0;
    } else {
        sender_count += 1;
    }
    sender_num = pid;
    total_send_time = 0;
    send_count = 0;

    # initialize data structures
    current_type = type;
    current_label = label;

    if (type != "vanilla" && type != "intnw") {
        printf("SAW UNKNOWN TYPE %s\n", type);
    }
    if (label != "foreground" && label != "background") {
        printf("SAW UNKNOWN LABEL %s\n", label);
    }
}

/Chunksize: [0-9]+/ {
    chunksize = $2;
    current_chunksize = chunksize;
}

# Timestamp            Response time (sec)     Throughput (bytes/sec)
# 1278534838.454732          35.580977               7367.532376
/[0-9]+\.[0-9+]+ +[0-9]+\.[0-9+]+ +[0-9]+\.[0-9+]+/ {
    timestamp = $1;
    response_time = $2;

    total_send_time += response_time;
    send_count += 1;
}

# print everything as it is processed
{
    print;
}

function print_summary(results_array)
{
    cur_run = 0;
    cur_sender = 0;
    for (run in results_array) {
        split(run, indices, SUBSEP);
        if (
            printf("Run %d:\n", );
            cur_run = indices[i];
            sender_num = indices[i+1];

        }
    }

}

END {
    # print summary

}

BEGIN {
    split("", vanilla_avg_fg_response_times);
    split("", vanilla_avg_bg_total_throughputs);
    split("", intnw_avg_fg_response_time);
    split("", intnw_avg_bg_total_throughputs);

    split("", vanilla_avg_fg_chunksizes);
    split("", vanilla_avg_bg_chunksizes);
    split("", intnw_avg_fg_chunksizes);
    split("", intnw_avg_bg_chunksizes);

    current_type = "";
    current_label = "";
}

/Results for run [0-9]+/ {
    run_num = $4;

    # initialize data structures

    total_send_times = 0;
    send_count = 0;
}

# Worker PID 12345 - intnw foreground sender results
/Worker PID [0-9]+ - .+ .+ sender results/ {
    pid = $3;
    type = $5; # 'intnw' or 'vanilla'
    label = $6; # 'foreground' or 'background'

    #if it's not the first time...
    if (current_type != "") {
        # add to running average - depends on whether I'm FG or BG
        # TODO: pick up here.  write down what figures we need
        #       and then figure out how to compute them.
        if (current_type == "vanilla") {
            if (current_label == "foreground") {
                #vanilla_avg_fg_response_times[
            } else if (current_label == "background") {

            }
        } else if (current_type == "intnw") {
            if (current_label == "foreground") {

            } else if (current_label == "background") {

            }
        }
    }

    # initialize data structures
    current_type = type;
    current_label = label;

    if (type != "vanilla" && type != "intnw") {
        printf("SAW UNKNOWN TYPE %s\n", type);
    }
    if (label != "foreground" && label != "background") {
        printf("SAW UNKNOWN LABEL %s\n", label);
    }
}

/Chunksize: [0-9]+/ {
    chunksize = $2;
    current_chunksize = chunksize;
}

# Timestamp            Response time (sec)     Throughput (bytes/sec)
# 1278534838.454732          35.580977               7367.532376
/[0-9]+\.[0-9+]+ +[0-9]+\.[0-9+]+ +[0-9]+\.[0-9+]+/ {
    timestamp = $1;
    response_time = $2;

}

# print everything as it is processed
{
    print;
}

END {
    # print summary

}


Accounting: better datanodes filtering

Accounting: better datanodes filtering

#
# Parsing output of:
#
# hdfs dfsadmin -report
#

function dbstr(s) {
	if (s) { return "'" s "'" }
	else { return "NULL" }
}

function dbi(i) {
	if (i >= 0) { return i }
	else { return "NULL" }
}

function reset() {
	name="(none)";
	full=-1;
	disk=-1;
	disk_free=-1;
	disk_used=-1;
	block_under=-1;
	block_corrupt=-1;
	block_missing=-1;
#	cache=-1;
#	cache_used=-1;
#	cache_free=-1;
}

BEGIN {
	reset();
	name="all";
	state=-1;

	FS="[: ]+";
	CONVFMT="%d";

	print "INSERT INTO measure (name) VALUES ('hdfs');";
}

/^Live datanodes.*/		{state=1}
/^Dead datanodes.*/		{state=2}
/^Decommissioned .*/		{state=3}

/^Hostname:.*/			{name=$2}
/^Name:.*/			{ip=$2}
/^Configured Capacity:.*/	{full=$3}
/^Present Capacity:.*/		{disk=$3}
/^DFS Remaining:.*/		{disk_free=$3}
/^DFS Used:.*/			{disk_used=$3}
/^Under replicated blocks:.*/	{block_under=$4}
/^Blocks with corrupt replicas:.*/	{block_corrupt=$5}
/^Missing blocks:.*/		{block_missing=$3}
#/^Configured Cache Capacity:.*/{cache=$4}
#/^Cache Used:.*/		{cache_used=$3}
#/^Cache Remaining:.*/		{cache_free=$3}

/^$/ {
	if (name != "(none)" && state != 2) {
		print "INSERT INTO hdfs (id_measure, hostname, state, full, disk, disk_free, disk_used, block_under, block_corrupt, block_missing) VALUES (last_insert_id(), " dbstr(name) ", " dbi(state) ", " dbi(full) ", IFNULL(" dbi(disk) ", " dbi(disk_free) " + " dbi(disk_used) "), " dbi(disk_free) ", " dbi(disk_used) ", " dbi(block_under) ", " dbi(block_corrupt) ", " dbi(block_missing) ");";
	}
	reset()
}

#
# Parsing output of:
#
# hdfs dfsadmin -report
#

function dbstr(s) {
	if (s) { return "'" s "'" }
	else { return "NULL" }
}

function dbi(i) {
	if (i >= 0) { return i }
	else { return "NULL" }
}

function reset() {
	name="(none)";
	full=-1;
	disk=-1;
	disk_free=-1;
	disk_used=-1;
	block_under=-1;
	block_corrupt=-1;
	block_missing=-1;
#	cache=-1;
#	cache_used=-1;
#	cache_free=-1;
}

BEGIN {
	reset();
	name="all";
	state=-1;

	FS="[: ]+";
	CONVFMT="%d";

	print "INSERT INTO measure (name) VALUES ('hdfs');";
}

/^Live datanodes.*/		{state=1}
/^Dead datanodes.*/		{state=2}
/^Decommissioned .*/		{state=3}

/^Hostname:.*/			{name=$2}
/^Name:.*/			{ip=$2}
/^Configured Capacity:.*/	{full=$3}
/^Present Capacity:.*/		{disk=$3}
/^DFS Remaining:.*/		{disk_free=$3}
/^DFS Used:.*/			{disk_used=$3}
/^Under replicated blocks:.*/	{block_under=$4}
/^Blocks with corrupt replicas:.*/	{block_corrupt=$5}
/^Missing blocks:.*/		{block_missing=$3}
#/^Configured Cache Capacity:.*/{cache=$4}
#/^Cache Used:.*/		{cache_used=$3}
#/^Cache Remaining:.*/		{cache_free=$3}

/^$/ {
	if (name != "(none)" && ip !~ /^10\./) {
		print "INSERT INTO hdfs (id_measure, hostname, state, full, disk, disk_free, disk_used, block_under, block_corrupt, block_missing) VALUES (last_insert_id(), " dbstr(name) ", " dbi(state) ", " dbi(full) ", IFNULL(" dbi(disk) ", " dbi(disk_free) " + " dbi(disk_used) "), " dbi(disk_free) ", " dbi(disk_used) ", " dbi(block_under) ", " dbi(block_corrupt) ", " dbi(block_missing) ");";
	}
	reset()
}


Shorten header; eliminate useless pubDate

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

BEGIN {
    # Print header
    printf ("Content Type\tContent ID\tTitle\tEntity ID\tGenre\tRating\t")
    printf ("Show Type\tDuration\tDate Type\tOriginal Date\tShow ID\tSeason ID\t")
    printf ("Sn #\tEp #\tDescription\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType
    # print "contentId = " contentId
    # Make sure no fields will be carried over due to missing keys
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    # print "dateType = " dateType
    # print "originalDate = " originalDate
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId
    next
}

# Print spacer between items
/<\/item>/ {
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "EntityId = " EntityId
    }
    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "EntityId = " EntityId
    }
    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", contentType, contentId, \
            title, EntityId, genre, rating, showType, duration, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber, description)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='png'&Quality=45&ImageId='204656'&EntityType='Item'&EntityId='23740'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/episode/The_Last_Furlong_23740
# https://www.britbox.com/us/episode/24_Hours_In_Police_Custody_23576

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

BEGIN {
    # Print header
    printf ("contentType\tcontentId\tpubDate\ttitle\tEntityId\tgenre\trating\t")
    printf ("showType\tduration\tdateType\toriginalDate\tshowContentId\tseasonContentId\t")
    printf ("seasonNumber\tepisodeNumber\tdescription\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType
    # print "contentId = " contentId
    # Make sure no fields will be carried over due to missing keys
    pubDate = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
}

# Grab pubDate
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    split ($0,fld,"[<>]")
    pubDate = fld[3]
    sub (/T/," ",pubDate)
    sub (/Z/,"",pubDate)
    # print "pubDate = " pubDate
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    # print "dateType = " dateType
    # print "originalDate = " originalDate
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId
    next
}

# Print spacer between items
/<\/item>/ {
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "EntityId = " EntityId
    }
    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "EntityId = " EntityId
    }
    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", contentType, contentId, \
            pubDate, title, EntityId, genre, rating, showType, duration, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber, description)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='png'&Quality=45&ImageId='204656'&EntityType='Item'&EntityId='23740'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/episode/The_Last_Furlong_23740
# https://www.britbox.com/us/episode/24_Hours_In_Police_Custody_23576


increase specificity

increase specificity

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology they are from. (curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed local identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri,  delim, char, max, c, l){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_*|_*$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri,  u){
	u = uri
	# shorten till longest uri in curi map is found (or not)

	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123

function deoboify(curie,  a){
	split(curie, a , ":")
	if(match(a[2], a[1]"_") > 0){
		sub("_", ":", a[2]);
		curie=a[2]
	}
	return curie
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	prefix["https_archive_monarchinitiative_org_"]="MonarchArchive"
	############################################################
	# Often re-visit whether these exceptions are still necessary
	# they may have been moved into the curie prefix mapping file
	# or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in IMPC (not httpS)
	prefix["http://www.mousephenotype.org"]="IMPC"
	# note in curie_map IMPC is:
	# http://www.mousephenotype.org/data/genes/

	# in panther
	# prefix["http://identifiers.org/wormbase/"]="WormBase"
	# note in curie_map WormBase is
	# 'https://www.wormbase.org/get?name='

	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# note in curie_map dbSNP is
	# 'http://www.ncbi.nlm.nih.gov/projects/SNP/snp_ref.cgi?rs='

	# Ensembl  transient
	prefix["http://identifiers.org/ensembl/"]="ENSEMBL"
	# various, transient
    prefix["http://www.ncbi.nlm.nih.gov/gene/"]="NCBIGene"
	# kegg
	prefix["http://identifiers.org/ec-code/"]="EC"
	# ensembl
	prefix["http://identifiers.org/hgnc/HGNC:"]="HGNC"

	# playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
	# and their IRI are unroutable, i.e in same machine (or local net)
	# this is motivated in part by not wanting to publish
	# urls to our own servers which go nowhere.
	prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
	prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
	prefix["http://0.0.0.0/"]="LCL"	                       # ID halfway house

	# LCL's obvious down side is zero options for global collisions
	# best you can do is go the uuid which is ugly, or
	# belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*': 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology they are from. (curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed local identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri,  delim, char, max, c, l){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_*|_*$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri,  u){
	u = uri
	# shorten till longest uri in curi map is found (or not)

	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123

function deoboify(curie,  a){
	split(curie, a , ":")
	if(match(a[2], a[1]) > 0){
		sub("_", ":", a[2]);
		curie=a[2]
	}
	return curie
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	prefix["https_archive_monarchinitiative_org_"]="MonarchArchive"
	############################################################
	# Often re-visit whether these exceptions are still necessary
	# they may have been moved into the curie prefix mapping file
	# or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in IMPC (not httpS)
	prefix["http://www.mousephenotype.org"]="IMPC"
	# note in curie_map IMPC is:
	# http://www.mousephenotype.org/data/genes/

	# in panther
	# prefix["http://identifiers.org/wormbase/"]="WormBase"
	# note in curie_map WormBase is
	# 'https://www.wormbase.org/get?name='

	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# note in curie_map dbSNP is
	# 'http://www.ncbi.nlm.nih.gov/projects/SNP/snp_ref.cgi?rs='

	# Ensembl  transient
	prefix["http://identifiers.org/ensembl/"]="ENSEMBL"
	# various, transient
    prefix["http://www.ncbi.nlm.nih.gov/gene/"]="NCBIGene"
	# kegg
	prefix["http://identifiers.org/ec-code/"]="EC"
	# ensembl
	prefix["http://identifiers.org/hgnc/HGNC:"]="HGNC"

	# playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
	# and their IRI are unroutable, i.e in same machine (or local net)
	# this is motivated in part by not wanting to publish
	# urls to our own servers which go nowhere.
	prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
	prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
	prefix["http://0.0.0.0/"]="LCL"	                       # ID halfway house

	# LCL's obvious down side is zero options for global collisions
	# best you can do is go the uuid which is ugly, or
	# belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*': 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}


changes values rule to be pattern to match w/ dp

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action == \"validate\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
}

$1 == "option" {
    options[$2]=$3;
}

$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]
    cat=params[3]
    field_index=header_index[field]
    test=""
    msg=""

    if (rule_type == "type") {
        type=params[2]
        if (type == "numeric"){
            test=field " && !is_numeric(" field ")"
            msg="Field " field " in \" FILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        test=field " == \"\""
        msg="Field " field " in \" FILENAME \" line \" NR \" is required"
    } else if (rule_type == "unique") {
        test="!is_unique(" field ")"
        msg="Field " field " in \" FILENAME \" line \" NR \" is a duplicate and should be unique"
    } else if (rule_type ~ /^min|max$/) {
        comparator=$2 == "max" ? ">" : "<"
        limit=params[2]
        term=$2 == "max" ? "less" : "greater"

        test=field " " comparator " " limit
        msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
    } else if (rule_type == "pattern") {
        pattern=params[2]

        test=field " != \"\" && " field " !~ " pattern
        msg=field " in line \" NR \" should match the following pattern " pattern " and was \" " field " \" "
    }

    if (!options["mode"]) options["mode"] = "text"
    if (!options["dcat"]) options["dcat"] = "na"
    if (!cat) cat = options["dcat"]
    handler_prefix="{ log_err(\"" cat "\"); "
    if (options["mode"] == "text") {
        err_handler= handler_prefix "print \"" msg "\" RS $0 RS; }"
    } else if (options["mode"] == "wrap") {
        acc="$" field_index
        err_handler= handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"
    } else if (options["mode"] == "append") {
        err_handler= handler_prefix "print $0 FS \"" msg "\" }"
    }

    if (rule_type !~ /^(none|colmap)$/) { print "action == \"validate\" && NR > 1 && " test " " err_handler " " }
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]

    if (rule_type ~ /^(min|max)$/ || (rule_type == "type" && params[2] == "numeric")) {
        types[field] = "numeric"
    } else if (rule_type != "colmap") {
        types[field] = "text"
    }

    if(rule_type == "colmap") mappings[field]=params[2]
}

END {
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in types) {
        type=types[field]
        field_index=header_index[field]
        dval= type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    gsub(\"Range\", \"rangeno\");"
    print "    print \"COPY " options["table"] " (\" $0 \") FROM stdin;\""
    print "}"

    print "action == \"insert\" && NR > 1 {"
    print "    gsub(\",\", \"\t\");"
    print "    print;"
    print "}"

    print "action == \"table\" && NR == 1 {"

    fields=""
    l=length(headers)
    for (i in headers) {
        c++
        field=headers[i]
        type=types[field]
        if (length(mappings) && mappings[field]) field=mappings[field]
        fields=fields field " " type
        if (c < l) fields=fields ","
    }

    if(options["pkey"]) {
        cols=options["pkey"]
        gsub(":", ",", cols)
        fields=fields ", PRIMARY KEY (" cols ") "
    }
    if(options["fkey"]) {
        split(options["fkey"], ref, " ")
        ftable=ref[1]
        fcols=ref[2]
        gsub(":", ",", fcols)
        fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
    }

    print "     print \"CREATE TABLE IF NOT EXISTS " options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }" RS

    print "# la fin"
    print "END {"
    print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\") print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); "
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action == \"validate\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
}

$1 == "option" {
    options[$2]=$3;
}

$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]
    cat=params[3]
    field_index=header_index[field]
    test=""
    msg=""

    if (rule_type == "type") {
        type=params[2]
        if (type == "numeric"){
            test=field " && !is_numeric(" field ")"
            msg="Field " field " in \" FILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        test=field " == \"\""
        msg="Field " field " in \" FILENAME \" line \" NR \" is required"
    } else if (rule_type == "unique") {
        test="!is_unique(" field ")"
        msg="Field " field " in \" FILENAME \" line \" NR \" is a duplicate and should be unique"
    } else if (rule_type ~ /^min|max$/) {
        comparator=$2 == "max" ? ">" : "<"
        limit=params[2]
        term=$2 == "max" ? "less" : "greater"

        test=field " " comparator " " limit
        msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
    } else if (rule_type == "values") {
        values=params[2]

        test=field " != \"\" && " field " !~ /^(" values ")$/"
        msg=field " in line \" NR \" should be one of " values " and was \" " field " \" "
    }

    if (!options["mode"]) options["mode"] = "text"
    if (!options["dcat"]) options["dcat"] = "na"
    if (!cat) cat = options["dcat"]
    handler_prefix="{ log_err(\"" cat "\"); "
    if (options["mode"] == "text") {
        err_handler= handler_prefix "print \"" msg "\" RS $0 RS; }"
    } else if (options["mode"] == "wrap") {
        acc="$" field_index
        err_handler= handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"
    } else if (options["mode"] == "append") {
        err_handler= handler_prefix "print $0 FS \"" msg "\" }"
    }

    if (rule_type !~ /^(none|colmap)$/) { print "action == \"validate\" && NR > 1 && " test " " err_handler " " }
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]

    if (rule_type ~ /^(min|max)$/ || (rule_type == "type" && params[2] == "numeric")) {
        types[field] = "numeric"
    } else if (rule_type != "colmap") {
        types[field] = "text"
    }

    if(rule_type == "colmap") mappings[field]=params[2]
}

END {
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in types) {
        type=types[field]
        field_index=header_index[field]
        dval= type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    gsub(\"Range\", \"rangeno\");"
    print "    print \"COPY " options["table"] " (\" $0 \") FROM stdin;\""
    print "}"

    print "action == \"insert\" && NR > 1 {"
    print "    gsub(\",\", \"\t\");"
    print "    print;"
    print "}"

    print "action == \"table\" && NR == 1 {"

    fields=""
    l=length(headers)
    for (i in headers) {
        c++
        field=headers[i]
        type=types[field]
        if (length(mappings) && mappings[field]) field=mappings[field]
        fields=fields field " " type
        if (c < l) fields=fields ","
    }

    if(options["pkey"]) {
        cols=options["pkey"]
        gsub(":", ",", cols)
        fields=fields ", PRIMARY KEY (" cols ") "
    }
    if(options["fkey"]) {
        split(options["fkey"], ref, " ")
        ftable=ref[1]
        fcols=ref[2]
        gsub(":", ",", fcols)
        fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
    }

    print "     print \"CREATE TABLE IF NOT EXISTS " options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }" RS

    print "# la fin"
    print "END {"
    print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\") print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); "
    print "}"
}


let stuff trickle through, clean up and close last slide when open

BEGIN {
    heading = 0
}

hunt && /class="section_anchor">/ {
    hier[heading] = $0
    sub(/<\/a>/, "", hier[heading])
    sub(/<\/a>/, "", hier[heading])
    if (slide) print "
"
    print "
"
    slide = 1
    if (heading > 1) {
	print "
" hier[heading] "
"
    }
    for (i=1; i < heading; ++i) {
	print "<h" i " style='color: #C0C0C0;'>" hier[i] "</h" i ">"
    }
    print "<h" heading ">" hier[heading]
    hunt = 0
    next
}

/
/ {
    heading = 1
    hunt = 1
    next
}

/<\/h[1-9]>/ {
    heading = 0
    print
    next
}

/
/ && heading == 0 {
    heading = 2
    hunt = 1
    next
}

/
/ && heading == 0 {
    heading = 3
    hunt = 1
    next
}

/
/ && heading == 0 {
    heading = 4
    hunt = 1
    next
}

{ print }

END {
    if (slide) print "
"
}

BEGIN {
    heading = 0
}

hunt && /class="section_anchor">/ {
    hier[heading] = $0
    sub(/<\/a>/, "", hier[heading])
    sub(/<\/a>/, "", hier[heading])
    if (slide) print "
"
    print "
"
    slide = 1
    if (heading > 1) {
	print "
" hier[heading] "
"
    }
    for (i=1; i < heading; ++i) {
	print "<h" i " style='color: #C0C0C0;'>" hier[i] "</h" i ">"
    }
    print "<h" heading ">" hier[heading]
    hunt = 0
    next
}

/
/ {
    #print "
"
    heading = 1
    hunt = 1
    #print
    next
}

/<\/h[1-9]>/ {
    heading = 0
    print
    next
}

/
/ && heading == 0 {
    #print "
"
    #print "
"
    #print "
" hier[1] "
"
    heading = 2
    hunt = 1
    #print
    next
}

/
/ && heading == 0 {
    #print "
"
    #print "
"
    #print "
" hier[1] "
"
    #print "
" hier[2] "
"
    heading = 3
    hunt = 1
    #print
    next
}

/
/ && heading == 3 {
    heading = 4
    hunt = 1
    #print
    next
}