KNF the output.

KNF the output.

#! /usr/bin/awk -f
#	$OpenBSD: devlist2h.awk,v 1.2 2005/05/07 22:39:43 miod Exp $
#
# Copyright (c) 2003, Miodrag Vallat.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
BEGIN {
	header = 0
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)

	printf("/*\t\$OpenBSD\$\t*/\n\n")
	printf("/*\n")
	printf(" * THIS FILE AUTOMATICALLY GENERATED.  DO NOT EDIT.\n")
	printf(" *\n")
	printf(" * generated from:\n")
	printf(" *\t%s\n", VERSION)
	printf(" */\n")

	next
}
$1 == "keyboard" || $1 == "mouse" || $1 == "idmodule" {

	if (header == 0) {
		printf("const struct hildevice hildevs[] = {\n")
		header = 1
	}

	printf("\t{ 0x%s, 0x%s, HIL_DEVICE_%s, \"",
	    $2, $3, toupper($1))

	# description, with optional ``#''-prefixed comments
	i = 4
	f = i
	while (f <= NF) {
		if ($f == "#") {
			break
		}
		if (f > i)
			printf(" ")
		printf("%s", $f)
		f++
	}
	printf("\" },\n")

	next
}
{
	if ($0 == "")
		blanklines++
	if (blanklines < 2)
		print $0
}
END {
	printf("\t{ -1, -1, -1, NULL }\n")
	printf("};\n")
}

#! /usr/bin/awk -f
#	$OpenBSD: devlist2h.awk,v 1.1 2003/02/11 19:39:30 miod Exp $
#
# Copyright (c) 2003, Miodrag Vallat.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
BEGIN {
	header = 0
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)

	printf("/*\t\$OpenBSD\$\t*/\n\n")
	printf("/*\n")
	printf(" * THIS FILE AUTOMATICALLY GENERATED.  DO NOT EDIT.\n")
	printf(" *\n")
	printf(" * generated from:\n")
	printf(" *\t%s\n", VERSION)
	printf(" */\n")

	next
}
$1 == "keyboard" || $1 == "mouse" || $1 == "idmodule" {

	if (header == 0) {
		printf("const struct hildevice hildevs[] = {\n")
		header = 1
	}

	printf("\t{ 0x%s, 0x%s, HIL_DEVICE_%s, \"",
	    $2, $3, toupper($1))

	# description, with optional ``#''-prefixed comments
	i = 4
	f = i
	while (f <= NF) {
		if ($f == "#") {
			break
		}
		if (f > i)
			printf(" ")
		printf("%s", $f)
		f++
	}
	printf("\" },\n")

	next
}
{
	if ($0 == "")
		blanklines++
	if (blanklines < 2)
		print $0
}
END {
	printf("\t{ -1, -1, -1, NULL }")
	printf("};\n")
}


Better names

Better names

BEGIN {
    FS="[<>]"
}

# Eliminate unneeded constant fields
/^<item>/ {
    sub (/^<item>/,"")
    sub (/<\/item>$/,"")
    gsub (/<value>/,"")
    gsub (/<\/value>/,"")
}

/\/us\// {
    URL = $3
    showTitle = $7

    # Titles starting with "The" should not sort based on "The"
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
}

# <URL>/us/episode/Vera_S3_E1_13502</URL><Title>Vera</Title><Subtitle>Season 3, Castles in The Air</Subtitle>\
# <Duration>88 min</Duration><Year>2013</Year><Rating>TV-MA</Rating>
/\/us\/episode\// {
    episodeTitle = $11
    episodeDuration = $15
    episodeYear = $19
    episodeRating = $23
    episodeDescription = $27

    nfields = split (URL,fld,"_")
    seasonNumber = fld[nfields - 2]
    sub (/S/,"",seasonNumber)
    episodeNumber = fld[nfields - 1]
    sub (/E/,"",episodeNumber)

    # Get rid of perfunctory "Season <n>, " from episode title
    episodePrefix = "Season " seasonNumber ", "
    sub (episodePrefix, "", episodeTitle)

    if (episodeNumber != "") {
        printf \
            ("%s S%02dE%02d\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, S%02dE%02d, %s\"\)\t\t%s\t%s\t%s\t%s\n", \
             showTitle, seasonNumber, episodeNumber, URL, showTitle, seasonNumber, episodeNumber, \
             episodeTitle, episodeDuration, episodeYear, episodeRating, episodeDescription)
    }
}

# <URL>/us/season/Vera_S8_15720</URL><Title>Vera</Title><SeasonNumber>8</SeasonNumber>\
# <Year>2018</Year><NumEpisodes>4</NumEpisodes>
/\/us\/season\// {
    seasonNumber = $11
    seasonYear = $15
    seasonEpisodes = $19

    if (seasonNumber != "") {
        printf \
            ("%s S%02d\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, Season %d\"\)\t%s\t\t%s\t%s\t%s\n", \
             showTitle, seasonNumber, URL, showTitle, seasonNumber, \
             seasonEpisodes, seasonYear, seasonRating, seasonDescription)
    }
}

BEGIN {
    FS="[<>]"
}

# Eliminate unneeded constant fields
/^<item>/ {
    sub (/^<item>/,"")
    sub (/<\/item>$/,"")
    gsub (/<value>/,"")
    gsub (/<\/value>/,"")
}

/\/us\// {
    URL = $3
    showTitle = $7
    subtitle = $11

    # Titles starting with "The" should not sort based on "The"
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
}

# <URL>/us/episode/Vera_S3_E1_13502</URL><Title>Vera</Title><Subtitle>Season 3, Castles in The Air</Subtitle>\
# <Duration>88 min</Duration><Year>2013</Year><Rating>TV-MA</Rating>
/\/us\/episode\// {
    episodeDuration = $15
    episodeYear = $19
    episodeRating = $23
    episodeDescription = $27

    nfields = split (URL,fld,"_")
    seasonNumber = fld[nfields - 2]
    sub (/S/,"",seasonNumber)
    episodeNumber = fld[nfields - 1]
    sub (/E/,"",episodeNumber)

    # Get rid of perfunctory "Season <n>, " from episode title
    episodePrefix = "Season " seasonNumber ", "
    sub (episodePrefix, "", subtitle)

    if (episodeNumber != "") {
        printf \
            ("%s S%02dE%02d\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, S%02dE%02d, %s\"\)\t\t%s\t%s\t%s\t%s\n", \
             showTitle, seasonNumber, episodeNumber, URL, showTitle, seasonNumber, episodeNumber, \
             subtitle, episodeDuration, episodeYear, episodeRating, episodeDescription)
    }
}

# <URL>/us/season/Vera_S8_15720</URL><Title>Vera</Title><SeasonNumber>8</SeasonNumber>\
# <Year>2018</Year><NumEpisodes>4</NumEpisodes>
/\/us\/season\// {
    seasonYear = $15
    seasonEpisodes = $19

    nfields = split (URL,fld,"_")
    seasonNumber = fld[nfields - 1]
    sub (/S/,"",seasonNumber)

    if (seasonNumber != "") {
        printf \
            ("%s S%02d\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, Season %d\"\)\t%s\t\t%s\t%s\t%s\n", \
             showTitle, seasonNumber, URL, showTitle, seasonNumber, \
             seasonEpisodes, seasonYear, seasonRating, seasonDescription)
    }
}


Theme: add comments

Theme: add comments

####################################################################
# Theme.awk                                                        #
####################################################################

# Prettify a string using corresponding SGR option.
function prettify(name, string,    i, temp) {
    temp = string
    if ("sgr-" name in Option)
        if (isarray(Option["sgr-" name]))
            for (i in Option["sgr-" name])
                temp = ansi(Option["sgr-" name][i], temp)
        else
            temp = ansi(Option["sgr-" name], temp)
    return temp
}

# Return a string representing a random color.
function randomColor(    i) {
    i = int(5 * rand())
    switch (i) {
    case 0: return "green"
    case 1: return "yellow"
    case 2: return "blue"
    case 3: return "magenta"
    case 4: return "cyan"
    default: return "default"
    }
}

# Set theme to: random.
function setRandomTheme(    i, n, temp) {
    srand(systime())
    for (i = 0; i < 3; i++) {
        do temp = randomColor(); while (belongsTo(temp, n))
        n[i] = temp
    }

    Option["sgr-prompt-message"] = Option["sgr-languages"] = n[0]
    Option["sgr-original-dictionary-detailed-word-class"][1] = n[0]
    Option["sgr-original-dictionary-detailed-word-class"][2] = "bold"
    Option["sgr-original-dictionary-synonyms"] = n[0]
    Option["sgr-original-dictionary-synonyms-word-class"][1] = n[0]
    Option["sgr-original-dictionary-synonyms-word-class"][2] = "bold"
    Option["sgr-original-dictionary-examples"] = n[0]
    Option["sgr-original-dictionary-see-also"] = n[0]
    Option["sgr-dictionary-word-class"][1] = n[0]
    Option["sgr-dictionary-word-class"][2] = "bold"

    Option["sgr-original"][1] = Option["sgr-original-phonetics"][1] = n[1]
    Option["sgr-original"][2] = Option["sgr-original-phonetics"][2] = "bold"
    Option["sgr-prompt-message-original"][1] = n[1]
    Option["sgr-prompt-message-original"][2] = "bold"
    Option["sgr-languages-sl"] = n[1]
    Option["sgr-original-dictionary-detailed-explanation"][1] = n[1]
    Option["sgr-original-dictionary-detailed-explanation"][2] = "bold"
    Option["sgr-original-dictionary-detailed-example"] = n[1]
    Option["sgr-original-dictionary-detailed-synonyms"] = n[1]
    Option["sgr-original-dictionary-detailed-synonyms-item"][1] = n[1]
    Option["sgr-original-dictionary-detailed-synonyms-item"][2] = "bold"
    Option["sgr-original-dictionary-synonyms-synonyms"] = n[1]
    Option["sgr-original-dictionary-synonyms-synonyms-item"][1] = n[1]
    Option["sgr-original-dictionary-synonyms-synonyms-item"][2] = "bold"
    Option["sgr-original-dictionary-examples-example"] = n[1]
    Option["sgr-original-dictionary-examples-original"][1] = n[1]
    Option["sgr-original-dictionary-examples-original"][2] = "bold"
    Option["sgr-original-dictionary-examples-original"][3] = "underline"
    Option["sgr-original-dictionary-see-also-phrases"] = n[1]
    Option["sgr-original-dictionary-see-also-phrases-item"][1] = n[1]
    Option["sgr-original-dictionary-see-also-phrases-item"][2] = "bold"
    Option["sgr-dictionary-explanation"] = n[1]
    Option["sgr-dictionary-explanation-item"][1] = n[1]
    Option["sgr-dictionary-explanation-item"][2] = "bold"
    Option["sgr-alternatives-original"][1] = n[1]
    Option["sgr-alternatives-original"][2] = "bold"

    Option["sgr-translation"][1] = Option["sgr-translation-phonetics"][1] = n[2]
    Option["sgr-translation"][2] = Option["sgr-translation-phonetics"][2] = "bold"
    Option["sgr-languages-tl"] = n[2]
    Option["sgr-dictionary-word"][1] = n[2]
    Option["sgr-dictionary-word"][2] = "bold"
    Option["sgr-alternatives-translations"] = n[2]
    Option["sgr-alternatives-translations-item"][1] = n[2]
    Option["sgr-alternatives-translations-item"][2] = "bold"
    Option["sgr-brief-translation"][1] = Option["sgr-brief-translation-phonetics"][1] = n[2]
    Option["sgr-brief-translation"][2] = Option["sgr-brief-translation-phonetics"][2] = "bold"
}

# Set theme to: default.
function setDefaultTheme() {
    Option["sgr-translation"] = Option["sgr-translation-phonetics"] = "bold"
    Option["sgr-prompt-message-original"] = "underline"
    Option["sgr-languages-sl"] = "underline"
    Option["sgr-languages-tl"] = "bold"
    Option["sgr-original-dictionary-detailed-explanation"] = "bold"
    Option["sgr-original-dictionary-detailed-synonyms-item"] = "bold"
    Option["sgr-original-dictionary-synonyms-synonyms-item"] = "bold"
    Option["sgr-original-dictionary-examples-original"][1] = "bold"
    Option["sgr-original-dictionary-examples-original"][2] = "underline"
    Option["sgr-original-dictionary-see-also-phrases-item"] = "bold"
    Option["sgr-dictionary-word"] = "bold"
    Option["sgr-alternatives-original"] = "underline"
    Option["sgr-alternatives-translations-item"] = "bold"
}

# Set theme using corresponding option.
function setTheme(    file, line, script) {
    if (Option["theme"] && Option["theme"] != "default") {
        file = Option["theme"]
        if (!fileExists(file))
            file = ENVIRON["HOME"] "/.translate-shell/" Option["theme"]
    }

    if (file && fileExists(file)) {
        # Read from theme file
        script = NULLSTR
        while (getline line < file)
            script = script "\n" line
        loadOptions(script)
    } else if (Option["theme"] == "none")
        ;# skip
    else if (Option["theme"] == "random")
        setRandomTheme()
    else
        setDefaultTheme()
}

####################################################################
# Theme.awk                                                        #
####################################################################

function prettify(name, string,    i, temp) {
    temp = string
    if ("sgr-" name in Option)
        if (isarray(Option["sgr-" name]))
            for (i in Option["sgr-" name])
                temp = ansi(Option["sgr-" name][i], temp)
        else
            temp = ansi(Option["sgr-" name], temp)
    return temp
}

function randomColor(    i) {
    i = int(5 * rand())
    switch (i) {
    case 0: return "green"
    case 1: return "yellow"
    case 2: return "blue"
    case 3: return "magenta"
    case 4: return "cyan"
    default: return "default"
    }
}

function setRandomTheme(    i, n, temp) {
    srand(systime())
    for (i = 0; i < 3; i++) {
        do temp = randomColor(); while (belongsTo(temp, n))
        n[i] = temp
    }

    Option["sgr-prompt-message"] = Option["sgr-languages"] = n[0]
    Option["sgr-original-dictionary-detailed-word-class"][1] = n[0]
    Option["sgr-original-dictionary-detailed-word-class"][2] = "bold"
    Option["sgr-original-dictionary-synonyms"] = n[0]
    Option["sgr-original-dictionary-synonyms-word-class"][1] = n[0]
    Option["sgr-original-dictionary-synonyms-word-class"][2] = "bold"
    Option["sgr-original-dictionary-examples"] = n[0]
    Option["sgr-original-dictionary-see-also"] = n[0]
    Option["sgr-dictionary-word-class"][1] = n[0]
    Option["sgr-dictionary-word-class"][2] = "bold"

    Option["sgr-original"][1] = Option["sgr-original-phonetics"][1] = n[1]
    Option["sgr-original"][2] = Option["sgr-original-phonetics"][2] = "bold"
    Option["sgr-prompt-message-original"][1] = n[1]
    Option["sgr-prompt-message-original"][2] = "bold"
    Option["sgr-languages-sl"] = n[1]
    Option["sgr-original-dictionary-detailed-explanation"][1] = n[1]
    Option["sgr-original-dictionary-detailed-explanation"][2] = "bold"
    Option["sgr-original-dictionary-detailed-example"] = n[1]
    Option["sgr-original-dictionary-detailed-synonyms"] = n[1]
    Option["sgr-original-dictionary-detailed-synonyms-item"][1] = n[1]
    Option["sgr-original-dictionary-detailed-synonyms-item"][2] = "bold"
    Option["sgr-original-dictionary-synonyms-synonyms"] = n[1]
    Option["sgr-original-dictionary-synonyms-synonyms-item"][1] = n[1]
    Option["sgr-original-dictionary-synonyms-synonyms-item"][2] = "bold"
    Option["sgr-original-dictionary-examples-example"] = n[1]
    Option["sgr-original-dictionary-examples-original"][1] = n[1]
    Option["sgr-original-dictionary-examples-original"][2] = "bold"
    Option["sgr-original-dictionary-examples-original"][3] = "underline"
    Option["sgr-original-dictionary-see-also-phrases"] = n[1]
    Option["sgr-original-dictionary-see-also-phrases-item"][1] = n[1]
    Option["sgr-original-dictionary-see-also-phrases-item"][2] = "bold"
    Option["sgr-dictionary-explanation"] = n[1]
    Option["sgr-dictionary-explanation-item"][1] = n[1]
    Option["sgr-dictionary-explanation-item"][2] = "bold"
    Option["sgr-alternatives-original"][1] = n[1]
    Option["sgr-alternatives-original"][2] = "bold"

    Option["sgr-translation"][1] = Option["sgr-translation-phonetics"][1] = n[2]
    Option["sgr-translation"][2] = Option["sgr-translation-phonetics"][2] = "bold"
    Option["sgr-languages-tl"] = n[2]
    Option["sgr-dictionary-word"][1] = n[2]
    Option["sgr-dictionary-word"][2] = "bold"
    Option["sgr-alternatives-translations"] = n[2]
    Option["sgr-alternatives-translations-item"][1] = n[2]
    Option["sgr-alternatives-translations-item"][2] = "bold"
    Option["sgr-brief-translation"][1] = Option["sgr-brief-translation-phonetics"][1] = n[2]
    Option["sgr-brief-translation"][2] = Option["sgr-brief-translation-phonetics"][2] = "bold"
}

function setDefaultTheme() {
    Option["sgr-translation"] = Option["sgr-translation-phonetics"] = "bold"
    Option["sgr-prompt-message-original"] = "underline"
    Option["sgr-languages-sl"] = "underline"
    Option["sgr-languages-tl"] = "bold"
    Option["sgr-original-dictionary-detailed-explanation"] = "bold"
    Option["sgr-original-dictionary-detailed-synonyms-item"] = "bold"
    Option["sgr-original-dictionary-synonyms-synonyms-item"] = "bold"
    Option["sgr-original-dictionary-examples-original"][1] = "bold"
    Option["sgr-original-dictionary-examples-original"][2] = "underline"
    Option["sgr-original-dictionary-see-also-phrases-item"] = "bold"
    Option["sgr-dictionary-word"] = "bold"
    Option["sgr-alternatives-original"] = "underline"
    Option["sgr-alternatives-translations-item"] = "bold"
}

function setTheme(    file, line, script) {
    if (Option["theme"] && Option["theme"] != "default") {
        file = Option["theme"]
        if (!fileExists(file))
            file = ENVIRON["HOME"] "/.translate-shell/" Option["theme"]
    }

    if (file && fileExists(file)) {
        # Read from theme file
        script = NULLSTR
        while (getline line < file)
            script = script "\n" line
        loadOptions(script)
    } else if (Option["theme"] == "none")
        ;# skip
    else if (Option["theme"] == "random")
        setRandomTheme()
    else
        setDefaultTheme()
}


Utils: parameterize data in curlPost()

Utils: parameterize data in curlPost()

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev" SUPERR
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev" SUPERR "| sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    }

    if (ENVIRON["TRANS_ENTRY"]) {
        command = Rlwrap " " ENVIRON["TRANS_ENTRY"] " "                 \
            parameterize("-no-rlwrap") # never fork rlwrap again!
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        command = Rlwrap " sh "                                         \
            parameterize(ENVIRON["TRANS_DIR"] "/" EntryScript)          \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            command = command " " parameterize(ARGV[i])

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    }

    params = ""
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            params = params " " parameterize(ARGV[i], "\"")
    if (ENVIRON["TRANS_ENTRY"]) {
        el = "(progn (setq explicit-shell-file-name \"" ENVIRON["TRANS_ENTRY"] "\") " \
            "(setq explicit-" Command "-args '(\"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        el = "(progn (setq explicit-shell-file-name \"" "sh" "\") "     \
            "(setq explicit-" "sh" "-args '(\"" ENVIRON["TRANS_DIR"] "/" EntryScript "\" \"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    command = Emacs " --eval " parameterize(el)

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Fetch the content of a URL. Return a null string if failed.
function curlPost(url, data, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " --request POST --data " parameterize(data)
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    # hexdump tricks:
    # (1) use -v (--no-squeezing)
    # (2) use "%u" (unsigned integers)
    command = "hexdump" " -v -e'1/1 \"%03u\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

# Base64 encode a string.
# NOTE: can only be ran once for each text! Build a cache.
function base64(text,    command, temp) {
    command = "base64"
    ("echo -n " parameterize(text) PIPE command) | getline temp
    return temp
}

# Print a Unicode-escaped string. (GNU only)
# FIXME: find a BSD/macOS equivalent
# NOTE: can only be ran once for each text! Build a cache.
function uprintf(text,    command, temp) {
    command = "printf"
    (command " -- " parameterize(text) | getline temp)
    return temp
}

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev" SUPERR
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev" SUPERR "| sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    }

    if (ENVIRON["TRANS_ENTRY"]) {
        command = Rlwrap " " ENVIRON["TRANS_ENTRY"] " "                 \
            parameterize("-no-rlwrap") # never fork rlwrap again!
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        command = Rlwrap " sh "                                         \
            parameterize(ENVIRON["TRANS_DIR"] "/" EntryScript)          \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            command = command " " parameterize(ARGV[i])

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    }

    params = ""
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            params = params " " parameterize(ARGV[i], "\"")
    if (ENVIRON["TRANS_ENTRY"]) {
        el = "(progn (setq explicit-shell-file-name \"" ENVIRON["TRANS_ENTRY"] "\") " \
            "(setq explicit-" Command "-args '(\"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        el = "(progn (setq explicit-shell-file-name \"" "sh" "\") "     \
            "(setq explicit-" "sh" "-args '(\"" ENVIRON["TRANS_DIR"] "/" EntryScript "\" \"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    command = Emacs " --eval " parameterize(el)

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Fetch the content of a URL. Return a null string if failed.
function curlPost(url, data, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " --request POST --data " data
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    # hexdump tricks:
    # (1) use -v (--no-squeezing)
    # (2) use "%u" (unsigned integers)
    command = "hexdump" " -v -e'1/1 \"%03u\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

# Base64 encode a string.
# NOTE: can only be ran once for each text! Build a cache.
function base64(text,    command, temp) {
    command = "base64"
    ("echo -n " parameterize(text) PIPE command) | getline temp
    return temp
}

# Print a Unicode-escaped string. (GNU only)
# FIXME: find a BSD/macOS equivalent
# NOTE: can only be ran once for each text! Build a cache.
function uprintf(text,    command, temp) {
    command = "printf"
    (command " -- " parameterize(text) | getline temp)
    return temp
}


Commons: add PIPE

Commons: add PIPE

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used constants and functions.                           #
####################################################################

# Initialize constants.
function initConst() {
    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "
}

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, "", line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    if (length(array)) {
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != "")
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array # array is empty string

    return temp
}

# Return 0 if the string starts with '0', 'f' or 'n';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^[0fn]/) ? 0 : 1
}

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return non-zero if file exists, otherwise return 0.
function fileExists(file) {
    return !system("test -f " file)
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

BEGIN {
    initConst()

    initUrlEncoding()
    initAnsiCode()
    initUriSchemes()
}

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used constants and functions.                           #
####################################################################

# Initialize constants.
function initConst() {
    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
}

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, "", line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    if (length(array)) {
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != "")
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array # array is empty string

    return temp
}

# Return 0 if the string starts with '0', 'f' or 'n';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^[0fn]/) ? 0 : 1
}

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return non-zero if file exists, otherwise return 0.
function fileExists(file) {
    return !system("test -f " file)
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

BEGIN {
    initConst()

    initUrlEncoding()
    initAnsiCode()
    initUriSchemes()
}


The bindings should include the BRLAPI_ERROR_... constants. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@3382 91a5dbb7-01b9-0310-9b5f-b28072856b6e

###############################################################################
# BRLTTY - A background process providing access to the console screen (when in
#          text mode) for a blind person using a refreshable braille display.
#
# Copyright (C) 1995-2007 by The BRLTTY Developers.
#
# BRLTTY comes with ABSOLUTELY NO WARRANTY.
#
# This is free software, placed under the terms of the
# GNU General Public License, as published by the Free Software
# Foundation.  Please see the file COPYING for details.
#
# Web Page: http://mielke.cc/brltty/
#
# This software is maintained by Dave Mielke <dave@mielke.cc>.
###############################################################################

BEGIN {
  apiRangeTypeCount = 0
}

/#define[ \t]*BRLAPI_(CURSOR|DISPLAY|ERROR|TTY)_/ {
  apiConstant(substr($2, 8), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_MAX/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_MASK/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_SHIFT/ {
  apiShift(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_TYPE_/ {
  apiType(substr($2, 17), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_SYM_/ {
  apiKey(substr($2, 16), $2, getDefineValue(), "")
  next
}

/^ *brlapi_rangeType_/ {
  gsub(",", "", $1)
  apiRangeType(substr($1, 18), $1, apiRangeTypeCount++, "")
  next
}

###############################################################################
# BRLTTY - A background process providing access to the console screen (when in
#          text mode) for a blind person using a refreshable braille display.
#
# Copyright (C) 1995-2007 by The BRLTTY Developers.
#
# BRLTTY comes with ABSOLUTELY NO WARRANTY.
#
# This is free software, placed under the terms of the
# GNU General Public License, as published by the Free Software
# Foundation.  Please see the file COPYING for details.
#
# Web Page: http://mielke.cc/brltty/
#
# This software is maintained by Dave Mielke <dave@mielke.cc>.
###############################################################################

BEGIN {
  apiRangeTypeCount = 0
}

/#define[ \t]*BRLAPI_(CURSOR|DISPLAY|TTY)_/ {
  apiConstant(substr($2, 8), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_MAX/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_MASK/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_SHIFT/ {
  apiShift(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_TYPE_/ {
  apiType(substr($2, 17), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_SYM_/ {
  apiKey(substr($2, 16), $2, getDefineValue(), "")
  next
}

/^ *brlapi_rangeType_/ {
  gsub(",", "", $1)
  apiRangeType(substr($1, 18), $1, apiRangeTypeCount++, "")
  next
}


Update isNumber regex to be more accurate

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_integer(x) { return x ~ /^-?[0-9]+$/ }"
    print "function is_number(x) { return x ~ /^-?[0-9]+(\.[0-9]+)?$/ }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print  NR FS CSVFILENAME FS \"header\" FS \"invalid-header\" FS \"error\" FS  \"headers are invalid \"; exit 0;} }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "violation"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type == "integer") {
            cat       = "error"
            test      = field " && !is_integer(" field ")"
            mini_msg  = field " should be an integer"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be an integer but was \" " field " \" "
        } else if (type == "number") {
            cat       = "error"
            test      = field " && !is_number(" field ")"
            mini_msg  = field " should be a decimal number"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a decimal number but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = field " value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = field " value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = field " value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = field " value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = field " value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = field " max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        } else if (options["mode"] == "violation") {
            key=file_options["pkey"]
            gsub(/\|/, " \"-\" ", key)
            err_handler = handler_prefix "print  NR FS CSVFILENAME FS " key " FS \"" rule_type "\" FS \"" cat "\" FS  \"" mini_msg "\" FS \"" field "\"}"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    print \"COPY \" schema \"" file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    print "    if (action == \"insert\") print \"\\\\.\" RS"
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_integer(x) { return x ~ /^-?[0-9]+$/ }"
    print "function is_number(x) { return x ~ /^-?[0-9]+\.[0-9]+$/ }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print  NR FS CSVFILENAME FS \"header\" FS \"invalid-header\" FS \"error\" FS  \"headers are invalid \"; exit 0;} }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "violation"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type == "integer") {
            cat       = "error"
            test      = field " && !is_integer(" field ")"
            mini_msg  = field " should be an integer"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be an integer but was \" " field " \" "
        } else if (type == "number") {
            cat       = "error"
            test      = field " && !is_number(" field ")"
            mini_msg  = field " should be a decimal number"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a decimal number but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = field " value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = field " value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = field " value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = field " value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = field " value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = field " max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        } else if (options["mode"] == "violation") {
            key=file_options["pkey"]
            gsub(/\|/, " \"-\" ", key)
            err_handler = handler_prefix "print  NR FS CSVFILENAME FS " key " FS \"" rule_type "\" FS \"" cat "\" FS  \"" mini_msg "\" FS \"" field "\"}"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    print \"COPY \" schema \"" file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    print "    if (action == \"insert\") print \"\\\\.\" RS"
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


Refactor BritBox season processing

Refactor BritBox season processing

# Print only the valid "Seasons" lines from a WebScraper csv file saved in tsv format

# Field numbers
#    1 web-scraper-order  2 web-scraper-start-url  3 PgmURL            4 PgmURL-href       5 SsnURL
#    6 SsnURL-href        7 EpiURL                 8 EpiURL-href       9 Program_Title    10 Sn_Title
#   11 Sn_Years           12 Sn_Epis              13 Sn_Description   14 More_Description

BEGIN {
    FS="\t"
    print "Sortkey\tURL\tYears\tEpisodes\tDescription"
}

# Only print the preceding line if the current line has a different Program_Title
# or the preceding line had a non-blank SsnURL-href
NR > 1 {
    if (oldProgram_Title != $9 || oldSsnURL_href != "")
        print savedLine
}

# Save fields from the current line so they can be printed after checking the following line
{
    # Raw fields for comparison
    oldSsnURL_href = $6
    oldProgram_Title = $9

    # Fields to print from current line
    sortkey = NR
    $6  == "" ? URL = $4 : URL = $6
    showTitle = $9
    $10 == "" ? seasonTitle = "Season 1" : seasonTitle = $10
    Years = $11
    Episodes = $12
    $14 != "" ? Description = $14 : Description = $13
    savedLine = sprintf \
        ("%s - %s\t=HYPERLINK(\"%s\";\"%s, %s\"\)\t%s\t%s\t%s",\
         showTitle, sortkey, URL, showTitle, seasonTitle, Years, Episodes, Description)

    next
}

END {
    print savedLine
}

# Print only the valid "Seasons" lines from a WebScraper csv file saved in tsv format

# Field numbers
#    1 web-scraper-order  2 web-scraper-start-url  3 PgmURL            4 PgmURL-href       5 SsnURL
#    6 SsnURL-href        7 EpiURL                 8 EpiURL-href       9 Program_Title    10 Sn_Title
#   11 Sn_Years           12 Sn_Epis              13 Sn_Description   14 More_Description

BEGIN {
    FS="\t"
}

# Only print the preceding line if the current line has a new Program_Title
# or the preceding line had a non-blank SsnURL-href
NR > 1 {
    SsnURL_href =$6
    Program_Title = $9
    if (Program_Title != oldProgram_Title || oldSsnURL_href != "")
        print oldline
}

# Always save the current line so it can be printed after chaecking the following line
{
    oldline = $0
    oldSsnURL_href = $6
    oldProgram_Title = $9
    next
}

END {
    print NR-1 " - "oldline
}


r2795/sofa-dev : WARNING: Removing binary dependencies and headers from SVN. Please use system-specific packages available at https://gforge.inria.fr/frs/?group_id=483
This implements task 2610.

 //partial sync (1/22 changes)//

#!/usr/bin/awk -f
# This script removes parts flagged by SOFA_DEV inside C source code files
# The file with the relevant parts removed is output to the standard output.

# If you never used awk, a good introduction is available at http://www.cs.hmc.edu/qref/awk.html

# init
BEGIN {
    # indicate whether the current code is filtered of not
    filtered=0;
    # indicate the number if impricated if seen so far.
    # The if conditions influence on the filtering will be stored in the ifcond array
    # i.e. if the outmost if is 'ifdef SOFA_DEV', it
    nbif=0;
}

# match preprocessor condition activating filtering
$1~/^#ifdef/ && $2=="SOFA_DEV" {
    filtered=1;
    nbif++;
    ifcond[nbif]=1;
    next;
}

# match preprocessor condition desactivating filtering
$1~/^#ifndef/ && $2=="SOFA_DEV" {
    nbif++;
    if (filtered==0)
	ifcond[nbif]=-1;
    else
	ifcond[nbif]=0; # if we are already filtered, this if has no influence
    next;
}

# match other preprocessor condition
$1~/^#if/ {
    nbif++;
    ifcond[nbif]=0;
}

# match else deactivating filtering
$1~/^#else/ && ifcond[nbif]==1 {
    filtered=0;
    ifcond[nbif]=-1;
    next;
}

# match else activating filtering
$1~/^#else/ && ifcond[nbif]==-1 {
    filtered=1;
    ifcond[nbif]=1;
    next;
}

# match other else
$1~/^#else/ && ifcond[nbif]==0 {
}

# match other else
$1~/^#elif/ && ifcond[nbif]==0 {
}

# match endif deactivating filtering
$1~/^#endif/ && ifcond[nbif]==1 {
    filtered=0;
    delete ifcond[nbif];
    nbif--;
    next;
}

# match endif activating filtering
$1~/^#endif/ && ifcond[nbif]==-1 {
    delete ifcond[nbif];
    nbif--;
    next;
}

# match other endif
$1~/^#endif/ && ifcond[nbif]==0 {
    delete ifcond[nbif];
    nbif--;
}

# debug print
#{ print filtered "(" nbif "): " $0; }

# non-filtered line: simply print the line
filtered==0 { print; }

#!/usr/bin/awk -f
# This script removes parts flagged by SOFA_DEV inside C source code files
# The file with the relevant parts removed is output to the standard output,
# while files mentionned inside removed parts are output to the standard error.
# This can be used to remove then from the repository by svn rm'ing them.

# If you never used awk, a good introduction is available at http://www.cs.hmc.edu/qref/awk.html

# init
BEGIN {
    # indicate whether the current code is filtered of not
    filtered=0;
    # indicate the number if impricated if seen so far.
    # The if conditions influence on the filtering will be stored in the ifcond array
    # i.e. if the outmost if is 'ifdef SOFA_DEV', it
    nbif=0;
}

# match preprocessor condition activating filtering
$1~/^#ifdef/ && $2=="SOFA_DEV" {
    filtered=1;
    nbif++;
    ifcond[nbif]=1;
    next;
}

# match preprocessor condition desactivating filtering
$1~/^#ifndef/ && $2=="SOFA_DEV" {
    nbif++;
    if (filtered==0)
	ifcond[nbif]=-1;
    else
	ifcond[nbif]=0; # if we are already filtered, this if has no influence
    next;
}

# match other preprocessor condition
$1~/^#if/ {
    nbif++;
    ifcond[nbif]=0;
}

# match else deactivating filtering
$1~/^#else/ && ifcond[nbif]==1 {
    filtered=0;
    ifcond[nbif]=-1;
    next;
}

# match else activating filtering
$1~/^#else/ && ifcond[nbif]==-1 {
    filtered=1;
    ifcond[nbif]=1;
    next;
}

# match other else
$1~/^#else/ && ifcond[nbif]==0 {
}

# match other else
$1~/^#elif/ && ifcond[nbif]==0 {
}

# match endif deactivating filtering
$1~/^#endif/ && ifcond[nbif]==1 {
    filtered=0;
    delete ifcond[nbif];
    nbif--;
    next;
}

# match endif activating filtering
$1~/^#endif/ && ifcond[nbif]==-1 {
    delete ifcond[nbif];
    nbif--;
    next;
}

# match other endif
$1~/^#endif/ && ifcond[nbif]==0 {
    delete ifcond[nbif];
    nbif--;
}

# debug print
#{ print filtered "(" nbif "): " $0; }

# non-filtered line: simply print the line
filtered==0 { print; }


PLParser: header comment

PLParser: header comment

####################################################################
# PLParser.awk                                                     #
####################################################################

# Parse a list of tokens and return an AST.
function plParse(returnAST, tokens,
                 leftBrackets,
                 rightBrackets,
                 separators,
                 ####
                 i, j, key, p, stack, token) {
    # Default parameters
    if (!leftBrackets[0]) {
        leftBrackets[0] = "(" # left parenthesis
        leftBrackets[1] = "[" # left bracket
        leftBrackets[2] = "{" # left brace
    }
    if (!rightBrackets[0]) {
        rightBrackets[0] = ")" # right parenthesis
        rightBrackets[1] = "]" # right bracket
        rightBrackets[2] = "}" # right brace
    }
    if (!separators[0]) {
        separators[0] = "," # comma
    }

    stack[p = 0] = 0
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, leftBrackets))
            stack[++p] = 0
        else if (belongsTo(token, rightBrackets))
            --p
        else if (belongsTo(token, separators))
            stack[p]++
        else {
            key = stack[0]
            for (j = 1; j <= p; j++)
                key = key SUBSEP stack[j]
            returnAST[key] = token
        }
    }
}

#!/usr/bin/gawk -f

# Parse a list of tokens and return an AST.
function plParse(returnAST, tokens,
                 leftBrackets,
                 rightBrackets,
                 separators,
                 ####
                 i, j, key, p, stack, token) {
    # Default parameters
    if (!leftBrackets[0]) {
        leftBrackets[0] = "(" # left parenthesis
        leftBrackets[1] = "[" # left bracket
        leftBrackets[2] = "{" # left brace
    }
    if (!rightBrackets[0]) {
        rightBrackets[0] = ")" # right parenthesis
        rightBrackets[1] = "]" # right bracket
        rightBrackets[2] = "}" # right brace
    }
    if (!separators[0]) {
        separators[0] = "," # comma
    }

    stack[p = 0] = 0
    for (i = 0; i < length(tokens); i++) {
        token = tokens[i]

        if (belongsTo(token, leftBrackets))
            stack[++p] = 0
        else if (belongsTo(token, rightBrackets))
            --p
        else if (belongsTo(token, separators))
            stack[p]++
        else {
            key = stack[0]
            for (j = 1; j <= p; j++)
                key = key SUBSEP stack[j]
            returnAST[key] = token
        }
    }
}


output more heavy headings; checkpoint

output more heavy headings; checkpoint

BEGIN {
    heading = 0
}

hunt && /class="section_anchor">/ {
    hier[heading] = $0
    sub(/<\/a>/, "", hier[heading])
    sub(/<\/a>/, "", hier[heading])
    print hier[heading]
    hunt = 0
    next
}

/
/ {
    print "
"
    heading = 1
    hunt = 1
    print
    next
}

/<\/h[1-9]>/ {
    heading = 0
    print
    next
}

/
/ && heading == 0 {
    print "
"
    print "
"
    print "
" hier[1] "
"
    heading = 2
    hunt = 1
    print
    next
}

/
/ && heading == 0 {
    print "
"
    print "
"
    print "
" hier[1] "
"
    print "
" hier[2] "
"
    heading = 3
    hunt = 1
    print
    next
}

/
/ && heading == 3 {
    heading = 4
    hunt = 1
    print
    next
}

BEGIN {
    heading = 0
}

hunt && /class="section_anchor">/ {
    hier[heading] = $0
    sub(/<\/a>/, "", hier[heading])
    sub(/<\/a>/, "", hier[heading])
    print "#############" hier[heading]
    hunt = 0
}

/
/ {
    print "
"
    heading = 1
    hunt = 1
    print
    next
}

/<\/h[1-9]>/ {
    heading -= 1
    print
    next
}

/
/ && heading == 1 {
    print "
"
    print "
"
    print "
" hier[heading] "
"
    heading = 2
    hunt = 1
    print
    next
}

/
/ && heading == 2 {
    heading = 3
    hunt = 1
    print
    next
}

/
/ && heading == 3 {
    heading = 4
    hunt = 1
    print
    next
}