Parser script displays number of compiler warnings

#
# MIT License
#
# Copyright (c) 2017 Jean-Sebastien CONAN
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

#
# Part of the camelSCAD library.
#
# Parses and displays the results of the unit tests.
#
# @package util/test
# @author jsconan
#

BEGIN {
    # will parse a CSV file
    FS = ";"

    # init counters
    packageCount = 0
    moduleCount  = 0
    assertCount  = 0
    successCount = 0
    failedCount  = 0
    warningCount = 0

    # define ANSI colors
    BLACK    = "\033[0;30m"
    GRAY     = "\033[1;30m"
    RED      = "\033[0;31m"
    LRED     = "\033[1;31m"
    GREEN    = "\033[0;32m"
    LGREEN   = "\033[1;32m"
    BROWN    = "\033[0;33m"
    YELLOW   = "\033[1;33m"
    BLUE     = "\033[0;34m"
    LBLUE    = "\033[1;34m"
    MAGENTA  = "\033[0;35m"
    LMAGENTA = "\033[1;35m"
    CYAN     = "\033[0;36m"
    LCYAN    = "\033[1;36m"
    GREY     = "\033[0;37m"
    WHITE    = "\033[1;37m"
    RESET    = "\033[0m"
}

# detect compiler warnings
$0~/^WARNING:/ {
    warningCount = warningCount + 1
    printf("\n%s%s\n", LRED, $0)
}

# detect package declarations
$1~/package/ {
    packageCount = packageCount + 1
    package = $2
    lastType = "package"
    printf("\n%s[%s] ", LBLUE, $2)
}

# detect module declarations
$1~/module/ {
    moduleCount = moduleCount + 1
    module = $2
    lastType = "module"
    printf("\n%s    %s ", CYAN, $2)
}

# detect assertion results
$1~/assert/ {
    assertCount = assertCount + 1
    lastType = "assert"

    if ($2 == "#OK") {
        # successful assertion
        successCount = successCount + 1
        printf("%s.", LGREEN)
    } else {
        # failed assertion
        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $3
        failedAssertsMessage[failedCount] = $4
        failedAssertsDetails[failedCount] = $5
        failedCount = failedCount + 1

        # notify the failed assertion
        printf("%sF", LRED)
    }
}

# detect assertions counters
$1~/expect/ {
    # is failed count ?
    if ($5 != "true") {
        if ($3 == "") {
            # no expected number, try to take the actual number
            if (!$4) {
                # no actual number, assume it is at least 1
                expected = 1
            } else {
                # take the actual number as the expected number
                expected = $4
            }
        } else {
            # provided expected number
            expected = $3
        }

        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $6
        failedAssertsMessage[failedCount] = $7
        assertCount = assertCount + expected - $4
        failedCount = failedCount + 1

        # put a space after package name in case of some result will be displayed after
        if ($2 == "package") {
            printf(" ")
        }

        # if the last was a module, the result need to be displayed on a new line
        if ($2 == "module") {
            printf("\n    ")
        }

        # notify the missing assertions
        printf("%sM", LRED)
    }
}

END {
    separator = "%s-----------\n"
    printf("\n")

    if (failedCount > 0) {
        printf(separator, GREY)
        printf("%sFAILED\n", RED)
        package = ""
        context = ""

        for (i = 0; i < failedCount; i++) {
            if (package != failedAssertsPackage[i]) {
                package = failedAssertsPackage[i]
                printf("%s[%s]\n", GRAY, package)
            }
            if (failedAssertsContext[i] && context != failedAssertsContext[i]) {
                context = failedAssertsContext[i]
                printf("%s  %s\n", MAGENTA, context)
            }
            if (failedAssertsContext[i]) {
                printf("  ")
            }
            printf("%s- %s\n", RED, failedAssertsMessage[i])
            if (failedAssertsDetails[i]) {
                printf("    %s\n", failedAssertsDetails[i])
            }
            printf("\n")
        }
    }

    printf(separator, GREY)
    printf("%s%d packages\n", GREEN, packageCount)
    printf("%s%d modules\n", GREEN, moduleCount)
    printf("%s%d/%d asserts\n", GREEN, successCount, assertCount)
    printf(separator, GREY)

    if (warningCount > 0) {
        printf("%s%d warnings!\n", RED, warningCount)
    }

    if (failedCount == 0) {
        printf("%sAll successful!\n", GREEN)
    } else {
        printf("%s%d failed asserts!\n", RED, failedCount)
    }
}

#
# MIT License
#
# Copyright (c) 2017 Jean-Sebastien CONAN
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

#
# Part of the camelSCAD library.
#
# Parses and displays the results of the unit tests.
#
# @package util/test
# @author jsconan
#

BEGIN {
    # will parse a CSV file
    FS = ";"

    # init counters
    packageCount = 0
    moduleCount  = 0
    assertCount  = 0
    successCount = 0
    failedCount  = 0

    # define ANSI colors
    BLACK    = "\033[0;30m"
    GRAY     = "\033[1;30m"
    RED      = "\033[0;31m"
    LRED     = "\033[1;31m"
    GREEN    = "\033[0;32m"
    LGREEN   = "\033[1;32m"
    BROWN    = "\033[0;33m"
    YELLOW   = "\033[1;33m"
    BLUE     = "\033[0;34m"
    LBLUE    = "\033[1;34m"
    MAGENTA  = "\033[0;35m"
    LMAGENTA = "\033[1;35m"
    CYAN     = "\033[0;36m"
    LCYAN    = "\033[1;36m"
    GREY     = "\033[0;37m"
    WHITE    = "\033[1;37m"
    RESET    = "\033[0m"
}

# detect compiler warnings
$0~/^WARNING:/ {
    printf("\n%s%s\n", LRED, $0)
}

# detect package declarations
$1~/package/ {
    packageCount = packageCount + 1
    package = $2
    lastType = "package"
    printf("\n%s[%s] ", LBLUE, $2)
}

# detect module declarations
$1~/module/ {
    moduleCount = moduleCount + 1
    module = $2
    lastType = "module"
    printf("\n%s    %s ", CYAN, $2)
}

# detect assertion results
$1~/assert/ {
    assertCount = assertCount + 1
    lastType = "assert"

    if ($2 == "#OK") {
        # successful assertion
        successCount = successCount + 1
        printf("%s.", LGREEN)
    } else {
        # failed assertion
        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $3
        failedAssertsMessage[failedCount] = $4
        failedAssertsDetails[failedCount] = $5
        failedCount = failedCount + 1

        # notify the failed assertion
        printf("%sF", LRED)
    }
}

# detect assertions counters
$1~/expect/ {
    # is failed count ?
    if ($5 != "true") {
        if ($3 == "") {
            # no expected number, try to take the actual number
            if (!$4) {
                # no actual number, assume it is at least 1
                expected = 1
            } else {
                # take the actual number as the expected number
                expected = $4
            }
        } else {
            # provided expected number
            expected = $3
        }

        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $6
        failedAssertsMessage[failedCount] = $7
        assertCount = assertCount + expected - $4
        failedCount = failedCount + 1

        # put a space after package name in case of some result will be displayed after
        if ($2 == "package") {
            printf(" ")
        }

        # if the last was a module, the result need to be displayed on a new line
        if ($2 == "module") {
            printf("\n    ")
        }

        # notify the missing assertions
        printf("%sM", LRED)
    }
}

END {
    separator = "%s-----------\n"
    printf("\n")

    if (failedCount > 0) {
        printf(separator, GREY)
        printf("%sFAILED\n", RED)
        package = ""
        context = ""

        for (i = 0; i < failedCount; i++) {
            if (package != failedAssertsPackage[i]) {
                package = failedAssertsPackage[i]
                printf("%s[%s]\n", GRAY, package)
            }
            if (failedAssertsContext[i] && context != failedAssertsContext[i]) {
                context = failedAssertsContext[i]
                printf("%s  %s\n", MAGENTA, context)
            }
            if (failedAssertsContext[i]) {
                printf("  ")
            }
            printf("%s- %s\n", RED, failedAssertsMessage[i])
            if (failedAssertsDetails[i]) {
                printf("    %s\n", failedAssertsDetails[i])
            }
            printf("\n")
        }
    }

    printf(separator, GREY)
    printf("%s%d packages\n", GREEN, packageCount)
    printf("%s%d modules\n", GREEN, moduleCount)
    printf("%s%d/%d asserts\n", GREEN, successCount, assertCount)
    printf(separator, GREY)

    if (failedCount == 0) {
        printf("%sAll successful!\n", GREEN)
    } else {
        printf("%s%d failed asserts!\n", RED, failedCount)
    }
}


http txn tweak filtering

http txn tweak filtering

{
        if ( $0~/#hasHash/ ) {}
   else if ( $0~/#TabularDigest/ ) {}
   else if ( $0~/#hashAlgorithm/ ) {}
   else if ( $0~/#hashValue/ ) {}
   else if ( $0~/#reproductionOf>.*<hash:Item/ ) {}
   else if ( $0~/<hash:Item.*frbr.core#>/ ) {}
   else if ( $0~/p.surfrdf/ ) {}
   else { print }
}

{
        if ( $0~/#hasHash/ ) {}
   else if ( $0~/#TabularDigest/ ) {}
   else if ( $0~/#hashAlgorithm/ ) {}
   else if ( $0~/#hashValue/ ) {}
   else if ( $0~/hash:Item/ ) {}
   else if ( $0~/p.surfrdf/ ) {}
   else { print }
}


TestCommons: test reverse()

TestCommons: test reverse()

BEGIN {
    START_TEST("Commons.awk")

    # Arrays
    T("anything()", 3)
    {
        delete something
        assertFalse(anything(nothing))
        something[0] = 0; assertFalse(anything(something)) # edge case
        something[0] = 1; assertTrue(anything(something))
    }
    T("exists()", 4)
    {
        delete something
        assertFalse(exists(something))
        something[0] = 0; assertFalse(exists(something)) # edge case
        something[0] = 1; assertTrue(exists(something))
        assertTrue(exists(something[0]))
    }
    T("belongsTo()", 3)
    {
        delete array; array[0] = "foo"; array[1] = "bar"
        assertTrue(belongsTo("foo", array))
        assertTrue(belongsTo("bar", array))
        assertFalse(belongsTo("world", array))
    }
    T("identical()", 4)
    {
        delete x; x[0] = 42
        delete y; y[0][0] = 42
        assertFalse(identical(x, y))
        assertTrue(identical(x, y[0]))
        assertFalse(identical(x[0], y[0]))
        assertTrue(identical(x[0], y[0][0]))
    }
    T("append()", 1)
    {
        delete array; array[0] = "foo"
        delete expected; expected[0] = "foo"; expected[1] = "bar"
        append(array, "bar")
        assertEqual(array, expected)
    }

    # Strings
    T("isnum()", 4)
    {
        assertTrue(isnum(0))
        assertTrue(isnum(42.0))
        assertFalse(isnum(""))
        assertFalse(isnum("hello world"))
    }
    T("startsWithAny()", 3)
    {
        delete substrings; substrings[0] = "A"; substrings[1] = "a"
        assertTrue(startsWithAny("absolute", substrings))
        assertTrue(startsWithAny("ABSOLUTE", substrings))
        assertFalse(startsWithAny("ZOO", substrings))
    }
    T("matchesAny()", 4)
    {
        delete patterns; patterns[0] = "[[:space:]]"; patterns[1] = "[0Oo]"
        assertTrue(matchesAny("  ", patterns))
        assertTrue(matchesAny("obsolete", patterns))
        assertTrue(matchesAny("0.0", patterns))
        assertFalse(matchesAny("1.0", patterns))
    }
    T("replicate()", 4)
    {
        assertEqual(replicate("", 0), "")
        assertEqual(replicate("", 2), "")
        assertEqual(replicate("foo bar", 1), "foo bar")
        assertEqual(replicate("foo bar", 3), "foo barfoo barfoo bar")
    }
    T("reverse()", 4)
    {
        assertEqual(reverse(""), "")
        assertEqual(reverse("god"), "dog")
        assertEqual(reverse("0123456789"), "9876543210")
        assertEqual(reverse("さしすせそ"), "そせすしさ")
    }
    T("join()", 4)
    {
        assertEqual(join("", "-"), "")
        delete array; array[0]
        assertEqual(join(array, "-"), "")
        delete array; array[0] = "foo"; array[1] = "bar"
        assertEqual(join(array, " "), "foo bar")
        assertEqual(join(array, ","), "foo,bar")
    }
    T("explode()", 1)
    {
        delete array
        delete expected; expected[1] = "f"; expected[2] = "o"; expected[3] = "o"
        explode("foo", array)
        assertEqual(array, expected)
    }
    T("escapeChar()", 8)
    {
        assertEqual(escapeChar("b"), "\b")
        assertEqual(escapeChar("f"), "\f")
        assertEqual(escapeChar("n"), "\n")
        assertEqual(escapeChar("r"), "\r")
        assertEqual(escapeChar("t"), "\t")
        assertEqual(escapeChar("v"), "\v")
        assertEqual(escapeChar("_"), "_")
        assertNotEqual(escapeChar("_"), "\\_")
    }
    T("literal()", 6)
    {
        assertEqual(literal(""), "")
        assertEqual(literal("foo"), "foo")
        assertEqual(literal("\"foo\""), "foo")
        assertEqual(literal("\"\\\"foo\\\"\""), "\"foo\"")
        assertEqual(literal("\"foo\\nbar\""), "foo\nbar")
        assertEqual(literal("\"foo\\u0026bar\""), "foo&bar")
    }
    T("escape()", 4)
    {
        assertEqual(escape(""), "")
        assertEqual(escape("foo"), "foo")
        assertEqual(escape("\""), "\\\"")
        assertEqual(escape("\"foo\""), "\\\"foo\\\"")
    }
    T("unescape()", 4)
    {
        assertEqual(unescape(""), "")
        assertEqual(unescape("foo"), "foo")
        assertEqual(unescape("\\\""), "\"")
        assertEqual(unescape("\\\"foo\\\""), "\"foo\"")
    }
    T("parameterize()", 10)
    {
        assertEqual(parameterize(""), "''")
        assertEqual(parameterize("foo"), "'foo'")
        assertEqual(parameterize("foo bar"), "'foo bar'")
        assertEqual(parameterize("foo 'bar"), "'foo '\\''bar'")
        assertEqual(parameterize("foo \"bar\""), "'foo \"bar\"'")
        assertEqual(parameterize("", "\""), "\"\"")
        assertEqual(parameterize("foo", "\""), "\"foo\"")
        assertEqual(parameterize("foo bar", "\""), "\"foo bar\"")
        assertEqual(parameterize("foo 'bar", "\""), "\"foo 'bar\"")
        assertEqual(parameterize("foo \"bar\"", "\""), "\"foo \\\"bar\\\"\"")
    }
    T("unparameterize()", 10)
    {
        assertEqual(unparameterize("''"), "")
        assertEqual(unparameterize("'foo'"), "foo")
        assertEqual(unparameterize("'foo bar'"), "foo bar")
        assertEqual(unparameterize("'foo '\\''bar'"), "foo 'bar")
        assertEqual(unparameterize("'foo \"bar\"'"), "foo \"bar\"")
        assertEqual(unparameterize("\"\""), "")
        assertEqual(unparameterize("\"foo\""), "foo")
        assertEqual(unparameterize("\"foo bar\""), "foo bar")
        assertEqual(unparameterize("\"foo 'bar\""), "foo 'bar")
        assertEqual(unparameterize("\"foo \\\"bar\\\"\""), "foo \"bar\"")
    }
    T("toString()", 4)
    {
        assertEqual(toString(""), "")
        assertEqual(toString(42), "42")
        assertEqual(toString("foo"), "foo")
        assertEqual(toString("\"foo bar\""), "\"foo bar\"")
    }
    T("squeeze()", 4)
    {
        assertEqual(squeeze(""), "")
        assertEqual(squeeze(" "), "")
        assertEqual(squeeze("	"), "")
        assertEqual(squeeze("  foo = bar #comments"), "foo = bar")
    }
    T("yn()", 12)
    {
        assertFalse(yn(0))
        assertFalse(yn("0"))
        assertFalse(yn("false"))
        assertFalse(yn("no"))
        assertFalse(yn("off"))
        assertFalse(yn("OFF"))
        assertTrue(yn(1))
        assertTrue(yn("1"))
        assertTrue(yn("true"))
        assertTrue(yn("yes"))
        assertTrue(yn("on"))
        assertTrue(yn("ON"))
    }

    # URLs
    T("quote()", 4)
    {
        assertEqual(quote(""), "")
        assertEqual(quote("foo"), "foo")
        assertEqual(quote("foo bar"), "foo%20bar")
        assertEqual(quote("\"hello, world!\""), "%22hello%2C%20world%21%22")
    }

    # System
    T("fileExists()", 4)
    {
        assertFalse(fileExists("README"))
        assertFalse(fileExists("README .md"))
        assertTrue(fileExists("README.md"))
        assertFalse(fileExists("."))
    }
    T("dirExists()", 4)
    {
        assertFalse(dirExists("README"))
        assertFalse(dirExists("README.md"))
        assertTrue(dirExists("."))
        assertFalse(dirExists(" ."))
    }
    T("detectProgram()", 2)
    {
        assertTrue(detectProgram("ls"))
        assertTrue(detectProgram("gawk", "--version"))
    }

    END_TEST()
}

BEGIN {
    START_TEST("Commons.awk")

    # Arrays
    T("anything()", 3)
    {
        delete something
        assertFalse(anything(nothing))
        something[0] = 0; assertFalse(anything(something)) # edge case
        something[0] = 1; assertTrue(anything(something))
    }
    T("exists()", 4)
    {
        delete something
        assertFalse(exists(something))
        something[0] = 0; assertFalse(exists(something)) # edge case
        something[0] = 1; assertTrue(exists(something))
        assertTrue(exists(something[0]))
    }
    T("belongsTo()", 3)
    {
        delete array; array[0] = "foo"; array[1] = "bar"
        assertTrue(belongsTo("foo", array))
        assertTrue(belongsTo("bar", array))
        assertFalse(belongsTo("world", array))
    }
    T("identical()", 4)
    {
        delete x; x[0] = 42
        delete y; y[0][0] = 42
        assertFalse(identical(x, y))
        assertTrue(identical(x, y[0]))
        assertFalse(identical(x[0], y[0]))
        assertTrue(identical(x[0], y[0][0]))
    }
    T("append()", 1)
    {
        delete array; array[0] = "foo"
        delete expected; expected[0] = "foo"; expected[1] = "bar"
        append(array, "bar")
        assertEqual(array, expected)
    }

    # Strings
    T("isnum()", 4)
    {
        assertTrue(isnum(0))
        assertTrue(isnum(42.0))
        assertFalse(isnum(""))
        assertFalse(isnum("hello world"))
    }
    T("startsWithAny()", 3)
    {
        delete substrings; substrings[0] = "A"; substrings[1] = "a"
        assertTrue(startsWithAny("absolute", substrings))
        assertTrue(startsWithAny("ABSOLUTE", substrings))
        assertFalse(startsWithAny("ZOO", substrings))
    }
    T("matchesAny()", 4)
    {
        delete patterns; patterns[0] = "[[:space:]]"; patterns[1] = "[0Oo]"
        assertTrue(matchesAny("  ", patterns))
        assertTrue(matchesAny("obsolete", patterns))
        assertTrue(matchesAny("0.0", patterns))
        assertFalse(matchesAny("1.0", patterns))
    }
    T("replicate()", 4)
    {
        assertEqual(replicate("", 0), "")
        assertEqual(replicate("", 2), "")
        assertEqual(replicate("foo bar", 1), "foo bar")
        assertEqual(replicate("foo bar", 3), "foo barfoo barfoo bar")
    }
    T("join()", 4)
    {
        assertEqual(join("", "-"), "")
        delete array; array[0]
        assertEqual(join(array, "-"), "")
        delete array; array[0] = "foo"; array[1] = "bar"
        assertEqual(join(array, " "), "foo bar")
        assertEqual(join(array, ","), "foo,bar")
    }
    T("explode()", 1)
    {
        delete array
        delete expected; expected[1] = "f"; expected[2] = "o"; expected[3] = "o"
        explode("foo", array)
        assertEqual(array, expected)
    }
    T("escapeChar()", 8)
    {
        assertEqual(escapeChar("b"), "\b")
        assertEqual(escapeChar("f"), "\f")
        assertEqual(escapeChar("n"), "\n")
        assertEqual(escapeChar("r"), "\r")
        assertEqual(escapeChar("t"), "\t")
        assertEqual(escapeChar("v"), "\v")
        assertEqual(escapeChar("_"), "_")
        assertNotEqual(escapeChar("_"), "\\_")
    }
    T("literal()", 6)
    {
        assertEqual(literal(""), "")
        assertEqual(literal("foo"), "foo")
        assertEqual(literal("\"foo\""), "foo")
        assertEqual(literal("\"\\\"foo\\\"\""), "\"foo\"")
        assertEqual(literal("\"foo\\nbar\""), "foo\nbar")
        assertEqual(literal("\"foo\\u0026bar\""), "foo&bar")
    }
    T("escape()", 4)
    {
        assertEqual(escape(""), "")
        assertEqual(escape("foo"), "foo")
        assertEqual(escape("\""), "\\\"")
        assertEqual(escape("\"foo\""), "\\\"foo\\\"")
    }
    T("unescape()", 4)
    {
        assertEqual(unescape(""), "")
        assertEqual(unescape("foo"), "foo")
        assertEqual(unescape("\\\""), "\"")
        assertEqual(unescape("\\\"foo\\\""), "\"foo\"")
    }
    T("parameterize()", 10)
    {
        assertEqual(parameterize(""), "''")
        assertEqual(parameterize("foo"), "'foo'")
        assertEqual(parameterize("foo bar"), "'foo bar'")
        assertEqual(parameterize("foo 'bar"), "'foo '\\''bar'")
        assertEqual(parameterize("foo \"bar\""), "'foo \"bar\"'")
        assertEqual(parameterize("", "\""), "\"\"")
        assertEqual(parameterize("foo", "\""), "\"foo\"")
        assertEqual(parameterize("foo bar", "\""), "\"foo bar\"")
        assertEqual(parameterize("foo 'bar", "\""), "\"foo 'bar\"")
        assertEqual(parameterize("foo \"bar\"", "\""), "\"foo \\\"bar\\\"\"")
    }
    T("unparameterize()", 10)
    {
        assertEqual(unparameterize("''"), "")
        assertEqual(unparameterize("'foo'"), "foo")
        assertEqual(unparameterize("'foo bar'"), "foo bar")
        assertEqual(unparameterize("'foo '\\''bar'"), "foo 'bar")
        assertEqual(unparameterize("'foo \"bar\"'"), "foo \"bar\"")
        assertEqual(unparameterize("\"\""), "")
        assertEqual(unparameterize("\"foo\""), "foo")
        assertEqual(unparameterize("\"foo bar\""), "foo bar")
        assertEqual(unparameterize("\"foo 'bar\""), "foo 'bar")
        assertEqual(unparameterize("\"foo \\\"bar\\\"\""), "foo \"bar\"")
    }
    T("toString()", 4)
    {
        assertEqual(toString(""), "")
        assertEqual(toString(42), "42")
        assertEqual(toString("foo"), "foo")
        assertEqual(toString("\"foo bar\""), "\"foo bar\"")
    }
    T("squeeze()", 4)
    {
        assertEqual(squeeze(""), "")
        assertEqual(squeeze(" "), "")
        assertEqual(squeeze("	"), "")
        assertEqual(squeeze("  foo = bar #comments"), "foo = bar")
    }
    T("yn()", 12)
    {
        assertFalse(yn(0))
        assertFalse(yn("0"))
        assertFalse(yn("false"))
        assertFalse(yn("no"))
        assertFalse(yn("off"))
        assertFalse(yn("OFF"))
        assertTrue(yn(1))
        assertTrue(yn("1"))
        assertTrue(yn("true"))
        assertTrue(yn("yes"))
        assertTrue(yn("on"))
        assertTrue(yn("ON"))
    }

    # URLs
    T("quote()", 4)
    {
        assertEqual(quote(""), "")
        assertEqual(quote("foo"), "foo")
        assertEqual(quote("foo bar"), "foo%20bar")
        assertEqual(quote("\"hello, world!\""), "%22hello%2C%20world%21%22")
    }

    # System
    T("fileExists()", 4)
    {
        assertFalse(fileExists("README"))
        assertFalse(fileExists("README .md"))
        assertTrue(fileExists("README.md"))
        assertFalse(fileExists("."))
    }
    T("dirExists()", 4)
    {
        assertFalse(dirExists("README"))
        assertFalse(dirExists("README.md"))
        assertTrue(dirExists("."))
        assertFalse(dirExists(" ."))
    }
    T("detectProgram()", 2)
    {
        assertTrue(detectProgram("ls"))
        assertTrue(detectProgram("gawk", "--version"))
    }

    END_TEST()
}


build.awk: readme() uses number of language codes with stable support

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/Languages.awk"
@include "metainfo.awk"

function init() {
    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    HomebrewFormula      = PagesPath "translate-shell.rb"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, num, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = length(group) == 1 ? group[1] "_language" : join(group, "_")
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    #print "***" length(Locale) "*** *languages in total. "
    print "*Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "*Version: [English](https://github.com/soimort/translate-shell/wiki/Languages) " \
        "| [Chinese Simplified](https://github.com/soimort/translate-shell/wiki/Languages-%28%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87%29)*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = length(group) == 1 ? group[1] "_language" :
            group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language, getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function readSqueezed(fileName, squeezed,    group, line, ret) {
    if (fileName ~ /\*$/) # glob simulation
        return readSqueezed(fileName ".awk", squeezed)

    ret = NULLSTR
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (ret) ret = ret RS
                ret = ret readSqueezed(group[1] ".awk", squeezed)
            } else if (!squeezed || line = squeeze(line)) { # effective LOC
                if (ret) ret = ret RS
                ret = ret line
            }
        }
    return ret
}

function build(target, type,    group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans

        print "#" > Trans
        if (fileExists("DISCLAIMER"))
            while (getline line < "DISCLAIMER")
                print "# " line > Trans
        print "#" > Trans

        print "if ! [[ $LANG =~ '[UTF|utf]-?8$' ]]; then export LANG=en_US.UTF-8; fi" > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        print readSqueezed(EntryPoint, TRUE) > Trans
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp
        print (temp == "Darwin" ?
               "#!/usr/bin/env gawk -f" : # OS X
               "#!/usr/bin/gawk -f") > TransAwk

        print readSqueezed(EntryPoint, TRUE) > TransAwk

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/translate-shell.rb ...")
    # Update gh-pages/translate-shell.rb
    ("sha1sum " PagesPath "translate-shell.tar.gz") | getline temp
    split(temp, group)
    sha1 = group[1]
    text = readFrom(HomebrewFormula ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, HomebrewFormula)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/Languages.awk"
@include "metainfo.awk"

function init() {
    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    HomebrewFormula      = PagesPath "translate-shell.rb"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    rows = int(length(Locale) / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = length(group) == 1 ? group[1] "_language" : join(group, "_")
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    #print "***" length(Locale) "*** *languages in total. "
    print "*Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "*Version: [English](https://github.com/soimort/translate-shell/wiki/Languages) " \
        "| [Chinese Simplified](https://github.com/soimort/translate-shell/wiki/Languages-%28%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87%29)*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = length(group) == 1 ? group[1] "_language" :
            group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language, getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function readSqueezed(fileName, squeezed,    group, line, ret) {
    if (fileName ~ /\*$/) # glob simulation
        return readSqueezed(fileName ".awk", squeezed)

    ret = NULLSTR
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (ret) ret = ret RS
                ret = ret readSqueezed(group[1] ".awk", squeezed)
            } else if (!squeezed || line = squeeze(line)) { # effective LOC
                if (ret) ret = ret RS
                ret = ret line
            }
        }
    return ret
}

function build(target, type,    group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans

        print "#" > Trans
        if (fileExists("DISCLAIMER"))
            while (getline line < "DISCLAIMER")
                print "# " line > Trans
        print "#" > Trans

        print "if ! [[ $LANG =~ '[UTF|utf]-?8$' ]]; then export LANG=en_US.UTF-8; fi" > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        print readSqueezed(EntryPoint, TRUE) > Trans
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp
        print (temp == "Darwin" ?
               "#!/usr/bin/env gawk -f" : # OS X
               "#!/usr/bin/gawk -f") > TransAwk

        print readSqueezed(EntryPoint, TRUE) > TransAwk

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/translate-shell.rb ...")
    # Update gh-pages/translate-shell.rb
    ("sha1sum " PagesPath "translate-shell.tar.gz") | getline temp
    split(temp, group)
    sha1 = group[1]
    text = readFrom(HomebrewFormula ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, HomebrewFormula)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}


UTF-8 errors are non-fatal

UTF-8 errors are non-fatal

Just skip broken characters and put the message on stderr.

#!/usr/bin/awk -f
#
# Takes a terminal formatted manual page and generates an identical
# looking HTML
 block.
#
# This script only supports UTF-8 encoding.
#

BEGIN {
	# The backspace character is used to combine characters
	BS = "\x08"

	# special HTML character substitutions
	SUB["&"] = "&"
	SUB["<"] = "<"
	SUB[">"] = ">"

	# formatting flags
	FMT_REGULAR    = 0
	FMT_BOLD       = 1
	FMT_UNDERLINE  = 2

	# formatting selectors
	SELECT["_"] = FMT_UNDERLINE

	# opening tags
	OPEN[FMT_BOLD]                 = ""
	OPEN[FMT_UNDERLINE]            = ""
	OPEN[FMT_BOLD + FMT_UNDERLINE] = ""

	# closing tags
	CLOSE[FMT_BOLD]                 = ""
	CLOSE[FMT_UNDERLINE]            = ""
	CLOSE[FMT_BOLD + FMT_UNDERLINE] = ""

	print "
"
}

END {
	print "
"
}

function u8error(msg)
{
	printf("utf-8 error:%d:%d:%d: %s\n\n%s\n% " p "s\n", NR, p, i, msg, $0, "^") > "/dev/stderr"
}

{
	# split characters
	delete chars

	n = split($0, chars, "")

	#
	# merge utf-8 multibyte chars
	#
	delete uchars

	p = 1
	mbyte = 0
	for (i = 1; i <= n; ++i) {
		# no leading bits, 1 byte characte
		if (mbyte)
		{
			if (chars[i] < "\x80" || chars[i] >= "\xc0") {
				u8error("unexected byte in multibyte character")
				mbyte = 0
				continue
			}
			uchars[p] = uchars[p] chars[i]
			p += !--mbyte
		}
		# regular 1 byte characters
		else if (chars[i] < "\x80") {
			uchars[p++] = chars[i]
		}
		# 2 leading bits, merge 2 bytes
		else if (chars[i] >= "\xc0" && chars[i] < "\xe0") {
			uchars[p] = chars[i]
			mbyte = 1
		}
		# 3 leading bits, merge 3 bytes
		else if (chars[i] >= "\xe0" && chars[i] < "\xf0") {
			uchars[p] = chars[i]
			mbyte = 2
		}
		# 4 byte characters
		else if (chars[i] >= "\xf0" && chars[i] < "\xf8") {
			uchars[p] = chars[i]
			mbyte = 3
		}
		# error
		else
		{
			u8error("byte is not a character")
		}
	}
	n = p

	#
	# detect per character formatting
	#
	delete format
	delete ochars

	olen = 0
	for (i = 1; i <= n; ++i) {
		++olen
		# check all formatting characters
		for (; uchars[i + 1] == BS; i += 2) {
			format[olen] = or(format[olen], SELECT[uchars[i]])
			# bold is any match in the chain of characters,
			# so check the current one against all the
			# following ones
			for (p = i + 2; uchars[p - 1] == BS; p += 2) {
				format[olen] = or(format[olen], uchars[i] == uchars[p] ? FMT_BOLD : FMT_REGULAR)
			}
		}
		# add character
		ochars[olen] = uchars[i]
	}

	#
	# replace special characters
	#
	for (i = 1; i <= olen; ++i) {
		ochars[i] in SUB && ochars[i] = SUB[ochars[i]]
	}

	#
	# output with HTML formatting
	#
	$0 = ""

	last = FMT_REGULAR
	for (i = 1; i <= olen; ++i) {
		if (format[i] != last) {
			$0 = $0 CLOSE[last] OPEN[format[i]]
		}
		$0 = $0 ochars[i]
		last = format[i]
	}
	$0 = $0 CLOSE[last]

	print
}

#!/usr/bin/awk -f
#
# Takes a terminal formatted manual page and generates an identical
# looking HTML
 block.
#
# This script only supports UTF-8 encoding.
#

BEGIN {
	# The backspace character is used to combine characters
	BS = "\x08"

	# special HTML character substitutions
	SUB["&"] = "&"
	SUB["<"] = "<"
	SUB[">"] = ">"

	# formatting flags
	FMT_REGULAR    = 0
	FMT_BOLD       = 1
	FMT_UNDERLINE  = 2

	# formatting selectors
	SELECT["_"] = FMT_UNDERLINE

	# opening tags
	OPEN[FMT_BOLD]                 = ""
	OPEN[FMT_UNDERLINE]            = ""
	OPEN[FMT_BOLD + FMT_UNDERLINE] = ""

	# closing tags
	CLOSE[FMT_BOLD]                 = ""
	CLOSE[FMT_UNDERLINE]            = ""
	CLOSE[FMT_BOLD + FMT_UNDERLINE] = ""

	print "
"
}

END {
	print "
"
}

function u8error(msg)
{
	printf("utf-8 error:%d:%d:%d: %s\n\n%s\n% " p "s\n", NR, p, i, msg, $0, "^")
	exit 1
}

{
	# split characters
	delete chars

	n = split($0, chars, "")

	#
	# merge utf-8 multibyte chars
	#
	delete uchars

	p = 1
	mbyte = 0
	for (i = 1; i <= n; ++i) {
		# no leading bits, 1 byte characte
		if (mbyte)
		{
			if (chars[i] < "\x80" || chars[i] >= "\xc0") {
				u8error("unexected byte in multibyte character")
			}
			uchars[p] = uchars[p] chars[i]
			p += !--mbyte
		}
		# regular 1 byte characters
		else if (chars[i] < "\x80") {
			uchars[p++] = chars[i]
		}
		# 2 leading bits, merge 2 bytes
		else if (chars[i] >= "\xc0" && chars[i] < "\xe0") {
			uchars[p] = chars[i]
			mbyte = 1
		}
		# 3 leading bits, merge 3 bytes
		else if (chars[i] >= "\xe0" && chars[i] < "\xf0") {
			uchars[p] = chars[i]
			mbyte = 2
		}
		# 4 byte characters
		else if (chars[i] >= "\xf0" && chars[i] < "\xf8") {
			uchars[p] = chars[i]
			mbyte = 3
		}
		# error
		else
		{
			u8error("byte is not a character")
		}
	}
	n = p

	#
	# detect per character formatting
	#
	delete format
	delete ochars

	olen = 0
	for (i = 1; i <= n; ++i) {
		++olen
		# check all formatting characters
		for (; uchars[i + 1] == BS; i += 2) {
			format[olen] = or(format[olen], SELECT[uchars[i]])
			# bold is any match in the chain of characters,
			# so check the current one against all the
			# following ones
			for (p = i + 2; uchars[p - 1] == BS; p += 2) {
				format[olen] = or(format[olen], uchars[i] == uchars[p] ? FMT_BOLD : FMT_REGULAR)
			}
		}
		# add character
		ochars[olen] = uchars[i]
	}

	#
	# replace special characters
	#
	for (i = 1; i <= olen; ++i) {
		ochars[i] in SUB && ochars[i] = SUB[ochars[i]]
	}

	#
	# output with HTML formatting
	#
	$0 = ""

	last = FMT_REGULAR
	for (i = 1; i <= olen; ++i) {
		if (format[i] != last) {
			$0 = $0 CLOSE[last] OPEN[format[i]]
		}
		$0 = $0 ochars[i]
		last = format[i]
	}
	$0 = $0 CLOSE[last]

	print
}


Beta release, add support for header files.

#!@include "functions.awk"

BEGIN {
	version = "0.1.2"
	printf("link-parser, v%s\n", version)

	true = 1
	false = 0
	g_pat_srccommon = "\\/sw\\/vobs.+"
	g_sroot = ""
	inst_root = ""
	tar_root = ""

	# Colors
	RED="\033[0;31m"
	GRN="\033[0;32m"
	BLU="\033[0;34m"
	NC="\033[0m"
	BOL="\033[1m"

	if (arg ~ /test/) {
		do_test();
		SKIP_END = true
		exit 0
	}

	if (arg ~ /pretend/) {
		pretend = true
	}
	count = 0

	# Skip head lines
	getline
}

/^[^-]/ {
	link = $0
	# Remove the ending ' \'
	gsub(/[ \t]+\\$/, "", link)

	if (g_sroot == "") {
		printf("Grabbing sources and headers ...\n")
		g_sroot = get_sroot(link)
		"date +%Y%m%d%H-%M%S" | getline date
		inst_root = g_sroot"-"date
	}

	# ---------
	# Sources
	# ---------

	cmd_file = link".cmd"
	while ((getline line < cmd_file) > 0) {
		# abosolute path
		if (line ~ /[ \t]+-c[ \t]+\/.+$/)
			source = gensub(/^.+[ \t]+-c[ \t]+(.+)$/, "\\1", "g", line)
		else # relative path
			source = gensub(/^cd (.+);.+-c (.+)/, "\\1/\\2", "g", line)

		rel_install(source, g_sroot, inst_root)
	}
	close(cmd_file)

	# ---------
	# Headers
	# ---------

	gsub(/\.(obj|o|lib)$/, "", link)
	dep_file = link ".D"

	if ((getline < dep_file) < 0)
		next

	# Get path prefix
	cur_path = $1
	gsub(/^#/, "", cur_path)

	while ((getline < dep_file) > 0) {
		gsub(/:/, " ")
		for (i = 1; i <= NF; i++) {
			if ($i !~ /\.(h|hpp)$/)
				continue
			header = cur_path "/" $i
			rel_install(header, g_sroot, inst_root)
		}
	}
	close(dep_file)
}

END {
	print "Total dumped files: " count
	if (pretend)
		exit 0

	tarball = inst_root ".tar.gz"
	printf("Generating source tarball %s%s%s%s ...\n", RED, BOL, tarball, NC)
	system("tar zcf " tarball " " inst_root)
	system("rm -r "inst_root)
}

function rel_install(source, rel_root, new_root) {
	count++
	dir = dirname(source)
	base = basename(source)
	patt = "^.+\\/"rel_root"\\/(sw.+)"
	rel_path = gensub(patt, "\\1", "g", dir)

	if (pretend)
		return 0
	system("mkdir -p " new_root "/" rel_path)
	system("cp -L " source " " new_root "/" rel_path)
}

function get_sroot(line) {
	if (line !~ g_pat_srccommon)
		return ""

	gsub(g_pat_srccommon, "", line);
	gsub(/.+\//, "", line);
	return line;
}

# dirname
function dirname(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)

	# Remove basename
	gsub(/[^\/]+$/, "", path)
	if (path == "") return "."

	# Remove the ending '/' again
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	return path
}

# basename
function basename(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	path = gensub(/^.+\/+(.+)$/, "\\1", "g", path)

	return path;
}

###### Test #########
function do_test() {
	printf("%s%s%s%s\n", BLU, BOL, "---- Test get_sroot ----", NC)
	test_get_sroot("/repo/lulinw/isam/sw/vobs/esam/build");
	test_get_sroot("/repo/lulinw/isam/sw/esam/build");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test dirname ----", NC)
	test_dirname("/home/abc");
	test_dirname("home///abc//");
	test_dirname("home///abc/de//");
	test_dirname("../abc//");
	test_dirname("abc");
	test_dirname("..");
	test_dirname("..///");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test basename ----", NC)
	test_basename("/home//abc")
	test_basename("///")
	test_basename("/")
	test_basename("..")
	test_basename("../abc//")
	test_basename("/../abc//")
	test_basename("/../abc/efg/")
}

function test_get_sroot(line) {
	if ((ret = get_sroot(line)) == "") {
		printf("Unexpected line [PAT=%s]:\n  %s%s%s\n",
				g_pat_srccommon, RED, line, NC)
	} else
		printf("Root of %s is\n  %s%s%s\n", line, GRN, get_sroot(line), NC)
}

function test_dirname(path) {
	printf("dirname(%s) is \n  %s%s%s\n", path, GRN, dirname(path), NC)
}

function test_basename(path) {
	printf("basename(%s) is \n %s%s%s\n", path, GRN, basename(path), NC)
}

#!@include "functions.awk"

BEGIN {
	true = 1
	false = 0
	g_pat_srccommon = "\\/sw\\/vobs.+"
	g_sroot = ""
	tar_root = ""

	# Colors
	RED="\033[0;31m"
	GRN="\033[0;32m"
	BLU="\033[0;34m"
	NC="\033[0m"
	BOL="\033[1m"

	if (arg ~ /test/) {
		do_test();
		SKIP_END = true
		exit 0
	}
	count = 0
}

/^[^-]/ {
	link = $0
	# Remove the ending ' \'
	gsub(/[ \t]+\\$/, "", link)

	if (g_sroot == "") {
		g_sroot = get_sroot(link)
		system("mkdir ." g_sroot)
	}

	cmd_file = link".cmd"
	while ((getline line < cmd_file) > 0) {
		# abosolute path
		if (line ~ /[ \t]+-c[ \t]+\/.+$/)
			source = gensub(/^.+[ \t]+-c[ \t]+(.+)$/, "\\1", "g", line)
		# relative path
		else
			source = gensub(/^cd (.+);.+-c (.+)/, "\\1/\\2", "g", line)

		print BLU source NC
		if (source == "") {
			print RED "+++++++"line NC
		}

		dir = dirname(source)
		base = basename(source)
		patt = "^.+\\/"g_sroot"\\/(sw.+)"
		rel_path = gensub(patt, "\\1", "g", dir)

		system("mkdir -p ." g_sroot "/" rel_path)
		system("cp " source " ." g_sroot "/" rel_path)

		count = count + 1
	}
}

END {
	print "Total dumped files: " count
	"date +%Y%m%d%H%M%S" | getline date
	tarball = g_sroot "." date ".tar.gz"
	printf("Generating source tarball %s%s%s%s ...\n", RED, BOL, tarball, NC)
	system("tar zcf " tarball " ." g_sroot)
	system("rm -r ."g_sroot)
}

function get_sroot(line) {
	if (line !~ g_pat_srccommon)
		return ""

	gsub(g_pat_srccommon, "", line);
	gsub(/.+\//, "", line);
	return line;
}

# dirname
function dirname(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)

	# Remove basename
	gsub(/[^\/]+$/, "", path)
	if (path == "") return "."

	# Remove the ending '/' again
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	return path
}

# basename
function basename(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	path = gensub(/^.+\/+(.+)$/, "\\1", "g", path)

	return path;
}

###### Test #########
function do_test() {
	printf("%s%s%s%s\n", BLU, BOL, "---- Test get_sroot ----", NC)
	test_get_sroot("/repo/lulinw/isam/sw/vobs/esam/build");
	test_get_sroot("/repo/lulinw/isam/sw/esam/build");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test dirname ----", NC)
	test_dirname("/home/abc");
	test_dirname("home///abc//");
	test_dirname("home///abc/de//");
	test_dirname("../abc//");
	test_dirname("abc");
	test_dirname("..");
	test_dirname("..///");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test basename ----", NC)
	test_basename("/home//abc")
	test_basename("///")
	test_basename("/")
	test_basename("..")
	test_basename("../abc//")
	test_basename("/../abc//")
	test_basename("/../abc/efg/")
}

function test_get_sroot(line) {
	if ((ret = get_sroot(line)) == "") {
		printf("Unexpected line [PAT=%s]:\n  %s%s%s\n",
				g_pat_srccommon, RED, line, NC)
	} else
		printf("Root of %s is\n  %s%s%s\n", line, GRN, get_sroot(line), NC)
}

function test_dirname(path) {
	printf("dirname(%s) is \n  %s%s%s\n", path, GRN, dirname(path), NC)
}

function test_basename(path) {
	printf("basename(%s) is \n %s%s%s\n", path, GRN, basename(path), NC)
}


- Add automatic post vop debug checks.  These work in both the success and    failure cases.

#!/usr/bin/awk -f

#
# Copyright (c) 1992, 1993
#	The Regents of the University of California.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#	This product includes software developed by the University of
#	California, Berkeley and its contributors.
# 4. Neither the name of the University nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#	@(#)vnode_if.sh	8.1 (Berkeley) 6/10/93
# $FreeBSD$
#
# Script to produce VFS front-end sugar.
#
# usage: vnode_if.awk <srcfile> [-c | -h]
#	(where <srcfile> is currently /sys/kern/vnode_if.src)
#

function usage()
{
	print "usage: vnode_if.awk <srcfile> [-c|-h]";
	exit 1;
}

function die(msg, what)
{
	printf msg "\n", what > "/dev/stderr";
	exit 1;
}

function t_spc(type)
{
	# Append a space if the type is not a pointer
	return (type ~ /\*$/) ? type : type " ";
}

# These are just for convenience ...
function printc(s) {print s > cfile;}
function printh(s) {print s > hfile;}

function add_debug_code(name, arg, pos)
{
	if (lockdata[name, arg, pos]) {
		# Add assertions for locking
		if (lockdata[name, arg, pos] == "L")
			printh("\tASSERT_VOP_LOCKED("arg", \""uname"\");");
		else if (lockdata[name, arg, pos] == "U")
			printh("\tASSERT_VOP_UNLOCKED("arg", \""uname"\");");
		else if (0) {
			# XXX More checks!
		}
	}
}

function add_debug_pre(name)
{
	if (lockdata[name, "pre"]) {
		printh("#ifdef	DEBUG_VFS_LOCKS");
		printh("\t"lockdata[name, "pre"]"(&a);");
		printh("#endif");
	}
}

function add_debug_post(name)
{
	if (lockdata[name, "post"]) {
		printh("#ifdef	DEBUG_VFS_LOCKS");
		printh("\t"lockdata[name, "post"]"(&a, rc);");
		printh("#endif");
	}
}

function find_arg_with_type (type)
{
	for (jj = 0; jj < numargs; jj++) {
		if (types[jj] == type) {
			return "VOPARG_OFFSETOF(struct " \
			    name "_args,a_" args[jj] ")";
		}
	}

	return "VDESC_NO_OFFSET";
}

BEGIN{

# Process the command line
for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[ch]+$/ && arg !~ /\.src$/)
		usage();
	if (arg ~ /^-.*c/)
		cfile = "vnode_if.c";
	if (arg ~ /^-.*h/)
		hfile = "vnode_if.h";
	if (arg ~ /\.src$/)
		srcfile = arg;
}
ARGC = 1;

if (!cfile && !hfile)
	exit 0;

if (!srcfile)
	usage();

common_head = \
    "/*\n" \
    " * This file is produced automatically.\n" \
    " * Do not modify anything in here by hand.\n" \
    " *\n" \
    " * Created from $FreeBSD$\n" \
    " */\n" \
    "\n";

if (hfile)
	printh(common_head "extern struct vnodeop_desc vop_default_desc;");

if (cfile) {
	printc(common_head \
	    "#include <sys/param.h>\n" \
	    "#include <sys/systm.h>\n" \
	    "#include <sys/vnode.h>\n" \
	    "\n" \
	    "struct vnodeop_desc vop_default_desc = {\n" \
	    "	1,\t\t\t/* special case, vop_default => 1 */\n" \
	    "	\"default\",\n" \
	    "	0,\n" \
	    "	NULL,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	NULL,\n" \
	    "};\n");
}

while ((getline < srcfile) > 0) {
	if (NF == 0)
		continue;
	if ($1 ~ /^#%/) {
		if (NF != 6  ||  $1 != "#%"  || \
		    $2 !~ /^[a-z]+$/  ||  $3 !~ /^[a-z]+$/  || \
		    $4 !~ /^.$/  ||  $5 !~ /^.$/  ||  $6 !~ /^.$/)
			continue;
		if ($3 == "vpp")
			$3 = "*vpp";
		lockdata["vop_" $2, $3, "Entry"] = $4;
		lockdata["vop_" $2, $3, "OK"]    = $5;
		lockdata["vop_" $2, $3, "Error"] = $6;
		continue;
	}

	if ($1 ~ /^#!/) {
		if (NF != 4 || $1 != "#!")
			continue;
		if ($3 != "pre" && $3 != "post")
			continue;
		lockdata["vop_" $2, $3] = $4;
		continue;
	}
	if ($1 ~ /^#/)
		continue;

	# Get the function name.
	name = $1;
	uname = toupper(name);
	# Get the function arguments.
	for (numargs = 0; ; ++numargs) {
		if ((getline < srcfile) <= 0) {
			die("Unable to read through the arguments for \"%s\"",
			    name);
		}
		if ($1 ~ /^\};/)
			break;

		# Delete comments, if any.
		gsub (/\/\*.*\*\//, "");

		# Condense whitespace and delete leading/trailing space.
		gsub(/[[:space:]]+/, " ");
		sub(/^ /, "");
		sub(/ $/, "");

		# Pick off direction.
		if ($1 != "INOUT" && $1 != "IN" && $1 != "OUT")
			die("No IN/OUT direction for \"%s\".", $0);
		dirs[numargs] = $1;
		sub(/^[A-Z]* /, "");

		if ((reles[numargs] = $1) == "WILLRELE")
			sub(/^[A-Z]* /, "");
		else
			reles[numargs] = "WONTRELE";

		# kill trailing ;
		if (sub(/;$/, "") < 1)
			die("Missing end-of-line ; in \"%s\".", $0);

		# pick off variable name
		if ((argp = match($0, /[A-Za-z0-9_]+$/)) < 1)
			die("Missing var name \"a_foo\" in \"%s\".", $0);
		args[numargs] = substr($0, argp);
		$0 = substr($0, 1, argp - 1);

		# what is left must be type
		# remove trailing space (if any)
		sub(/ $/, "");
		types[numargs] = $0;
	}

	if (hfile) {
		# Print out the vop_F_args structure.
		printh("struct "name"_args {\n\tstruct vnodeop_desc *a_desc;");
		for (i = 0; i < numargs; ++i)
			printh("\t" t_spc(types[i]) "a_" args[i] ";");
		printh("};");

		# Print out extern declaration.
		printh("extern struct vnodeop_desc " name "_desc;");

		# Print out function.
		printh("static __inline int " uname "(");
		for (i = 0; i < numargs; ++i) {
			printh("\t" t_spc(types[i]) args[i] \
			    (i < numargs - 1 ? "," : ")"));
		}
		printh("{\n\tstruct " name "_args a;");
		printh("\tint rc;");
		printh("\ta.a_desc = VDESC(" name ");");
		for (i = 0; i < numargs; ++i)
			printh("\ta.a_" args[i] " = " args[i] ";");
		for (i = 0; i < numargs; ++i)
			add_debug_code(name, args[i], "Entry");
		add_debug_pre(name);
		printh("\trc = VCALL(" args[0] ", VOFFSET(" name "), &a);");
		printh("if (rc == 0) {");
		for (i = 0; i < numargs; ++i)
			add_debug_code(name, args[i], "OK");
		printh("} else {");
		for (i = 0; i < numargs; ++i)
			add_debug_code(name, args[i], "Error");
		printh("}");
		add_debug_post(name);
		printh("\treturn (rc);\n}");
	}

	if (cfile) {
		# Print out the vop_F_vp_offsets structure.  This all depends
		# on naming conventions and nothing else.
		printc("static int " name "_vp_offsets[] = {");
		# as a side effect, figure out the releflags
		releflags = "";
		vpnum = 0;
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode *") {
				printc("\tVOPARG_OFFSETOF(struct " name \
				    "_args,a_" args[i] "),");
				if (reles[i] == "WILLRELE") {
					releflags = releflags \
					    "|VDESC_VP" vpnum "_WILLRELE";
				}
				vpnum++;
			}
		}

		sub(/^\|/, "", releflags);
		printc("\tVDESC_NO_OFFSET");
		printc("};");

		# Print out the vnodeop_desc structure.
		printc("struct vnodeop_desc " name "_desc = {");
		# offset
		printc("\t0,");
		# printable name
		printc("\t\"" name "\",");
		# flags
		vppwillrele = "";
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode **" && \
			    reles[i] == "WILLRELE") {
				vppwillrele = "|VDESC_VPP_WILLRELE";
			}
		}

		if (!releflags)
			releflags = "0";
		printc("\t" releflags vppwillrele ",");

		# vp offsets
		printc("\t" name "_vp_offsets,");
		# vpp (if any)
		printc("\t" find_arg_with_type("struct vnode **") ",");
		# cred (if any)
		printc("\t" find_arg_with_type("struct ucred *") ",");
		# thread (if any)
		printc("\t" find_arg_with_type("struct thread *") ",");
		# componentname
		printc("\t" find_arg_with_type("struct componentname *") ",");
		# transport layer information
		printc("\tNULL,\n};\n");
	}
}

if (hfile)
	close(hfile);
if (cfile)
	close(cfile);
close(srcfile);

exit 0;

}

#!/usr/bin/awk -f

#
# Copyright (c) 1992, 1993
#	The Regents of the University of California.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#	This product includes software developed by the University of
#	California, Berkeley and its contributors.
# 4. Neither the name of the University nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#	@(#)vnode_if.sh	8.1 (Berkeley) 6/10/93
# $FreeBSD$
#
# Script to produce VFS front-end sugar.
#
# usage: vnode_if.awk <srcfile> [-c | -h]
#	(where <srcfile> is currently /sys/kern/vnode_if.src)
#

function usage()
{
	print "usage: vnode_if.awk <srcfile> [-c|-h]";
	exit 1;
}

function die(msg, what)
{
	printf msg "\n", what > "/dev/stderr";
	exit 1;
}

function t_spc(type)
{
	# Append a space if the type is not a pointer
	return (type ~ /\*$/) ? type : type " ";
}

# These are just for convenience ...
function printc(s) {print s > cfile;}
function printh(s) {print s > hfile;}

function add_debug_code(name, arg)
{
	if (lockdata[name, arg, "Entry"]) {
		# Add assertions for locking
		if (lockdata[name, arg, "Entry"] == "L")
			printh("\tASSERT_VOP_LOCKED("arg", \""uname"\");");
		else if (lockdata[name, arg, "Entry"] == "U")
			printh("\tASSERT_VOP_UNLOCKED("arg", \""uname"\");");
		else if (0) {
			# XXX More checks!
		}
	}
}

function add_debug_pre(name)
{
	if (lockdata[name, "pre"]) {
		printh("#ifdef	DEBUG_VFS_LOCKS");
		printh("\t"lockdata[name, "pre"]"(&a);");
		printh("#endif");
	}
}

function add_debug_post(name)
{
	if (lockdata[name, "post"]) {
		printh("#ifdef	DEBUG_VFS_LOCKS");
		printh("\t"lockdata[name, "post"]"(&a, rc);");
		printh("#endif");
	}
}

function find_arg_with_type (type)
{
	for (jj = 0; jj < numargs; jj++) {
		if (types[jj] == type) {
			return "VOPARG_OFFSETOF(struct " \
			    name "_args,a_" args[jj] ")";
		}
	}

	return "VDESC_NO_OFFSET";
}

BEGIN{

# Process the command line
for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[ch]+$/ && arg !~ /\.src$/)
		usage();
	if (arg ~ /^-.*c/)
		cfile = "vnode_if.c";
	if (arg ~ /^-.*h/)
		hfile = "vnode_if.h";
	if (arg ~ /\.src$/)
		srcfile = arg;
}
ARGC = 1;

if (!cfile && !hfile)
	exit 0;

if (!srcfile)
	usage();

common_head = \
    "/*\n" \
    " * This file is produced automatically.\n" \
    " * Do not modify anything in here by hand.\n" \
    " *\n" \
    " * Created from $FreeBSD$\n" \
    " */\n" \
    "\n";

if (hfile)
	printh(common_head "extern struct vnodeop_desc vop_default_desc;");

if (cfile) {
	printc(common_head \
	    "#include <sys/param.h>\n" \
	    "#include <sys/systm.h>\n" \
	    "#include <sys/vnode.h>\n" \
	    "\n" \
	    "struct vnodeop_desc vop_default_desc = {\n" \
	    "	1,\t\t\t/* special case, vop_default => 1 */\n" \
	    "	\"default\",\n" \
	    "	0,\n" \
	    "	NULL,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	NULL,\n" \
	    "};\n");
}

while ((getline < srcfile) > 0) {
	if (NF == 0)
		continue;
	if ($1 ~ /^#%/) {
		if (NF != 6  ||  $1 != "#%"  || \
		    $2 !~ /^[a-z]+$/  ||  $3 !~ /^[a-z]+$/  || \
		    $4 !~ /^.$/  ||  $5 !~ /^.$/  ||  $6 !~ /^.$/)
			continue;
		lockdata["vop_" $2, $3, "Entry"] = $4;
		lockdata["vop_" $2, $3, "OK"]    = $5;
		lockdata["vop_" $2, $3, "Error"] = $6;
		continue;
	}

	if ($1 ~ /^#!/) {
		if (NF != 4 || $1 != "#!")
			continue;
		if ($3 != "pre" && $3 != "post")
			continue;
		lockdata["vop_" $2, $3] = $4;
		continue;
	}
	if ($1 ~ /^#/)
		continue;

	# Get the function name.
	name = $1;
	uname = toupper(name);
	# Get the function arguments.
	for (numargs = 0; ; ++numargs) {
		if ((getline < srcfile) <= 0) {
			die("Unable to read through the arguments for \"%s\"",
			    name);
		}
		if ($1 ~ /^\};/)
			break;

		# Delete comments, if any.
		gsub (/\/\*.*\*\//, "");

		# Condense whitespace and delete leading/trailing space.
		gsub(/[[:space:]]+/, " ");
		sub(/^ /, "");
		sub(/ $/, "");

		# Pick off direction.
		if ($1 != "INOUT" && $1 != "IN" && $1 != "OUT")
			die("No IN/OUT direction for \"%s\".", $0);
		dirs[numargs] = $1;
		sub(/^[A-Z]* /, "");

		if ((reles[numargs] = $1) == "WILLRELE")
			sub(/^[A-Z]* /, "");
		else
			reles[numargs] = "WONTRELE";

		# kill trailing ;
		if (sub(/;$/, "") < 1)
			die("Missing end-of-line ; in \"%s\".", $0);

		# pick off variable name
		if ((argp = match($0, /[A-Za-z0-9_]+$/)) < 1)
			die("Missing var name \"a_foo\" in \"%s\".", $0);
		args[numargs] = substr($0, argp);
		$0 = substr($0, 1, argp - 1);

		# what is left must be type
		# remove trailing space (if any)
		sub(/ $/, "");
		types[numargs] = $0;
	}

	if (hfile) {
		# Print out the vop_F_args structure.
		printh("struct "name"_args {\n\tstruct vnodeop_desc *a_desc;");
		for (i = 0; i < numargs; ++i)
			printh("\t" t_spc(types[i]) "a_" args[i] ";");
		printh("};");

		# Print out extern declaration.
		printh("extern struct vnodeop_desc " name "_desc;");

		# Print out function.
		printh("static __inline int " uname "(");
		for (i = 0; i < numargs; ++i) {
			printh("\t" t_spc(types[i]) args[i] \
			    (i < numargs - 1 ? "," : ")"));
		}
		printh("{\n\tstruct " name "_args a;");
		printh("\tint rc;");
		printh("\ta.a_desc = VDESC(" name ");");
		for (i = 0; i < numargs; ++i)
			printh("\ta.a_" args[i] " = " args[i] ";");
		for (i = 0; i < numargs; ++i)
			add_debug_code(name, args[i]);
		add_debug_pre(name);
		printh("\trc = VCALL(" args[0] ", VOFFSET(" name "), &a);");
		add_debug_post(name);
		printh("\treturn (rc);\n}");
	}

	if (cfile) {
		# Print out the vop_F_vp_offsets structure.  This all depends
		# on naming conventions and nothing else.
		printc("static int " name "_vp_offsets[] = {");
		# as a side effect, figure out the releflags
		releflags = "";
		vpnum = 0;
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode *") {
				printc("\tVOPARG_OFFSETOF(struct " name \
				    "_args,a_" args[i] "),");
				if (reles[i] == "WILLRELE") {
					releflags = releflags \
					    "|VDESC_VP" vpnum "_WILLRELE";
				}
				vpnum++;
			}
		}

		sub(/^\|/, "", releflags);
		printc("\tVDESC_NO_OFFSET");
		printc("};");

		# Print out the vnodeop_desc structure.
		printc("struct vnodeop_desc " name "_desc = {");
		# offset
		printc("\t0,");
		# printable name
		printc("\t\"" name "\",");
		# flags
		vppwillrele = "";
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode **" && \
			    reles[i] == "WILLRELE") {
				vppwillrele = "|VDESC_VPP_WILLRELE";
			}
		}

		if (!releflags)
			releflags = "0";
		printc("\t" releflags vppwillrele ",");

		# vp offsets
		printc("\t" name "_vp_offsets,");
		# vpp (if any)
		printc("\t" find_arg_with_type("struct vnode **") ",");
		# cred (if any)
		printc("\t" find_arg_with_type("struct ucred *") ",");
		# thread (if any)
		printc("\t" find_arg_with_type("struct thread *") ",");
		# componentname
		printc("\t" find_arg_with_type("struct componentname *") ",");
		# transport layer information
		printc("\tNULL,\n};\n");
	}
}

if (hfile)
	close(hfile);
if (cfile)
	close(cfile);
close(srcfile);

exit 0;

}


added two prefixes for geonamesid and govtrack

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:         <http://open.vocab.org/terms/> ."
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}