TestCommons: update

TestCommons: update

BEGIN {
    START_TEST("Commons.awk")

    # Arrays
    T("anything()", 3)
    {
        delete something
        assertFalse(anything(nothing))
        something[0] = 0; assertFalse(anything(something)) # edge case
        something[0] = 1; assertTrue(anything(something))
    }
    T("exists()", 4)
    {
        delete something
        assertFalse(exists(something))
        something[0] = 0; assertFalse(exists(something)) # edge case
        something[0] = 1; assertTrue(exists(something))
        assertTrue(exists(something[0]))
    }
    T("belongsTo()", 3)
    {
        delete array; array[0] = "foo"; array[1] = "bar"
        assertTrue(belongsTo("foo", array))
        assertTrue(belongsTo("bar", array))
        assertFalse(belongsTo("world", array))
    }
    T("identical()", 4)
    {
        delete x; x[0] = 42
        delete y; y[0][0] = 42
        assertFalse(identical(x, y))
        assertTrue(identical(x, y[0]))
        assertFalse(identical(x[0], y[0]))
        assertTrue(identical(x[0], y[0][0]))
    }
    T("append()", 1)
    {
        delete array; array[0] = "foo"
        delete expected; expected[0] = "foo"; expected[1] = "bar"
        append(array, "bar")
        assertEqual(array, expected)
    }

    # Strings
    T("isnum()", 4)
    {
        assertTrue(isnum(0))
        assertTrue(isnum(42.0))
        assertFalse(isnum(""))
        assertFalse(isnum("hello world"))
    }
    T("startsWithAny()", 3)
    {
        delete substrings; substrings[0] = "A"; substrings[1] = "a"
        assertTrue(startsWithAny("absolute", substrings))
        assertTrue(startsWithAny("ABSOLUTE", substrings))
        assertFalse(startsWithAny("ZOO", substrings))
    }
    T("matchesAny()", 4)
    {
        delete patterns; patterns[0] = "[[:space:]]"; patterns[1] = "[0Oo]"
        assertTrue(matchesAny("  ", patterns))
        assertTrue(matchesAny("obsolete", patterns))
        assertTrue(matchesAny("0.0", patterns))
        assertFalse(matchesAny("1.0", patterns))
    }
    T("replicate()", 4)
    {
        assertEqual(replicate("", 0), "")
        assertEqual(replicate("", 2), "")
        assertEqual(replicate("foo bar", 1), "foo bar")
        assertEqual(replicate("foo bar", 3), "foo barfoo barfoo bar")
    }
    T("join()", 4)
    {
        assertEqual(join("", "-"), "")
        delete array; array[0]
        assertEqual(join(array, "-"), "")
        delete array; array[0] = "foo"; array[1] = "bar"
        assertEqual(join(array, " "), "foo bar")
        assertEqual(join(array, ","), "foo,bar")
    }
    T("explode()", 1)
    {
        delete array
        delete expected; expected[1] = "f"; expected[2] = "o"; expected[3] = "o"
        explode("foo", array)
        assertEqual(array, expected)
    }
    T("escapeChar()", 8)
    {
        assertEqual(escapeChar("b"), "\b")
        assertEqual(escapeChar("f"), "\f")
        assertEqual(escapeChar("n"), "\n")
        assertEqual(escapeChar("r"), "\r")
        assertEqual(escapeChar("t"), "\t")
        assertEqual(escapeChar("v"), "\v")
        assertEqual(escapeChar("_"), "_")
        assertNotEqual(escapeChar("_"), "\\_")
    }
    T("literal()", 4)
    {
        assertEqual(literal(""), "")
        assertEqual(literal("foo"), "foo")
        assertEqual(literal("\"foo\""), "foo")
        assertEqual(literal("\"\\\"foo\\\"\""), "\"foo\"")
    }
    T("escape()", 4)
    {
        assertEqual(escape(""), "")
        assertEqual(escape("foo"), "foo")
        assertEqual(escape("\""), "\\\\\"")
        assertEqual(escape("\"foo\""), "\\\\\"foo\\\\\"")
    }
    T("parameterize()", 10)
    {
        assertEqual(parameterize(""), "''")
        assertEqual(parameterize("foo"), "'foo'")
        assertEqual(parameterize("foo bar"), "'foo bar'")
        assertEqual(parameterize("foo 'bar"), "'foo '\\''bar'")
        assertEqual(parameterize("foo \"bar\""), "'foo \"bar\"'")
        assertEqual(parameterize("", "\""), "\"\"")
        assertEqual(parameterize("foo", "\""), "\"foo\"")
        assertEqual(parameterize("foo bar", "\""), "\"foo bar\"")
        assertEqual(parameterize("foo 'bar", "\""), "\"foo 'bar\"")
        assertEqual(parameterize("foo \"bar\"", "\""), "\"foo \\\\\"bar\\\\\"\"")
    }
    T("toString()", 4)
    {
        assertEqual(toString(""), "")
        assertEqual(toString(42), "42")
        assertEqual(toString("foo"), "foo")
        assertEqual(toString("\"foo bar\""), "\"foo bar\"")
    }
    T("squeeze()", 4)
    {
        assertEqual(squeeze(""), "")
        assertEqual(squeeze(" "), "")
        assertEqual(squeeze("	"), "")
        assertEqual(squeeze("  foo = bar #comments"), "foo = bar")
    }
    T("yn()", 12)
    {
        assertFalse(yn(0))
        assertFalse(yn("0"))
        assertFalse(yn("false"))
        assertFalse(yn("no"))
        assertFalse(yn("off"))
        assertFalse(yn("OFF"))
        assertTrue(yn(1))
        assertTrue(yn("1"))
        assertTrue(yn("true"))
        assertTrue(yn("yes"))
        assertTrue(yn("on"))
        assertTrue(yn("ON"))
    }

    # URLs
    T("quote()", 4)
    {
        assertEqual(quote(""), "")
        assertEqual(quote("foo"), "foo")
        assertEqual(quote("foo bar"), "foo%20bar")
        assertEqual(quote("\"hello, world!\""), "%22hello%2C%20world%21%22")
    }

    # System
    T("fileExists()", 4)
    {
        assertFalse(fileExists("README"))
        assertFalse(fileExists("README .md"))
        assertTrue(fileExists("README.md"))
        assertFalse(fileExists("."))
    }
    T("dirExists()", 4)
    {
        assertFalse(dirExists("README"))
        assertFalse(dirExists("README.md"))
        assertTrue(dirExists("."))
        assertFalse(dirExists(" ."))
    }
    T("detectProgram()", 2)
    {
        assertTrue(detectProgram("ls"))
        assertTrue(detectProgram("gawk", "--version"))
    }

    END_TEST()
}

BEGIN {
    START_TEST("Commons.awk")

    # Arrays
    T("anything()", 3)
    {
        delete something
        assertFalse(anything(nothing))
        something[0] = 0; assertFalse(anything(something)) # edge case
        something[0] = 1; assertTrue(anything(something))
    }
    T("exists()", 4)
    {
        delete something
        assertFalse(exists(something))
        something[0] = 0; assertFalse(exists(something)) # edge case
        something[0] = 1; assertTrue(exists(something))
        assertTrue(exists(something[0]))
    }
    T("belongsTo()", 3)
    {
        delete array; array[0] = "foo"; array[1] = "bar"
        assertTrue(belongsTo("foo", array))
        assertTrue(belongsTo("bar", array))
        assertFalse(belongsTo("world", array))
    }
    T("identical()", 4)
    {
        delete x; x[0] = 42
        delete y; y[0][0] = 42
        assertFalse(identical(x, y))
        assertTrue(identical(x, y[0]))
        assertFalse(identical(x[0], y[0]))
        assertTrue(identical(x[0], y[0][0]))
    }
    T("append()", 1)
    {
        delete array; array[0] = "foo"
        delete expected; expected[0] = "foo"; expected[1] = "bar"
        append(array, "bar")
        assertTrue(identical(array, expected))
    }

    # Strings
    T("isnum()", 4)
    {
        assertTrue(isnum(0))
        assertTrue(isnum(42.0))
        assertFalse(isnum(""))
        assertFalse(isnum("hello world"))
    }
    T("startsWithAny()", 3)
    {
        delete substrings; substrings[0] = "A"; substrings[1] = "a"
        assertTrue(startsWithAny("absolute", substrings))
        assertTrue(startsWithAny("ABSOLUTE", substrings))
        assertFalse(startsWithAny("ZOO", substrings))
    }
    T("matchesAny()", 4)
    {
        delete patterns; patterns[0] = "[[:space:]]"; patterns[1] = "[0Oo]"
        assertTrue(matchesAny("  ", patterns))
        assertTrue(matchesAny("obsolete", patterns))
        assertTrue(matchesAny("0.0", patterns))
        assertFalse(matchesAny("1.0", patterns))
    }
    T("replicate()", 4)
    {
        assertEqual(replicate("", 0), "")
        assertEqual(replicate("", 2), "")
        assertEqual(replicate("foo bar", 1), "foo bar")
        assertEqual(replicate("foo bar", 3), "foo barfoo barfoo bar")
    }
    T("join()", 4)
    {
        assertEqual(join("", "-"), "")
        delete array; array[0]
        assertEqual(join(array, "-"), "")
        delete array; array[0] = "foo"; array[1] = "bar"
        assertEqual(join(array, " "), "foo bar")
        assertEqual(join(array, ","), "foo,bar")
    }
    T("explode()", 1)
    {
        delete array
        delete expected; expected[1] = "f"; expected[2] = "o"; expected[3] = "o"
        explode("foo", array)
        assertTrue(identical(array, expected))
    }
    T("escapeChar()", 8)
    {
        assertEqual(escapeChar("b"), "\b")
        assertEqual(escapeChar("f"), "\f")
        assertEqual(escapeChar("n"), "\n")
        assertEqual(escapeChar("r"), "\r")
        assertEqual(escapeChar("t"), "\t")
        assertEqual(escapeChar("v"), "\v")
        assertEqual(escapeChar("_"), "_")
        assertNotEqual(escapeChar("_"), "\\_")
    }
    T("literal()", 4)
    {
        assertEqual(literal(""), "")
        assertEqual(literal("foo"), "foo")
        assertEqual(literal("\"foo\""), "foo")
        assertEqual(literal("\"\\\"foo\\\"\""), "\"foo\"")
    }
    T("escape()", 4)
    {
        assertEqual(escape(""), "")
        assertEqual(escape("foo"), "foo")
        assertEqual(escape("\""), "\\\\\"")
        assertEqual(escape("\"foo\""), "\\\\\"foo\\\\\"")
    }
    T("parameterize()", 10)
    {
        assertEqual(parameterize(""), "''")
        assertEqual(parameterize("foo"), "'foo'")
        assertEqual(parameterize("foo bar"), "'foo bar'")
        assertEqual(parameterize("foo 'bar"), "'foo '\\''bar'")
        assertEqual(parameterize("foo \"bar\""), "'foo \"bar\"'")
        assertEqual(parameterize("", "\""), "\"\"")
        assertEqual(parameterize("foo", "\""), "\"foo\"")
        assertEqual(parameterize("foo bar", "\""), "\"foo bar\"")
        assertEqual(parameterize("foo 'bar", "\""), "\"foo 'bar\"")
        assertEqual(parameterize("foo \"bar\"", "\""), "\"foo \\\\\"bar\\\\\"\"")
    }
    T("toString()", 4)
    {
        assertEqual(toString(""), "")
        assertEqual(toString(42), "42")
        assertEqual(toString("foo"), "foo")
        assertEqual(toString("\"foo bar\""), "\"foo bar\"")
    }
    T("squeeze()", 4)
    {
        assertEqual(squeeze(""), "")
        assertEqual(squeeze(" "), "")
        assertEqual(squeeze("	"), "")
        assertEqual(squeeze("  foo = bar #comments"), "foo = bar")
    }
    T("yn()", 12)
    {
        assertFalse(yn(0))
        assertFalse(yn("0"))
        assertFalse(yn("false"))
        assertFalse(yn("no"))
        assertFalse(yn("off"))
        assertFalse(yn("OFF"))
        assertTrue(yn(1))
        assertTrue(yn("1"))
        assertTrue(yn("true"))
        assertTrue(yn("yes"))
        assertTrue(yn("on"))
        assertTrue(yn("ON"))
    }

    # URLs
    T("quote()", 4)
    {
        assertEqual(quote(""), "")
        assertEqual(quote("foo"), "foo")
        assertEqual(quote("foo bar"), "foo%20bar")
        assertEqual(quote("\"hello, world!\""), "%22hello%2C%20world%21%22")
    }

    # System
    T("fileExists()", 4)
    {
        assertFalse(fileExists("README"))
        assertFalse(fileExists("README .md"))
        assertTrue(fileExists("README.md"))
        assertFalse(fileExists("."))
    }
    T("dirExists()", 4)
    {
        assertFalse(dirExists("README"))
        assertFalse(dirExists("README.md"))
        assertTrue(dirExists("."))
        assertFalse(dirExists(" ."))
    }
    T("detectProgram()", 2)
    {
        assertTrue(detectProgram("ls"))
        assertTrue(detectProgram("gawk", "--version"))
    }

    END_TEST()
}


Work on adding episode types

Work on adding episode types

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode type, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-.[[:digit:]]{3,4}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2) + 0
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a Trailer or Finale (i.e. First look), set episodeType to "T"
    if (episodeURL ~ /-trailer-|-finale-/)
        episodeType = "T"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeURL ~ /-c-x[[:digit:]]{3,4}$/)
        episodeType = "X"
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number, and correct if necessary
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0 + 0
    if (episodeType == "T") {
        printf ("==> Corrected %s%02d to T01: %s\n", episodeType, episodeNumber, \
                shortURL) >> ERROR_FILE
        episodeNumber = 1
    }
    if ((episodeURL ~ /-c-.[[:digit:]]{3,4}$/) && episodeNumber != episodeNumberFromURL) {
        printf ("==> Corrected %s%02d to %s%02d: %s\n", episodeType, episodeNumber, episodeType, \
                episodeNumberFromURL, shortURL) >> ERROR_FILE
        episodeNumber = episodeNumberFromURL
    }
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected %s00 to %s%02d: %s\n", episodeType, episodeType,
                        episodeNumberFromURL, shortURL) >> ERROR_FILE
                episodeNumber = episodeNumberFromURL
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
                SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeType, episodeNumber, \
                episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " shortURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    if (episodeURL ~ /-c-[[:digit:]]{5}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2)
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0
    if (seriesTitle == "Detective Montalbano" && page2 == "yes") {
        oldEpisodeNumber = episodeNumber
        episodeNumber += 24
        printf ("==> Corrected E%02d to E%02d: %s\n", oldEpisodeNumber, episodeNumber, \
                shortURL) >> ERROR_FILE
    }
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected E00 to E%02d: %s\n", episodeNumber, shortURL) >> ERROR_FILE
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02dE%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
            SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeNumber, episodeTitle, \
            episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " shortURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}


Commons: add initAnsiCode()

Commons: add initAnsiCode()

#!/usr/bin/gawk -f

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if(ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[21m"
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        if (preserveNull || array[i] != "")
            temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Debug an array.
function da(array, formatString, sortedIn,
                ####
                i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

#!/usr/bin/gawk -f

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        if (preserveNull || array[i] != "")
            temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Debug an array.
function da(array, formatString, sortedIn,
                ####
                i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}


Changes in marksession.awk awk script to have a better session ID, especially when Frames do not have Call-ID (before: session ID was previous one, after: session-ID is {0}

git-svn-id: 392d0e5ce652024192c571322d40dcc23d6d83b0@17 8a65f31f-a56f-4bd6-98e8-2755f64920ec

BEGIN {
  session_count = 0;
}

{
  if ($1 ~ "[0-9]+") {
    # See if the current line is already in the table
    frame_file = destDir "/frames/Frame"$1".html"
    session_line = ""
    "grep -i \"" session_token "\" " frame_file | getline session_line

    if (session_line == "" ) {
      $4 = "{0}"
      print $0

    } else {
      found = 0;
      split(session_line, array)
      session = array[2]
      for (i=0; i < session_count; i++) {
        if (sessions[i] == session) {
          found = 1
          $4 = "{" i+1 "}"
          print $0
          break
        }
      }

      if (found == 0) {
        $4 = "{" session_count+1 "}"
        sessions[session_count++] = session
        print $0
        s = sprintf("echo 'New session in frame %s: %s' >&2", $1, session)
        system(s)
      }
    }

  } else {
    # Comment line, line starting with "#".
    # Requires no processing, just print it
    print $0
  }
}

BEGIN {
   session_count = 0;
}

{
   if ($1 ~ "[0-9]+") {
	  /* See if the current line is already in the table */
	  frame_file = destDir "/frames/Frame"$1".html";
	  "grep -i \"" session_token "\" " frame_file | getline session_line;
	  found = 0;
	  split(session_line, array);
	  session = array[2];

	  for (i=0; i < session_count; i++) {
		 if (sessions[i] == session) {
			found = 1;
			$4 = "{" i+1 "}";
			print $0;
			break;
		 }
	  }
	  if (found == 0) {
		 $4 = "{" session_count+1 "}";
		 sessions[session_count++] = session;
		 print $0;
		 s = sprintf("echo 'New session in frame %s: %s' >&2", $1, session);
		 system(s);
	  }

   } else {
	  print $0;
   }
}


Document the head of the script

Document the head of the script

#!/usr/bin/awk -f
#
# Combine multiple pkg-plist files into a single pkg-plist.
#
# This takes a list of all options as command line arguments. The
# order of the options determines the order option specific plist
# entries appear in the combined output.
#
# The plists must be fed into stdin and preceded by a line starting
# with `OPTIONS:` followed by a white space separated list of the
# active options.
#
# Files that appear with all options are listed first, followed by
# the option specific output.
#
# @param ARGV
#	A sorted list of options
#

# Get the order of options
BEGIN {
	OPTION_STR["DOCS"] =     "%%PORTDOCS%%"
	OPTION_STR["EXAMPLES"] = "%%PORTEXAMPLES%%"
	for (i = 1; i < ARGC; ++i) {
		OPTIONS_ORDERD[i] = ARGV[i]
		CNT_OPT_FILES[ARGV[i]] = 0
		if (!(ARGV[i] in OPTION_STR)) {
			OPTION_STR[ARGV[i]] = "%%" ARGV[i] "%%"
		}
		if (!("NO_" ARGV[i] in OPTION_STR)) {
			OPTION_STR["NO_" ARGV[i]] = "%%NO_" ARGV[i] "%%"
		}
		delete ARGV[i]
	}
	CNT_FILES = 0
}

# Get the options the following files were staged with
/^OPTIONS:/ {
	delete aoptions
	sub(/^OPTIONS: */, "")
	cnt_aoptions = split($0, aoptions)
	for (i = 1; i <= cnt_aoptions; ++i) {
		++OPTIONS[aoptions[i]]
	}
	++CONFIGS
	next
}

# Collect files
{
	# Record order of file
	if (!($0 in FILES)) {
		FILES_ORDERED[++CNT_FILES] = $0
	}
	# Count occurence of file
	++FILES[$0]
	# The same book keepin per option
	for (i = 1; i <= cnt_aoptions; ++i) {
		option = aoptions[i]
		# Record order of file for option
		if (!OPT_FILES[option, $0]) {
			OPT_FILES_ORDERED[option, ++CNT_OPT_FILES[option]] = $0
		}
		# Count occurence of file by option
		++OPT_FILES[option, $0]
	}
}

# Print files
END {
	# Print files common to all configurations
	for (i = 1; i <= CNT_FILES; ++i) {
		file = FILES_ORDERED[i]
		if (FILES[file] == CONFIGS) {
			print file
			delete FILES[file]
		}
	}
	# Print option specific files
	for (i = 1; OPTIONS_ORDERD[i]; ++i) {
		option = OPTIONS_ORDERD[i]
		for (p = 1; p <= CNT_OPT_FILES[option]; ++p) {
			file = OPT_FILES_ORDERED[option, p]
			# Skip files that have already been printed
			if (!(file in FILES)) { continue }
			# Print file if it only occurs for the current
			# option
			if (OPT_FILES[option, file] == OPTIONS[option] &&
			    OPT_FILES[option, file] == FILES[file]) {
				print OPTION_STR[option] file
				delete FILES[file]
			}
		}
		for (p = 1; p <= CNT_FILES; ++p) {
			file = FILES_ORDERED[p]
			if (!(file in FILES)) { continue }
			# Print file if it occurs everywhere but with
			# this option
			if (!OPT_FILES[option, file] &&
			    FILES[file] + OPTIONS[option] == CONFIGS) {
				print OPTION_STR["NO_" option] file
				delete FILES[file]
			}
		}
	}
	# Print all files that have not been printed
	for (i = 1; i <= CNT_FILES; ++i) {
		file = FILES_ORDERED[i]
		if (!(file in FILES)) { continue }
		print "@fail " file " could not be mapped to an option!"
		msg = "@fail Candidates:"
		for (p = 1; OPTIONS_ORDERD[p]; ++p) {
			if (OPT_FILES[OPTIONS_ORDERD[p], file]) {
				msg = msg " " OPTIONS_ORDERD[p]
			}
		}
		print msg
	}
}


Improve regex portability

Improve regex portability

BEGIN {
	RS = "{\""
}

{
	len = split($0, items, /,"/)
	if (len > 1) {
		sub(/^{"/, "", items[1])
		sub(/}\s*$/, "", items[len])
	}
	for (i = 1; i <= len; i++) {
		item = items[i]
		j = index(item, "\":")
		key = substr(item, 1, j-1)
		value = substr(item, j+2)
		object[key] = value
	}
}

function amount(key) {
	return object[key]/10000
}

function str(key) {
	value = object[key]
	sub(/^"/, "", value)
	sub(/"[}\]]*$/, "", value)
	return value
}

BEGIN {
	RS = "{\""
}

{
	len = split($0, items, /,"/)
	if (len > 1) {
		sub(/^{"/, "", items[1])
		sub(/}$/, "", items[len])
	}
	for (i = 1; i <= len; i++) {
		item = items[i]
		j = index(item, "\":")
		key = substr(item, 1, j-1)
		value = substr(item, j+2)
		object[key] = value
	}
}

function amount(key) {
	return object[key]/10000
}

function str(key) {
	value = object[key]
	sub(/^"/, "", value)
	sub(/"[}\]]*$/, "", value)
	return value
}


Update min/max length to be inclusive

Update min/max length to be inclusive

see: https://trello.com/c/RU4BWrYA/340-as-a-data-manager-adding-min-max-values-in-the-jts-they-should-be-inclusive

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">=" : "<="
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">=" : "<="
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">=" : "<="
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


Remove no longer needed kludge

Remove no longer needed kludge

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-[[:digit:]]{5}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2)
    episodeTitle = fld[4]
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected E00: " episodeURL " to E" episodeNumber >> ERROR_FILE
            } else {
                print "==> Episode number is 00: " episodeURL >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02dE%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
            SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeNumber, episodeTitle, \
            episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " episodeURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-[[:digit:]]{5}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2)
    episodeTitle = fld[4]
    # kludge for missing episode number in mammon/season:1/videos/the-murder
    if (episodeTitle ~ /Sn 1 Ep 5/)
        episodeNumberFromURL = "05"
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number
/<h4 class="/ {
    sub (/.*Episode /,"")
    sub (/<\/span>.*/,"")
    episodeNumber = $0
    next
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/<div class="transparent padding-top-medium"/,/<\/div>/ {
    if ($0 ~ /
 Corrected E00: " episodeURL " to E" episodeNumber >> ERROR_FILE
            } else {
                print "==> Episode number is 00: " episodeURL >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02dE%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
            SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeNumber, episodeTitle, \
            episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "")
            print "==> No description: " episodeURL >> ERROR_FILE
        print episodeDescription >> EPISODE_INFO_FILE
    }
}


minor: whitespace

minor: whitespace

#
# Simple AWK script to process lines of the corpus and print summed instance count for each unique n-gram.
#
# NB:-
# * n-grams containing characters other than a-z and quote are ignored.  You may wish to modify the regex.
# * Assumes the input corpus is sorted.
#
{
    if ( $1 ~ /^[a-zA-Z' ]+$/ ) {
        if ( $1 == last ) {
            count+=$3
        } else if ( last != "" ) {
            print last"\t"count
            count=$3
        }
        last=$1
    }
}
END { print last"\t"count }

#
# Simple AWK script to process lines of the corpus and print summed instance count for each unique n-gram.
#
# NB:-
# * n-grams containing characters other than a-z and quote are ignored.  You may wish to modify the regex.
# * Assumes the input corpus is sorted.
#
{
    if ($1 ~ /^[a-zA-Z' ]+$/) {
        if ( $1 == last ) {
            count+=$3
        } else if ( last != "" ) {
            print last"\t"count
            count=$3
        }
        last=$1
    }
}
END { print last"\t"count }


Add the SENSES option to linked-isas.awk

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    if (length(SENSES) == 0) SENSES = 0;
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0 || nisas == 0) next;

    for (sense in senses) {
        for (isa in isas) {
            if (SENSES) {
                sub(/(:|$)/, OFS, isas[isa]);
            } else {
                sub(/#[[:digit:]]+$/, "", senses[sense]);
                sub(/#[[:digit:]]+(:|$)/, OFS, isas[isa]);
            }
            print senses[sense], isas[isa] | "sort --parallel=$(nproc) -t \"\t\" -S1G -s -g -k3r | cut -f1,2";
        }
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0 || nisas == 0) next;

    for (sense in senses) {
        for (isa in isas) {
            sub(/#[[:digit:]]+$/, "", senses[sense]);
            sub(/#[[:digit:]]+(:|$)/, OFS, isas[isa]);
            print senses[sense], isas[isa] | "sort --parallel=$(nproc) -t \"\t\" -S1G -s -g -k3r | cut -f1,2";
        }
    }
}


Revert.

Revert.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@38824 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back] *.tdb

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if ending year is gt or eq to contract year
  current_year = 2014
  end_year = 0
  incontract = 0
  test_year = ""
  if ($7 == "") {
    test_year = $3
  } else {
    test_year = $7
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  #printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$7,end_year,test_year
  if (end_year >= test_year) {
    if ((end_year < current_year) || ((end_year >= current_year) && (incl_cur == 1))) {
      incontract = 1
    }
  }

  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      if ($7 == "") {
        k[pn] = ".."
      } else {
        k[pn] = $7
      }
      d[pn] = $4
      r[pn] = $5
      pn++
    }
    b[$1,lp2,$4]++
    c[$1,lp2,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "crawling"
  s[8] = "frozen"
  s[9] = "ingNotReady"
  s[10] = "finished"
  s[11] = "down"
  s[12] = "superseded"
  s[13] = "zapped"
  s[14] = "doNotProcess"
  sn = 15

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "craw"
  sc[8] = "froz"
  sc[9] = "ingN"
  sc[10] = "fini"
  sc[11] = "down"
  sc[12] = "supe"
  sc[13] = "zapp"
  sc[14] = "doNP"
  scn = 15

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],d[i],s[j]] == 0) {
        printf "\t.."
      } else {
        printf "\t%d", c[p[i],n[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
# tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back] *.tdb

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if ending year is gt or eq to contract year
  current_year = 2015
  end_year = 0
  incontract = 0
  test_year = ""
  if ($7 == "") {
    test_year = $3
  } else {
    test_year = $7
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  #printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$7,end_year,test_year
  if (end_year >= test_year) {
    if ((end_year < current_year) || ((end_year >= current_year) && (incl_cur == 1))) {
      incontract = 1
    }
  }

  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      if ($7 == "") {
        k[pn] = ".."
      } else {
        k[pn] = $7
      }
      d[pn] = $4
      r[pn] = $5
      pn++
    }
    b[$1,lp2,$4]++
    c[$1,lp2,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "crawling"
  s[8] = "frozen"
  s[9] = "ingNotReady"
  s[10] = "finished"
  s[11] = "down"
  s[12] = "superseded"
  s[13] = "zapped"
  s[14] = "doNotProcess"
  sn = 15

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "craw"
  sc[8] = "froz"
  sc[9] = "ingN"
  sc[10] = "fini"
  sc[11] = "down"
  sc[12] = "supe"
  sc[13] = "zapp"
  sc[14] = "doNP"
  scn = 15

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],d[i],s[j]] == 0) {
        printf "\t.."
      } else {
        printf "\t%d", c[p[i],n[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}


Compute the CloudFront records dynamically.

function printUrlQueryString(queryString)
{
    split(queryString, pairs, "&", seps)
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=", seps2)
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    print "{";
}

END {
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ", seps)
    numFields = length(fields)
}

! /^#/ {
    split($0, fieldsThisLine, " ", sepsThrowAway)
    print "\t{"
    for(i = 1; i < numFields; i++)
    {
        printf "\t\t\"%s\": \"%s\",\n", fields[i], fieldsThisLine[i]
    }
    print "\t},"
}

BEGIN {
    print "[";
}

function printUrlQueryString(queryString)
{
    split(queryString, pairs, "&", seps)
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=", seps2)
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

/^#/ {
  # Ignore log comment lines
}

/^[1-9][0-9]*/ {
    date = $1                         # date
    time = $2                         # time
    x_edge_location = $3              # x-edge-location
    sc_bytes = $4                     # sc-bytes
    c_ip = $5                         # c-ip
    cs_method = $6                    # cs-method
    cs_Host = $7                      # cs(Host)
    cs_uri_stem = $8                  # cs-uri-stem
    sc_status = $9                    # sc-status
    cs_Referer = $10                  # cs(Referer)
    cs_User_Agent = $11               # cs(User-Agent)
    cs_uri_query = $12                # cs-uri-query
    cs_Cookie = $13                   # cs(Cookie)
    x_edge_result_type = $14          # x-edge-result-type
    x_edge_request_id = $15           # x-edge-request-id
    x_host_header = $16               # x-host-header
    cs_protocol = $17                 # cs-protocol
    cs_bytes = $18                    # cs-bytes
    time_taken = $19                  # time-taken
    x_forwarded_for = $20             # x-forwarded-for
    ssl_protocol = $21                # ssl-protocol
    ssl_cipher = $22                  # ssl-cipher
    x_edge_response_result_type = $23 # x-edge-response-result-type
    printf "{\n\t\"date\": \"%s\",\n\t\"time\":\"%s\",\n\t\"x-edge-location\":\"\%s\",\n\t\"sc-bytes\": \"%s\",\n\t\"c-ip\": \"%s\",\n\t\"cs-method\": \"%s\",\n\t\"cs(Host)\": \"%s\",\n\t\"cs-uri-stream\": \"%s\",\n\t\"sc-status\": \"%s\",\n\t\"cs(Referer)\": \"%s\",\n\t\"cs(User-Agent)\": \"%s\",\n\t\"cs(Cookie)\": \"%s\",\n\t\"x-edge-result-type\": \"%s\",\n\t\"x-edge-request-id\": \"%s\",\n\t\"x-host-header\": \"%s\"\n\t\"cs-protocol\": \"%s\",\n\t\"cs-bytes\": \"%s\",\n\t\"time-taken\": \"%s\",\n\t\"x-forwarded-for\": \"%s\",\n\t\"ssl-protocol\": \"%s\",\n\t\"ssl-cipher\": \"%s\",\n\t\"x-edge-response-result-type\": \"%s\",\n", date, time, x_edge_location, sc_bytes, c_ip, cs_method, cs_Host, cs_uri_stem, sc_status, cs_Referer, urlDecode(urlDecode(cs_User_Agent)), cs_Cookie, x_edge_result_type, x_edge_request_id, x_host_header, cs_protocol, cs_bytes, time_taken, x_forwarded_for, ssl_protocol, ssl_cipher, x_edge_response_result_type
  printf "\t\"cs-uri-query\": {\n"
  printUrlQueryString(cs_uri_query)
  printf "\t}\n"
}

END {
  print "]";
}


add a diff line so you can check your commit before hitting enter...

#!/usr/bin/awk -f
#
# $FreeBSD$
#

BEGIN {
	CVSROOT="ncvs:/home/ncvs"
	UPDATEOPTS="-kk"
}

/^>/ {
	sub(">[ 	]*", "")
}

/^Revision/ || $1 == "" {
	next
}

{
	if (sub("1.", "") != 1)
		next
	if (!(match($2, "\\+[0-9]") && match($3, "-[0-9]")))
		next
	printf("cvs -d %s update %s -j 1.%d -j 1.%d %s\n", CVSROOT, UPDATEOPTS, $1 - 1, $1, $4)
	files[$4] = 1
}

END {
	for (i in files)
		fl = fl " " i
	printf("cvs -d %s diff -kk %s | less\n", CVSROOT, fl);
	printf("cvs -d %s commit %s\n", CVSROOT, fl);
}

#!/usr/bin/awk -f
#
# $FreeBSD$
#

BEGIN {
	CVSROOT="ncvs:/home/ncvs"
	UPDATEOPTS="-kk"
}

/^>/ {
	sub(">[ 	]*", "")
}

/^Revision/ || $1 == "" {
	next
}

{
	if (sub("1.", "") != 1)
		next
	if (!(match($2, "\\+[0-9]") && match($3, "-[0-9]")))
		next
	printf("cvs -d %s update %s -j 1.%d -j 1.%d %s\n", CVSROOT, UPDATEOPTS, $1 - 1, $1, $4)
	files = files " " $4
}

END {
	printf("cvs -d %s commit %s\n", CVSROOT, files);
}


Only link classes and insert paragraph breaks.

# Qick hack to convert pydoc to something decent looking in html.
# Author: Thomas Moeller
# Usage: pydoc3.3 inventor | awk -f pydoc2html.awk > inventor.html
#
# Example for updating a man directory:
# for f in *.html; do pydoc3.3 ${f%%.html} | awk -f ../../PyInventor/utils/pydoc2html.awk > $f; done
#
# Copyright (C) the PyInventor contributors. All rights reserved.
# This file is part of PyInventor, distributed under the BSD 3-Clause
# License. For full terms see the included COPYING file.
#

BEGIN   {
          eatEmptyLines = 0;
          indendet = 0;
          dataDesc = 0;
          isBullet = 0;
          print "<!DOCTYPE html>";
          print "<meta name='description' content='PyInventor : 3D Graphics in Python with Open Inventor' />";
          print "<link rel='stylesheet' type='text/css' media='screen' href='../stylesheets/stylesheet.css'>";
          print "<title>PyInventor</title>";
          print "
";
          print "View on GitHub";
          print "
PyInventor

3D Graphics in Python with Open Inventor
";
          print "
";

          print "

";
        }

END     {
          print "
";
          print "
";
          print "
PyInventor maintained by TheHubbit
";
          print "
Published with GitHub Pages

";
          print "";
        }

        { sub(/^[ \t\|]+/, ""); gsub(/[\<\>]/, ""); }
/^Help on/ { getline; next; }
/ = class / { print "
" $1 "

" $3 " " $4 "
"; module = substr($1, 0, index($1, ".") - 1); next; }
/^[ ]*FILE$/ { getline; next; }
/^[ ]*NAME$/ { getline; print "
" $1 "

module " $0 "
"; module = $1; next; }
/^[ ]*DESCRIPTION$/ { next; }
/^[ ]*FUNCTIONS$/ { print "
Functions:
"; next; }
/^[ ]*[A-Z]+$/ { print "
" $0 "
"; next; }
/PACKAGE CONTENTS/ {
            print "
Package contents:

";
            while (!match($0, /^[ \|\t\n]*$/))
            {
              getline;
              if ((length($0) > 2) && !match($0, "__main__"))
              {
                print "
" $0 "";
              }
            }
            print "
";
            isBullet = 0; next;
          }
/Method resolution order:/ {
            print "
" $0 "

";
            while (!match($0, /^[ \|\t]*$/))
            {
              getline;
              if (match($2, /PySide\./)) { link = $2; gsub(/\./, "/", link); print "
" $2 ""; } else
              if (match($2, /FieldContainer/)) print "
" $2 ""; else
              if (length($2) > 0) print "
" $2;
            }
            print "
";
            isBullet = 0; next;
          }
/Methods inherited from PySide/ { exit; }
/^[ ]*[A-Z][a-z]+ [A-Za-z \.]+:$/ { header = "
 " $0 "
"; eatEmptyLines = 1; if (match($0, /[ ]*Data /)) dataDesc = 1; next; }
/__/    { getline; next; }
/^[ \t]*[a-zA-Z0-9_]+\([A-Za-z0-9_\., ]+\)$/ {
          sub(/^[ \t]*/, "");
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          if (length(header) > 0)
          {
            print header;
            header = "";
          }
          print "
" $0 "
"; next;
        }
/Args:|Returns:|Note:/ {
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          print "
" $0 "
"; print "
"; indendet = 1; next;
        }
/^[ ]*\- [a-zA-Z0-9_, ]+: / {
          if (!isBullet) print "
";
          target = substr($2, 0, length($2) -1);
          if (target == "inventor") print "
" $2 "";
          else if (match(target, /^[A-Z]/)) print "
" $2 "";
          else print "
" $2;

          sub(/^[ ]*\- [a-zA-Z0-9_, ]+: /, "");
          print;
          isBullet = 1; next;
        }
/^[ -]*[a-zA-Z0-9_, ]+: / { sub(/^[ -]+/, ""); print "
" $0; isBullet = 1; next; }
/-----/ { print ""; dataDesc = 0; next; }
        {
          sub(/^[ \t]*/, "");
          if (dataDesc)
          {
            if (indendet)
            {
              indendet = 0;
              print "
";
            }
            if ((length(header) > 0) && (length($0) > 0))
            {
              print header;
              header = "";
            }
            if (match($0, /^[ \t]*[a-zA-Z0-9_]+$/))
            {
              print "
" $0 "
";
            }
            else if (length($0) > 0)
            {
              if (index($0, " = ") > 0)
              {
                print $0 "
";
              }
              else print $0;
            }
            else print "
";
          }
          else if (!eatEmptyLines || length($0) > 0)
          {
            print;
          }

          if (!isBullet && !eatEmptyLines && length($0) == 0)
          {
            print "
";
          }

          if (length($0) > 0) eatEmptyLines = 0; else if (isBullet)
          {
            print "
";
            isBullet = 0;
          }
        }

# Qick hack to convert pydoc to something decent looking in html.
# Author: Thomas Moeller
# Usage: pydoc3.3 inventor | awk -f pydoc2html.awk > inventor.html
#
# Example for updating a man directory:
# for f in *.html; do pydoc3.3 ${f%%.html} | awk -f ../../PyInventor/utils/pydoc2html.awk > $f; done
#
# Copyright (C) the PyInventor contributors. All rights reserved.
# This file is part of PyInventor, distributed under the BSD 3-Clause
# License. For full terms see the included COPYING file.
#

BEGIN   {
          eatEmptyLines = 0;
          indendet = 0;
          dataDesc = 0;
          isBullet = 0;
          print "<!DOCTYPE html>";
          print "<meta name='description' content='PyInventor : 3D Graphics in Python with Open Inventor' />";
          print "<link rel='stylesheet' type='text/css' media='screen' href='../stylesheets/stylesheet.css'>";
          print "<title>PyInventor</title>";
          print "
";
          print "View on GitHub";
          print "
PyInventor

3D Graphics in Python with Open Inventor
";
          print "
";

          print "

";
        }

END     {
          print "
";
          print "
";
          print "
PyInventor maintained by TheHubbit
";
          print "
Published with GitHub Pages

";
          print "";
        }

        { sub(/^[ \t\|]+/, ""); gsub(/[\<\>]/, ""); }
/^Help on/ { getline; next; }
/ = class / { print "
" $1 "

" $3 " " $4 "
"; module = substr($1, 0, index($1, ".") - 1); next; }
/^[ ]*FILE$/ { getline; next; }
/^[ ]*NAME$/ { getline; print "
" $1 "

module " $0 "
"; module = $1; next; }
/^[ ]*DESCRIPTION$/ { next; }
/^[ ]*FUNCTIONS$/ { print "
Functions:
"; next; }
/^[ ]*[A-Z]+$/ { print "
" $0 "
"; next; }
/PACKAGE CONTENTS/ {
            print "
Package contents:

";
            while (!match($0, /^[ \|\t\n]*$/))
            {
              getline;
              if ((length($0) > 2) && !match($0, "__main__"))
              {
                print "
" $0 "";
              }
            }
            print "
";
            isBullet = 0; next;
          }
/Method resolution order:/ {
            print "
" $0 "

";
            while (!match($0, /^[ \|\t]*$/))
            {
              getline;
              if (match($2, /PySide\./)) { link = $2; gsub(/\./, "/", link); print "
" $2 ""; } else
              if (match($2, /FieldContainer/)) print "
" $2 ""; else
              if (length($2) > 0) print "
" $2;
            }
            print "
";
            isBullet = 0; next;
          }
/Methods inherited from PySide/ { exit; }
/^[ ]*[A-Z][a-z]+ [A-Za-z \.]+:$/ { header = "
 " $0 "
"; eatEmptyLines = 1; if (match($0, /[ ]*Data /)) dataDesc = 1; next; }
/__/    { getline; next; }
/^[ \t]*[a-zA-Z0-9_]+\([A-Za-z0-9_\., ]+\)$/ {
          sub(/^[ \t]*/, "");
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          if (length(header) > 0)
          {
            print header;
            header = "";
          }
          print "
" $0 "
"; next;
        }
/Args:|Returns:|Note:/ {
          if (indendet)
          {
            indendet = 0;
            print "
";
          }
          print "
" $0 "
"; print "
"; indendet = 1; next;
        }
/^[ ]*\- [a-zA-Z0-9_, ]+: / {
          if (!isBullet) print "
";
          target = substr($2, 0, length($2) -1);
          if (target == "inventor") print "
" $2 "";
          else print "
" $2 "";
          sub(/^[ ]*\- [a-zA-Z0-9_, ]+: /, "");
          print;
          isBullet = 1; next;
        }
/^[ -]*[a-zA-Z0-9_, ]+: / { sub(/^[ -]+/, ""); print "
" $0; isBullet = 1; next; }
/-----/ { print ""; dataDesc = 0; next; }
        {
          sub(/^[ \t]*/, "");
          if (dataDesc)
          {
            if (indendet)
            {
              indendet = 0;
              print "
";
            }
            if ((length(header) > 0) && (length($0) > 0))
            {
              print header;
              header = "";
            }
            if (match($0, /^[ \t]*[a-zA-Z0-9_]+$/))
            {
              print "
" $0 "
";
            }
            else if (length($0) > 0)
            {
              if (index($0, " = ") > 0)
              {
                print $0 "
";
              }
              else print $0;
            }
          }
          else if (!eatEmptyLines || length($0) > 0)
          {
            print;
          }
          if (length($0) > 0) eatEmptyLines = 0; else if (isBullet)
          {
            print "
";
            isBullet = 0;
          }
        }


hinter: use same hint for all occurrences of match

This patch changes the hinter to use the same hint for all occurrences
of a match, thereby reducing the need to use multi-character hints and
also eliminating redundant hints to simplify the user's hint decision.

BEGIN {
  n_matches = 0;
  COMPACT_HINTS = ENVIRON["COMPACT_HINTS"];

  HINTS[0] = "p"
  HINTS[1] = "o"
  HINTS[2] = "i"
  HINTS[3] = "u"
  HINTS[4] = "l"
  HINTS[5] = "k"
  HINTS[6] = "j"
  HINTS[7] = "t"
  HINTS[8] = "r"
  HINTS[9] = "e"
  HINTS[10] = "wj"
  HINTS[11] = "wt"
  HINTS[12] = "wr"
  HINTS[13] = "we"
  HINTS[14] = "ww"
  HINTS[15] = "wq"
  HINTS[16] = "wf"
  HINTS[17] = "wd"
  HINTS[18] = "ws"
  HINTS[19] = "wa"
  HINTS[20] = "qp"
  HINTS[21] = "qo"
  HINTS[22] = "qi"
  HINTS[23] = "qu"
  HINTS[24] = "ql"
  HINTS[25] = "qk"
  HINTS[26] = "qj"
  HINTS[27] = "qt"
  HINTS[28] = "qr"
  HINTS[29] = "qe"
  HINTS[30] = "qw"
  HINTS[31] = "qq"
  HINTS[32] = "qf"
  HINTS[33] = "qd"
  HINTS[34] = "qs"
  HINTS[35] = "qa"
  HINTS[36] = "fp"
  HINTS[37] = "fo"
  HINTS[38] = "fi"
  HINTS[39] = "fu"
  HINTS[40] = "fl"
  HINTS[41] = "fk"
  HINTS[42] = "fj"
  HINTS[43] = "ft"
  HINTS[44] = "fr"
  HINTS[45] = "fe"
  HINTS[46] = "fw"
  HINTS[47] = "fq"
  HINTS[48] = "ff"
  HINTS[49] = "fd"
  HINTS[50] = "fs"
  HINTS[51] = "fa"
  HINTS[52] = "dp"
  HINTS[53] = "do"
  HINTS[54] = "di"
  HINTS[55] = "du"
  HINTS[56] = "dl"
  HINTS[57] = "dk"
  HINTS[58] = "dj"
  HINTS[59] = "dt"
  HINTS[60] = "dr"
  HINTS[61] = "de"
  HINTS[62] = "dw"
  HINTS[63] = "dq"
  HINTS[64] = "df"
  HINTS[65] = "dd"
  HINTS[66] = "ds"
  HINTS[67] = "da"
  HINTS[68] = "sp"
  HINTS[69] = "so"
  HINTS[70] = "si"
  HINTS[71] = "su"
  HINTS[72] = "sl"
  HINTS[73] = "sk"
  HINTS[74] = "sj"
  HINTS[75] = "st"
  HINTS[76] = "sr"
  HINTS[77] = "se"
  HINTS[78] = "sw"
  HINTS[79] = "sq"
  HINTS[80] = "sf"
  HINTS[81] = "sd"
  HINTS[82] = "ss"
  HINTS[83] = "sa"
  HINTS[84] = "ap"
  HINTS[85] = "ao"
  HINTS[86] = "ai"
  HINTS[87] = "au"
  HINTS[88] = "al"
  HINTS[89] = "ak"
  HINTS[90] = "aj"
  HINTS[91] = "at"
  HINTS[92] = "ar"
  HINTS[93] = "ae"
  HINTS[94] = "aw"
  HINTS[95] = "aq"
  HINTS[96] = "af"
  HINTS[97] = "ad"
  HINTS[98] = "as"
  HINTS[99] = "aa"

  finger_patterns = ENVIRON["FINGER_PATTERNS"];

  if (COMPACT_HINTS) {
    hint_format = "\033[30;1;43m%s\033[0m"
    highlight_format = "\033[1;33m%s\033[0m"
    compound_format = hint_format highlight_format
  } else {
    hint_format = "\033[1;33m[%s]\033[0m"
    highlight_format = "\033[1;33m%s\033[0m "
    compound_format = highlight_format hint_format
  }

  hint_lookup = ""
}

{
  output_line = $0;

  # insert hints into `output_line` and accumulate hints in `hint_lookup`
  line = output_line;
  pos = col_pos_correction = 0;
  while (match(line, finger_patterns)) {
    pos += RSTART;
    col_pos = pos + col_pos_correction
    pre_match = substr(output_line, 0, col_pos - 1);
    post_match = substr(output_line, col_pos + RLENGTH, length(line) - 1);
    line_match = substr(line, RSTART, RLENGTH);

    hint = hint_by_match[line_match]
    if (!hint) {
      hint = HINTS[n_matches++]
      hint_by_match[line_match] = hint
    }
    hint_lookup = hint_lookup hint ":" line_match "\n"

    if (COMPACT_HINTS) {
      hint_len = length(hint)
      line_match = substr(line_match, hint_len + 1, length(line_match) - hint_len);
      hint_match = sprintf(compound_format, hint, line_match);
    } else {
      hint_match = sprintf(compound_format, line_match, hint);
    }
    output_line = pre_match hint_match post_match;

    col_pos_correction += length(sprintf(highlight_format, line_match)) + length(sprintf(hint_format, hint)) - 1;

    line = post_match;
  }

  printf "\n%s", output_line
}

END {
  print hint_lookup | "cat 1>&3"
}

BEGIN {
  n_matches = 0;
  line_pos = 0;
  col_pos = 0;
  COMPACT_HINTS = ENVIRON["COMPACT_HINTS"];

  HINTS[0] = "p"
  HINTS[1] = "o"
  HINTS[2] = "i"
  HINTS[3] = "u"
  HINTS[4] = "l"
  HINTS[5] = "k"
  HINTS[6] = "j"
  HINTS[7] = "t"
  HINTS[8] = "r"
  HINTS[9] = "e"
  HINTS[10] = "wj"
  HINTS[11] = "wt"
  HINTS[12] = "wr"
  HINTS[13] = "we"
  HINTS[14] = "ww"
  HINTS[15] = "wq"
  HINTS[16] = "wf"
  HINTS[17] = "wd"
  HINTS[18] = "ws"
  HINTS[19] = "wa"
  HINTS[20] = "qp"
  HINTS[21] = "qo"
  HINTS[22] = "qi"
  HINTS[23] = "qu"
  HINTS[24] = "ql"
  HINTS[25] = "qk"
  HINTS[26] = "qj"
  HINTS[27] = "qt"
  HINTS[28] = "qr"
  HINTS[29] = "qe"
  HINTS[30] = "qw"
  HINTS[31] = "qq"
  HINTS[32] = "qf"
  HINTS[33] = "qd"
  HINTS[34] = "qs"
  HINTS[35] = "qa"
  HINTS[36] = "fp"
  HINTS[37] = "fo"
  HINTS[38] = "fi"
  HINTS[39] = "fu"
  HINTS[40] = "fl"
  HINTS[41] = "fk"
  HINTS[42] = "fj"
  HINTS[43] = "ft"
  HINTS[44] = "fr"
  HINTS[45] = "fe"
  HINTS[46] = "fw"
  HINTS[47] = "fq"
  HINTS[48] = "ff"
  HINTS[49] = "fd"
  HINTS[50] = "fs"
  HINTS[51] = "fa"
  HINTS[52] = "dp"
  HINTS[53] = "do"
  HINTS[54] = "di"
  HINTS[55] = "du"
  HINTS[56] = "dl"
  HINTS[57] = "dk"
  HINTS[58] = "dj"
  HINTS[59] = "dt"
  HINTS[60] = "dr"
  HINTS[61] = "de"
  HINTS[62] = "dw"
  HINTS[63] = "dq"
  HINTS[64] = "df"
  HINTS[65] = "dd"
  HINTS[66] = "ds"
  HINTS[67] = "da"
  HINTS[68] = "sp"
  HINTS[69] = "so"
  HINTS[70] = "si"
  HINTS[71] = "su"
  HINTS[72] = "sl"
  HINTS[73] = "sk"
  HINTS[74] = "sj"
  HINTS[75] = "st"
  HINTS[76] = "sr"
  HINTS[77] = "se"
  HINTS[78] = "sw"
  HINTS[79] = "sq"
  HINTS[80] = "sf"
  HINTS[81] = "sd"
  HINTS[82] = "ss"
  HINTS[83] = "sa"
  HINTS[84] = "ap"
  HINTS[85] = "ao"
  HINTS[86] = "ai"
  HINTS[87] = "au"
  HINTS[88] = "al"
  HINTS[89] = "ak"
  HINTS[90] = "aj"
  HINTS[91] = "at"
  HINTS[92] = "ar"
  HINTS[93] = "ae"
  HINTS[94] = "aw"
  HINTS[95] = "aq"
  HINTS[96] = "af"
  HINTS[97] = "ad"
  HINTS[98] = "as"
  HINTS[99] = "aa"

  finger_patterns = ENVIRON["FINGER_PATTERNS"];

  if (COMPACT_HINTS) {
    hint_format = "\033[30;1;43m%s\033[0m"
    highlight_format = "\033[1;33m%s\033[0m"
    compound_format = hint_format highlight_format
  } else {
    hint_format = "\033[1;33m[%s]\033[0m"
    highlight_format = "\033[1;33m%s\033[0m "
    compound_format = highlight_format hint_format
  }

  hint_lookup = ""
}

{
  line = $0;
  pos = 0;
  col_pos = 0;
  col_pos_correction = 0;

  output_line = line;

  while (match(line, finger_patterns)) {
    n_matches += 1;

    hint = HINTS[n_matches - 1]
    pos += RSTART;

    col_pos = pos;

    line_match = substr(line, RSTART, RLENGTH);
    full_line_match = line_match

    if (COMPACT_HINTS) {
      hint_len = length(hint)
      line_match = substr(line_match, hint_len + 1, length(line_match) - hint_len);
    }

    col_pos = col_pos + col_pos_correction

    line_pos = NR;

    pre_match = substr(output_line, 0, col_pos - 1);

    if (COMPACT_HINTS) {
      hint_match = sprintf(compound_format, hint, line_match);
    } else {
      hint_match = sprintf(compound_format, line_match, hint);
    }

    post_match = substr(output_line, col_pos + RLENGTH, length(line) - 1);

    output_line = pre_match hint_match post_match;

    line = post_match;

    col_pos_correction += length(sprintf(highlight_format, line_match)) + length(sprintf(hint_format, hint)) - 1;

    hint_lookup = hint_lookup hint ":" full_line_match "\n"
  }

  printf "\n%s", output_line
}

END {
  print hint_lookup | "cat 1>&3"
}