LanguageHelper: improve getDetails() for writing systems

####################################################################
# LanguageHelper.awk                                               #
####################################################################

# Get locale key by language code or alias.
function getCode(code,    group) {
    if (code == "auto" || code in Locale)
        return code
    else if (code in LocaleAlias)
        return LocaleAlias[code]
    else if (tolower(code) in LocaleAlias)
        return LocaleAlias[tolower(code)]

    # Remove unidentified region or script code
    match(code, /^([[:alpha:]][[:alpha:]][[:alpha:]]?)-(.*)$/, group)
    if (group[1])
        return group[1]

    return # return nothing if not found
}

# Return the name of a language.
function getName(code) {
    return Locale[getCode(code)]["name"]
}

# Return all the names of a language, separated by "/"
function getNames(code) {
    if ("name2" in Locale[getCode(code)])
        return Locale[getCode(code)]["name"] " / " Locale[getCode(code)]["name2"]
    else
        return Locale[getCode(code)]["name"]
}

# Return the endonym of a language.
function getEndonym(code) {
    return Locale[getCode(code)]["endonym"]
}

# Return the string for displaying the endonym of a language.
function getDisplay(code) {
    return Locale[getCode(code)]["display"]
}

# Return formatted text for "translations of".
function showTranslationsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["translations-of"]
    if (!fmt) fmt = Locale["en"]["translations-of"]
    return sprintf(fmt, text)
}

# Return formatted text for "definitions of".
function showDefinitionsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["definitions-of"]
    if (!fmt) fmt = Locale["en"]["definitions-of"]
    return sprintf(fmt, text)
}

# Return a string of "synonyms".
function showSynonyms(code,    tmp) {
    tmp = Locale[getCode(code)]["synonyms"]
    if (!tmp) tmp = Locale["en"]["synonyms"]
    return tmp
}

# Return a string of "examples".
function showExamples(code,    tmp) {
    tmp = Locale[getCode(code)]["examples"]
    if (!tmp) tmp = Locale["en"]["examples"]
    return tmp
}

# Return a string of "see also".
function showSeeAlso(code,    tmp) {
    tmp = Locale[getCode(code)]["see-also"]
    if (!tmp) tmp = Locale["en"]["see-also"]
    return tmp
}

# Return the family of a language.
function getFamily(code) {
    return Locale[getCode(code)]["family"]
}

# Return the branch of a language.
function getBranch(code) {
    return Locale[getCode(code)]["branch"]
}

# Return the ISO 639-3 code of a language.
function getISO(code) {
    return Locale[getCode(code)]["iso"]
}

# Return the Glottocode of a language.
function getGlotto(code) {
    return Locale[getCode(code)]["glotto"]
}

# Return the ISO 15924 script code of a language.
function getScript(code) {
    return Locale[getCode(code)]["script"]
}

# Return 1 if a language is R-to-L; otherwise return 0.
function isRTL(code) {
    return Locale[getCode(code)]["rtl"] ? 1 : 0
}

# Return 1 if Google provides dictionary data for a language; otherwise return 0.
function hasDictionary(code) {
    return Locale[getCode(code)]["dictionary"] ? 1 : 0
}

# Comparator using getName().
function compName(i1, v1, i2, v2) {
    if (getName(i1) < getName(i2))
        return -1
    else
        return (getName(i1) != getName(i2))
}

# Return the name of script (writing system).
# See: <https://en.wikipedia.org/wiki/ISO_15924#List_of_codes>
#      <http://unicode.org/iso15924/iso15924-codes.html>
function scriptName(code) {
    switch (code) {
    case "Arab": return "Arabic"
    case "Armn": return "Armenian"
    case "Beng": return "Bengali"
    case "Cher": return "Cherokee"
    case "Cyrl": return "Cyrillic"
    case "Deva": return "Devanagari"
    case "Ethi": return "Ethiopic (Geʻez)"
    case "Geor": return "Georgian (Mkhedruli)"
    case "Grek": return "Greek"
    case "Gujr": return "Gujarati"
    case "Guru": return "Gurmukhi"
    case "Hani": return "Han"
    case "Hans": return "Han (Simplified)"
    case "Hant": return "Han (Traditional)"
    case "Hebr": return "Hebrew"
    case "Jpan": return "Japanese (Han + Hiragana + Katakana)"
    case "Khmr": return "Khmer"
    case "Knda": return "Kannada"
    case "Kore": return "Korean (Hangul + Han)"
    case "Laoo": return "Lao"
    case "Latn": return "Latin"
    case "Mtei": return "Meitei Mayek"
    case "Mlym": return "Malayalam"
    case "Mymr": return "Myanmar"
    case "Orya": return "Oriya"
    case "Piqd": return "Klingon (pIqaD)"
    case "Sinh": return "Sinhala"
    case "Taml": return "Tamil"
    case "Telu": return "Telugu"
    case "Thaa": return "Thaana"
    case "Thai": return "Thai"
    case "Tibt": return "Tibetan"
    default: return "Unknown"
    }
}

# Return the regions that a language is spoken in, as an English string.
function getSpokenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["spoken-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return the regions that a script is written in, as an English string.
function getWrittenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["written-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return 1 if a language is supported by Google; otherwise return 0.
function isSupportedByGoogle(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "google") return 1
    }
    return 0
}

# Return 1 if a language is supported by Bing; otherwise return 0.
function isSupportedByBing(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "bing") return 1
    }
    return 0
}

# Return detailed information of a language as a string.
function getDetails(code,    article, desc, group, iso, name, names, script, writing) {
    if (code == "auto" || !getCode(code)) {
        e("[ERROR] Language not found: " code "\n"                      \
          "        Run '-reference / -R' to see a list of available languages.")
        exit 1
    }

    script = scriptName(getScript(code))
    if (isRTL(code)) script = script " (R-to-L)"

    split(getISO(code), group, "-")
    iso = group[1]

    name = getName(code)
    names = getNames(code)
    if (match(name, /\(.*\)/)) {
        writing = substr(name, match(name, /\(.*\)/) + 1)
        writing = substr(writing, 1, length(writing) - 1)
        name = substr(name, 1, match(name, /\(.*\)/) - 2)
    }

    if (getBranch(code)) {
        article = match(tolower(getBranch(code)), /^[aeiou]/) ? "an" : "a"
        desc = sprintf("%s is %s %s language spoken mainly in %s.",
                       names, article, getBranch(code), getSpokenIn(code))
    } else if (getFamily(code) == NULLSTR || tolower(getFamily(code)) == "language isolate") {
        desc = sprintf("%s is a language spoken mainly in %s.",
                       names, getSpokenIn(code))
    } else
        desc = sprintf("%s is a language of the %s family, spoken mainly in %s.",
                       names, getFamily(code), getSpokenIn(code))
    if (writing && getWrittenIn(code))
        desc = desc sprintf(" The %s writing system is officially used in %s.",
                            tolower(writing), getWrittenIn(code))

    return ansi("bold", sprintf("%s\n", getDisplay(code)))              \
        sprintf("%-22s%s\n", "Name", ansi("bold", names))               \
        sprintf("%-22s%s\n", "Family", ansi("bold", getFamily(code)))   \
        sprintf("%-22s%s\n", "Writing system", ansi("bold", script))    \
        sprintf("%-22s%s\n", "Code", ansi("bold", getCode(code)))       \
        sprintf("%-22s%s\n", "ISO 639-3", ansi("bold", iso))            \
        sprintf("%-22s%s\n", "SIL", ansi("bold", "https://iso639-3.sil.org/code/" iso)) \
        sprintf("%-22s%s\n", "Glottolog", getGlotto(code) ?
                ansi("bold", "https://glottolog.org/resource/languoid/id/" getGlotto(code)) : "") \
        sprintf("%-22s%s\n", "Wikipedia", ansi("bold", "https://en.wikipedia.org/wiki/ISO_639:" iso)) \
        (Locale[getCode(code)]["supported-by"] ? # FIXME
        sprintf("%-22s%s\n", "Translator support", sprintf("Google [%s]    Bing [%s]",
                isSupportedByGoogle(code) ? "✔" : "✘", isSupportedByBing(code) ? "✔" : "✘")) : "") \
        (Locale[getCode(code)]["spoken-in"] ? # FIXME
        ansi("bold", sprintf("\n%s", desc)) : "")
}

# Add /slashes/ for IPA phonemic notations and (parentheses) for others.
# Parameters:
#     code
function showPhonetics(phonetics, code) {
    if (code && getCode(code) == "en")
        return "/" phonetics "/"
    else
        return "(" phonetics ")"
}

# Convert a logical string to visual; don't right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
function show(text, code,    command, temp) {
    if (!code || isRTL(code)) {
        if (Cache[text][0])
            return Cache[text][0]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDiNoPad) {
                command = "echo " parameterize(text) PIPE BiDiNoPad
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][0] = temp
        }
    } else
        return text
}

# Convert a logical string to visual and right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
#     width: ignore to use default width for padding
function s(text, code, width,    command, temp) {
    if (!code || isRTL(code)) {
        if (!width) width = Option["width"]
        if (Cache[text][width])
            return Cache[text][width]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDi) {
                command = "echo " parameterize(text) PIPE sprintf(BiDi, width)
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][width] = temp
        }
    } else
        return text
}

# Convert a logical string to visual with a certain level of indentation.
# Parameters:
#     level: level of indentation
#     code: ignore to apply left indentation
#     width: ignore to use default width for padding
function ins(level, text, code, width,    i, temp) {
    if (code && isRTL(code)) {
        if (!width) width = Option["width"]
        return s(text, code, width - Option["indent"] * level)
    } else
        return replicate(" ", Option["indent"] * level) text
}

# Parse a POSIX locale identifier and return the language code;
# Identified by both language identifier and region identifier.
# Parameters:
#     lang = [language[_territory][.codeset][@modifier]]
# See: <https://en.wikipedia.org/wiki/Locale>
function parseLang(lang,    code, group) {
    match(lang, /^([a-z][a-z][a-z]?)(_|$)/, group)
    code = getCode(group[1])

    # Detect region identifier
    ## Regions using Chinese Simplified: China, Singapore
    if (lang ~ /^zh_(CN|SG)/) code = "zh-CN"
    ## Regions using Chinese Traditional: Taiwan, Hong Kong
    else if (lang ~ /^zh_(TW|HK)/) code = "zh-TW"

    # FIXME: handle unrecognized language code
    if (!code) code = "en"

    return code
}

# Initialize `UserLang`.
function initUserLang(    lang, utf) {
    if (lang = ENVIRON["LC_ALL"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (lang = ENVIRON["LANG"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (!UserLocale) {
        UserLocale = "en_US.UTF-8"
        utf = 1
    }
    if (!utf)
        w("[WARNING] Your locale codeset (" UserLocale ") is not UTF-8.")

    UserLang = parseLang(UserLocale)
}

####################################################################
# LanguageHelper.awk                                               #
####################################################################

# Get locale key by language code or alias.
function getCode(code,    group) {
    if (code == "auto" || code in Locale)
        return code
    else if (code in LocaleAlias)
        return LocaleAlias[code]
    else if (tolower(code) in LocaleAlias)
        return LocaleAlias[tolower(code)]

    # Remove unidentified region or script code
    match(code, /^([[:alpha:]][[:alpha:]][[:alpha:]]?)-(.*)$/, group)
    if (group[1])
        return group[1]

    return # return nothing if not found
}

# Return the name of a language.
function getName(code) {
    return Locale[getCode(code)]["name"]
}

# Return all the names of a language, separated by "/"
function getNames(code) {
    if ("name2" in Locale[getCode(code)])
        return Locale[getCode(code)]["name"] " / " Locale[getCode(code)]["name2"]
    else
        return Locale[getCode(code)]["name"]
}

# Return the endonym of a language.
function getEndonym(code) {
    return Locale[getCode(code)]["endonym"]
}

# Return the string for displaying the endonym of a language.
function getDisplay(code) {
    return Locale[getCode(code)]["display"]
}

# Return formatted text for "translations of".
function showTranslationsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["translations-of"]
    if (!fmt) fmt = Locale["en"]["translations-of"]
    return sprintf(fmt, text)
}

# Return formatted text for "definitions of".
function showDefinitionsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["definitions-of"]
    if (!fmt) fmt = Locale["en"]["definitions-of"]
    return sprintf(fmt, text)
}

# Return a string of "synonyms".
function showSynonyms(code,    tmp) {
    tmp = Locale[getCode(code)]["synonyms"]
    if (!tmp) tmp = Locale["en"]["synonyms"]
    return tmp
}

# Return a string of "examples".
function showExamples(code,    tmp) {
    tmp = Locale[getCode(code)]["examples"]
    if (!tmp) tmp = Locale["en"]["examples"]
    return tmp
}

# Return a string of "see also".
function showSeeAlso(code,    tmp) {
    tmp = Locale[getCode(code)]["see-also"]
    if (!tmp) tmp = Locale["en"]["see-also"]
    return tmp
}

# Return the family of a language.
function getFamily(code) {
    return Locale[getCode(code)]["family"]
}

# Return the branch of a language.
function getBranch(code) {
    return Locale[getCode(code)]["branch"]
}

# Return the ISO 639-3 code of a language.
function getISO(code) {
    return Locale[getCode(code)]["iso"]
}

# Return the Glottocode of a language.
function getGlotto(code) {
    return Locale[getCode(code)]["glotto"]
}

# Return the ISO 15924 script code of a language.
function getScript(code) {
    return Locale[getCode(code)]["script"]
}

# Return 1 if a language is R-to-L; otherwise return 0.
function isRTL(code) {
    return Locale[getCode(code)]["rtl"] ? 1 : 0
}

# Return 1 if Google provides dictionary data for a language; otherwise return 0.
function hasDictionary(code) {
    return Locale[getCode(code)]["dictionary"] ? 1 : 0
}

# Comparator using getName().
function compName(i1, v1, i2, v2) {
    if (getName(i1) < getName(i2))
        return -1
    else
        return (getName(i1) != getName(i2))
}

# Return the name of script (writing system).
# See: <https://en.wikipedia.org/wiki/ISO_15924#List_of_codes>
#      <http://unicode.org/iso15924/iso15924-codes.html>
function scriptName(code) {
    switch (code) {
    case "Arab": return "Arabic"
    case "Armn": return "Armenian"
    case "Beng": return "Bengali"
    case "Cher": return "Cherokee"
    case "Cyrl": return "Cyrillic"
    case "Deva": return "Devanagari"
    case "Ethi": return "Ethiopic (Geʻez)"
    case "Geor": return "Georgian (Mkhedruli)"
    case "Grek": return "Greek"
    case "Gujr": return "Gujarati"
    case "Guru": return "Gurmukhi"
    case "Hani": return "Han"
    case "Hans": return "Han (Simplified)"
    case "Hant": return "Han (Traditional)"
    case "Hebr": return "Hebrew"
    case "Jpan": return "Japanese (Han + Hiragana + Katakana)"
    case "Khmr": return "Khmer"
    case "Knda": return "Kannada"
    case "Kore": return "Korean (Hangul + Han)"
    case "Laoo": return "Lao"
    case "Latn": return "Latin"
    case "Mtei": return "Meitei Mayek"
    case "Mlym": return "Malayalam"
    case "Mymr": return "Myanmar"
    case "Orya": return "Oriya"
    case "Piqd": return "Klingon (pIqaD)"
    case "Sinh": return "Sinhala"
    case "Taml": return "Tamil"
    case "Telu": return "Telugu"
    case "Thaa": return "Thaana"
    case "Thai": return "Thai"
    case "Tibt": return "Tibetan"
    default: return "Unknown"
    }
}

# Return the regions that a language is spoken in, as an English string.
function getSpokenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["spoken-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return 1 if a language is supported by Google; otherwise return 0.
function isSupportedByGoogle(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "google") return 1
    }
    return 0
}

# Return 1 if a language is supported by Bing; otherwise return 0.
function isSupportedByBing(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "bing") return 1
    }
    return 0
}

# Return detailed information of a language as a string.
function getDetails(code,    article, desc, group, iso, script) {
    if (code == "auto" || !getCode(code)) {
        e("[ERROR] Language not found: " code "\n"                      \
          "        Run '-reference / -R' to see a list of available languages.")
        exit 1
    }

    script = scriptName(getScript(code))
    if (isRTL(code)) script = script " (R-to-L)"
    split(getISO(code), group, "-")
    iso = group[1]

    if (getBranch(code)) {
        article = match(tolower(getBranch(code)), /^[aeiou]/) ? "an" : "a"
        desc = sprintf("%s is %s %s language spoken mainly in %s.",
                       getNames(code), article, getBranch(code), getSpokenIn(code))
    } else if (getFamily(code) == NULLSTR || tolower(getFamily(code)) == "language isolate") {
        desc = sprintf("%s is a language spoken mainly in %s.",
                       getNames(code), getSpokenIn(code))
    } else
        desc = sprintf("%s is a language of the %s family, spoken mainly in %s.",
                       getNames(code), getFamily(code), getSpokenIn(code))

    return ansi("bold", sprintf("%s\n", getDisplay(code)))              \
        sprintf("%-22s%s\n", "Name", ansi("bold", getNames(code)))      \
        sprintf("%-22s%s\n", "Family", ansi("bold", getFamily(code)))   \
        sprintf("%-22s%s\n", "Writing system", ansi("bold", script))    \
        sprintf("%-22s%s\n", "Code", ansi("bold", getCode(code)))       \
        sprintf("%-22s%s\n", "ISO 639-3", ansi("bold", iso))            \
        sprintf("%-22s%s\n", "SIL", ansi("bold", "https://iso639-3.sil.org/code/" iso)) \
        sprintf("%-22s%s\n", "Glottolog", getGlotto(code) ?
                ansi("bold", "https://glottolog.org/resource/languoid/id/" getGlotto(code)) : "") \
        sprintf("%-22s%s\n", "Wikipedia", ansi("bold", "https://en.wikipedia.org/wiki/ISO_639:" iso)) \
        (Locale[getCode(code)]["supported-by"] ? # FIXME
        sprintf("%-22s%s\n", "Translator support", sprintf("Google [%s]    Bing [%s]",
                isSupportedByGoogle(code) ? "✔" : "✘", isSupportedByBing(code) ? "✔" : "✘")) : "") \
        (Locale[getCode(code)]["spoken-in"] ? # FIXME
        ansi("bold", sprintf("\n%s", desc)) : "")
}

# Add /slashes/ for IPA phonemic notations and (parentheses) for others.
# Parameters:
#     code
function showPhonetics(phonetics, code) {
    if (code && getCode(code) == "en")
        return "/" phonetics "/"
    else
        return "(" phonetics ")"
}

# Convert a logical string to visual; don't right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
function show(text, code,    command, temp) {
    if (!code || isRTL(code)) {
        if (Cache[text][0])
            return Cache[text][0]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDiNoPad) {
                command = "echo " parameterize(text) PIPE BiDiNoPad
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][0] = temp
        }
    } else
        return text
}

# Convert a logical string to visual and right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
#     width: ignore to use default width for padding
function s(text, code, width,    command, temp) {
    if (!code || isRTL(code)) {
        if (!width) width = Option["width"]
        if (Cache[text][width])
            return Cache[text][width]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDi) {
                command = "echo " parameterize(text) PIPE sprintf(BiDi, width)
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][width] = temp
        }
    } else
        return text
}

# Convert a logical string to visual with a certain level of indentation.
# Parameters:
#     level: level of indentation
#     code: ignore to apply left indentation
#     width: ignore to use default width for padding
function ins(level, text, code, width,    i, temp) {
    if (code && isRTL(code)) {
        if (!width) width = Option["width"]
        return s(text, code, width - Option["indent"] * level)
    } else
        return replicate(" ", Option["indent"] * level) text
}

# Parse a POSIX locale identifier and return the language code;
# Identified by both language identifier and region identifier.
# Parameters:
#     lang = [language[_territory][.codeset][@modifier]]
# See: <https://en.wikipedia.org/wiki/Locale>
function parseLang(lang,    code, group) {
    match(lang, /^([a-z][a-z][a-z]?)(_|$)/, group)
    code = getCode(group[1])

    # Detect region identifier
    ## Regions using Chinese Simplified: China, Singapore
    if (lang ~ /^zh_(CN|SG)/) code = "zh-CN"
    ## Regions using Chinese Traditional: Taiwan, Hong Kong
    else if (lang ~ /^zh_(TW|HK)/) code = "zh-TW"

    # FIXME: handle unrecognized language code
    if (!code) code = "en"

    return code
}

# Initialize `UserLang`.
function initUserLang(    lang, utf) {
    if (lang = ENVIRON["LC_ALL"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (lang = ENVIRON["LANG"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (!UserLocale) {
        UserLocale = "en_US.UTF-8"
        utf = 1
    }
    if (!utf)
        w("[WARNING] Your locale codeset (" UserLocale ") is not UTF-8.")

    UserLang = parseLang(UserLocale)
}


Utils: add a short note for dump()

Utils: add a short note for dump()

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev"
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev | sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                       "\"${TRANS_PROGRAM}\"" :
                                       "-f " EntryPoint)                \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                command = command " " parameterize(ARGV[i])

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        params = ""
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                params = params " " parameterize(ARGV[i], "\"")
        if (ENVIRON["TRANS_PROGRAM"]) {
            el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
                "(setq explicit-shell-file-name \"" Gawk "\") "         \
                "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        } else {
            el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
                "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        }
        command = Emacs " --eval " parameterize(el)

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    } else {
        command = Curl " --location --silent " url
        content = NULLSTR
        while ((command |& getline line) > 0)
            content = (content ? content "\n" : NULLSTR) line
        return content
    }
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    command = "hexdump" " -e'1/1 \"%03d\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev"
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev | sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                       "\"${TRANS_PROGRAM}\"" :
                                       "-f " EntryPoint)                \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                command = command " " parameterize(ARGV[i])

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        params = ""
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                params = params " " parameterize(ARGV[i], "\"")
        if (ENVIRON["TRANS_PROGRAM"]) {
            el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
                "(setq explicit-shell-file-name \"" Gawk "\") "         \
                "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        } else {
            el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
                "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        }
        command = Emacs " --eval " parameterize(el)

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    } else {
        command = Curl " --location --silent " url
        content = NULLSTR
        while ((command |& getline line) > 0)
            content = (content ? content "\n" : NULLSTR) line
        return content
    }
}

# Dump a Unicode string into a byte array. Return the length of the array.
function dump(text, group,    command, temp) {
    command = "hexdump" " -e'1/1 \"%03d\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}


Fix bugs

Fix bugs


Reset the len value

Reset the len value

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    gsub("(#[[:digit:]]+|, $)", "");

    len = split($3, words, ", ");

    if (len >= 10000) next;

    for (i = 1; i <= len; i++) uniq[words[i]];

    len = 0;

    for (word in uniq) {
        synset = synset sep word;
        sep = ", ";
        len++;
    }

    print $1, len, synset | "sort --parallel=$(nproc) -t \"\t\" -S1G -k2nr -k1n -s";

    synset = sep = len = ""; delete uniq;
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    gsub("(#[[:digit:]]+|, $)", "");

    len = split($3, words, ", ");

    if (len >= 10000) next;

    for (i = 1; i <= len; i++) uniq[words[i]];

    for (word in uniq) {
        synset = synset sep word;
        sep = ", ";
        len++;
    }

    print $1, len, synset | "sort --parallel=$(nproc) -t \"\t\" -S1G -k2nr -k1n -s";

    synset = sep = len = ""; delete uniq;
}


dynamic regexp

dynamic regexp

Effective awk programming : universal text processing and pattern matching / Arnold Robbins

echo aaaabcd | awk '{ sub(/a+/, "<A>"); print }'

BEGIN { digits_regexp = "[[:digit:]]+" }
$0 ~ digits_regexp    { print }


Indent line after } else {

Indent line after } else {

# Generate a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, episodeDuration, & episodeDescription with the same columns
# as the primary spreadsheet so they can be combined into one.
# Add to the number of episodes but with no terminating newline
#
# INVOCATION:
#       curl -s https://watch.mhzchoice.com/a-french-village/season:1 \
#           | awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getMHzFrom-episodePages.awk
#
# INPUT:
#        <title>A French Village - MHz Choice</title>
#  ---
#        <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://watch.mhzchoice.com/detective-montalbano?page=2">
#  ---
#

#          18 Episodes
#

#  ---
#
47:58

#  ---
#        The Landing (June 12, 1940)\
#
#  ---
#
Episode 1

#

#
A doctor oversees the birth of a child. Directed by Phillipe Triboit, 2009.

#
#
This episode will be made available on August 8.

#

# OUTPUT:
#       $EPISODE_INFO_FILE & the number of episodes
#

# Extract whether this is a page 2
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
}

# Extract only the number of episodes, print them with no terminating newline
# but if this is page 2 skip to avoid double counting
/
> NUM_EPISODES_FILE
        next
    }
}

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ - MHz Choice<.*/,"")
    gsub (/'/,"'")
    gsub (/"/,"\"\"")
    gsub (/&/,"\\&")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    seriesTitle = $0
    next
}

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    next
}

# Extract the episode URL, the episode type, the episode title, and the seasonNumber
/<strong title="/ {
    gsub (/'/,"'")
    split ($0,fld,"\"")
    episodeURL = fld[2]
    if (episodeURL ~ /-c-.[[:digit:]]{3,4}$/)
        episodeNumberFromURL = substr (episodeURL, length(episodeURL)-1, 2) + 0
    shortURL = episodeURL
    sub (/.*watch/,"watch",shortURL)
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeURL ~ /-c-x[[:digit:]]{3,4}$|montme-c-01001|richard-sammel-inetrview/)
        episodeType = "X"
    # Take care of extra fields introduced in February 2020
    titleString = substr ($0, match ($0, /strong title/))
    split (titleString,fld,"\"")
    episodeTitle = fld[2]
    if (episodeTitle ~ /^PR \|/) {
        # Episode is a Trailer (i.e. First look), set episodeType to "T"
        episodeType = "T"
        printf ("-1") >> NUM_EPISODES_FILE
        # print episodeTitle "\t" shortURL >> ERROR_FILE
    }
    #
    # If start of episodeTitle == seriesTitle ": ", remove the redundant part.
    if ((match (episodeTitle, seriesTitle ": ")) == 1) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/.*season:/,"")
    sub (/\/.*/,"")
    seasonNumber = $0
    next
}

# Extract the episode number, and correct if necessary
# Detective Montalbano requires special processing for page 2 episodeNumber
/<h4 class="/ {
    episodeNumber = $0
    sub (/.*Episode /,"",episodeNumber)
    sub (/<\/span>.*/,"",episodeNumber)
    episodeNumber = episodeNumber + 0
    if (seriesTitle == "Detective Montalbano" && page2 == "yes") {
        oldEpisodeNumber = episodeNumber
        episodeNumber += 24
        printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                episodeNumber, shortURL) >> ERROR_FILE
    }
}

# WARNING - other scripts depend on the number and order of the fields below
# Extract the episode description and print the composed info
/
<span class=/,/<\/div>/ {
    if ($0 ~ /
 Changed %s00 to %s%02d: %s\n", episodeType, episodeType,
                        episodeNumberFromURL, shortURL) >> ERROR_FILE
                episodeNumber = episodeNumberFromURL
            } else {
                printf ("==> Episode number is 00: %s\n", shortURL) >> ERROR_FILE
            }
        }
        printf ("%d\t=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)\t\t\t%s\t\t\t\t\t", \
                SERIES_NUMBER, episodeURL, seriesTitle, seasonNumber, episodeType, episodeNumber, \
                episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
        # Need to account for multiple lines of description
        descriptionLinesFound = 0
        episodeDescription = ""
        # make sure there is no carryforward
        episodeNumber = ""
        episodeNumberFromURL = ""
    }
    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/"/,"\"",paragraph)
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
    }
    if ($0 ~ /<\/div>/) {
        if (episodeDescription == "") {
            if (episodeType == "T") {
                episodeDescription = episodeTitle
            } else {
                print "==> No description: " shortURL >> ERROR_FILE
            }
        }
        sub (/^PR \| /,"",episodeDescription)
        print episodeDescription >> EPISODE_INFO_FILE
    }
}


Utils: fix curl()

Utils: fix curl()

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev"
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev | sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                       "\"${TRANS_PROGRAM}\"" :
                                       "-f " EntryPoint)                \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                command = command " " parameterize(ARGV[i])

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        params = ""
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                params = params " " parameterize(ARGV[i], "\"")
        if (ENVIRON["TRANS_PROGRAM"]) {
            el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
                "(setq explicit-shell-file-name \"" Gawk "\") "         \
                "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        } else {
            el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
                "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        }
        command = Emacs " --eval " parameterize(el)

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    } else {
        command = Curl " --location --silent " url
        content = NULLSTR
        while ((command |& getline line) > 0)
            content = (content ? content "\n" : NULLSTR) line
        return content
    }
}

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev"
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev | sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                       "\"${TRANS_PROGRAM}\"" :
                                       "-f " EntryPoint)                \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                command = command " " parameterize(ARGV[i])

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    } else if (!(ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
        l(">> not found: $TRANS_PROGRAM or EntryPoint")
        return 1
    } else {
        params = ""
        for (i = 1; i < length(ARGV); i++)
            if (ARGV[i])
                params = params " " parameterize(ARGV[i], "\"")
        if (ENVIRON["TRANS_PROGRAM"]) {
            el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
                "(setq explicit-shell-file-name \"" Gawk "\") "         \
                "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        } else {
            el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
                "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
                "(command-execute 'shell) (rename-buffer \"" Name "\"))"
        }
        command = Emacs " --eval " parameterize(el)

        l(">> forking: " command)
        if (!system(command)) {
            l(">> process exited with code 0")
            exit ExitCode
        } else {
            l(">> process exited with non-zero return code")
            return 1
        }
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    } else {
        command = Curl " --location --silent " url
        content = NULLSTR
        while ((command |& getline line) > 0)
            content = content line "\n"
        return content
    }
}


space/tab

space/tab

#! /usr/bin/gawk -f

# label_dot_edge.awk
# Attempts to augment opaque ontological terms with their current label
# example usage:
# label_dot_edge.awk predicate_curie_term_label.tab  in.dot > out.dot
#
# Arguments: two files
# Loads the first file as a map from terms to labels:
# (expects: [namespace:]<tab><term><tab><label>[<tab><whatever>])
# the first set was generated via cypher queries against our scigraph
# (neo4j) instance loaded with the ontologies we use.
# Reads the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\t"; OFS="\t"}
# load labels
FNR == NR && $3 != "null"{label[$1 $2]=$3}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available
FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)
	split(row[4], term, /"/)
	if (term[2] in label){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
	line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}


added dgtwc: http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#  to enhancement template maker

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "   ];"
   printf(".");
}


Bell-Labs AWK does not support POSIX "bracket expressions" (POSIXese for
"character classes", basically).  So change them to their character
representation.

Submitted by:	David Wolfskill <david@catwhisker.org>

#!/usr/bin/awk -f
#
# $FreeBSD$
#
# Merge two boot loader help files for FreeBSD 3.0
# Joe Abley <jabley@patho.gen.nz>

BEGIN \
{
  state = 0;
  first = 0;
  ind = 0;
}

# beginning of first command
/^###/ && (state == 0) \
{
  state = 1;
  next;
}

# entry header
/^# T[\x21-\x7e]+ (S[\x21-\x7e]+ )*D[\x21-\x7e][\x20-\x7e]*$/ && (state == 1) \
{
  match($0, " T[\x21-\x7e]+");
  T = substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " S[\x21-\x7e]+");
  S = (RLENGTH == -1) ? "" : substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " D[\x21-\x7e][\x20-\x7e]*$");
  D = substr($0, RSTART + 2);

  # find a suitable place to store this one...
  ind++;
  if (ind == 1)
  {
    first = ind;
    help[ind, "T"] = T;
    help[ind, "S"] = S;
    help[ind, "link"] = -1;
  } else {
    i = first; j = -1;
    while (help[i, "T"] help[i, "S"] < T S)
    {
      j = i;
      i = help[i, "link"];
      if (i == -1) break;
    }

    if (i == -1)
    {
      help[j, "link"] = ind;
      help[ind, "link"] = -1;
    } else {
      help[ind, "link"] = i;
      if (j == -1)
        first = ind;
      else
        help[j, "link"] = ind;
    }
  }
  help[ind, "T"] = T;
  help[ind, "S"] = S;
  help[ind, "D"] = D;

  # set our state
  state = 2;
  help[ind, "text"] = 0;
  next;
}

# end of last command, beginning of next one
/^###/ && (state == 2) \
{
  state = 1;
}

(state == 2) \
{
  sub("[[:blank:]]+$", "");
  if (help[ind, "text"] == 0 && $0 ~ /^[[:blank:]]*$/) next;
  help[ind, "text", help[ind, "text"]] = $0;
  help[ind, "text"]++;
  next;
}

# show them what we have (it's already sorted in help[])
END \
{
  node = first;
  while (node != -1)
  {
    printf "################################################################################\n";
    printf "# T%s ", help[node, "T"];
    if (help[node, "S"] != "") printf "S%s ", help[node, "S"];
    printf "D%s\n\n", help[node, "D"];
    for (i = 0; i < help[node, "text"]; i++)
      printf "%s\n", help[node, "text", i];
    node = help[node, "link"];
  }
  printf "################################################################################\n";
}

#!/usr/bin/awk -f
#
# $FreeBSD$
#
# Merge two boot loader help files for FreeBSD 3.0
# Joe Abley <jabley@patho.gen.nz>

BEGIN \
{
  state = 0;
  first = 0;
  ind = 0;
}

# beginning of first command
/^###/ && (state == 0) \
{
  state = 1;
  next;
}

# entry header
/^# T[[:graph:]]+ (S[[:graph:]]+ )*D[[:graph:]][[:print:]]*$/ && (state == 1) \
{
  match($0, " T[[:graph:]]+");
  T = substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " S[[:graph:]]+");
  S = (RLENGTH == -1) ? "" : substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " D[[:graph:]][[:print:]]*$");
  D = substr($0, RSTART + 2);

  # find a suitable place to store this one...
  ind++;
  if (ind == 1)
  {
    first = ind;
    help[ind, "T"] = T;
    help[ind, "S"] = S;
    help[ind, "link"] = -1;
  } else {
    i = first; j = -1;
    while (help[i, "T"] help[i, "S"] < T S)
    {
      j = i;
      i = help[i, "link"];
      if (i == -1) break;
    }

    if (i == -1)
    {
      help[j, "link"] = ind;
      help[ind, "link"] = -1;
    } else {
      help[ind, "link"] = i;
      if (j == -1)
        first = ind;
      else
        help[j, "link"] = ind;
    }
  }
  help[ind, "T"] = T;
  help[ind, "S"] = S;
  help[ind, "D"] = D;

  # set our state
  state = 2;
  help[ind, "text"] = 0;
  next;
}

# end of last command, beginning of next one
/^###/ && (state == 2) \
{
  state = 1;
}

(state == 2) \
{
  sub("[[:blank:]]+$", "");
  if (help[ind, "text"] == 0 && $0 ~ /^[[:blank:]]*$/) next;
  help[ind, "text", help[ind, "text"]] = $0;
  help[ind, "text"]++;
  next;
}

# show them what we have (it's already sorted in help[])
END \
{
  node = first;
  while (node != -1)
  {
    printf "################################################################################\n";
    printf "# T%s ", help[node, "T"];
    if (help[node, "S"] != "") printf "S%s ", help[node, "S"];
    printf "D%s\n\n", help[node, "D"];
    for (i = 0; i < help[node, "text"]; i++)
      printf "%s\n", help[node, "text", i];
    node = help[node, "link"];
  }
  printf "################################################################################\n";
}


Commons: bold red error e()

Commons: bold red error e()

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used functions for string and array operations, logging.#
####################################################################

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line) {
    # Remove preceding spaces
    gsub(/^[[:space:]]+/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    if (length(array)) {
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != "")
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array # array == ""

    return temp
}

# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > "/dev/stderr"
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("red", text)) > "/dev/stderr"
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > "/dev/stderr"
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return non-zero if file exists, otherwise return 0.
function fileExists(file) {
    return !system("test -f " file)
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

BEGIN {
    initUrlEncoding()
    initAnsiCode()
    initUriSchemes()
}

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used functions for string and array operations, logging.#
####################################################################

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line) {
    # Remove preceding spaces
    gsub(/^[[:space:]]+/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    if (length(array)) {
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != "")
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array # array == ""

    return temp
}

# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > "/dev/stderr"
}

# Print error message.
function e(text) {
    print ansi("red", text) > "/dev/stderr"
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > "/dev/stderr"
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return non-zero if file exists, otherwise return 0.
function fileExists(file) {
    return !system("test -f " file)
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

BEGIN {
    initUrlEncoding()
    initAnsiCode()
    initUriSchemes()
}