Translators/BingTranslator: fix for Mongolian (Cyrillic)

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 8 Jun 2022
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Set IG, IID, and BingTokenKey (a URL-encoded string containing token and key).
function bingSetup(    ast, content, cookie, group, header, isBody, key,
                       location, status, token, tokens, url) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie
    Cookie = cookie

    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
        l(IG, "IG")
    } else {
        e("[ERROR] Failed to extract IG.")
        exit 1
    }

    match(content, /data-iid="([^"]+)"/, group)
    if (group[1]) {
        IID = group[1]
        l(IID, "IID")
    } else {
        e("[ERROR] Failed to extract IID.")
        exit 1
    }

    match(content, /params_RichTranslateHelper = ([^;]+);/, group)
    if (group[1]) {
        tokenize(tokens, group[1])
        parseJson(ast, tokens)
        key = ast[0 SUBSEP 0]
        token = unparameterize(ast[0 SUBSEP 1])
        BingTokenKey = sprintf("&token=%s&key=%s", quote(token), quote(key))
        l(BingTokenKey, "BingTokenKey")
    } else {
        e("[ERROR] Failed to extract token & key.")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3" sprintf("?IG=%s&IID=%s", IG, IID)
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestUserAgent(text, sl, tl, hl, type) {
    return ""
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl BingTokenKey
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetup()

    # Hot-patches for Bing's own translator language codes
    # See: <https://docs.microsoft.com/en-us/azure/cognitive-services/Translator/language-support>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "tl")    _sl = "fil" # Bing uses 'fil' for Filipino
    if (_sl == "hmn")   _sl = "mww" # Bing uses 'mww' for Hmong Daw
    if (_sl == "ku")    _sl = "kmr" # Bing uses 'kmr' for Northern Kurdish
    else if (_sl == "ckb") _sl = "ku" # and 'ku' for Central Kurdish
    if (_sl == "mn")    _sl = "mn-Cyrl" # Bing uses 'mn-Cyrl' for Mongolian (Cyrillic)
    if (_sl == "no")    _sl = "nb"  # Bing uses Norwegian Bokmål
    # Bing uses 'pt' or 'pt-br' for Brazilian Portuguese, 'pt-pt' for European Portuguese
    if (_sl == "pt-BR") _sl = "pt" # just pt-br
    else if (_sl == "pt-PT") _sl = "pt" # FIXME: support pt-pt
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "tl")    _tl = "fil"
    if (_tl == "hmn")   _tl = "mww"
    if (_tl == "ku")    _tl = "kmr"
    else if (_tl == "ckb") _tl = "ku"
    if (_tl == "mn")    _tl = "mn-Cyrl"
    if (_tl == "no")    _tl = "nb"
    if (_tl == "pt-BR") _tl = "pt"
    else if (_tl == "pt-PT") _tl = "pt-pt"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 8 Jun 2022
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Set IG, IID, and BingTokenKey (a URL-encoded string containing token and key).
function bingSetup(    ast, content, cookie, group, header, isBody, key,
                       location, status, token, tokens, url) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie
    Cookie = cookie

    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
        l(IG, "IG")
    } else {
        e("[ERROR] Failed to extract IG.")
        exit 1
    }

    match(content, /data-iid="([^"]+)"/, group)
    if (group[1]) {
        IID = group[1]
        l(IID, "IID")
    } else {
        e("[ERROR] Failed to extract IID.")
        exit 1
    }

    match(content, /params_RichTranslateHelper = ([^;]+);/, group)
    if (group[1]) {
        tokenize(tokens, group[1])
        parseJson(ast, tokens)
        key = ast[0 SUBSEP 0]
        token = unparameterize(ast[0 SUBSEP 1])
        BingTokenKey = sprintf("&token=%s&key=%s", quote(token), quote(key))
        l(BingTokenKey, "BingTokenKey")
    } else {
        e("[ERROR] Failed to extract token & key.")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3" sprintf("?IG=%s&IID=%s", IG, IID)
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestUserAgent(text, sl, tl, hl, type) {
    return ""
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl BingTokenKey
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetup()

    # Hot-patches for Bing's own translator language codes
    # See: <https://docs.microsoft.com/en-us/azure/cognitive-services/Translator/language-support>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "tl")    _sl = "fil" # Bing uses 'fil' for Filipino
    if (_sl == "hmn")   _sl = "mww" # Bing uses 'mww' for Hmong Daw
    if (_sl == "ku")    _sl = "kmr" # Bing uses 'kmr' for Northern Kurdish
    else if (_sl == "ckb") _sl = "ku" # and 'ku' for Central Kurdish
    if (_sl == "no")    _sl = "nb"  # Bing uses Norwegian Bokmål
    # Bing uses 'pt' or 'pt-br' for Brazilian Portuguese, 'pt-pt' for European Portuguese
    if (_sl == "pt-BR") _sl = "pt" # just pt-br
    else if (_sl == "pt-PT") _sl = "pt" # FIXME: support pt-pt
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "tl")    _tl = "fil"
    if (_tl == "hmn")   _tl = "mww"
    if (_tl == "ku")    _tl = "kmr"
    else if (_tl == "ckb") _tl = "ku"
    if (_tl == "no")    _tl = "nb"
    if (_tl == "pt-BR") _tl = "pt"
    else if (_tl == "pt-PT") _tl = "pt-pt"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}


su/tag_dll.awk: fixed whitespace

su/tag_dll.awk: fixed whitespace

darcs-hash:20081127232613-db55f-d8a29c04b6f8dd685f6b1cf6b7c6b07f27b416cc.gz

#! /usr/bin/env awk
#
# Create reference tags and WIN32 initialization functions for tags.
#
# --------------------------------------------------------------------
#
# This file is part of the Sofia-SIP package
#
# Copyright (C) 2005 Nokia Corporation.
#
# Contact: Pekka Pessi <pekka.pessi@nokia.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# --------------------------------------------------------------------
#
# Author: Pekka Pessi <Pekka.Pessi@nokia.com>
#
# Created: Tue Feb 26 14:11:25 2002 ppessi
#

BEGIN {
  DEFS = 0;
  HEADER = 1;
}

HEADER {
  HEADER = 0;

  if (OUTNAME != "") {
    fn = OUTNAME;
  } else {
    fn = FILENAME;
  }

  if (REF == "") {
    REF = fn;
    sub(/[.]c(at)?/, "_ref.c", REF);
  }

  if (NODLL) DLL = "/dev/null";

  if (DLL == "") {
    DLL = fn;
    sub(/[.]c(at)?/, "_dll.c", DLL);
  }

  base = fn;
  sub(/.*\//, "", base);

  printf("#ifdef _WIN32\n"\
	 "/*\n" \
	 " * PLEASE NOTE: \n" \
	 " * \n" \
	 " * This file is automatically generated by tag_dll.awk.\n"\
	 " * It contains magic required by Win32 DLLs to initialize\n"\
	 " * tag_typedef_t variables.\n"\
	 " * \n"\
	 " * Do not, repeat, do not edit this file. Edit '%s' instead.\n"\
	 " * \n"\
	 " */\n\n"\
	 "#define EXPORT __declspec(dllexport)\n\n", base) > DLL;

  if (DLLREF) {
    print \
      "#include \"config.h\"\n\n" \
      "#include <sofia-sip/su_tag_class.h>\n\n" > DLL;
  }

  printf("/*\n" \
	 " * PLEASE NOTE: \n" \
	 " * \n" \
	 " * This file is automatically generated by tag_dll.awk.\n"\
	 " * It contains tag_typedef_t for tag references.\n"\
	 " * \n"\
	 " * Do not, repeat, do not edit this file. Edit '%s' instead.\n"\
	 " * \n"\
	 " */\n\n"\
	 "#include \"config.h\"\n\n"\
	 "#include <sofia-sip/su_tag_class.h>\n"\
	 "\n"\
	 "#if defined _WIN32 || defined HAVE_OPEN_C\n"\
	 "#define EXPORT __declspec(dllexport)\n"\
	 "#else\n"\
	 "#define EXPORT\n"\
	 "#endif\n\n",
	 base) > REF;

  if (LIST) {
    print "#include <stddef.h>" >REF;
  }
  print "" > REF;
}

/^#define TAG_NAMESPACE/ {
  print "#undef TAG_NAMESPACE" > REF;
  print $0 > REF;
  print "" > REF;
  print "#undef TAG_NAMESPACE" > DLL;
  print $0 > DLL;
  print "" > DLL;
}

/SU_HAVE_EXPERIMENTAL/ {
  print $0 > REF;
  print $0 > DLL;
}

!DEFS && /^tag_typedef_t/ { DEFS = 1; }

DEFS && /tag_typedef_t/ {
  tag = $2;
  typedefs[tag] = $0;

  if ($0 !~ /NSTAG_TYPEDEF/) {
    print "extern tag_typedef_t "tag";" > REF;
    print "EXPORT tag_typedef_t "tag"_ref;" > DLL;
    print "EXPORT tag_typedef_t "tag"_ref = \n  REFTAG_TYPEDEF("tag");" > REF;
  }

  gsub(/ = .*$/, ";");
}

!DLLREF { print $0 > DLL; }

END {
  if (LIST) {
    print "\nEXPORT tag_type_t " LIST "[] =\n{" > REF;
    print "\nEXPORT tag_type_t " LIST "[] =\n{" > DLL;
    for (tag in typedefs) {
      if (typedefs[tag] !~ /NSTAG_TYPEDEF/) {
        print "  " tag "," > REF;
        print "  " tag "," > DLL;
      }
    }
    print "  NULL\n};" > REF;
    print "  NULL\n};" > DLL;
  }

  printf("\n" \
	 "#include <windows.h>\n"\
	 "\n"\
	 "BOOL WINAPI DllMain(HINSTANCE hInst, DWORD fwdReason, LPVOID fImpLoad)\n"\
	 "{\n") > DLL;

   for (tag in typedefs) {
     def = typedefs[tag];
     if (!DLLREF) {
       sub(/^tag_typedef_t /, "  tag_typedef_t _", def);
       print def > DLL;
     }
     else {
       print "  extern tag_typedef_t "tag";" > DLL;
     }
     print "  tag_typedef_t _"tag"_ref =\n    REFTAG_TYPEDEF("tag");" > DLL;
   }

   print "" > DLL;

   for (tag in typedefs) {
     if (!DLLREF) {
       print "  *(struct tag_type_s *)" tag " = *_" tag ";" > DLL;
     }
     print "  *(struct tag_type_s *)" tag "_ref = *_" tag "_ref;" > DLL;
   }

   print "\n  return TRUE;" > DLL;
   print "}" > DLL;
   print "\n#endif" > DLL;
}


added namespaces to enhance params template creation

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw"
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
      print "#@prefix dcterms:    <http://purl.org/dc/elements/1.1/> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
   }
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
                                     print ":dataset a void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:dataset_version    \"%s\";\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                     print "      a conversion:RawConversionProcess;"
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   }
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS);
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw"
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
      print "#@prefix dcterms:    <http://purl.org/dc/elements/1.1/> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
   }
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
                                     print ":dataset a void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:dataset_version    \"%s\";\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                     print "      a conversion:RawConversionProcess;"
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   }
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS);
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}


2013-06-05: parsegaspriceshtml.awk: Added substitutions for Marathon and BP/Duchess pictures.

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

/
<div class=\"p","")
   sub("^[ ]*

[ ]*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && /<a href="/ {
   gsub("","Marathon")
   gsub("","Duchess")
   gsub("","Thorntons")
   gsub("","Thorntons")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.*<a href=\"/Profile[.]aspx[?]member=","")
   sub("\".*$","")
   reporter[i]=$0
   # print
}

inGasPriceRecord && expectRelativeTime {
   expectRelativeTime=False
   sub("^[ ]*","")
   sub("[ ]ago[ ]*$","")
   timeAgo[i]=$0
   # print
}

inGasPriceRecord && /<div class="tm" / {
   sub("^[ ]*
[ ]*$","")
   time[i]=$0
   expectRelativeTime=True
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

/
<div class=\"p","")
   sub("^[ ]*

[ ]*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && /<a href="/ {
   gsub("","Thorntons")
   gsub("","Thorntons")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.*<a href=\"/Profile[.]aspx[?]member=","")
   sub("\".*$","")
   reporter[i]=$0
   # print
}

inGasPriceRecord && expectRelativeTime {
   expectRelativeTime=False
   sub("^[ ]*","")
   sub("[ ]ago[ ]*$","")
   timeAgo[i]=$0
   # print
}

inGasPriceRecord && /<div class="tm" / {
   sub("^[ ]*
[ ]*$","")
   time[i]=$0
   expectRelativeTime=True
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}


Help: display translation engine in getVersion()

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead, platform) {
    initAudioPlayer()
    initPager()
    platform = detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", platform)                      \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", Option["sl"])           \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "translation engine", Option["engine"])    \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCE") "]"                            \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-S") ", " ansi("bold", "-list-engines")) RS \
        ins(2, "List available translation engines and exit.") RS       \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Translator options:" RS                                     \
        ins(1, ansi("bold", "-e ") ansi("underline", "ENGINE")          \
            ", " ansi("bold", "-engine ") ansi("underline", "ENGINE")) RS \
        ins(2, "Specify the translation engine to use.") RS             \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-identify")) RS                            \
        ins(2, "Language identification.") RS                           \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-speak")) RS                               \
        ins(2, "Listen to the original text.") RS                       \
        ins(1, ansi("bold", "-n ") ansi("underline", "VOICE")           \
            ", " ansi("bold", "-narrator ") ansi("underline", "VOICE")) RS \
        ins(2, "Specify the narrator, and listen to the translation.") RS \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view")) RS                             \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-sl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-source ") ansi("underline", "CODE")     \
            ", " ansi("bold", "-from ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify the source language.") RS                       \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-to ") ansi("underline", "CODES")) RS    \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists("man/" Command ".1"))
        system("man man/" Command ".1" SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, num, r, rows, saveSortedIn,
                      t1, t2) {
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        # show languages only with stable support
        if (Locale[code]["support"] != "unstable") {
            col = int(i / rows)
            append(cols[col], code)
            i++
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getDisplay(cols[j][i])
                    switch (cols[j][i]) { # fix rendered text width
                    case "he":
                        t1 = sprintf(" %-18s", t1)
                        break
                    case "ur":
                        t1 = sprintf(" %-17s", t1)
                        break
                    case "hi": case "gu": case "km": case "kn":
                    case "my": case "ne": case "pa": case "si":
                    case "ta": case "te": case "yi":
                        t1 = sprintf(" %-16s", t1)
                        break
                    case "ja": case "ko":
                        t1 = sprintf(" %-12s", t1)
                        break
                    case "zh-CN": case "zh-TW":
                        t1 = sprintf(" %-11s", t1)
                        break
                    default:
                        if (length(t1) <= 15)
                            t1 = sprintf(" %-15s", t1)
                    }
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s |", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s|", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s|", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    } else {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getName(cols[j][i])
                    if (length(t1) > 15)
                        t1 = substr(t1, 1, 12) "..."
                    t1 = sprintf(" %-15s", t1)
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s |", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s|", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s|", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}

####################################################################
# Help.awk                                                         #
####################################################################

# Return version as a string.
function getVersion(    build, gitHead, platform) {
    initAudioPlayer()
    initPager()
    platform = detectProgram("uname", "-s", 1)
    if (ENVIRON["TRANS_BUILD"])
        build = "-" ENVIRON["TRANS_BUILD"]
    else {
        gitHead = getGitHead()
        build = gitHead ? "-git:" gitHead : ""
    }

    return ansi("bold", sprintf("%-22s%s%s\n\n", Name, Version, build)) \
        sprintf("%-22s%s\n", "platform", platform)                      \
        sprintf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])     \
        sprintf("%s\n", FriBidi ? FriBidi :
                "fribidi (GNU FriBidi) [NOT INSTALLED]")                \
        sprintf("%-22s%s\n", "audio player", AudioPlayer ? AudioPlayer :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal pager", Pager ? Pager :
                "[NOT INSTALLED]")                                      \
        sprintf("%-22s%s\n", "terminal type", ENVIRON["TERM"])          \
        sprintf("%-22s%s (%s)\n", "user locale", UserLocale, getName(UserLang)) \
        sprintf("%-22s%s\n", "home language", Option["hl"])             \
        sprintf("%-22s%s\n", "source language", Option["sl"])           \
        sprintf("%-22s%s\n", "target language", join(Option["tl"], "+")) \
        sprintf("%-22s%s\n", "proxy", Option["proxy"] ? Option["proxy"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "user-agent", Option["user-agent"] ? Option["user-agent"] :
                "[NONE]")                                               \
        sprintf("%-22s%s\n", "theme", Option["theme"])                  \
        sprintf("%-22s%s\n", "init file", InitScript ? InitScript : "[NONE]") \
        sprintf("\n%-22s%s", "Report bugs to:", "https://github.com/soimort/translate-shell/issues")
}

# Return help message as a string.
function getHelp() {
    return "Usage:  " ansi("bold", Command)                             \
        " [" ansi("underline", "OPTIONS") "]"                           \
        " [" ansi("underline", "SOURCE") "]"                            \
        ":[" ansi("underline", "TARGETS") "]"                           \
        " [" ansi("underline", "TEXT") "]..." RS                        \
        RS "Information options:" RS                                    \
        ins(1, ansi("bold", "-V") ", " ansi("bold", "-version")) RS     \
        ins(2, "Print version and exit.") RS                            \
        ins(1, ansi("bold", "-H") ", " ansi("bold", "-help")) RS        \
        ins(2, "Print help message and exit.") RS                       \
        ins(1, ansi("bold", "-M") ", " ansi("bold", "-man")) RS         \
        ins(2, "Show man page and exit.") RS                            \
        ins(1, ansi("bold", "-T") ", " ansi("bold", "-reference")) RS   \
        ins(2, "Print reference table of languages and exit.") RS       \
        ins(1, ansi("bold", "-R") ", " ansi("bold", "-reference-english")) RS \
        ins(2, "Print reference table of languages (in English names) and exit.") RS \
        ins(1, ansi("bold", "-L ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-list ") ansi("underline", "CODES")) RS  \
        ins(2, "Print details of languages and exit.") RS               \
        ins(1, ansi("bold", "-S") ", " ansi("bold", "-list-engines")) RS \
        ins(2, "List available translation engines and exit.") RS       \
        ins(1, ansi("bold", "-U") ", " ansi("bold", "-upgrade")) RS     \
        ins(2, "Check for upgrade of this program.") RS                 \
        RS "Translator options:" RS                                     \
        ins(1, ansi("bold", "-e ") ansi("underline", "ENGINE")          \
            ", " ansi("bold", "-engine ") ansi("underline", "ENGINE")) RS \
        ins(2, "Specify the translation engine to use.") RS             \
        RS "Display options:" RS                                        \
        ins(1, ansi("bold", "-verbose")) RS                             \
        ins(2, "Verbose mode. (default)") RS                            \
        ins(1, ansi("bold", "-b") ", " ansi("bold", "-brief")) RS       \
        ins(2, "Brief mode.") RS                                        \
        ins(1, ansi("bold", "-d") ", " ansi("bold", "-dictionary")) RS  \
        ins(2, "Dictionary mode.") RS                                   \
        ins(1, ansi("bold", "-identify")) RS                            \
        ins(2, "Language identification.") RS                           \
        ins(1, ansi("bold", "-show-original ") ansi("underline", "Y/n")) RS \
        ins(2, "Show original text or not.") RS                         \
        ins(1, ansi("bold", "-show-original-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of original text or not.") RS    \
        ins(1, ansi("bold", "-show-translation ") ansi("underline", "Y/n")) RS \
        ins(2, "Show translation or not.") RS                           \
        ins(1, ansi("bold", "-show-translation-phonetics ") ansi("underline", "Y/n")) RS \
        ins(2, "Show phonetic notation of translation or not.") RS      \
        ins(1, ansi("bold", "-show-prompt-message ") ansi("underline", "Y/n")) RS \
        ins(2, "Show prompt message or not.") RS                        \
        ins(1, ansi("bold", "-show-languages ") ansi("underline", "Y/n")) RS \
        ins(2, "Show source and target languages or not.") RS           \
        ins(1, ansi("bold", "-show-original-dictionary ") ansi("underline", "y/N")) RS \
        ins(2, "Show dictionary entry of original text or not.") RS     \
        ins(1, ansi("bold", "-show-dictionary ") ansi("underline", "Y/n")) RS \
        ins(2, "Show dictionary entry of translation or not.") RS       \
        ins(1, ansi("bold", "-show-alternatives ") ansi("underline", "Y/n")) RS \
        ins(2, "Show alternative translations or not.") RS              \
        ins(1, ansi("bold", "-w ") ansi("underline", "NUM")             \
            ", " ansi("bold", "-width ") ansi("underline", "NUM")) RS   \
        ins(2, "Specify the screen width for padding.") RS              \
        ins(1, ansi("bold", "-indent ") ansi("underline", "NUM")) RS    \
        ins(2, "Specify the size of indent (number of spaces).") RS     \
        ins(1, ansi("bold", "-theme ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the theme to use.") RS                          \
        ins(1, ansi("bold", "-no-theme")) RS                            \
        ins(2, "Do not use any other theme than default.") RS           \
        ins(1, ansi("bold", "-no-ansi")) RS                             \
        ins(2, "Do not use ANSI escape codes.") RS                      \
        RS "Audio options:" RS                                          \
        ins(1, ansi("bold", "-p, -play")) RS                            \
        ins(2, "Listen to the translation.") RS                         \
        ins(1, ansi("bold", "-speak")) RS                               \
        ins(2, "Listen to the original text.") RS                       \
        ins(1, ansi("bold", "-n ") ansi("underline", "VOICE")           \
            ", " ansi("bold", "-narrator ") ansi("underline", "VOICE")) RS \
        ins(2, "Specify the narrator, and listen to the translation.") RS \
        ins(1, ansi("bold", "-player ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the audio player to use, and listen to the translation.") RS \
        ins(1, ansi("bold", "-no-play")) RS                             \
        ins(2, "Do not listen to the translation.") RS                  \
        RS "Terminal paging and browsing options:" RS                   \
        ins(1, ansi("bold", "-v") ", " ansi("bold", "-view")) RS        \
        ins(2, "View the translation in a terminal pager.") RS          \
        ins(1, ansi("bold", "-pager ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the terminal pager to use, and view the translation.") RS \
        ins(1, ansi("bold", "-no-view")) RS                             \
        ins(2, "Do not view the translation in a terminal pager.") RS   \
        ins(1, ansi("bold", "-browser ") ansi("underline", "PROGRAM")) RS \
        ins(2, "Specify the web browser to use.") RS                    \
        RS "Networking options:" RS                                     \
        ins(1, ansi("bold", "-x ") ansi("underline", "HOST:PORT")       \
            ", " ansi("bold", "-proxy ") ansi("underline", "HOST:PORT")) RS \
        ins(2, "Use HTTP proxy on given port.") RS                      \
        ins(1, ansi("bold", "-u ") ansi("underline", "STRING")          \
            ", " ansi("bold", "-user-agent ") ansi("underline", "STRING")) RS \
        ins(2, "Specify the User-Agent to identify as.") RS             \
        RS "Interactive shell options:" RS                              \
        ins(1, ansi("bold", "-I") ", " ansi("bold", "-interactive") ", " ansi("bold", "-shell")) RS \
        ins(2, "Start an interactive shell.") RS                        \
        ins(1, ansi("bold", "-E") ", " ansi("bold", "-emacs")) RS       \
        ins(2, "Start the GNU Emacs front-end for an interactive shell.") RS \
        ins(1, ansi("bold", "-no-rlwrap")) RS                           \
        ins(2, "Do not invoke rlwrap when starting an interactive shell.") RS \
        RS "I/O options:" RS                                            \
        ins(1, ansi("bold", "-i ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-input ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the input file.") RS                            \
        ins(1, ansi("bold", "-o ") ansi("underline", "FILENAME")        \
            ", " ansi("bold", "-output ") ansi("underline", "FILENAME")) RS \
        ins(2, "Specify the output file.") RS                           \
        RS "Language preference options:" RS                            \
        ins(1, ansi("bold", "-l ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-hl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-lang ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify your home language.") RS                        \
        ins(1, ansi("bold", "-s ") ansi("underline", "CODE")            \
            ", " ansi("bold", "-sl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-source ") ansi("underline", "CODE")     \
            ", " ansi("bold", "-from ") ansi("underline", "CODE")) RS   \
        ins(2, "Specify the source language.") RS                       \
        ins(1, ansi("bold", "-t ") ansi("underline", "CODES")           \
            ", " ansi("bold", "-tl ") ansi("underline", "CODE")         \
            ", " ansi("bold", "-target ") ansi("underline", "CODES")    \
            ", " ansi("bold", "-to ") ansi("underline", "CODES")) RS    \
        ins(2, "Specify the target language(s), joined by '+'.") RS     \
        RS "Other options:" RS                                          \
        ins(1, ansi("bold", "-no-init")) RS                             \
        ins(2, "Do not load any initialization script.") RS             \
        RS "See the man page " Command "(1) for more information."
}

# Show man page.
function showMan(    temp) {
    if (ENVIRON["TRANS_MANPAGE"]) {
        initPager()
        Groff = detectProgram("groff", "--version")
        if (Pager && Groff) {
            temp = "echo -E \"${TRANS_MANPAGE}\""
            temp = temp PIPE                                            \
                Groff " -Wall -mtty-char -mandoc -Tutf8 "               \
                "-rLL=" Option["width"] "n -rLT=" Option["width"] "n"
            switch (Pager) {
            case "less":
                temp = temp PIPE                                        \
                    Pager " -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\""
                break
            case "most":
                temp = temp PIPE Pager " -Cs"
                break
            default: # more
                temp = temp PIPE Pager
            }
            system(temp)
            return
        }
    }
    if (fileExists("man/" Command ".1"))
        system("man man/" Command ".1" SUPERR)
    else if (system("man " Command SUPERR))
        print getHelp()
}

# Return a reference table of languages as a string.
# Parameters:
#     displayName = "endonym" or "name"
function getReference(displayName,
                      ####
                      code, col, cols, i, j, name, num, r, rows, saveSortedIn,
                      t1, t2) {
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = displayName == "endonym" ? "@ind_num_asc" :
        "compName"
    for (code in Locale) {
        # show languages only with stable support
        if (Locale[code]["support"] != "unstable") {
            col = int(i / rows)
            append(cols[col], code)
            i++
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn

    if (displayName == "endonym") {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getDisplay(cols[j][i])
                    switch (cols[j][i]) { # fix rendered text width
                    case "he":
                        t1 = sprintf(" %-18s", t1)
                        break
                    case "ur":
                        t1 = sprintf(" %-17s", t1)
                        break
                    case "hi": case "gu": case "km": case "kn":
                    case "my": case "ne": case "pa": case "si":
                    case "ta": case "te": case "yi":
                        t1 = sprintf(" %-16s", t1)
                        break
                    case "ja": case "ko":
                        t1 = sprintf(" %-12s", t1)
                        break
                    case "zh-CN": case "zh-TW":
                        t1 = sprintf(" %-11s", t1)
                        break
                    default:
                        if (length(t1) <= 15)
                            t1 = sprintf(" %-15s", t1)
                    }
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s |", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s|", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s|", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    } else {
        r = "┌" replicate("─", 23) "┬" replicate("─", 23) "┬" replicate("─", 23) "┐" RS
        for (i = 0; i < rows; i++) {
            r = r "│"
            for (j = 0; j < 3; j++) {
                if (cols[j][i]) {
                    t1 = getName(cols[j][i])
                    if (length(t1) > 15)
                        t1 = substr(t1, 1, 12) "..."
                    t1 = sprintf(" %-15s", t1)
                    switch (length(cols[j][i])) {
                    case 1: case 2: case 3: case 4:
                        t2 = sprintf("- %s |", ansi("bold", sprintf("%4s", cols[j][i])))
                        break
                    case 5:
                        t2 = sprintf("- %s|", ansi("bold", cols[j][i]))
                        break
                    case 6:
                        t2 = sprintf("-%s|", ansi("bold", cols[j][i]))
                        break
                    case 7:
                        t2 = sprintf("-%s", ansi("bold", cols[j][i]))
                        break
                    default:
                        t2 = ansi("bold", cols[j][i])
                    }
                    r = r t1 t2
                } else
                    r = r sprintf("%23s│", NULLSTR)
            }
            r = r RS
        }
        r = r "└" replicate("─", 23) "┴" replicate("─", 23) "┴" replicate("─", 23) "┘"
    }
    return r
}

# Return detailed information of languages as a string.
function getList(codes,    code, i, r, saveSortedIn) {
    r = NULLSTR
    if (!isarray(codes))
        r = getDetails(codes)
    else if (anything(codes)) {
        saveSortedIn = PROCINFO["sorted_in"]
        PROCINFO["sorted_in"] = "@ind_num_asc"
        for (i in codes)
            r = (r ? r RS prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])) RS \
                 : r) getDetails(codes[i])
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        r = getDetails(Option["hl"])
    return r
}


Don't fail on 'out og sequence'

Don't fail on 'out og sequence'

BEGIN {
	active=0;
	print "BatchId, jobtype, starttime, jobid, endtime, totalmaptime, totalreducetime, maptasks";
}

# Output:
# BatchId, job type, start time, end time, duration, jobId, #map input records, map time, reduce time

#2014-08-12 07:50:07.901 INFO  d.s.m.autonomous.AutonomousComponent - Found batch B400026955287-RT2
/Found/ {
	if (!active) {
		print "RECORD START" | "cat 1>&2"
		split($0, s, "batch ");
		printf "%s", s[2];
		printf ", %s", jobtype
		active=1
	} else {
		print "ERROR - out of sequence!" $0 | "cat 1>&2"
		print "FAILED"
		print "RECORD START" | "cat 1>&2"
		split($0, s, "batch ");
		printf "%s", s[2];
		printf ", %s", jobtype
		active=1
	}
}

# 2014-08-11 10:30:14.415 INFO  org.apache.hadoop.mapreduce.Job - Running job: job_1407155868271_0080
/Running job:/ {
	if (active==1) {
		#printf "\tSTART: %s\n", $0
		split($0,s," INFO ");
		printf ", %s", s[1];
		split($0,s,"Running job: ")
		printf ", %s", s[2];
	}
}

# This is the last log message in a record
#		Map input records=11680
/Map input records/ {
	if (active==1) {
		#printf "\t%s, \n", $0
		split($0, s, "=");
		printf ", %s\n", s[2];
		print "RECORD END" | "cat 1>&2"
		active = 0
	}
}

#		Total time spent by all maps in occupied slots (ms)=5512646
/Total time spent by all maps in occupied slots/ {
	if (active==1) {
		#printf "\t%s, \n", $0
		split($0, s, "=");
		printf ", %s", s[2];
	}
}

#		Total time spent by all reduces in occupied slots (ms)=81308394
/Total time spent by all reduces in occupied slots/ {
	if (active==1) {
		#printf "\t%s, \n", $0
		split($0, s, "=");
		printf ", %s", s[2];
	}
}

/failed with state FAILED due/ {
	if (active==1) {
		#print "\tFAILED: " $0
		printf ", FAILED\n";
		active = 0
		print "RECORD END" | "cat 1>&2"
	}
}

#2014-08-12 04:38:00.649 INFO  org.apache.hadoop.mapreduce.Job - Job job_1407155868271_0084 completed successfully
/completed successfully/ {
	if (active==1) {
		#print "\tSTOP: " $0
		split($0, s, " INFO ");
		printf ", %s", s[1];
	}
}

BEGIN {
	active=0;
	print "BatchId, jobtype, starttime, jobid, endtime, totalmaptime, totalreducetime, maptasks";
}

# Output:
# BatchId, job type, start time, end time, duration, jobId, #map input records, map time, reduce time

#2014-08-12 07:50:07.901 INFO  d.s.m.autonomous.AutonomousComponent - Found batch B400026955287-RT2
/Found/ {
	if (!active) {
		print "RECORD START" | "cat 1>&2"
		split($0, s, "batch ");
		printf "%s", s[2];
		printf ", %s", jobtype
		active=1
	} else {
		print "ERROR - out of sequence!" $0 | "cat 1>&2"
		exit
	}
}

# 2014-08-11 10:30:14.415 INFO  org.apache.hadoop.mapreduce.Job - Running job: job_1407155868271_0080
/Running job:/ {
	if (active==1) {
		#printf "\tSTART: %s\n", $0
		split($0,s," INFO ");
		printf ", %s", s[1];
		split($0,s,"Running job: ")
		printf ", %s", s[2];
	}
}

# This is the last log message in a record
#		Map input records=11680
/Map input records/ {
	if (active==1) {
		#printf "\t%s, \n", $0
		split($0, s, "=");
		printf ", %s\n", s[2];
		print "RECORD END" | "cat 1>&2"
		active = 0
	}
}

#		Total time spent by all maps in occupied slots (ms)=5512646
/Total time spent by all maps in occupied slots/ {
	if (active==1) {
		#printf "\t%s, \n", $0
		split($0, s, "=");
		printf ", %s", s[2];
	}
}

#		Total time spent by all reduces in occupied slots (ms)=81308394
/Total time spent by all reduces in occupied slots/ {
	if (active==1) {
		#printf "\t%s, \n", $0
		split($0, s, "=");
		printf ", %s", s[2];
	}
}

/failed with state FAILED due/ {
	if (active==1) {
		#print "\tFAILED: " $0
		printf ", FAILED\n";
		active = 0
		print "RECORD END" | "cat 1>&s"
	}
}

#2014-08-12 04:38:00.649 INFO  org.apache.hadoop.mapreduce.Job - Job job_1407155868271_0084 completed successfully
/completed successfully/ {
	if (active==1) {
		#print "\tSTOP: " $0
		split($0, s, " INFO ");
		printf ", %s", s[1];
	}
}


Simplify toloka.awk

Simplify toloka.awk

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    print "INPUT:hyponym", "INPUT:hypernym", "INPUT:genitive", "GOLDEN:judgement", "HINT:text";
}
NR == 1 {
    for (i=1; i<=NF; i++) ix[$i] = i;
    next;
}
!$ix["found"] {
    next;
}
{
    hypernyms[$ix["hypernym"] OFS $ix["genitive"]];
}
$ix["hyponym"] != hyponym {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }

    hyponym = $ix["hyponym"];
    delete hypernyms;
}
END {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
    print "INPUT:hyponym", "INPUT:hypernym", "INPUT:genitive", "GOLDEN:judgement", "HINT:text";
}
NR == 1 {
    for (i=1; i<=NF; i++) ix[$i] = i;
}
NR > 1 && $ix["found"] {
    hypernyms[$ix["hypernym"] OFS $ix["genitive"]];
}
NR > 1 && $ix["found"] && $ix["hyponym"] != hyponym {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }

    hyponym = $ix["hyponym"];
    delete hypernyms;
}
END {
    for (hypernym_genitive in hypernyms) {
        print $ix["hyponym"], hypernym_genitive, "", "";
    }
}