- This should not be "unsigned".

- This should not be "unsigned".

git-svn-id: 74e0a9e6c5d9c3e57f79acf98f28a92415314dbb@290 76bb4867-1e22-0410-9289-e9e6e9b3b6f2

#!/usr/bin/awk -f
#
# $Id$
#
# Description: a script that generates a single byte code set to Unicode
# mapping table.
#

function conv(str) {
	if (!match(str, "^0[xX]")) {
		return 0 + str
	}

	retval = 0

	for (i = 3; i <= length(str); i++) {
		n = index("0123456789abcdefABCDEF", substr(str, i, 1)) - 1

		if (n < 0) {
			return 0 + str;
		} else if (n >= 16) {
			n -= 6;
		}

		retval = retval * 16 + n
	}

	return retval
}

BEGIN {
	FS="[ \t#]"
}

/^#/ {
	# Do nothing
}

{
	tbl[conv($1)] = conv($2)
}

END {
	print "/* This file is automatically generated. Do not edit! */"
	if (IFNDEF_NAME) {
		print "#ifndef " IFNDEF_NAME
	}

	print "static const int " TABLE_NAME "[] = {"
	i = 160;
	for (;;) {
		printf("\t0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x", tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++]);
		if (i != 256) {
			printf(",\n");
		} else {
			print ""
			break;
		}
	}
	print "};"

	if (IFNDEF_NAME) {
		print "#endif /* " IFNDEF_NAME " */"
	}
}

#!/usr/bin/awk -f
#
# $Id$
#
# Description: a script that generates a single byte code set to Unicode
# mapping table.
#

function conv(str) {
	if (!match(str, "^0[xX]")) {
		return 0 + str
	}

	retval = 0

	for (i = 3; i <= length(str); i++) {
		n = index("0123456789abcdefABCDEF", substr(str, i, 1)) - 1

		if (n < 0) {
			return 0 + str;
		} else if (n >= 16) {
			n -= 6;
		}

		retval = retval * 16 + n
	}

	return retval
}

BEGIN {
	FS="[ \t#]"
}

/^#/ {
	# Do nothing
}

{
	tbl[conv($1)] = conv($2)
}

END {
	print "/* This file is automatically generated. Do not edit! */"
	if (IFNDEF_NAME) {
		print "#ifndef " IFNDEF_NAME
	}

	print "static const unsigned int " TABLE_NAME "[] = {"
	i = 160;
	for (;;) {
		printf("\t0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x, 0x%04x", tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++], tbl[i++]);
		if (i != 256) {
			printf(",\n");
		} else {
			print ""
			break;
		}
	}
	print "};"

	if (IFNDEF_NAME) {
		print "#endif /* " IFNDEF_NAME " */"
	}
}


Translators/BingTranslator: trivial

Translators/BingTranslator: trivial

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 25 Jul 2019
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# [OBSOLETE?] Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    #bingSetIG() # set IG (no longer needed)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 25 Jul 2019
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# [OBSOLETE?] Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    #bingSetIG() # set IG (no longer needed)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code  # FIXME
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            delete ast  # purge old AST
            content = postResponse(text, il, il, _hl, "translate")
            tokenize(tokens, content)
            parseJson(ast, tokens)
            oPhonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 \
                                            SUBSEP "transliteration" SUBSEP "text"])
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "dicContent", 1, 1)
                l(dicTokens, "dicTokens", 1, 0, 1)
                l(dicAst, "dicAst")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}


the_t world_t would_t be_t a_t better_t place_t if_t some_t people_t did_t not_t feel_t the_t need_t to_t typedef_t everything_t

#! /usr/bin/awk -f
#	$OpenBSD: devlist2h.awk,v 1.10 2007/02/28 22:51:07 deraadt Exp $
#	$NetBSD: devlist2h.awk,v 1.9 2001/01/18 20:28:22 jdolecek Exp $
#
# Copyright (c) 1995, 1996 Christopher G. Demetriou
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Christopher G. Demetriou.
# 4. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
BEGIN {
	nproducts = nvendors = 0
	dfile="usbdevs_data.h"
	hfile="usbdevs.h"
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)

	if (os == "NetBSD")
		printf("/*\t\$NetBSD\$\t*/\n\n") > dfile
	else if (os == "FreeBSD")
		printf("/*\t\$FreeBSD\$\t*/\n\n") > dfile
	else if (os == "OpenBSD")
		printf("/*\t\$OpenBSD\$\t*/\n\n") > dfile
	else
		printf("/* ??? */\n\n") > dfile
	printf("/*\n") > dfile
	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > dfile
	printf(" *\n") > dfile
	printf(" * generated from:\n") > dfile
	printf(" *\t%s\n", VERSION) > dfile
	printf(" */\n") > dfile

	if (os == "NetBSD")
		printf("/*\t\$NetBSD\$\t*/\n\n") > hfile
	else if (os == "FreeBSD")
		printf("/*\t\$FreeBSD\$\t*/\n\n") > hfile
	else if (os == "OpenBSD")
		printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
	else
		printf("/* ??? */\n\n") > hfile
	printf("/*\n") > hfile
	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > hfile
	printf(" *\n") > hfile
	printf(" * generated from:\n") > hfile
	printf(" *\t%s\n", VERSION) > hfile
	printf(" */\n") > hfile

	next
}
$1 == "vendor" {
	nvendors++

	vendorindex[$2] = nvendors;		# record index for this name, for later.
	vendors[nvendors, 1] = $2;		# name
	vendors[nvendors, 2] = $3;		# id
	printf("#define\tUSB_VENDOR_%s\t%s\t", vendors[nvendors, 1],
	    vendors[nvendors, 2]) > hfile

	i = 3; f = 4;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			printf("%s", $f) > hfile
			if (f < NF)
				printf(" ") > hfile
			f++
			continue
		}
		vendors[nvendors, i] = $f
		printf("%s", vendors[nvendors, i]) > hfile
		if (f < NF)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen)
		printf(")") > hfile
	if (ocomment)
		printf(" */") > hfile
	printf("\n") > hfile

	next
}
$1 == "product" {
	nproducts++

	products[nproducts, 1] = $2;		# vendor name
	products[nproducts, 2] = $3;		# product id
	products[nproducts, 3] = $4;		# id
	printf("#define\tUSB_PRODUCT_%s_%s\t%s\t", products[nproducts, 1],
	    products[nproducts, 2], products[nproducts, 3]) > hfile

	i=4; f = 5;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			printf("%s", $f) > hfile
			if (f < NF)
				printf(" ") > hfile
			f++
			continue
		}
		products[nproducts, i] = $f
		printf("%s", products[nproducts, i]) > hfile
		if (f < NF)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen)
		printf(")") > hfile
	if (ocomment)
		printf(" */") > hfile
	printf("\n") > hfile

	next
}
{
	if ($0 == "")
		blanklines++
	print $0 > hfile
	if (blanklines < 2)
		print $0 > dfile
}
END {
	# print out the match tables

	printf("\n") > dfile

	printf("/*\n") > dfile
	printf(" * Descriptions of known vendors and devices (\"products\").\n") > dfile
	printf(" */\n") > dfile
	printf("struct usb_known_vendor {\n") > dfile
	printf("	u_int16_t	vendor;\n") > dfile
	printf("	char		*vendorname;\n") > dfile
	printf("};\n\n") > dfile

	printf("struct usb_known_product {\n") > dfile
	printf("	u_int16_t	vendor;\n") > dfile
	printf("	u_int16_t	product;\n") > dfile
	printf("	char		*productname;\n") > dfile
	printf("};\n\n") > dfile
	printf("const struct usb_known_product usb_known_products[] = {\n") \
	    > dfile
	for (i = 1; i <= nproducts; i++) {
		printf("\t{\n") > dfile
		printf("\t    USB_VENDOR_%s, USB_PRODUCT_%s_%s,\n",
		    products[i, 1], products[i, 1], products[i, 2]) \
		    > dfile

		printf("\t    \"") > dfile
		j = 4;
		needspace = 0;
		while (products[i, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", products[i, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile
		printf("\t},\n") > dfile
	}
	printf("\t{ 0, 0, NULL }\n") > dfile
	printf("};\n\n") > dfile

	printf("const struct usb_known_vendor usb_known_vendors[] = {\n") \
	     > dfile
	for (i = 1; i <= nvendors; i++) {
		printf("\t{\n") > dfile
		printf("\t    USB_VENDOR_%s,\n", vendors[i, 1]) \
		    > dfile

		printf("\t    \"") > dfile
		j = 3;
		needspace = 0;
		while (vendors[i, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", vendors[i, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile
		printf("\t},\n") > dfile
	}
	printf("\t{ 0, NULL }\n") > dfile
	printf("};\n") > dfile
}

#! /usr/bin/awk -f
#	$OpenBSD: devlist2h.awk,v 1.9 2007/02/28 22:30:55 deraadt Exp $
#	$NetBSD: devlist2h.awk,v 1.9 2001/01/18 20:28:22 jdolecek Exp $
#
# Copyright (c) 1995, 1996 Christopher G. Demetriou
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Christopher G. Demetriou.
# 4. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
BEGIN {
	nproducts = nvendors = 0
	dfile="usbdevs_data.h"
	hfile="usbdevs.h"
}
NR == 1 {
	VERSION = $0
	gsub("\\$", "", VERSION)

	if (os == "NetBSD")
		printf("/*\t\$NetBSD\$\t*/\n\n") > dfile
	else if (os == "FreeBSD")
		printf("/*\t\$FreeBSD\$\t*/\n\n") > dfile
	else if (os == "OpenBSD")
		printf("/*\t\$OpenBSD\$\t*/\n\n") > dfile
	else
		printf("/* ??? */\n\n") > dfile
	printf("/*\n") > dfile
	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > dfile
	printf(" *\n") > dfile
	printf(" * generated from:\n") > dfile
	printf(" *\t%s\n", VERSION) > dfile
	printf(" */\n") > dfile

	if (os == "NetBSD")
		printf("/*\t\$NetBSD\$\t*/\n\n") > hfile
	else if (os == "FreeBSD")
		printf("/*\t\$FreeBSD\$\t*/\n\n") > hfile
	else if (os == "OpenBSD")
		printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
	else
		printf("/* ??? */\n\n") > hfile
	printf("/*\n") > hfile
	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > hfile
	printf(" *\n") > hfile
	printf(" * generated from:\n") > hfile
	printf(" *\t%s\n", VERSION) > hfile
	printf(" */\n") > hfile

	next
}
$1 == "vendor" {
	nvendors++

	vendorindex[$2] = nvendors;		# record index for this name, for later.
	vendors[nvendors, 1] = $2;		# name
	vendors[nvendors, 2] = $3;		# id
	printf("#define\tUSB_VENDOR_%s\t%s\t", vendors[nvendors, 1],
	    vendors[nvendors, 2]) > hfile

	i = 3; f = 4;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			printf("%s", $f) > hfile
			if (f < NF)
				printf(" ") > hfile
			f++
			continue
		}
		vendors[nvendors, i] = $f
		printf("%s", vendors[nvendors, i]) > hfile
		if (f < NF)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen)
		printf(")") > hfile
	if (ocomment)
		printf(" */") > hfile
	printf("\n") > hfile

	next
}
$1 == "product" {
	nproducts++

	products[nproducts, 1] = $2;		# vendor name
	products[nproducts, 2] = $3;		# product id
	products[nproducts, 3] = $4;		# id
	printf("#define\tUSB_PRODUCT_%s_%s\t%s\t", products[nproducts, 1],
	    products[nproducts, 2], products[nproducts, 3]) > hfile

	i=4; f = 5;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			printf("%s", $f) > hfile
			if (f < NF)
				printf(" ") > hfile
			f++
			continue
		}
		products[nproducts, i] = $f
		printf("%s", products[nproducts, i]) > hfile
		if (f < NF)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen)
		printf(")") > hfile
	if (ocomment)
		printf(" */") > hfile
	printf("\n") > hfile

	next
}
{
	if ($0 == "")
		blanklines++
	print $0 > hfile
	if (blanklines < 2)
		print $0 > dfile
}
END {
	# print out the match tables

	printf("\n") > dfile

	printf("typedef u_int16_t usb_vendor_id_t;\n") > dfile
	printf("typedef u_int16_t usb_product_id_t;\n\n") > dfile

	printf("/*\n") > dfile
	printf(" * Descriptions of known vendors and devices (\"products\").\n") > dfile
	printf(" */\n") > dfile
	printf("struct usb_known_vendor {\n") > dfile
	printf("	usb_vendor_id_t		vendor;\n") > dfile
	printf("	char			*vendorname;\n") > dfile
	printf("};\n\n") > dfile

	printf("struct usb_known_product {\n") > dfile
	printf("	usb_vendor_id_t		vendor;\n") > dfile
	printf("	usb_product_id_t	product;\n") > dfile
	printf("	char			*productname;\n") > dfile
	printf("};\n\n") > dfile
	printf("const struct usb_known_product usb_known_products[] = {\n") \
	    > dfile
	for (i = 1; i <= nproducts; i++) {
		printf("\t{\n") > dfile
		printf("\t    USB_VENDOR_%s, USB_PRODUCT_%s_%s,\n",
		    products[i, 1], products[i, 1], products[i, 2]) \
		    > dfile

		printf("\t    \"") > dfile
		j = 4;
		needspace = 0;
		while (products[i, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", products[i, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile
		printf("\t},\n") > dfile
	}
	printf("\t{ 0, 0, NULL }\n") > dfile
	printf("};\n\n") > dfile

	printf("const struct usb_known_vendor usb_known_vendors[] = {\n") \
	     > dfile
	for (i = 1; i <= nvendors; i++) {
		printf("\t{\n") > dfile
		printf("\t    USB_VENDOR_%s,\n", vendors[i, 1]) \
		    > dfile

		printf("\t    \"") > dfile
		j = 3;
		needspace = 0;
		while (vendors[i, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", vendors[i, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile
		printf("\t},\n") > dfile
	}
	printf("\t{ 0, NULL }\n") > dfile
	printf("};\n") > dfile
}


Parser script displays compiler warnings

#
# MIT License
#
# Copyright (c) 2017 Jean-Sebastien CONAN
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

#
# Part of the camelSCAD library.
#
# Parses and displays the results of the unit tests.
#
# @package util/test
# @author jsconan
#

BEGIN {
    # will parse a CSV file
    FS = ";"

    # init counters
    packageCount = 0
    moduleCount  = 0
    assertCount  = 0
    successCount = 0
    failedCount  = 0

    # define ANSI colors
    BLACK    = "\033[0;30m"
    GRAY     = "\033[1;30m"
    RED      = "\033[0;31m"
    LRED     = "\033[1;31m"
    GREEN    = "\033[0;32m"
    LGREEN   = "\033[1;32m"
    BROWN    = "\033[0;33m"
    YELLOW   = "\033[1;33m"
    BLUE     = "\033[0;34m"
    LBLUE    = "\033[1;34m"
    MAGENTA  = "\033[0;35m"
    LMAGENTA = "\033[1;35m"
    CYAN     = "\033[0;36m"
    LCYAN    = "\033[1;36m"
    GREY     = "\033[0;37m"
    WHITE    = "\033[1;37m"
    RESET    = "\033[0m"
}

# detect compiler warnings
$0~/^WARNING:/ {
    printf("\n%s%s\n", LRED, $0)
}

# detect package declarations
$1~/package/ {
    packageCount = packageCount + 1
    package = $2
    lastType = "package"
    printf("\n%s[%s] ", LBLUE, $2)
}

# detect module declarations
$1~/module/ {
    moduleCount = moduleCount + 1
    module = $2
    lastType = "module"
    printf("\n%s    %s ", CYAN, $2)
}

# detect assertion results
$1~/assert/ {
    assertCount = assertCount + 1
    lastType = "assert"

    if ($2 == "#OK") {
        # successful assertion
        successCount = successCount + 1
        printf("%s.", LGREEN)
    } else {
        # failed assertion
        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $3
        failedAssertsMessage[failedCount] = $4
        failedAssertsDetails[failedCount] = $5
        failedCount = failedCount + 1

        # notify the failed assertion
        printf("%sF", LRED)
    }
}

# detect assertions counters
$1~/expect/ {
    # is failed count ?
    if ($5 != "true") {
        if ($3 == "") {
            # no expected number, try to take the actual number
            if (!$4) {
                # no actual number, assume it is at least 1
                expected = 1
            } else {
                # take the actual number as the expected number
                expected = $4
            }
        } else {
            # provided expected number
            expected = $3
        }

        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $6
        failedAssertsMessage[failedCount] = $7
        assertCount = assertCount + expected - $4
        failedCount = failedCount + 1

        # put a space after package name in case of some result will be displayed after
        if ($2 == "package") {
            printf(" ")
        }

        # if the last was a module, the result need to be displayed on a new line
        if ($2 == "module") {
            printf("\n    ")
        }

        # notify the missing assertions
        printf("%sM", LRED)
    }
}

END {
    separator = "%s-----------\n"
    printf("\n")

    if (failedCount > 0) {
        printf(separator, GREY)
        printf("%sFAILED\n", RED)
        package = ""
        context = ""

        for (i = 0; i < failedCount; i++) {
            if (package != failedAssertsPackage[i]) {
                package = failedAssertsPackage[i]
                printf("%s[%s]\n", GRAY, package)
            }
            if (failedAssertsContext[i] && context != failedAssertsContext[i]) {
                context = failedAssertsContext[i]
                printf("%s  %s\n", MAGENTA, context)
            }
            if (failedAssertsContext[i]) {
                printf("  ")
            }
            printf("%s- %s\n", RED, failedAssertsMessage[i])
            if (failedAssertsDetails[i]) {
                printf("    %s\n", failedAssertsDetails[i])
            }
            printf("\n")
        }
    }

    printf(separator, GREY)
    printf("%s%d packages\n", GREEN, packageCount)
    printf("%s%d modules\n", GREEN, moduleCount)
    printf("%s%d/%d asserts\n", GREEN, successCount, assertCount)
    printf(separator, GREY)

    if (failedCount == 0) {
        printf("%sAll successful!\n", GREEN)
    } else {
        printf("%s%d failed asserts!\n", RED, failedCount)
    }
}

#
# MIT License
#
# Copyright (c) 2017 Jean-Sebastien CONAN
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

#
# Part of the camelSCAD library.
#
# Parses and displays the results of the unit tests.
#
# @package util/test
# @author jsconan
#

BEGIN {
    # will parse a CSV file
    FS = ";"

    # init counters
    packageCount = 0
    moduleCount  = 0
    assertCount  = 0
    successCount = 0
    failedCount  = 0

    # define ANSI colors
    BLACK    = "\033[0;30m"
    GRAY     = "\033[1;30m"
    RED      = "\033[0;31m"
    LRED     = "\033[1;31m"
    GREEN    = "\033[0;32m"
    LGREEN   = "\033[1;32m"
    BROWN    = "\033[0;33m"
    YELLOW   = "\033[1;33m"
    BLUE     = "\033[0;34m"
    LBLUE    = "\033[1;34m"
    MAGENTA  = "\033[0;35m"
    LMAGENTA = "\033[1;35m"
    CYAN     = "\033[0;36m"
    LCYAN    = "\033[1;36m"
    GREY     = "\033[0;37m"
    WHITE    = "\033[1;37m"
    RESET    = "\033[0m"
}

# detect package declarations
$1~/package/ {
    packageCount = packageCount + 1
    package = $2
    lastType = "package"
    printf("\n%s[%s] ", LBLUE, $2)
}

# detect module declarations
$1~/module/ {
    moduleCount = moduleCount + 1
    module = $2
    lastType = "module"
    printf("\n%s    %s ", CYAN, $2)
}

# detect assertion results
$1~/assert/ {
    assertCount = assertCount + 1
    lastType = "assert"

    if ($2 == "#OK") {
        # successful assertion
        successCount = successCount + 1
        printf("%s.", LGREEN)
    } else {
        # failed assertion
        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $3
        failedAssertsMessage[failedCount] = $4
        failedAssertsDetails[failedCount] = $5
        failedCount = failedCount + 1

        # notify the failed assertion
        printf("%sF", LRED)
    }
}

# detect assertions counters
$1~/expect/ {
    # is failed count ?
    if ($5 != "true") {
        if ($3 == "") {
            # no expected number, try to take the actual number
            if (!$4) {
                # no actual number, assume it is at least 1
                expected = 1
            } else {
                # take the actual number as the expected number
                expected = $4
            }
        } else {
            # provided expected number
            expected = $3
        }

        failedAsserts[failedCount] = $0
        failedAssertsPackage[failedCount] = package
        failedAssertsModule[failedCount] = module
        failedAssertsContext[failedCount] = $6
        failedAssertsMessage[failedCount] = $7
        assertCount = assertCount + expected - $4
        failedCount = failedCount + 1

        # put a space after package name in case of some result will be displayed after
        if ($2 == "package") {
            printf(" ")
        }

        # if the last was a module, the result need to be displayed on a new line
        if ($2 == "module") {
            printf("\n    ")
        }

        # notify the missing assertions
        printf("%sM", LRED)
    }
}

END {
    separator = "%s-----------\n"
    printf("\n")

    if (failedCount > 0) {
        printf(separator, GREY)
        printf("%sFAILED\n", RED)
        package = ""
        context = ""

        for (i = 0; i < failedCount; i++) {
            if (package != failedAssertsPackage[i]) {
                package = failedAssertsPackage[i]
                printf("%s[%s]\n", GRAY, package)
            }
            if (failedAssertsContext[i] && context != failedAssertsContext[i]) {
                context = failedAssertsContext[i]
                printf("%s  %s\n", MAGENTA, context)
            }
            if (failedAssertsContext[i]) {
                printf("  ")
            }
            printf("%s- %s\n", RED, failedAssertsMessage[i])
            if (failedAssertsDetails[i]) {
                printf("    %s\n", failedAssertsDetails[i])
            }
            printf("\n")
        }
    }

    printf(separator, GREY)
    printf("%s%d packages\n", GREEN, packageCount)
    printf("%s%d modules\n", GREEN, moduleCount)
    printf("%s%d/%d asserts\n", GREEN, successCount, assertCount)
    printf(separator, GREY)

    if (failedCount == 0) {
        printf("%sAll successful!\n", GREEN)
    } else {
        printf("%s%d failed asserts!\n", RED, failedCount)
    }
}


fix csv by stripping double-quotes from committer names

#!/bin/awk
# commit 989fb6102add218589e8779c76fd61f91e113684
# Author: Tad Hunt <tadhunt@tad-hunts-macbook-pro.local>
# Date:   2009-11-30
#
#     ignore objects
#
# 4       0       .gitignore
#
# commit 54f97a5baf887f0d2a73745efaa0a647ed20512e
# Author: Tad Hunt <tadhunt@tad-hunts-macbook-pro.local>
# Date:   2009-11-30
#
#     reorg & add pitch
#
# 0       46      Makefile
# 0       62      bwcalc.c
# 0       32      dedupgain.sh
# 0       56      dedupgaintree.sh
# 0       391     sha1block.c
# 0       73      sha1speed.c
# 46      0       src/Makefile
# 62      0       src/bwcalc.c
# 32      0       src/dedupgain.sh
# 56      0       src/dedupgaintree.sh
# 391     0       src/sha1block.c
# 73      0       src/sha1speed.c
#
# commit 509aff7e53df7406749274d943ed573a0a78af02
# ...
#
BEGIN {
	stderr = "/dev/stderr"

#	printf("\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n",
#		"Date",
#		"Number of Insertions",
#		"Number of Deletions",
#		"Author",
#		"Date",
#		"Filename")

	last = "none"
}

/^commit[ 	].*/ {
	commit = $2

	last = "commit"
	next
}

/^Author:[ 	].*/ {
	author = $0
	sub(/^Author: /, "", author)

	last = "author"
	next
}

/^Date:[ 	].*/ {
	date = $2

	last = "date"
	next
}

/^Merge:[ 	].*/ {
	ismerge = 1
	last = "merge"
	next						# don't track merge history
}

/^[ 	]+.*/ {
	last = "logmsg"
	next						# don't track log message
}

/^-[ 	]+-[ 	]+.*/ {
	last = "binary"
	next						# don't track binary files
}

/^[0-9]+[ 	]+[0-9]+[ 	]+.*/ {
	ninsertions = $1
	ndeletions = $2
	file = $0
	sub(/^[0-9]+[ 	]+[0-9]+[ 	]+/, "", file)	# easier this way because the filename might contain spaces

	gsub(/"/, "'", file)
	gsub(/"/, "'", author)

	if(ninsertions > 0 || ndeletions > 0) {
		printf("\"%s\",%d,%d,\"%s\",\"%s\",\"%s\"\n",
			date,
			ninsertions,
			ndeletions,
			commit,
			author,
			file)
	}

	last = "nlines"
	next
}

/^$/ {
	if(last == "date")
		next
	if(last == "logmsg")
		next
	if(last == "nlines" || last == "binary") {
		commit = ""
		author = ""
		date=""
		skip = 0
		ninsertions = 0
		ndeletions = 0
		file = ""
		ismerge = 0
		last = "blank"
		next
	}

	printf("ERROR: unexpected blank line %d last %s\n", NR, last) > stderr
	exit 1
}

{
	printf("ERROR: Line %d: unhandled\n", NR) > stderr
	printf("ERROR: \"%s\"\n", $0) > stderr
	exit 1
}

#!/bin/awk
# commit 989fb6102add218589e8779c76fd61f91e113684
# Author: Tad Hunt <tadhunt@tad-hunts-macbook-pro.local>
# Date:   2009-11-30
#
#     ignore objects
#
# 4       0       .gitignore
#
# commit 54f97a5baf887f0d2a73745efaa0a647ed20512e
# Author: Tad Hunt <tadhunt@tad-hunts-macbook-pro.local>
# Date:   2009-11-30
#
#     reorg & add pitch
#
# 0       46      Makefile
# 0       62      bwcalc.c
# 0       32      dedupgain.sh
# 0       56      dedupgaintree.sh
# 0       391     sha1block.c
# 0       73      sha1speed.c
# 46      0       src/Makefile
# 62      0       src/bwcalc.c
# 32      0       src/dedupgain.sh
# 56      0       src/dedupgaintree.sh
# 391     0       src/sha1block.c
# 73      0       src/sha1speed.c
#
# commit 509aff7e53df7406749274d943ed573a0a78af02
# ...
#
BEGIN {
	stderr = "/dev/stderr"

#	printf("\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n",
#		"Date",
#		"Number of Insertions",
#		"Number of Deletions",
#		"Author",
#		"Date",
#		"Filename")

	last = "none"
}

/^commit[ 	].*/ {
	commit = $2

	last = "commit"
	next
}

/^Author:[ 	].*/ {
	author = $0
	sub(/^Author: /, "", author)

	last = "author"
	next
}

/^Date:[ 	].*/ {
	date = $2

	last = "date"
	next
}

/^Merge:[ 	].*/ {
	ismerge = 1
	last = "merge"
	next						# don't track merge history
}

/^[ 	]+.*/ {
	last = "logmsg"
	next						# don't track log message
}

/^-[ 	]+-[ 	]+.*/ {
	last = "binary"
	next						# don't track binary files
}

/^[0-9]+[ 	]+[0-9]+[ 	]+.*/ {
	ninsertions = $1
	ndeletions = $2
	file = $0
	sub(/^[0-9]+[ 	]+[0-9]+[ 	]+/, "", file)	# easier this way because the filename might contain spaces
	gsub(/"/, "'", file)

	if(ninsertions > 0 || ndeletions > 0) {
		printf("\"%s\",%d,%d,\"%s\",\"%s\",\"%s\"\n",
			date,
			ninsertions,
			ndeletions,
			commit,
			author,
			file)
	}

	last = "nlines"
	next
}

/^$/ {
	if(last == "date")
		next
	if(last == "logmsg")
		next
	if(last == "nlines" || last == "binary") {
		commit = ""
		author = ""
		date=""
		skip = 0
		ninsertions = 0
		ndeletions = 0
		file = ""
		ismerge = 0
		last = "blank"
		next
	}

	printf("ERROR: unexpected blank line %d last %s\n", NR, last) > stderr
	exit 1
}

{
	printf("ERROR: Line %d: unhandled\n", NR) > stderr
	printf("ERROR: \"%s\"\n", $0) > stderr
	exit 1
}