Translate: localize variable i in preprocessByDump() (fixed a regression bug that affects Google Translate engine)

####################################################################
# Translate.awk                                                    #
####################################################################

function provides(engineName) {
    Translator[tolower(engineName)] = TRUE
}

function engineMethod(methodName,    engine, translator) {
    if (!Translator[Option["engine"]]) {
        # case-insensitive match engine name
        engine = tolower(Option["engine"])
        if (!Translator[engine]) # fuzzy match engine name
            for (translator in Translator)
                if (Translator[translator] && # there IS such a translator
                    translator ~ "^"engine) {
                    engine = translator
                    break
                }
        if (!Translator[engine]) {
            e("[ERROR] Translator not found: " Option["engine"] "\n"    \
              "        Run '-list-engines / -S' to see a list of available engines.")
            exit 1
        }
        Option["engine"] = engine
    }
    return Option["engine"] methodName
}

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mpv" SUPOUT SUPERR) ?
        "mpv --no-config" :
        (!system("mplayer" SUPOUT SUPERR) ?
         "mplayer" :
         (!system("mpg123 --version" SUPOUT SUPERR) ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say ''" SUPOUT SUPERR) ?
        "say" :
        (!system("espeak ''" SUPOUT SUPERR) ?
         "espeak" :
         "")
}

# Detect external terminal pager (less, more, most).
function initPager() {
    Pager = !system("less -V" SUPOUT SUPERR) ?
        "less" :
        (!system("more -V" SUPOUT SUPERR) ?
         "more" :
         (!system("most" SUPOUT SUPERR) ?
          "most" :
          ""))
}

# Initialize `HttpService`.
function initHttpService(    inet) {
    _Init()

    inet = "inet"
    if (Option["ip-version"])
        inet = inet Option["ip-version"]
    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?(([^:]+):([^@]+)@)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpAuthUser = HttpProxySpec[3]
        HttpAuthPass = HttpProxySpec[4]
        HttpAuthCredentials = base64(unquote(HttpAuthUser) ":" HttpAuthPass)
        HttpService = "/" inet "/tcp/0/" HttpProxySpec[5] "/" HttpProxySpec[6]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/" inet "/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }

    # How long to wait in milliseconds for IPv6 before trying IPv4
    PROCINFO[HttpService, "READ_TIMEOUT"] = 2000
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# Pre-process string (using hexdump to URL-encode everything).
function preprocessByDump(text,    arr, i, len, temp) {
    len = dumpX(text, arr)
    temp = ""
    for (i = 1; i <= len; i++)
        temp = temp "%" arr[i]
    return temp
}

# [OBSOLETE] Is this function still relevant?
# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP GET request and get response from an online translator.
function getResponse(text, sl, tl, hl,
                     ####
                     content, header, isBody, url, group, status, location) {
    url = _RequestUrl(text, sl, tl, hl)

    header = "GET " url " HTTP/1.1\r\n"           \
        "Host: " HttpHost "\r\n"                  \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print header |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if ((status == "301" || status == "302") && location)
        content = curl(location)

    return assert(content, "[ERROR] Null response.")
}

# Send an HTTP POST request and return response from an online translator.
function postResponse(text, sl, tl, hl, type,
                      ####
                      content, contentLength, contentType, group,
                      header, isBody, reqBody, url, status, location, userAgent) {
    url = _PostRequestUrl(text, sl, tl, hl, type)
    contentType = _PostRequestContentType(text, sl, tl, hl, type)
    userAgent = _PostRequestUserAgent(text, sl, tl, hl, type)
    reqBody = _PostRequestBody(text, sl, tl, hl, type)
    if (DumpContentengths[reqBody])
        contentLength = DumpContentengths[reqBody]
    else
        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)

    header = "POST " url " HTTP/1.1\r\n"                  \
        "Host: " HttpHost "\r\n"                          \
        "Connection: close\r\n"                           \
        "Content-Length: " contentLength "\r\n"           \
        "Content-Type: " contentType "\r\n"     # must!
    if (Option["user-agent"] && !userAgent)
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (userAgent)
        header = header "User-Agent: " userAgent "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print (header "\r\n" reqBody) |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if (status == "404") {
        e("[ERROR] 404 Not Found")
        exit 1
    }
    if ((status == "301" || status == "302") && location) {
        url = "https" substr(url, 5) # switch to HTTPS; don't use location!
        content = curlPost(url, reqBody)
    }

    return content
}

# Print a string (to output file or terminal pager).
function p(string) {
    if (Option["view"]) {
        print string | Option["pager"]
        close(Option["pager"])
    } else
        print string > Option["output"]
}

# Play using a Text-to-Speech engine.
function play(text, tl,    url, status) {
    url = _TTSUrl(text, tl)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    status = system(Option["player"] " " parameterize(url) SUPOUT SUPERR)

    if (status)
        w("Voice output isn't available for " getName(tl))

    return status
}

# Download audio from a Text-to-Speech engine.
function download_audio(text, tl,    url, output) {
    url = _TTSUrl(text, tl)

    if (Option["download-audio-as"])
        output = Option["download-audio-as"]
    else
        output = text " [" Option["engine"] "] (" Option["narrator"] ").ts"

    if (url ~ /^\//)
        system("mv -- " parameterize(url) " " parameterize(output))
    else
        curl(url, output)
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist, returnIl) {
    return _Translate(text, sl, tl, hl,
                      isVerbose, toSpeech, returnPlaylist, returnIl)
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl,    temp) {
    temp = _WebTranslateUrl(uri, sl, tl, hl)
    if (temp) {
        p(temp)
        if (Option["browser"] != NONE)
            system(Option["browser"] " " parameterize(temp) SUPOUT SUPERR)
    }
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, il, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (isRTL(Option["hl"])) {
        # Check if home language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["hl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (isRTL(Option["sl"])) {
        # Check if source language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["sl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            # translate URL only from command-line parameters (inline)
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            # translate URL only from command-line parameters (inline)
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            if (!Option["no-translate"])
                p(getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"] || Option["download-audio"], playlist, il))
            else
                il[0] = Option["sl"] == "auto" ? "en" : Option["sl"]

            if (Option["play"] == 1) {
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
            } else if (Option["play"] == 2) {
                if (Option["player"])
                    play(text, il[0])
                else if (SpeechSynthesizer)
                    print text | SpeechSynthesizer
            }

            if (Option["download-audio"] == 1) {
                # Download the translation unless used with -sp or -no-trans
                if (Option["play"] != 2 && !Option["no-translate"])
                    download_audio(playlist[length(playlist) - 1]["text"], \
                                   playlist[length(playlist) - 1]["tl"])
                else
                    download_audio(text, il[0])
            }
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Translate the source text (from all source languages).
function translates(text, inline,
                    ####
                    i) {
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["sls"]) {
        # Non-interactive verbose mode: separator between sources
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        Option["sl"] = Option["sls"][i]
        translate(text, inline)
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    if (Option["input"] == STDIN || fileExists(Option["input"])) {
        i = 0
        while (getline line < Option["input"])
            if (line) {
                # Non-interactive verbose mode: separator between sources
                if (!Option["interactive"])
                    if (Option["verbose"] && i++ > 0)
                        p(prettify("source-seperator",
                                   replicate(Option["chr-source-seperator"],
                                             Option["width"])))

                if (Option["interactive"])
                    repl(line)
                else
                    translates(line)
            } else {
                # Non-interactive brief mode: preserve line breaks
                if (!Option["interactive"])
                    if (!Option["verbose"])
                        p(line)
            }
    } else
        e("[ERROR] File not found: " Option["input"])
}

####################################################################
# Translate.awk                                                    #
####################################################################

function provides(engineName) {
    Translator[tolower(engineName)] = TRUE
}

function engineMethod(methodName,    engine, translator) {
    if (!Translator[Option["engine"]]) {
        # case-insensitive match engine name
        engine = tolower(Option["engine"])
        if (!Translator[engine]) # fuzzy match engine name
            for (translator in Translator)
                if (Translator[translator] && # there IS such a translator
                    translator ~ "^"engine) {
                    engine = translator
                    break
                }
        if (!Translator[engine]) {
            e("[ERROR] Translator not found: " Option["engine"] "\n"    \
              "        Run '-list-engines / -S' to see a list of available engines.")
            exit 1
        }
        Option["engine"] = engine
    }
    return Option["engine"] methodName
}

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mpv" SUPOUT SUPERR) ?
        "mpv --no-config" :
        (!system("mplayer" SUPOUT SUPERR) ?
         "mplayer" :
         (!system("mpg123 --version" SUPOUT SUPERR) ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say ''" SUPOUT SUPERR) ?
        "say" :
        (!system("espeak ''" SUPOUT SUPERR) ?
         "espeak" :
         "")
}

# Detect external terminal pager (less, more, most).
function initPager() {
    Pager = !system("less -V" SUPOUT SUPERR) ?
        "less" :
        (!system("more -V" SUPOUT SUPERR) ?
         "more" :
         (!system("most" SUPOUT SUPERR) ?
          "most" :
          ""))
}

# Initialize `HttpService`.
function initHttpService(    inet) {
    _Init()

    inet = "inet"
    if (Option["ip-version"])
        inet = inet Option["ip-version"]
    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?(([^:]+):([^@]+)@)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpAuthUser = HttpProxySpec[3]
        HttpAuthPass = HttpProxySpec[4]
        HttpAuthCredentials = base64(unquote(HttpAuthUser) ":" HttpAuthPass)
        HttpService = "/" inet "/tcp/0/" HttpProxySpec[5] "/" HttpProxySpec[6]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/" inet "/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }

    # How long to wait in milliseconds for IPv6 before trying IPv4
    PROCINFO[HttpService, "READ_TIMEOUT"] = 2000
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# Pre-process string (using hexdump to URL-encode everything).
function preprocessByDump(text,    arr, len, temp) {
    len = dumpX(text, arr)
    temp = ""
    for (i = 1; i <= len; i++)
        temp = temp "%" arr[i]
    return temp
}

# [OBSOLETE] Is this function still relevant?
# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP GET request and get response from an online translator.
function getResponse(text, sl, tl, hl,
                     ####
                     content, header, isBody, url, group, status, location) {
    url = _RequestUrl(text, sl, tl, hl)

    header = "GET " url " HTTP/1.1\r\n"           \
        "Host: " HttpHost "\r\n"                  \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print header |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if ((status == "301" || status == "302") && location)
        content = curl(location)

    return assert(content, "[ERROR] Null response.")
}

# Send an HTTP POST request and return response from an online translator.
function postResponse(text, sl, tl, hl, type,
                      ####
                      content, contentLength, contentType, group,
                      header, isBody, reqBody, url, status, location, userAgent) {
    url = _PostRequestUrl(text, sl, tl, hl, type)
    contentType = _PostRequestContentType(text, sl, tl, hl, type)
    userAgent = _PostRequestUserAgent(text, sl, tl, hl, type)
    reqBody = _PostRequestBody(text, sl, tl, hl, type)
    if (DumpContentengths[reqBody])
        contentLength = DumpContentengths[reqBody]
    else
        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)

    header = "POST " url " HTTP/1.1\r\n"                  \
        "Host: " HttpHost "\r\n"                          \
        "Connection: close\r\n"                           \
        "Content-Length: " contentLength "\r\n"           \
        "Content-Type: " contentType "\r\n"     # must!
    if (Option["user-agent"] && !userAgent)
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (userAgent)
        header = header "User-Agent: " userAgent "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    while (1) {
        print (header "\r\n" reqBody) |& HttpService
        while ((HttpService |& getline) > 0) {
            if (isBody)
                content = content ? content "\n" $0 : $0
            else if (length($0) <= 1)
                isBody = 1
            else { # interesting fields in header
                match($0, /^HTTP[^ ]* ([^ ]*)/, group)
                if (RSTART) status = group[1]
                match($0, /^Location: (.*)/, group)
                if (RSTART) location = squeeze(group[1]) # squeeze the URL!
            }
            l(sprintf("%4s bytes > %s", length($0), $0))
        }
        close(HttpService)

        if (ERRNO == "Connection timed out") {
            w("[WARNING] " ERRNO ". Retrying IPv4 connection.")
            Option["ip-version"] = 4
            initHttpService()
            PROCINFO[HttpService, "READ_TIMEOUT"] = 0
            ERRNO = ""
        } else
            break
    }

    if (status == "404") {
        e("[ERROR] 404 Not Found")
        exit 1
    }
    if ((status == "301" || status == "302") && location) {
        url = "https" substr(url, 5) # switch to HTTPS; don't use location!
        content = curlPost(url, reqBody)
    }

    return content
}

# Print a string (to output file or terminal pager).
function p(string) {
    if (Option["view"]) {
        print string | Option["pager"]
        close(Option["pager"])
    } else
        print string > Option["output"]
}

# Play using a Text-to-Speech engine.
function play(text, tl,    url, status) {
    url = _TTSUrl(text, tl)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    status = system(Option["player"] " " parameterize(url) SUPOUT SUPERR)

    if (status)
        w("Voice output isn't available for " getName(tl))

    return status
}

# Download audio from a Text-to-Speech engine.
function download_audio(text, tl,    url, output) {
    url = _TTSUrl(text, tl)

    if (Option["download-audio-as"])
        output = Option["download-audio-as"]
    else
        output = text " [" Option["engine"] "] (" Option["narrator"] ").ts"

    if (url ~ /^\//)
        system("mv -- " parameterize(url) " " parameterize(output))
    else
        curl(url, output)
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist, returnIl) {
    return _Translate(text, sl, tl, hl,
                      isVerbose, toSpeech, returnPlaylist, returnIl)
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl,    temp) {
    temp = _WebTranslateUrl(uri, sl, tl, hl)
    if (temp) {
        p(temp)
        if (Option["browser"] != NONE)
            system(Option["browser"] " " parameterize(temp) SUPOUT SUPERR)
    }
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, il, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (isRTL(Option["hl"])) {
        # Check if home language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["hl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (isRTL(Option["sl"])) {
        # Check if source language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["sl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            # translate URL only from command-line parameters (inline)
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            # translate URL only from command-line parameters (inline)
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            if (!Option["no-translate"])
                p(getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"] || Option["download-audio"], playlist, il))
            else
                il[0] = Option["sl"] == "auto" ? "en" : Option["sl"]

            if (Option["play"] == 1) {
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
            } else if (Option["play"] == 2) {
                if (Option["player"])
                    play(text, il[0])
                else if (SpeechSynthesizer)
                    print text | SpeechSynthesizer
            }

            if (Option["download-audio"] == 1) {
                # Download the translation unless used with -sp or -no-trans
                if (Option["play"] != 2 && !Option["no-translate"])
                    download_audio(playlist[length(playlist) - 1]["text"], \
                                   playlist[length(playlist) - 1]["tl"])
                else
                    download_audio(text, il[0])
            }
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Translate the source text (from all source languages).
function translates(text, inline,
                    ####
                    i) {
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["sls"]) {
        # Non-interactive verbose mode: separator between sources
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        Option["sl"] = Option["sls"][i]
        translate(text, inline)
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    if (Option["input"] == STDIN || fileExists(Option["input"])) {
        i = 0
        while (getline line < Option["input"])
            if (line) {
                # Non-interactive verbose mode: separator between sources
                if (!Option["interactive"])
                    if (Option["verbose"] && i++ > 0)
                        p(prettify("source-seperator",
                                   replicate(Option["chr-source-seperator"],
                                             Option["width"])))

                if (Option["interactive"])
                    repl(line)
                else
                    translates(line)
            } else {
                # Non-interactive brief mode: preserve line breaks
                if (!Option["interactive"])
                    if (!Option["verbose"])
                        p(line)
            }
    } else
        e("[ERROR] File not found: " Option["input"])
}


Replace split with loop for POSIX compatibility

split: null string for third arg is a gawk extension

#!/usr/bin/awk -f

# calculate levenshtein distance
#
# @param str1  string to compare
# @param str2  the other string to compare
# @returns     levenshtein distance as integer
#
function levenshtein_distance(str1, str2,    len1, len2, chars1, chars2, i, j, matrix, cost, x, y, z) {
    len1 = length(str1)
    len2 = length(str2)
    if (len1 == 0) return len2
    if (len2 == 0) return len1
    if (str1 == str2) return 0
    for(i = 1; i <= len1; i++) {
        chars1[i] = substr(str1, i, 1)
    }
	for(j = 1; j <= len2; j++) {
        chars2[j] = substr(str2, j, 1)
    }
    for (i = 0; i <= len1; i++) {
        matrix[i, 0] = i
    }
    for (j = 0; j <= len2; j++) {
        matrix[0, j] = j
    }
    for (i = 1; i <= len1; i++) {
        for (j = 1; j <= len2; j++) {
            cost = (chars1[i] == chars2[j]) ? 0 : 1
            x = matrix[i-1, j] + 1
            y = matrix[i, j-1] + 1
            z = matrix[i-1, j-1] + cost
            if (x <= y && x <= z) matrix[i, j] = x
            if (y <= x && y <= z) matrix[i, j] = y
            if (z <= x && z <= y) matrix[i, j] = z
        }
    }
    return matrix[len1, len2]
}

#!/usr/bin/awk -f

# calculate levenshtein distance
#
# @param str1  string to compare
# @param str2  the other string to compare
# @returns     levenshtein distance as integer
#
function levenshtein_distance(str1, str2,    len1, len2, chars1, chars2, i, j, matrix, cost, x, y, z) {
    len1 = length(str1)
    len2 = length(str2)
    if (len1 == 0) return len2
    if (len2 == 0) return len1
    if (str1 == str2) return 0
    split(str1, chars1, "")
    split(str2, chars2, "")
    for (i = 0; i <= len1; i++) {
        matrix[i, 0] = i
    }
    for (j = 0; j <= len2; j++) {
        matrix[0, j] = j
    }
    for (i = 1; i <= len1; i++) {
        for (j = 1; j <= len2; j++) {
            cost = (chars1[i] == chars2[j]) ? 0 : 1
            x = matrix[i-1, j] + 1
            y = matrix[i, j-1] + 1
            z = matrix[i-1, j-1] + cost
            if (x <= y && x <= z) matrix[i, j] = x
            if (y <= x && y <= z) matrix[i, j] = y
            if (z <= x && z <= y) matrix[i, j] = z
        }
    }
    return matrix[len1, len2]
}


Better variable names.

Better variable names.

#!/usr/bin/awk -f

# Add domain name to global variable domains.
function add(domain) {
	domain_count = domain_count + 1
	domains[domain_count] = domain
}

# Extract domain name from non-regex line
function extract(line, original_line) {
	sub(/.*:\/\//, "", line) # remove everything till ://, such as http:// https://
	sub(/\/.*/, "", line)    # remove everything from /
	sub(/:.*/, "", line)     # remove everything from :
	sub(/\*/, "", line)      # remove *
	sub(/^\./, "", line)     # remove leading dot

	if (match(line, /([a-zA-Z0-9\-]+\.)+[a-zA-Z0-9\-]+/)) {
		add(line)
	} else {
		print "[WARN] <NR>" NR "</NR><original>" original_line "</original><extracted>" line "</extracted>" | "cat >&2"
	}
}

# Extract domain name from regex line
function extract_regex(line, original_line) {

	# Expand the lines like (aa|bb|cc) into multiple records
	pos = match(line, /\([a-zA-Z0-9\.\|]+\)/)
	if (pos != 0) {
		in_bracket = substr(line, RSTART + 1, RLENGTH - 2)
		n = split(in_bracket, arr, "|")
		for (i = 1; i <= n; i++) {
			expanded_line = line
			sub(/\([a-zA-Z0-9\.\|]+\)/, arr[i], expanded_line)
			extract_regex(expanded_line, original_line)
		}
	} else {
		sub(/.*:\\\/\\\//, "", line)    # remove everything till :\/\/ in regex, :// in plain text, such as http:// https://
		gsub(/\([^\)]+\)\*?/, "", line) # remove (...)*? such as ([^\/]+\.)
		gsub(/\[.*\]\+/, "", line)      # remove [...]
		sub(/\\\/.*/, "", line)         # remove everything from \/ in regex, / in plaint text
		gsub(/\\\./, ".", line)         # replace \. to .
		extract(line, original_line)
	}
}

BEGIN {
	"date \"+%Y-%m-%d %H:%M:%S\"" | getline now
	print "# gfw list ipset rules for dnsmasq"
	print "# updated on " now
	print "#"

	if (host == "") {
		host = "127.0.0.1"
	}
	if (port == "") {
		port = 5353
	}

	domain_count = 0
}
{
	original_line = $0

	if (/^#/) {
		# #
		# comments in user_rule.txt
	} else if (/^\!/) {
		# !
		# https://adblockplus.org/en/filters#comments
		# comments
	} else if (/^\[/) {
		# [
		# comments
	} else if (/^@@/) {
		# @@
		# https://adblockplus.org/en/filters#whitelist
	} else if (/^$/) {
		# Empty line
	} else if (/^\|\|/) {
		# ||
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|\|/, "") # remove leading ||
		extract($0, original_line)
	} else if (/^\|/) {
		# |
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|/, "") # removing leading |
		extract($0, original_line)
	} else if (/^\/.*\/$/) {
		# /.../
		# https://adblockplus.org/en/filters#regexps
		sub(/^\//, "") # remove leading /
		sub(/\/$/, "") # remove tailing /
		extract_regex($0, original_line)
	} else if (/^\./) {
		# start with dot
		extract($0, original_line)
	} else {
		extract($0, original_line)
	}
}
END {
	for (i = 1; i <= domain_count; i++) {
		printf "server=/.%s/%s#%d\n", domains[i], host, port | "sort | uniq"
	}
}

#!/usr/bin/awk -f

# Add domain name to global variable domains.
function add(domain) {
	domain_count = domain_count + 1
	domains[domain_count] = domain
}

# Extract domain name from non-regex line
function extract(line, original_line) {
	sub(/.*:\/\//, "", line) # remove everything till ://, such as http:// https://
	sub(/\/.*/, "", line)    # remove everything from /
	sub(/:.*/, "", line)     # remove everything from :
	sub(/\*/, "", line)      # remove *
	sub(/^\./, "", line)     # remove leading dot

	if (match(line, /([a-zA-Z0-9\-]+\.)+[a-zA-Z0-9\-]+/)) {
		add(line)
	} else {
		print "[WARN] <NR>" NR "</NR><original>" original_line "</original><extracted>" line "</extracted>" | "cat >&2"
	}
}

# Extract domain name from regex line
function extract_regex(line, original_line) {

	# Expand the lines like (aa|bb|cc) into multiple records
	pos = match(line, /\([a-zA-Z0-9\.\|]+\)/)
	if (pos != 0) {
		bracket = substr(line, RSTART + 1, RLENGTH - 2)
		n = split(bracket, a, "|")
		for (i = 1; i <= n; i++) {
			expanded_line = line
			sub(/\([a-zA-Z0-9\.\|]+\)/, a[i], expanded_line)
			extract_regex(expanded_line, original_line)
		}
	} else {
		sub(/.*:\\\/\\\//, "", line)    # remove everything till :\/\/ in regex, :// in plain text, such as http:// https://
		gsub(/\([^\)]+\)\*?/, "", line) # remove (...)*? such as ([^\/]+\.)
		gsub(/\[.*\]\+/, "", line)      # remove [...]
		sub(/\\\/.*/, "", line)         # remove everything from \/ in regex, / in plaint text
		gsub(/\\\./, ".", line)         # replace \. to .
		extract(line, original_line)
	}
}

BEGIN {
	"date \"+%Y-%m-%d %H:%M:%S\"" | getline now
	print "# gfw list ipset rules for dnsmasq"
	print "# updated on " now
	print "#"

	if (host == "") {
		host = "127.0.0.1"
	}
	if (port == "") {
		port = 5353
	}

	domain_count = 0
}
{
	original_line = $0

	if (/^#/) {
		# #
		# comments in user_rule.txt
	} else if (/^\!/) {
		# !
		# https://adblockplus.org/en/filters#comments
		# comments
	} else if (/^\[/) {
		# [
		# comments
	} else if (/^@@/) {
		# @@
		# https://adblockplus.org/en/filters#whitelist
	} else if (/^$/) {
		# Empty line
	} else if (/^\|\|/) {
		# ||
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|\|/, "") # remove leading ||
		extract($0, original_line)
	} else if (/^\|/) {
		# |
		# https://adblockplus.org/en/filters#anchors
		sub(/^\|/, "") # removing leading |
		extract($0, original_line)
	} else if (/^\/.*\/$/) {
		# /.../
		# https://adblockplus.org/en/filters#regexps
		sub(/^\//, "") # remove leading /
		sub(/\/$/, "") # remove tailing /
		extract_regex($0, original_line)
	} else if (/^\./) {
		# start with dot
		extract($0, original_line)
	} else {
		extract($0, original_line)
	}
}
END {
	for (i = 1; i <= domain_count; i++) {
		printf "server=/.%s/%s#%d\n", domains[i], host, port | "sort | uniq"
	}
}


Regress because of errors.

Regress because of errors.

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb

BEGIN {
  FS="\t"
  pn = 0
    #str = "date +%m-%d-%Y";
    #str | getline date;
    #close str;
}
#$3>date

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2005 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2015
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "Mz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "NA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  #printf "%s", date
  #printf "\n"
}

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb

BEGIN {
  FS="\t"
  pn = 0
    str = "date +%m-%d-%Y";
    str | getline date;
    #close str;
}
$3>date

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2005 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2015
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "Mz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,2) >= "NA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  printf "%s", date
  printf "\n"
}


Translators/BingTranslator: fix dictionary lookup

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 25 Jul 2019
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# [OBSOLETE?] Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# [OBSOLETE?] Old dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    #else if (type == "transliterate")
    #    return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    #else if (type == "transliterate")
    #    return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast, dicContent, dicTokens, dicAst,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation, phonetics, oPhonetics,
                       wordClasses, words, wordBackTranslations,
                       wShowOriginal, wShowOriginalPhonetics,
                       wShowTranslation, wShowTranslationPhonetics,
                       wShowLanguages, wShowDictionary,
                       i, j, k, group, temp, saveSortedIn) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    #bingSetIG() # set IG (no longer needed)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code  # FIXME
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            # FIXME!
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = postResponse(text, il, _tl, _hl, "lookup")
            if (dicContent != "") {
                tokenize(dicTokens, dicContent)
                parseJson(dicAst, dicTokens)

                l(dicContent, "content", 1, 1)
                l(dicTokens, "tokens", 1, 0, 1)
                l(dicAst, "ast")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "compareByIndexFields"
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "posTag$", group))
                        wordClasses[group[1]] = tolower(literal(dicAst[i]))
                }
                for (i in dicAst) {
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "displayTarget$", group))
                        words[wordClasses[group[1]]][group[1]] = literal(dicAst[i])
                    if (match(i, "^0" SUBSEP "0" SUBSEP "translations" SUBSEP "([[:digit:]]+)" SUBSEP \
                              "backTranslations" SUBSEP "([[:digit:]]+)" SUBSEP "displayText$", group))
                        wordBackTranslations[wordClasses[group[1]]][group[1]][group[2]] = literal(dicAst[i])
                }
                PROCINFO["sorted_in"] = saveSortedIn

                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")
                for (i = 0; i < length(words); i++) {
                    r = (i > 0 ? r RS : r) RS prettify("dictionary-word-class", s(wordClasses[i], hl))

                    for (j in words[wordClasses[i]]) {
                        r = r RS prettify("dictionary-word", ins(1, words[wordClasses[i]][j], tl))

                        if (isRTL(il))
                            explanation = join(wordBackTranslations[wordClasses[i]][j], ", ")
                        else {
                            explanation = prettify("dictionary-explanation-item",
                                                   wordBackTranslations[wordClasses[i]][j][0])
                            for (k = 1; k < length(wordBackTranslations[wordClasses[i]][j]); k++)
                                explanation = explanation prettify("dictionary-explanation", ", ") \
                                    prettify("dictionary-explanation-item",
                                             wordBackTranslations[wordClasses[i]][j][k])
                        }

                        if (isRTL(il))
                            r = r RS prettify("dictionary-explanation-item", ins(2, explanation, il))
                        else
                            r = r RS ins(2, explanation)
                    }
                }
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 24 Jul 2019
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# [OBSOLETE] Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

# FIXME!
function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?"                                          \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl,    _sl, _tl) {
    # Hot-patches for language codes
    _sl = sl; _tl = tl
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh")    _sl = "zh-CHS" # still old format
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh")    _tl = "zh-CHS"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    return "https://www.translatetheweb.com/?" "from=" _sl "&to=" _tl "&a=" uri
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Main Bing Translator API (via HTTP POST).
function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "lookup")
        return HttpPathPrefix "/tlookupv3"
    else if (type == "transliterate")
        return HttpPathPrefix "/ttransliteratev3"
    else # type == "translate"
        return HttpPathPrefix "/ttranslatev3"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "lookup")
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl  # FIXME!
    else if (type == "transliterate")
        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"  # FIXME!
    else # type == "translate"
        return "&text=" quote(text) "&fromLang=" sl "&to=" tl
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    #bingSetIG() # set IG (no longer needed)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "auto-detect"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code  # FIXME
    if (_sl == "zh-CN") _sl = "zh-Hans"
    if (_sl == "zh-TW") _sl = "zh-Hant"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-Hans"
    if (_tl == "zh-TW") _tl = "zh-Hant"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (content == "") {
        # Empty content. Assume "301 Moved Permanently" and use cn.bing.com
        HttpHost = "cn.bing.com"
        # Just dirty hack
        if (Option["proxy"]) {
            HttpPathPrefix = HttpProtocol HttpHost
        } else {
            HttpService = "/" "inet" "/tcp/0/" HttpHost "/" HttpPort  # FIXME: inet version
        }

        # Try again
        content = postResponse(text, _sl, _tl, _hl, "translate")
    }
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = _sl == "auto-detect" ?
        unparameterize(ast[0 SUBSEP 0 SUBSEP "detectedLanguage" SUBSEP "language"]) : _sl
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        #content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        #phonetics = unparameterize(content)
        phonetics = unparameterize(ast[0 SUBSEP 0 SUBSEP "translations" SUBSEP 0 SUBSEP "transliteration" \
                                       SUBSEP "text"])
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            #content = postResponse(text, group[1], group[1], _hl, "transliterate")
            #oPhonetics = unparameterize(content)
            # TODO
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, _sl, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = _tl
    }

    return r
}


Translators/BingTranslator: support original phonetics

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 17 Mar 2018
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/translator/api/language/Speak?"                   \
        "locale=" tl "&text=" preprocess(text)                          \
        "&gender=" gender "&media=audio/mp3" " HTTP/1.1\r\n"              \
        "Host: " HttpHost "\r\n"                                          \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n" # must!

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl) {
    return "http://www.microsofttranslator.com/bv.aspx?"        \
        "from=" sl "&to=" tl "&a=" uri
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "translate")
        return HttpPathPrefix "/ttranslate?&IG=" IG "&IID=translator.5032.2"
    else if (type == "translationlookup")
        return HttpPathPrefix "/ttranslationlookup?&IG=" IG "&IID=translator.5032.2"
    else if (type == "transliterate")
        return HttpPathPrefix "/ttransliterate?&IG=" IG "&IID=translator.5032.1"
    else # type == "detect"
        return HttpPathPrefix "/tdetect?&IG=" IG "&IID=translator.5032.2"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "translate")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "translationlookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "transliterate")
        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "detect"
        return "&text=" quote(text)
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetIG() # set IG

    # Language identification
    il = postResponse(text, _sl, _tl, _hl, "detect")
    if (!il) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    returnIl[0] = il
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = il
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    } else if (ast[0 SUBSEP "statusCode"] != "200") {
        e("[ERROR] statusCode: " ast[0 SUBSEP "statusCode"])
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "translationResponse"])

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        phonetics = unparameterize(content)
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(il, group, "-")
            content = postResponse(text, group[1], group[1], _hl, "transliterate")
            oPhonetics = unparameterize(content)
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), il))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, il, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 17 Mar 2018
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/translator/api/language/Speak?"                   \
        "locale=" tl "&text=" preprocess(text)                          \
        "&gender=" gender "&media=audio/mp3" " HTTP/1.1\r\n"              \
        "Host: " HttpHost "\r\n"                                          \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n" # must!

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl) {
    return "http://www.microsofttranslator.com/bv.aspx?"        \
        "from=" sl "&to=" tl "&a=" uri
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "translate")
        return HttpPathPrefix "/ttranslate?&IG=" IG "&IID=translator.5032.2"
    else if (type == "translationlookup")
        return HttpPathPrefix "/ttranslationlookup?&IG=" IG "&IID=translator.5032.2"
    else if (type == "transliterate")
        return HttpPathPrefix "/ttransliterate?&IG=" IG "&IID=translator.5032.1"
    else # type == "detect"
        return HttpPathPrefix "/tdetect?&IG=" IG "&IID=translator.5032.2"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "translate")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "translationlookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "transliterate")
        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "detect"
        return "&text=" quote(text)
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetIG() # set IG

    # Language identification
    il = postResponse(text, _sl, _tl, _hl, "detect")
    if (!il) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    returnIl[0] = il
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = il
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    } else if (ast[0 SUBSEP "statusCode"] != "200") {
        e("[ERROR] statusCode: " ast[0 SUBSEP "statusCode"])
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "translationResponse"])

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        phonetics = unparameterize(content)
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, il, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}


port.wm: make it actually read it's input

@include "ctx.awk"
@include "class.awk"
@include "class.x11.awk"

BEGIN {
    teslawm_fifo = ENVIRON["TESLAWM_fifo"]
}

get("xprop.wm_class_name") == "Sonata" {
    set("xprop.wm_name", HOME "/mnt/ssh/rfigura@wirsing/home/rfigura/pub/audio/" get("xprop.wm_name"))
}

get("xwin") && teslawm_fifo {
    # todo: improve teslawm to accept a window id in commands:
    label("close"); menu("echo close " get("xwin") " > " Q(teslawm_fifo))
    label("movemouse"); menu("echo movemouse " get("xwin") " > " Q(teslawm_fifo))
    label("resizemouse"); menu("echo resizemouse " get("xwin") " > " Q(teslawm_fifo))
}
get("xwin") {
    menu("xkill -id " get("xwin"))
    menu(pager("xwininfo -id " get("xwin")))
    menu(pager("xprop -id " get("xwin")))
    label("wm_name..."); menu(nest(get("xprop.wm_name"), "open"))
#    label("teslawm rule (sel)"); menu("@xsel " get("xprop.wm_class_name") ":" get("xprop.wm_class") ":" get("xprop.wm_name"))
}

{
    set("pid", get("xprop.net_wm_pid"))
}
@include "port.pid.awk"
@include "port.default.awk"

@include "ctx.awk"
@include "class.awk"
@include "class.x11.awk"

BEGIN {
    teslawm_fifo = ENVIRON["TESLAWM_fifo"]
}

get("ctx.xprop.wm_class_name") == "Sonata" {
    set("ctx.xprop.wm_name", HOME "/mnt/ssh/rfigura@wirsing/home/rfigura/pub/audio/" get("ctx.xprop.wm_name"))
}

get("ctx.xwin") && teslawm_fifo {
    # todo: improve teslawm to accept a window id in commands:
    label("close"); menu("echo close " get("ctx.xwin") " > " Q(teslawm_fifo))
    label("movemouse"); menu("echo movemouse " get("ctx.xwin") " > " Q(teslawm_fifo))
    label("resizemouse"); menu("echo resizemouse " get("ctx.xwin") " > " Q(teslawm_fifo))
}
get("ctx.xwin") {
    menu("xkill -id " get("ctx.xwin"))
    menu(pager("xwininfo -id " get("ctx.xwin")))
    menu(pager("xprop -id " get("ctx.xwin")))
    label("wm_name..."); menu(nest(get("ctx.xprop.wm_name"), "open"))
#    label("teslawm rule (sel)"); menu("@xsel " get("ctx.xprop.wm_class_name") ":" get("ctx.xprop.wm_class") ":" get("ctx.xprop.wm_name"))
}

{
    set("pid", get("ctx.xprop.net_wm_pid"))
}
@include "port.pid.awk"
@include "port.default.awk"


ahrf.awk: Clean the snippet which was written yesterday

#!/usr/bin/awk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0,":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9_]+/ {
	printf("
")
	for (p=1; p<NF; p++) {
		if ($p ~ / +$/) {
			gsub(/ +$/,"",$p)
			printf("%s
\n", $p)
		} else {
			printf("%s\n", $p)
		}
	}
	printf("%s
\n\n", $p)
	next
}

# Code
/^====+/ {
	if (NF > 2 && $NF ~ /====+/) {
		gsub(/^[\t ]*====+[\t ]*\n|\n[\t ]*====+[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		#gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\* +/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l ~ /\* +/) {
			match($l,"\\* +")
			str = substr($l,RSTART+RLENGTH)
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u,"\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}

#!/usr/bin/awk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0,":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9_]+/ {
	#printf("
%s
\n\n", $0)
	printf("
")
	for (p=1; p<=NF; p++) {
		if ($p ~ / +$/) {
			printf("%s
\n", $p)
		} else {
			printf("%s\n", $p)
		}
	}
	printf("
\n\n")
	next
}

# Code
/^====+/ {
	if (NF > 2 && $NF ~ /====+/) {
		gsub(/^[\t ]*====+[\t ]*\n|\n[\t ]*====+[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		#gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\* +/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l ~ /\* +/) {
			match($l,"\\* +")
			str = substr($l,RSTART+RLENGTH)
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u,"\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}


Add processing for movies

Add processing for movies