Translate: postResponse()

Translate: postResponse()

####################################################################
# Translate.awk                                                    #
####################################################################

function provides(engineName) {
    Translator[tolower(engineName)] = TRUE
}

function engineMethod(methodName,    engine, translator) {
    if (!Translator[Option["engine"]]) {
        # case-insensitive match engine name
        engine = tolower(Option["engine"])
        if (!Translator[engine]) # fuzzy match engine name
            for (translator in Translator)
                if (Translator[translator] && # there IS such a translator
                    translator ~ "^"engine) {
                    engine = translator
                    break
                }
        if (!Translator[engine]) {
            e("[ERROR] Translator not found: " Option["engine"] "\n"    \
              "        Run '-list-engines / -S' to see a list of available engines.")
            exit 1
        }
        Option["engine"] = engine
    }
    return Option["engine"] methodName
}

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mpv" SUPOUT SUPERR) ?
        "mpv --loop-file=no" :
        (!system("mplayer" SUPOUT SUPERR) ?
         "mplayer" :
         (!system("mpg123 --version" SUPOUT SUPERR) ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say ''" SUPOUT SUPERR) ?
        "say" :
        (!system("espeak ''" SUPOUT SUPERR) ?
         "espeak" :
         "")
}

# Detect external terminal pager (less, more, most).
function initPager() {
    Pager = !system("less -V" SUPOUT SUPERR) ?
        "less" :
        (!system("more -V" SUPOUT SUPERR) ?
         "more" :
         (!system("most" SUPOUT SUPERR) ?
          "most" :
          ""))
}

# Initialize `HttpService`.
function initHttpService() {
    _Init()

    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?(([^:]+):([^@]+)@)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpAuthUser = HttpProxySpec[3]
        HttpAuthPass = HttpProxySpec[4]
        HttpAuthCredentials = base64(unquote(HttpAuthUser) ":" HttpAuthPass)
        HttpService = "/inet/tcp/0/" HttpProxySpec[5] "/" HttpProxySpec[6]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/inet/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# [OBSOLETE] Is this function still relevant?
# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP GET request and get response from an online translator.
function getResponse(text, sl, tl, hl,
                     ####
                     content, header, isBody, url, group, status, location) {
    url = _RequestUrl(text, sl, tl, hl)

    header = "GET " url " HTTP/1.1\r\n"           \
        "Host: " HttpHost "\r\n"                  \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (status == "301" && location)
        content = curl(location)

    return assert(content, "[ERROR] Null response.")
}

# Send an HTTP POST request and return response from an online translator.
function postResponse(text, sl, tl, hl, type,
                      ####
                      content, contentLength, contentType, group,
                      header, isBody, reqBody, url) {
    url = _PostRequestUrl(text, sl, tl, hl, type)
    contentType = _PostRequestContentType(text, sl, tl, hl, type)
    reqBody = _PostRequestBody(text, sl, tl, hl, type)
    if (DumpContentengths[reqBody])
        contentLength = DumpContentengths[reqBody]
    else
        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)

    header = "POST " url " HTTP/1.1\r\n"                  \
        "Host: " HttpHost "\r\n"                          \
        "Connection: close\r\n"                           \
        "Content-Length: " contentLength "\r\n"           \
        "Content-Type: " contentType "\r\n"     # must!
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    print (header "\r\n" reqBody) |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    return assert(content, "[ERROR] Null response.")
}

# Print a string (to output file or terminal pager).
function p(string) {
    if (Option["view"])
        print string | Option["pager"]
    else
        print string > Option["output"]
}

# Play using a Text-to-Speech engine.
function play(text, tl,    url) {
    url = _TTSUrl(text, tl)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    system(Option["player"] " " parameterize(url) SUPOUT SUPERR)
}

# Download audio from a Text-to-Speech engine.
function download_audio(text, tl,    url, output) {
    url = _TTSUrl(text, tl)

    if (Option["download-audio-as"])
        output = Option["download-audio-as"]
    else
        output = text " [" Option["engine"] "] (" Option["narrator"] ").ts"

    if (url ~ /^\//)
        system("mv -- " parameterize(url) " " parameterize(output))
    else
        curl(url, output)
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist, returnIl) {
    return _Translate(text, sl, tl, hl,
                      isVerbose, toSpeech, returnPlaylist, returnIl)
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl) {
    system(Option["browser"] " "                                \
           parameterize(_WebTranslateUrl(uri, sl, tl, hl)) "&")
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, il, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (isRTL(Option["hl"])) {
        # Check if home language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["hl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (isRTL(Option["sl"])) {
        # Check if source language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["sl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            # translate URL only from command-line parameters (inline)
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            # translate URL only from command-line parameters (inline)
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            if (!Option["no-translate"])
                p(getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"] || Option["download-audio"], playlist, il))
            else
                il[0] = Option["sl"] == "auto" ? "en" : Option["sl"]

            if (Option["play"] == 1) {
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
            } else if (Option["play"] == 2) {
                if (Option["player"])
                    play(text, il[0])
                else if (SpeechSynthesizer)
                    print text | SpeechSynthesizer
            }

            if (Option["download-audio"] == 1) {
                # Download the translation unless used with -sp or -no-trans
                if (Option["play"] != 2 && !Option["no-translate"])
                    download_audio(playlist[length(playlist) - 1]["text"], \
                                   playlist[length(playlist) - 1]["tl"])
                else
                    download_audio(text, il[0])
            }
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    if (Option["input"] == STDIN || fileExists(Option["input"])) {
        i = 0
        while (getline line < Option["input"])
            if (line) {
                # Non-interactive verbose mode: separator between sources
                if (!Option["interactive"])
                    if (Option["verbose"] && i++ > 0)
                        p(prettify("source-seperator",
                                   replicate(Option["chr-source-seperator"],
                                             Option["width"])))

                if (Option["interactive"])
                    repl(line)
                else
                    translate(line)
            } else {
                # Non-interactive brief mode: preserve line breaks
                if (!Option["interactive"])
                    if (!Option["verbose"])
                        p(line)
            }
    } else
        e("[ERROR] File not found: " Option["input"])
}

####################################################################
# Translate.awk                                                    #
####################################################################

function provides(engineName) {
    Translator[tolower(engineName)] = TRUE
}

function engineMethod(methodName,    engine, translator) {
    if (!Translator[Option["engine"]]) {
        # case-insensitive match engine name
        engine = tolower(Option["engine"])
        if (!Translator[engine]) # fuzzy match engine name
            for (translator in Translator)
                if (Translator[translator] && # there IS such a translator
                    translator ~ "^"engine) {
                    engine = translator
                    break
                }
        if (!Translator[engine]) {
            e("[ERROR] Translator not found: " Option["engine"] "\n"    \
              "        Run '-list-engines / -S' to see a list of available engines.")
            exit 1
        }
        Option["engine"] = engine
    }
    return Option["engine"] methodName
}

# Detect external audio player (mplayer, mpv, mpg123).
function initAudioPlayer() {
    AudioPlayer = !system("mpv" SUPOUT SUPERR) ?
        "mpv --loop-file=no" :
        (!system("mplayer" SUPOUT SUPERR) ?
         "mplayer" :
         (!system("mpg123 --version" SUPOUT SUPERR) ?
          "mpg123" :
          ""))
}

# Detect external speech synthesizer (say, espeak).
function initSpeechSynthesizer() {
    SpeechSynthesizer = !system("say ''" SUPOUT SUPERR) ?
        "say" :
        (!system("espeak ''" SUPOUT SUPERR) ?
         "espeak" :
         "")
}

# Detect external terminal pager (less, more, most).
function initPager() {
    Pager = !system("less -V" SUPOUT SUPERR) ?
        "less" :
        (!system("more -V" SUPOUT SUPERR) ?
         "more" :
         (!system("most" SUPOUT SUPERR) ?
          "most" :
          ""))
}

# Initialize `HttpService`.
function initHttpService() {
    _Init()

    if (Option["proxy"]) {
        match(Option["proxy"], /^(http:\/*)?(([^:]+):([^@]+)@)?([^\/]*):([^\/:]*)/, HttpProxySpec)
        HttpAuthUser = HttpProxySpec[3]
        HttpAuthPass = HttpProxySpec[4]
        HttpAuthCredentials = base64(unquote(HttpAuthUser) ":" HttpAuthPass)
        HttpService = "/inet/tcp/0/" HttpProxySpec[5] "/" HttpProxySpec[6]
        HttpPathPrefix = HttpProtocol HttpHost
    } else {
        HttpService = "/inet/tcp/0/" HttpHost "/" HttpPort
        HttpPathPrefix = ""
    }
}

# Pre-process string (URL-encode before send).
function preprocess(text) {
    return quote(text)
}

# [OBSOLETE] Is this function still relevant?
# Post-process string (remove any redundant whitespace).
function postprocess(text) {
    text = gensub(/ ([.,;:?!"])/, "\\1", "g", text)
    text = gensub(/(["]) /, "\\1", "g", text)
    return text
}

# Send an HTTP GET request and get response from an online translator.
function getResponse(text, sl, tl, hl,
                     ####
                     content, header, isBody, url, group, status, location) {
    url = _RequestUrl(text, sl, tl, hl)

    header = "GET " url " HTTP/1.1\r\n"           \
        "Host: " HttpHost "\r\n"                  \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"
    if (HttpAuthUser && HttpAuthPass)
        # TODO: digest auth
        header = header "Proxy-Authorization: Basic " HttpAuthCredentials "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (status == "301" && location)
        content = curl(location)

    return assert(content, "[ERROR] Null response.")
}

# Print a string (to output file or terminal pager).
function p(string) {
    if (Option["view"])
        print string | Option["pager"]
    else
        print string > Option["output"]
}

# Play using a Text-to-Speech engine.
function play(text, tl,    url) {
    url = _TTSUrl(text, tl)

    # Don't use getline from pipe here - the same pipe will be run only once for each AWK script!
    system(Option["player"] " " parameterize(url) SUPOUT SUPERR)
}

# Download audio from a Text-to-Speech engine.
function download_audio(text, tl,    url, output) {
    url = _TTSUrl(text, tl)

    if (Option["download-audio-as"])
        output = Option["download-audio-as"]
    else
        output = text " [" Option["engine"] "] (" Option["narrator"] ").ts"

    if (url ~ /^\//)
        system("mv -- " parameterize(url) " " parameterize(output))
    else
        curl(url, output)
}

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist, returnIl) {
    return _Translate(text, sl, tl, hl,
                      isVerbose, toSpeech, returnPlaylist, returnIl)
}

# Translate a file.
function fileTranslation(uri,    group, temp1, temp2) {
    temp1 = Option["input"]
    temp2 = Option["verbose"]

    match(uri, /^file:\/\/(.*)/, group)
    Option["input"] = group[1]
    Option["verbose"] = 0

    translateMain()

    Option["input"] = temp1
    Option["verbose"] = temp2
}

# Start a browser session and translate a web page.
function webTranslation(uri, sl, tl, hl) {
    system(Option["browser"] " "                                \
           parameterize(_WebTranslateUrl(uri, sl, tl, hl)) "&")
}

# Translate the source text (into all target languages).
function translate(text, inline,
                   ####
                   i, j, playlist, il, saveSortedIn) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (isRTL(Option["hl"])) {
        # Check if home language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["hl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (isRTL(Option["sl"])) {
        # Check if source language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(Option["sl"]) " is a right-to-left language, but FriBidi is not found.")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between targets
        if (!Option["interactive"])
            if (Option["verbose"] && i > 1)
                p(prettify("target-seperator", replicate(Option["chr-target-seperator"], Option["width"])))

        if (inline &&
            startsWithAny(text, UriSchemes) == "file://") {
            # translate URL only from command-line parameters (inline)
            fileTranslation(text)
        } else if (inline &&
                   startsWithAny(text, UriSchemes) == "http://" ||
                   startsWithAny(text, UriSchemes) == "https://") {
            # translate URL only from command-line parameters (inline)
            webTranslation(text, Option["sl"], Option["tl"][i], Option["hl"])
        } else {
            if (!Option["no-translate"])
                p(getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"] || Option["download-audio"], playlist, il))
            else
                il[0] = Option["sl"] == "auto" ? "en" : Option["sl"]

            if (Option["play"] == 1) {
                if (Option["player"])
                    for (j in playlist)
                        play(playlist[j]["text"], playlist[j]["tl"])
                else if (SpeechSynthesizer)
                    for (j in playlist)
                        print playlist[j]["text"] | SpeechSynthesizer
            } else if (Option["play"] == 2) {
                if (Option["player"])
                    play(text, il[0])
                else if (SpeechSynthesizer)
                    print text | SpeechSynthesizer
            }

            if (Option["download-audio"] == 1) {
                # Download the translation unless used with -sp or -no-trans
                if (Option["play"] != 2 && !Option["no-translate"])
                    download_audio(playlist[length(playlist) - 1]["text"], \
                                   playlist[length(playlist) - 1]["tl"])
                else
                    download_audio(text, il[0])
            }
        }
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Read from input and translate each line.
function translateMain(    i, line) {
    if (Option["interactive"])
        prompt()

    if (Option["input"] == STDIN || fileExists(Option["input"])) {
        i = 0
        while (getline line < Option["input"])
            if (line) {
                # Non-interactive verbose mode: separator between sources
                if (!Option["interactive"])
                    if (Option["verbose"] && i++ > 0)
                        p(prettify("source-seperator",
                                   replicate(Option["chr-source-seperator"],
                                             Option["width"])))

                if (Option["interactive"])
                    repl(line)
                else
                    translate(line)
            } else {
                # Non-interactive brief mode: preserve line breaks
                if (!Option["interactive"])
                    if (!Option["verbose"])
                        p(line)
            }
    } else
        e("[ERROR] File not found: " Option["input"])
}


don't hardcode array indexes and add a line that's in some files

BEGIN {
	IN_BLOCK = 0;

	PRE_LINE = "^[ \t]*";
	POST_LINE = "[ \t]*$";

	# tmp variables
	X = "((\\*|#( *\\*)?) *)?";
	PRE_BLOCK = "(/\\* *|# *(/\\* *)?)?";
	POST_BLOCK = " *(\\*/)?";

	BLOCK_END = 0;

	BLOCK[BLOCK_END++] = PRE_BLOCK "\\*\\*\\*\\*\\* BEGIN LICENSE BLOCK \\*\\*\\*\\*\\*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "BBN Address and AS Number PKI Database/repository software";
	BLOCK[BLOCK_END++] = X "Version [^ ]*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "US government users are permitted unrestricted rights as";
	BLOCK[BLOCK_END++] = X "defined in the FAR.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "This software is distributed on an \"AS IS\" basis, WITHOUT";
	BLOCK[BLOCK_END++] = X "WARRANTY OF ANY KIND, either express or implied.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "Copyright .*";
	BLOCK[BLOCK_END++] = X "All Rights Reserved.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "Contributor(\\(s\\)|s)?: .*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "\\*\\*\\*\\*\\* END LICENSE BLOCK \\*\\*\\*\\*\\*" POST_BLOCK;
}

{
	if (!IN_BLOCK) {
		if ($0 ~ PRE_LINE BLOCK[0] POST_LINE) {
			IN_BLOCK = 1;
			BLOCK_LINENO = 0;
			SAVED_BLOCK = $0;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
		} else {
			print;
		}
	} else {
		if ($0 ~ PRE_LINE BLOCK[BLOCK_LINENO + 1] POST_LINE) {
			SAVED_BLOCK = SAVED_BLOCK RS $0;
			BLOCK_LINENO += 1;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
			if (BLOCK_LINENO == BLOCK_END) {
				IN_BLOCK = 0;
			}
		} else {
			print SAVED_BLOCK;
			print;
			IN_BLOCK = 0;
		}
	}
}

END {
	if (IN_BLOCK) {
		print SAVED_BLOCK;
	}
}

BEGIN {
	IN_BLOCK = 0;

	PRE_LINE = "^[ \t]*";
	POST_LINE = "[ \t]*$";

	# tmp variables
	X = "((\\*|#( *\\*)?) *)?";
	PRE_BLOCK = "(/\\* *|# *(/\\* *)?)?";
	POST_BLOCK = " *(\\*/)?";

	BLOCK[ 0] = PRE_BLOCK "\\*\\*\\*\\*\\* BEGIN LICENSE BLOCK \\*\\*\\*\\*\\*";
	BLOCK[ 1] = X;
	BLOCK[ 2] = X "BBN Address and AS Number PKI Database/repository software";
	BLOCK[ 3] = X "Version [^ ]*";
	BLOCK[ 4] = X;
	BLOCK[ 5] = X "US government users are permitted unrestricted rights as";
	BLOCK[ 6] = X "defined in the FAR.";
	BLOCK[ 7] = X;
	BLOCK[ 8] = X "This software is distributed on an \"AS IS\" basis, WITHOUT";
	BLOCK[ 9] = X "WARRANTY OF ANY KIND, either express or implied.";
	BLOCK[10] = X;
	BLOCK[11] = X "Copyright .*";
	BLOCK[12] = X;
	BLOCK[13] = X "Contributor(\\(s\\)|s)?: .*";
	BLOCK[14] = X;
	BLOCK[15] = X "\\*\\*\\*\\*\\* END LICENSE BLOCK \\*\\*\\*\\*\\*" POST_BLOCK;

	BLOCK_END = 15;
}

{
	if (!IN_BLOCK) {
		if ($0 ~ PRE_LINE BLOCK[0] POST_LINE) {
			IN_BLOCK = 1;
			BLOCK_LINENO = 0;
			SAVED_BLOCK = $0;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
		} else {
			print;
		}
	} else {
		if ($0 ~ PRE_LINE BLOCK[BLOCK_LINENO + 1] POST_LINE) {
			SAVED_BLOCK = SAVED_BLOCK RS $0;
			BLOCK_LINENO += 1;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
			if (BLOCK_LINENO == BLOCK_END) {
				IN_BLOCK = 0;
			}
		} else {
			print SAVED_BLOCK;
			print;
			IN_BLOCK = 0;
		}
	}
}

END {
	if (IN_BLOCK) {
		print SAVED_BLOCK;
	}
}


Fix getMHzFromSitemap after site changed

44:15

Has been split into 6 lines.

# Produce a raw data spreadsheet from URLs found in https://watch.mhzchoice.com/sitemap.xml

# INVOCATION:
#    awk -v ERRORS=$ERRORS -v RAW_CREDITS=$RAW_CREDITS -v RAW_TITLES=$RAW_TITLES \
#        -f getMHzFromSitemap.awk >>$UNSORTED

# Field numbers
# 1 Title  2 Seasons  3 Episodes  4 Duration  5 Genre  6 Country  7 Language  8 Rating  9 Description

### Begin season processing

# Determine whether this is a page 2
#    <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://mhzchoice.vhx.tv/detective-montalbano?page=2">
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
    next
}

# Season Title & Season URL
#    <form class="form"><input value="https://watch.mhzchoice.com/gasmamman/season:1" type="text" \
#        class="text text-center border-none site-background-color site-font-secondary-color" /></form>
# This returns incorrect values if page2 == "yes", but those values are never used
/<form class="form">/ {
    split ($0,fld,"\"")
    seasonURL = fld[4]
    shortSeasonURL = seasonURL
    sub (/.*watch/,"watch",shortSeasonURL)
    split (seasonURL,fld,":")
    seasonNumber = fld[3]
    seasonTitle = "Season " seasonNumber
    # print "==> seasonURL = " seasonURL > "/dev/stderr"
    # print "==> seasonNumber = " seasonNumber > "/dev/stderr"
    # print "==> seasonTitle = " seasonTitle > "/dev/stderr"
    next
}

# Season episodes
#

#      9 Episodes
#

# Extract only the number of episodes, # but if this is page 2 skip to avoid double counting
/<h2 class=.*content-label/,/<\/h2>/ {
    sub (/^ */,"")
    if ($0 ~ / Episode/) {
        seasonEpisodes = $1
        next
    }
}

### Begin show processing

# Show Title
#    <title>Gasmamman - MHz Choice</title>
/<title>/ {
    split ($0,fld,"[<>]")
    showTitle = fld[3]
    sub (/ - MHz Choice/,"",showTitle)
    gsub (/'/,"'",showTitle)
    print showTitle >> RAW_TITLES
    # print "==> showTitle = " showTitle > "/dev/stderr"
    next
}

# Show Descriptor
#    <meta name="description" content="DRAMA - CRIME | SWEDEN | SWEDISH WITH ENGLISH SUBTITLES | TV-14
#    Alexandra Rapaport stars as a suburban mom pulled into the criminal \
#        underworld to pay off her murdered husband’s debts.">
#
# There should be four header fields prior to the description
/<meta name="description" content="/,/">/ {
     # if we're on the first line of this block ...
    if ($0 ~ /name="description" content="/) {
        headerAdded = "no"
        sub (/.*name="description" content="/,"")
    }
    # If we find a header, clean it up and put it before the description
    if ($0 ~ /\|[ ]+TV-/) {
        sub (/WITH ENGLISH SUBTITLES /,"")
        sub (/SCANDINAVIAN CRIME FICTION/,"Sweden")
        sub (/NONFICTION - DOCUMENTARY/,"Documentary")
        sub (/NON-FICTION - DOCUMENTARY/,"Documentary")
        gsub (/ \|[ ]+/,"|")
        sub (/\r/,"")
        # Split out header fields
        numFields = split ($0,fld,"\|")
        # lowercase everything but the first character in all fields but the last
        for (i = 1; i < numFields; ++i) {
            fld[i] = substr(fld[i],1,1) tolower(substr(fld[i],2))
            # uppercase the first character of any second word
            if (match (fld[i],/ /)) {
                fld[i] = substr(fld[i],1,RSTART) toupper(substr(fld[i],RSTART+1,1)) (substr(fld[i],RSTART+2))
            }
        }
        # Put the finalized header before the description
        if (numFields > 3 ) {
            sub (/Drama-crime/,"Drama - crime",fld[1])
            gsub (/ /,"",fld[4])
            showDescriptor =  fld[1] "\t" fld[2] "\t" fld[(numFields-1)] "\t" fld[(numFields)] "\t"
            headerAdded = "yes"
        }
    } else {
        # if we didn't find a header in this block, add a blank one
        if (headerAdded == "no") {
            printf ("==> Added blank header for \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = "\t\t\t\t"
        }
        # We found a description, clean it up and add it
        gsub (/  */," ")
        gsub (/'/,"'")
        gsub (/"/,"\"")
        gsub (/&/,"\\&")
        gsub (/‘/,"’")
        gsub (/’/,"’")
        gsub (/“/,"“")
        gsub (/”/,"”")
        gsub (/\r/," ")
        # if it's not the last line of the description, add it
        if ($0 !~ /"\>/) {
            printf ("==> Multi line description in \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = showDescriptor $0
        } else {
            # if it's the last line of the description, add it
            sub (/"\>/,"")
            showDescriptor = showDescriptor $0
        }
    }
    # print "==> showDescriptor = " showDescriptor > "/dev/stderr"
    next
}

# Show URL
#    <meta property="og:url" content="https://watch.mhzchoice.com/gasmamman">
/<meta property="og:url" content=/ {
    split ($0,fld,"\"")
    showURL = fld[4]
    # print "==> showURL = " showURL > "/dev/stderr"
    next
}

# Show seasons
#

#        3 Seasons
#

# Extract only the number of seasons, # but if this is page 2 skip to avoid double counting
/<h2 class=.*collection-stats"/,/<\/h2>/ {
    sub (/^ */,"")
    # Only if there are two or more seasons, a show with only one season doesn't have this
    if ($0 ~ / Season/) {
        showSeasons = $1
        next
    }
}

### Begin episode processing

# Episode URL(s)
#

#
/
/,/<a href="https:/ {
    if ($0 ~ / episodeURL = " episodeURL > "/dev/stderr"
        next
    }
}

# Episode Duration(s)
# Used to be on one line
#
44:15

#
# Now it's on six
#

#      47:58
#

# Extract the duration
/<div class="duration-container/,/<\/div>/ {
    gsub (/ /,"")
    if ($0 ~ /[[:digit:]]+:[[:digit:]]+/)
        episodeDuration = $0
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    if ($0 ~ /<\/div>/) {
        # print "==> episodeDuration = " episodeDuration > "/dev/stderr"
        next
    }
}

# Episode Title(s)
#
 \
#        Gåsmamman: Episode 01 (Sn 1 Ep 1)

/<h3 class="tooltip-item-title/{
    split ($0,fld,"[<>]")
    episodeTitle = fld[5]
    gsub (/'/,"'",episodeTitle)
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/^[[:space:]]/,"",episodeTitle)
    sub (/[[:space:]]+$/,"",episodeTitle)
    # If start of episodeTitle == showTitle followed by ": " or " - ", remove the redundant part.
    if (match (episodeTitle, showTitle ": ") == 1 || \
        (match (episodeTitle, showTitle " - ") == 1)) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    #
    # If there is a trailing (Sn 1 Ep 1), remove it
    # Handle normal (Sn 1 Ep 1) with variations in spacing and capitalization
    # and ones missing the second letter (Sn 1 E 1), (S1 E1), or wrong second letter (Sm 1 Ep 1)
    if (match (episodeTitle,\
        /[ ]*\(S[Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/))  {
            if (episodeTitle !~ / \(Sn [[:digit:]]+ Ep [[:digit:]]+\)/)
                printf ("==> Malformed Sn/Ep in \"%s: %s\"\n", showTitle, episodeTitle) >> ERRORS
            sub (/[ ]*\(S[Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/,\
                "",episodeTitle)
            # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    }
    #
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeTitle ~ /BONUS|Montalbano and Me/)
        episodeType = "X"
    # If episode is a Trailer (i.e. First look), set episodeType to "T"
    if (episodeTitle ~ /^PR \|/) {
        episodeType = "T"
        seasonEpisodes = seasonEpisodes - 1
    }
    # print "==> episodeType = " episodeType > "/dev/stderr"
    next
}

# Episode Number(s)
#

#         Episode 1
#

/
<span class='media-identifier/,/<\/div>/ {
    sub (/^ */,"")
    if ($0 ~ /Episode /) {
        episodeNumber = $2
        if (showTitle == "Detective Montalbano" && page2 == "yes") {
            oldEpisodeNumber = episodeNumber
            episodeNumber += 24
            printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                    episodeNumber, shortEpisodeURL) >> ERRORS
        }
        # print "==> episodeNumber = " episodeNumber > "/dev/stderr"
        next
    }

### Wrap-up episode processing when Episode Description is found
### print only on LONG_SPREADSHEET
    # print $0 > "/dev/stderr"

    # Episode Description(s)
    #

    #
Preparing for her sister's wedding, Sonja's idyllic life is shattered by tragedy.
    #

    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        if (paragraph != "")
            episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
        gsub (/&/,"\\&",episodeDescription)
        gsub (/'/,"'",episodeDescription)
        # print "descriptionLinesFound = " descriptionLinesFound > "/dev/stderr"
        # print "==> episodeDescription from
 = \n" episodeDescription > "/dev/stderr"
        next
    }
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1/videos/gasmamman-episode-01-sn-1-ep-1-1";\
    #  "Gasmamman, S01E01, Gåsmamman: Episode 01 (Sn 1 Ep 1)")
    if ($0 ~ /<\/div>/) {
        # Make sure episodeDescription is valid
        if (episodeDescription == "") {
            if (episodeType == "T") {
                episodeDescription = episodeTitle
                # print "==> episodeDescription from title = " episodeDescription > "/dev/stderr"
            } else {
                print "==> No description: " shortEpisodeURL >> ERRORS
            }
        }
        if (episodeDescription ~ /^PR \|/)
            sub (/^PR \| /,"",episodeDescription)
        # print "==> episodeDescription = \n" episodeDescription > "/dev/stderr"
        episodeLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\")", episodeURL, showTitle,
                    seasonNumber, episodeType, episodeNumber, episodeTitle)
        #
        # Make sure episodeDuration is valid
        if (split (episodeDuration, tm, ":") == 3) {
            # Canonicalize episodeDuration to 3 parts like other services
            episodeDuration = sprintf ("%02d:%02d:%02d",tm[1],tm[2],tm[3])
        } else {
            printf ("==> Bad episodeDuration %s in \"%s: %s\"\n", episodeDuration, showTitle,
                    episodeTitle) >> ERRORS
            episodeDuration = ""
        }
        #
        # Print "episode" line
        printf ("%s\t\t\t%s\t\t\t\t\t%s\n", episodeLink, episodeDuration, episodeDescription)
        #
        # Look for directors
        director_name = episodeDescription
        if (gsub (/\xc2\xa0/," ",director_name)) {
            printf ("==> UTF-8 non-breaking-space in %s, S%02d%s%02d, %s\n", showTitle, seasonNumber,
                    episodeType, episodeNumber, episodeTitle) >> ERRORS
            print "    " episodeDescription >> ERRORS
        }
        if (director_name ~ /[Dd]irected by /) {
            sub (/.*[Dd]irected by /,"",director_name)
            sub (/,.*$/,"",director_name)
            sub (/ [0-9][0-9][0-9][0-9]\./,"",director_name)
            sub (/[[:space:]]+$/,"",director_name)
            sub (/\.$/,"",director_name)
            # Special cases
            if (director_name ~ /Manetti Bros/)
                director_name = "The Manetti Bros."
            if (director_name ~ /Andrea and Antonio Frazzi/)
                director_name = "Andrea Frazzi and Antonio Frazzi"
            #
            if (match (director_name, " and ") || match (director_name, " & ")) {
                dname = substr(director_name,1,RSTART-1)
                director_name = substr(director_name,RSTART+RLENGTH)
                printf ("%s\tdirector\ttv_show\t%s\n", dname, showTitle) >> RAW_CREDITS
            }
            printf ("%s\tdirector\ttv_show\t%s\n", director_name, showTitle) >> RAW_CREDITS
        }
        # Make sure there is no carryover
        descriptionLinesFound = 0
        episodeLink = ""
        episodeURL =""
        shortEpisodeURL = ""
        episodeType = ""
        episodeNumber = ""
        episodeTitle = ""
        episodeDuration = ""
        episodeDescription =""
    }
}

### Wrap-up show processing - print on standard out

# Footer at end of file
#
/ "/dev/stderr"
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman";"Gasmamman")
    showLink = "=HYPERLINK(\"" showURL "\";\"" showTitle "\")"
    #
    # Shows with only one season have a blank showSeasons
    if (showSeasons == "")
        showSeasons = 1
    # in most cases, showSeasons = the final season number, i.e.
    # a show with two seasons would have seasons 1 and 2
    finalSeason = showSeasons
    # Special case any discontinuous seasons, e.g. 2 seasons, but numbered 1 and 3
    if (showURL ~ /\/wallander$/)
        finalSeason = 3
    #
    # Wrap show, i.e. but only on final season in order to prevent multiples
    # print "==> " showURL " - seasonNumber = " seasonNumber  > "/dev/stderr"
    # print "==> " showTitle " - finalSeason = " finalSeason  > "/dev/stderr"
    if (seasonNumber > finalSeason ) {
        # This should be OK, as printing any single valid season will work - however ...
        printf ("==> Season number %d beyond final season %d: %s\n", seasonNumber, finalSeason,
                shortSeasonURL) >> ERRORS
    }
    if (seasonNumber == finalSeason ) {
        # Print "show" line
        printf ("%s\t%s\t\t\t%s\n", showLink, showSeasons, showDescriptor)
    }
    #
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1";"Gasmamman, S01, Season 1")
    if (page2 != "yes") {
        seasonLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d, %s\")", seasonURL, showTitle, seasonNumber,
                   seasonTitle)
        # print "==> seasonURL = " seasonURL  > "/dev/stderr"
        # print "==> seasonLink = " seasonLink  > "/dev/stderr"
        # Don't print redundant showDescriptor for season 1
        if (seasonNumber == 1)
            showDescriptor = ""
        # Print "season" line
        printf ("%s\t\t%s\t\t%s\n", seasonLink, seasonEpisodes, showDescriptor)
    }
    #
    # Make sure there is no carryover
    showLink = ""
    showURL = ""
    showTitle = ""
    showSeasons = ""
    showDescriptor = ""
    # print "" > "/dev/stderr"
    #
    page2 = ""
    seasonLink = ""
    seasonURL = ""
    shortSeasonURL = ""
    seasonNumber = ""
    seasonTitle = ""
    seasonEpisodes = ""
}

# Produce a raw data spreadsheet from URLs found in https://watch.mhzchoice.com/sitemap.xml

# INVOCATION:
#    awk -v ERRORS=$ERRORS -v RAW_CREDITS=$RAW_CREDITS -v RAW_TITLES=$RAW_TITLES \
#        -f getMHzFromSitemap.awk >>$UNSORTED

# Field numbers
# 1 Title  2 Seasons  3 Episodes  4 Duration  5 Genre  6 Country  7 Language  8 Rating  9 Description

### Begin season processing

# Determine whether this is a page 2
#    <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://mhzchoice.vhx.tv/detective-montalbano?page=2">
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
    next
}

# Season Title & Season URL
#    <form class="form"><input value="https://watch.mhzchoice.com/gasmamman/season:1" type="text" \
#        class="text text-center border-none site-background-color site-font-secondary-color" /></form>
# This returns incorrect values if page2 == "yes", but those values are never used
/<form class="form">/ {
    split ($0,fld,"\"")
    seasonURL = fld[4]
    shortSeasonURL = seasonURL
    sub (/.*watch/,"watch",shortSeasonURL)
    split (seasonURL,fld,":")
    seasonNumber = fld[3]
    seasonTitle = "Season " seasonNumber
    # print "==> seasonURL = " seasonURL > "/dev/stderr"
    # print "==> seasonNumber = " seasonNumber > "/dev/stderr"
    # print "==> seasonTitle = " seasonTitle > "/dev/stderr"
    next
}

# Season episodes
#

#      9 Episodes
#

# Extract only the number of episodes, # but if this is page 2 skip to avoid double counting
/<h2 class=.*content-label/,/<\/h2>/ {
    sub (/^ */,"")
    if ($0 ~ / Episode/) {
        seasonEpisodes = $1
        next
    }
}

### Begin show processing

# Show Title
#    <title>Gasmamman - MHz Choice</title>
/<title>/ {
    split ($0,fld,"[<>]")
    showTitle = fld[3]
    sub (/ - MHz Choice/,"",showTitle)
    gsub (/'/,"'",showTitle)
    print showTitle >> RAW_TITLES
    # print "==> showTitle = " showTitle > "/dev/stderr"
    next
}

# Show Descriptor
#    <meta name="description" content="DRAMA - CRIME | SWEDEN | SWEDISH WITH ENGLISH SUBTITLES | TV-14
#    Alexandra Rapaport stars as a suburban mom pulled into the criminal \
#        underworld to pay off her murdered husband’s debts.">
#
# There should be four header fields prior to the description
/<meta name="description" content="/,/">/ {
     # if we're on the first line of this block ...
    if ($0 ~ /name="description" content="/) {
        headerAdded = "no"
        sub (/.*name="description" content="/,"")
    }
    # If we find a header, clean it up and put it before the description
    if ($0 ~ /\|[ ]+TV-/) {
        sub (/WITH ENGLISH SUBTITLES /,"")
        sub (/SCANDINAVIAN CRIME FICTION/,"Sweden")
        sub (/NONFICTION - DOCUMENTARY/,"Documentary")
        sub (/NON-FICTION - DOCUMENTARY/,"Documentary")
        gsub (/ \|[ ]+/,"|")
        sub (/\r/,"")
        # Split out header fields
        numFields = split ($0,fld,"\|")
        # lowercase everything but the first character in all fields but the last
        for (i = 1; i < numFields; ++i) {
            fld[i] = substr(fld[i],1,1) tolower(substr(fld[i],2))
            # uppercase the first character of any second word
            if (match (fld[i],/ /)) {
                fld[i] = substr(fld[i],1,RSTART) toupper(substr(fld[i],RSTART+1,1)) (substr(fld[i],RSTART+2))
            }
        }
        # Put the finalized header before the description
        if (numFields > 3 ) {
            sub (/Drama-crime/,"Drama - crime",fld[1])
            gsub (/ /,"",fld[4])
            showDescriptor =  fld[1] "\t" fld[2] "\t" fld[(numFields-1)] "\t" fld[(numFields)] "\t"
            headerAdded = "yes"
        }
    } else {
        # if we didn't find a header in this block, add a blank one
        if (headerAdded == "no") {
            printf ("==> Added blank header for \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = "\t\t\t\t"
        }
        # We found a description, clean it up and add it
        gsub (/  */," ")
        gsub (/'/,"'")
        gsub (/"/,"\"")
        gsub (/&/,"\\&")
        gsub (/‘/,"’")
        gsub (/’/,"’")
        gsub (/“/,"“")
        gsub (/”/,"”")
        gsub (/\r/," ")
        # if it's not the last line of the description, add it
        if ($0 !~ /"\>/) {
            printf ("==> Multi line description in \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = showDescriptor $0
        } else {
            # if it's the last line of the description, add it
            sub (/"\>/,"")
            showDescriptor = showDescriptor $0
        }
    }
    # print "==> showDescriptor = " showDescriptor > "/dev/stderr"
    next
}

# Show URL
#    <meta property="og:url" content="https://watch.mhzchoice.com/gasmamman">
/<meta property="og:url" content=/ {
    split ($0,fld,"\"")
    showURL = fld[4]
    # print "==> showURL = " showURL > "/dev/stderr"
    next
}

# Show seasons
#

#        3 Seasons
#

# Extract only the number of seasons, # but if this is page 2 skip to avoid double counting
/<h2 class=.*collection-stats"/,/<\/h2>/ {
    sub (/^ */,"")
    # Only if there are two or more seasons, a show with only one season doesn't have this
    if ($0 ~ / Season/) {
        showSeasons = $1
        next
    }
}

### Begin episode processing

# Episode URL(s)
#

#
/
/,/<a href="https:/ {
    if ($0 ~ / episodeURL = " episodeURL > "/dev/stderr"
        next
    }
}

# Episode Duration(s)
#
44:15

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    # print "==> episodeDuration = " episodeDuration > "/dev/stderr"
    next
}

# Episode Title(s)
#
 \
#        Gåsmamman: Episode 01 (Sn 1 Ep 1)

/<h3 class="tooltip-item-title/{
    split ($0,fld,"[<>]")
    episodeTitle = fld[5]
    gsub (/'/,"'",episodeTitle)
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/^[[:space:]]/,"",episodeTitle)
    sub (/[[:space:]]+$/,"",episodeTitle)
    # If start of episodeTitle == showTitle followed by ": " or " - ", remove the redundant part.
    if (match (episodeTitle, showTitle ": ") == 1 || \
        (match (episodeTitle, showTitle " - ") == 1)) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    #
    # If there is a trailing (Sn 1 Ep 1), remove it
    # Handle normal (Sn 1 Ep 1) with variations in spacing and capitalization
    # and ones missing the second letter (Sn 1 E 1), (S1 E1), or wrong second letter (Sm 1 Ep 1)
    if (match (episodeTitle,\
        /[ ]*\(S[Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/))  {
            if (episodeTitle !~ / \(Sn [[:digit:]]+ Ep [[:digit:]]+\)/)
                printf ("==> Malformed Sn/Ep in \"%s: %s\"\n", showTitle, episodeTitle) >> ERRORS
            sub (/[ ]*\(S[Nnm]*[ ]*[[:digit:]]+[ ]+[Ee][Pp]*[ ]*[[:digit:]]+[[:space:]]*\)/,\
                "",episodeTitle)
            # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    }
    #
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeTitle ~ /BONUS|Montalbano and Me/)
        episodeType = "X"
    # If episode is a Trailer (i.e. First look), set episodeType to "T"
    if (episodeTitle ~ /^PR \|/) {
        episodeType = "T"
        seasonEpisodes = seasonEpisodes - 1
    }
    # print "==> episodeType = " episodeType > "/dev/stderr"
    next
}

# Episode Number(s)
#

#         Episode 1
#

/
<span class='media-identifier/,/<\/div>/ {
    sub (/^ */,"")
    if ($0 ~ /Episode /) {
        episodeNumber = $2
        if (showTitle == "Detective Montalbano" && page2 == "yes") {
            oldEpisodeNumber = episodeNumber
            episodeNumber += 24
            printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                    episodeNumber, shortEpisodeURL) >> ERRORS
        }
        # print "==> episodeNumber = " episodeNumber > "/dev/stderr"
        next
    }

### Wrap-up episode processing when Episode Description is found
### print only on LONG_SPREADSHEET
    # print $0 > "/dev/stderr"

    # Episode Description(s)
    #

    #
Preparing for her sister's wedding, Sonja's idyllic life is shattered by tragedy.
    #

    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        if (paragraph != "")
            episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
        gsub (/&/,"\\&",episodeDescription)
        gsub (/'/,"'",episodeDescription)
        # print "descriptionLinesFound = " descriptionLinesFound > "/dev/stderr"
        # print "==> episodeDescription from
 = \n" episodeDescription > "/dev/stderr"
        next
    }
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1/videos/gasmamman-episode-01-sn-1-ep-1-1";\
    #  "Gasmamman, S01E01, Gåsmamman: Episode 01 (Sn 1 Ep 1)")
    if ($0 ~ /<\/div>/) {
        # Make sure episodeDescription is valid
        if (episodeDescription == "") {
            if (episodeType == "T") {
                episodeDescription = episodeTitle
                # print "==> episodeDescription from title = " episodeDescription > "/dev/stderr"
            } else {
                print "==> No description: " shortEpisodeURL >> ERRORS
            }
        }
        if (episodeDescription ~ /^PR \|/)
            sub (/^PR \| /,"",episodeDescription)
        # print "==> episodeDescription = \n" episodeDescription > "/dev/stderr"
        episodeLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\")", episodeURL, showTitle,
                    seasonNumber, episodeType, episodeNumber, episodeTitle)
        #
        # Make sure episodeDuration is valid
        if (split (episodeDuration, tm, ":") == 3) {
            # Canonicalize episodeDuration to 3 parts like other services
            episodeDuration = sprintf ("%02d:%02d:%02d",tm[1],tm[2],tm[3])
        } else {
            printf ("==> Bad episodeDuration %s in \"%s: %s\"\n", episodeDuration, showTitle,
                    episodeTitle) >> ERRORS
            episodeDuration = ""
        }
        #
        # Print "episode" line
        printf ("%s\t\t\t%s\t\t\t\t\t%s\n", episodeLink, episodeDuration, episodeDescription)
        #
        # Look for directors
        director_name = episodeDescription
        if (gsub (/\xc2\xa0/," ",director_name)) {
            printf ("==> UTF-8 non-breaking-space in %s, S%02d%s%02d, %s\n", showTitle, seasonNumber,
                    episodeType, episodeNumber, episodeTitle) >> ERRORS
            print "    " episodeDescription >> ERRORS
        }
        if (director_name ~ /[Dd]irected by /) {
            sub (/.*[Dd]irected by /,"",director_name)
            sub (/,.*$/,"",director_name)
            sub (/ [0-9][0-9][0-9][0-9]\./,"",director_name)
            sub (/[[:space:]]+$/,"",director_name)
            sub (/\.$/,"",director_name)
            # Special cases
            if (director_name ~ /Manetti Bros/)
                director_name = "The Manetti Bros."
            if (director_name ~ /Andrea and Antonio Frazzi/)
                director_name = "Andrea Frazzi and Antonio Frazzi"
            #
            if (match (director_name, " and ") || match (director_name, " & ")) {
                dname = substr(director_name,1,RSTART-1)
                director_name = substr(director_name,RSTART+RLENGTH)
                printf ("%s\tdirector\ttv_show\t%s\n", dname, showTitle) >> RAW_CREDITS
            }
            printf ("%s\tdirector\ttv_show\t%s\n", director_name, showTitle) >> RAW_CREDITS
        }
        # Make sure there is no carryover
        descriptionLinesFound = 0
        episodeLink = ""
        episodeURL =""
        shortEpisodeURL = ""
        episodeType = ""
        episodeNumber = ""
        episodeTitle = ""
        episodeDuration = ""
        episodeDescription =""
    }
}

### Wrap-up show processing - print on standard out

# Footer at end of file
#
/ "/dev/stderr"
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman";"Gasmamman")
    showLink = "=HYPERLINK(\"" showURL "\";\"" showTitle "\")"
    #
    # Shows with only one season have a blank showSeasons
    if (showSeasons == "")
        showSeasons = 1
    # in most cases, showSeasons = the final season number, i.e.
    # a show with two seasons would have seasons 1 and 2
    finalSeason = showSeasons
    # Special case any discontinuous seasons, e.g. 2 seasons, but numbered 1 and 3
    if (showURL ~ /\/wallander$/)
        finalSeason = 3
    #
    # Wrap show, i.e. but only on final season in order to prevent multiples
    # print "==> " showURL " - seasonNumber = " seasonNumber  > "/dev/stderr"
    # print "==> " showTitle " - finalSeason = " finalSeason  > "/dev/stderr"
    if (seasonNumber > finalSeason ) {
        # This should be OK, as printing any single valid season will work - however ...
        printf ("==> Season number %d beyond final season %d: %s\n", seasonNumber, finalSeason,
                shortSeasonURL) >> ERRORS
    }
    if (seasonNumber == finalSeason ) {
        # Print "show" line
        printf ("%s\t%s\t\t\t%s\n", showLink, showSeasons, showDescriptor)
    }
    #
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1";"Gasmamman, S01, Season 1")
    if (page2 != "yes") {
        seasonLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d, %s\")", seasonURL, showTitle, seasonNumber,
                   seasonTitle)
        # print "==> seasonURL = " seasonURL  > "/dev/stderr"
        # print "==> seasonLink = " seasonLink  > "/dev/stderr"
        # Don't print redundant showDescriptor for season 1
        if (seasonNumber == 1)
            showDescriptor = ""
        # Print "season" line
        printf ("%s\t\t%s\t\t%s\n", seasonLink, seasonEpisodes, showDescriptor)
    }
    #
    # Make sure there is no carryover
    showLink = ""
    showURL = ""
    showTitle = ""
    showSeasons = ""
    showDescriptor = ""
    # print "" > "/dev/stderr"
    #
    page2 = ""
    seasonLink = ""
    seasonURL = ""
    shortSeasonURL = ""
    seasonNumber = ""
    seasonTitle = ""
    seasonEpisodes = ""
}