Translators/YandexTranslate: support dictionary service

####################################################################
# YandexTranslate.awk                                              #
####################################################################
#
# Last Updated: 16 May 2016
BEGIN { provides("yandex") }

function genSID(    content, group, temp) {
    content = curl("http://translate.yandex.com")

    match(content, /SID:[[:space:]]*'([^']+)'/, group)
    if (group[1]) {
        split(group[1], temp, ".")
        SID = reverse(temp[1]) "." reverse(temp[2]) "." reverse(temp[3])
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function yandexInit() {
    genSID() # generate a one-time key
    YandexWebTranslate = "z5h64q92x9.net" # host for web translation

    HttpProtocol = "http://"
    HttpHost = "translate.yandex.net"
    HttpPort = 80
}

function yandexRequestUrl(text, sl, tl, hl,    group) {
    # Quick hack: Yandex doesn't support digraphia code (yet)
    split(sl, group, "-"); sl = group[1]
    split(tl, group, "-"); tl = group[1]

    return HttpPathPrefix "/api/v1/tr.json/translate?"                  \
        "id=" SID "-0-0&srv=tr-text"                                    \
        "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)
}

function yandexGetDictionaryResponse(text, sl, tl, hl,    content, header, isBody, url) {
    # Quick hack: Yandex doesn't support digraphia code (yet)
    split(sl, group, "-"); sl = group[1]
    split(tl, group, "-"); tl = group[1]

    url = HttpPathPrefix "/dicservice.json/lookup?"                     \
        "sid=" SID                                                      \
        "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)

    header = "GET " url " HTTP/1.1\n"           \
        "Host: " "dictionary.yandex.net" "\n"   \
        "Connection: close\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    return assert(content, "[ERROR] Null response.")
}

function yandexTTSUrl(text, tl) {
    switch (tl) { # List of available TTS language codes
    case "ar": tl = "ar_AE"; break
    case "cs": tl = "cs_CZ"; break
    case "da": tl = "da_DK"; break
    case "de": tl = "de_DE"; break
    case "el": tl = "el_GR"; break
    case "en": tl = "en_GB"; break
    case "es": tl = "es_ES"; break
    case "fi": tl = "fi_FI"; break
    case "fr": tl = "fr_FR"; break
    case "it": tl = "it_IT"; break
    case "nl": tl = "nl_NL"; break
    case "no": tl = "no_NO"; break
    case "pl": tl = "pl_PL"; break
    case "pt": tl = "pt_PT"; break
    case "ru": tl = "ru_RU"; break
    case "sv": tl = "sv_SE"; break
    case "tr": tl = "tr_TR"; break
    default: tl = NULLSTR
    }
    return HttpProtocol "tts.voicetech.yandex.net" "/tts?"              \
        "text=" preprocess(text) (tl ? "&lang=" tl : tl)                \
        "&format=mp3" "&quality=hi"
}

function yandexWebTranslateUrl(uri, sl, tl, hl) {
    gsub(/:\/\//, "/", uri)
    return HttpProtocol YandexWebTranslate "/proxy_u/"  \
        (sl == "auto" ? tl : sl "-" tl)"/" uri
}

# Get the translation of a string.
function yandexTranslate(text, sl, tl, hl,
                         isVerbose, toSpeech, returnPlaylist, returnIl,
                         ####
                         r,
                         content, tokens, ast,
                         _sl, _tl, _hl, il,
                         translation,
                         wShowOriginal, wShowTranslation, wShowLanguages,
                         wShowDictionary, dicContent, dicTokens, dicAst,
                         i, syn, mean,
                         group, temp) {
    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    content = getResponse(text, _sl, _tl, _hl)
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    if (ast[0 SUBSEP "code"] != "200") {
        e("[ERROR] " unparameterize(ast[0 SUBSEP "message"]))
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "text" SUBSEP 0])

    split(unparameterize(ast[0 SUBSEP "lang"]), group, "-")
    returnIl[0] = il = group[1]
    if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode
        r = translation

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary) {
            # Dictionary API
            dicContent = yandexGetDictionaryResponse(text, _sl, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens)

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                saveSortedIn = PROCINFO["sorted_in"]
                PROCINFO["sorted_in"] = "@ind_num_asc"
                for (i in dicAst) {
                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
                        "pos$") {
                        r = r RS prettify("dictionary-word-class", s((literal(dicAst[i])), hl))
                        syn = mean = ""
                    }

                    # TODO: ex, gen, ...

                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
                        "tr" SUBSEP "[[:digit:]]+" SUBSEP               \
                        "mean" SUBSEP "[[:digit:]]+" SUBSEP "text") {
                        if (mean) {
                            mean = mean prettify("dictionary-explanation", ", ") \
                                prettify("dictionary-explanation-item", s((literal(dicAst[i])), sl))
                        } else {
                            mean = prettify("dictionary-explanation-item", s((literal(dicAst[i])), sl))
                        }
                    }

                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
                        "tr" SUBSEP "[[:digit:]]+" SUBSEP               \
                        "syn" SUBSEP "[[:digit:]]+" SUBSEP "text") {
                        if (syn) {
                            syn = syn prettify("dictionary-explanation", ", ") \
                                prettify("dictionary-word", s((literal(dicAst[i])), il))
                        } else {
                            syn = prettify("dictionary-word", s((literal(dicAst[i])), il))
                        }
                    }

                    if (i ~ "^0" SUBSEP "def" SUBSEP "[[:digit:]]+" SUBSEP \
                        "tr" SUBSEP "[[:digit:]]+" SUBSEP "text$") {
                        text = prettify("dictionary-word", s((literal(dicAst[i])), il))
                        if (syn) {
                            r = r RS ins(1, text prettify("dictionary-explanation", ", ") syn)
                        } else {
                            r = r RS ins(1, text)
                        }
                        r = r RS ins(2, mean)
                        syn = mean = ""
                    }
                }
                PROCINFO["sorted_in"] = saveSortedIn
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}

####################################################################
# YandexTranslate.awk                                              #
####################################################################
#
# Last Updated: 16 May 2016
BEGIN { provides("yandex") }

function genSID(    content, group, temp) {
    content = curl("http://translate.yandex.com")

    match(content, /SID:[[:space:]]*'([^']+)'/, group)
    if (group[1]) {
        split(group[1], temp, ".")
        SID = reverse(temp[1]) "." reverse(temp[2]) "." reverse(temp[3])
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function yandexInit() {
    genSID() # generate a one-time key
    YandexWebTranslate = "z5h64q92x9.net" # host for web translation

    HttpProtocol = "http://"
    HttpHost = "translate.yandex.net"
    HttpPort = 80
}

function yandexRequestUrl(text, sl, tl, hl,    group) {
    # Quick hack: Yandex doesn't support digraphia code (yet)
    split(sl, group, "-"); sl = group[1]
    split(tl, group, "-"); tl = group[1]

    return HttpPathPrefix "/api/v1/tr.json/translate?"                  \
        "id=" SID "-0-0&srv=tr-text"                                    \
        "&text=" preprocess(text) "&lang=" (sl == "auto" ? tl : sl "-" tl)
}

function yandexTTSUrl(text, tl) {
    switch (tl) { # List of available TTS language codes
    case "ar": tl = "ar_AE"; break
    case "cs": tl = "cs_CZ"; break
    case "da": tl = "da_DK"; break
    case "de": tl = "de_DE"; break
    case "el": tl = "el_GR"; break
    case "en": tl = "en_GB"; break
    case "es": tl = "es_ES"; break
    case "fi": tl = "fi_FI"; break
    case "fr": tl = "fr_FR"; break
    case "it": tl = "it_IT"; break
    case "nl": tl = "nl_NL"; break
    case "no": tl = "no_NO"; break
    case "pl": tl = "pl_PL"; break
    case "pt": tl = "pt_PT"; break
    case "ru": tl = "ru_RU"; break
    case "sv": tl = "sv_SE"; break
    case "tr": tl = "tr_TR"; break
    default: tl = NULLSTR
    }
    return HttpProtocol "tts.voicetech.yandex.net" "/tts?"              \
        "text=" preprocess(text) (tl ? "&lang=" tl : tl)                \
        "&format=mp3" "&quality=hi"
}

function yandexWebTranslateUrl(uri, sl, tl, hl) {
    gsub(/:\/\//, "/", uri)
    return HttpProtocol YandexWebTranslate "/proxy_u/"  \
        (sl == "auto" ? tl : sl "-" tl)"/" uri
}

# Get the translation of a string.
function yandexTranslate(text, sl, tl, hl,
                         isVerbose, toSpeech, returnPlaylist, returnIl,
                         ####
                         r,
                         content, tokens, ast,
                         _sl, _tl, _hl, il,
                         translation,
                         wShowOriginal, wShowTranslation, wShowLanguages,
                         group, temp) {
    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    content = getResponse(text, _sl, _tl, _hl)
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    if (ast[0 SUBSEP "code"] != "200") {
        e("[ERROR] " unparameterize(ast[0 SUBSEP "message"]))
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "text" SUBSEP 0])

    split(unparameterize(ast[0 SUBSEP "lang"]), group, "-")
    returnIl[0] = il = group[1]
    if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode
        r = translation

    } else {
        # Verbose mode
        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}


Convert duration in seconds to HH:MM:SS

Convert duration in seconds to HH:MM:SS

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

# INVOCATION
#   awk -v ERROR_FILE=$ERROR_FILE -f getBBoxCatalogFrom-sitemap.awk $SITEMAP_FILE >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title           3 Seasons          4 Episodes      5 Duration     6 Year
#     7 Rating        8 Description     9 Content Type    10 Content ID   11 Entity ID   12 Genre
#    13 Show Type    14 Date Type      15 Original Date   16 Show ID      16 Season ID   13 Sn #   14 Ep #

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear\tRating\tDescription\t")
    printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    printf ("Show ID\tSeason ID\tSn #\tEp #\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType  > "/dev/stderr"
    # print "contentId = " contentId  > "/dev/stderr"
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = 111
    numEpisodes = 222
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title  > "/dev/stderr"
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre  > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description  > "/dev/stderr"
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating  > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType  > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType  > "/dev/stderr"
    # print "originalDate = " originalDate  > "/dev/stderr"
    # print "year = " year  > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId  > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId  > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber  > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber  > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId  > "/dev/stderr"
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    # Add missing EntityId
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERROR_FILE
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERROR_FILE
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERROR_FILE
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERROR_FILE
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> originalDate = " originalDate  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) S%s", title, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) S%02d", showTitle, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, \
                episodeNumber, showContentId)
    }

    # Copied from printing header to make it easier to coordinate arranging fields
    # printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    # printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    # printf ("Show ID\tSeason ID\tSn #\tEp #\n")

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
            sortkey, title, numSeasons, numEpisodes, duration, year, rating, description, \
            contentType, contentId, EntityId, genre, showType, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

END {
    printf ("In getBBoxCatalogFrom-sitemap.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
}

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

# INVOCATION
#   awk -v ERROR_FILE=$ERROR_FILE -f getBBoxCatalogFrom-sitemap.awk $SITEMAP_FILE >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title           3 Seasons          4 Episodes      5 Duration     6 Year
#     7 Rating        8 Description     9 Content Type    10 Content ID   11 Entity ID   12 Genre
#    13 Show Type    14 Date Type      15 Original Date   16 Show ID      16 Season ID   13 Sn #   14 Ep #

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear\tRating\tDescription\t")
    printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    printf ("Show ID\tSeason ID\tSn #\tEp #\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType  > "/dev/stderr"
    # print "contentId = " contentId  > "/dev/stderr"
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = 111
    numEpisodes = 222
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title  > "/dev/stderr"
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre  > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description  > "/dev/stderr"
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating  > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType  > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration  > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType  > "/dev/stderr"
    # print "originalDate = " originalDate  > "/dev/stderr"
    # print "year = " year  > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId  > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId  > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber  > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber  > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId  > "/dev/stderr"
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    # Add missing EntityId
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERROR_FILE
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERROR_FILE
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERROR_FILE
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERROR_FILE
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> originalDate = " originalDate  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) S%s", title, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) S%02d", showTitle, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, \
                episodeNumber, showContentId)
    }

    # Copied from printing header to make it easier to coordinate arranging fields
    # printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    # printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    # printf ("Show ID\tSeason ID\tSn #\tEp #\n")

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
            sortkey, title, numSeasons, numEpisodes, duration, year, rating, description, \
            contentType, contentId, EntityId, genre, showType, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

END {
    printf ("In getBBoxCatalogFrom-sitemap.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
}


fix start_date value

fix start_date value

# EUCѴƤ餳ΥץȤȤθ塢UTF᤹
# cat tenpomaster | nkf -e | awk -F, -f lawson2osm.awk | nkf -w > tenpomaster.osm

BEGIN {
    printf("<?xml version='1.0' encoding='UTF-8'?>\n");
    printf("<osm version='0.6'>\n");
}
{

    if($1 !~ /^[0-9]*$/) next

    cmd="echo "$11"|sed -e s//-/g -e s/*.$//g -e s//\\ /g| nkf -eZ|kakasi -Ha"
    cmd | getline t
    close(cmd)

    cmd2="echo "$17"|sed s/^0/+81-/g"
    cmd2|getline p
    close(cmd2)

    printf("  <node id='-%s' lat='%f' lon='%f'>\n",$1,$2/3600000,$3/3600000);
    printf("    <tag k='shop' v='convenience' />\n");
    printf("    <tag k='toilet' v='yes' />\n");
    printf("    <tag k='atm' v='yes' />\n");
    printf("    <tag k='internet_access' v='wlan' />\n");
    if($18==2){
	printf("    <tag k='name' v='ʥ %sŹ' />\n",$9);
	printf("    <tag k='name:en' v='Natural Lawson %s' />\n",t);
	printf("    <tag k='brand' v='ʥ' />\n");
	printf("    <tag k='brand:en' v='Natural Lawson' />\n");
    }else if($18==4){
	printf("    <tag k='name' v='󥹥ȥ100 %sŹ' />\n",$9);
	printf("    <tag k='name:en' v='Lawson Store 100 %s' />\n",t);
	printf("    <tag k='brand' v='󥹥ȥ100' />\n");
	printf("    <tag k='brand:en' v='Lawson Store 100' />\n");
    }else{
	printf("    <tag k='name' v=' %sŹ' />\n",$9);
	printf("    <tag k='name:en' v='Lawson %s' />\n",t);
	printf("    <tag k='brand' v='' />\n");
	printf("    <tag k='brand:en' v='Lawson' />\n");
    }
    printf("    <tag k='branch' v='%sŹ' />\n",$9);
    printf("    <tag k='branch:ja_kana' v='%sƤ' />\n",$11);
    printf("    <tag k='branch:en' v='%s' />\n",t);
    printf("    <tag k='operator' v='' />\n");
    printf("    <tag k='operator:en' v='Lawson' />\n");
    printf("    <tag k='phone' v='%s' />\n",p);
    if($19==1){
	printf("    <tag k='opening_hours' v='24/7' />\n");
    }else{
	if($20 > $21){
	    printf("    <tag k='opening_hours' v='Mo-Su %s-%d:%s' />\n",$20,substr($21,1,2)+24,substr($21,4,2));
	}else{
	    printf("    <tag k='opening_hours' v='Mo-Su %s-%s' />\n",$20,$21);
	}
    }
    if($5!=""){
	printf("    <tag k='start_date' v='%s-%s-%s' />\n",substr($5,1,4),substr($5,5,2),substr($5,7,2));
    }
    printf("    <tag k='addr:full' v='%s' />\n",$10);
    printf("    <tag k='ref' v='%s' />\n",$1);
    printf("  </node>\n");
}
END {
    printf("</osm>\n");
}

# EUCѴƤ餳ΥץȤȤθ塢UTF᤹
# cat tenpomaster | nkf -e | awk -F, -f lawson2osm.awk | nkf -w > tenpomaster.osm

BEGIN {
    printf("<?xml version='1.0' encoding='UTF-8'?>\n");
    printf("<osm version='0.6'>\n");
}
{

    if($1 !~ /^[0-9]*$/) next

    cmd="echo "$11"|sed -e s//-/g -e s/*.$//g -e s//\\ /g| nkf -eZ|kakasi -Ha"
    cmd | getline t
    close(cmd)

    cmd2="echo "$17"|sed s/^0/+81-/g"
    cmd2|getline p
    close(cmd2)

    printf("  <node id='-%s' lat='%f' lon='%f'>\n",$1,$2/3600000,$3/3600000);
    printf("    <tag k='shop' v='convenience' />\n");
    printf("    <tag k='toilet' v='yes' />\n");
    printf("    <tag k='atm' v='yes' />\n");
    printf("    <tag k='internet_access' v='wlan' />\n");
    if($18==2){
	printf("    <tag k='name' v='ʥ %sŹ' />\n",$9);
	printf("    <tag k='name:en' v='Natural Lawson %s' />\n",t);
	printf("    <tag k='brand' v='ʥ' />\n");
	printf("    <tag k='brand:en' v='Natural Lawson' />\n");
    }else if($18==4){
	printf("    <tag k='name' v='󥹥ȥ100 %sŹ' />\n",$9);
	printf("    <tag k='name:en' v='Lawson Store 100 %s' />\n",t);
	printf("    <tag k='brand' v='󥹥ȥ100' />\n");
	printf("    <tag k='brand:en' v='Lawson Store 100' />\n");
    }else{
	printf("    <tag k='name' v=' %sŹ' />\n",$9);
	printf("    <tag k='name:en' v='Lawson %s' />\n",t);
	printf("    <tag k='brand' v='' />\n");
	printf("    <tag k='brand:en' v='Lawson' />\n");
    }
    printf("    <tag k='branch' v='%sŹ' />\n",$9);
    printf("    <tag k='branch:ja_kana' v='%sƤ' />\n",$11);
    printf("    <tag k='branch:en' v='%s' />\n",t);
    printf("    <tag k='operator' v='' />\n");
    printf("    <tag k='operator:en' v='Lawson' />\n");
    printf("    <tag k='phone' v='%s' />\n",p);
    if($19==1){
	printf("    <tag k='opening_hours' v='24/7' />\n");
    }else{
	if($20 > $21){
	    printf("    <tag k='opening_hours' v='Mo-Su %s-%d:%s' />\n",$20,substr($21,1,2)+24,substr($21,4,2));
	}else{
	    printf("    <tag k='opening_hours' v='Mo-Su %s-%s' />\n",$20,$21);
	}
    }
    if($5!=""){
	printf("    <tag k='start_date' v='%s' />\n",$5);
    }
    printf("    <tag k='addr:full' v='%s' />\n",$10);
    printf("    <tag k='ref' v='%s' />\n",$1);
    printf("  </node>\n");
}
END {
    printf("</osm>\n");
}


Change assignments.

Change assignments.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@31188 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[tester],status,year

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either status is _not_ manifest or ending year is gt or eq to contract year
  current_year = 2012
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "American R")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "American S") && (substr(p[pn],1,2) <= "Hz")) {
          r[pn] = "4"
#        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Pz")) {
#          r[pn] = "5"
        } else if (substr(p[pn],1,1) >= "IA") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[tester],status,year

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if either status is _not_ manifest or ending year is gt or eq to contract year
  current_year = 2012
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2005
  if (length($5) > 3) {
    end_year = substr($5,length($5)-3,4)
  }
  #printf "%s\n", $0
  if (($4 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      if (r[pn] == "") {
        if ((substr(p[pn],1,2) <= "American R")) {
          r[pn] = "5"
        } else if ((substr(p[pn],1,2) >= "American S") && (substr(p[pn],1,2) <= "Bz")) {
          r[pn] = "4"
        } else if ((substr(p[pn],1,2) >= "CA") && (substr(p[pn],1,2) <= "Pz")) {
          r[pn] = "5"
        } else if (substr(p[pn],1,1) >= "Q") {
          r[pn] = "8"
        }
      }
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}


geo_utils.awk	-- added GU_get_style_defaults()

function GU_pi() {
	return 4*atan2(1, 1)
}
function GU_d2r(d) {
	return (d+0)*GU_pi()/180.0
}
function GU_r2d(r) {
	return (r+0)*180.0/GU_pi()
}
function GU_get_style_defaults(sd) {
	sd["marker-color"] = "#aae"
	sd["marker-size"]  = "small"
}
function GU_is_righthanded(count, first, x, y,   area, p, x1, y1, x2, y2) {
	area = 0
	for(p = 0; p < p_count[poly]; p++){
		x1 = x[p_first[poly] + p]
		y1 = y[p_first[poly] + p]
		x2 = x[p_first[poly] + p + 1]
		y2 = y[p_first[poly] + p + 1]
		area += GU_d2r(x2 - x1) * (2.0 + sin(GU_d2r(y1)) + sin(GU_d2r(y2)))
	}
	return area >= 0
}
function GU_pr_header(sc_cfg, meta_data) {

	printf("{\n")
	if(sc_cfg != ""){
		printf("\"scaleConfig\": ")
		for( ; (getline sc_line < sc_cfg) > 0; )
			printf("%s\n", sc_line)
		printf(",\n")
		close(sc_cfg)
	}
	printf("\"geojson\": {\n")
	printf("\"type\": \"FeatureCollection\",\n")
	printf("\"metadata\": {\n")
	printf("  \"createdAt\": \"%s\"%s\n", strftime("%Y%m%dT%H%M%S%Z"), meta_data["count"] > 0 ? "," : "")
	for(i = 1; i <= meta_data["count"]; i++){
		printf("  \"%s\": ", meta_data[i, "key"])
		if(!meta_data[i, "is_str"])
			printf("%s", meta_data[i, "value"])
		else
			printf("\"%s\"", GU_str_escape(meta_data[i, "value"]))
		printf("%s\n", i < meta_data["count"] ? "," : "")
	}
	printf("},\n")
	printf("\"features\": [\n")
}
function GU_pr_trailer() {
	printf("]\n")
	printf("}\n")
	printf("}\n")
}
function GU_find_pgroups(start, count, longs, lats, pg_starts, pg_counts,   n_pgroups, l_geo, geo, i) {

	n_pgroups = 1
	pg_starts[n_pgroups] = start
	pg_counts[n_pgroups] = 1
	l_geo[1] = longs[start]
	l_geo[2] = lats[start]
	for(i = 1; i < count; i++){
		geo[1] = longs[start + i]
		geo[2] = longs[start + i]
		if(GU_geo_equal(geo, l_geo)){
			pg_counts[n_pgroups]++
		}else{
			n_pgroups++
			pg_starts[n_pgroups] = start + i
			pg_counts[n_pgroups] = 1
		}
		l_geo[1] = geo[1]
		l_geo[2] = geo[2]
	}

	return n_pgroups
}
function GU_geo_equal(g1, g2) {
	return g1[1] == g2[1] && g1[2] == g2[2]
}
function GU_geo_isnull(g) {
	return g[1] == "" || g[2] == ""
}
function GU_geo_adjust(long, lat, n, long_adj, lat_adj,   PI, RAD, i, a) {

	PI = 4.0 * atan2(1, 1)
	RAD = 0.0001

	long_adj[1] = 0
	lat_adj[1] = 0
	if(n > 1){
		a = 2.0*PI/(n-1)
		for(i = 2; i <= n; i++){
			long_adj[i] = RAD * sin(a * (i-2))
			lat_adj[i] = RAD * cos(a * (i-2))
		}
	}
}
function GU_mk_point(file, color, size, long, lat, title, last,   h_color, h_size) {
 	h_color = color != "."
	h_size = size != "."
 	printf("{\n")									> file
 	printf("  \"type\": \"Feature\",\n")						> file
 	printf("  \"geometry\": {")							> file
 	printf("\"type\": \"Point\", ")							> file
 	printf("\"coordinates\": [%.5f, %.5f]", long, lat)				> file
 	printf("},\n")									> file
 	printf("  \"properties\": {\n")							> file
 	printf("    \"title\": \"%s\",\n", GU_str_escape(title))			> file
	if(h_color)
		printf("    \"marker-color\": \"%s\"%s\n", color, h_size ? "," : "")	> file
	if(h_size)
		printf("    \"marker-size\": \"%s\"\n", size)				> file
	printf("  }\n")									> file
	printf("}%s\n", !last ? "," : "")						> file
}
function GU_mk_line(file, lng_1, lat_1, lng_2, lat_2, last) {
	printf("{\n")																> file
	printf("  \"type\": \"Feature\",\n")													> file
	printf("  \"geometry\": {\"type\": \"LineString\", \"coordinates\": [[%.5f, %.5f], [%.5f, %.5f]]}\n", lng_1, lat_1, lng_2, lat_2)	> file
	printf("}%s\n", !last ? "," : "")													> file
}
function GU_gc_dist(lng_1, lat_1, lng_2, lat_2,   R_EARTH, D2R, ph1_1, phi_2, delta_phi, delta_lambda, a, c) {

	R_EARTH = 3.959e3	# mean radius in miles
	D2R = (2*atan2(0, -1))/360.0
	phi_1 = lat_1 * D2R
	phi_2 = lat_2 * D2R
	delta_phi = (lat_2 - lat_1) * D2R
	delta_lambda = (lng_2 - lng_1) * D2R
	a = sin(delta_phi/2.0) * sin(delta_phi/2.0) + cos(phi_1) * cos(phi_2) * sin(delta_lambda/2.0) * sin(delta_lambda/2.0)
	c = 2 * atan2(sqrt(a), sqrt(1.0-a))
	return R_EARTH * c
}
function GU_str_escape(str,   work) {

	work = str
	gsub(/\\/, "\\\\", work)
	gsub(/\"/, "\\\"", work)
	gsub(/\//, "\\/", work)
	return work
}

function GU_pi() {
	return 4*atan2(1, 1)
}
function GU_d2r(d) {
	return (d+0)*GU_pi()/180.0
}
function GU_r2d(r) {
	return (r+0)*180.0/GU_pi()
}
function GU_is_righthanded(count, first, x, y,   area, p, x1, y1, x2, y2) {
	area = 0
	for(p = 0; p < p_count[poly]; p++){
		x1 = x[p_first[poly] + p]
		y1 = y[p_first[poly] + p]
		x2 = x[p_first[poly] + p + 1]
		y2 = y[p_first[poly] + p + 1]
		area += GU_d2r(x2 - x1) * (2.0 + sin(GU_d2r(y1)) + sin(GU_d2r(y2)))
	}
	return area >= 0
}
function GU_pr_header(sc_cfg, meta_data) {

	printf("{\n")
	if(sc_cfg != ""){
		printf("\"scaleConfig\": ")
		for( ; (getline sc_line < sc_cfg) > 0; )
			printf("%s\n", sc_line)
		printf(",\n")
		close(sc_cfg)
	}
	printf("\"geojson\": {\n")
	printf("\"type\": \"FeatureCollection\",\n")
	printf("\"metadata\": {\n")
	printf("  \"createdAt\": \"%s\"%s\n", strftime("%Y%m%dT%H%M%S%Z"), meta_data["count"] > 0 ? "," : "")
	for(i = 1; i <= meta_data["count"]; i++){
		printf("  \"%s\": ", meta_data[i, "key"])
		if(!meta_data[i, "is_str"])
			printf("%s", meta_data[i, "value"])
		else
			printf("\"%s\"", GU_str_escape(meta_data[i, "value"]))
		printf("%s\n", i < meta_data["count"] ? "," : "")
	}
	printf("},\n")
	printf("\"features\": [\n")
}
function GU_pr_trailer() {
	printf("]\n")
	printf("}\n")
	printf("}\n")
}
function GU_find_pgroups(start, count, longs, lats, pg_starts, pg_counts,   n_pgroups, l_geo, geo, i) {

	n_pgroups = 1
	pg_starts[n_pgroups] = start
	pg_counts[n_pgroups] = 1
	l_geo[1] = longs[start]
	l_geo[2] = lats[start]
	for(i = 1; i < count; i++){
		geo[1] = longs[start + i]
		geo[2] = longs[start + i]
		if(GU_geo_equal(geo, l_geo)){
			pg_counts[n_pgroups]++
		}else{
			n_pgroups++
			pg_starts[n_pgroups] = start + i
			pg_counts[n_pgroups] = 1
		}
		l_geo[1] = geo[1]
		l_geo[2] = geo[2]
	}

	return n_pgroups
}
function GU_geo_equal(g1, g2) {
	return g1[1] == g2[1] && g1[2] == g2[2]
}
function GU_geo_isnull(g) {
	return g[1] == "" || g[2] == ""
}
function GU_geo_adjust(long, lat, n, long_adj, lat_adj,   PI, RAD, i, a) {

	PI = 4.0 * atan2(1, 1)
	RAD = 0.0001

	long_adj[1] = 0
	lat_adj[1] = 0
	if(n > 1){
		a = 2.0*PI/(n-1)
		for(i = 2; i <= n; i++){
			long_adj[i] = RAD * sin(a * (i-2))
			lat_adj[i] = RAD * cos(a * (i-2))
		}
	}
}
function GU_mk_point(file, color, size, long, lat, title, last,   h_color, h_size) {
 	h_color = color != "."
	h_size = size != "."
 	printf("{\n")									> file
 	printf("  \"type\": \"Feature\",\n")						> file
 	printf("  \"geometry\": {")							> file
 	printf("\"type\": \"Point\", ")							> file
 	printf("\"coordinates\": [%.5f, %.5f]", long, lat)				> file
 	printf("},\n")									> file
 	printf("  \"properties\": {\n")							> file
 	printf("    \"title\": \"%s\",\n", GU_str_escape(title))			> file
	if(h_color)
		printf("    \"marker-color\": \"%s\"%s\n", color, h_size ? "," : "")	> file
	if(h_size)
		printf("    \"marker-size\": \"%s\"\n", size)				> file
	printf("  }\n")									> file
	printf("}%s\n", !last ? "," : "")						> file
}
function GU_mk_line(file, lng_1, lat_1, lng_2, lat_2, last) {
	printf("{\n")																> file
	printf("  \"type\": \"Feature\",\n")													> file
	printf("  \"geometry\": {\"type\": \"LineString\", \"coordinates\": [[%.5f, %.5f], [%.5f, %.5f]]}\n", lng_1, lat_1, lng_2, lat_2)	> file
	printf("}%s\n", !last ? "," : "")													> file
}
function GU_gc_dist(lng_1, lat_1, lng_2, lat_2,   R_EARTH, D2R, ph1_1, phi_2, delta_phi, delta_lambda, a, c) {

	R_EARTH = 3.959e3	# mean radius in miles
	D2R = (2*atan2(0, -1))/360.0
	phi_1 = lat_1 * D2R
	phi_2 = lat_2 * D2R
	delta_phi = (lat_2 - lat_1) * D2R
	delta_lambda = (lng_2 - lng_1) * D2R
	a = sin(delta_phi/2.0) * sin(delta_phi/2.0) + cos(phi_1) * cos(phi_2) * sin(delta_lambda/2.0) * sin(delta_lambda/2.0)
	c = 2 * atan2(sqrt(a), sqrt(1.0-a))
	return R_EARTH * c
}
function GU_str_escape(str,   work) {

	work = str
	gsub(/\\/, "\\\\", work)
	gsub(/\"/, "\\\"", work)
	gsub(/\//, "\\/", work)
	return work
}


testuite report: minor refactoring

testuite report: minor refactoring

#**************************************************************************
#*                                                                        *
#*                                 OCaml                                  *
#*                                                                        *
#*         Damien Doligez, projet Gallium, INRIA Rocquencourt             *
#*                                                                        *
#*   Copyright 2013 Institut National de Recherche en Informatique et     *
#*     en Automatique.                                                    *
#*                                                                        *
#*   All rights reserved.  This file is distributed under the terms of    *
#*   the GNU Lesser General Public License version 2.1, with the          *
#*   special exception on linking described in the file LICENSE.          *
#*                                                                        *
#**************************************************************************

function check() {
    if (!in_test){
        printf("error at line %d: found test result without test start\n", NR);
        errored = 1;
    }
}

function clear() {
    curfile = "";
    in_test = 0;
}

function record_pass() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "p";
    delete SKIPPED[curdir];
    clear();
}

function record_skip() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "s";
    if (curdir in SKIPPED) SKIPPED[curdir] = 1;
    clear();
}

function record_na() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "n";
    if (curdir in SKIPPED) SKIPPED[curdir] = 1;
    clear();
}

# The output cares only if the test passes at least once so if a test passes,
# but then fails in a re-run triggered by a different test, ignore it.
function record_fail() {
    check();
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "f";
    }
    delete SKIPPED[curdir];
    clear();
}

function record_unexp() {
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "e";
    }
    delete SKIPPED[curdir];
    clear();
}

/Running tests from '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /Running tests from '[^']*'/);
    curdir = substr($0, RSTART+20, RLENGTH - 21);
    # Use SKIPPED[curdir] as a sentinel to detect no output
    SKIPPED[curdir] = 0;
    key = curdir;
    DIRS[key] = key;
    curfile = "";
}

/ ... testing.* ... testing/ {
    printf("error at line %d: found two test results on the same line\n", NR);
    errored = 1;
}

/^ ... testing '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /... testing '[^']*'/);
    curfile = substr($0, RSTART+13, RLENGTH-14);
    if (match($0, /... testing '[^']*' with [^:=]*/)){
        curfile = substr($0, RSTART+12, RLENGTH-12);
    }
    key = sprintf ("%s/%s", curdir, curfile);
    DIRS[key] = curdir;
    in_test = 1;
}

/^ ... testing (with|[^'])/ {
    if (in_test) record_unexp();
    key = curdir;
    DIRS[key] = curdir;
    in_test = 1;
}

/=> passed/ {
    record_pass();
}

/=> skipped/ {
    record_skip();
}

/=> n\/a/ {
    record_na();
}

/=> failed/ {
    record_fail();
}

/=> unexpected error/ {
    record_unexp();
}

/^re-ran / {
    if (in_test){
        printf("error at line %d: found re-ran inside a test\n", NR);
        errored = 1;
    }else{
        RERAN[substr($0, 8, length($0)-7)] += 1;
        ++ reran;
    }
}

END {
    if (errored){
        printf ("\n#### Some fatal error occurred during testing.\n\n");
        exit (3);
    }else{
        if (!retries){
            for (key in SKIPPED){
                if (!SKIPPED[key]){
                    ++ empty;
                    blanks[emptyidx++] = key;
                    delete SKIPPED[key];
                }
            }
            for (key in RESULTS){
                r = RESULTS[key];
                if (r == "p"){
                    ++ passed;
                }else if (r == "f"){
                    ++ failed;
                    fail[failidx++] = key;
                }else if (r == "e"){
                    ++ unexped;
                    unexp[unexpidx++] = key;
                }else if (r == "s"){
                    ++ skipped;
                    curdir = DIRS[key];
                    if (curdir in SKIPPED){
                        if (SKIPPED[curdir]){
                            SKIPPED[curdir] = 0;
                            skips[skipidx++] = curdir;
                        }
                    }else{
                        skips[skipidx++] = key;
                    }
                }else if (r == "n"){
                    ++ ignored;
                }
            }
            printf("\n");
            if (failed != 0){
                printf("\nList of failed tests:\n");
                for (i=0; i < failed; i++) printf("    %s\n", fail[i]);
            }
            if (unexped != 0){
                printf("\nList of unexpected errors:\n");
                for (i=0; i < unexped; i++) printf("    %s\n", unexp[i]);
            }
            if (skipped != 0){
                printf("\nList of skipped tests:\n");
                for (i=0; i < skipidx; i++) printf("    %s\n", skips[i]);
            }
            if (empty != 0){
                printf("\nList of directories returning no results:\n");
                for (i=0; i < empty; i++) printf("    %s\n", blanks[i]);
            }
            printf("\n");
            printf("Summary:\n");
            printf("  %3d tests passed\n", passed);
            printf("  %3d tests skipped\n", skipped);
            printf("  %3d tests failed\n", failed);
            printf("  %3d tests not started (parent test skipped or failed)\n",
                   ignored);
            printf("  %3d unexpected errors\n", unexped);
            printf("  %3d tests considered", nresults);
            if (nresults != passed + skipped + ignored + failed + unexped){
                printf (" (totals don't add up??)");
            }
            printf ("\n");
            if (reran != 0){
                printf("  %3d test dir re-runs\n", reran);
            }
            if (failed || unexped){
                printf("#### Something failed. Exiting with error status.\n\n");
                exit 4;
            }
        }else{
            for (key in RESULTS){
                if (RESULTS[key] == "f" || RESULTS[key] == "e"){
                    key = DIRS[key];
                    if (!(key in RERUNS)){
                        RERUNS[key] = 1;
                        if (RERAN[key] < max_retries){
                            printf("%s\n", key);
                        }
                    }
                }
            }
        }
    }
}

#**************************************************************************
#*                                                                        *
#*                                 OCaml                                  *
#*                                                                        *
#*         Damien Doligez, projet Gallium, INRIA Rocquencourt             *
#*                                                                        *
#*   Copyright 2013 Institut National de Recherche en Informatique et     *
#*     en Automatique.                                                    *
#*                                                                        *
#*   All rights reserved.  This file is distributed under the terms of    *
#*   the GNU Lesser General Public License version 2.1, with the          *
#*   special exception on linking described in the file LICENSE.          *
#*                                                                        *
#**************************************************************************

function check() {
    if (!in_test){
        printf("error at line %d: found test result without test start\n", NR);
        errored = 1;
    }
}

function clear() {
    curfile = "";
    in_test = 0;
}

function record_pass() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "p";
    delete SKIPPED[curdir];
    clear();
}

function record_skip() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "s";
    if (curdir in SKIPPED) SKIPPED[curdir] = 1;
    clear();
}

function record_na() {
    check();
    if (!(key in RESULTS)) ++nresults;
    RESULTS[key] = "n";
    if (curdir in SKIPPED) SKIPPED[curdir] = 1;
    clear();
}

# The output cares only if the test passes at least once so if a test passes,
# but then fails in a re-run triggered by a different test, ignore it.
function record_fail() {
    check();
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "f";
    }
    delete SKIPPED[curdir];
    clear();
}

function record_unexp() {
    if (!(key in RESULTS) || RESULTS[key] == "s"){
        if (!(key in RESULTS)) ++nresults;
        RESULTS[key] = "e";
    }
    delete SKIPPED[curdir];
    clear();
}

/Running tests from '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /Running tests from '[^']*'/);
    curdir = substr($0, RSTART+20, RLENGTH - 21);
    # Use SKIPPED[curdir] as a sentinel to detect no output
    SKIPPED[curdir] = 0;
    key = curdir;
    DIRS[key] = key;
    curfile = "";
}

/ ... testing.* ... testing/ {
    printf("error at line %d: found two test results on the same line\n", NR);
    errored = 1;
}

/^ ... testing '[^']*'/ {
    if (in_test) record_unexp();
    match($0, /... testing '[^']*'/);
    curfile = substr($0, RSTART+13, RLENGTH-14);
    if (match($0, /... testing '[^']*' with [^:=]*/)){
        curfile = substr($0, RSTART+12, RLENGTH-12);
    }
    key = sprintf ("%s/%s", curdir, curfile);
    DIRS[key] = curdir;
    in_test = 1;
}

/^ ... testing (with|[^'])/ {
    if (in_test) record_unexp();
    key = curdir;
    DIRS[key] = curdir;
    in_test = 1;
}

/=> passed/ {
    record_pass();
}

/=> skipped/ {
    record_skip();
}

/=> n\/a/ {
    record_na();
}

/=> failed/ {
    record_fail();
}

/=> unexpected error/ {
    record_unexp();
}

/^re-ran / {
    if (in_test){
        printf("error at line %d: found re-ran inside a test\n", NR);
        errored = 1;
    }else{
        RERAN[substr($0, 8, length($0)-7)] += 1;
        ++ reran;
    }
}

END {
    if (errored){
        printf ("\n#### Some fatal error occurred during testing.\n\n");
        exit (3);
    }else{
        if (!retries){
            for (key in SKIPPED){
                if (!SKIPPED[key]){
                    ++ empty;
                    blanks[emptyidx++] = key;
                    delete SKIPPED[key];
                }
            }
            for (key in RESULTS){
                r = RESULTS[key];
                if (r == "p"){
                    ++ passed;
                }else if (r == "f"){
                    ++ failed;
                    fail[failidx++] = key;
                }else if (r == "e"){
                    ++ unexped;
                    unexp[unexpidx++] = key;
                }else if (r == "s"){
                    ++ skipped;
                    curdir = DIRS[key];
                    if (curdir in SKIPPED){
                        if (SKIPPED[curdir]){
                            SKIPPED[curdir] = 0;
                            skips[skipidx++] = curdir;
                        }
                    }else{
                        skips[skipidx++] = key;
                    }
                }else if (r == "n"){
                    ++ ignored;
                }
            }
            printf("\n");
            if (failed != 0){
                printf("\nList of failed tests:\n");
                for (i=0; i < failed; i++) printf("    %s\n", fail[i]);
            }
            if (unexped != 0){
                printf("\nList of unexpected errors:\n");
                for (i=0; i < unexped; i++) printf("    %s\n", unexp[i]);
            }
            if (skipped != 0){
                printf("\nList of skipped tests:\n");
                for (i=0; i < skipidx; i++) printf("    %s\n", skips[i]);
            }
            if (empty != 0){
                printf("\nList of directories returning no results:\n");
                for (i=0; i < empty; i++) printf("    %s\n", blanks[i]);
            }
            printf("\n");
            printf("Summary:\n");
            printf("  %3d tests passed\n", passed);
            printf("  %3d tests skipped\n", skipped);
            printf("  %3d tests failed\n", failed);
            printf("  %3d tests not started (parent test skipped or failed)\n",
                   ignored);
            printf("  %3d unexpected errors\n", unexped);
            printf("  %3d tests considered", nresults);
            if (nresults == passed + skipped + ignored + failed + unexped){
                printf ("\n");
            }else{
                printf (" (totals don't add up??)\n");
            }
            if (reran != 0){
                printf("  %3d test dir re-runs\n", reran);
            }
            if (failed || unexped){
                printf("#### Something failed. Exiting with error status.\n\n");
                exit 4;
            }
        }else{
            for (key in RESULTS){
                if (RESULTS[key] == "f" || RESULTS[key] == "e"){
                    key = DIRS[key];
                    if (!(key in RERUNS)){
                        RERUNS[key] = 1;
                        if (RERAN[key] < max_retries){
                            printf("%s\n", key);
                        }
                    }
                }
            }
        }
    }
}