Translators/BingTranslator: change to new API (fix #218)

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 17 Mar 2018
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Retrieve the Cookie and set IG.
function bingSetCookie(    content, cookie, group, header, isBody, url) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/translator/api/language/Speak?"                   \
        "locale=" tl "&text=" preprocess(text)                          \
        "&gender=" gender "&media=audio/mp3" " HTTP/1.1\r\n"              \
        "Host: " HttpHost "\r\n"                                          \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n" # must!

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl) {
    return "http://www.microsofttranslator.com/bv.aspx?"        \
        "from=" sl "&to=" tl "&a=" uri
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "translate")
        return HttpPathPrefix "/ttranslate?&IG=" IG "&IID=translator.5032.2"
    else if (type == "translationlookup")
        return HttpPathPrefix "/ttranslationlookup?&IG=" IG "&IID=translator.5032.2"
    else if (type == "transliterate")
        return HttpPathPrefix "/ttransliterate?&IG=" IG "&IID=translator.5032.1"
    else # type == "detect"
        return HttpPathPrefix "/tdetect?&IG=" IG "&IID=translator.5032.2"
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "translate")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "translationlookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "transliterate")
        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "detect"
        return "&text=" quote(text)
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetCookie() # set IG

    # Language identification
    il = postResponse(text, _sl, _tl, _hl, "detect")
    if (!il) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    returnIl[0] = il
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = il
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    } else if (ast[0 SUBSEP "statusCode"] != "200") {
        e("[ERROR] statusCode: " ast[0 SUBSEP "statusCode"])
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "translationResponse"])

    # Generate output
    if (!isVerbose) {
        # Brief mode
        r = translation

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, il, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 18 May 2016
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Retrieve the Cookie needed.
function bingSetCookie(    cookie, group, header, url) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                   \
        "Host: " HttpHost "\r\n"                                          \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0 && length($0) > 1) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)
    Cookie = cookie
}

function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/translator/api/language/Speak?"                   \
        "locale=" tl "&text=" preprocess(text)                          \
        "&gender=" gender "&media=audio/mp3" " HTTP/1.1\r\n"              \
        "Host: " HttpHost "\r\n"                                          \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n" # must!

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl) {
    return "http://www.microsofttranslator.com/bv.aspx?"        \
        "from=" sl "&to=" tl "&a=" uri
}

# Send an HTTP POST request and get response from Bing Translator.
function bingPost(text, sl, tl, hl,
                  ####
                  content, contentLength, group,
                  header, isBody, reqBody, url) {
    reqBody = "[{" parameterize("text") ":" parameterize(text, "\"") "}]"
    if (DumpContentengths[reqBody])
        contentLength = DumpContentengths[reqBody]
    else
        contentLength = DumpContentengths[reqBody] = dump(reqBody, group)

    url = HttpPathPrefix "/translator/api/Translate/TranslateArray?"    \
        "from=" sl "&to=" tl

    header = "POST " url " HTTP/1.1\r\n"                  \
        "Host: " HttpHost "\r\n"                          \
        "Connection: close\r\n"                           \
        "Content-Length: " contentLength "\r\n"           \
        "Content-Type: application/json\r\n"     # must!
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n" # must!

    content = NULLSTR; isBody = 0
    print (header "\r\n" reqBody) |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\r\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    return assert(content, "[ERROR] Null response.")
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = "-"
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    bingSetCookie() # must!

    content = bingPost(text, _sl, _tl, _hl)
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    } else if (ast[0 SUBSEP "Message"]) {
        e("[ERROR] " unparameterize(ast[0 SUBSEP "Message"]))
        e("[ERROR] " unparameterize(ast[0 SUBSEP "Details" SUBSEP 0]))
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "items" SUBSEP 0 SUBSEP "text"])

    returnIl[0] = il = unparameterize(ast[0 SUBSEP "from"])
    if (Option["verbose"] < -1)
        return il
    else if (Option["verbose"] < 0)
        return getList(il)

    # Generate output
    if (!isVerbose) {
        # Brief mode
        r = translation

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        if (wShowOriginal) {
            # Display: original text
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
        }

        if (wShowTranslation) {
            # Display: major translation
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, il, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}


Move to 2018

Move to 2018

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb
#           #1        #2      #3                     #4    #5
#tdbout -t publisher,plugin,auid,publisher:info[tester],status,year *.tdb
#           #1        #2      #3   #4                     #5     #6

BEGIN {
  FS="\t"
  pn = 0
    #str = "date +%m-%d-%Y";
    #str | getline date;
    #close str;
}

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2010 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2018
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2010
  if (length($6) > 3) {
    end_year = substr($6,length($6)-3,4)
  }
  #printf "%s\n", $0
  if (($5 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    #split up the plugin. nn=number of fields. na is the name of the array.
    nn = split($2,na,/\./)
    #save the last element into 1p2.
    lp2 = na[nn]
    p[pn] = $1        #publisher
    n[pn] = lp2       #plugin
    auid[pn] = $3     #auid
    status[pn] = $5   #status
    r[pn] = $4        #tester
    if (r[pn] == "") {
      if ((substr(p[pn],1,2) <= "Mz")) {
        r[pn] = "5"
      } else if (substr(p[pn],1,2) >= "NA") {
        r[pn] = "8"
      }
    }
    pn++
    x[$5]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tAUid\tT\tTotal\tStatus"
  #for (j = 0 ; j < scn ; j++) {
  #  if (x[s[j]] > 0) {
  #  printf "\t%s", sc[j]
  #  }
  #}
  printf "\n"

  #print out publisher, plugin, auid, tester, status
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t1", p[i], n[i], auid[i], r[i], status[i]
    #for (j = 0 ; j < sn ; j++) {
    #  if (x[s[j]] > 0){
#   #   if (status[i] == 0) {
    #  printf "\t.."
    #} else {
    #    printf "1"
    #  }
    #}
#    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tAUid\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  #printf "%s", date
  #printf "\n"
}

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb
#           #1        #2      #3                     #4    #5
#tdbout -t publisher,plugin,auid,publisher:info[tester],status,year *.tdb
#           #1        #2      #3   #4                     #5     #6

BEGIN {
  FS="\t"
  pn = 0
    #str = "date +%m-%d-%Y";
    #str | getline date;
    #close str;
}

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2010 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2017
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2010
  if (length($6) > 3) {
    end_year = substr($6,length($6)-3,4)
  }
  #printf "%s\n", $0
  if (($5 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    #split up the plugin. nn=number of fields. na is the name of the array.
    nn = split($2,na,/\./)
    #save the last element into 1p2.
    lp2 = na[nn]
    p[pn] = $1        #publisher
    n[pn] = lp2       #plugin
    auid[pn] = $3     #auid
    status[pn] = $5   #status
    r[pn] = $4        #tester
    if (r[pn] == "") {
      if ((substr(p[pn],1,2) <= "Mz")) {
        r[pn] = "5"
      } else if (substr(p[pn],1,2) >= "NA") {
        r[pn] = "8"
      }
    }
    pn++
    x[$5]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tAUid\tT\tTotal\tStatus"
  #for (j = 0 ; j < scn ; j++) {
  #  if (x[s[j]] > 0) {
  #  printf "\t%s", sc[j]
  #  }
  #}
  printf "\n"

  #print out publisher, plugin, auid, tester, status
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t1", p[i], n[i], auid[i], r[i], status[i]
    #for (j = 0 ; j < sn ; j++) {
    #  if (x[s[j]] > 0){
#   #   if (status[i] == 0) {
    #  printf "\t.."
    #} else {
    #    printf "1"
    #  }
    #}
#    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tAUid\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  #printf "%s", date
  #printf "\n"
}


SpecialCase "A Midsummer Night's Dream"

SpecialCase "A Midsummer Night's Dream"

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

BEGIN {
    # Print header
    printf ("Sortkey\tContent Type\tContent ID\tTitle\tEntity ID\tGenre\tRating\t")
    printf ("Show Type\tDuration\tDate Type\tOriginal Date\tShow ID\tSeason ID\t")
    printf ("Sn #\tEp #\tDescription\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType
    # print "contentId = " contentId
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    # print "dateType = " dateType
    # print "originalDate = " originalDate
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "==> EntityId = " EntityId
    }

    if (title == "Porridge") {
        if (EntityId == "_9509")
            title = "Porridge (1974-1977)"
        else if (EntityId == "_14747")
            title = "Porridge (2016-2017)"
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179")
            title = "A Midsummer Night's Dream (1981)"
        else if (EntityId == "_15999")
            title = "A Midsummer Night's Dream (2016)"
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "==> EntityId = " EntityId
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) %s S%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) %s S%02d", showTitle, originalDate, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) %s S%02dE%03d", showTitle, originalDate, seasonNumber, episodeNumber)
    }

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", sortkey, contentType, \
            contentId, title, EntityId, genre, rating, showType, duration, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber, description)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

BEGIN {
    # Print header
    printf ("Sortkey\tContent Type\tContent ID\tTitle\tEntity ID\tGenre\tRating\t")
    printf ("Show Type\tDuration\tDate Type\tOriginal Date\tShow ID\tSeason ID\t")
    printf ("Sn #\tEp #\tDescription\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType
    # print "contentId = " contentId
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    # print "dateType = " dateType
    # print "originalDate = " originalDate
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "==> EntityId = " EntityId
    }

    if (title == "Porridge") {
        if (EntityId == "_9509")
            title = "Porridge (1974-1977)"
        else if (EntityId == "_14747")
            title = "Porridge (2016-2017)"
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "==> EntityId = " EntityId
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) %s S%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) %s S%02d", showTitle, originalDate, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) %s S%02dE%03d", showTitle, originalDate, seasonNumber, episodeNumber)
    }

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", sortkey, contentType, \
            contentId, title, EntityId, genre, rating, showType, duration, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber, description)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}


Added a shebang to `wrap.awk`

Added a shebang to `wrap.awk`

#! /usr/bin/awk -f
#
# Like the Un*x `fmt` command, but for markdown.
#
# https://github.com/wernsey/d.awk
#
# It does its best to preserve markdown headings,
# lists, pre-formatted code blocks and block quotes.
#
# (c) 2016 Werner Stoop
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved. This file is offered as-is,
# without any warranty.

BEGIN { if(!Width) Width = 80; }

# Preserve headings
/^[[:space:]]*[=\-][=\-][=\-]+/ {
    if(Buf)
        Out = Out Buf "\n" $0 "\n";
    else
        Out = Out $0 "\n";
    Buf = "";
    next;
}

# Preformatted text is sent to the output verbatim
# Ditto for links/abbreviations
/^(    |\t)+/ && match(last,/^[[:space:]]*$/) || /^[[:space:]]*\*?\[.*\]:/ {
    Out = Out $0 "\n";
    next;
}

# GitHub-style ``` code blocks:
/^```/ {
    Out = Out Buf "\n" $0 "\n";
    Buf = "";
    Code = !Code;
    next;
}
Code {
    Out = Out $0 "\n";
    next;
}

$0 !~ /^[[:space:]]*$/ && match(last,/^[[:space:]]*$/) {
    # how much is the current line indented
    match(str, /^[[:space:]]+/);
    Indent = substr(str, 1, RLENGTH);
}

# Blank lines cause blank lines in the output
/^[[:space:]]*$/ {
    # You need to preserve $0 for blank lines in preformatted blocks
    if(Buf)
        Out = Out Buf "\n" $0 "\n";
    else
        Out = Out "\n" $0;
    Buf = "";
    last = $0;
    InList = 0;
    next;
}

# Handle every other input line
{ last = $0; fmt($0); }

# Write output when done
END {
    Out = Out Buf;
    print Out;
}

function fmt(str,          loc,word,n,indent) {

    # Get the current line's indentation level.
    match(str, /^[[:space:]]+/);
    indent = substr(str, 1, RLENGTH);

    # Trim leading whitespace
    str = substr(str, RLENGTH+1);
    gsub(/\r/, "", str); # Windows :(

    # Lines starting with list item characters
    # force a line break in the output
    if(match(str,/^([*+\-]|[[:digit:]]+\.)/)) {
        if(Buf) Out = Out Buf "\n";
        Buf = "";
        # Preserve the indentation in the global Indent
        # if it is a list that is going to be split.
        Indent = indent;
        InList = 1; # remember we're in a list on subsequent calls.
    } else if(match(str,/^>[[:space:]]+/)) {
        Indent = indent substr(str, 1, RLENGTH);
        str = substr(str,RLENGTH+1);
    }

    # Current indentation level = global Indent
    indent = Indent;

    # This implements the simple algorithm from the wikipedia
    # https://en.wikipedia.org/wiki/Line_wrap_and_word_wrap
    # There is a better way. See #Minimum_raggedness on that wiki page.
    # The C code example on https://www.rosettacode.org/wiki/Word_wrap#C
    # may actually be easy to port to Awk (the Awk version on that page
    # implements the greedy algorithm, like I do here).

    loc = match(str, /[[:space:]]+/);
    while(loc) {
        word = substr(str, 1, RSTART-1);
        n = RSTART+RLENGTH;

        # Handle forced line breaks
        if(match(str,/(  |[[:space:]]+\\)$/) == loc) {
            if(length(Buf) + length(str) + 1 >= Width)
                Out = Out Buf "\n" indent str "\n" indent;
            else
                Out = Out Buf " " str "\n" indent;
            Buf = "";
            return;
        }

        # If the buffer + the word exceeds the allowed width
        # then insert a line break. Otherwise, just append the
        # word to the buffer.
        # Also, preserve the indentation.
        if(length(Buf) + length(word) + 1 >= Width) {
            Out = Out Buf "\n";
            if(InList) indent = Indent "  ";
            Buf = indent word;
        } else if(length(Buf))
            Buf = Buf " " word;
        else
            Buf = indent word;

        str = substr(str, n);
        loc = match(str, /[[:space:]]+/);
    }

    # Append the remainder of str to Buf
    if(length(str)) {
        if(length(Buf) + length(str) + 1 >= Width) {
            Out = Out Buf "\n";
            if(InList) indent = Indent "  ";
            Buf = indent str;
        } else if(length(Buf))
            Buf = Buf " " str;
        else
            Buf = indent str;
    }
}


A better lexicon extraction script

A better lexicon extraction script

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    for (i = 1; i <= split($3, words, ", "); i++) {
        print words[i] | "sort --parallel=$(nproc) -S1G -us";
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = OFS = "\t";
}
{
    for (i = 1; i <= split($3, words, ", "); i++) {
        gsub("_", " ", words[i]);
        print words[i] | "sort -us";
    }
}


Fix awk script to work with cairo generated postscript

BEGIN{doit = 0}
 { if (doit) print $0 }
 $0 = /%%End.*Setup/ { doit = 1 }

BEGIN{doit = 0}
 { if (doit) print $0 }
 $0 == "%%EndSetup" { doit = 1 }


include straight http too

include straight http too

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology they are from. (curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed local identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri,  delim, char, max, c, l){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_*|_*$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri,  u){
	u = uri
	# shorten till longest uri in curi map is found (or not)

	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123

function deoboify(curie,  a){
	split(curie, a , ":")
	if(match(a[2], a[1]"_") > 0){
		sub("_", ":", a[2]);
		curie=a[2]
	}
	return curie
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	prefix["https://archive.monarchinitiative.org/"]="MonarchArchive"
	############################################################
	# not expected to be added to curie map
	# in HPOA
	prefix["http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part="]="NCBIBook"
	prefix["http://www.ncbi.nlm.nih.gov/books/NBK"]="NCBINBK"

	############################################################
	# Often re-visit whether these exceptions are still necessary
	# they may have been moved into the curie prefix mapping file
	# or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in IMPC (not httpS  --- hmm all three IRI exist)
	prefix["http://www.mousephenotype.org"]="IMPC"
	# note in curie_map IMPC is:
	# http://www.mousephenotype.org/data/genes/

	# in panther
	# prefix["http://identifiers.org/wormbase/"]="WormBase"
	# note in curie_map WormBase is
	# 'https://www.wormbase.org/get?name='

	# various, transient?
    prefix["http://www.ncbi.nlm.nih.gov/gene/"]="NCBIGene"
    prefix["http://www.ncbi.nlm.nih.gov/genome/"]="NCBIGenome"
    prefix["http://www.ncbi.nlm.nih.gov/assembly?term="]="NCBIAssembly"
	# kegg
	prefix["http://www.genome.jp/kegg/pathway/map/map"]="KEGG-img"
	# in EOM
	prefix["https://elementsofmorphology.nih.gov/images/terms/"]="EOM_IMG"
	prefix["http://elementsofmorphology.nih.gov/index.cgi?tid="]="EOM"  # w/o httpS

	# playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
	# and their IRI are unroutable, i.e in same machine (or local net)
	# this is motivated in part by not wanting to publish
	# urls to our own servers which go nowhere.
	prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
	prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
	prefix["http://0.0.0.0/"]="LCL"	                       # ID halfway house

	# LCL's obvious down side is zero options for global collisions
	# best you can do is go the uuid which is ugly, or
	# belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*': 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology they are from. (curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed local identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri,  delim, char, max, c, l){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:|^https:\/\/monarchinitiative.org\/.well-known\/genid\//))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_*|_*$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri,  u){
	u = uri
	# shorten till longest uri in curi map is found (or not)

	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123

function deoboify(curie,  a){
	split(curie, a , ":")
	if(match(a[2], a[1]"_") > 0){
		sub("_", ":", a[2]);
		curie=a[2]
	}
	return curie
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	prefix["https://archive.monarchinitiative.org/"]="MonarchArchive"
	############################################################
	# not expected to be added to curie map
	# in HPOA
	prefix["http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part="]="NCBIBook"
	prefix["http://www.ncbi.nlm.nih.gov/books/NBK"]="NCBINBK"

	############################################################
	# Often re-visit whether these exceptions are still necessary
	# they may have been moved into the curie prefix mapping file
	# or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in IMPC (not httpS  --- hmm all three IRI exist)
	prefix["http://www.mousephenotype.org"]="IMPC"
	# note in curie_map IMPC is:
	# http://www.mousephenotype.org/data/genes/

	# in panther
	# prefix["http://identifiers.org/wormbase/"]="WormBase"
	# note in curie_map WormBase is
	# 'https://www.wormbase.org/get?name='

	# various, transient?
    prefix["http://www.ncbi.nlm.nih.gov/gene/"]="NCBIGene"
    prefix["http://www.ncbi.nlm.nih.gov/genome/"]="NCBIGenome"
    prefix["http://www.ncbi.nlm.nih.gov/assembly?term="]="NCBIAssembly"
	# kegg
	prefix["http://www.genome.jp/kegg/pathway/map/map"]="KEGG-img"
	# in EOM
	prefix["https://elementsofmorphology.nih.gov/images/terms/"]="EOM_IMG"

	# playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
	# and their IRI are unroutable, i.e in same machine (or local net)
	# this is motivated in part by not wanting to publish
	# urls to our own servers which go nowhere.
	prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
	prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
	prefix["http://0.0.0.0/"]="LCL"	                       # ID halfway house

	# LCL's obvious down side is zero options for global collisions
	# best you can do is go the uuid which is ugly, or
	# belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*': 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}