Clarify UTF-8 messages

Clarify UTF-8 messages

# Produce a raw data spreadsheet from URLs found in https://watch.mhzchoice.com/sitemap.xml

# INVOCATION:
#    awk -v ERRORS=$ERRORS -v RAW_CREDITS=$RAW_CREDITS -v RAW_TITLES=$RAW_TITLES \
#        -f getMHzFromSitemap.awk >>$UNSORTED

# Field numbers
# 1 Title  2 Seasons  3 Episodes  4 Duration  5 Genre  6 Country  7 Language  8 Rating  9 Description

### Begin season processing

# Determine whether this is a page 2
#    <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://mhzchoice.vhx.tv/detective-montalbano?page=2">
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
    next
}

# Season Title & Season URL
#    <form class="form"><input value="https://watch.mhzchoice.com/gasmamman/season:1" type="text" \
#        class="text text-center border-none site-background-color site-font-secondary-color" /></form>
# This returns incorrect values if page2 == "yes", but those values are never used
/<form class="form">/ {
    split ($0,fld,"\"")
    seasonURL = fld[4]
    shortSeasonURL = seasonURL
    sub (/.*watch/,"watch",shortSeasonURL)
    split (seasonURL,fld,":")
    seasonNumber = fld[3]
    seasonTitle = "Season " seasonNumber
    # print "==> seasonURL = " seasonURL > "/dev/stderr"
    # print "==> seasonNumber = " seasonNumber > "/dev/stderr"
    # print "==> seasonTitle = " seasonTitle > "/dev/stderr"
    next
}

# Season episodes
#

#      9 Episodes
#

# Extract only the number of episodes, # but if this is page 2 skip to avoid double counting
/<h2 class=.*content-label/,/<\/h2>/ {
    sub (/^ */,"")
    if ($0 ~ / Episode/) {
        seasonEpisodes = $1
        next
    }
}

### Begin show processing

# Show Title
#    <title>Gasmamman - MHz Choice</title>
/<title>/ {
    split ($0,fld,"[<>]")
    showTitle = fld[3]
    sub (/ - MHz Choice/,"",showTitle)
    gsub (/'/,"'",showTitle)
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
    print showTitle >> RAW_TITLES
    # print "==> showTitle = " showTitle > "/dev/stderr"
    next
}

# Show Descriptor
#    <meta name="description" content="DRAMA - CRIME | SWEDEN | SWEDISH WITH ENGLISH SUBTITLES | TV-14
#    Alexandra Rapaport stars as a suburban mom pulled into the criminal \
#        underworld to pay off her murdered husband’s debts.">
#
# There should be four header fields prior to the description
/<meta name="description" content="/,/">/ {
     # if we're on the first line of this block ...
    if ($0 ~ /name="description" content="/) {
        headerAdded = "no"
        sub (/.*name="description" content="/,"")
    }
    # If we find a header, clean it up and put it before the description
    if ($0 ~ /\| {1,2}TV-/) {
        sub (/WITH ENGLISH SUBTITLES /,"")
        sub (/SCANDINAVIAN CRIME FICTION/,"Sweden")
        sub (/NONFICTION - DOCUMENTARY/,"Documentary")
        sub (/NON-FICTION - DOCUMENTARY/,"Documentary")
        gsub (/ \| {1,2}/,"|")
        sub (/\r/,"")
        # Split out header fields
        numFields = split ($0,fld,"\|")
        # lowercase everything but the first character in all fields but the last
        for (i = 1; i < numFields; ++i) {
            fld[i] = substr(fld[i],1,1) tolower(substr(fld[i],2))
            # uppercase the first character of any second word
            if (match (fld[i],/ /)) {
                fld[i] = substr(fld[i],1,RSTART) toupper(substr(fld[i],RSTART+1,1)) (substr(fld[i],RSTART+2))
            }
        }
        # Put the finalized header before the description
        if (numFields > 3 ) {
            sub (/Drama-crime/,"Drama - crime",fld[1])
            gsub (/ /,"",fld[4])
            showDescriptor =  fld[1] "\t" fld[2] "\t" fld[(numFields-1)] "\t" fld[(numFields)] "\t"
            headerAdded = "yes"
        }
    } else {
        # if we didn't find a header in this block, add a blank one
        if (headerAdded == "no") {
            printf ("==> Added blank header for \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = "\t\t\t\t"
        }
        # We found a description, clean it up and add it
        gsub (/  */," ")
        gsub (/'/,"'")
        gsub (/"/,"\"")
        gsub (/&/,"\\&")
        gsub (/‘/,"’")
        gsub (/’/,"’")
        gsub (/“/,"“")
        gsub (/”/,"”")
        gsub (/\r/," ")
        # if it's not the last line of the description, add it
        if ($0 !~ /"\>/) {
            printf ("==> Multi line description in \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = showDescriptor $0
        } else {
            # if it's the last line of the description, add it
            sub (/"\>/,"")
            showDescriptor = showDescriptor $0
        }
    }
    # print "==> showDescriptor = " showDescriptor > "/dev/stderr"
    next
}

# Show URL
#    <meta property="og:url" content="https://watch.mhzchoice.com/gasmamman">
/<meta property="og:url" content=/ {
    split ($0,fld,"\"")
    showURL = fld[4]
    # print "==> showURL = " showURL > "/dev/stderr"
    next
}

# Show seasons
#

#        3 Seasons
#

# Extract only the number of seasons, # but if this is page 2 skip to avoid double counting
/<h2 class=.*collection-stats"/,/<\/h2>/ {
    sub (/^ */,"")
    # Only if there are two or more seasons, a show with only one season doesn't have this
    if ($0 ~ / Season/) {
        showSeasons = $1
        next
    }
}

### Begin episode processing

# Episode URL(s)
#

#
/
/,/<a href="https:/ {
    if ($0 ~ / episodeURL = " episodeURL > "/dev/stderr"
        next
    }
}

# Episode Duration(s)
#
44:15

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    # print "==> episodeDuration = " episodeDuration > "/dev/stderr"
    next
}

# Episode Title(s)
#
 \
#        Gåsmamman: Episode 01 (Sn 1 Ep 1)

/<h3 class="tooltip-item-title/{
    split ($0,fld,"[<>]")
    episodeTitle = fld[5]
    gsub (/'/,"'",episodeTitle)
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/^[[:space:]]/,"",episodeTitle)
    sub (/[[:space:]]+$/,"",episodeTitle)
    # If start of episodeTitle == showTitle followed by ": " or " - ", remove the redundant part.
    if (match (episodeTitle, showTitle ": ") == 1 || \
        (match (episodeTitle, showTitle " - ") == 1)) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    #
    # If there is a trailing (Sn 1 Ep 1), remove it
    # Handle normal (Sn 1 Ep 1) with variations in spacing and capitalization
    # and ones missing the second letter (Sn 1 E 1), (S1 E1), or wrong second letter (Sm 1 Ep 1)
    if (match (episodeTitle,\
        /[ ]*\(S[Nnm]*[ ]*[[:digit:]]{1,2}[ ]+[Ee][Pp]*[ ]*[[:digit:]]{1,3}[[:space:]]*\)/))  {
            if (episodeTitle !~ / \(Sn [[:digit:]]{1,2} Ep [[:digit:]]{1,3}\)/)
                printf ("==> Malformed Sn/Ep in \"%s: %s\"\n", showTitle, episodeTitle) >> ERRORS
            sub (/[ ]*\(S[Nnm]*[ ]*[[:digit:]]{1,2}[ ]+[Ee][Pp]*[ ]*[[:digit:]]{1,3}[[:space:]]*\)/,\
                "",episodeTitle)
            # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    }
    #
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeTitle ~ /BONUS|Montalbano and Me/)
        episodeType = "X"
    # If episode is a Trailer (i.e. First look), set episodeType to "T"
    if (episodeTitle ~ /^PR \|/) {
        episodeType = "T"
        seasonEpisodes = seasonEpisodes - 1
    }
    # print "==> episodeType = " episodeType > "/dev/stderr"
    next
}

# Episode Number(s)
#

#         Episode 1
#

/
<span class='media-identifier/,/<\/div>/ {
    sub (/^ */,"")
    if ($0 ~ /Episode /) {
        episodeNumber = $2
        if (showTitle == "Detective Montalbano" && page2 == "yes") {
            oldEpisodeNumber = episodeNumber
            episodeNumber += 24
            printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                    episodeNumber, shortEpisodeURL) >> ERRORS
        }
        # print "==> episodeNumber = " episodeNumber > "/dev/stderr"
        next
    }

### Wrap-up episode processing when Episode Description is found
### print only on LONG_SPREADSHEET
    # print $0 > "/dev/stderr"

    # Episode Description(s)
    #

    #
Preparing for her sister's wedding, Sonja's idyllic life is shattered by tragedy.
    #

    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        if (paragraph != "")
            episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
        gsub (/&/,"\\&",episodeDescription)
        gsub (/'/,"'",episodeDescription)
        # print "descriptionLinesFound = " descriptionLinesFound > "/dev/stderr"
        # print "==> episodeDescription from
 = \n" episodeDescription > "/dev/stderr"
        next
    }
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1/videos/gasmamman-episode-01-sn-1-ep-1-1";\
    #  "Gasmamman, S01E01, Gåsmamman: Episode 01 (Sn 1 Ep 1)")
    if ($0 ~ /<\/div>/) {
        # Make sure episodeDescription is valid
        if (episodeDescription == "") {
            if (episodeType == "T") {
                episodeDescription = episodeTitle
                # print "==> episodeDescription from title = " episodeDescription > "/dev/stderr"
            } else {
                print "==> No description: " shortEpisodeURL >> ERRORS
            }
        }
        if (episodeDescription ~ /^PR \|/)
            sub (/^PR \| /,"",episodeDescription)
        # print "==> episodeDescription = \n" episodeDescription > "/dev/stderr"
        episodeLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)", episodeURL, showTitle,
                    seasonNumber, episodeType, episodeNumber, episodeTitle)
        #
        # Make sure episodeDuration is valid
        if (split (episodeDuration, tm, ":") == 3) {
            # Canonicalize episodeDuration to 3 parts like other services
            episodeDuration = sprintf ("%02d:%02d:%02d",tm[1],tm[2],tm[3])
        } else {
            printf ("==> Bad episodeDuration %s in \"%s: %s\"\n", episodeDuration, showTitle,
                    episodeTitle) >> ERRORS
            episodeDuration = ""
        }
        #
        # Print "episode" line
        printf ("%s\t\t\t%s\t\t\t\t\t%s\n", episodeLink, episodeDuration, episodeDescription)
        #
        # Look for directors
        director_name = episodeDescription
        if (gsub (/\xc2\xa0/," ",director_name)) {
            printf ("==> UTF-8 non-breaking-space in %s, S%02d%s%02d, %s\n", showTitle, seasonNumber,
                    episodeType, episodeNumber, episodeTitle) >> ERRORS
            print "    " episodeDescription >> ERRORS
        }
        if (director_name ~ /[Dd]irected by /) {
            sub (/.*[Dd]irected by /,"",director_name)
            sub (/,.*$/,"",director_name)
            sub (/ [[:digit:]]{4}\./,"",director_name)
            sub (/[[:space:]]+$/,"",director_name)
            sub (/\.$/,"",director_name)
            # Special cases
            if (director_name ~ /Manetti Bros/)
                director_name = "The Manetti Bros."
            if (director_name ~ /Andrea and Antonio Frazzi/)
                director_name = "Andrea Frazzi and Antonio Frazzi"
            #
            if (match (director_name, " and ") || match (director_name, " & ")) {
                dname = substr(director_name,1,RSTART-1)
                director_name = substr(director_name,RSTART+RLENGTH)
                printf ("%s\tdirector\ttv_show\t%s\n", dname, showTitle) >> RAW_CREDITS
            }
            printf ("%s\tdirector\ttv_show\t%s\n", director_name, showTitle) >> RAW_CREDITS
        }
        # Make sure there is no carryover
        descriptionLinesFound = 0
        episodeLink = ""
        episodeURL =""
        shortEpisodeURL = ""
        episodeType = ""
        episodeNumber = ""
        episodeTitle = ""
        episodeDuration = ""
        episodeDescription =""
    }
}

### Wrap-up show processing - print on standard out

# Footer at end of file
#
/ "/dev/stderr"
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman";"Gasmamman")
    showLink = "=HYPERLINK(\"" showURL "\";\"" showTitle "\")"
    #
    # Shows with only one season have a blank showSeasons
    if (showSeasons == "")
        showSeasons = 1
    # in most cases, showSeasons = the final season number, i.e.
    # a show with two seasons would have seasons 1 and 2
    finalSeason = showSeasons
    # Special case any discontinuous seasons, e.g. 2 seasons, but numbered 1 and 3
    if (showURL ~ /\/wallander$/)
        finalSeason = 3
    #
    # Wrap show, i.e. but only on final season in order to prevent multiples
    # print "==> " showURL " - seasonNumber = " seasonNumber  > "/dev/stderr"
    # print "==> " showTitle " - finalSeason = " finalSeason  > "/dev/stderr"
    if (seasonNumber > finalSeason ) {
        # This should be OK, as printing any single valid season will work - however ...
        printf ("==> Season number %d beyond final season %d: %s\n", seasonNumber, finalSeason,
                shortSeasonURL) >> ERRORS
    }
    if (seasonNumber == finalSeason ) {
        # Print "show" line
        printf ("%s\t%s\t\t\t%s\n", showLink, showSeasons, showDescriptor)
    }
    #
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1";"Gasmamman, S01, Season 1")
    if (page2 != "yes") {
        seasonLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d, %s\"\)", seasonURL, showTitle, seasonNumber,
                   seasonTitle)
        # print "==> seasonURL = " seasonURL  > "/dev/stderr"
        # print "==> seasonLink = " seasonLink  > "/dev/stderr"
        # Don't print redundant showDescriptor for season 1
        if (seasonNumber == 1)
            showDescriptor = ""
        # Print "season" line
        printf ("%s\t\t%s\t\t%s\n", seasonLink, seasonEpisodes, showDescriptor)
    }
    #
    # Make sure there is no carryover
    showLink = ""
    showURL = ""
    showTitle = ""
    showSeasons = ""
    showDescriptor = ""
    # print "" > "/dev/stderr"
    #
    page2 = ""
    seasonLink = ""
    seasonURL = ""
    shortSeasonURL = ""
    seasonNumber = ""
    seasonTitle = ""
    seasonEpisodes = ""
}

# Produce a raw data spreadsheet from URLs found in https://watch.mhzchoice.com/sitemap.xml

# INVOCATION:
#    awk -v ERRORS=$ERRORS -v RAW_CREDITS=$RAW_CREDITS -v RAW_TITLES=$RAW_TITLES \
#        -f getMHzFromSitemap.awk >>$UNSORTED

# Field numbers
# 1 Title  2 Seasons  3 Episodes  4 Duration  5 Genre  6 Country  7 Language  8 Rating  9 Description

### Begin season processing

# Determine whether this is a page 2
#    <meta name="apple-itunes-app" content="app-id=1096194231, \
#        app-argument=https://mhzchoice.vhx.tv/detective-montalbano?page=2">
/<meta name="apple-itunes-app" content=".*?page=2">/ {
    page2 = "yes"
    next
}

# Season Title & Season URL
#    <form class="form"><input value="https://watch.mhzchoice.com/gasmamman/season:1" type="text" \
#        class="text text-center border-none site-background-color site-font-secondary-color" /></form>
# This returns incorrect values if page2 == "yes", but those values are never used
/<form class="form">/ {
    split ($0,fld,"\"")
    seasonURL = fld[4]
    shortSeasonURL = seasonURL
    sub (/.*watch/,"watch",shortSeasonURL)
    split (seasonURL,fld,":")
    seasonNumber = fld[3]
    seasonTitle = "Season " seasonNumber
    # print "==> seasonURL = " seasonURL > "/dev/stderr"
    # print "==> seasonNumber = " seasonNumber > "/dev/stderr"
    # print "==> seasonTitle = " seasonTitle > "/dev/stderr"
    next
}

# Season episodes
#

#      9 Episodes
#

# Extract only the number of episodes, # but if this is page 2 skip to avoid double counting
/<h2 class=.*content-label/,/<\/h2>/ {
    sub (/^ */,"")
    if ($0 ~ / Episode/) {
        seasonEpisodes = $1
        next
    }
}

### Begin show processing

# Show Title
#    <title>Gasmamman - MHz Choice</title>
/<title>/ {
    split ($0,fld,"[<>]")
    showTitle = fld[3]
    sub (/ - MHz Choice/,"",showTitle)
    gsub (/'/,"'",showTitle)
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
    print showTitle >> RAW_TITLES
    # print "==> showTitle = " showTitle > "/dev/stderr"
    next
}

# Show Descriptor
#    <meta name="description" content="DRAMA - CRIME | SWEDEN | SWEDISH WITH ENGLISH SUBTITLES | TV-14
#    Alexandra Rapaport stars as a suburban mom pulled into the criminal \
#        underworld to pay off her murdered husband’s debts.">
#
# There should be four header fields prior to the description
/<meta name="description" content="/,/">/ {
     # if we're on the first line of this block ...
    if ($0 ~ /name="description" content="/) {
        headerAdded = "no"
        sub (/.*name="description" content="/,"")
    }
    # If we find a header, clean it up and put it before the description
    if ($0 ~ /\| {1,2}TV-/) {
        sub (/WITH ENGLISH SUBTITLES /,"")
        sub (/SCANDINAVIAN CRIME FICTION/,"Sweden")
        sub (/NONFICTION - DOCUMENTARY/,"Documentary")
        sub (/NON-FICTION - DOCUMENTARY/,"Documentary")
        gsub (/ \| {1,2}/,"|")
        sub (/\r/,"")
        # Split out header fields
        numFields = split ($0,fld,"\|")
        # lowercase everything but the first character in all fields but the last
        for (i = 1; i < numFields; ++i) {
            fld[i] = substr(fld[i],1,1) tolower(substr(fld[i],2))
            # uppercase the first character of any second word
            if (match (fld[i],/ /)) {
                fld[i] = substr(fld[i],1,RSTART) toupper(substr(fld[i],RSTART+1,1)) (substr(fld[i],RSTART+2))
            }
        }
        # Put the finalized header before the description
        if (numFields > 3 ) {
            sub (/Drama-crime/,"Drama - crime",fld[1])
            gsub (/ /,"",fld[4])
            showDescriptor =  fld[1] "\t" fld[2] "\t" fld[(numFields-1)] "\t" fld[(numFields)] "\t"
            headerAdded = "yes"
        }
    } else {
        # if we didn't find a header in this block, add a blank one
        if (headerAdded == "no") {
            printf ("==> Added blank header for \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = "\t\t\t\t"
        }
        # We found a description, clean it up and add it
        gsub (/  */," ")
        gsub (/'/,"'")
        gsub (/"/,"\"")
        gsub (/&/,"\\&")
        gsub (/‘/,"’")
        gsub (/’/,"’")
        gsub (/“/,"“")
        gsub (/”/,"”")
        gsub (/\r/," ")
        # if it's not the last line of the description, add it
        if ($0 !~ /"\>/) {
            printf ("==> Multi line description in \"%s\"\n", showTitle) >> ERRORS
            showDescriptor = showDescriptor $0
        } else {
            # if it's the last line of the description, add it
            sub (/"\>/,"")
            showDescriptor = showDescriptor $0
        }
    }
    # print "==> showDescriptor = " showDescriptor > "/dev/stderr"
    next
}

# Show URL
#    <meta property="og:url" content="https://watch.mhzchoice.com/gasmamman">
/<meta property="og:url" content=/ {
    split ($0,fld,"\"")
    showURL = fld[4]
    # print "==> showURL = " showURL > "/dev/stderr"
    next
}

# Show seasons
#

#        3 Seasons
#

# Extract only the number of seasons, # but if this is page 2 skip to avoid double counting
/<h2 class=.*collection-stats"/,/<\/h2>/ {
    sub (/^ */,"")
    # Only if there are two or more seasons, a show with only one season doesn't have this
    if ($0 ~ / Season/) {
        showSeasons = $1
        next
    }
}

### Begin episode processing

# Episode URL(s)
#

#
/
/,/<a href="https:/ {
    if ($0 ~ / episodeURL = " episodeURL > "/dev/stderr"
        next
    }
}

# Episode Duration(s)
#
44:15

# Extract the duration
/<div class="duration-container/ {
    split ($0,fld,"[<>]")
    episodeDuration = fld[3]
    gsub (/ /,"",episodeDuration)
    # Spreadsheets decipher 2 part durations as time-of-day so make sure they're 3 parts
    if (split (episodeDuration, tm, ":") == 2)
        episodeDuration = "00:" episodeDuration
    # print "==> episodeDuration = " episodeDuration > "/dev/stderr"
    next
}

# Episode Title(s)
#
 \
#        Gåsmamman: Episode 01 (Sn 1 Ep 1)

/<h3 class="tooltip-item-title/{
    split ($0,fld,"[<>]")
    episodeTitle = fld[5]
    gsub (/'/,"'",episodeTitle)
    gsub (/"/,"\"\"",episodeTitle)
    gsub (/&/,"\\&",episodeTitle)
    sub (/^[[:space:]]/,"",episodeTitle)
    sub (/[[:space:]]+$/,"",episodeTitle)
    # If start of episodeTitle == showTitle followed by ": " or " - ", remove the redundant part.
    if (match (episodeTitle, showTitle ": ") == 1 || \
        (match (episodeTitle, showTitle " - ") == 1)) {
        episodeTitle = substr(episodeTitle, RLENGTH + 1)
    }
    # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    #
    # If there is a trailing (Sn 1 Ep 1), remove it
    # Handle normal (Sn 1 Ep 1) with variations in spacing and capitalization
    # and ones missing the second letter (Sn 1 E 1), (S1 E1), or wrong second letter (Sm 1 Ep 1)
    if (match (episodeTitle,\
        /[ ]*\(S[Nnm]*[ ]*[[:digit:]]{1,2}[ ]+[Ee][Pp]*[ ]*[[:digit:]]{1,3}[[:space:]]*\)/))  {
            if (episodeTitle !~ / \(Sn [[:digit:]]{1,2} Ep [[:digit:]]{1,3}\)/)
                printf ("==> Malformed Sn/Ep in \"%s: %s\"\n", showTitle, episodeTitle) >> ERRORS
            sub (/[ ]*\(S[Nnm]*[ ]*[[:digit:]]{1,2}[ ]+[Ee][Pp]*[ ]*[[:digit:]]{1,3}[[:space:]]*\)/,\
                "",episodeTitle)
            # print "==> episodeTitle = " episodeTitle > "/dev/stderr"
    }
    #
    # Default episodeType to "E"
    episodeType = "E"
    # If episode is a BONUS:, set episodeType to "X"
    if (episodeTitle ~ /BONUS|Montalbano and Me/)
        episodeType = "X"
    # If episode is a Trailer (i.e. First look), set episodeType to "T"
    if (episodeTitle ~ /^PR \|/) {
        episodeType = "T"
        seasonEpisodes = seasonEpisodes - 1
    }
    # print "==> episodeType = " episodeType > "/dev/stderr"
    next
}

# Episode Number(s)
#

#         Episode 1
#

/
<span class='media-identifier/,/<\/div>/ {
    sub (/^ */,"")
    if ($0 ~ /Episode /) {
        episodeNumber = $2
        if (showTitle == "Detective Montalbano" && page2 == "yes") {
            oldEpisodeNumber = episodeNumber
            episodeNumber += 24
            printf ("==> Changed E%02d to %s%02d: %s\n", oldEpisodeNumber, episodeType, \
                    episodeNumber, shortEpisodeURL) >> ERRORS
        }
        # print "==> episodeNumber = " episodeNumber > "/dev/stderr"
        next
    }

### Wrap-up episode processing when Episode Description is found
### print only on LONG_SPREADSHEET
    # print $0 > "/dev/stderr"

    # Episode Description(s)
    #

    #
Preparing for her sister's wedding, Sonja's idyllic life is shattered by tragedy.
    #

    if ($0 ~ /
/) {
        split ($0,fld,"[<>]")
        paragraph = fld[3]
        gsub (/&/,"\\&",paragraph)
        # Could be in multiple paragraphs
        descriptionLinesFound += 1
        if (paragraph != "")
            episodeDescription = episodeDescription (descriptionLinesFound == 1 ? "" : " ") paragraph
        gsub (/&/,"\\&",episodeDescription)
        gsub (/'/,"'",episodeDescription)
        # print "descriptionLinesFound = " descriptionLinesFound > "/dev/stderr"
        # print "==> episodeDescription from
 = \n" episodeDescription > "/dev/stderr"
        next
    }
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1/videos/gasmamman-episode-01-sn-1-ep-1-1";\
    #  "Gasmamman, S01E01, Gåsmamman: Episode 01 (Sn 1 Ep 1)")
    if ($0 ~ /<\/div>/) {
        # Make sure episodeDescription is valid
        if (episodeDescription == "") {
            if (episodeType == "T") {
                episodeDescription = episodeTitle
                # print "==> episodeDescription from title = " episodeDescription > "/dev/stderr"
            } else {
                print "==> No description: " shortEpisodeURL >> ERRORS
            }
        }
        if (episodeDescription ~ /^PR \|/)
            sub (/^PR \| /,"",episodeDescription)
        # print "==> episodeDescription = \n" episodeDescription > "/dev/stderr"
        episodeLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d%s%02d, %s\"\)", episodeURL, showTitle,
                    seasonNumber, episodeType, episodeNumber, episodeTitle)
        #
        # Make sure episodeDuration is valid
        if (split (episodeDuration, tm, ":") == 3) {
            # Canonicalize episodeDuration to 3 parts like other services
            episodeDuration = sprintf ("%02d:%02d:%02d",tm[1],tm[2],tm[3])
        } else {
            printf ("==> Bad episodeDuration %s in \"%s: %s\"\n", episodeDuration, showTitle,
                    episodeTitle) >> ERRORS
            episodeDuration = ""
        }
        #
        # Print "episode" line
        printf ("%s\t\t\t%s\t\t\t\t\t%s\n", episodeLink, episodeDuration, episodeDescription)
        #
        # Look for directors
        director_name = episodeDescription
        if (gsub (/\xc2\xa0/," ",director_name))
            printf ("==> UTF8 non-breaking-space in %s\n", shortEpisodeURL) >> ERRORS
        if (director_name ~ /[Dd]irected by /) {
            sub (/.*[Dd]irected by /,"",director_name)
            sub (/,.*$/,"",director_name)
            sub (/ [[:digit:]]{4}\./,"",director_name)
            sub (/[[:space:]]+$/,"",director_name)
            sub (/\.$/,"",director_name)
            # Special cases
            if (director_name ~ /Manetti Bros/)
                director_name = "The Manetti Bros."
            if (director_name ~ /Andrea and Antonio Frazzi/)
                director_name = "Andrea Frazzi and Antonio Frazzi"
            #
            if (match (director_name, " and ") || match (director_name, " & ")) {
                dname = substr(director_name,1,RSTART-1)
                director_name = substr(director_name,RSTART+RLENGTH)
                printf ("%s\tdirector\ttv_show\t%s\n", dname, showTitle) >> RAW_CREDITS
            }
            printf ("%s\tdirector\ttv_show\t%s\n", director_name, showTitle) >> RAW_CREDITS
        }
        # Make sure there is no carryover
        descriptionLinesFound = 0
        episodeLink = ""
        episodeURL =""
        shortEpisodeURL = ""
        episodeType = ""
        episodeNumber = ""
        episodeTitle = ""
        episodeDuration = ""
        episodeDescription =""
    }
}

### Wrap-up show processing - print on standard out

# Footer at end of file
#
/ "/dev/stderr"
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman";"Gasmamman")
    showLink = "=HYPERLINK(\"" showURL "\";\"" showTitle "\")"
    #
    # Shows with only one season have a blank showSeasons
    if (showSeasons == "")
        showSeasons = 1
    # in most cases, showSeasons = the final season number, i.e.
    # a show with two seasons would have seasons 1 and 2
    finalSeason = showSeasons
    # Special case any discontinuous seasons, e.g. 2 seasons, but numbered 1 and 3
    if (showURL ~ /\/wallander$/)
        finalSeason = 3
    #
    # Wrap show, i.e. but only on final season in order to prevent multiples
    # print "==> " showURL " - seasonNumber = " seasonNumber  > "/dev/stderr"
    # print "==> " showTitle " - finalSeason = " finalSeason  > "/dev/stderr"
    if (seasonNumber > finalSeason ) {
        # This should be OK, as printing any single valid season will work - however ...
        printf ("==> Season number %d beyond final season %d: %s\n", seasonNumber, finalSeason,
                shortSeasonURL) >> ERRORS
    }
    if (seasonNumber == finalSeason ) {
        # Print "show" line
        printf ("%s\t%s\t\t\t%s\n", showLink, showSeasons, showDescriptor)
    }
    #
    #  =HYPERLINK("https://watch.mhzchoice.com/gasmamman/season:1";"Gasmamman, S01, Season 1")
    if (page2 != "yes") {
        seasonLink = sprintf ("=HYPERLINK(\"%s\";\"%s, S%02d, %s\"\)", seasonURL, showTitle, seasonNumber,
                   seasonTitle)
        # print "==> seasonURL = " seasonURL  > "/dev/stderr"
        # print "==> seasonLink = " seasonLink  > "/dev/stderr"
        # Don't print redundant showDescriptor for season 1
        if (seasonNumber == 1)
            showDescriptor = ""
        # Print "season" line
        printf ("%s\t\t%s\t\t%s\n", seasonLink, seasonEpisodes, showDescriptor)
    }
    #
    # Make sure there is no carryover
    showLink = ""
    showURL = ""
    showTitle = ""
    showSeasons = ""
    showDescriptor = ""
    # print "" > "/dev/stderr"
    #
    page2 = ""
    seasonLink = ""
    seasonURL = ""
    shortSeasonURL = ""
    seasonNumber = ""
    seasonTitle = ""
    seasonEpisodes = ""
}


add option -upgrade

add option -upgrade

####################################################################
# Main.awk                                                         #
####################################################################

# Initialization.
function init() {
    initGawk()
    initBiDi()

    # Languages.awk
    initLocale()
    initUserLang()

    RS = "\n"

    ExitCode = 0

    Option["debug"] = 0

    # Display
    Option["verbose"] = 1
    Option["show-original"] = 1
    Option["show-original-phonetics"] = 1
    Option["show-translation"] = 1
    Option["show-translation-phonetics"] = 1
    Option["show-prompt-message"] = 1
    Option["show-languages"] = 1
    Option["show-original-dictionary"] = 0
    Option["show-dictionary"] = 1
    Option["show-alternatives"] = 1
    Option["width"] = ENVIRON["COLUMNS"] ? ENVIRON["COLUMNS"] : 64
    Option["indent"] = 4
    Option["no-ansi"] = 0
    Option["theme"] = "default"

    # Audio
    Option["play"] = 0
    Option["player"] = ENVIRON["PLAYER"]

    # Terminal paging and web
    Option["view"] = 0
    Option["pager"] = ENVIRON["PAGER"]
    Option["browser"] = ENVIRON["BROWSER"]

    # Networking
    Option["proxy"] = ENVIRON["HTTP_PROXY"] ? ENVIRON["HTTP_PROXY"] : ENVIRON["http_proxy"]
    Option["user-agent"] = ENVIRON["USER_AGENT"]

    # Interactive shell
    Option["no-rlwrap"] = 0
    Option["interactive"] = 0
    Option["emacs"] = 0

    # I/O
    Option["input"] = NULLSTR
    Option["output"] = STDOUT

    # Language preference
    Option["hl"] = ENVIRON["HOME_LANG"] ? ENVIRON["HOME_LANG"] : UserLang
    Option["sl"] = ENVIRON["SOURCE_LANG"] ? ENVIRON["SOURCE_LANG"] : "auto"
    Option["tl"][1] = ENVIRON["TARGET_LANG"] ? ENVIRON["TARGET_LANG"] : UserLang
}

# Initialization script.
function initScript(    file, line, script, temp) {
    # Find the initialization file
    file = ".trans"
    if (!fileExists(file)) {
        file = ENVIRON["HOME"] "/.translate-shell/init.trans"
        if (!fileExists(file)) {
            file = "/etc/translate-shell"
            if (!fileExists(file)) return
        }
    }

    InitScript = file
    script = NULLSTR
    while (getline line < InitScript)
        script = script "\n" line
    loadOptions(script)

    # Hack: Option["tl"] must be an array
    if (!isarray(Option["tl"])) {
        temp = Option["tl"]
        delete Option["tl"]
        Option["tl"][1] = temp
    }
}

# Miscellany initialization.
function initMisc(    group) {
    # Translate.awk
    initHttpService()

    # Disable ANSI escape codes if required
    if (Option["no-ansi"])
        delete AnsiCode

    # Initialize audio player or speech synthesizer
    if (Option["play"]) {
        if (!Option["player"]) {
            initAudioPlayer()
            Option["player"] = AudioPlayer ? AudioPlayer : Option["player"]
            if (!Option["player"])
                initSpeechSynthesizer()
        }

        if (!Option["player"] && !SpeechSynthesizer) {
            w("[WARNING] No available audio player or speech synthesizer.")
            Option["play"] = 0
        }
    }

    # Initialize pager
    if (Option["view"]) {
        if (!Option["pager"]) {
            initPager()
            Option["pager"] = Pager
        }

        if (!Option["pager"]) {
            w("[WARNING] No available terminal pager.")
            Option["view"] = 0
        }
    }

    # Initialize browser
    if (!Option["browser"]) {
        "xdg-mime query default text/html" SUPERR |& getline Option["browser"]
        match(Option["browser"], "(.*).desktop$", group)
        Option["browser"] = group[1]
    }
}

# Main entry point.
BEGIN {
    init()

    if (!(belongsTo("-no-init", ARGV) || belongsTo("--no-init", ARGV)))
        initScript() # initialization script overrides default setting

    # Command-line options override initialization script
    pos = 0
    while (ARGV[++pos]) {
        ## Information options

        # -V, -version
        match(ARGV[pos], /^--?(V|vers(i(on?)?)?)$/)
        if (RSTART) {
            InfoOnly = "version"
            continue
        }

        # -H, -help
        match(ARGV[pos], /^--?(H|h(e(lp?)?)?)$/)
        if (RSTART) {
            InfoOnly = "help"
            continue
        }

        # -M, -manual
        match(ARGV[pos], /^--?(M|m(a(n(u(al?)?)?)?)?)$/)
        if (RSTART) {
            InfoOnly = "manual"
            continue
        }

        # -T, -reference
        match(ARGV[pos], /^--?(T|ref(e(r(e(n(ce?)?)?)?)?)?)$/)
        if (RSTART) {
            InfoOnly = "reference"
            continue
        }

        # FIXME[1.0]: to be removed
        match(ARGV[pos], /^--?r$/)
        if (RSTART) {
            w("[WARNING] Option '-r' has been deprecated since version 0.9.\n" \
              "          Use option '-T' or '-reference' instead.")
            exit 1
        }

        # -R, -reference-english
        match(ARGV[pos], /^--?(R|reference-e(n(g(l(i(sh?)?)?)?)?)?)$/)
        if (RSTART) {
            InfoOnly = "reference-english"
            continue
        }

        # -L CODE(S), -list CODE(S)
        match(ARGV[pos], /^--?(L|list)(=(.*)?)?$/, group)
        if (RSTART) {
            InfoOnly = "list"
            if (group[2]) {
                if (group[3]) split(group[3], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # -U, -upgrade
        match(ARGV[pos], /^--?(U|upgrade)$/)
        if (RSTART) {
            InfoOnly = "upgrade"
            continue
        }

        # -N, -nothing
        match(ARGV[pos], /^--?(N|nothing)$/)
        if (RSTART) {
            InfoOnly = "nothing"
            continue
        }

        ## Display options

        # -verbose
        match(ARGV[pos], /^--?verbose$/)
        if (RSTART) {
            Option["verbose"] = 1 # default value
            continue
        }

        # -b, -brief
        match(ARGV[pos], /^--?b(r(i(ef?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 0
            continue
        }

        # -d, -dictionary
        match(ARGV[pos], /^--?d(i(c(t(i(o(n(a(ry?)?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            Option["show-original-dictionary"] = 1
            Option["show-dictionary"] = 0
            Option["show-alternatives"] = 0
            continue
        }

        # -show-original Y|n
        match(ARGV[pos], /^--?show-original(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-phonetics Y|n
        match(ARGV[pos], /^--?show-original-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation Y|n
        match(ARGV[pos], /^--?show-translation(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation-phonetics Y|n
        match(ARGV[pos], /^--?show-translation-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-prompt-message Y|n
        match(ARGV[pos], /^--?show-prompt-message(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-prompt-message"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-languages Y|n
        match(ARGV[pos], /^--?show-languages(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-languages"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-dictionary y|N
        match(ARGV[pos], /^--?show-original-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-dictionary Y|n
        match(ARGV[pos], /^--?show-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-alternatives Y|n
        match(ARGV[pos], /^--?show-alternatives(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-alternatives"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -w NUM, -width NUM
        match(ARGV[pos], /^--?w(i(d(th?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["width"] = group[4] ?
                (group[5] ? group[5] : Option["width"]) :
                ARGV[++pos]
            continue
        }

        # -indent NUM
        match(ARGV[pos], /^--?indent(=(.*)?)?$/, group)
        if (RSTART) {
            Option["indent"] = group[1] ?
                (group[2] ? group[2] : Option["indent"]) :
                ARGV[++pos]
            continue
        }

        # -no-ansi
        match(ARGV[pos], /^--?no-ansi$/)
        if (RSTART) {
            Option["no-ansi"] = 1
            continue
        }

        # -no-theme
        match(ARGV[pos], /^--?no-theme$/)
        if (RSTART) {
            Option["theme"] = NULLSTR
            continue
        }

        # -theme FILENAME
        match(ARGV[pos], /^--?theme(=(.*)?)?$/, group)
        if (RSTART) {
            Option["theme"] = group[1] ?
                (group[2] ? group[2] : Option["theme"]) :
                ARGV[++pos]
            continue
        }

        ## Audio options

        # -no-play
        match(ARGV[pos], /^--?no-play$/)
        if (RSTART) {
            Option["play"] = 0
            continue
        }

        # -p, -play
        match(ARGV[pos], /^--?p(l(ay?)?)?$/)
        if (RSTART) {
            Option["play"] = 1
            continue
        }

        # -player PROGRAM
        match(ARGV[pos], /^--?player(=(.*)?)?$/, group)
        if (RSTART) {
            Option["play"] = 1
            Option["player"] = group[1] ?
                (group[2] ? group[2] : Option["player"]) :
                ARGV[++pos]
            continue
        }

        ## Terminal paging and web options

        # -no-view
        match(ARGV[pos], /^--?no-view$/)
        if (RSTART) {
            Option["view"] = 0
            continue
        }

        # -v, -view
        match(ARGV[pos], /^--?v(i(ew?)?)?$/)
        if (RSTART) {
            Option["view"] = 1
            continue
        }

        # -pager PROGRAM
        match(ARGV[pos], /^--?pager(=(.*)?)?$/, group)
        if (RSTART) {
            Option["view"] = 1
            Option["pager"] = group[1] ?
                (group[2] ? group[2] : Option["pager"]) :
                ARGV[++pos]
            continue
        }

        # -browser PROGRAM
        match(ARGV[pos], /^--?browser(=(.*)?)?$/, group)
        if (RSTART) {
            Option["browser"] = group[1] ?
                (group[2] ? group[2] : Option["browser"]) :
                ARGV[++pos]
            continue
        }

        ## Networking options

        # -x HOST:PORT, -proxy HOST:PORT
        match(ARGV[pos], /^--?(x|proxy)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["proxy"] = group[2] ?
                (group[3] ? group[3] : Option["proxy"]) :
                ARGV[++pos]
            continue
        }

        # -u STRING, -user-agent STRING
        match(ARGV[pos], /^--?(u|user-agent)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["user-agent"] = group[2] ?
                (group[3] ? group[3] : Option["user-agent"]) :
                ARGV[++pos]
            continue
        }

        ## Interactive shell options

        # -no-rlwrap
        match(ARGV[pos], /^--?no-rlwrap$/)
        if (RSTART) {
            Option["no-rlwrap"] = 1
            continue
        }

        # -I, -interactive, -shell
        match(ARGV[pos], /^--?(I|int(e(r(a(c(t(i(ve?)?)?)?)?)?)?)?|shell)$/)
        if (RSTART) {
            Option["interactive"] = 1
            continue
        }

        # -E, -emacs
        match(ARGV[pos], /^--?(E|emacs)$/)
        if (RSTART) {
            Option["emacs"] = 1
            continue
        }

        # FIXME[1.0]: to be removed
        # -prompt PROMPT_STRING
        match(ARGV[pos], /^--?prompt(=(.*)?)?$/, group)
        if (RSTART) {
            w("[ERROR] Option '-prompt' has been deprecated since version 0.9.\n" \
              "        Use configuration variable 'fmt-prompt' instead.")
            exit 1
        }

        # FIXME[1.0]: to be removed
        # -prompt-color COLOR_CODE
        match(ARGV[pos], /^--?prompt-color(=(.*)?)?$/, group)
        if (RSTART) {
            w("[ERROR] Option '-prompt-color' has been deprecated since version 0.9.\n" \
              "        Use configuration variable 'sgr-prompt' instead.")
            exit 1
        }

        ## I/O options

        # -i FILENAME, -input FILENAME
        match(ARGV[pos], /^--?i(n(p(ut?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["input"] = group[4] ?
                (group[5] ? group[5] : Option["input"]) :
                ARGV[++pos]
            continue
        }

        # -o FILENAME, -output FILENAME
        match(ARGV[pos], /^--?o(u(t(p(ut?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["output"] = group[5] ?
                (group[6] ? group[6] : Option["output"]) :
                ARGV[++pos]
            continue
        }

        ## Language preference options

        # -l CODE, -lang CODE
        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["hl"] = group[3] ?
                (group[4] ? group[4] : Option["hl"]) :
                ARGV[++pos]
            continue
        }

        # -s CODE, -source CODE
        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["sl"] = group[5] ?
                (group[6] ? group[6] : Option["sl"]) :
                ARGV[++pos]
            continue
        }

        # -t CODE(S), -target CODE(S)
        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            if (group[5]) {
                if (group[6]) split(group[6], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # Shortcut format
        # 'CODE:CODE+...' or 'CODE=CODE+...'
        match(ARGV[pos], /^[{(\[]?([[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?(:|=)((@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?\+)*(@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?)[})\]]?$/, group)
        if (RSTART) {
            if (group[1]) Option["sl"] = group[1]
            if (group[4]) split(group[4], Option["tl"], "+")
            continue
        }

        ## Other options

        # -D, -debug
        match(ARGV[pos], /^--?(D|debug)$/)
        if (RSTART) {
            Option["debug"] = 1
            continue
        }

        # -no-init
        match(ARGV[pos], /^--?no-init/)
        if (RSTART) continue # skip

        # -, -no-op
        match(ARGV[pos], /^-(-?no-op)?$/)
        if (RSTART) continue # no operation, skip

        # --
        match(ARGV[pos], /^--$/)
        if (RSTART) {
            ++pos # skip the end-of-options option
            break # no more option from here
        }

        break # no more option from here
    }

    setTheme() # theme overrides command-line options

    # Handle interactive shell
    if (Option["interactive"] && !Option["no-rlwrap"])
        rlwrapMe() # interactive mode
    else if (Option["emacs"] && !Option["interactive"] && !Option["no-rlwrap"])
        if (emacsMe()) # emacs front-end
            Option["interactive"] = 1 # fallback to interactive mode

    # Get started

    initMisc()

    # Handle information options
    switch (InfoOnly) {
    case "version":
        print getVersion()
        exit
    case "help":
        print getHelp()
        exit
    case "manual":
        showMan()
        exit
    case "reference":
        print getReference("endonym")
        exit
    case "reference-english":
        print getReference("name")
        exit
    case "list":
        print getList(Option["tl"])
        exit
    case "upgrade":
        upgrade()
        exit
    case "nothing":
        exit
    }

    if (Option["interactive"])
        welcome()

    if (pos < ARGC) {
        # More parameters

        # Translate remaining parameters
        for (i = pos; i < ARGC; i++) {
            # Verbose mode: separator between sources
            if (Option["verbose"] && i > pos)
                p(prettify("source-seperator", replicate(Option["chr-source-seperator"], Option["width"])))

            translate(ARGV[i], 1) # inline mode
        }

        # If input not specified, we're done
    } else {
        # No more parameter besides options

        # If input not specified, use stdin
        if (!Option["input"]) Option["input"] = STDIN
    }

    # If input specified, start translating
    if (Option["input"])
        translateMain()

    exit ExitCode
}

####################################################################
# Main.awk                                                         #
####################################################################

# Initialization.
function init() {
    initGawk()
    initBiDi()

    # Languages.awk
    initLocale()
    initUserLang()

    RS = "\n"

    ExitCode = 0

    Option["debug"] = 0

    # Display
    Option["verbose"] = 1
    Option["show-original"] = 1
    Option["show-original-phonetics"] = 1
    Option["show-translation"] = 1
    Option["show-translation-phonetics"] = 1
    Option["show-prompt-message"] = 1
    Option["show-languages"] = 1
    Option["show-original-dictionary"] = 0
    Option["show-dictionary"] = 1
    Option["show-alternatives"] = 1
    Option["width"] = ENVIRON["COLUMNS"] ? ENVIRON["COLUMNS"] : 64
    Option["indent"] = 4
    Option["no-ansi"] = 0
    Option["theme"] = "default"

    # Audio
    Option["play"] = 0
    Option["player"] = ENVIRON["PLAYER"]

    # Terminal paging and web
    Option["view"] = 0
    Option["pager"] = ENVIRON["PAGER"]
    Option["browser"] = ENVIRON["BROWSER"]

    # Networking
    Option["proxy"] = ENVIRON["HTTP_PROXY"] ? ENVIRON["HTTP_PROXY"] : ENVIRON["http_proxy"]
    Option["user-agent"] = ENVIRON["USER_AGENT"]

    # Interactive shell
    Option["no-rlwrap"] = 0
    Option["interactive"] = 0
    Option["emacs"] = 0

    # I/O
    Option["input"] = NULLSTR
    Option["output"] = STDOUT

    # Language preference
    Option["hl"] = ENVIRON["HOME_LANG"] ? ENVIRON["HOME_LANG"] : UserLang
    Option["sl"] = ENVIRON["SOURCE_LANG"] ? ENVIRON["SOURCE_LANG"] : "auto"
    Option["tl"][1] = ENVIRON["TARGET_LANG"] ? ENVIRON["TARGET_LANG"] : UserLang
}

# Initialization script.
function initScript(    file, line, script, temp) {
    # Find the initialization file
    file = ".trans"
    if (!fileExists(file)) {
        file = ENVIRON["HOME"] "/.translate-shell/init.trans"
        if (!fileExists(file)) {
            file = "/etc/translate-shell"
            if (!fileExists(file)) return
        }
    }

    InitScript = file
    script = NULLSTR
    while (getline line < InitScript)
        script = script "\n" line
    loadOptions(script)

    # Hack: Option["tl"] must be an array
    if (!isarray(Option["tl"])) {
        temp = Option["tl"]
        delete Option["tl"]
        Option["tl"][1] = temp
    }
}

# Miscellany initialization.
function initMisc(    group) {
    # Translate.awk
    initHttpService()

    # Disable ANSI escape codes if required
    if (Option["no-ansi"])
        delete AnsiCode

    # Initialize audio player or speech synthesizer
    if (Option["play"]) {
        if (!Option["player"]) {
            initAudioPlayer()
            Option["player"] = AudioPlayer ? AudioPlayer : Option["player"]
            if (!Option["player"])
                initSpeechSynthesizer()
        }

        if (!Option["player"] && !SpeechSynthesizer) {
            w("[WARNING] No available audio player or speech synthesizer.")
            Option["play"] = 0
        }
    }

    # Initialize pager
    if (Option["view"]) {
        if (!Option["pager"]) {
            initPager()
            Option["pager"] = Pager
        }

        if (!Option["pager"]) {
            w("[WARNING] No available terminal pager.")
            Option["view"] = 0
        }
    }

    # Initialize browser
    if (!Option["browser"]) {
        "xdg-mime query default text/html" SUPERR |& getline Option["browser"]
        match(Option["browser"], "(.*).desktop$", group)
        Option["browser"] = group[1]
    }
}

# Main entry point.
BEGIN {
    init()

    if (!(belongsTo("-no-init", ARGV) || belongsTo("--no-init", ARGV)))
        initScript() # initialization script overrides default setting

    # Command-line options override initialization script
    pos = 0
    while (ARGV[++pos]) {
        ## Information options

        # -V, -version
        match(ARGV[pos], /^--?(V|vers(i(on?)?)?)$/)
        if (RSTART) {
            InfoOnly = "version"
            continue
        }

        # -H, -help
        match(ARGV[pos], /^--?(H|h(e(lp?)?)?)$/)
        if (RSTART) {
            InfoOnly = "help"
            continue
        }

        # -M, -manual
        match(ARGV[pos], /^--?(M|m(a(n(u(al?)?)?)?)?)$/)
        if (RSTART) {
            InfoOnly = "manual"
            continue
        }

        # -T, -reference
        match(ARGV[pos], /^--?(T|ref(e(r(e(n(ce?)?)?)?)?)?)$/)
        if (RSTART) {
            InfoOnly = "reference"
            continue
        }

        # FIXME[1.0]: to be removed
        match(ARGV[pos], /^--?r$/)
        if (RSTART) {
            w("[WARNING] Option '-r' has been deprecated since version 0.9.\n" \
              "          Use option '-T' or '-reference' instead.")
            exit 1
        }

        # -R, -reference-english
        match(ARGV[pos], /^--?(R|reference-e(n(g(l(i(sh?)?)?)?)?)?)$/)
        if (RSTART) {
            InfoOnly = "reference-english"
            continue
        }

        # -L CODE(S), -list CODE(S)
        match(ARGV[pos], /^--?(L|list)(=(.*)?)?$/, group)
        if (RSTART) {
            InfoOnly = "list"
            if (group[2]) {
                if (group[3]) split(group[3], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # -N, -nothing
        match(ARGV[pos], /^--?(N|nothing)$/)
        if (RSTART) {
            InfoOnly = "nothing"
            continue
        }

        ## Display options

        # -verbose
        match(ARGV[pos], /^--?verbose$/)
        if (RSTART) {
            Option["verbose"] = 1 # default value
            continue
        }

        # -b, -brief
        match(ARGV[pos], /^--?b(r(i(ef?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 0
            continue
        }

        # -d, -dictionary
        match(ARGV[pos], /^--?d(i(c(t(i(o(n(a(ry?)?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            Option["show-original-dictionary"] = 1
            Option["show-dictionary"] = 0
            Option["show-alternatives"] = 0
            continue
        }

        # -show-original Y|n
        match(ARGV[pos], /^--?show-original(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-phonetics Y|n
        match(ARGV[pos], /^--?show-original-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation Y|n
        match(ARGV[pos], /^--?show-translation(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation-phonetics Y|n
        match(ARGV[pos], /^--?show-translation-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-prompt-message Y|n
        match(ARGV[pos], /^--?show-prompt-message(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-prompt-message"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-languages Y|n
        match(ARGV[pos], /^--?show-languages(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-languages"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-dictionary y|N
        match(ARGV[pos], /^--?show-original-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-dictionary Y|n
        match(ARGV[pos], /^--?show-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-alternatives Y|n
        match(ARGV[pos], /^--?show-alternatives(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-alternatives"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -w NUM, -width NUM
        match(ARGV[pos], /^--?w(i(d(th?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["width"] = group[4] ?
                (group[5] ? group[5] : Option["width"]) :
                ARGV[++pos]
            continue
        }

        # -indent NUM
        match(ARGV[pos], /^--?indent(=(.*)?)?$/, group)
        if (RSTART) {
            Option["indent"] = group[1] ?
                (group[2] ? group[2] : Option["indent"]) :
                ARGV[++pos]
            continue
        }

        # -no-ansi
        match(ARGV[pos], /^--?no-ansi$/)
        if (RSTART) {
            Option["no-ansi"] = 1
            continue
        }

        # -no-theme
        match(ARGV[pos], /^--?no-theme$/)
        if (RSTART) {
            Option["theme"] = NULLSTR
            continue
        }

        # -theme FILENAME
        match(ARGV[pos], /^--?theme(=(.*)?)?$/, group)
        if (RSTART) {
            Option["theme"] = group[1] ?
                (group[2] ? group[2] : Option["theme"]) :
                ARGV[++pos]
            continue
        }

        ## Audio options

        # -no-play
        match(ARGV[pos], /^--?no-play$/)
        if (RSTART) {
            Option["play"] = 0
            continue
        }

        # -p, -play
        match(ARGV[pos], /^--?p(l(ay?)?)?$/)
        if (RSTART) {
            Option["play"] = 1
            continue
        }

        # -player PROGRAM
        match(ARGV[pos], /^--?player(=(.*)?)?$/, group)
        if (RSTART) {
            Option["play"] = 1
            Option["player"] = group[1] ?
                (group[2] ? group[2] : Option["player"]) :
                ARGV[++pos]
            continue
        }

        ## Terminal paging and web options

        # -no-view
        match(ARGV[pos], /^--?no-view$/)
        if (RSTART) {
            Option["view"] = 0
            continue
        }

        # -v, -view
        match(ARGV[pos], /^--?v(i(ew?)?)?$/)
        if (RSTART) {
            Option["view"] = 1
            continue
        }

        # -pager PROGRAM
        match(ARGV[pos], /^--?pager(=(.*)?)?$/, group)
        if (RSTART) {
            Option["view"] = 1
            Option["pager"] = group[1] ?
                (group[2] ? group[2] : Option["pager"]) :
                ARGV[++pos]
            continue
        }

        # -browser PROGRAM
        match(ARGV[pos], /^--?browser(=(.*)?)?$/, group)
        if (RSTART) {
            Option["browser"] = group[1] ?
                (group[2] ? group[2] : Option["browser"]) :
                ARGV[++pos]
            continue
        }

        ## Networking options

        # -x HOST:PORT, -proxy HOST:PORT
        match(ARGV[pos], /^--?(x|proxy)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["proxy"] = group[2] ?
                (group[3] ? group[3] : Option["proxy"]) :
                ARGV[++pos]
            continue
        }

        # -u STRING, -user-agent STRING
        match(ARGV[pos], /^--?(u|user-agent)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["user-agent"] = group[2] ?
                (group[3] ? group[3] : Option["user-agent"]) :
                ARGV[++pos]
            continue
        }

        ## Interactive shell options

        # -no-rlwrap
        match(ARGV[pos], /^--?no-rlwrap$/)
        if (RSTART) {
            Option["no-rlwrap"] = 1
            continue
        }

        # -I, -interactive, -shell
        match(ARGV[pos], /^--?(I|int(e(r(a(c(t(i(ve?)?)?)?)?)?)?)?|shell)$/)
        if (RSTART) {
            Option["interactive"] = 1
            continue
        }

        # -E, -emacs
        match(ARGV[pos], /^--?(E|emacs)$/)
        if (RSTART) {
            Option["emacs"] = 1
            continue
        }

        # FIXME[1.0]: to be removed
        # -prompt PROMPT_STRING
        match(ARGV[pos], /^--?prompt(=(.*)?)?$/, group)
        if (RSTART) {
            w("[ERROR] Option '-prompt' has been deprecated since version 0.9.\n" \
              "        Use configuration variable 'fmt-prompt' instead.")
            exit 1
        }

        # FIXME[1.0]: to be removed
        # -prompt-color COLOR_CODE
        match(ARGV[pos], /^--?prompt-color(=(.*)?)?$/, group)
        if (RSTART) {
            w("[ERROR] Option '-prompt-color' has been deprecated since version 0.9.\n" \
              "        Use configuration variable 'sgr-prompt' instead.")
            exit 1
        }

        ## I/O options

        # -i FILENAME, -input FILENAME
        match(ARGV[pos], /^--?i(n(p(ut?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["input"] = group[4] ?
                (group[5] ? group[5] : Option["input"]) :
                ARGV[++pos]
            continue
        }

        # -o FILENAME, -output FILENAME
        match(ARGV[pos], /^--?o(u(t(p(ut?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["output"] = group[5] ?
                (group[6] ? group[6] : Option["output"]) :
                ARGV[++pos]
            continue
        }

        ## Language preference options

        # -l CODE, -lang CODE
        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["hl"] = group[3] ?
                (group[4] ? group[4] : Option["hl"]) :
                ARGV[++pos]
            continue
        }

        # -s CODE, -source CODE
        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["sl"] = group[5] ?
                (group[6] ? group[6] : Option["sl"]) :
                ARGV[++pos]
            continue
        }

        # -t CODE(S), -target CODE(S)
        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            if (group[5]) {
                if (group[6]) split(group[6], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # Shortcut format
        # 'CODE:CODE+...' or 'CODE=CODE+...'
        match(ARGV[pos], /^[{(\[]?([[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?(:|=)((@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?\+)*(@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?)[})\]]?$/, group)
        if (RSTART) {
            if (group[1]) Option["sl"] = group[1]
            if (group[4]) split(group[4], Option["tl"], "+")
            continue
        }

        ## Other options

        # -D, -debug
        match(ARGV[pos], /^--?(D|debug)$/)
        if (RSTART) {
            Option["debug"] = 1
            continue
        }

        # -no-init
        match(ARGV[pos], /^--?no-init/)
        if (RSTART) continue # skip

        # -, -no-op
        match(ARGV[pos], /^-(-?no-op)?$/)
        if (RSTART) continue # no operation, skip

        # --
        match(ARGV[pos], /^--$/)
        if (RSTART) {
            ++pos # skip the end-of-options option
            break # no more option from here
        }

        break # no more option from here
    }

    setTheme() # theme overrides command-line options

    # Handle interactive shell
    if (Option["interactive"] && !Option["no-rlwrap"])
        rlwrapMe() # interactive mode
    else if (Option["emacs"] && !Option["interactive"] && !Option["no-rlwrap"])
        if (emacsMe()) # emacs front-end
            Option["interactive"] = 1 # fallback to interactive mode

    # Get started

    initMisc()

    # Handle information options
    switch (InfoOnly) {
    case "version":
        print getVersion()
        exit
    case "help":
        print getHelp()
        exit
    case "manual":
        showMan()
        exit
    case "reference":
        print getReference("endonym")
        exit
    case "reference-english":
        print getReference("name")
        exit
    case "list":
        print getList(Option["tl"])
        exit
    case "nothing":
        exit
    }

    if (Option["interactive"])
        welcome()

    if (pos < ARGC) {
        # More parameters

        # Translate remaining parameters
        for (i = pos; i < ARGC; i++) {
            # Verbose mode: separator between sources
            if (Option["verbose"] && i > pos)
                p(prettify("source-seperator", replicate(Option["chr-source-seperator"], Option["width"])))

            translate(ARGV[i], 1) # inline mode
        }

        # If input not specified, we're done
    } else {
        # No more parameter besides options

        # If input not specified, use stdin
        if (!Option["input"]) Option["input"] = STDIN
    }

    # If input specified, start translating
    if (Option["input"])
        translateMain()

    exit ExitCode
}


cleaning global.e1.params.ttl with bin/util/update-e-params-subject-discrim.awk

# update-e-params-subject-discrim.awk
# params:
#    baseURI
#    sourceID
#    dataset_identifier
#    datasetVersion
#    layerID
#    subjectDiscriminator

{
   if( $1 == "@prefix" && $2 == ":" ) {
      # Fix up default namespace.
      printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/enhancement/%s/> .",baseURI,sourceID,dataset_identifier,datasetVersion,layerID);

   }else if( $0 == ":dataset a void:Dataset;" ) {
      # Change subject to name of layer dataset itself.
      printf("%s/source/%s/dataset/%s/version/%s/conversion/enhancement/%s",baseURI,sourceID,dataset_identifier,datasetVersion,layerID);

   }else if( $0~"conversion:subject_discriminator" ) {
      printf("      conversion:subject_discriminator \"%s\";\n",subjectDiscriminator)

   }else if( $0~"conversion:dataset_version" ) {                          # TODO: DEPRECATED (but converter not recognizing it)
      printf("   conversion:dataset_version    \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:version_identifier" ) {
      printf("   conversion:version_identifier \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:dataset_identifier" && length(dataset_identifier) > 0) {
      printf("   conversion:dataset_identifier \"%s\";\n",dataset_identifier)

   }else {
      print
   }
}

# update-e-params-subject-discrim.awk
# params:
#    baseURI
#    sourceID
#    dataset_identifier
#    datasetVersion
#    layerID
#    subjectDiscriminator

{
   if( $1 == "@prefix" && $2 == ":" ) {
      # Fix up default namespace.
      printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/enhancement/%s/> .",baseURI,sourceID,dataset_identifier,);

   }else if( $0 == ":dataset a void:Dataset;" ) {
      # Change subject to name of layer dataset itself.
      printf("%s/source/%s/dataset/%s/version/%s/conversion/enhancement/%s",baseURI,sourceID,dataset_identifier,datasetVersion,$layerID);

   }else if( $0~"conversion:subject_discriminator" ) {
      printf("      conversion:subject_discriminator \"%s\";\n",subjectDiscriminator)

   }else if( $0~"conversion:dataset_version" ) {                          # TODO: DEPRECATED (but converter not recognizing it)
      printf("   conversion:dataset_version    \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:version_identifier" ) {
      printf("   conversion:version_identifier \"%s\";\n",datasetVersion)

   }else if( $0~"conversion:dataset_identifier" && length(dataset_identifier) > 0) {
      printf("   conversion:dataset_identifier \"%s\";\n",dataset_identifier)

   }else {
      print
   }
}


Fix dependencies generation

Fix dependencies generation

BEGIN{
  ORS=""
  bootstrap = ENVIRON["BOOTSTRAPDIR"]
  split(ENVIRON["FORCEINCLUDELIST"], force_include, " ")
  if (compiler=="gcc")
  {
    out="$(OBJDIR)";
    obj="o";
    rc="rc.o"
    dirsep="/";
  }
  else
  {
    out="$(INTDIR)";
    obj="obj";
    rc="res"
    dirsep="\\";
  }
}
{
  if (match($0, /(.+)\.(.+)/, a))
  {
    filename = a[1]
    ext = a[2]
  }

  if (match(filename, /(.+\\).+/, a))
  {
    path_part = a[1]
  }

  is_cpp = ext == "cpp"

  if(ext == "cpp" || ext == "c")
    ext=obj;
  if(ext == "rc")
    ext=rc;

  if(path_part == "" && (ext == obj || ext == rc))
  {
    print out dirsep filename "." ext ":";
    print " " $0;

    if (is_cpp)
    {
      for (i in force_include)
        print " " force_include[i];
    }
    else if (ext == obj)
        print " disabled_warnings.hpp"
  }
  else
  {
    print filename "." ext ":";
  }

  while((getline lnsrc < ($0)) > 0)
  {
    if(substr(lnsrc,1,length("#include \"")) == "#include \"")
    {
      lnsrc=gensub(/^#include[ \t]*"([^"]+)".*$/, "\\1", "g", lnsrc);
      if(lnsrc != "" && lnsrc != $0)
        if(substr(lnsrc,1,length("bootstrap/")) == "bootstrap/")
          lnsrc = bootstrap substr(lnsrc, length("bootstrap/") + 1)
        print " " path_part lnsrc;
    }
  }
  print "\n\n"
}

BEGIN{
  ORS=""
  bootstrap = ENVIRON["BOOTSTRAPDIR"]
  split(ENVIRON["FORCEINCLUDELIST"], force_include, " ")
  if (compiler=="gcc")
  {
    out="$(OBJDIR)";
    obj="o";
    rc="rc.o"
    dirsep="/";
  }
  else
  {
    out="$(INTDIR)";
    obj="obj";
    rc="res"
    dirsep="\\";
  }
}
{
  if (match($0, /(.+)\.(.+)/, a))
  {
    filename = a[1]
    ext = a[2]
  }

  if (match(filename, /(.+\\).+/, a))
  {
    path_part = a[1]
  }

  is_cpp = ext == "cpp"

  if(ext == "cpp" || ext == "c")
    ext=obj;
  if(ext == "rc")
    ext=rc;

  if(path_part == "" && (ext == obj || ext == rc))
  {
    print out dirsep filename "." ext ":";
    print " " $0;

    if (is_cpp)
    {
      for (i in force_include)
        print " " force_include[i];
    }
  }
  else
  {
    print filename "." ext ":";
  }

  while((getline lnsrc < ($0)) > 0)
  {
    if(substr(lnsrc,1,length("#include \"")) == "#include \"")
    {
      lnsrc=gensub(/^#include[ \t]*"([^"]+)".*$/, "\\1", "g", lnsrc);
      if(lnsrc != "" && lnsrc != $0)
        if(substr(lnsrc,1,length("bootstrap/")) == "bootstrap/")
          lnsrc = bootstrap substr(lnsrc, length("bootstrap/") + 1)
        print " " path_part lnsrc;
    }
  }
  print "\n\n"
}


Translate: handle 301 redirection via curl() (often HTTPS)


Update min/max messaging

Update min/max messaging

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


reworked to improve splitting namespace from id

#! /usr/bin/awk -f

#  Reduce the subject and object of RDF triples (ntriples format)
# down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier and
# the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
# labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
# a handy way to differentiate uri for subjects and objects
# which may belong to a "structural" ontology as opposed to
# a data uri.

#  Perhaps an improvement would be to also
# express subject and objects as individual curies (ala predicates)
# iff their namespace is also used by a predicate.

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(1 == match(uri, /^_:/))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0) # we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) && (0 < length(u)))
		u = stripid(substr(u,1,length(u)-1))

	if(u in prefix)
		return prefix[u]
	else
		for(ex in exception){
			if(0 < match(uri, ex))
				return exception[substr(uri, 1, RLENGTH)]
		}
		return "___" simplify(uri)
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	prefix["BNODE"] = "BNODE"  # is a fixed point
	# revisit if exceptions are still necessary
	exception["http://www.w3.org/1999/02/22-rdf-syntax-ns#"]="rdf"
	exception["http://www.w3.org/2000/01/rdf-schema#"]="rdfs"
	exception["http://www.w3.org/2002/07/owl#"]="owl"
	# in mgi
	exception["https://www.mousephenotype.org"]="IMPC"
	# in panther
	exception["http://identifiers.org/wormbase"]="WormBase"
	# just until skolemized bnodes get in the curie map?
	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
	# till all non httpS: are purged
	exception["http://monarchinitiave.org"]="MONARCH"
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	prefix[arr[4]]=arr[2]
}
# process the ntriple file(s)  which are not the first file
### case when subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
}
### case when the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	### not a uri
	o = "LITERAL"
	edgelist[s, ns ":" p, o]++
	nodelist[o " [shape=record];"]++
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist){
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	for(node in nodelist) print node
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"

}

#! /usr/bin/awk -f

#  Reduce the subject and object of RDF triples (ntriples format)
# down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier and
# the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
# labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
# a handy way to differentiate uri for subjects and objects
# which may belong to a "structural" ontology as opposed to
# a data uri.

#  Perhaps an improvement would be to to also
# express subject and objects as individual curies (ala predicates)
# iff their namespace is also used by a predicate.

##########################################################
# remove first and last chars from input
function trim(str){
	return substr(str,2,length(str)-2)
}

# remove final element in slashed path
# this leaves the namespace of the removed identifier
function stripid(uri){
	l = match(uri,/.*\//)  # side effect is RLENGTH
	if(l<=0)
		return uri
	else
		return substr(uri,1,RLENGTH-1)
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	if(uri in prefix)
		return prefix[uri]
	else
		for(ex in exception){
			start = match(uri, ex)
			if(0 < start)
				return exception[substr(uri, start, RLENGTH)]
		}
		return simplify(uri)
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# Monarchinitiave's dipper ingest files have cases
	# which escape the general rules
	# in everything
	exception["http://www.w3.org/2002/07"]="OWL_"
	exception["http://www.w3.org/1999/02"]="RDF"
	exception["http://www.w3.org/2000/01"]="RDFS"
	# in go,mgi,wormbase
	exception["http://dx.doi.org"]="DOI"
	# in mgi
	exception["https://www.mousephenotype.org"]="IMPC_"
	# in panther
	exception["http://identifiers.org/wormbase"]="WormBase"
	# just until skolemized bnodes get in the curie map?
	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
	# till all non httpS: are purged
	exception["http://monarchinitiave.org"]="MONARCH"
}

# main loop
# parse and stash the curie file (first file)
(FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ {
	split($0,a,"'")
	prefix[stripid(a[4])]=a[2]
}
# process the ntriple file(s)  which are not the first file
### case when subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	q = final(trim($2))
	p = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist[s " -> " o " [label=\"" p ":" q]++
}
### case when the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	q = final(trim($2))
	p = contract(stripid(trim($2)))
	### not a uri
	o = "LITERAL"
	edgelist[s " -> " o " [label=\"" p":"q ]++
	nodelist[o " [shape=record];"]++
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist) print edge " (" edgelist[edge] ")\"];"
	for(node in nodelist) print node
	print "labelloc=\"t\";"
	title = final(FILENAME)
	now = systime()
	# return date(1)-style output
	datestamp = strftime("%Y%m%d", now)
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}