Remove extraneous \

Remove extraneous \

# Produce a raw data spreadsheet from fields in BritBox Catalog --  apple_catalogue_feed.xml
# The "showContentId" from "tv_show" items are needed to process "tv_season" and "tv_episode" items,
# so you should sort the catalog to ensure items are in the proper order.

# INVOCATION:
#    awk -v ERRORS=$ERRORS -v IDS_SEASONS=$IDS_SEASONS -v IDS_EPISODES=$IDS_EPISODES \
#        -v RAW_TITLES=$RAW_TITLES -v RAW_CREDITS=$RAW_CREDITS -f getBBoxCatalogFromSitemap.awk \
#        $SORTED_SITEMAP >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title         3 Seasons          4 Episodes         5 Duration      6 Genre
#     7 Year          8 Rating        9 Description     10 Content_Type    11 Content_ID   12 Entity_ID
#    13 Show_Type    14 Date_Type    15 Original_Date   16 Show_ID         17 Season_ID    18 Sn_#
#    19 Ep_#         20 1st_#        21 Last_#

BEGIN {
    # Print spreadsheet header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tGenre\tYear\tRating\tDescription\t")
    printf ("Content_Type\tContent_ID\tEntity_ID\tShow_Type\tDate_Type\tOriginal_Date\t")
    printf ("Show_ID\tSeason_ID\tSn_#\tEp_#\t1st_#\tLast_#\n")

    # Print credits  header
    printf ("Person\tRole\tShow_Type\tShow_Title\tCharacter_Name\n") > RAW_CREDITS
}

# Process credits
/<credit role=/ {
    split ($0,fld,"\"")
    person_role = fld[2]
    next
}
#
/<name locale="en-US">/ {
    split ($0,fld,"[<>]")
    person_name = fld[3]
    next
}
#
/<characterName locale="en-US">/ {
    split ($0,fld,"[<>]")
    char_name = fld[3]
    sub (/^[[:space:]]+/,"",char_name)
    # Special case
    # Fix anomalous line with embedded tab ".^I Charlotte Edalji" in "Arthur and George"
    sub (/^.[[:space:]]+/,"",char_name)
    next
}
#
/<\/credit>/ {
    if (contentType == "movie" || contentType == "tv_show") {
        totalCredits += 1
        # "movie" is too common to be in a key field, use "tv_movie"
        contentType == "movie" ? tvShowType = "tv_movie" : tvShowType = contentType
        printf ("%s\t%s\t%s\t%s\t%s\n", person_name, person_role, tvShowType, title,
                char_name) >> RAW_CREDITS
    }
    person_role = ""
    person_name = ""
    char_name = ""
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    firstLineNum = NR
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType > "/dev/stderr"
    # print "contentId = " contentId > "/dev/stderr"
    #
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = ""
    numEpisodes = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title > "/dev/stderr"
    gsub (/&/,"\\&",title)
    sub (/[[:space:]]+$/,"",title)
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    genre = toupper(substr(genre,1,1)) substr(genre,2)
    sub (/Sci_fi/,"Sci-Fi",genre)
    sub (/Special_interest/,"Special interest",genre)
    # print "genre = " genre > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description > "/dev/stderr"
    gsub (/&/,"\\&",description)
    gsub (/\t/," – ",description)
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType > "/dev/stderr"
    # print "originalDate = " originalDate > "/dev/stderr"
    # print "year = " year > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId > "/dev/stderr"
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    lastLineNum = NR

    # "The Moonstone" needs to be revised to avoid duplicate names
    if (title == "The Moonstone") {
        if (EntityId == "_9283") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'The Moonstone (2016)'\n", title) >> ERRORS
            title = "The Moonstone (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERRORS
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERRORS
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night's Dream (1981)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night's Dream (2016)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERRORS
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERRORS
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    if (EntityId == "") {
        missingEntityIds += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s at line %d\n", contentType, contentId,
                title, showContentId, firstLineNum) >> ERRORS
    }

    if (contentType == "movie") {
        totalMovies += 1
        # print "\ntv_movie" > "/dev/stderr"
        # Wish I didn't have to do this, but "movie" is too common to be in a key field
        contentType = "tv_movie"
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
    }

    if (contentType == "tv_show") {
        totalShows += 1
        # print "\ntv_show" > "/dev/stderr"
        showArray[totalShows] = contentId
        titleArray[totalShows] = title
        sortkey = sprintf ("%s (1) S%s %s", title, EntityId, contentId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
        cmd1 = "grep -c " contentId " " IDS_SEASONS
        cmd1 | getline numSeasons
        close (cmd1)
        #
        cmd2 = "grep -c " contentId " " IDS_EPISODES
        cmd2 | getline numEpisodes
        close (cmd2)
    }

    if (contentType == "tv_season") {
        totalSeasons += 1
        # print "\ntv_season" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02d %s", showTitle, seasonNumber, showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02d %s", showTitle, seasonNumber, showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02d, %s", showTitle, seasonNumber, title)
    }

    if (contentType == "tv_episode") {
        totalEpisodes += 1
        # print "\ntv_episode" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                    showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02dE%03d, %s", showTitle, seasonNumber, episodeNumber, title)
    }

    # Generate a link that will lead to the show on BritBox
    # https://www.britbox.com/us/movie/A_Queen_Is_Crowned_13551
    # https://www.britbox.com/us/movie/_13551
    #
    # https://www.britbox.com/us/show/All_Creatures_Great_and_Small_23737
    # https://www.britbox.com/us/show/_23737
    #
    # https://www.britbox.com/us/season/All_Creatures_Great_and_Small_S2_23752
    # https://www.britbox.com/us/season/_23752
    #
    # https://www.britbox.com/us/episode/All_Creatures_Great_and_Small_S2_E2_23764
    # https://www.britbox.com/us/episode/_23764
    showType_URL = contentType "/"
    sub (/tv_/,"",showType_URL)
    full_URL = "https://www.britbox.com/us/" showType_URL EntityId
    fullTitle = "=HYPERLINK(\"" full_URL "\";\"" title "\")"
    # print "fullTitle = " fullTitle > "/dev/stderr"

    # A seasonless "show" should have blank rather than 0 seasons
    if (showType == "seasonless")
        numSeasons = ""
    # If an "episode" sortkey or fullTitle contains "S00E" delete the "S00" part
    sub (/S00E/,"E",sortkey)
    sub (/S00E/,"E",fullTitle)
    # Print everything except tv_seasons
    if (contentType != "tv_season")
        printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%06d\t%06d\n",
                sortkey, fullTitle, numSeasons, numEpisodes, duration, genre, year, rating, description,
                contentType, contentId, EntityId, showType, dateType, originalDate, showContentId,
                seasonContentId, seasonNumber, episodeNumber, firstLineNum, lastLineNum)
}

END {
    printf ("In getBBoxCatalogFromSitemap.awk\n") > "/dev/stderr"

    totalMovies == 1 ? pluralMovies = "movie" : pluralMovies = "movies"
    totalShows == 1 ? pluralShows = "show" : pluralShows = "shows"
    totalSeasons == 1 ? pluralSeasons = "season" : pluralSeasons = "seasons"
    totalEpisodes == 1 ? pluralEpisodes = "episode" : pluralEpisodes = "episodes"
    totalCredits == 1 ? pluralCredits = "credit" : pluralCredits = "credits"
    #
    printf ("    Processed %d %s, %d %s, %d %s, %d %s, %d %s\n", totalMovies, pluralMovies,
            totalShows, pluralShows, totalSeasons, pluralSeasons, totalEpisodes, pluralEpisodes,
            totalCredits, pluralCredits) > "/dev/stderr"

    if (missingEntityIds > 0 ) {
        missingEntityIds == 1 ? plural = "EntityId" : plural = "EntityIds"
        printf ("%8d missing %s in %s\n", missingEntityIds, plural, FILENAME) > "/dev/stderr"
    }

    if (missingShowContentIds > 0 ) {
        missingShowContentIds == 1 ? plural = "showContentId" : plural = "showContentIds"
        printf ("%8d missing %s in %s\n", missingShowContentIds, plural, FILENAME) > "/dev/stderr"
    }

    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? plural = "title" : plural = "titles"
        printf ("%8d %s revised in %s\n", revisedTitles, plural, FILENAME) > "/dev/stderr"
    }
}

# Produce a raw data spreadsheet from fields in BritBox Catalog --  apple_catalogue_feed.xml
# The "showContentId" from "tv_show" items are needed to process "tv_season" and "tv_episode" items,
# so you should sort the catalog to ensure items are in the proper order.

# INVOCATION:
#    awk -v ERRORS=$ERRORS -v IDS_SEASONS=$IDS_SEASONS -v IDS_EPISODES=$IDS_EPISODES \
#        -v RAW_TITLES=$RAW_TITLES -v RAW_CREDITS=$RAW_CREDITS -f getBBoxCatalogFromSitemap.awk \
#        $SORTED_SITEMAP >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title         3 Seasons          4 Episodes         5 Duration      6 Genre
#     7 Year          8 Rating        9 Description     10 Content_Type    11 Content_ID   12 Entity_ID
#    13 Show_Type    14 Date_Type    15 Original_Date   16 Show_ID         17 Season_ID    18 Sn_#
#    19 Ep_#         20 1st_#        21 Last_#

BEGIN {
    # Print spreadsheet header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tGenre\tYear\tRating\tDescription\t")
    printf ("Content_Type\tContent_ID\tEntity_ID\tShow_Type\tDate_Type\tOriginal_Date\t")
    printf ("Show_ID\tSeason_ID\tSn_#\tEp_#\t1st_#\tLast_#\n")

    # Print credits  header
    printf ("Person\tRole\tShow_Type\tShow_Title\tCharacter_Name\n") > RAW_CREDITS
}

# Process credits
/<credit role=/ {
    split ($0,fld,"\"")
    person_role = fld[2]
    next
}
#
/<name locale="en-US">/ {
    split ($0,fld,"[<>]")
    person_name = fld[3]
    next
}
#
/<characterName locale="en-US">/ {
    split ($0,fld,"[<>]")
    char_name = fld[3]
    sub (/^[[:space:]]+/,"",char_name)
    # Special case
    # Fix anomalous line with embedded tab ".^I Charlotte Edalji" in "Arthur and George"
    sub (/^.[[:space:]]+/,"",char_name)
    next
}
#
/<\/credit>/ {
    if (contentType == "movie" || contentType == "tv_show") {
        totalCredits += 1
        # "movie" is too common to be in a key field, use "tv_movie"
        contentType == "movie" ? tvShowType = "tv_movie" : tvShowType = contentType
        printf ("%s\t%s\t%s\t%s\t%s\n", person_name, person_role, tvShowType, title,
                char_name) >> RAW_CREDITS
    }
    person_role = ""
    person_name = ""
    char_name = ""
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    firstLineNum = NR
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType > "/dev/stderr"
    # print "contentId = " contentId > "/dev/stderr"
    #
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = ""
    numEpisodes = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title > "/dev/stderr"
    gsub (/&/,"\\&",title)
    sub (/[[:space:]]+$/,"",title)
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    genre = toupper(substr(genre,1,1)) substr(genre,2)
    sub (/Sci_fi/,"Sci-Fi",genre)
    sub (/Special_interest/,"Special interest",genre)
    # print "genre = " genre > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description > "/dev/stderr"
    gsub (/&/,"\\&",description)
    gsub (/\t/," – ",description)
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType > "/dev/stderr"
    # print "originalDate = " originalDate > "/dev/stderr"
    # print "year = " year > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId > "/dev/stderr"
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    lastLineNum = NR

    # "The Moonstone" needs to be revised to avoid duplicate names
    if (title == "The Moonstone") {
        if (EntityId == "_9283") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'The Moonstone (2016)'\n", title) >> ERRORS
            title = "The Moonstone (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERRORS
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERRORS
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERRORS
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERRORS
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    if (EntityId == "") {
        missingEntityIds += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s at line %d\n", contentType, contentId,
                title, showContentId, firstLineNum) >> ERRORS
    }

    if (contentType == "movie") {
        totalMovies += 1
        # print "\ntv_movie" > "/dev/stderr"
        # Wish I didn't have to do this, but "movie" is too common to be in a key field
        contentType = "tv_movie"
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
    }

    if (contentType == "tv_show") {
        totalShows += 1
        # print "\ntv_show" > "/dev/stderr"
        showArray[totalShows] = contentId
        titleArray[totalShows] = title
        sortkey = sprintf ("%s (1) S%s %s", title, EntityId, contentId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
        cmd1 = "grep -c " contentId " " IDS_SEASONS
        cmd1 | getline numSeasons
        close (cmd1)
        #
        cmd2 = "grep -c " contentId " " IDS_EPISODES
        cmd2 | getline numEpisodes
        close (cmd2)
    }

    if (contentType == "tv_season") {
        totalSeasons += 1
        # print "\ntv_season" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02d %s", showTitle, seasonNumber, showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02d %s", showTitle, seasonNumber, showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02d, %s", showTitle, seasonNumber, title)
    }

    if (contentType == "tv_episode") {
        totalEpisodes += 1
        # print "\ntv_episode" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                    showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02dE%03d, %s", showTitle, seasonNumber, episodeNumber, title)
    }

    # Generate a link that will lead to the show on BritBox
    # https://www.britbox.com/us/movie/A_Queen_Is_Crowned_13551
    # https://www.britbox.com/us/movie/_13551
    #
    # https://www.britbox.com/us/show/All_Creatures_Great_and_Small_23737
    # https://www.britbox.com/us/show/_23737
    #
    # https://www.britbox.com/us/season/All_Creatures_Great_and_Small_S2_23752
    # https://www.britbox.com/us/season/_23752
    #
    # https://www.britbox.com/us/episode/All_Creatures_Great_and_Small_S2_E2_23764
    # https://www.britbox.com/us/episode/_23764
    showType_URL = contentType "/"
    sub (/tv_/,"",showType_URL)
    full_URL = "https://www.britbox.com/us/" showType_URL EntityId
    fullTitle = "=HYPERLINK(\"" full_URL "\";\"" title "\")"
    # print "fullTitle = " fullTitle > "/dev/stderr"

    # A seasonless "show" should have blank rather than 0 seasons
    if (showType == "seasonless")
        numSeasons = ""
    # If an "episode" sortkey or fullTitle contains "S00E" delete the "S00" part
    sub (/S00E/,"E",sortkey)
    sub (/S00E/,"E",fullTitle)
    # Print everything except tv_seasons
    if (contentType != "tv_season")
        printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%06d\t%06d\n",
                sortkey, fullTitle, numSeasons, numEpisodes, duration, genre, year, rating, description,
                contentType, contentId, EntityId, showType, dateType, originalDate, showContentId,
                seasonContentId, seasonNumber, episodeNumber, firstLineNum, lastLineNum)
}

END {
    printf ("In getBBoxCatalogFromSitemap.awk\n") > "/dev/stderr"

    totalMovies == 1 ? pluralMovies = "movie" : pluralMovies = "movies"
    totalShows == 1 ? pluralShows = "show" : pluralShows = "shows"
    totalSeasons == 1 ? pluralSeasons = "season" : pluralSeasons = "seasons"
    totalEpisodes == 1 ? pluralEpisodes = "episode" : pluralEpisodes = "episodes"
    totalCredits == 1 ? pluralCredits = "credit" : pluralCredits = "credits"
    #
    printf ("    Processed %d %s, %d %s, %d %s, %d %s, %d %s\n", totalMovies, pluralMovies,
            totalShows, pluralShows, totalSeasons, pluralSeasons, totalEpisodes, pluralEpisodes,
            totalCredits, pluralCredits) > "/dev/stderr"

    if (missingEntityIds > 0 ) {
        missingEntityIds == 1 ? plural = "EntityId" : plural = "EntityIds"
        printf ("%8d missing %s in %s\n", missingEntityIds, plural, FILENAME) > "/dev/stderr"
    }

    if (missingShowContentIds > 0 ) {
        missingShowContentIds == 1 ? plural = "showContentId" : plural = "showContentIds"
        printf ("%8d missing %s in %s\n", missingShowContentIds, plural, FILENAME) > "/dev/stderr"
    }

    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? plural = "title" : plural = "titles"
        printf ("%8d %s revised in %s\n", revisedTitles, plural, FILENAME) > "/dev/stderr"
    }
}


Don't pass /& to csv files

Don't pass /& to csv files

BEGIN {
    FS="[<>]"
}

# Eliminate unneeded constant fields from XML
/^<item>/ {
    sub (/^<item>/,"")
    sub (/<\/item>$/,"")
    gsub (/<value>/,"")
    gsub (/<\/value>/,"")
    gsub (/&/,"\\&")
}

# if needed for debugging record placment, replace "/nosuchrecord/" below
/nosuchrecord/ {
    print ""
    print NR " - " $0
    for ( i = 1; i <= NF; i++ ) {
        if ($i ~ /parse/ )
            print "field " i " = " $i
    }
}

# Example input lines
#
# <URL>/us/movie/70_Glorious_Years_13550</URL><Title>70 Glorious Years</Title><Subtitle></Subtitle>\
# <Duration>60</Duration><Year>1996</Year><Rating>TV-PG</Rating>
#
# <URL>/us/episode/Vera_S3_E1_13502</URL><Title>Vera</Title><Subtitle>Season 3, Castles in The Air</Subtitle>\
# <Duration>88 min</Duration><Year>2013</Year><Rating>TV-MA</Rating>
#
# <URL>/us/season/Vera_S8_15720</URL><Title>Vera</Title><SeasonNumber>8</SeasonNumber>\
# <Year>2018</Year><NumEpisodes>4</NumEpisodes>

/^<URL>\/us\// {
    URL = $3
    showTitle = $7

    # Titles starting with "The" should not sort based on "The"
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
}

/^<URL>\/us\/episode\/|^<URL>\/us\/movie\// {
    Title = $11
    Duration = $15
    Year = $19
    Rating = $23
    Description = $27
    topURL = $31
    functionName = $35

    # Convert duration from minutes to HMS
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
}

/^<URL>\/us\/movie\// {
    # Extract movie sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_[[:digit:]]*$/) {
        sortkey = sprintf ("M%05d", fld[nflds])
        printf \
            ("%s %s - mv\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description, topURL, functionName)
    }
}

/^<URL>\/us\/episode\// {
    # Extract episode sortkey from URL
    nflds = split (URL,fld,"_")
    seasonNumber = fld[nflds - 2]
    sub (/S/,"",seasonNumber)
    episodeNumber = fld[nflds - 1]
    sub (/E/,"",episodeNumber)
    sortkey = sprintf ("S%02dE%02d", seasonNumber, episodeNumber)

    # Use X plus shownumber as sortkey for specials
    if (URL ~ /_Special_[[:digit:]]*$/)
        sortkey = sprintf ("X%05d", fld[nflds])

    # Get rid of perfunctory "Season <n>, " from episode title
    episodePrefix = "^Season " seasonNumber ", "
    sub (episodePrefix, "", Title)

    if (episodeNumber != "") {
        printf \
            ("%s %s - ep\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description, topURL, functionName)
    }
}

/^<URL>\/us\/season\// {
    seasonNumber = $11
    seasonYear = $15
    seasonEpisodes = $19
    seasonDescription = $23
    topURL = $27
    functionName = $31

    if (seasonNumber != "") {
        printf \
            ("%s S%02d - sn\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, Season %d\"\)\t%s\t\t%s\t%s\t%s\t%s\t%s\n", \
                 showTitle, seasonNumber, URL, showTitle, seasonNumber, \
                 seasonEpisodes, seasonYear, seasonRating, seasonDescription, topURL, functionName)
    }
}

BEGIN {
    FS="[<>]"
}

# Eliminate unneeded constant fields from XML
/^<item>/ {
    sub (/^<item>/,"")
    sub (/<\/item>$/,"")
    gsub (/<value>/,"")
    gsub (/<\/value>/,"")
}

# if needed for debugging record placment, replace "/nosuchrecord/" below
/nosuchrecord/ {
    print ""
    print NR " - " $0
    for ( i = 1; i <= NF; i++ ) {
        if ($i ~ /parse/ )
            print "field " i " = " $i
    }
}

# Example input lines
#
# <URL>/us/movie/70_Glorious_Years_13550</URL><Title>70 Glorious Years</Title><Subtitle></Subtitle>\
# <Duration>60</Duration><Year>1996</Year><Rating>TV-PG</Rating>
#
# <URL>/us/episode/Vera_S3_E1_13502</URL><Title>Vera</Title><Subtitle>Season 3, Castles in The Air</Subtitle>\
# <Duration>88 min</Duration><Year>2013</Year><Rating>TV-MA</Rating>
#
# <URL>/us/season/Vera_S8_15720</URL><Title>Vera</Title><SeasonNumber>8</SeasonNumber>\
# <Year>2018</Year><NumEpisodes>4</NumEpisodes>

/^<URL>\/us\// {
    URL = $3
    showTitle = $7

    # Titles starting with "The" should not sort based on "The"
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
}

/^<URL>\/us\/episode\/|^<URL>\/us\/movie\// {
    Title = $11
    Duration = $15
    Year = $19
    Rating = $23
    Description = $27
    topURL = $31
    functionName = $35

    # Convert duration from minutes to HMS
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
}

/^<URL>\/us\/movie\// {
    # Extract movie sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_[[:digit:]]*$/) {
        sortkey = sprintf ("M%05d", fld[nflds])
        printf \
            ("%s %s - mv\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description, topURL, functionName)
    }
}

/^<URL>\/us\/episode\// {
    # Extract episode sortkey from URL
    nflds = split (URL,fld,"_")
    seasonNumber = fld[nflds - 2]
    sub (/S/,"",seasonNumber)
    episodeNumber = fld[nflds - 1]
    sub (/E/,"",episodeNumber)
    sortkey = sprintf ("S%02dE%02d", seasonNumber, episodeNumber)

    # Use X plus shownumber as sortkey for specials
    if (URL ~ /_Special_[[:digit:]]*$/)
        sortkey = sprintf ("X%05d", fld[nflds])

    # Get rid of perfunctory "Season <n>, " from episode title
    episodePrefix = "^Season " seasonNumber ", "
    sub (episodePrefix, "", Title)

    if (episodeNumber != "") {
        printf \
            ("%s %s - ep\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description, topURL, functionName)
    }
}

/^<URL>\/us\/season\// {
    seasonNumber = $11
    seasonYear = $15
    seasonEpisodes = $19
    seasonDescription = $23
    topURL = $27
    functionName = $31

    if (seasonNumber != "") {
        printf \
            ("%s S%02d - sn\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, Season %d\"\)\t%s\t\t%s\t%s\t%s\t%s\t%s\n", \
                 showTitle, seasonNumber, URL, showTitle, seasonNumber, \
                 seasonEpisodes, seasonYear, seasonRating, seasonDescription, topURL, functionName)
    }
}


Filter out current year

Filter out current year

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@24401 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back]

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if ending year is gt or eq to contract year
  current_year = 2013
  end_year = 0
  incontract = 0
  test_year = ""
  if ($7 == "") {
    test_year = $3
  } else {
    test_year = $7
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  #printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$7,end_year,test_year
  if (end_year >= test_year) {
    if ((end_year < current_year) || ((end_year >= current_year) && (incl_cur == 1))) {
      incontract = 1
    }
  }

  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      k[pn] = $7
      d[pn] = $4
      r[pn] = $5
      pn++
    }
    b[$1,lp2,$4]++
    c[$1,lp2,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],d[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back]

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if ending year is gt or eq to contract year
  end_year = 0
  incontract = 0
  test_year = ""
  if ($7 == "") {
    test_year = $3
  } else {
    test_year = $7
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  #printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$7,end_year,test_year
  if (end_year >= test_year) {
    incontract = 1
  }

  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      k[pn] = $7
      d[pn] = $4
      r[pn] = $5
      pn++
    }
    b[$1,lp2,$4]++
    c[$1,lp2,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],d[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}


bump version to 0.9.1-dev

bump version to 0.9.1-dev

BEGIN {
    Name        = "Translate Shell"
    Description = "Google Translate to serve as a command-line tool"
    Version     = "0.9.1-dev"
    ReleaseDate = "2015-06-08"
    Command     = "trans"
    EntryPoint  = "translate.awk"
}

BEGIN {
    Name        = "Translate Shell"
    Description = "Google Translate to serve as a command-line tool"
    Version     = "0.9.0.7"
    ReleaseDate = "2015-06-08"
    Command     = "trans"
    EntryPoint  = "translate.awk"
}


Fixes for untime

Fixes for untime

Skip RIL files, and a sp fix I missed

# remove time stamp
/^--- stock_/  { print $1 " " $2; next; }
/^+++ modded_/ { print $1 " " $2; next; }

# convert obsolete form
/^diff -Npru /  { sub(/diff -Npru /, "diff -NpruEbB -x '*RIL'"); }
/stock-/  { sub(/stock-/, "stock_"); sub(/modded-/, "modded_"); print; next; }

# keep rest of file as is:
{ print; }

# remove time stamp
/^--- stock_/  { print $1 " " $2; next; }
/^+++ moded_/ { print $1 " " $2; next; }

# convert obsolete form
/^diff -Npru /  { sub(/diff -Npru /, "diff -NpruEbB "); }
/stock-/  { sub(/stock-/, "stock_"); sub(/modded-/, "modded_"); print; next; }

# keep rest of file as is:
{ print; }


Use float for calculating fractional part of STARTTIME to support busybox awk.
Also print used start time is various representations.

Compared to gawk, busybox awk does only support 32 bit integers.

{
    if (1==NR)
    {
        startTimestamp = substr($1, 2, length($1) - 2)
        printf(";$FILEVERSION=1.3\n")
        printf(";$STARTTIME=%u.%.0f\n", mktime(strftime("%Y %m %d 0 0 0", startTimestamp)) / 86400 + 25569, \
                10000000000.0 / 86400.0 * (mktime(strftime("%Y %m %d %H %M %S", startTimestamp)) - mktime(strftime("%Y %m %d 0 0 0", startTimestamp))))
        printf(";\n; Start time: %s\n", startTimestamp)
        printf(";             %u\n", mktime(strftime("%Y %m %d %H %M %S", startTimestamp)))
        printf(";             %s\n;\n", strftime("%Y-%m-%dT%H:%M:%S%z (%Z)", startTimestamp))
        currentTimeOffset = 0.0
    }
    else
    {
        previousTimeOffset = currentTimeOffset
        currentTimestamp = substr($1, 2, length($1) - 2)
        currentTimeOffset = currentTimestamp - startTimestamp
        if (currentTimeOffset < 0)
        {
            startTimestamp = currentTimestamp + currentTimeOffset
            currentTimeOffset = previousTimeOffset
        }
    }
    printf("%u) %.3f 1 Rx %s - %s", NR, currentTimeOffset * 1000, $3, substr($4, 2, length($4) - 2))
    if (5<=NF)
    {
        for(i=5;i<=NF;i++)
        {
            printf " "$i
        }
    }
    printf "\n"
}

{
    if (1==NR)
    {
        startTimestamp = substr($1, 2, length($1) - 2)
        printf(";$FILEVERSION=1.3\n")
        printf(";$STARTTIME=%u.%u\n", mktime(strftime("%Y %m %d 0 0 0", startTimestamp)) / 86400 + 25569, \
                10000000000 / 86400 * (mktime(strftime("%Y %m %d %H %M %S", startTimestamp)) - mktime(strftime("%Y %m %d 0 0 0", startTimestamp))))
        currentTimeOffset = 0.0
    }
    else
    {
        previousTimeOffset = currentTimeOffset
        currentTimestamp = substr($1, 2, length($1) - 2)
        currentTimeOffset = currentTimestamp - startTimestamp
        if (currentTimeOffset < 0)
        {
            startTimestamp = currentTimestamp + currentTimeOffset
            currentTimeOffset = previousTimeOffset
        }
    }
    printf("%u) %.3f 1 Rx %s - %s", NR, currentTimeOffset * 1000, $3, substr($4, 2, length($4) - 2))
    if (5<=NF)
    {
        for(i=5;i<=NF;i++)
        {
            printf " "$i
        }
    }
    printf "\n"
}


make this extensible and do not omit inherited without patterns

Change-Id: I97a554169637740aab4282b303a32f476f294e2f

#!/usr/bin/gawk -f
# -*- Mode: awk; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
#
# Copyright 2012 LibreOffice contributors.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Usage: gawk -f list-dateacceptancepattern.awk *.xml [--html]
#
# Outputs three lists of locales, one with DateAcceptancePattern elements
# defined, one with inherited LC_FORMAT elements and thus date patterns, and
# one where no DateAcceptancePattern are defined.
#
# If --html is given as the last parameter, format output suitable for
# inclusion in HTML.

BEGIN {
    html = 0
    if (ARGV[ARGC-1] == "--html") {
        html = 1
        --ARGC
    }
    file = ""
    offlocale = 0
    offpatterns = 1
    offinherit = 2
    inheritedcount = 0
    if (html)
        print "
"
    else
        print ""
    print "Locales with explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
}

file != FILENAME {
    if (file)
        endFile()
    file = FILENAME
    patterns = 0
    inherited = ""
}

/<DateAcceptancePattern>/ {
    split( $0, a, /<|>/ )
    pattern[patterns++] = a[3]
}

# pattern inherited as well
/<LC_FORMAT[^>]* ref="[^>"]+"[^>]*>/ {
    split( $0, a, /.* ref="|"/ )
    inherited = a[2]
}

END {
    if (file)
        endFile()

    PROCINFO["sorted_in"] = "@ind_str_asc"

    fillAllInherited()

    if (html)
    {
        print "
"
        print "\n
"
    }
    else
        print "\n"

    print "Locales inheriting patterns:"
    if (html)
    {
        print "
"
        for (i in LocaleList)
        {
            if (LocaleList[i][offinherit] && LocaleList[i][offpatterns])
                print LocaleList[i][offlocale] " = " LocaleList[i][offinherit] "    "
        }
        print "\n
"
    }
    else
    {
        for (i in LocaleList)
        {
            if (LocaleList[i][offinherit] && LocaleList[i][offpatterns])
                print LocaleList[i][offlocale] " = " LocaleList[i][offinherit]
        }
        print "\n"
    }

    print "Locales without explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
    print "(one implicit full date pattern is always generated)"
    if (html)
        print "
"
    if (html)
    {
        for (i in LocaleList)
        {
            if (!LocaleList[i][offpatterns])
                print LocaleList[i][offlocale] "    "
        }
    }
    else
    {
        for (i in LocaleList)
        {
            if (!LocaleList[i][offpatterns])
                print LocaleList[i][offlocale]
        }
    }
}

function endFile() {
    locale =  getLocale( file)
    LocaleList[locale][offlocale] = locale
    LocaleList[locale][offpatterns] = patterns
    if (patterns)
    {
        if (html)
        {
            print "
 " locale ":"
            print "
"
            for ( i=0; i<patterns; ++i )
            {
                print "
 " pattern[i]
            }
            print "
"
        }
        else
        {
            print locale ":"
            for ( i=0; i<patterns; ++i )
            {
                print "    " pattern[i]
            }
        }
    }
    else if (inherited)
        LocaleList[locale][offinherit] = inherited
}

function getLocale( file,       tmp ) {
    tmp = file
    gsub( /.*\//, "", tmp )
    gsub( /\.xml/, "", tmp )
    return tmp
}

function fillInherited( locale ) {
    if (!LocaleList[locale][offpatterns] && LocaleList[locale][offinherit])
        LocaleList[locale][offpatterns] = fillInherited( LocaleList[locale][offinherit])
    return LocaleList[locale][offpatterns]
}

function fillAllInherited(      i ) {
    for (i in LocaleList)
    {
        if (!LocaleList[i][offpatterns] && LocaleList[i][offinherit])
            LocaleList[i][offpatterns] = fillInherited( LocaleList[i][offinherit])
    }
}

# vim:set shiftwidth=4 softtabstop=4 expandtab:

#!/usr/bin/gawk -f
# -*- Mode: awk; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
#
# Copyright 2012 LibreOffice contributors.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Usage: gawk -f list-dateacceptancepattern.awk *.xml [--html]
#
# Outputs three lists of locales, one with DateAcceptancePattern elements
# defined, one with inherited LC_FORMAT elements and thus date patterns, and
# one where no DateAcceptancePattern are defined.
#
# If --html is given as the last parameter, format output suitable for
# inclusion in HTML.

BEGIN {
    html = 0
    if (ARGV[ARGC-1] == "--html") {
        html = 1
        --ARGC
    }
    file = ""
    nopatterns = 0
    inheritedcount = 0
    if (html)
        print "
"
    else
        print ""
    print "Locales with explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
}

file != FILENAME {
    if (file)
        endFile()
    file = FILENAME
    patterns = 0
    inherited = ""
}

/<DateAcceptancePattern>/ {
    split( $0, a, /<|>/ )
    pattern[patterns++] = a[3]
}

# pattern inherited as well
/<LC_FORMAT[^>]* ref="[^>"]+"[^>]*>/ {
    split( $0, a, /.* ref="|"/ )
    inherited = a[2]
}

END {
    if (file)
        endFile()
    if (html)
    {
        print "
"
        print "\n
"
    }
    else
        print "\n"

    print "Locales inheriting patterns:"
    if (html)
    {
        print "
"
        for (i=0; i<inheritedcount; ++i)
        {
            if (LocaleHasPatterns[InheritedList[i][1]])
                print InheritedList[i][0] " = " InheritedList[i][1] "    "
        }
        print "\n
"
    }
    else
    {
        for (i=0; i<inheritedcount; ++i)
        {
            if (LocaleHasPatterns[InheritedList[i][1]])
                print InheritedList[i][0] " = " InheritedList[i][1]
        }
        print "\n"
    }

    print "Locales without explicit DateAcceptancePattern elements:"
    if (html)
        print "
"
    print "(one implicit full date pattern is always generated)"
    if (html)
        print "
"
    if (html)
    {
        for (i=0; i<nopatterns; ++i)
        {
            print NoPatternList[i] "    "
        }
    }
    else
    {
        for (i=0; i<nopatterns; ++i)
        {
            print NoPatternList[i]
        }
    }
}

function endFile() {
    locale =  getLocale( file)
    LocaleHasPatterns[locale] = patterns
    if (patterns)
    {
        if (html)
        {
            print "
 " locale ":"
            print "
"
            for ( i=0; i<patterns; ++i )
            {
                print "
 " pattern[i]
            }
            print "
"
        }
        else
        {
            print locale ":"
            for ( i=0; i<patterns; ++i )
            {
                print "    " pattern[i]
            }
        }
    }
    else if (inherited)
    {
        InheritedList[inheritedcount][0] = locale
        InheritedList[inheritedcount][1] = inherited
        ++inheritedcount
    }
    else
        NoPatternList[nopatterns++] = locale
}

function getLocale( file, tmp ) {
    tmp = file
    gsub( /.*\//, "", tmp )
    gsub( /\.xml/, "", tmp )
    return tmp
}

# vim:set shiftwidth=4 softtabstop=4 expandtab:


bump version to 0.9.1-dev

bump version to 0.9.1-dev

BEGIN {
    Name        = "Translate Shell"
    Description = "Google Translate to serve as a command-line tool"
    Version     = "0.9.1-dev"
    ReleaseDate = "2015-09-01"
    Command     = "trans"
    EntryPoint  = "translate.awk"
}

BEGIN {
    Name        = "Translate Shell"
    Description = "Google Translate to serve as a command-line tool"
    Version     = "0.9.0.8"
    ReleaseDate = "2015-09-01"
    Command     = "trans"
    EntryPoint  = "translate.awk"
}


Callouts in docs are now processed language-specific. Python uses '#' to comment them out, all other languages will currently fall back to '//'.

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

#
# @author Daniel Kuppitz (http://gremlin.guru)
#
BEGIN {
  p = 0
  c = "//"
}

function escape_string(string) {
  str = gensub(/\\/, "\\\\\\\\", "g", string)
  return gensub(/'/, "\\\\'", "g", str)
}

function print_string(string) {
  print "pb(" p++ "); '" escape_string(string) "'"
}

function transform_callouts(code) {
  return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, " " c c " \\1", "g", code)
}

function remove_callouts(code) {
  return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, "", "g", code)
}

/^----$/ {
  if (inCodeSection) {
    if (prepared) {
      inCodeSection = 0
      prepared = 0
    } else {
      prepared = 1
    }
  }
  print_string($0)
}

!/^----$/ {
  if (inCodeSection) {
    if ($0 ~ /^:/) {
      print "'" escape_string(transform_callouts($0)) "'"
      print remove_callouts($0)
    } else {
      print transform_callouts($0)
    }
  } else {
    print_string($0)
  }
}

/^\[gremlin-/ {
  inCodeSection = 1
  split($0, a, "-")
  b = gensub(/]'/, "", "g", a[2])
  split(b, l, ",")
  lang = l[1]
  switch (lang) {
    case "python":
      c = "#"
      break
    default:
      c = "//"
      break
  }
}

END {
  print_string("// LAST LINE")
}

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

#
# @author Daniel Kuppitz (http://gremlin.guru)
#
BEGIN {
  p = 0
}

function escape_string(string) {
  str = gensub(/\\/, "\\\\\\\\", "g", string)
  return gensub(/'/, "\\\\'", "g", str)
}

function print_string(string) {
  print "pb(" p++ "); '" escape_string(string) "'"
}

function transform_callouts(code) {
  return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, " //// \\1", "g", code)
}

function remove_callouts(code) {
  return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, "", "g", code)
}

/^----$/ {
  if (inCodeSection) {
    if (prepared) {
      inCodeSection = 0
      prepared = 0
    } else {
      prepared = 1
    }
  }
  print_string($0)
}

!/^----$/ {
  if (inCodeSection) {
    if ($0 ~ /^:/) {
      print "'" escape_string(transform_callouts($0)) "'"
      print remove_callouts($0)
    } else {
      print transform_callouts($0)
    }
  } else {
    print_string($0)
  }
}

/^\[gremlin-/ {
  inCodeSection = 1
}

END {
  print_string("// LAST LINE")
}


documented CSV2RDF4LOD_CONVERT_PERSON_URI and MACHINE and changed output if missing headers2params2.awk

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,whoami);
      if(length(person_uri)) {
                                    printf("   sioc:account_of <%s>;\n",machine_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }
                                     print
                                     print ":dataset a void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami);
   }else if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # TODO: same as above.
   }else if(length(person_uri)  && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n",nowXSD);
   }
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   } # END if(showConversionProcess > 0)
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS);
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,machine_uri,whoami);
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n   sioc:account_of <%s>;\n.\n",machine_uri,whoami,whoami,person_uri);
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier <%s> .\n",person_uri,whoami);
   }
                                     print
                                     print ":dataset a void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami);
   }else if(length(person_uri)) {
                                    printf("      dcterms:creator \"%s\";\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator \"%s\";\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n",nowXSD);
   }
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   } # END if(showConversionProcess > 0)
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS);
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}


fixes comment

fixes comment

# Count discordant paired-end alignments & mates > 500k bases away

# Absolute value
function abs(v) {
	return v < 0 ? -v : v
}

# Round to nearest 100
function rnd(v) {
	r = v / 100
	i = int(r)
	d = r - i
	return d >= 0.5 ? (i + 1) * 100 : i * 100
}

# Parse SAM
# If mate mapped AND (mate is on different chr OR mate is more than 500k away)
# print alignments rounded to nearest 100
# Format: mate 1 chrom TAB mate 1 rounded coordinate TAB mate 2 chrom TAB mate 2 rounded coordinate TAB 1 if secondary else 0
$7 != "*" && ($7 != "=" && $7 != $3 || (abs($4-$8) > 500000)) {
	print $3 "\t" rnd($4) "\t" ($7 == "=" ? $3 : $7) "\t" rnd($8) "\t" (int($2 / 256) % 2)
}

# Count discordant paired-end alignments & mates > 500k bases away

# Absolute value
function abs(v) {
	return v < 0 ? -v : v
}

# Round to nearest 100
function rnd(v) {
	r = v / 100
	i = int(r)
	d = r - i
	return d >= 0.5 ? (i + 1) * 100 : i * 100
}

# Parse SAM
# If primary alignment AND mate mapped AND (mate is on different chr OR mate is more than 500k away)
# print alignments rounded to nearest 100
# Format: mate 1 chrom TAB mate 1 rounded coordinate TAB mate 2 chrom TAB mate 2 rounded coordinate TAB 1 if secondary else 0
$7 != "*" && ($7 != "=" && $7 != $3 || (abs($4-$8) > 500000)) {
	print $3 "\t" rnd($4) "\t" ($7 == "=" ? $3 : $7) "\t" rnd($8) "\t" (int($2 / 256) % 2)
}


Typo.

Typo.

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@7046 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f

#######
# BEGIN
#######

BEGIN {
 # Boolean
 false = 0;
 true = 1;

 # Directives
 DIRECTIVE_FIXED_VALUES        = "@fixedValues"
 DIRECTIVE_FORMAT_STRING       = "@formatString"
 DIRECTIVE_NAME_PREFIX         = "@namePrefix"
 DIRECTIVE_PUBLISHER_TITLE_SET = "@publisherTitleSet"

 # Format codes
 CODE_ATTRIBUTE      = "@"
 CODE_CODEN          = "c"
 CODE_ESTIMATED_SIZE = "e"
 CODE_ISSN           = "i"
 CODE_JOURNAL_TITLE  = "j"
 CODE_PARAMETER      = "%"
 CODE_PLUGIN         = "p"
 CODE_PUBLISHER      = "P"
 CODE_RIGHTS         = "R"
 CODE_SKIP_1         = "x"
 CODE_SKIP_2         = "X"
 CODE_STATUS         = "S"
 CODE_TITLE          = "t"

 # Archival unit statuses
 STATUS_DOES_NOT_EXIST = "does_not_exist"
 STATUS_DO_NOT_PROCESS = "do_not_process"
 STATUS_EXISTS         = "exists"
 STATUS_MANIFEST       = "manifest"
 STATUS_TESTING        = "testing"
 STATUS_READY          = "ready"
 STATUS_PRE_RELEASED   = "pre_released"
 STATUS_RELEASED       = "released"
 STATUS_DOWN           = "down"
 STATUS_SUPERSEDED     = "superseded"
 STATUS_RETRACTED      = "retracted"

 # Output levels and corresponding archival unit statuses
 LEVEL_PRODUCTION              = "production"
 STATUSES_PRODUCTION           = STATUS_RETRACTED "," STATUS_SUPERSEDED "," STATUS_DOWN "," STATUS_RELEASED
 LEVEL_CONTENT_TESTING         = "contentTesting"
 STATUSES_CONTENT_TESTING      = STATUSES_PRODUCTION "," STATUS_PRE_RELEASED "," STATUS_READY "," STATUS_TESTING "," STATUS_MANIFEST "," STATUS_EXISTS
 LEVEL_DANGEROUS               = "dangerous"
 STATUSES_DANGEROUS            = STATUSES_CONTENT_TESTING "," STATUS_DO_NOT_PROCESS
 LEVEL_VERY_DANGEROUS          = "veryDangerous"
 STATUSES_VERY_DANGEROUS       = STATUSES_DANGEROUS "," STATUS_DOES_NOT_EXIST
 LEVEL_PRE_RELEASE             = "preRelease"
 STATUSES_PRE_RELEASE          = STATUS_PRE_RELEASED
 LEVEL_CONSERVATIVE_TESTING    = "conservativeTesting"
 STATUSES_CONSERVATIVE_TESTING = STATUS_RELEASED "," STATUS_PRE_RELEASED "," STATUS_READY "," STATUS_TESTING "," STATUS_MANIFEST
 LEVEL_DEFAULT                 = LEVEL_PRODUCTION

 # Output styles
 STYLE_XML         = "xml"
 STYLE_XML_ENTRIES = "xmlEntries"
 STYLE_XML_LEGACY  = "xmlLegacy"
 STYLE_DEFAULT     = STYLE_XML_ENTRIES

 # Exit codes
 EXIT_NO_FORMAT_STRING = 11

 # Initial setup
 FS="\t"
 parseCommandLine()

 # Preamble
 if (outputStyle == STYLE_XML || outputStyle == STYLE_XML_LEGACY) { preambleXml() }
}

###########
# FUNCTIONS
###########

#
# parseCommandLine
#
function parseCommandLine() {
 # Defaults
 if (outputStyle == "") { outputStyle = STYLE_DEFAULT }
 if (outputLevel == "") { outputLevel = LEVEL_DEFAULT }

 # Parse
 parseOutputLevel()
}

#
# parseOutputLevel
#
function parseOutputLevel(        _splitOutputLevel, _i, _statuses, _splitStatuses, _j) {
 # Reset output level information
 delete outputLevels

 # For each output level...
 split(outputLevel, _splitOutputLevel, ",")
 for (_i in _splitOutputLevel) {
  # Synonyms
  if (_splitOutputLevel[_i] == LEVEL_PRODUCTION) { _statuses = STATUSES_PRODUCTION }
  else if (_splitOutputLevel[_i] == LEVEL_PRE_RELEASE) { _statuses = STATUSES_PRE_RELEASE }
  else if (_splitOutputLevel[_i] == LEVEL_CONTENT_TESTING) { _statuses = STATUSES_CONTENT_TESTING }
  else if (_splitOutputLevel[_i] == LEVEL_CONSERVATIVE_TESTING) { _statuses = STATUSES_CONSERVATIVE_TESTING }
  else if (_splitOutputLevel[_i] == LEVEL_DANGEROUS) { _statuses = STATUSES_DANGEROUS }
  else if (_splitOutputLevel[_i] == LEVEL_VERY_DANGEROUS) { _statuses = STATUSES_VERY_DANGEROUS }
  else { _statuses = _splitOutputLevel[_i] } # Not a synonym
  # For each status...
  split(_statuses, _splitStatuses, ",")
  for (_j in _splitStatuses) { outputLevels[_splitStatuses[_j]] = true }
 }
}

#
# parseFormatString
#
function parseFormatString(        _splitFormatString, _numFields, _field, _code) {
 # Reset format information
 delete indexOf
 numAdditionalParameters = 0
 delete indexAdditionalAttributes
 numParameters = 0
 delete indexParameters
 delete fixedValue

 # For each format code...
 _numFields = split(formatString, _splitFormatString, ",")
 for (_field = 1 ; _field <= _numFields ; ++_field) {
  _code = _splitFormatString[_field]
  if (_code == CODE_SKIP_1 || _code == CODE_SKIP_2) { continue }
  else if (_code == CODE_CODEN || _code == CODE_ESTIMATED_SIZE || _code == CODE_ISSN || _code == CODE_JOURNAL_TITLE || _code == CODE_PLUGIN || _code == CODE_PUBLISHER || _code == CODE_RIGHTS || _code == CODE_STATUS || _code == CODE_TITLE) { indexOf[_code] = _field }
  else if (_code == CODE_ATTRIBUTE) { indexAdditionalAttributes[++numAdditionalAttributes] = _field }
  else if (_code == CODE_PARAMETER) { indexParameters[++numParameters] = _field }
  # FIXME: unknown code
 }
}

#
# parseFixedValues
#
function parseFixedValues(        _i, _code) {
 for (_i = 3 ; _i <= NF ; ++_i) {
  _code = getKey($_i)
  if (_code == "") { continue }
  if (_code == CODE_CODEN || _code == CODE_ISSN || _code == CODE_JOURNAL_TITLE || _code == CODE_PLUGIN || _code == CODE_PUBLISHER || _code == CODE_RIGHTS) {
   indexOf[_code] = -1
   fixedValue[_code] = getValue($_i)
  }
  # FIXME: unknown or illegal code
 }
}

#
# preambleXml
#
function preambleXml() {
 xmlInOrgLockssTitle = false;

 printf   "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 printf   "<!DOCTYPE lockss-config [\n"
 printf   "<!ELEMENT lockss-config (if|property)+>\n"
 printf   "<!ELEMENT property (property|list|value|if)*>\n"
 printf   "<!ELEMENT list (value)+>\n"
 printf   "<!ELEMENT value (#PCDATA)>\n"
 printf   "<!ELEMENT test EMPTY>\n"
 printf   "<!ELEMENT and (and|or|not|test)*>\n"
 printf   "<!ELEMENT or (and|or|not|test)*>\n"
 printf   "<!ELEMENT not (and|or|not|test)*>\n"
 printf   "<!ELEMENT if (and|or|not|then|else|test|property)*>\n"
 printf   "<!ELEMENT then (if|property)*>\n"
 printf   "<!ELEMENT else (if|property)*>\n"
 printf   "<!ATTLIST property name CDATA #REQUIRED>\n"
 printf   "<!ATTLIST property value CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test hostname CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test group CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test daemonVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test daemonVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test daemonVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformName CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if hostname CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if group CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if daemonVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if daemonVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if daemonVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformName CDATA #IMPLIED>\n"
 printf   "<!ATTLIST list append CDATA #IMPLIED>\n"
 printf   "]>\n"
 printf   "\n"
 printf   "<lockss-config>\n"
 printf   "\n"
}

#
# postambleXml
#
function postambleXml() {
 if (xmlInOrgLockssTitle) {
  xmlInOrgLockssTitle = false;
  printf  " </property>\n"
  printf  "\n"
 }
 printf   "</lockss-config>\n"
}

#
# printOneXmlEntry
#
function printOneXmlEntry(        _status, _i) {
 # Skip based on output level
 _status = get(CODE_STATUS)
 if (!outputLevels[_status]) { return }

 # Start config subtree if needed
 if ((outputStyle == STYLE_XML || outputStyle == STYLE_XML_LEGACY) && !xmlInOrgLockssTitle) {
  xmlInOrgLockssTitle = true;
  printf  " <property name=\"org.lockss.title\">\n"
  printf  "\n"
 }

 # Begin
 printf   "  <property name=\"%s\">\n", getOpaqueName(get(CODE_TITLE))

 # First the publisher
 printOneXmlAttribute("publisher=" xml(get(CODE_PUBLISHER)))

 # Then the ISSN, CODEN, journal title, archival unit title and plugin
 printOneXmlProperty("issn=" get(CODE_ISSN))
 printOneXmlProperty("coden=" get(CODE_CODEN))
 printOneXmlProperty("journalTitle=" xml(get(CODE_JOURNAL_TITLE)))
 printOneXmlProperty("title=" xml(get(CODE_TITLE)) (_status == STATUS_PRE_RELEASED ? " (pre-release)" : "") (_status == STATUS_SUPERSEDED ? " (superseded)" : ""))
 printOneXmlProperty("plugin=" get(CODE_PLUGIN))

 # Then the parameters
 for (_i = 1 ; _i <= numParameters ; ++_i) { printOneXmlParameter(_i, $indexParameters[_i]) }
 # Down AUs are denoted by a parameter
 if (_status == STATUS_DOWN || _status == STATUS_SUPERSEDED) { printOneXmlParameter(99, "pub_down=true") }

 # Then any additional attributes
 for (_i = 1 ; _i <= numAdditionalAttributes ; ++_i) { printOneXmlAttribute($indexAdditionalAttributes[_i]) }

 # Then other tidbits
 if (get(CODE_RIGHTS) == "openaccess") { printOneXmlAttribute("rights=openaccess") }
 printOneXmlProperty("estSize=" get(CODE_ESTIMATED_SIZE))
 if (_status == STATUS_PRE_RELEASED) { printOneXmlAttribute("releaseStatus=pre-release") }

 # End
 printf   "  </property>\n"
 printf   "\n"
}

#
# printOneXmlPublisherTitleSet
#
function printOneXmlPublisherTitleSet(        _pub) {
 if (getKey($3) == "publisher") {
  if (xmlInOrgLockssTitle) {
   xmlInOrgLockssTitle = false;
   printf " </property>\n"
   printf "\n"
  }
  _pub = xml(getValue($3))
  printf  " <property name=\"org.lockss.titleSet\">\n"
  printf  "\n"
  printf  "  <property name=\"%s\">\n", _pub
  printOneXmlProperty("name=All " _pub " Titles")
  printOneXmlProperty("class=xpath")
  printOneXmlProperty("xpath=[attributes/publisher='" _pub "']")
  printf  "  </property>\n"
  printf  "\n"
  printf  " </property>\n"
  printf  "\n"
 }
 # FIXME: invalid format
}

#
# printOneXmlParameter
#
function printOneXmlParameter(_num, _pair) {
 if (_pair == "") { return }
 printf   "   <property name=\"param.%d\">\n", _num
 printOneXmlProperty("key=" getKey(_pair), " ")
 printOneXmlProperty("value=" getValue(_pair), " ")
 printf   "   </property>\n"
}

#
# printOneXmlAttribute
#
function printOneXmlAttribute(_pair) {
 printOneXmlProperty("attributes." _pair)
}

#
# printOneXmlProperty
#
function printOneXmlProperty(_pair, _pad) {
 if (_pair == "" || getValue(_pair) == "") { return }
 if (_pad != "") { printf "%s", _pad }
 printf   "   <property name=\"%s\" value=\"%s\" />\n", getKey(_pair), getValue(_pair)
}

#
# get
#
function get(_code) {
 if (indexOf[_code] < 0) { return fixedValue[_code] }
 if (indexOf[_code] > 0) { return $indexOf[_code] }
 return "";
}

#
# getOpaqueName
#
function getOpaqueName(_str) {
 gsub(/ Volume /, "", _str); # FIXME
 gsub(/&/, "and", _str);
 gsub(/[^a-zA-Z0-9]/, "", _str);
 return namePrefix _str
}

#
# getKey
#
# Returns the key in a key=value pair.
#
function getKey(_str) {
 return match(_str, /^[^=]+=/) ? substr(_str, 1, RLENGTH - 1) : _str
}

#
# getValue
#
# Returns the value in a key=value pair.
#
function getValue(_str) {
 return match(_str, /^[^=]+=/) ? substr(_str, RLENGTH + 1, length(_str) - RLENGTH) : _str
}

#
# xml
#
# Encodes a string's ampersands and angle brackets for XML output.
# Assumes UTF-8 input; produces UTF-8 output.
#
function xml(_str) {
 gsub(/&/, "\&", _str);
 gsub(/</, "\<", _str);
 gsub(/>/, "\>", _str);
 return _str
}

##########
# PATTERNS
##########

#
# Beginning of file
# unconditional
#
FNR == 1 {
 formatString = ""
 namePrefix = ""
}

#
# Blank line
#
/^\t*$/ {
 next
}

#
# Fixed values line
#
$1 == "#" && $2 == DIRECTIVE_FIXED_VALUES {
 parseFixedValues()
 next
}

#
# Format line
#
$1 == "#" && $2 == DIRECTIVE_FORMAT_STRING {
 formatString = $3
 parseFormatString()
 next
}

#
# Name prefix line
#
$1 == "#" && $2 == DIRECTIVE_NAME_PREFIX {
 namePrefix = $3
 next
}

#
# Publisher title set line
#
$1 == "#" && $2 == DIRECTIVE_PUBLISHER_TITLE_SET {
 if (outputStyle == STYLE_XML) { printOneXmlPublisherTitleSet() }
 next
}

#
# Comment line
#
/^#/ {
 next
}

# Normal line
# unconditional
{
 if (formatString == "") { exit EXIT_NO_FORMAT_STRING }
 if (outputStyle == STYLE_XML || outputStyle == STYLE_XML_ENTRIES || outputStyle == STYLE_XML_LEGACY) { printOneXmlEntry() }
 next
}

#####
# END
#####

END {
 # Postamble
 if (outputStyle == STYLE_XML || outputStyle == STYLE_XML_LEGACY) { postambleXml() }
}

#!/usr/bin/awk -f

#######
# BEGIN
#######

BEGIN {
 # Boolean
 false = 0;
 true = 1;

 # Directives
 DIRECTIVE_FIXED_VALUES        = "@fixedValues"
 DIRECTIVE_FORMAT_STRING       = "@formatString"
 DIRECTIVE_NAME_PREFIX         = "@namePrefix"
 DIRECTIVE_PUBLISHER_TITLE_SET = "@publisherTitleSet"

 # Format codes
 CODE_ATTRIBUTE      = "@"
 CODE_CODEN          = "c"
 CODE_ESTIMATED_SIZE = "e"
 CODE_ISSN           = "i"
 CODE_JOURNAL_TITLE  = "j"
 CODE_PARAMETER      = "%"
 CODE_PLUGIN         = "p"
 CODE_PUBLISHER      = "P"
 CODE_RIGHTS         = "R"
 CODE_SKIP_1         = "x"
 CODE_SKIP_2         = "X"
 CODE_STATUS         = "S"
 CODE_TITLE          = "t"

 # Archival unit statuses
 STATUS_DOES_NOT_EXIST = "does_not_exist"
 STATUS_DO_NOT_PROCESS = "do_not_process"
 STATUS_EXISTS         = "exists"
 STATUS_MANIFEST       = "manifest"
 STATUS_TESTING        = "testing"
 STATUS_READY          = "ready"
 STATUS_PRE_RELEASED   = "pre_released"
 STATUS_RELEASED       = "released"
 STATUS_DOWN           = "down"
 STATUS_SUPERSEDED     = "superseded"
 STATUS_RETRACTED      = "retracted"

 # Output levels and corresponding archival unit statuses
 LEVEL_PRODUCTION              = "production"
 STATUSES_PRODUCTION           = STATUS_RETRACTED "," STATUS_SUPERSEDED "," STATUS_DOWN "," STATUS_RELEASED
 LEVEL_CONTENT_TESTING         = "contentTesting"
 STATUSES_CONTENT_TESTING      = STATUSES_PRODUCTION "," STATUS_PRE_RELEASED "," STATUS_READY "," STATUS_TESTING "," STATUS_MANIFEST "," STATUS_EXISTS
 LEVEL_DANGEROUS               = "dangerous"
 STATUSES_DANGEROUS            = STATUSES_CONTENT_TESTING "," STATUS_DO_NOT_PROCESS
 LEVEL_VERY_DANGEROUS          = "veryDangerous"
 STATUSES_VERY_DANGEROUS       = STATUSES_DANGEROUS "," STATUS_DOES_NOT_EXIST
 LEVEL_PRE_RELEASE             = "preRelease"
 STATUSES_PRE_RELEASE          = STATUS_PRE_RELEASED
 LEVEL_CONSERVATIVE_TESTING    = "conservativeTesting"
 STATUSES_CONSERVATIVE_TESTING = STATUS_RELEASED "," STATUS_PRE_RELEASED "," STATUS_READY "," STATUS_TESTING "," STATUS_MANIFEST
 LEVEL_DEFAULT                 = LEVEL_PRODUCTION

 # Output styles
 STYLE_XML         = "xml"
 STYLE_XML_ENTRIES = "xmlEntries"
 STYLE_XML_LEGACY  = "xmlLegacy"
 STYLE_DEFAULT     = STYLE_XML_ENTRIES

 # Exit codes
 EXIT_NO_FORMAT_STRING = 11

 # Initial setup
 FS="\t"
 parseCommandLine()

 # Preamble
 if (outputStyle == STYLE_XML || outputStyle == STYLE_XML_LEGACY) { preambleXml() }
}

###########
# FUNCTIONS
###########

#
# parseCommandLine
#
function parseCommandLine() {
 # Defaults
 if (outputStyle == "") { outputStyle = STYLE_DEFAULT }
 if (outputLevel == "") { outputLevel = LEVEL_DEFAULT }

 # Parse
 parseOutputLevel()
}

#
# parseOutputLevel
#
function parseOutputLevel(        _splitOutputLevel, _i, _statuses, _splitStatuses, _j) {
 # Reset output level information
 delete outputLevels

 # For each output level...
 split(outputLevel, _splitOutputLevel, ",")
 for (_i in _splitOutputLevel) {
  # Synonyms
  if (_splitOutputLevel[_i] == LEVEL_PRODUCTION) { _statuses = STATUSES_PRODUCTION }
  else if (_splitOutputLevel[_i] == LEVEL_PRE_RELEASE) { _statuses = STATUSES_PRE_RELEASE }
  else if (_splitOutputLevel[_i] == LEVEL_CONTENT_TESTING) { _statuses = STATUSES_CONTENT_TESTING }
  else if (_splitOutputLevel[_i] == LEVEL_CONSERVATIVE_TESTING) { _statuses = STATUSES_CONSERVATIVE_TESTING }
  else if (_splitOutputLevel[_i] == LEVEL_DANGEROUS) { _statuses = STATUSES_DANGEROUS }
  else if (_splitOutputLevel[_i] == LEVEL_VERY_DANGEROUS) { _statuses = STATUSES_VERY_DANGEROUS }
  else { _statuses = _splitOutputLevel[_i] } # Not a synonym
  # For each status...
  split(_statuses, _splitStatuses, ",")
  for (_j in _splitStatuses) { outputLevels[_splitStatuses[_j]] = true }
 }
}

#
# parseFormatString
#
function parseFormatString(        _splitFormatString, _numFields, _field, _code) {
 # Reset format information
 delete indexOf
 numAdditionalParameters = 0
 delete indexAdditionalAttributes
 numParameters = 0
 delete indexParameters
 delete fixedValue

 # For each format code...
 _numFields = split(formatString, _splitFormatString, ",")
 for (_field = 1 ; _field <= _numFields ; ++_field) {
  _code = _splitFormatString[_field]
  if (_code == CODE_SKIP_1 || _code == CODE_SKIP_2) { continue }
  else if (_code == CODE_CODEN || _code == CODE_ESTIMATED_SIZE || _code == CODE_ISSN || _code == CODE_JOURNAL_TITLE || _code == CODE_PLUGIN || _code == CODE_PUBLISHER || _code == CODE_RIGHTS || _code == CODE_STATUS || _code == CODE_TITLE) { indexOf[_code] = _field }
  else if (_code == CODE_ATTRIBUTE) { indexAdditionalAttributes[++numAdditionalAttributes] = _field }
  else if (_code == CODE_PARAMETER) { indexParameters[++numParameters] = _field }
  # FIXME: unknown code
 }
}

#
# parseFixedValues
#
function parseFixedValues(        _i, _code) {
 for (_i = 3 ; _i <= NF ; ++_i) {
  _code = getKey($_i)
  if (_code == "") { continue }
  if (_code == CODE_CODEN || _code == CODE_ISSN || _code == CODE_JOURNAL_TITLE || _code == CODE_PLUGIN || _code == CODE_PUBLISHER || _code == CODE_RIGHTS) {
   indexOf[_code] = -1
   fixedValue[_code] = getValue($_i)
  }
  # FIXME: unknown or illegal code
 }
}

#
# preambleXml
#
function preambleXml() {
 xmlInOrgLockssTitle = false;

 printf   "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 printf   "<!DOCTYPE lockss-config [\n"
 printf   "<!ELEMENT lockss-config (if|property)+>\n"
 printf   "<!ELEMENT property (property|list|value|if)*>\n"
 printf   "<!ELEMENT list (value)+>\n"
 printf   "<!ELEMENT value (#PCDATA)>\n"
 printf   "<!ELEMENT test EMPTY>\n"
 printf   "<!ELEMENT and (and|or|not|test)*>\n"
 printf   "<!ELEMENT or (and|or|not|test)*>\n"
 printf   "<!ELEMENT not (and|or|not|test)*>\n"
 printf   "<!ELEMENT if (and|or|not|then|else|test|property)*>\n"
 printf   "<!ELEMENT then (if|property)*>\n"
 printf   "<!ELEMENT else (if|property)*>\n"
 printf   "<!ATTLIST property name CDATA #REQUIRED>\n"
 printf   "<!ATTLIST property value CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test hostname CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test group CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test daemonVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test daemonVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test daemonVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST test platformName CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if hostname CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if group CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if daemonVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if daemonVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if daemonVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformVersionMin CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformVersionMax CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformVersion CDATA #IMPLIED>\n"
 printf   "<!ATTLIST if platformName CDATA #IMPLIED>\n"
 printf   "<!ATTLIST list append CDATA #IMPLIED>\n"
 printf   "]>\n"
 printf   "\n"
 printf   "<lockss-config>\n"
 printf   "\n"
}

#
# postambleXml
#
function postambleXml() {
 if (xmlInOrgLockssTitle) {
  xmlInOrgLockssTitle = false;
  printf  " </property>\n"
  printf  "\n"
 }
 printf   "</lockss-config>\n"
}

#
# printOneXmlEntry
#
function printOneXmlEntry(        _status, _i) {
 # Skip based on output level
 _status = get(CODE_STATUS)
 if (!outputLevels[_status]) { return }

 # Start config subtree if needed
 if ((outputStyle == STYLE_XML || outputStyle == STYLE_XML_LEGACY) && !xmlInOrgLockssTitle) {
  xmlInOrgLockssTitle = true;
  printf  " <property name=\"org.lockss.title\">\n"
  printf  "\n"
 }

 # Begin
 printf   "  <property name=\"%s\">\n", getOpaqueName(get(CODE_TITLE))

 # First the publisher
 printOneXmlAttribute("publisher=" xml(get(CODE_PUBLISHER)))

 # Then the ISSN, CODEN, journal title, archival unit title and plugin
 printOneXmlProperty("issn=" get(CODE_ISSN))
 printOneXmlProperty("coden=" get(CODE_CODEN))
 printOneXmlProperty("journalTitle=" xml(get(CODE_JOURNAL_TITLE)))
 printOneXmlProperty("title=" xml(get(CODE_TITLE)) (_status == STATUS_PRE_RELEASED ? " (pre-release)" : "") (_status == STATUS_SUPERSEDED ? " (superseded)" : ""))
 printOneXmlProperty("plugin=" get(CODE_PLUGIN))

 # Then the parameters
 for (_i = 1 ; _i <= numParameters ; ++_i) { printOneXmlParameter(_i, $indexParameters[_i]) }
 # Down AUs are denoted by a parameter
 if (_status == STATUS_DOWN || _status == STATUS_SUPERSEDED) { printOneXmlParameter(99, "pub_down=true") }

 # Then any additional attributes
 for (_i = 1 ; _i <= numAdditionalAttributes ; ++_i) { printOneXmlAttribute($indexAdditionalAttributes[_i]) }

 # Then other tidbits
 if (get(CODE_RIGHTS) == "openaccess") { printOneXmlAttribute("rights=openaccess") }
 printOneXmlProperty("estSize=" get(CODE_ESTIMATED_SIZE))
 if (_status == STATUS_PRE_RELEASED) { printOneXmlAttribute("releaseStatus=pre-release") }

 # End
 printf   "  </property>\n"
 printf   "\n"
}

#
# printOneXmlPublisherTitleSet
#
function printOneXmlPublisherTitleSet(        _pub) {
 if (getKey($3) == "publisher") {
  if (xmlInOrgLockssTitle) {
   xmlInOrgLockssTitle = false;
   printf " </property>\n"
   printf "\n"
  }
  _pub = xml(getValue($3))
  printf  " <property name=\"org.lockss.titleSet\">\n"
  printf  "\n"
  printf  "  <property name=\"%s\">\n", _pub)
  printOneXmlProperty("name=All " _pub " Titles")
  printOneXmlProperty("class=xpath")
  printOneXmlProperty("xpath=[attributes/publisher='" _pub "']")
  printf  "  </property>\n"
  printf  "\n"
  printf  " </property>\n"
  printf  "\n"
 }
 # FIXME: invalid format
}

#
# printOneXmlParameter
#
function printOneXmlParameter(_num, _pair) {
 if (_pair == "") { return }
 printf   "   <property name=\"param.%d\">\n", _num
 printOneXmlProperty("key=" getKey(_pair), " ")
 printOneXmlProperty("value=" getValue(_pair), " ")
 printf   "   </property>\n"
}

#
# printOneXmlAttribute
#
function printOneXmlAttribute(_pair) {
 printOneXmlProperty("attributes." _pair)
}

#
# printOneXmlProperty
#
function printOneXmlProperty(_pair, _pad) {
 if (_pair == "" || getValue(_pair) == "") { return }
 if (_pad != "") { printf "%s", _pad }
 printf   "   <property name=\"%s\" value=\"%s\" />\n", getKey(_pair), getValue(_pair)
}

#
# get
#
function get(_code) {
 if (indexOf[_code] < 0) { return fixedValue[_code] }
 if (indexOf[_code] > 0) { return $indexOf[_code] }
 return "";
}

#
# getOpaqueName
#
function getOpaqueName(_str) {
 gsub(/ Volume /, "", _str); # FIXME
 gsub(/&/, "and", _str);
 gsub(/[^a-zA-Z0-9]/, "", _str);
 return namePrefix _str
}

#
# getKey
#
# Returns the key in a key=value pair.
#
function getKey(_str) {
 return match(_str, /^[^=]+=/) ? substr(_str, 1, RLENGTH - 1) : _str
}

#
# getValue
#
# Returns the value in a key=value pair.
#
function getValue(_str) {
 return match(_str, /^[^=]+=/) ? substr(_str, RLENGTH + 1, length(_str) - RLENGTH) : _str
}

#
# xml
#
# Encodes a string's ampersands and angle brackets for XML output.
# Assumes UTF-8 input; produces UTF-8 output.
#
function xml(_str) {
 gsub(/&/, "\&", _str);
 gsub(/</, "\<", _str);
 gsub(/>/, "\>", _str);
 return _str
}

##########
# PATTERNS
##########

#
# Beginning of file
# unconditional
#
FNR == 1 {
 formatString = ""
 namePrefix = ""
}

#
# Blank line
#
/^\t*$/ {
 next
}

#
# Fixed values line
#
$1 == "#" && $2 == DIRECTIVE_FIXED_VALUES {
 parseFixedValues()
 next
}

#
# Format line
#
$1 == "#" && $2 == DIRECTIVE_FORMAT_STRING {
 formatString = $3
 parseFormatString()
 next
}

#
# Name prefix line
#
$1 == "#" && $2 == DIRECTIVE_NAME_PREFIX {
 namePrefix = $3
 next
}

#
# Publisher title set line
#
$1 == "#" && $2 == DIRECTIVE_PUBLISHER_TITLE_SET {
 if (outputStyle == STYLE_XML) { printOneXmlPublisherTitleSet() }
 next
}

#
# Comment line
#
/^#/ {
 next
}

# Normal line
# unconditional
{
 if (formatString == "") { exit EXIT_NO_FORMAT_STRING }
 if (outputStyle == STYLE_XML || outputStyle == STYLE_XML_ENTRIES || outputStyle == STYLE_XML_LEGACY) { printOneXmlEntry() }
 next
}

#####
# END
#####

END {
 # Postamble
 if (outputStyle == STYLE_XML || outputStyle == STYLE_XML_LEGACY) { postambleXml() }
}