Define KEY_MAX for Python. (dm)

Define KEY_MAX for Python. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@2776 91a5dbb7-01b9-0310-9b5f-b28072856b6e

###############################################################################
# BRLTTY - A background process providing access to the console screen (when in
#          text mode) for a blind person using a refreshable braille display.
#
# Copyright (C) 1995-2006 by The BRLTTY Developers.
#
# BRLTTY comes with ABSOLUTELY NO WARRANTY.
#
# This is free software, placed under the terms of the
# GNU General Public License, as published by the Free Software
# Foundation.  Please see the file COPYING for details.
#
# Web Page: http://mielke.cc/brltty/
#
# This software is maintained by Dave Mielke <dave@mielke.cc>.
###############################################################################

/#define[ \t]*BRLAPI_KEY_MAX/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_MASK/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_SHIFT/ {
  apiShift(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_TYPE_/ {
  apiType(substr($2, 17), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_SYM_/ {
  apiKey(substr($2, 16), $2, getDefineValue(), "")
  next
}

###############################################################################
# BRLTTY - A background process providing access to the console screen (when in
#          text mode) for a blind person using a refreshable braille display.
#
# Copyright (C) 1995-2006 by The BRLTTY Developers.
#
# BRLTTY comes with ABSOLUTELY NO WARRANTY.
#
# This is free software, placed under the terms of the
# GNU General Public License, as published by the Free Software
# Foundation.  Please see the file COPYING for details.
#
# Web Page: http://mielke.cc/brltty/
#
# This software is maintained by Dave Mielke <dave@mielke.cc>.
###############################################################################

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_MASK/ {
  apiMask(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_[A-Z_]+_SHIFT/ {
  apiShift(substr($2, 12), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_TYPE_/ {
  apiType(substr($2, 17), $2, getDefineValue(), "")
  next
}

/#define[ \t]*BRLAPI_KEY_SYM_/ {
  apiKey(substr($2, 16), $2, getDefineValue(), "")
  next
}


Use NF.

Use NF.

function printUrlQueryString(queryString)
{
    split(queryString, pairs, "&", seps)
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=", seps2)
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    print "{";
}

END {
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ", seps)
    numFields = length(fields)
}

/^#/ {
    # ignore anything else starting with "#"
}

! /^#/ {
    split($0, fieldsThisLine, " ", sepsThrowAway)
    if (NF == numFields) {
        print "\t{"
        for(i = 1; i < NF; i++)
        {
            printf "\t\t\"%s\": \"%s\",\n", fields[i], fieldsThisLine[i]
        }
        print "\t},"
    }
}

function printUrlQueryString(queryString)
{
    split(queryString, pairs, "&", seps)
    len = length(pairs)
    for (j = 1; j < len; ++j) {
        split(pairs[j], keyValue, "=", seps2)
        k = keyValue[1]
        v = keyValue[2]
        printf "\t\t\"%s\": \"%s\",\n", k, urlDecode(urlDecode(v))
    }
}

function base64Decode(thing)
{
  "echo "$thing" | base64 --decode" | getline x
  return x
}

function urlDecode(url) {
    for (i = 0x20; i < 0x40; ++i) {
        repl = sprintf("%c", i);
        if ((repl == "&") || (repl == "\\")) {
            repl = "\\" repl;
        }
        url = gensub(sprintf("%%%02X", i), repl, "g", url);
        url = gensub(sprintf("%%%02x", i), repl, "g", url);
    }
    return url;
}

BEGIN {
    print "{";
}

END {
    print "}"
}

/^#Version: / {
    printf "\t\"version\": \"%s\",\n", substr($0, length("#Version: ") + 1)
}

/^#Fields: / {
    rest = substr($0, length("#Fields: ") + 1)
    split(rest, fields, " ", seps)
    numFields = length(fields)
}

/^#/ {
    # ignore anything else starting with "#"
}

! /^#/ {
    split($0, fieldsThisLine, " ", sepsThrowAway)
    print "\t{"
    for(i = 1; i < numFields; i++)
    {
        printf "\t\t\"%s\": \"%s\",\n", fields[i], fieldsThisLine[i]
    }
    print "\t},"
}


Fix comments.

Fix comments.

#!/usr/bin/awk -f

{
	if (/^$/ || /^#/ || /^\!/ || /^\[/ || /^@@/) {
		# Empty line, or line starts with #, !, [ or @@
		# https://adblockplus.org/en/filters#comments
		# https://adblockplus.org/en/filters#whitelist
	} else if (/(^\|\|)|(^\|)/) {
		# Line starts with || or |
		# https://adblockplus.org/en/filters#anchors
		sub(/(^\|\|)|(^\|)/, "") # remove leading || or |
		extract($0)
	} else if (/^\/.*\/$/) {
		# Line in two slashs, like /.../, is a regular expression.
		# https://adblockplus.org/en/filters#regexps
		gsub(/(^\/)|(\/$)/, "") # remove leading and tailing /
		extract_regex($0)
	} else {
		extract($0)
	}
}

END {
	for (i = 1; i <= domain_count; i++) {
		print domains[i] | "sort | uniq"
	}
	close("sort | uniq")
}

# Extract domain name from non-regex line
function extract(line) {

	# Remove everything till ://, such as http:// https://
	sub(/.*:\/\//, "", line)

	# Remove everything from /
	sub(/\/.*/, "", line)

	# Remove everything from :
	sub(/:.*/, "", line)

	# Remove segments(dot separated) contain *
	sub(/[^\.]*\*[^\.]*/, "", line)

	# Remove leading dot
	sub(/^\./, "", line)

	if (line ~ /^((([0-9]{1,2})|(1[0-9]{2})|(2[0-4][0-9])|(25[0-5]))\.){3}(([0-9]{1,2})|(1[0-9]{2})|(2[0-4][0-9])|(25[0-5]))$/) {
		# IPv4 string
		print "Skipping line " NR ". " line | "cat >&2"
		close("cat >&2")
	} else if (line ~ /^([A-Za-z0-9\-]+\.)+[A-Za-z0-9\-]+$/) {
		domains[++domain_count] = line
	} else {
		print "Skipping line " NR ". " line | "cat >&2"
		close("cat >&2")
	}
}

# Extract domain name from regex line
function extract_regex(line) {
	# Expand the lines like (aa|bb|cc) into multiple records
	pos = match(line, /\([A-Za-z0-9\.\|]+\)/)
	if (pos != 0) {
		in_bracket = substr(line, RSTART + 1, RLENGTH - 2)
		n = split(in_bracket, arr, "|")
		for (i = 1; i <= n; i++) {
			expanded_line = line
			sub(/\([A-Za-z0-9\.\|]+\)/, arr[i], expanded_line)
			extract_regex(expanded_line)
		}
	} else {

		# Remove everything till :\/\/ in regex(:// in plain text, such as http:// https://)
		sub(/.*:\\\/\\\//, "", line)

		# Remove (...)*? such as ([^\/]+\.)
		gsub(/\([^\)]+\)\*?/, "", line)

		# Remove [...]
		gsub(/\[.*\]\+/, "", line)

		# Remove everything from \/ in regex(/ in plaint text)
		sub(/\\\/.*/, "", line)

		# Replace \. to .
		gsub(/\\\./, ".", line)

		extract(line)
	}
}

#!/usr/bin/awk -f

{
	if (/^$/ || /^#/ || /^\!/ || /^\[/ || /^@@/) {
		# Empty line, or line starts with #, !, [ or @@
		# https://adblockplus.org/en/filters#comments
		# https://adblockplus.org/en/filters#whitelist
	} else if (/(^\|\|)|(^\|)/) {
		# Line starts with || or |
		# https://adblockplus.org/en/filters#anchors
		sub(/(^\|\|)|(^\|)/, "") # remove leading || or |
		extract($0)
	} else if (/^\/.*\/$/) {
		# Line in two slashs, like /.../, is a regular expression.
		# https://adblockplus.org/en/filters#regexps
		gsub(/(^\/)|(\/$)/, "") # remove leading and tailing /
		extract_regex($0)
	} else {
		extract($0)
	}
}

END {
	for (i = 1; i <= domain_count; i++) {
		print domains[i] | "sort | uniq"
	}
	close("sort | uniq")
}

# Extract domain name from non-regex line
function extract(line) {
	sub(/.*:\/\//, "", line) # remove everything till ://, such as http:// https://
	sub(/\/.*/, "", line)    # remove everything from /
	sub(/:.*/, "", line)     # remove everything from :
	sub(/[^\.]*\*[^\.]*/, "", line)      # remove segments(dot separated) contain *
	sub(/^\./, "", line)     # remove leading dot

	if (line ~ /^((([0-9]{1,2})|(1[0-9]{2})|(2[0-4][0-9])|(25[0-5]))\.){3}(([0-9]{1,2})|(1[0-9]{2})|(2[0-4][0-9])|(25[0-5]))$/) {
		# IPv4 string
		print "Skipping line " NR ". " line | "cat >&2"
		close("cat >&2")
	} else if (line ~ /^([A-Za-z0-9\-]+\.)+[A-Za-z0-9\-]+$/) {
		domains[++domain_count] = line
	} else {
		print "Skipping line " NR ". " line | "cat >&2"
		close("cat >&2")
	}
}

# Extract domain name from regex line
function extract_regex(line) {
	# Expand the lines like (aa|bb|cc) into multiple records
	pos = match(line, /\([A-Za-z0-9\.\|]+\)/)
	if (pos != 0) {
		in_bracket = substr(line, RSTART + 1, RLENGTH - 2)
		n = split(in_bracket, arr, "|")
		for (i = 1; i <= n; i++) {
			expanded_line = line
			sub(/\([A-Za-z0-9\.\|]+\)/, arr[i], expanded_line)
			extract_regex(expanded_line)
		}
	} else {
		sub(/.*:\\\/\\\//, "", line)    # remove everything till :\/\/ in regex(:// in plain text, such as http:// https://)
		gsub(/\([^\)]+\)\*?/, "", line) # remove (...)*? such as ([^\/]+\.)
		gsub(/\[.*\]\+/, "", line)      # remove [...]
		sub(/\\\/.*/, "", line)         # remove everything from \/ in regex(/ in plaint text)
		gsub(/\\\./, ".", line)         # replace \. to .
		extract(line)
	}
}


Commons: add escape(), enable double quotes for parameterize()

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used functions for string and array operations, logging.#
####################################################################

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line) {
    # Remove preceding spaces
    gsub(/^[[:space:]]+/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        if (preserveNull || array[i] != "")
            temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if(ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[21m"
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Print warning message.
function w(text) {
    print AnsiCode["yellow"] text AnsiCode[0] > "/dev/stderr"
}

# Print error message.
function e(text) {
    print AnsiCode["red"] AnsiCode["bold"] text AnsiCode[0] > "/dev/stderr"
}

# Print debugging message.
function d(text) {
    print AnsiCode["gray"] text AnsiCode[0] > "/dev/stderr"
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

BEGIN {
    initUrlEncoding()
    initAnsiCode()
}

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used functions for string and array operations, logging.#
####################################################################

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the single-quoted string.
function parameterize(string) {
    gsub(/'/, "'\\''", string)
    return "'" string "'"
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line) {
    # Remove preceding spaces
    gsub(/^[[:space:]]+/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        if (preserveNull || array[i] != "")
            temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Initialize ANSI escape codes (ANSI X3.64 Standard Control Sequences).
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if(ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[21m"
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Print warning message.
function w(text) {
    print AnsiCode["yellow"] text AnsiCode[0] > "/dev/stderr"
}

# Print error message.
function e(text) {
    print AnsiCode["red"] AnsiCode["bold"] text AnsiCode[0] > "/dev/stderr"
}

# Print debugging message.
function d(text) {
    print AnsiCode["gray"] text AnsiCode[0] > "/dev/stderr"
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

BEGIN {
    initUrlEncoding()
    initAnsiCode()
}


Commons: add assert()

Commons: add assert()

#!/usr/bin/gawk -f

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        if (preserveNull || array[i] != "")
            temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Debug an array.
function da(array, formatString, sortedIn,
                ####
                i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

#!/usr/bin/gawk -f

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array)
        if (preserveNull || array[i] != "")
            temp = j++ ? temp separator array[i] : array[i]
    PROCINFO["sorted_in"] = saveSortedIn

    return temp
}

# Debug an array.
function da(array, formatString, sortedIn,
                ####
                i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}


add another line

add another line

BEGIN {
	IN_BLOCK = 0;

	PRE_LINE = "^[ \t]*";
	POST_LINE = "[ \t]*$";

	# tmp variables
	X = "((\\*|#( *\\*)?) *)?";
	PRE_BLOCK = "(/\\* *|# *(/\\* *)?)?";
	POST_BLOCK = " *(\\*/)?";

	BLOCK_END = 0;

	BLOCK[BLOCK_END++] = PRE_BLOCK "\\*\\*\\*\\*\\* BEGIN LICENSE BLOCK \\*\\*\\*\\*\\*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "BBN Address and AS Number PKI Database/repository software";
	BLOCK[BLOCK_END++] = X "Version [^ ]*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "COMMERCIAL COMPUTER SOFTWARE RESTRICTED RIGHTS.*";
	BLOCK[BLOCK_END++] = X "US government users are permitted unrestricted rights as";
	BLOCK[BLOCK_END++] = X "defined in the FAR.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "This software is distributed on an \"AS IS\" basis, WITHOUT";
	BLOCK[BLOCK_END++] = X "WARRANTY OF ANY KIND, either express or implied.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "Copyright .*";
	BLOCK[BLOCK_END++] = X "All Rights Reserved.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "Contributor(\\(s\\)|s)?: .*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "\\*\\*\\*\\*\\* END LICENSE BLOCK \\*\\*\\*\\*\\*" POST_BLOCK;

	BLOCK_END--;
}

{
	if (!IN_BLOCK) {
		if ($0 ~ PRE_LINE BLOCK[0] POST_LINE) {
			IN_BLOCK = 1;
			BLOCK_LINENO = 0;
			SAVED_BLOCK = $0;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
		} else {
			print;
		}
	} else {
		if ($0 ~ PRE_LINE BLOCK[BLOCK_LINENO + 1] POST_LINE) {
			SAVED_BLOCK = SAVED_BLOCK RS $0;
			BLOCK_LINENO += 1;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
			if (BLOCK_LINENO == BLOCK_END) {
				IN_BLOCK = 0;
			}
		} else {
			print SAVED_BLOCK;
			print;
			IN_BLOCK = 0;
		}
	}
}

END {
	if (IN_BLOCK) {
		print SAVED_BLOCK;
	}
}

BEGIN {
	IN_BLOCK = 0;

	PRE_LINE = "^[ \t]*";
	POST_LINE = "[ \t]*$";

	# tmp variables
	X = "((\\*|#( *\\*)?) *)?";
	PRE_BLOCK = "(/\\* *|# *(/\\* *)?)?";
	POST_BLOCK = " *(\\*/)?";

	BLOCK_END = 0;

	BLOCK[BLOCK_END++] = PRE_BLOCK "\\*\\*\\*\\*\\* BEGIN LICENSE BLOCK \\*\\*\\*\\*\\*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "BBN Address and AS Number PKI Database/repository software";
	BLOCK[BLOCK_END++] = X "Version [^ ]*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "US government users are permitted unrestricted rights as";
	BLOCK[BLOCK_END++] = X "defined in the FAR.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "This software is distributed on an \"AS IS\" basis, WITHOUT";
	BLOCK[BLOCK_END++] = X "WARRANTY OF ANY KIND, either express or implied.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "Copyright .*";
	BLOCK[BLOCK_END++] = X "All Rights Reserved.";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "Contributor(\\(s\\)|s)?: .*";
	BLOCK[BLOCK_END++] = X;
	BLOCK[BLOCK_END++] = X "\\*\\*\\*\\*\\* END LICENSE BLOCK \\*\\*\\*\\*\\*" POST_BLOCK;

	BLOCK_END--;
}

{
	if (!IN_BLOCK) {
		if ($0 ~ PRE_LINE BLOCK[0] POST_LINE) {
			IN_BLOCK = 1;
			BLOCK_LINENO = 0;
			SAVED_BLOCK = $0;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
		} else {
			print;
		}
	} else {
		if ($0 ~ PRE_LINE BLOCK[BLOCK_LINENO + 1] POST_LINE) {
			SAVED_BLOCK = SAVED_BLOCK RS $0;
			BLOCK_LINENO += 1;
			print "Matched line " BLOCK_LINENO >> "/dev/stderr";
			if (BLOCK_LINENO == BLOCK_END) {
				IN_BLOCK = 0;
			}
		} else {
			print SAVED_BLOCK;
			print;
			IN_BLOCK = 0;
		}
	}
}

END {
	if (IN_BLOCK) {
		print SAVED_BLOCK;
	}
}


handling #conversion:label

handling #conversion:label

# cell-ify-eparams.awk
#
# columns 12 to 72 should be cellified: jot -s "," `expr 71 - 12 + 1` 12
#
#
# Parameters:
#    cellify:     What columns?     comma-separated list of column indexes to cellify. e.g. 65,66,67,68,69
#                                     hint: use `jot -s "," 5 65` or seq
#
#    label_up:    Primary label?
#
#    up_object:   Primary object?   Object of up value. If '.', use ov:csvHeader as a literal. If ':', use ov:csvHeader as a resource.
#
#    up_range:    Primary object?   Object of up value
#
#    out_range:   ? e.g. Type of out value.                                        e.g. xsd:gYear
#
#    labels_up:   What range?       comma-separated list of secondary predicate labels.
#
# Example:
#      conversion:enhance [
#         ov:csvCol         65;
#         ov:csvHeader     "BAR00";
#         conversion:label "BAR00";
#         conversion:range  todo:Literal;
#      ];
#
# (params: )
#
#      conversion:enhance [
#          ov:csvCol         65;
#          ov:csvHeader     "BAR00";
#          conversion:label "Year";                      # WAS: BAR00
#          conversion:object "2000"^^xsd:gYear;          # new: value represented by column header
#          conversion:range  rdfs:decimal;               # WAS: todo:Literal
#       ];
#       conversion:enhance [                             # new:
#          ov:csvCol         65;                         # new:
#          conversion:predicate "Place";                 # new:
#          conversion:object    "Bar";                   # new:
#       ];                                               # new:
#
# Example usage:
#
#    (input)
#
#    conversion:enhance [
#       ov:csvCol         12;
#       ov:csvHeader     "Sum_WORK90";
#       conversion:label "Sum_WORK90";
#       conversion:range  todo:Literal;
#    ];
#
#    cat cities-pc-cnty.csv.e1.params.ttl | awk -f $CSV2RDF4LOD_HOME/bin/util/cell-ify-eparams.awk -v cellify=`jot -s "," 60 12` -v label_up="Year" -v up_range=xsd:gYear -v out_range=xsd:decimal -v labels_up=Venue
#
#    (output)
#
#    conversion:enhance [
#       ov:csvCol         12;
#       ov:csvHeader     "Sum_WORK90";
#       a scovo:Item;                                               # :
#       conversion:label  "Year";                                   # : was "Sum_WORK90";
#       conversion:object ""^^xsd:gYear;                            # :
#       conversion:range  xsd:decimal;                              # : was todo:Literal;
#    ];
#    conversion:enhance [                                           # :
#       ov:csvCol            12;                                    # :
#       conversion:predicate "Venue";                               # :
#       conversion:object    "";                                    # :
#    ];                                                             # :
#
# NOTE: The conversion:objects must still be populated.

BEGIN {
   PADDING = 70;

   num_cells = split(cellify,cell_column,",");
   for(i=1; i<=num_cells; i++) {
      c = cell_column[i];
      a_cell[c] = "true";
   }
   num_labels_up = split(labels_up,secondaryPs,",");

   in_cell_enhancement = "false";
}

$1 == "ov:csvCol" && in_cell_enhancement ==  "false" {
   column=$2;
   gsub(/;/,"",column);
   gsub(/\"/,"",column);
   if( a_cell[column] == "true" ) {
      in_cell_enhancement = "true";
   }
}

function annotate(pre,pad,annotation)
{
   line=pre;
   spaces="";
   for(i=1; i<(pad-length(line)); i++) {
      spaces=spaces" ";
   }
   return sprintf("%s%s# : %s",line,spaces,annotation);
}

in_cell_enhancement == "true" {
   if( $1 == "ov:csvCol" ) {
      lastCol=$2;
      gsub(/;/,"",lastCol);
      gsub(/\"/,"",lastCol);
   }
   line=$0;

   if( length(label_up) > 0 && $1 == "conversion:label" || $1 == "#conversion:label" ) {
      print annotate("         %s  \""label_up"\";",$1,PADDING,"was "$2);
   }else if( length(out_range) > 0 && $1 == "conversion:range" ) {
      print annotate("         conversion:range  "out_range";",PADDING,"was "$2);
   }else if( $1 == "];" ) {
      print $0
      for(i=1; i<=num_labels_up; i++) {
          what=sprintf("         conversion:predicate \"%s\";",secondaryPs[i]);
         print annotate(        "      conversion:enhance [",PADDING,"");
         print annotate(        "         ov:csvCol            "lastCol";",PADDING,"");
         print annotate(what ,PADDING,"");
         print annotate(        "         conversion:object    \"\";",PADDING,"");
         print annotate(        "      ];",PADDING,"");
      }
   }else {
      print
   }

   if( $1 == "ov:csvHeader" ) {
      current_header=$2;
      gsub(/"/,"",current_header);
      gsub(/;/,"",current_header);
      print annotate("         a scovo:Item;",PADDING,"");
   }
   if( length(label_up) > 0 && $1 == "conversion:label" ) {
      datatype = length(up_range) > 0 ? "^^"up_range : "";
      if( up_object == ":" ) {
         print annotate("         conversion:object :"current_header";",PADDING,"");
      }else if( up_object == "." ) {
         print annotate("         conversion:object \""current_header"\""datatype";",PADDING,"");
      }else {
         print annotate("         conversion:object \""up_object"\""datatype";",PADDING,"");
      }
   }
}

{
   if( in_cell_enhancement == "true" && $1 == "];" ) {
      in_cell_enhancement = "false";
   }else if( in_cell_enhancement == "false" ) {
      print $0
   }
}

END {
   #if( num_cells == 0 ) {
      print "# params:"
      print "#   -v cellify=`jot -s \",\" 5 10` -v label_up=Year -v up_object={. : Blah} -v up_range=xsd:gYear -v out_range=xsd:decimal -v labels_up=Venue,Another,YAnother"
      print "#   -v cellify=`echo \"\" | awk '{for(i=5;i<=10;i++){printf(\",%s\",i)}}'` -v label_up=Year -v up_object=Blah -v up_range=xsd:gYear -v out_range=xsd:decimal -v labels_up=Venue,Another,YAnother"
   #}
}

# cell-ify-eparams.awk
#
# columns 12 to 72 should be cellified: jot -s "," `expr 71 - 12 + 1` 12
#
#
# Parameters:
#    cellify:     What columns?     comma-separated list of column indexes to cellify. e.g. 65,66,67,68,69
#                                     hint: use `jot -s "," 5 65` or seq
#
#    label_up:    Primary label?
#
#    up_object:   Primary object?   Object of up value. If '.', use ov:csvHeader as a literal. If ':', use ov:csvHeader as a resource.
#
#    up_range:    Primary object?   Object of up value
#
#    out_range:   ? e.g. Type of out value.                                        e.g. xsd:gYear
#
#    labels_up:   What range?       comma-separated list of secondary predicate labels.
#
# Example:
#      conversion:enhance [
#         ov:csvCol         65;
#         ov:csvHeader     "BAR00";
#         conversion:label "BAR00";
#         conversion:range  todo:Literal;
#      ];
#
# (params: )
#
#      conversion:enhance [
#          ov:csvCol         65;
#          ov:csvHeader     "BAR00";
#          conversion:label "Year";                      # WAS: BAR00
#          conversion:object "2000"^^xsd:gYear;          # new: value represented by column header
#          conversion:range  rdfs:decimal;               # WAS: todo:Literal
#       ];
#       conversion:enhance [                             # new:
#          ov:csvCol         65;                         # new:
#          conversion:predicate "Place";                 # new:
#          conversion:object    "Bar";                   # new:
#       ];                                               # new:
#
# Example usage:
#
#    (input)
#
#    conversion:enhance [
#       ov:csvCol         12;
#       ov:csvHeader     "Sum_WORK90";
#       conversion:label "Sum_WORK90";
#       conversion:range  todo:Literal;
#    ];
#
#    cat cities-pc-cnty.csv.e1.params.ttl | awk -f $CSV2RDF4LOD_HOME/bin/util/cell-ify-eparams.awk -v cellify=`jot -s "," 60 12` -v label_up="Year" -v up_range=xsd:gYear -v out_range=xsd:decimal -v labels_up=Venue
#
#    (output)
#
#    conversion:enhance [
#       ov:csvCol         12;
#       ov:csvHeader     "Sum_WORK90";
#       a scovo:Item;                                               # :
#       conversion:label  "Year";                                   # : was "Sum_WORK90";
#       conversion:object ""^^xsd:gYear;                            # :
#       conversion:range  xsd:decimal;                              # : was todo:Literal;
#    ];
#    conversion:enhance [                                           # :
#       ov:csvCol            12;                                    # :
#       conversion:predicate "Venue";                               # :
#       conversion:object    "";                                    # :
#    ];                                                             # :
#
# NOTE: The conversion:objects must still be populated.

BEGIN {
   PADDING = 70;

   num_cells = split(cellify,cell_column,",");
   for(i=1; i<=num_cells; i++) {
      c = cell_column[i];
      a_cell[c] = "true";
   }
   num_labels_up = split(labels_up,secondaryPs,",");

   in_cell_enhancement = "false";
}

$1 == "ov:csvCol" && in_cell_enhancement ==  "false" {
   column=$2;
   gsub(/;/,"",column);
   gsub(/\"/,"",column);
   if( a_cell[column] == "true" ) {
      in_cell_enhancement = "true";
   }
}

function annotate(pre,pad,annotation)
{
   line=pre;
   spaces="";
   for(i=1; i<(pad-length(line)); i++) {
      spaces=spaces" ";
   }
   return sprintf("%s%s# : %s",line,spaces,annotation);
}

in_cell_enhancement == "true" {
   if( $1 == "ov:csvCol" ) {
      lastCol=$2;
      gsub(/;/,"",lastCol);
      gsub(/\"/,"",lastCol);
   }
   line=$0;

   if( length(label_up) > 0 && $1 == "conversion:label" ) {
      print annotate("         conversion:label  \""label_up"\";",PADDING,"was "$2);
   }else if( length(out_range) > 0 && $1 == "conversion:range" ) {
      print annotate("         conversion:range  "out_range";",PADDING,"was "$2);
   }else if( $1 == "];" ) {
      print $0
      for(i=1; i<=num_labels_up; i++) {
          what=sprintf("         conversion:predicate \"%s\";",secondaryPs[i]);
         print annotate(        "      conversion:enhance [",PADDING,"");
         print annotate(        "         ov:csvCol            "lastCol";",PADDING,"");
         print annotate(what ,PADDING,"");
         print annotate(        "         conversion:object    \"\";",PADDING,"");
         print annotate(        "      ];",PADDING,"");
      }
   }else {
      print
   }

   if( $1 == "ov:csvHeader" ) {
      current_header=$2;
      gsub(/"/,"",current_header);
      gsub(/;/,"",current_header);
      print annotate("         a scovo:Item;",PADDING,"");
   }
   if( length(label_up) > 0 && $1 == "conversion:label" ) {
      datatype = length(up_range) > 0 ? "^^"up_range : "";
      if( up_object == ":" ) {
         print annotate("         conversion:object :"current_header";",PADDING,"");
      }else if( up_object == "." ) {
         print annotate("         conversion:object \""current_header"\""datatype";",PADDING,"");
      }else {
         print annotate("         conversion:object \""up_object"\""datatype";",PADDING,"");
      }
   }
}

{
   if( in_cell_enhancement == "true" && $1 == "];" ) {
      in_cell_enhancement = "false";
   }else if( in_cell_enhancement == "false" ) {
      print $0
   }
}

END {
   #if( num_cells == 0 ) {
      print "# params:"
      print "#   cellify=`jot -s \",\" 5 10` -v label_up=Year -v up_object={. : Blah} -v up_range=xsd:gYear -v out_range=xsd:decimal -v labels_up=Venue,Another,YAnother"
      print "#   cellify=`echo \"\" | awk '{for(i=5;i<=10;i++){printf(\",%s\",i)}}'` -v label_up=Year -v up_object=Blah -v up_range=xsd:gYear -v out_range=xsd:decimal -v labels_up=Venue,Another,YAnother"
   #}
}


Handle embedded tab in BBox description

Handle embedded tab in BBox description

# Produce a raw data spreadsheet from fields in BritBox Catalog --  apple_catalogue_feed.xml
# The "showContentId" from "tv_show" items are needed to process "tv_season" and "tv_episode" items,
# so you should sort the catalog to ensure items are in the proper order.

# INVOCATION
#   awk -v ERRORS=$ERRORS -v IDS_SEASONS=$IDS_SEASONS -v IDS_EPISODES=$IDS_EPISODES \
#       -v RAW_TITLES=$RAW_TITLES -f getBBoxCatalogFromSitemap.awk $SORTED_SITEMAPS >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title         3 Seasons          4 Episodes         5 Duration      6 Genre
#     7 Year          8 Rating        9 Description     10 Content_Type    11 Content_ID   12 Entity_ID
#    13 Show_Type    14 Date_Type    15 Original_Date   16 Show_ID         17 Season_ID    18 Sn_#
#    19 Ep_#         20 1st_#        21 Last_#

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tGenre\tYear\tRating\tDescription\t")
    printf ("Content_Type\tContent_ID\tEntity_ID\tShow_Type\tDate_Type\tOriginal_Date\t")
    printf ("Show_ID\tSeason_ID\tSn_#\tEp_#\t1st_#\tLast_#\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    firstLineNum = NR
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType > "/dev/stderr"
    # print "contentId = " contentId > "/dev/stderr"
    #
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = ""
    numEpisodes = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title > "/dev/stderr"
    gsub (/&/,"\\&",title)
    sub (/[[:space:]]+$/,"",title)
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    genre = toupper(substr(genre,1,1)) substr(genre,2)
    sub ("Sci_fi","Sci-Fi",genre)
    sub ("Special_interest","Special interest",genre)
    # print "genre = " genre > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description > "/dev/stderr"
    gsub (/&/,"\\&",description)
    gsub (/\t/," – ",description)
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType > "/dev/stderr"
    # print "originalDate = " originalDate > "/dev/stderr"
    # print "year = " year > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId > "/dev/stderr"
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    lastLineNum = NR

    # "The Moonstone" needs to be revised to avoid duplicate names
    if (title == "The Moonstone") {
        if (EntityId == "_9283") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'The Moonstone (2016)'\n", title) >> ERRORS
            title = "The Moonstone (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERRORS
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERRORS
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERRORS
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERRORS
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    if (EntityId == "") {
        missingEntityIds += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s at line %d\n", contentType, contentId,
                title, showContentId, firstLineNum) >> ERRORS
    }

    if (contentType == "movie") {
        totalMovies += 1
        # print "\ntv_movie" > "/dev/stderr"
        # Wish I didn't have to do this, but "movie" is too common to be in a key field
        contentType = "tv_movie"
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
    }

    if (contentType == "tv_show") {
        totalShows += 1
        # print "\ntv_show" > "/dev/stderr"
        showArray[totalShows] = contentId
        titleArray[totalShows] = title
        sortkey = sprintf ("%s (1) S%s %s", title, EntityId, contentId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
        #
        str = "grep -c " contentId " " IDS_SEASONS
        str | getline numSeasons
        close str
        #
        str = "grep -c " contentId " " IDS_EPISODES
        str | getline numEpisodes
        close str
    }

    if (contentType == "tv_season") {
        totalSeasons += 1
        # print "\ntv_season" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02d %s", showTitle, seasonNumber, showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02d %s", showTitle, seasonNumber, showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02d, %s", showTitle, seasonNumber, title)
    }

    if (contentType == "tv_episode") {
        totalEpisodes += 1
        # print "\ntv_episode" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                    showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02dE%03d, %s", showTitle, seasonNumber, episodeNumber, title)
    }

    # Generate a link that will lead to the show on Britbox
    # https://www.britbox.com/us/movie/A_Queen_Is_Crowned_13551
    # https://www.britbox.com/us/movie/_13551
    #
    # https://www.britbox.com/us/show/All_Creatures_Great_and_Small_23737
    # https://www.britbox.com/us/show/_23737
    #
    # https://www.britbox.com/us/season/All_Creatures_Great_and_Small_S2_23752
    # https://www.britbox.com/us/season/_23752
    #
    # https://www.britbox.com/us/episode/All_Creatures_Great_and_Small_S2_E2_23764
    # https://www.britbox.com/us/episode/_23764
    showType_URL = contentType "/"
    sub ("tv_","",showType_URL)
    full_URL = "https://www.britbox.com/us/" showType_URL EntityId
    fullTitle = "=HYPERLINK(\"" full_URL "\";\"" title "\")"
    # print "fullTitle = " fullTitle > "/dev/stderr"

    # A seasonless "show" should have blank rather than 0 seasons
    if (showType == "seasonless")
        numSeasons = ""
    # If an "episode" sortkey or fullTitle contains "S00E" delete the "S00" part
    sub ("S00E","E",sortkey)
    sub ("S00E","E",fullTitle)
    # Print everything except tv_seasons
    if (contentType != "tv_season")
        printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%06d\t%06d\n",
                sortkey, fullTitle, numSeasons, numEpisodes, duration, genre, year, rating, description,
                contentType, contentId, EntityId, showType, dateType, originalDate, showContentId,
                seasonContentId, seasonNumber, episodeNumber, firstLineNum, lastLineNum)
}

END {
    printf ("In getBBoxCatalogFromSitemap.awk\n") > "/dev/stderr"

    totalMovies == 1 ? pluralMovies = "movie" : pluralMovies = "movies"
    totalShows == 1 ? pluralShows = "show" : pluralShows = "shows"
    totalSeasons == 1 ? pluralSeasons = "season" : pluralSeasons = "seasons"
    totalEpisodes == 1 ? pluralEpisodes = "episode" : pluralEpisodes = "episodes"
    #
    printf ("    Processed %d %s, %d %s, %d %s, %d %s\n", totalMovies, pluralMovies, totalShows,
            pluralShows, totalSeasons, pluralSeasons, totalEpisodes, pluralEpisodes) > "/dev/stderr"

    if (missingEntityIds > 0 ) {
        missingEntityIds == 1 ? plural = "EntityId" : plural = "EntityIds"
        printf ("%8d missing %s in %s\n", missingEntityIds, plural, FILENAME) > "/dev/stderr"
    }

    if (missingShowContentIds > 0 ) {
        missingShowContentIds == 1 ? plural = "showContentId" : plural = "showContentIds"
        printf ("%8d missing %s in %s\n", missingShowContentIds, plural, FILENAME) > "/dev/stderr"
    }

    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? plural = "title" : plural = "titles"
        printf ("%8d %s revised in %s\n", revisedTitles, plural, FILENAME) > "/dev/stderr"
    }
}

# Produce a raw data spreadsheet from fields in BritBox Catalog --  apple_catalogue_feed.xml
# The "showContentId" from "tv_show" items are needed to process "tv_season" and "tv_episode" items,
# so you should sort the catalog to ensure items are in the proper order.

# INVOCATION
#   awk -v ERRORS=$ERRORS -v IDS_SEASONS=$IDS_SEASONS -v IDS_EPISODES=$IDS_EPISODES \
#       -v RAW_TITLES=$RAW_TITLES -f getBBoxCatalogFromSitemap.awk $SORTED_SITEMAPS >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title         3 Seasons          4 Episodes         5 Duration      6 Genre
#     7 Year          8 Rating        9 Description     10 Content_Type    11 Content_ID   12 Entity_ID
#    13 Show_Type    14 Date_Type    15 Original_Date   16 Show_ID         17 Season_ID    18 Sn_#
#    19 Ep_#         20 1st_#        21 Last_#

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tGenre\tYear\tRating\tDescription\t")
    printf ("Content_Type\tContent_ID\tEntity_ID\tShow_Type\tDate_Type\tOriginal_Date\t")
    printf ("Show_ID\tSeason_ID\tSn_#\tEp_#\t1st_#\tLast_#\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    firstLineNum = NR
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType > "/dev/stderr"
    # print "contentId = " contentId > "/dev/stderr"
    #
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = ""
    numEpisodes = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title > "/dev/stderr"
    gsub (/&/,"\\&",title)
    sub (/[[:space:]]+$/,"",title)
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    genre = toupper(substr(genre,1,1)) substr(genre,2)
    sub ("Sci_fi","Sci-Fi",genre)
    sub ("Special_interest","Special interest",genre)
    # print "genre = " genre > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description > "/dev/stderr"
    gsub (/&/,"\\&",description)
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType > "/dev/stderr"
    # print "originalDate = " originalDate > "/dev/stderr"
    # print "year = " year > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId > "/dev/stderr"
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    lastLineNum = NR

    # "The Moonstone" needs to be revised to avoid duplicate names
    if (title == "The Moonstone") {
        if (EntityId == "_9283") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'The Moonstone (2016)'\n", title) >> ERRORS
            title = "The Moonstone (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERRORS
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERRORS
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERRORS
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERRORS
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    if (EntityId == "") {
        missingEntityIds += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s at line %d\n", contentType, contentId,
                title, showContentId, firstLineNum) >> ERRORS
    }

    if (contentType == "movie") {
        totalMovies += 1
        # print "\ntv_movie" > "/dev/stderr"
        # Wish I didn't have to do this, but "movie" is too common to be in a key field
        contentType = "tv_movie"
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
    }

    if (contentType == "tv_show") {
        totalShows += 1
        # print "\ntv_show" > "/dev/stderr"
        showArray[totalShows] = contentId
        titleArray[totalShows] = title
        sortkey = sprintf ("%s (1) S%s %s", title, EntityId, contentId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
        #
        str = "grep -c " contentId " " IDS_SEASONS
        str | getline numSeasons
        close str
        #
        str = "grep -c " contentId " " IDS_EPISODES
        str | getline numEpisodes
        close str
    }

    if (contentType == "tv_season") {
        totalSeasons += 1
        # print "\ntv_season" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02d %s", showTitle, seasonNumber, showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02d %s", showTitle, seasonNumber, showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02d, %s", showTitle, seasonNumber, title)
    }

    if (contentType == "tv_episode") {
        totalEpisodes += 1
        # print "\ntv_episode" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                    showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02dE%03d, %s", showTitle, seasonNumber, episodeNumber, title)
    }

    # Generate a link that will lead to the show on Britbox
    # https://www.britbox.com/us/movie/A_Queen_Is_Crowned_13551
    # https://www.britbox.com/us/movie/_13551
    #
    # https://www.britbox.com/us/show/All_Creatures_Great_and_Small_23737
    # https://www.britbox.com/us/show/_23737
    #
    # https://www.britbox.com/us/season/All_Creatures_Great_and_Small_S2_23752
    # https://www.britbox.com/us/season/_23752
    #
    # https://www.britbox.com/us/episode/All_Creatures_Great_and_Small_S2_E2_23764
    # https://www.britbox.com/us/episode/_23764
    showType_URL = contentType "/"
    sub ("tv_","",showType_URL)
    full_URL = "https://www.britbox.com/us/" showType_URL EntityId
    fullTitle = "=HYPERLINK(\"" full_URL "\";\"" title "\")"
    # print "fullTitle = " fullTitle > "/dev/stderr"

    # A seasonless "show" should have blank rather than 0 seasons
    if (showType == "seasonless")
        numSeasons = ""
    # If an "episode" sortkey or fullTitle contains "S00E" delete the "S00" part
    sub ("S00E","E",sortkey)
    sub ("S00E","E",fullTitle)
    # Print everything except tv_seasons
    if (contentType != "tv_season")
        printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%06d\t%06d\n",
                sortkey, fullTitle, numSeasons, numEpisodes, duration, genre, year, rating, description,
                contentType, contentId, EntityId, showType, dateType, originalDate, showContentId,
                seasonContentId, seasonNumber, episodeNumber, firstLineNum, lastLineNum)
}

END {
    printf ("In getBBoxCatalogFromSitemap.awk\n") > "/dev/stderr"

    totalMovies == 1 ? pluralMovies = "movie" : pluralMovies = "movies"
    totalShows == 1 ? pluralShows = "show" : pluralShows = "shows"
    totalSeasons == 1 ? pluralSeasons = "season" : pluralSeasons = "seasons"
    totalEpisodes == 1 ? pluralEpisodes = "episode" : pluralEpisodes = "episodes"
    #
    printf ("    Processed %d %s, %d %s, %d %s, %d %s\n", totalMovies, pluralMovies, totalShows,
            pluralShows, totalSeasons, pluralSeasons, totalEpisodes, pluralEpisodes) > "/dev/stderr"

    if (missingEntityIds > 0 ) {
        missingEntityIds == 1 ? plural = "EntityId" : plural = "EntityIds"
        printf ("%8d missing %s in %s\n", missingEntityIds, plural, FILENAME) > "/dev/stderr"
    }

    if (missingShowContentIds > 0 ) {
        missingShowContentIds == 1 ? plural = "showContentId" : plural = "showContentIds"
        printf ("%8d missing %s in %s\n", missingShowContentIds, plural, FILENAME) > "/dev/stderr"
    }

    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? plural = "title" : plural = "titles"
        printf ("%8d %s revised in %s\n", revisedTitles, plural, FILENAME) > "/dev/stderr"
    }
}


Use URN in rdf-n3.awk

Use URN in rdf-n3.awk

#!/usr/bin/awk -f
BEGIN {
    FS = "\t";

    LABEL         = "<http://www.w3.org/2000/01/rdf-schema#label>";
    TYPE          = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";

    CONCEPT       = "<http://www.w3.org/2004/02/skos/core#Concept>";
    LEXICALSENSE  = "<http://lemon-model.net/lemon#LexicalSense>";
    WORD          = "<http://lemon-model.net/lemon#Word>";
    FORM          = "<http://lemon-model.net/lemon#Form>";

    WRITTENREP    = "<http://lemon-model.net/lemon#writtenRep>";
    CANONICALFORM = "<http://lemon-model.net/lemon#canonicalForm>";
    BROADER       = "<http://lemon-model.net/lemon#broader>";
    NARROWER      = "<http://lemon-model.net/lemon#narrower>";
    SENSE         = "<http://lemon-model.net/lemon#sense>";
    ISSENSEOF     = "<http://lemon-model.net/lemon#isSenseOf>";
    REFERENCE     = "<http://lemon-model.net/lemon#reference>";
    ISREFERENCEOF = "<http://lemon-model.net/lemon#isReferenceOf>";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0 || nisas == 0) next;

    print "<urn:linked:c" $1 ">", TYPE,  CONCEPT,           ".";
    print "<urn:linked:c" $1 ">", LABEL, "\"synset" $1 "\"", ".";

    for (sense_id in senses) {
        sense = senses[sense_id];

        sub(":.*$", "", sense);

        if (!(sense in sindex)) {
            sindex[sense] = scount++;

            word = sense;
            gsub("_", " ",  word);
            sub("#.*$", "", word);

            if (!(word in windex)) {
                windex[word] = wcount++;

                print "<urn:linked:w" windex[word] ">", TYPE,          WORD,                             ".";
                print "<urn:linked:w" windex[word] ">", LABEL,         "\"" word "\"",                   ".";
                print "<urn:linked:f" windex[word] ">", TYPE,          FORM,                             ".";
                print "<urn:linked:f" windex[word] ">", LABEL,         "\"" word "\"",                   ".";
                print "<urn:linked:f" windex[word] ">", WRITTENREP,    "\"" word "\"",                   ".";
                print "<urn:linked:w" windex[word] ">", CANONICALFORM, "<urn:linked:f" windex[word] ">", ".";
            }

            print "<urn:linked:s" sindex[sense] ">", TYPE,          LEXICALSENSE,       ".";
            print "<urn:linked:s" sindex[sense] ">", LABEL,         "\"" sense "\"",    ".";
            print "<urn:linked:s" sindex[sense] ">", ISSENSEOF,     "<urn:linked:w" windex[word] ">",  ".";
            print "<urn:linked:w" windex[word] ">",  SENSE,         "<urn:linked:s" sindex[sense] ">", ".";
            print "<urn:linked:s" sindex[sense] ">", REFERENCE,     "<urn:linked:c" $1 ">",            ".";
            print "<urn:linked:c" $1 ">",            ISREFERENCEOF, "<urn:linked:s" sindex[sense] ">", ".";
        }

        for (isa_id in isas) {
            isa = isas[isa_id];

            sub(":.*$", "", isa);

            if (!(isa in sindex)) {
                sindex[isa] = scount++;

                word = isa;
                gsub("_", " ",  word);
                sub("#.*$", "", word);

                if (!(word in windex)) {
                    windex[word] = wcount++;

                    print "<urn:linked:w" windex[word] ">", TYPE,          WORD,              ".";
                    print "<urn:linked:w" windex[word] ">", LABEL,         "\"" word "\"",    ".";
                    print "<urn:linked:f" windex[word] ">", TYPE,          FORM,              ".";
                    print "<urn:linked:f" windex[word] ">", LABEL,         "\"" word "\"",    ".";
                    print "<urn:linked:f" windex[word] ">", WRITTENREP,    "\"" word "\"",    ".";
                    print "<urn:linked:w" windex[word] ">", CANONICALFORM, "<urn:linked:f" windex[word] ">", ".";
                }

                print "<urn:linked:s" sindex[isa] ">",  TYPE,      LEXICALSENSE,      ".";
                print "<urn:linked:s" sindex[isa] ">",  LABEL,     "\"" isa "\"",     ".";
                print "<urn:linked:s" sindex[isa] ">",  ISSENSEOF, "<urn:linked:w" windex[word] ">", ".";
                print "<urn:linked:w" windex[word] ">", SENSE,     "<urn:linked:s" sindex[isa] ">",  ".";
            }

            print "<urn:linked:s" sindex[sense] ">", BROADER,  "<urn:linked:s" sindex[isa] ">",   ".";
            print "<urn:linked:s" sindex[isa] ">",   NARROWER, "<urn:linked:s" sindex[sense] ">", ".";
        }
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = "\t";

    LABEL         = "<http://www.w3.org/2000/01/rdf-schema#label>";
    TYPE          = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";

    CONCEPT       = "<http://www.w3.org/2004/02/skos/core#Concept>";
    LEXICALSENSE  = "<http://lemon-model.net/lemon#LexicalSense>";
    WORD          = "<http://lemon-model.net/lemon#Word>";
    FORM          = "<http://lemon-model.net/lemon#Form>";

    WRITTENREP    = "<http://lemon-model.net/lemon#writtenRep>";
    CANONICALFORM = "<http://lemon-model.net/lemon#canonicalForm>";
    BROADER       = "<http://lemon-model.net/lemon#broader>";
    NARROWER      = "<http://lemon-model.net/lemon#narrower>";
    SENSE         = "<http://lemon-model.net/lemon#sense>";
    ISSENSEOF     = "<http://lemon-model.net/lemon#isSenseOf>";
    REFERENCE     = "<http://lemon-model.net/lemon#reference>";
    ISREFERENCEOF = "<http://lemon-model.net/lemon#isReferenceOf>";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0 || nisas == 0) next;

    print ":c" $1, TYPE, CONCEPT, ".";

    for (sense_id in senses) {
        sense = senses[sense_id];

        sub(":.*$", "", sense);

        if (!(sense in sindex)) {
            sindex[sense] = scount++;

            word = sense;
            gsub("_", " ",  word);
            sub("#.*$", "", word);

            if (!(word in windex)) {
                windex[word] = wcount++;

                print ":w" windex[word], TYPE,          WORD,              ".";
                print ":w" windex[word], LABEL,         "\"" word "\"",    ".";
                print ":f" windex[word], TYPE,          FORM,              ".";
                print ":f" windex[word], LABEL,         "\"" word "\"",    ".";
                print ":f" windex[word], WRITTENREP,    "\"" word "\"",    ".";
                print ":w" windex[word], CANONICALFORM, ":f" windex[word], ".";
            }

            print ":s" sindex[sense], TYPE,          LEXICALSENSE,       ".";
            print ":s" sindex[sense], LABEL,         "\"" sense "\"",    ".";
            print ":s" sindex[sense], ISSENSEOF,     ":w" windex[word],  ".";
            print ":w" windex[word],  SENSE,         ":s" sindex[sense], ".";
            print ":s" sindex[sense], REFERENCE,     ":c" $1,            ".";
            print ":c" $1,            ISREFERENCEOF, ":s" sindex[sense], ".";
        }

        for (isa_id in isas) {
            isa = isas[isa_id];

            sub(":.*$", "", isa);

            if (!(isa in sindex)) {
                sindex[isa] = scount++;

                word = isa;
                gsub("_", " ",  word);
                sub("#.*$", "", word);

                if (!(word in windex)) {
                    windex[word] = wcount++;

                    print ":w" windex[word], TYPE,          WORD,              ".";
                    print ":w" windex[word], LABEL,         "\"" word "\"",    ".";
                    print ":f" windex[word], TYPE,          FORM,              ".";
                    print ":f" windex[word], LABEL,         "\"" word "\"",    ".";
                    print ":f" windex[word], WRITTENREP,    "\"" word "\"",    ".";
                    print ":w" windex[word], CANONICALFORM, ":f" windex[word], ".";
                }

                print ":s" sindex[isa],  TYPE,      LEXICALSENSE,      ".";
                print ":s" sindex[isa],  LABEL,     "\"" isa "\"",     ".";
                print ":s" sindex[isa],  ISSENSEOF, ":w" windex[word], ".";
                print ":w" windex[word], SENSE,     ":s" sindex[isa],  ".";
            }

            print ":s" sindex[sense], BROADER,  ":s" sindex[isa],   ".";
            print ":s" sindex[isa],   NARROWER, ":s" sindex[sense], ".";
        }
    }
}