Commons: add NULLSTR

Commons: add NULLSTR

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used constants and functions.                           #
####################################################################

# Initialize constants.
function initConst() {
    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "

    NULLSTR = ""
}

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, "", line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    if (length(array)) {
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != "")
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array # array is empty string

    return temp
}

# Return 0 if the string starts with '0', 'f' or 'n';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^[0fn]/) ? 0 : 1
}

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return non-zero if file exists, otherwise return 0.
function fileExists(file) {
    return !system("test -f " file)
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

BEGIN {
    initConst()

    initUrlEncoding()
    initAnsiCode()
    initUriSchemes()
}

####################################################################
# Commons.awk                                                      #
#                                                                  #
# Commonly used constants and functions.                           #
####################################################################

# Initialize constants.
function initConst() {
    STDIN  = "/dev/stdin"
    STDOUT = "/dev/stdout"
    STDERR = "/dev/stderr"

    SUPOUT = " > /dev/null "  # suppress output
    SUPERR = " 2> /dev/null " # suppress error
    PIPE = " | "
}

# Initialize `UrlEncoding`.
# See: <https://en.wikipedia.org/wiki/Percent-encoding>
function initUrlEncoding() {
    UrlEncoding["\n"] = "%0A"
    UrlEncoding[" "]  = "%20"
    UrlEncoding["!"]  = "%21"
    UrlEncoding["\""] = "%22"
    UrlEncoding["#"]  = "%23"
    UrlEncoding["$"]  = "%24"
    UrlEncoding["%"]  = "%25"
    UrlEncoding["&"]  = "%26"
    UrlEncoding["'"]  = "%27"
    UrlEncoding["("]  = "%28"
    UrlEncoding[")"]  = "%29"
    UrlEncoding["*"]  = "%2A"
    UrlEncoding["+"]  = "%2B"
    UrlEncoding[","]  = "%2C"
    UrlEncoding["-"]  = "%2D"
    UrlEncoding["."]  = "%2E"
    UrlEncoding["/"]  = "%2F"
    UrlEncoding[":"]  = "%3A"
    UrlEncoding[";"]  = "%3B"
    UrlEncoding["<"]  = "%3C"
    UrlEncoding["="]  = "%3D"
    UrlEncoding[">"]  = "%3E"
    UrlEncoding["?"]  = "%3F"
    UrlEncoding["@"]  = "%40"
    UrlEncoding["["]  = "%5B"
    UrlEncoding["\\"] = "%5C"
    UrlEncoding["]"]  = "%5D"
    UrlEncoding["^"]  = "%5E"
    UrlEncoding["_"]  = "%5F"
    UrlEncoding["`"]  = "%60"
    UrlEncoding["{"]  = "%7B"
    UrlEncoding["|"]  = "%7C"
    UrlEncoding["}"]  = "%7D"
    UrlEncoding["~"]  = "%7E"
}

# Return the real character represented by an escape sequence.
# Example: escapeChar("n") returns "\n".
# See: <https://en.wikipedia.org/wiki/Escape_character>
#      <https://en.wikipedia.org/wiki/Escape_sequences_in_C>
function escapeChar(char) {
    switch (char) {
    case "b":
        return "\b" # Backspace
    case "f":
        return "\f" # Formfeed
    case "n":
        return "\n" # Newline (Line Feed)
    case "r":
        return "\r" # Carriage Return
    case "t":
        return "\t" # Horizontal Tab
    case "v":
        return "\v" # Vertical Tab
    default:
        return char
    }
}

# Convert a literal-formatted string into its original string.
function literal(string,
                 ####
                 c, escaping, i, s) {
    if (string !~ /^".*"$/)
        return string

    split(string, s, "")
    string = ""
    escaping = 0
    for (i = 2; i < length(s); i++) {
        c = s[i]
        if (escaping) {
            string = string escapeChar(c)
            escaping = 0 # escape ends
        } else {
            if (c == "\\")
                escaping = 1 # escape begins
            else
                string = string c
        }
    }
    return string
}

# Return the escaped string.
function escape(string) {
    gsub(/"/, "\\\"", string)
    gsub(/\\/, "\\\\", string)
    return string
}

# Return the escaped, quoted string.
function parameterize(string, quotationMark) {
    if (!quotationMark)
        quotationMark = "'"

    if (quotationMark == "'") {
        gsub(/'/, "'\\''", string)
        return "'" string "'"
    } else {
        return "\"" escape(string) "\""
    }
}

# Return the URL-encoded string.
function quote(string,    i, r, s) {
    r = ""
    split(string, s, "")
    for (i = 1; i <= length(s); i++)
        r = r (s[i] in UrlEncoding ? UrlEncoding[s[i]] : s[i])
    return r
}

# Replicate a string.
function replicate(string, len,
                   ####
                   i, temp) {
    temp = ""
    for (i = 0; i < len; i++)
        temp = temp string
    return temp
}

# Squeeze a source line of AWK code.
function squeeze(line, preserveIndent) {
    # Remove preceding spaces if indentation not preserved
    if (!preserveIndent)
        gsub(/^[[:space:]]+/, "", line)
    # Remove comment
    gsub(/^[[:space:]]*#.*$/, "", line)
    # Remove in-line comment
    gsub(/#[^"]*$/, "", line)
    # Remove trailing spaces
    gsub(/[[:space:]]+$/, "", line)
    gsub(/[[:space:]]+\\$/, "\\", line)

    return line
}

# Return 1 if the array contains anything; otherwise return 0.
function anything(array,
                  ####
                  i) {
    for (i in array)
        if (array[i]) return 1
    return 0
}

# Append an element into an array (zero-based).
function append(array, element) {
    array[anything(array) ? length(array) : 0] = element
}

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Join an array into one string;
# Return the string.
function join(array, separator, sortedIn, preserveNull,
              ####
              i, j, saveSortedIn, temp) {
    # Default parameters
    if (!separator)
        separator = " "
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    temp = ""
    j = 0
    saveSortedIn = PROCINFO["sorted_in"]
    if (length(array)) {
        PROCINFO["sorted_in"] = sortedIn
        for (i in array)
            if (preserveNull || array[i] != "")
                temp = j++ ? temp separator array[i] : array[i]
        PROCINFO["sorted_in"] = saveSortedIn
    } else
        temp = array # array is empty string

    return temp
}

# Return 0 if the string starts with '0', 'f' or 'n';
# Otherwise, return 1.
function yn(string) {
    return (tolower(string) ~ /^[0fn]/) ? 0 : 1
}

# Initialize ANSI escape codes for SGR (Select Graphic Rendition).
# (ANSI X3.64 Standard Control Sequences)
# See: <https://en.wikipedia.org/wiki/ANSI_escape_code>
function initAnsiCode() {
    # Dumb terminal: no ANSI escape code whatsoever
    if (ENVIRON["TERM"] == "dumb") return

    AnsiCode["reset"]         = AnsiCode[0] = "\33[0m"

    AnsiCode["bold"]          = "\33[1m"
    AnsiCode["underline"]     = "\33[4m"
    AnsiCode["negative"]      = "\33[7m"
    AnsiCode["no bold"]       = "\33[22m" # SGR code 21 (bold off) not widely supported
    AnsiCode["no underline"]  = "\33[24m"
    AnsiCode["positive"]      = "\33[27m"

    AnsiCode["black"]         = "\33[30m"
    AnsiCode["red"]           = "\33[31m"
    AnsiCode["green"]         = "\33[32m"
    AnsiCode["yellow"]        = "\33[33m"
    AnsiCode["blue"]          = "\33[34m"
    AnsiCode["magenta"]       = "\33[35m"
    AnsiCode["cyan"]          = "\33[36m"
    AnsiCode["gray"]          = "\33[37m"

    AnsiCode["default"]       = "\33[39m"

    AnsiCode["dark gray"]     = "\33[90m"
    AnsiCode["light red"]     = "\33[91m"
    AnsiCode["light green"]   = "\33[92m"
    AnsiCode["light yellow"]  = "\33[93m"
    AnsiCode["light blue"]    = "\33[94m"
    AnsiCode["light magenta"] = "\33[95m"
    AnsiCode["light cyan"]    = "\33[96m"
    AnsiCode["white"]         = "\33[97m"
}

# Return ANSI escaped string.
function ansi(code, text) {
    switch (code) {
    case "bold":
        return AnsiCode[code] text AnsiCode["no bold"]
    case "underline":
        return AnsiCode[code] text AnsiCode["no underline"]
    case "negative":
        return AnsiCode[code] text AnsiCode["positive"]
    default:
        return AnsiCode[code] text AnsiCode[0]
    }
}

# Print warning message.
function w(text) {
    print ansi("yellow", text) > STDERR
}

# Print error message.
function e(text) {
    print ansi("bold", ansi("red", text)) > STDERR
}

# Print debugging message.
function d(text) {
    print ansi("gray", text) > STDERR
}

# Log an array.
function da(array, formatString, sortedIn,
            ####
            i, j, saveSortedIn) {
    # Default parameters
    if (!formatString)
        formatString = "_[%s]='%s'"
    if (!sortedIn)
        sortedIn = "@ind_num_asc"

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = sortedIn
    for (i in array) {
        split(i, j, SUBSEP)
        d(sprintf(formatString, join(j, ","), array[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn
}

# Naive assertion.
function assert(x, message) {
    if (!message)
        message = "[ERROR] Assertion failed."

    if (x)
        return x
    else
        e(message)
}

# Return non-zero if file exists, otherwise return 0.
function fileExists(file) {
    return !system("test -f " file)
}

# Initialize `UriSchemes`.
function initUriSchemes() {
    UriSchemes[0] = "file://"
    UriSchemes[1] = "http://"
    UriSchemes[2] = "https://"
}

BEGIN {
    initConst()

    initUrlEncoding()
    initAnsiCode()
    initUriSchemes()
}


Replace iterator with a diff.

Replace iterator with a diff.

BEGIN {
  year=strftime("%Y");
  starttime=0;
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

# divide and rund up anything that is not an onteger
# e.g. 1.1 becomes 2
function updiv(x,y) {
  t=1.0*x/y;
  dstr=sprintf("%d",t+0.99);
  return int(dstr);
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,2,length(shadow)-2)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID: /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }

  if (starttime>0) {
    joblength=ended-starttime;
  } else {
    # protect from incomplete data
    joblength=0;
  }

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}

/^===NewFile===/{
  if (starttime>0) {
    print "Termination event for job",jid,"from",shadow,"not found"
  }
  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}
########################################################################
END {
  tj=updiv(ended_jobs,parallelism);

  #since it is rounded, the sum could exceed tj
  tcnt=0;
  tgnz=updiv(goodjobsNZ,parallelism);tcnt=tcnt+tgnz;
  if (tcnt>tj) {tgnz=tgnz-(tcnt-tj); tcnt=tj;}
  tbjs=updiv(badjobsSignal,parallelism);tcnt=tcnt+tbjs;
  if (tcnt>tj) {tbjs=tbjs-(tcnt-tj); tcnt=tj;}
  tbjo=updiv(badjobsOther,parallelism);tcnt=tcnt+tbjo;
  if (tcnt>tj) {tbjo=tbjo-(tcnt-tj); tcnt=tj;}

  # if needed, penalize the "good jobs" the most
  # we mostly want to monitor failures
  tgz=updiv(goodjobsZ,parallelism); tcnt=tcnt+tgz;
  if (tcnt>tj) {tgz=tgz-(tcnt-tj); tcnt=tj;}
  # please notice that the percentages will still be accurate

  print "=================";
  print "Total jobs",tj,"utilization",int(total_used/parallelism);
  print "Total goodZ jobs",tgz, " (" percent(goodjobsZ,ended_jobs) "%)","utilization",int(goodputZ/parallelism) " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",tgnz, " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",int(goodputNZ/parallelism) " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",tbjs, " (" percent(badjobsSignal,ended_jobs) "%)","utilization",int(badputSignal/parallelism) " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",tbjo, " (" percent(badjobsOther,ended_jobs) "%)","utilization",int(badputOther/parallelism) " (" percent(badputOther,total_used) "%)";
}

BEGIN {
  year=strftime("%Y");
  starttime=0;
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

# divide and rund up anything that is not an onteger
# e.g. 1.1 becomes 2
function updiv(x,y) {
  t=1.0*x/y;
  dstr=sprintf("%d",t+0.99);
  return int(dstr);
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,2,length(shadow)-2)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID: /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }

  if (starttime>0) {
    joblength=ended-starttime;
  } else {
    # protect from incomplete data
    joblength=0;
  }

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}

/^===NewFile===/{
  if (starttime>0) {
    print "Termination event for job",jid,"from",shadow,"not found"
  }
  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}
########################################################################
END {
  tj=updiv(ended_jobs,parallelism);

  #since it is rounded, the sum could exceed tj
  tcnt=0;
  tgnz=updiv(goodjobsNZ,parallelism);tcnt=tcnt+tgnz;
  while (tcnt>tj) {tgnz=tgnz-1; tcnt=tcnt-1;}
  tbjs=updiv(badjobsSignal,parallelism);tcnt=tcnt+tbjs;
  while (tcnt>tj) {tbjs=tbjs-1; tcnt=tcnt-1;}
  tbjo=updiv(badjobsOther,parallelism);tcnt=tcnt+tbjo;
  while (tcnt>tj) {tbjo=tbjo-1; tcnt=tcnt-1;}

  # if needed, penalize the "good jobs" the most
  # we mostly want to monitor failures
  tgz=updiv(goodjobsZ,parallelism); tcnt=tcnt+tgz;
  while (tcnt>tj) {tgz=tgz-1; tcnt=tcnt-1;}
  # please notice that the percentages will still be accurate

  print "=================";
  print "Total jobs",tj,"utilization",int(total_used/parallelism);
  print "Total goodZ jobs",tgz, " (" percent(goodjobsZ,ended_jobs) "%)","utilization",int(goodputZ/parallelism) " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",tgnz, " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",int(goodputNZ/parallelism) " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",tbjs, " (" percent(badjobsSignal,ended_jobs) "%)","utilization",int(badputSignal/parallelism) " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",tbjo, " (" percent(badjobsOther,ended_jobs) "%)","utilization",int(badputOther/parallelism) " (" percent(badputOther,total_used) "%)";
}


Fix executing movingavg filter using mawk

The mawk interpreter does not support lshift().

#!/usr/bin/awk -f
#
# Applies a moving average.
#
# The user is expected to set the following parameters via `-v var=value`
# command line arguments:
#
# | Variable  | Description                                              |
# |-----------|----------------------------------------------------------|
# | `COLUMNS` | A `FS` separated list of column numbers                  |
# | `PRE`     | The number of time slices to use before the sample point |
# | `POST`    | The number of time slices to use behind the sample point |
#
# Algorithm
# ---------
#
# The moving average algorithm takes the mean over a set of buffered
# values, the user provided PRE and POST values determine the amount
# of samples used.
#
# The following global variables control the algorithm and maintain
# its state:
#
# | Variable   | Type           | Description                          |
# |------------|----------------|--------------------------------------|
# | `PRE`      | int            | The target pre-sample buffer size    |
# | `POST`     | int            | The target post-sample buffer size   |
# | `SIZE`     | int            | The target buffer size (PRE + POST)  |
# | `SUM`      | int[co]        | The sums of buffered input columns   |
# | `SUM_TO`   | double         | Factor for conversion to fix-point   |
# | `SUM_FROM` | double         | Factor for conversion from fix-point |
# | `BUF_IN`   | double[li, co] | Buffered inputs                      |
# | `BUF_OUT`  | double[li, co] | Buffered outputs                     |
# | `BUF_SIZE` | int            | The actual buffer size               |
#
# ### Computation of the Moving Average
#
# The `SIZE = PRE + POST` value represents the desired buffer size.
# The actual buffer size is kept in `BUF_SIZE`. At the beginning of
# the file `BUF_SIZE` grows with every line read until it has reached
# `SIZE`.
#
# For each input column an output buffer is kept, untouched columns
# are put straight into the output buffer `BUF_OUT`. This buffer
# is kept around for the last `BUF_SIZE` lines of processed input.
# Filtered columns are first stored in the `BUF_IN` buffer. The per
# column `SUM` values represent the sum of all `BUF_IN` values in
# a fix-point representation.
#
# The mean is derived from the `SUM` for each line of output, once
# `BUF_SIZE` has reached `SIZE` the oldest value in `BUF_IN` is subtracted
# from the `SUM` and deleted with each additional line of input. The
# fix-point representation of `SUM` ensures that the sum does not
# accumulate drift through floating point rounding.
#
# ### Output
#
# How the mean is output depends on the values of `PRE` and `POST`.
# If only `PRE` is set every line of input causes an immediate output
# of the updated mean. If `POST` is set the output is delayed so
# *future values* become part of the mean.
#
# If `POST` is set the remaining lines of output have to be produced
# after reading the last input line. The `POST` buffer effect is
# reduced with every line of output until the last line, which only
# is made up of PRE buffer values.
#

#
# Setup global constants.
#
BEGIN {
	# extract columns
	split(COLUMNS, columns)
	for (i in columns) { SUM[columns[i]] = 0 }
	delete columns

	# sampling buffer properties
	PRE  = int(PRE)    # pre sample buffer size
	POST = int(POST)   # post sample buffer size
	SIZE = PRE + POST  # buffer size
	if (!SIZE) {
		print "error: movingavg: PRE + POST buffer must contain at least 1 value" > "/dev/stderr"
		exit 1
	}

	# use 20 fix-point fraction bits to calculate the buffer sum
	SUM_TO   = 1048576
	SUM_FROM = 1. / SUM_TO
}

#
# Retrieve/Update the buffer sum for the given column.
#
# This function is the single point of access to the SUM buffer, it
# enforces integer arithmetic with a fixed number of fraction bits
# to avoid floating point drift.
#
# @param co
#	The column to retrieve/update
# @param offset
#	The change to the sum (optional)
# @param SUM
#	Global per column storage array
# @param SUM_TO,SUM_FROM
#	To and from fixed point conversion factors
# @return
#	The current buffer sum for the given column
#
function sum(co, offset) {
	return (SUM[co] += int(offset * SUM_TO + .5)) * SUM_FROM
}

#
# Print a buffered line, and trim the output buffer.
#
# Overwrites the input fields.
#
# @param NR,POST
#	The current line and the output line offset
# @param BUF_OUT
#	The output buffer to output a line from and trim
#
function printbuf(_li, _co) {
	# Fetch columns from output buffer
	_li = NR - POST
	for (_co = 1; _co <= NF; ++_co) {
		$_co = BUF_OUT[_li, _co]
		delete BUF_OUT[_li, _co]
	}
	print
}

#
# Update the buffer size until the target is reached.
#
NR <= SIZE { BUF_SIZE = NR }

#
# Update buffers.
#
# - Forward unaffected columns to the output buffer
# - Update BUF_IN, SUM and BUF_OUT for managed columns
# - Trim obsolete input buffer
#
{
	for (co = 1; co <= NF; ++co) {
		if (co in SUM) {
			# remove buffered input moving out of the
			# buffering window
			sum(co, -BUF_IN[NR - SIZE, co])
			delete  BUF_IN[NR - SIZE, co]

			# add latest value to the input buffer
			BUF_IN[NR, co] = $co
			# add latest value to the buffer sum and
			# update the output buffer
			BUF_OUT[NR - POST, co] = sum(co, +$co) / BUF_SIZE
		} else {
			# forward column to the output buffer
			BUF_OUT[NR, co] = $co
		}
	}
}

#
# For every line of input, produce a line of output, starting when
# the POST buffer is full.
#
NR > POST { printbuf() }

#
# Flush the remaining post buffer.
#
END {
	for (o = 1; o <= POST; ++o) {
		++NR
		--BUF_SIZE
		for (co in SUM) {
			# remove buffered input moving out of the
			# buffering window and update the output buffer
			BUF_OUT[NR - POST, co] = sum(co, -BUF_IN[NR - SIZE, co]) / BUF_SIZE
			delete  BUF_IN[NR - SIZE, co]
		}
		printbuf()
	}
}

#!/usr/bin/awk -f
#
# Applies a moving average.
#
# The user is expected to set the following parameters via `-v var=value`
# command line arguments:
#
# | Variable  | Description                                              |
# |-----------|----------------------------------------------------------|
# | `COLUMNS` | A `FS` separated list of column numbers                  |
# | `PRE`     | The number of time slices to use before the sample point |
# | `POST`    | The number of time slices to use behind the sample point |
#
# Algorithm
# ---------
#
# The moving average algorithm takes the mean over a set of buffered
# values, the user provided PRE and POST values determine the amount
# of samples used.
#
# The following global variables control the algorithm and maintain
# its state:
#
# | Variable   | Type           | Description                          |
# |------------|----------------|--------------------------------------|
# | `PRE`      | int            | The target pre-sample buffer size    |
# | `POST`     | int            | The target post-sample buffer size   |
# | `SIZE`     | int            | The target buffer size (PRE + POST)  |
# | `SUM`      | int[co]        | The sums of buffered input columns   |
# | `SUM_TO`   | double         | Factor for conversion to fix-point   |
# | `SUM_FROM` | double         | Factor for conversion from fix-point |
# | `BUF_IN`   | double[li, co] | Buffered inputs                      |
# | `BUF_OUT`  | double[li, co] | Buffered outputs                     |
# | `BUF_SIZE` | int            | The actual buffer size               |
#
# ### Computation of the Moving Average
#
# The `SIZE = PRE + POST` value represents the desired buffer size.
# The actual buffer size is kept in `BUF_SIZE`. At the beginning of
# the file `BUF_SIZE` grows with every line read until it has reached
# `SIZE`.
#
# For each input column an output buffer is kept, untouched columns
# are put straight into the output buffer `BUF_OUT`. This buffer
# is kept around for the last `BUF_SIZE` lines of processed input.
# Filtered columns are first stored in the `BUF_IN` buffer. The per
# column `SUM` values represent the sum of all `BUF_IN` values in
# a fix-point representation.
#
# The mean is derived from the `SUM` for each line of output, once
# `BUF_SIZE` has reached `SIZE` the oldest value in `BUF_IN` is subtracted
# from the `SUM` and deleted with each additional line of input. The
# fix-point representation of `SUM` ensures that the sum does not
# accumulate drift through floating point rounding.
#
# ### Output
#
# How the mean is output depends on the values of `PRE` and `POST`.
# If only `PRE` is set every line of input causes an immediate output
# of the updated mean. If `POST` is set the output is delayed so
# *future values* become part of the mean.
#
# If `POST` is set the remaining lines of output have to be produced
# after reading the last input line. The `POST` buffer effect is
# reduced with every line of output until the last line, which only
# is made up of PRE buffer values.
#

#
# Setup global constants.
#
BEGIN {
	# extract columns
	split(COLUMNS, columns)
	for (i in columns) { SUM[columns[i]] = 0 }
	delete columns

	# sampling buffer properties
	PRE  = int(PRE)    # pre sample buffer size
	POST = int(POST)   # post sample buffer size
	SIZE = PRE + POST  # buffer size
	if (!SIZE) {
		print "error: movingavg: PRE + POST buffer must contain at least 1 value" > "/dev/stderr"
		exit 1
	}

	# use 20 fix-point fraction bits to calculate the buffer sum
	SUM_TO   = lshift(1, 20)
	SUM_FROM = 1. / SUM_TO
}

#
# Retrieve/Update the buffer sum for the given column.
#
# This function is the single point of access to the SUM buffer, it
# enforces integer arithmetic with a fixed number of fraction bits
# to avoid floating point drift.
#
# @param co
#	The column to retrieve/update
# @param offset
#	The change to the sum (optional)
# @param SUM
#	Global per column storage array
# @param SUM_TO,SUM_FROM
#	To and from fixed point conversion factors
# @return
#	The current buffer sum for the given column
#
function sum(co, offset) {
	return (SUM[co] += int(offset * SUM_TO + .5)) * SUM_FROM
}

#
# Print a buffered line, and trim the output buffer.
#
# Overwrites the input fields.
#
# @param NR,POST
#	The current line and the output line offset
# @param BUF_OUT
#	The output buffer to output a line from and trim
#
function printbuf(_li, _co) {
	# Fetch columns from output buffer
	_li = NR - POST
	for (_co = 1; _co <= NF; ++_co) {
		$_co = BUF_OUT[_li, _co]
		delete BUF_OUT[_li, _co]
	}
	print
}

#
# Update the buffer size until the target is reached.
#
NR <= SIZE { BUF_SIZE = NR }

#
# Update buffers.
#
# - Forward unaffected columns to the output buffer
# - Update BUF_IN, SUM and BUF_OUT for managed columns
# - Trim obsolete input buffer
#
{
	for (co = 1; co <= NF; ++co) {
		if (co in SUM) {
			# remove buffered input moving out of the
			# buffering window
			sum(co, -BUF_IN[NR - SIZE, co])
			delete  BUF_IN[NR - SIZE, co]

			# add latest value to the input buffer
			BUF_IN[NR, co] = $co
			# add latest value to the buffer sum and
			# update the output buffer
			BUF_OUT[NR - POST, co] = sum(co, +$co) / BUF_SIZE
		} else {
			# forward column to the output buffer
			BUF_OUT[NR, co] = $co
		}
	}
}

#
# For every line of input, produce a line of output, starting when
# the POST buffer is full.
#
NR > POST { printbuf() }

#
# Flush the remaining post buffer.
#
END {
	for (o = 1; o <= POST; ++o) {
		++NR
		--BUF_SIZE
		for (co in SUM) {
			# remove buffered input moving out of the
			# buffering window and update the output buffer
			BUF_OUT[NR - POST, co] = sum(co, -BUF_IN[NR - SIZE, co]) / BUF_SIZE
			delete  BUF_IN[NR - SIZE, co]
		}
		printbuf()
	}
}


fix comment that mislabeled field numbers

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

# INVOCATION
#   awk -v ERROR_FILE=$ERROR_FILE -f getBBoxCatalogFrom-sitemap.awk $SITEMAP_FILE >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title           3 Seasons          4 Episodes      5 Duration     6 Year
#     7 Rating        8 Description     9 Content Type    10 Content ID   11 Entity ID   12 Genre
#    13 Show Type    14 Date Type      15 Original Date   16 Show ID      17 Season ID   18 Sn #   19 Ep #

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear\tRating\tDescription\t")
    printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    printf ("Show ID\tSeason ID\tSn #\tEp #\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType  > "/dev/stderr"
    # print "contentId = " contentId  > "/dev/stderr"
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = 111
    numEpisodes = 222
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title  > "/dev/stderr"
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre  > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description  > "/dev/stderr"
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating  > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType  > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType  > "/dev/stderr"
    # print "originalDate = " originalDate  > "/dev/stderr"
    # print "year = " year  > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId  > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId  > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber  > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber  > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId  > "/dev/stderr"
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERROR_FILE
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERROR_FILE
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERROR_FILE
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERROR_FILE
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> originalDate = " originalDate  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    if (EntityId == "") {
        missingIDs += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s\n", contentType, contentId, title, \
                showContentId) >> ERROR_FILE
        # Add missing EntityId
        if (contentId == "p07gnw9f") {
            EntityId = "_23842"
            printf ("==> Added EntityId %s for %s %s '%s' in show %s\n", EntityId, contentType, contentId,
                    title, showContentId) >> ERROR_FILE
            # print "==> EntityId = " EntityId  > "/dev/stderr"
        }
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) S%s", title, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) S%02d", showTitle, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, \
                episodeNumber, showContentId)
    }

    # Copied from printing header to make it easier to coordinate arranging fields
    # printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    # printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    # printf ("Show ID\tSeason ID\tSn #\tEp #\n")

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
            sortkey, title, numSeasons, numEpisodes, duration, year, rating, description, \
            contentType, contentId, EntityId, genre, showType, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

END {
    printf ("In getBBoxCatalogFrom-sitemap.awk\n") > "/dev/stderr"
    if (missingIDs > 0 ) {
        missingIDs == 1 ? field = "EntityId" : field = "EntityIds"
        printf ("    %2d missing %s in %s\n", missingIDs, field, FILENAME) > "/dev/stderr"
    }
    #
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
}

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

# INVOCATION
#   awk -v ERROR_FILE=$ERROR_FILE -f getBBoxCatalogFrom-sitemap.awk $SITEMAP_FILE >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title           3 Seasons          4 Episodes      5 Duration     6 Year
#     7 Rating        8 Description     9 Content Type    10 Content ID   11 Entity ID   12 Genre
#    13 Show Type    14 Date Type      15 Original Date   16 Show ID      16 Season ID   13 Sn #   14 Ep #

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear\tRating\tDescription\t")
    printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    printf ("Show ID\tSeason ID\tSn #\tEp #\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType  > "/dev/stderr"
    # print "contentId = " contentId  > "/dev/stderr"
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = 111
    numEpisodes = 222
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title  > "/dev/stderr"
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre  > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description  > "/dev/stderr"
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating  > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType  > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType  > "/dev/stderr"
    # print "originalDate = " originalDate  > "/dev/stderr"
    # print "year = " year  > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId  > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId  > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber  > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber  > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId  > "/dev/stderr"
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERROR_FILE
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERROR_FILE
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERROR_FILE
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERROR_FILE
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title  > "/dev/stderr"
        # print "==> originalDate = " originalDate  > "/dev/stderr"
        # print "==> contentId = " contentId  > "/dev/stderr"
        # print "==> EntityId = " EntityId  > "/dev/stderr"
        # print "---"  > "/dev/stderr"
    }

    if (EntityId == "") {
        missingIDs += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s\n", contentType, contentId, title, \
                showContentId) >> ERROR_FILE
        # Add missing EntityId
        if (contentId == "p07gnw9f") {
            EntityId = "_23842"
            printf ("==> Added EntityId %s for %s %s '%s' in show %s\n", EntityId, contentType, contentId,
                    title, showContentId) >> ERROR_FILE
            # print "==> EntityId = " EntityId  > "/dev/stderr"
        }
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) S%s", title, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) S%02d", showTitle, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, \
                episodeNumber, showContentId)
    }

    # Copied from printing header to make it easier to coordinate arranging fields
    # printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    # printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    # printf ("Show ID\tSeason ID\tSn #\tEp #\n")

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
            sortkey, title, numSeasons, numEpisodes, duration, year, rating, description, \
            contentType, contentId, EntityId, genre, showType, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

END {
    printf ("In getBBoxCatalogFrom-sitemap.awk\n") > "/dev/stderr"
    if (missingIDs > 0 ) {
        missingIDs == 1 ? field = "EntityId" : field = "EntityIds"
        printf ("    %2d missing %s in %s\n", missingIDs, field, FILENAME) > "/dev/stderr"
    }
    #
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
}


Remove if statement.

Remove if statement.

Unnecessary if statement.

BEGIN { if ( NF == 0 ) print "Hello, world!" }


ahrf.awk: Reduce URL length, in case it's >60

#!/home/ypnose/Dev/AWK/nawk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0, ":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====/ {
	if (NF > 2 && $NF == "====") {
		gsub(/^[\t ]*====[\t ]*\n|\n[\t ]*====[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\*/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l != "") {
			syb = substr($l,2,1)
			if (syb == " ") {
				str = substr($l,3)
			} else {
				str = substr($l,2)
			}
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u, "\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			if (length(url) >= 60) {
				printf("\t
%s %.60s...
\n", num, url, url)
			} else {
				printf("\t
%s %s
\n", num, url, url)
			}
		}
	}
	printf("
\n")
	next
}

#!/home/ypnose/Dev/AWK/nawk -f

BEGIN { FS = "\n"; RS = "" }

# Common shell symbols to HTML
{
	# Two backslashes for nawk(1) and OpenBSD awk(1)
	gsub(/\&/,"\\&")
	gsub(/</,"\\<")
	gsub(/>/,"\\>")
}

# h1, h2, h3, h4, h5, h6
/^[\t ]*:/ {
	match($0, ":+")
	cnt = RLENGTH
	gsub(/^[\t ]*:+[\t ]*|[\t ]*:+[\t ]*$/,"")
	# length($0) would also work
	if (cnt <= 6 && $0 != "") {
		printf("<h%s>%s</h%s>\n", cnt, $0, cnt)
	}
	next
}

# Paragraph
/^[A-Za-z0-9]+/ {
	printf("
%s
\n\n", $0)
	next
}

# Code
/^====/ {
	if (NF > 2 && $NF == "====") {
		gsub(/^[\t ]*====[\t ]*\n|\n[\t ]*====[\t ]*$/,"")
		printf("
")
		for (c=1; c<NF; c++) {
			gsub(/^ +$/,"",$c)
			printf("%s\n", $c)
		}
		# Solve "space" bug! Less crado (gsub).
		gsub(/^ +/,"",$NF)
		printf("%s
\n", $NF)
	}
	next
}

# List
/^[\t ]*\*/ {
	printf("
\n")
	for (l=1; l<=NF; l++) {
		gsub(/^[\t ]*/,"",$l)
		if ($l != "") {
			syb = substr($l,2,1)
			if (syb == " ") {
				str = substr($l,3)
			} else {
				str = substr($l,2)
			}
			printf("\t
%s
\n", str)
		}
	}
	printf("
\n")
	next
}

# End links
/^[\t ]*\[[01]/ {
	printf("
\n")
	for (u=1; u<=NF; u++) {
		gsub(/^[\t ]*/,"",$u)
		if (match($u, "\\[[0-9]+\\]")) {
			# RSTART can be replaced by 1, because we removed useless tabs/spaces
			num = substr($u,RSTART,RLENGTH)
			url = substr($u,RSTART+RLENGTH+1)
			printf("\t
%s %s
\n", num, url, url)
		}
	}
	printf("
\n")
	next
}


use `0-9` instead of [:digit:] to match digit char

BEGIN{
    split(ARGV[1],arr, "-");
    if (arr[2] == "") {
	while ((getline line < ARGV[2]) > 0) {
	    if (line ~ arr[1]) {
		if (line ~ /\[-[0-9.]+\]/) {
		    match(line, /[0-9.]+/);
		    print arr[1], substr(line, RSTART, RLENGTH);
		}
	    }
	}
    }else{
	print arr[1],arr[2];
    }
}

BEGIN{
    split(ARGV[1],arr, "-");
    if (arr[2] == "") {
	while ((getline line < ARGV[2]) > 0) {
	    if (line ~ arr[1]) {
		if (line ~ /\[-[[:digit:].]*\]/) {
		    match(line, /[[:digit:].]+/);
		    print arr[1], substr(line, RSTART, RLENGTH);
		}
	    }
	}
    }else{
	print arr[1],arr[2];
    }
}


sysconfig -> rc.conf Submitted by:	Igor Vinokurov <igor@cs.ibank.ru>

# $Id: ndcedit.awk,v 1.4 1997/02/22 16:08:19 peter Exp $
NR == 3 {
	print "#"
	print "# This file is generated automatically, do not edit it here!"
	print "# Please change src/usr.sbin/ndc/ndcedit.awk instead"
	print "#"
	print ""

	print "# If there is a global system configuration file, suck it in."
	print "if [ -f /etc/rc.conf ]; then"
	print "\t. /etc/rc.conf"
	print "fi\n"
}
{
	if ($1 == "named") {
		printf "\t\t# $namedflags is imported from /etc/rc.conf\n"
		printf "\t\tif [ \"X${named_flags}\" != X\"NO\" ]; then\n"
		printf "\t\t\tnamed ${named_flags} && {\n"
		getline
		printf "\t%s\n", $0
		getline
		printf "\t%s\n", $0
		getline
		printf "\t%s\n", $0
		printf "\t\tfi\n"
	} else {
	        if (/PATH=/) {
			gsub(":/usr/ucb:", ":", $0);
		      	if (!/export/) {
				$0=$0"\nexport PATH";
			}
		}
		print;
	}
}

# $Id$
NR == 3 {
	print "#"
	print "# This file is generated automatically, do not edit it here!"
	print "# Please change src/usr.sbin/ndc/ndcedit.awk instead"
	print "#"
	print ""

	print "# If there is a global system configuration file, suck it in."
	print "if [ -f /etc/sysconfig ]; then"
	print "\t. /etc/sysconfig"
	print "fi\n"
}
{
	if ($1 == "named") {
		printf "\t\t# $namedflags is imported from /etc/sysconfig\n"
		printf "\t\tif [ \"X${namedflags}\" != \"XNO\" ]; then\n"
		printf "\t\t\tnamed ${namedflags} && {\n"
		getline
		printf "\t%s\n", $0
		getline
		printf "\t%s\n", $0
		getline
		printf "\t%s\n", $0
		printf "\t\tfi\n"
	} else {
	        if (/PATH=/) {
			gsub(":/usr/ucb:", ":", $0);
		      	if (!/export/) {
				$0=$0"\nexport PATH";
			}
		}
		print;
	}
}


Fix .rc processing issue on FreeBSD

Fix .rc processing issue on FreeBSD

This change fixes issue that occurs on FreeBSD when compiling the
.rc files into the gettext format. For some reason, the shell
expression evaluation returns the resulting resource id as a string
datatype. That leads to a failure when trying to format the result
as a hex number using sprintf.
The change ensures that if that happens, the type is changed to numeric.

# Parse string resources from Windows native resource file
# and pass them to the writestringentry function that
# is responsible for writing the resource id and string
# to a platform specific resource file.
# A script containing this function needs to be specified
# using the -f command line parameter before this script.

BEGIN {
    inStringTable = 0;
    inBeginEnd = 0;
    writeheader();
}
{
    if ($1 == "STRINGTABLE" && $2 == "DISCARDABLE")
    {
        inStringTable = 1;
    }
    else if ($1 == "BEGIN")
    {
        inBeginEnd = inStringTable;
    }
    else if (inBeginEnd && $1 == "END")
    {
        inBeginEnd = 0;
        inStringTable = 0;
    }
    else if (inBeginEnd && $1 != "")
    {
        # combine all items until the first string and remove them
        # from the line
        i = 1
        expression = ""
        # string starts with either a quote or L followed by a quote
        while (substr($i, 1, 1) != "\"" && substr($i, 1, 2) != "L\"")
        {
            # some of the resource ids contain cast to HRESULT
            gsub(/\(HRESULT\)/, "", $i);
            # some of the resource ids have trailing L
            gsub(/L/, "", $i);
            expression = expression $i;
            $i="";
            i++;
        }
        # evaluate the resource id expression and format it as hex number
        cmd = "echo $(("expression"))";
        cmd | getline var;
        close(cmd);
        # in case shell returned the result as a string, ensure the var has numeric type
        var = var + 0;
        # sprintf can only handle signed ints, so we need to convert
        # values >= 0x80000000 to negative values
        if (var >= 2147483648)
        {
            var = var - 4294967296;
        }
        var = sprintf("%X", var);
        # remove the L prefix from strings
        gsub(/L"/, "\"", $0);
        # join strings "..." "..." into one
        gsub(/" +"/, "", $0);
        # remove all terminating newlines from the string - the msgfmt fails on those
        # since it expects them to be at the end of the msgid as well
        while(gsub(/\\n"/, "\"", $0)) {}

        # write the resource entry to the target file
        writestringentry(var, $0);
    }
}
END {
    writefooter();
}

# Parse string resources from Windows native resource file
# and pass them to the writestringentry function that
# is responsible for writing the resource id and string
# to a platform specific resource file.
# A script containing this function needs to be specified
# using the -f command line parameter before this script.

BEGIN {
    inStringTable = 0;
    inBeginEnd = 0;
    writeheader();
}
{
    if ($1 == "STRINGTABLE" && $2 == "DISCARDABLE")
    {
        inStringTable = 1;
    }
    else if ($1 == "BEGIN")
    {
        inBeginEnd = inStringTable;
    }
    else if (inBeginEnd && $1 == "END")
    {
        inBeginEnd = 0;
        inStringTable = 0;
    }
    else if (inBeginEnd && $1 != "")
    {
        # combine all items until the first string and remove them
        # from the line
        i = 1
        expression = ""
        # string starts with either a quote or L followed by a quote
        while (substr($i, 1, 1) != "\"" && substr($i, 1, 2) != "L\"")
        {
            # some of the resource ids contain cast to HRESULT
            gsub(/\(HRESULT\)/, "", $i);
            # some of the resource ids have trailing L
            gsub(/L/, "", $i);
            expression = expression $i;
            $i="";
            i++;
        }
        # evaluate the resource id expression and format it as hex number
        cmd = "echo $(("expression"))";
        cmd | getline var;
        close(cmd);
        # sprintf can only handle signed ints, so we need to convert
        # values >= 0x80000000 to negative values
        if (var >= 2147483648)
        {
            var = var - 4294967296;
        }
        var = sprintf("%X", var);
        # remove the L prefix from strings
        gsub(/L"/, "\"", $0);
        # join strings "..." "..." into one
        gsub(/" +"/, "", $0);
        # remove all terminating newlines from the string - the msgfmt fails on those
        # since it expects them to be at the end of the msgid as well
        while(gsub(/\\n"/, "\"", $0)) {}

        # write the resource entry to the target file
        writestringentry(var, $0);
    }
}
END {
    writefooter();
}


Print info on changed titles

Print info on changed titles

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

# INVOCATION
#   awk -v ERROR_FILE=$ERROR_FILE -f getBBoxCatalogFrom-sitemap.awk $SITEMAP_FILE >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title           3 Seasons          4 Episodes      5 Duration     6 Year
#     7 Rating        8 Description     9 Content Type    10 Content ID   11 Entity ID   12 Genre
#    13 Show Type    14 Date Type      15 Original Date   16 Show ID      16 Season ID   13 Sn #   14 Ep #

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear\tRating\tDescription\t")
    printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    printf ("Show ID\tSeason ID\tSn #\tEp #\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType
    # print "contentId = " contentId
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = 111
    numEpisodes = 222
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType
    # print "originalDate = " originalDate
    # print "year = " year
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "==> EntityId = " EntityId
    }

    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERROR_FILE
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERROR_FILE
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERROR_FILE
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "==> EntityId = " EntityId
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) S%s", title, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) S%02d", showTitle, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, \
                episodeNumber, showContentId)
    }

    # Copied from printing header to make it easier to coordinate arranging fields
    # printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    # printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    # printf ("Show ID\tSeason ID\tSn #\tEp #\n")

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
            sortkey, title, numSeasons, numEpisodes, duration, year, rating, description, \
            contentType, contentId, EntityId, genre, showType, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}

END {
    printf ("In getBBoxCatalogFrom-sitemap.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
}

# Broduce a raw data spreadsheet of fields in BritBox Catalog --  apple_catalogue_feed.xml

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    printf ("Show ID\tSeason ID\tSn #\tEp #\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType
    # print "contentId = " contentId
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    years = ""
    # New fields that haven't been created yet
    numSeasons = 111
    numEpisodes = 222
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    # print "genre = " genre
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    duration = fld[3]
    # print "duration = " duration
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    years = substr (originalDate,1,4)
    # print "dateType = " dateType
    # print "originalDate = " originalDate
    # print "years = " years
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId
    next
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    if (title == "Shallow Grave") {
        EntityId = "_23576"
        # print "==> EntityId = " EntityId
    }

    if (title == "Porridge") {
        if (EntityId == "_9509")
            title = "Porridge (1974-1977)"
        else if (EntityId == "_14747")
            title = "Porridge (2016-2017)"
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179")
            title = "A Midsummer Night's Dream (1981)"
        else if (EntityId == "_15999")
            title = "A Midsummer Night's Dream (2016)"
        # print "==> title = " title
        # print "==> EntityId = " EntityId
    }

    if (contentId == "p07gnw9f") {
        EntityId = "_23842"
        # print "==> EntityId = " EntityId
    }

    if (contentType == "movie") {
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
    }

    if (contentType == "tv_show") {
        showTitle = title
        sortkey = sprintf ("%s (1) S%s", title, EntityId)
    }

    if (contentType == "tv_season") {
        sortkey = sprintf ("%s (2) S%02d", showTitle, seasonNumber)
    }

    if (contentType == "tv_episode") {
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, \
                episodeNumber, showContentId)
    }

    # Copied from printing header to make it easier to coordinate arranging fields
    # printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription\t")
    # printf ("Content Type\tContent ID\tEntity ID\tGenre\tShow Type\tDate Type\tOriginal Date\t")
    # printf ("Show ID\tSeason ID\tSn #\tEp #\n")

    printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
            sortkey, title, numSeasons, numEpisodes, duration, years, rating, description, \
            contentType, contentId, EntityId, genre, showType, dateType, originalDate, \
            showContentId, seasonContentId, seasonNumber, episodeNumber)
}

# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
# https://www.britbox.com/us/movie/300_Years_of_French_and_Saunders_16103
/<item contentType="movie"/,/<\/item>/ {
    # print
}