build.awk: remove findIncludes()

build.awk: remove findIncludes()

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/Languages.awk"
@include "metainfo.awk"

function init() {
    EntryScript          = "translate"

    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, num, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = length(group) == 1 ? group[1] "_language" : join(group, "_")
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    #print "***" length(Locale) "*** *languages in total. "
    print "*Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "*Version: [English](https://github.com/soimort/translate-shell/wiki/Languages) " \
        "| [Chinese Simplified](https://github.com/soimort/translate-shell/wiki/Languages-%28%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87%29)*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = length(group) == 1 ? group[1] "_language" :
            group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language, getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function readSqueezed(fileName, squeezed,    group, line, ret) {
    if (fileName ~ /\*$/) # glob simulation
        return readSqueezed(fileName ".awk", squeezed)

    ret = NULLSTR
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (group[1] ~ /\.awk$/)
                    append(Includes, group[1])

                if (ret) ret = ret RS
                ret = ret readSqueezed(group[1], squeezed)
            } else if (!squeezed || line = squeeze(line)) { # effective LOC
                if (ret) ret = ret RS
                ret = ret line
            }
        }
    return ret
}

function build(target, type,    i, group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans

        if (fileExists("DISCLAIMER")) {
            print "#" > Trans
            while (getline line < "DISCLAIMER")
                print "# " line > Trans
            print "#" > Trans
        }

        print "export TRANS_ENTRY=\"$0\"" > Trans
        print "if [[ ! $LANG =~ (UTF|utf)-?8$ ]]; then export LANG=en_US.UTF-8; fi" > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        print readSqueezed(EntryPoint, TRUE) > Trans
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline

        # Rebuild EntryScript
        print "#!/bin/sh" > EntryScript
        print "TRANS_DIR=`dirname $0`" > EntryScript
        print "gawk \\" > EntryScript
        for (i = 0; i < length(Includes) - 1; i++)
            print "-i \"${TRANS_DIR}/" Includes[i] "\" \\" > EntryScript
        print "-f \"${TRANS_DIR}/" Includes[i] "\" -- \"$@\"" > EntryScript
        ("chmod +x " parameterize(EntryScript)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp
        print (temp == "Darwin" ?
               "#!/usr/bin/env gawk -f" : # macOS
               "#!/usr/bin/gawk -f") > TransAwk

        print readSqueezed(EntryPoint, TRUE) > TransAwk

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/Languages.awk"
@include "metainfo.awk"

function init() {
    EntryScript          = "translate"

    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, num, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = length(group) == 1 ? group[1] "_language" : join(group, "_")
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    #print "***" length(Locale) "*** *languages in total. "
    print "*Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "*Version: [English](https://github.com/soimort/translate-shell/wiki/Languages) " \
        "| [Chinese Simplified](https://github.com/soimort/translate-shell/wiki/Languages-%28%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87%29)*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = length(group) == 1 ? group[1] "_language" :
            group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language, getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function readSqueezed(fileName, squeezed,    group, line, ret) {
    if (fileName ~ /\*$/) # glob simulation
        return readSqueezed(fileName ".awk", squeezed)

    ret = NULLSTR
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (group[1] ~ /\.awk$/)
                    append(Includes, group[1])

                if (ret) ret = ret RS
                ret = ret readSqueezed(group[1], squeezed)
            } else if (!squeezed || line = squeeze(line)) { # effective LOC
                if (ret) ret = ret RS
                ret = ret line
            }
        }
    return ret
}

function findIncludes(fileName,    line) {
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (group[1] ~ /\*$/)
                    findIncludes(group[1] ".awk")
                else
                    append(Includes, group[1])
            }
        }
}

function build(target, type,    i, group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans

        if (fileExists("DISCLAIMER")) {
            print "#" > Trans
            while (getline line < "DISCLAIMER")
                print "# " line > Trans
            print "#" > Trans
        }

        print "export TRANS_ENTRY=\"$0\"" > Trans
        print "if [[ ! $LANG =~ (UTF|utf)-?8$ ]]; then export LANG=en_US.UTF-8; fi" > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        print readSqueezed(EntryPoint, TRUE) > Trans
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline

        # Rebuild EntryScript
        print "#!/bin/sh" > EntryScript
        print "TRANS_DIR=`dirname $0`" > EntryScript
        print "gawk \\" > EntryScript
        for (i = 0; i < length(Includes) - 1; i++)
            print "-i \"${TRANS_DIR}/" Includes[i] "\" \\" > EntryScript
        print "-f \"${TRANS_DIR}/" Includes[i] "\" -- \"$@\"" > EntryScript
        ("chmod +x " parameterize(EntryScript)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp
        print (temp == "Darwin" ?
               "#!/usr/bin/env gawk -f" : # macOS
               "#!/usr/bin/gawk -f") > TransAwk

        print readSqueezed(EntryPoint, TRUE) > TransAwk

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}


Set missing Christmas Specials episodeType to "X"

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Empire | Acorn TV</title>
#  ---
#        Empire
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ \| Acorn TV<.*/,"")
    gsub (/&/,"\\&")
    gsub (/"/,"\"\"")
    gsub (/'/,"'")
    gsub (/’/,"'")
    # gsub (/'/,"'")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    split ($0,fld,"\"")
    episodeLinesFound += 1
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if ((numSeasons + 0) == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/"/,"\"\"",description)
    gsub (/'/,"'",description)
    gsub (/'/,"'",description)
    gsub (/’/,"'",description)
    # fix unmatched quotes
    numQuotes = gsub(/"/,"\"",description)
    if ((numQuotes % 2) == 1) {
        printf ("==> Corrected unmatched quote (%d): %d\t%s\n", numQuotes, SERIES_NUMBER, \
                seriesTitle) >> ERROR_FILE
        description = description " \""
    }
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %s  %s\n", shortURL, seriesTitle) >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    gsub (/'/,"'",episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default bonus, parttwo, and christmas specials episodeType to "X"
    if (episodeURL ~ /\/bonus|[0-9]{1,2}parttwo\/|christmas[-]?special/)
        episodeType = "X"
    # If episode is a Trailer, set episodeType to "T"
    if (episodeURL ~ /\/trailer/)
        episodeType = "T"
    # jamaicainn, newworlds & newtonslaw bonus or trailer seasonNumber should be 1
    if (episodeURL ~ \
        /\/jamaicainn\/bonus\/|\/jamaicainn\/trailer\/|\/newworlds\/bonus\/|\/newtonslaw\/bonus\// \
        && seasonNumber != 1) {
        split (episodeURL, part, "/")
        printf ("==> Corrected S%02d to S01: acorn.tv/%s/%s\n", \
               seasonNumber, part[4], part[5]) >> ERROR_FILE
        seasonNumber = 1
    }
    # Plain christmasspecial, seriesfinale don't increment seasonNumber
    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
        split (episodeURL, part, "/")
        printf ("==> Corrected S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, seasonNumber-1, part[4], part[5]) >> ERROR_FILE
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, Vera, and others have problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|[0-9]{1,2}bonus\/|[0-9]{1,2}parttwo\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/bonus/,"",URLseasonNumber)
        # Foyle's War has sets: foyleswar/series9set8bonus/
        sub (/set[0-9]/,"",URLseasonNumber)
        # Midsomer Murders has two parts: midsomermurders/series19parttwo/
        sub (/parttwo/,"",URLseasonNumber)
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Corrected S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, URLseasonNumber, part[4], part[5]) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }

    if ((episodeNumber + 0) == 0)
        print "==> Episode number is 00: " shortURL >> ERROR_FILE
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

/-- Viewers Also Watched --/ {
    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    if (description == "")
        printf ("==> No description: %d\t%s  %s\n", SERIES_NUMBER, shortURL, seriesTitle) >> ERROR_FILE
    description = ""
    #
    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
    print seriesDuration >> DURATION_FILE
    if (seriesDuration == "00:00:00")
        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, shortURL, seriesTitle) >> ERROR_FILE
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Empire | Acorn TV</title>
#  ---
#        Empire
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ \| Acorn TV<.*/,"")
    gsub (/&/,"\\&")
    gsub (/"/,"\"\"")
    gsub (/'/,"'")
    gsub (/’/,"'")
    # gsub (/'/,"'")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    split ($0,fld,"\"")
    episodeLinesFound += 1
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if ((numSeasons + 0) == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/"/,"\"\"",description)
    gsub (/'/,"'",description)
    gsub (/'/,"'",description)
    gsub (/’/,"'",description)
    # fix unmatched quotes
    numQuotes = gsub(/"/,"\"",description)
    if ((numQuotes % 2) == 1) {
        printf ("==> Corrected unmatched quote (%d): %d\t%s\n", numQuotes, SERIES_NUMBER, \
                seriesTitle) >> ERROR_FILE
        description = description " \""
    }
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %s  %s\n", shortURL, seriesTitle) >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    gsub (/'/,"'",episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default bonus, parttwo, and christmasspecials episodeType to "X"
    if (episodeURL ~ /bonus\/|[0-9]{1,2}parttwo\/|christmasspecial\//)
        episodeType = "X"
    # If episode is a Trailer, set episodeType to "T"
    if (episodeURL ~ /\/trailer/)
        episodeType = "T"
    # jamaicainn, newworlds & newtonslaw bonus or trailer seasonNumber should be 1
    if (episodeURL ~ \
        /\/jamaicainn\/bonus\/|\/jamaicainn\/trailer\/|\/newworlds\/bonus\/|\/newtonslaw\/bonus\// \
        && seasonNumber != 1) {
        split (episodeURL, part, "/")
        printf ("==> Corrected S%02d to S01: acorn.tv/%s/%s\n", \
               seasonNumber, part[4], part[5]) >> ERROR_FILE
        seasonNumber = 1
    }
    # Plain christmasspecial, seriesfinale don't increment seasonNumber
    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
        split (episodeURL, part, "/")
        printf ("==> Corrected S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, seasonNumber-1, part[4], part[5]) >> ERROR_FILE
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, Vera, and others have problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|[0-9]{1,2}bonus\/|[0-9]{1,2}parttwo\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/bonus/,"",URLseasonNumber)
        # Foyle's War has sets: foyleswar/series9set8bonus/
        sub (/set[0-9]/,"",URLseasonNumber)
        # Midsomer Murders has two parts: midsomermurders/series19parttwo/
        sub (/parttwo/,"",URLseasonNumber)
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Corrected S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, URLseasonNumber, part[4], part[5]) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }

    if ((episodeNumber + 0) == 0)
        print "==> Episode number is 00: " shortURL >> ERROR_FILE
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

/-- Viewers Also Watched --/ {
    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    if (description == "")
        printf ("==> No description: %d\t%s  %s\n", SERIES_NUMBER, shortURL, seriesTitle) >> ERROR_FILE
    description = ""
    #
    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
    print seriesDuration >> DURATION_FILE
    if (seriesDuration == "00:00:00")
        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, shortURL, seriesTitle) >> ERROR_FILE
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}


Fix for changes in Acorn page layout

Fix for changes in Acorn page layout

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Empire | Acorn TV</title>
#  ---
#        Empire
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/span itemprop="name"/ {
    split ($0,fld,"[<>]")
    seriesTitle = fld[3]
    gsub (/&/,"\\&",seriesTitle)
    gsub (/"/,"\"\"",seriesTitle)
    gsub (/"/,"\"\"",seriesTitle)
    gsub (/'/,"'",seriesTitle)
    gsub (/’/,"'",seriesTitle)
    gsub (/–/,"-",seriesTitle)
    # gsub (/'/,"'",seriesTitle)
    if (match (seriesTitle, /^The /)) {
        seriesTitle = substr(seriesTitle, 5) ", The"
    }
    episodeLinesFound = 0
    seasonLinesFound = 0
    descriptionLinesFound = 0
    durationLinesFound = 0
    numEpisodesStr = ""
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    episodeLinesFound += 1
    split ($0,fld,"\"")
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    seasonLinesFound += 1
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if ((numSeasons + 0) == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    descriptionLinesFound += 1
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/"/,"\"\"",description)
    gsub (/'/,"'",description)
    gsub (/'/,"'",description)
    gsub (/’/,"'",description)
    # fix unmatched quotes
    numQuotes = gsub(/"/,"\"",description)
    if ((numQuotes % 2) == 1) {
        printf ("==> Changed unmatched quote (%d): %d\t%s\n", numQuotes, SERIES_NUMBER, \
                seriesTitle) >> ERROR_FILE
        description = description " \""
    }
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    durationLinesFound += 1
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %s  %s\n", shortURL, seriesTitle) >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    gsub (/'/,"'",episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default bonus, parttwo, and christmas specials episodeType to "X"
    if (episodeURL ~ /\/bonus|[0-9]{1,2}parttwo\/|christmas[-]?special/)
        episodeType = "X"
    # If episode is a Trailer, set episodeType to "T"
    if (episodeURL ~ /\/trailer/)
        episodeType = "T"
    # jamaicainn, newworlds & newtonslaw bonus or trailer seasonNumber should be 1
    if (episodeURL ~ \
        /\/jamaicainn\/bonus\/|\/jamaicainn\/trailer\/|\/newworlds\/bonus\/|\/newtonslaw\/bonus\// \
        && seasonNumber != 1) {
        split (episodeURL, part, "/")
        printf ("==> Changed S%02d to S01: acorn.tv/%s/%s\n", \
               seasonNumber, part[4], part[5]) >> ERROR_FILE
        seasonNumber = 1
    }
    # Plain christmasspecial, seriesfinale don't increment seasonNumber
    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
        split (episodeURL, part, "/")
        printf ("==> Changed S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, seasonNumber-1, part[4], part[5]) >> ERROR_FILE
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, Vera, and others have problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|[0-9]{1,2}bonus\/|[0-9]{1,2}parttwo\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/bonus/,"",URLseasonNumber)
        # Foyle's War has sets: foyleswar/series9set8bonus/
        sub (/set[0-9]/,"",URLseasonNumber)
        # Midsomer Murders has two parts: midsomermurders/series19parttwo/
        sub (/parttwo/,"",URLseasonNumber)
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Changed S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, URLseasonNumber, part[4], part[5]) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }

    if ((episodeNumber + 0) == 0)
        print "==> Episode number is 00: " shortURL >> ERROR_FILE
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

// {
    if (episodeLinesFound == 0) {
        print 0  >> NUM_EPISODES_FILE
        printf ("==> No numberOfEpisodes: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    } else {
        print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    }
    #
    if (seasonLinesFound == 0) {
        print seasonLinesFound >> NUM_SEASONS_FILE
        printf ("==> No numberOfSeasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    }
    #
    if (descriptionLinesFound == 0) {
        print "-- No Description --" >> DESCRIPTION_FILE
        printf ("==> No franchise-description: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    } else {
        if (description == "")
            printf ("==> No description: %d\t%s  %s\n", SERIES_NUMBER, shortURL, \
                    seriesTitle) >> ERROR_FILE
    }
    description = ""
    #
    if (durationLinesFound == 0) {
        print "00:00:00"  >> DURATION_FILE
        printf ("==> No timeRequired: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    } else {
        seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
        print seriesDuration >> DURATION_FILE
        if (seriesDuration == "00:00:00")
            printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, shortURL, i\
                    seriesTitle) >> ERROR_FILE
   }
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Empire | Acorn TV</title>
#  ---
#        Empire
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>/,"")
    sub (/ \| Acorn TV<.*/,"")
    gsub (/&/,"\\&")
    gsub (/"/,"\"\"")
    gsub (/'/,"'")
    gsub (/’/,"'")
    gsub (/–/,"-")
    # gsub (/'/,"'")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    seasonLinesFound = 0
    descriptionLinesFound = 0
    durationLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    episodeLinesFound += 1
    split ($0,fld,"\"")
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    seasonLinesFound += 1
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if ((numSeasons + 0) == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    descriptionLinesFound += 1
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/"/,"\"\"",description)
    gsub (/'/,"'",description)
    gsub (/'/,"'",description)
    gsub (/’/,"'",description)
    # fix unmatched quotes
    numQuotes = gsub(/"/,"\"",description)
    if ((numQuotes % 2) == 1) {
        printf ("==> Changed unmatched quote (%d): %d\t%s\n", numQuotes, SERIES_NUMBER, \
                seriesTitle) >> ERROR_FILE
        description = description " \""
    }
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    durationLinesFound += 1
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %s  %s\n", shortURL, seriesTitle) >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    gsub (/'/,"'",episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default bonus, parttwo, and christmas specials episodeType to "X"
    if (episodeURL ~ /\/bonus|[0-9]{1,2}parttwo\/|christmas[-]?special/)
        episodeType = "X"
    # If episode is a Trailer, set episodeType to "T"
    if (episodeURL ~ /\/trailer/)
        episodeType = "T"
    # jamaicainn, newworlds & newtonslaw bonus or trailer seasonNumber should be 1
    if (episodeURL ~ \
        /\/jamaicainn\/bonus\/|\/jamaicainn\/trailer\/|\/newworlds\/bonus\/|\/newtonslaw\/bonus\// \
        && seasonNumber != 1) {
        split (episodeURL, part, "/")
        printf ("==> Changed S%02d to S01: acorn.tv/%s/%s\n", \
               seasonNumber, part[4], part[5]) >> ERROR_FILE
        seasonNumber = 1
    }
    # Plain christmasspecial, seriesfinale don't increment seasonNumber
    if (episodeURL ~ /\/christmasspecial\/|\/seriesfinale\//) {
        split (episodeURL, part, "/")
        printf ("==> Changed S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, seasonNumber-1, part[4], part[5]) >> ERROR_FILE
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, Vera, and others have problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|[0-9]{1,2}bonus\/|[0-9]{1,2}parttwo\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/bonus/,"",URLseasonNumber)
        # Foyle's War has sets: foyleswar/series9set8bonus/
        sub (/set[0-9]/,"",URLseasonNumber)
        # Midsomer Murders has two parts: midsomermurders/series19parttwo/
        sub (/parttwo/,"",URLseasonNumber)
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Changed S%02d to S%02d: acorn.tv/%s/%s\n", \
               seasonNumber, URLseasonNumber, part[4], part[5]) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }

    if ((episodeNumber + 0) == 0)
        print "==> Episode number is 00: " shortURL >> ERROR_FILE
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

// {
    if (episodeLinesFound == 0) {
        print 0  >> NUM_EPISODES_FILE
        printf ("==> No numberOfEpisodes: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    } else {
        print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    }
    #
    if (seasonLinesFound == 0) {
        print seasonLinesFound >> NUM_SEASONS_FILE
        printf ("==> No numberOfSeasons: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    }
    #
    if (descriptionLinesFound == 0) {
        print "-- No Description --" >> DESCRIPTION_FILE
        printf ("==> No franchise-description: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    } else {
        if (description == "")
            printf ("==> No description: %d\t%s  %s\n", SERIES_NUMBER, shortURL, \
                    seriesTitle) >> ERROR_FILE
    }
    description = ""
    #
    if (durationLinesFound == 0) {
        print "00:00:00"  >> DURATION_FILE
        printf ("==> No timeRequired: %d\t%s\n", SERIES_NUMBER, seriesTitle) >> ERROR_FILE
    } else {
        seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
        print seriesDuration >> DURATION_FILE
        if (seriesDuration == "00:00:00")
            printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, shortURL, i\
                    seriesTitle) >> ERROR_FILE
   }
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}


Remove some unused and broken code that attempted to not invoke locking asserts on NULL vnode pointers.  All the vnode assertion routines already check for NULL vnode pointers.


+docs

+docs

FastQ.toFastA.awk has now help message.

#!/usr/bin/env awk -f
#
# @author  Luis M. Rodriguez-R
# @update  Dec-26-2015
# @license artistic license 2.0
#

BEGIN {
  for (i = 0; i < ARGC; i++) {
     if(ARGV[i] == "--help"){
       print "Description:\n"
       print "  Translates FastQ files into FastA.\n"
       print "Usage:\n"
       print "  FastQ.toFastA.awk < in.fq > out.fa\n"
       exit
     }
  }
}

NR%4 == 1, NR%4 == 2 {
   if(NR%4 == 1){ gsub(/^@/,">") }
   print $0
}

#!/usr/bin/env awk -f
#
# @author: Luis M. Rodriguez-R
# @update: May-10-2015
# @license: artistic license 2.0
#

NR%4 == 1, NR%4 == 2 {
   if(NR%4 == 1){ gsub(/^@/,">") }
   print $0
}


Use functions to make the program clearer.

#! /usr/bin/awk -f
#	$NetBSD: usb/devlist2h.awk,v 1.9 2001/01/18 20:28:22 jdolecek Exp $
#  $FreeBSD$
#
# Copyright (c) 1995, 1996 Christopher G. Demetriou
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Christopher G. Demetriou.
# 4. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

function usage()
{
	print "usage: usbdevs2h.awk <srcfile> [-d|-h]";
	exit 1;
}

function dheader(dfile)
{
	if (os == "NetBSD")
		printf("/*\t\$NetBSD\$\t*/\n\n") > dfile
	else if (os == "FreeBSD")
		printf("/* \$FreeBSD\$ */\n\n") > dfile
	else if (os == "OpenBSD")
		printf("/*\t\$OpenBSD\$\t*/\n\n") > dfile
	else
		printf("/* ??? */\n\n") > dfile
	printf("/*\n") > dfile
	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > dfile
	printf(" *\n") > dfile
	printf(" * generated from:\n") > dfile
	printf(" *\t%s\n", VERSION) > dfile
	printf(" */\n") > dfile
}

function hheader(hfile)
{
	if (os == "NetBSD")
		printf("/*\t\$NetBSD\$\t*/\n\n") > hfile
	else if (os == "FreeBSD")
		printf("/* \$FreeBSD\$ */\n\n") > hfile
	else if (os == "OpenBSD")
		printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
	else
		printf("/* ??? */\n\n") > hfile
	printf("/*\n") > hfile
	printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
	    > hfile
	printf(" *\n") > hfile
	printf(" * generated from:\n") > hfile
	printf(" *\t%s\n", VERSION) > hfile
	printf(" */\n") > hfile
}

function vendor(hfile)
{
	nvendors++

	vendorindex[$2] = nvendors;		# record index for this name, for later.
	vendors[nvendors, 1] = $2;		# name
	vendors[nvendors, 2] = $3;		# id
	if (hfile)
		printf("#define\tUSB_VENDOR_%s\t%s\t", vendors[nvendors, 1],
		    vendors[nvendors, 2]) > hfile
	i = 3; f = 4;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		if (hfile)
			printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			if (hfile)
				printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			if (hfile)
				printf("%s", $f) > hfile
			if (f < NF && hfile)
				printf(" ") > hfile
			f++
			continue
		}
		vendors[nvendors, i] = $f
		if (hfile)
			printf("%s", vendors[nvendors, i]) > hfile
		if (f < NF && hfile)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen && hfile)
		printf(")") > hfile
	if (ocomment && hfile)
		printf(" */") > hfile
	if (hfile)
		printf("\n") > hfile
}

function product(hfile)
{
	nproducts++

	products[nproducts, 1] = $2;		# vendor name
	products[nproducts, 2] = $3;		# product id
	products[nproducts, 3] = $4;		# id
	if (hfile)
		printf("#define\tUSB_PRODUCT_%s_%s\t%s\t", \
		  products[nproducts, 1], products[nproducts, 2], \
		  products[nproducts, 3]) > hfile

	i=4; f = 5;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		if (hfile)
			printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			if (hfile)
				printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			if (hfile)
				printf("%s", $f) > hfile
			if (f < NF && hfile)
				printf(" ") > hfile
			f++
			continue
		}
		products[nproducts, i] = $f
		if (hfile)
			printf("%s", products[nproducts, i]) > hfile
		if (f < NF && hfile)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen && hfile)
		printf(")") > hfile
	if (ocomment && hfile)
		printf(" */") > hfile
	if (hfile)
		printf("\n") > hfile
}

function dump_dfile(dfile)
{
	printf("\n") > dfile
	printf("const struct usb_knowndev usb_knowndevs[] = {\n") > dfile
	for (i = 1; i <= nproducts; i++) {
		printf("\t{\n") > dfile
		printf("\t    USB_VENDOR_%s, USB_PRODUCT_%s_%s,\n",
		    products[i, 1], products[i, 1], products[i, 2]) > dfile
		printf("\t    ") > dfile
		printf("0") > dfile
		printf(",\n") > dfile

		vendi = vendorindex[products[i, 1]];
		printf("\t    \"") > dfile
		j = 3;
		needspace = 0;
		while (vendors[vendi, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", vendors[vendi, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile

		printf("\t    \"") > dfile
		j = 4;
		needspace = 0;
		while (products[i, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", products[i, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile
		printf("\t},\n") > dfile
	}
	for (i = 1; i <= nvendors; i++) {
		printf("\t{\n") > dfile
		printf("\t    USB_VENDOR_%s, 0,\n", vendors[i, 1]) > dfile
		printf("\t    USB_KNOWNDEV_NOPROD,\n") > dfile
		printf("\t    \"") > dfile
		j = 3;
		needspace = 0;
		while (vendors[i, j] != "") {
			if (needspace)
				printf(" ") > dfile
			printf("%s", vendors[i, j]) > dfile
			needspace = 1
			j++
		}
		printf("\",\n") > dfile
		printf("\t    NULL,\n") > dfile
		printf("\t},\n") > dfile
	}
	printf("\t{ 0, 0, 0, NULL, NULL, }\n") > dfile
	printf("};\n") > dfile
}

BEGIN {

nproducts = nvendors = 0
# Process the command line
for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[dh]+$/ && arg !~ /devs$/)
		usage();
	if (arg ~ /^-.*d/)
		dfile="usbdevs_data.h"
	if (arg ~ /^-.*h/)
		hfile="usbdevs.h"
	if (arg ~ /devs$/)
		srcfile = arg;
}
ARGC = 1;
line=0;

while ((getline < srcfile) > 0) {
	line++;
	if (line == 1) {
		VERSION = $0
		gsub("\\$", "", VERSION)

		if (dfile)
			dheader(dfile)

		if (hfile)
			hheader(hfile)
		continue;
	}
	if ($1 == "vendor") {
		vendor(hfile)
		continue
	}
	if ($1 == "product") {
		product(hfile)
		continue
	}
	if ($0 == "")
		blanklines++
	if (hfile)
		print $0 > hfile
	if (blanklines < 2 && dfile)
	    print $0 > dfile
}

# print out the match tables

if (dfile)
	dump_dfile(dfile)
}

#! /usr/bin/awk -f
#	$NetBSD: usb/devlist2h.awk,v 1.9 2001/01/18 20:28:22 jdolecek Exp $
#  $FreeBSD$
#
# Copyright (c) 1995, 1996 Christopher G. Demetriou
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Christopher G. Demetriou.
# 4. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
function usage()
{
	print "usage: usbdevs2h.awk <srcfile> [-d|-h]";
	exit 1;
}

BEGIN {

nproducts = nvendors = 0
# Process the command line
for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[dh]+$/ && arg !~ /devs$/)
		usage();
	if (arg ~ /^-.*d/)
		dfile="usbdevs_data.h"
	if (arg ~ /^-.*h/)
		hfile="usbdevs.h"
	if (arg ~ /devs$/)
		srcfile = arg;
}
ARGC = 1;
line=0;

while ((getline < srcfile) > 0) {
    line++;
    if (line == 1) {
	VERSION = $0
	gsub("\\$", "", VERSION)

	if (dfile) {
		if (os == "NetBSD")
			printf("/*\t\$NetBSD\$\t*/\n\n") > dfile
		else if (os == "FreeBSD")
			printf("/* \$FreeBSD\$ */\n\n") > dfile
		else if (os == "OpenBSD")
			printf("/*\t\$OpenBSD\$\t*/\n\n") > dfile
		else
			printf("/* ??? */\n\n") > dfile
		printf("/*\n") > dfile
		printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
		    > dfile
		printf(" *\n") > dfile
		printf(" * generated from:\n") > dfile
		printf(" *\t%s\n", VERSION) > dfile
		printf(" */\n") > dfile
	}

	if (hfile) {
		if (os == "NetBSD")
			printf("/*\t\$NetBSD\$\t*/\n\n") > hfile
		else if (os == "FreeBSD")
			printf("/* \$FreeBSD\$ */\n\n") > hfile
		else if (os == "OpenBSD")
			printf("/*\t\$OpenBSD\$\t*/\n\n") > hfile
		else
			printf("/* ??? */\n\n") > hfile
		printf("/*\n") > hfile
		printf(" * THIS FILE IS AUTOMATICALLY GENERATED.  DO NOT EDIT.\n") \
		    > hfile
		printf(" *\n") > hfile
		printf(" * generated from:\n") > hfile
		printf(" *\t%s\n", VERSION) > hfile
		printf(" */\n") > hfile
	}
	continue;
    }
   if ($1 == "vendor") {
	nvendors++

	vendorindex[$2] = nvendors;		# record index for this name, for later.
	vendors[nvendors, 1] = $2;		# name
	vendors[nvendors, 2] = $3;		# id
	if (hfile)
		printf("#define\tUSB_VENDOR_%s\t%s\t", vendors[nvendors, 1],
		    vendors[nvendors, 2]) > hfile
	i = 3; f = 4;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		if (hfile)
			printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			if (hfile)
				printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			if (hfile)
				printf("%s", $f) > hfile
			if (f < NF && hfile)
				printf(" ") > hfile
			f++
			continue
		}
		vendors[nvendors, i] = $f
		if (hfile)
			printf("%s", vendors[nvendors, i]) > hfile
		if (f < NF && hfile)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen && hfile)
		printf(")") > hfile
	if (ocomment && hfile)
		printf(" */") > hfile
	if (hfile)
		printf("\n") > hfile

	continue;
    }
    if ($1 == "product") {
	nproducts++

	products[nproducts, 1] = $2;		# vendor name
	products[nproducts, 2] = $3;		# product id
	products[nproducts, 3] = $4;		# id
	if (hfile)
		printf("#define\tUSB_PRODUCT_%s_%s\t%s\t", \
		  products[nproducts, 1], products[nproducts, 2], \
		  products[nproducts, 3]) > hfile

	i=4; f = 5;

	# comments
	ocomment = oparen = 0
	if (f <= NF) {
		if (hfile)
			printf("\t/* ") > hfile
		ocomment = 1;
	}
	while (f <= NF) {
		if ($f == "#") {
			if (hfile)
				printf("(") > hfile
			oparen = 1
			f++
			continue
		}
		if (oparen) {
			if (hfile)
				printf("%s", $f) > hfile
			if (f < NF && hfile)
				printf(" ") > hfile
			f++
			continue
		}
		products[nproducts, i] = $f
		if (hfile)
			printf("%s", products[nproducts, i]) > hfile
		if (f < NF && hfile)
			printf(" ") > hfile
		i++; f++;
	}
	if (oparen && hfile)
		printf(")") > hfile
	if (ocomment && hfile)
		printf(" */") > hfile
	if (hfile)
		printf("\n") > hfile

	continue;
    }
	if ($0 == "")
		blanklines++
	if (hfile)
		print $0 > hfile
	if (blanklines < 2 && dfile)
	    print $0 > dfile
}
	# print out the match tables

	if (dfile) {
		printf("\n") > dfile

		printf("const struct usb_knowndev usb_knowndevs[] = {\n") > dfile
		for (i = 1; i <= nproducts; i++) {
			printf("\t{\n") > dfile
			printf("\t    USB_VENDOR_%s, USB_PRODUCT_%s_%s,\n",
			    products[i, 1], products[i, 1], products[i, 2]) \
			    > dfile
			printf("\t    ") > dfile
			printf("0") > dfile
			printf(",\n") > dfile

			vendi = vendorindex[products[i, 1]];
			printf("\t    \"") > dfile
			j = 3;
			needspace = 0;
			while (vendors[vendi, j] != "") {
				if (needspace)
					printf(" ") > dfile
				printf("%s", vendors[vendi, j]) > dfile
				needspace = 1
				j++
			}
			printf("\",\n") > dfile

			printf("\t    \"") > dfile
			j = 4;
			needspace = 0;
			while (products[i, j] != "") {
				if (needspace)
					printf(" ") > dfile
				printf("%s", products[i, j]) > dfile
				needspace = 1
				j++
			}
			printf("\",\n") > dfile
			printf("\t},\n") > dfile
		}
		for (i = 1; i <= nvendors; i++) {
			printf("\t{\n") > dfile
			printf("\t    USB_VENDOR_%s, 0,\n", vendors[i, 1]) \
			    > dfile
			printf("\t    USB_KNOWNDEV_NOPROD,\n") \
			    > dfile
			printf("\t    \"") > dfile
			j = 3;
			needspace = 0;
			while (vendors[i, j] != "") {
				if (needspace)
					printf(" ") > dfile
				printf("%s", vendors[i, j]) > dfile
				needspace = 1
				j++
			}
			printf("\",\n") > dfile
			printf("\t    NULL,\n") > dfile
			printf("\t},\n") > dfile
		}
		printf("\t{ 0, 0, 0, NULL, NULL, }\n") > dfile
		printf("};\n") > dfile
	}
}


close verbatim blocks more faithfully

close verbatim blocks more faithfully

BEGIN {
  print "%TCIDATA{Version=5.00.0.2606}";
  print "%TCIDATA{LaTeXparent=0,0,classes.tex}";
  print "                      ";
  print "";
  print "%%%%% BEGINNING OF DOCUMENT BODY %%%%%";
  print "% html: Beginning of file: `clean.html'";
  print "% DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\"";
  print "%  This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. ";
  print "";
  quoting = 0;
  itemizing = 0;
  coding = 0;
}

END {
  print "% html: End of file: `clean.html'";
  print "";
  print "%%%%% END OF DOCUMENT BODY %%%%%";
  print "% In the future, we might want to put some additional data here, such";
  print "% as when the documentation was converted from wiki to TeX.";
  print "%";
}

{ origLine = $0; }

## HANDLE CODING
## make sure that control sequences are always at the front
/ *{{{/ { sub(/ */, "") }
/ *}}}/ { sub(/ */, "") }

!coding && /^{{{/ {

  if (quoting) {
    print "\\end{quotation}";
    print ""
    quoting = 0;
  }

  print "\\begin{verbatim}";
  coding = 1;
  next;
}

coding && /^}}}/ {
  sub(/}}}.*$/, "", origLine);
  if (origLine) {
    print "{\\small " origLine
    print "}"
  }
  print "\\end{verbatim}";
  print ""
  coding = 0;
  next;
}

coding {
  sub(/^/, "{\\small ");
  print;
  print "}";
  next;
}

## NO CODING BELOW THIS POINT
coding { print "###code???####"; next; }

## pragmas do not count
/^\#labels/ {
  next;
}

## make sure that control sequences are always at the front
/^ *=/ { sub(/^ *=/, "=") }

## check itemness
{ inItem = 0 }
/^   *\*/ {
  inItem = 1
  ## print " ########## ITEM"
}

itemizing && !inItem {
  print "\\end{itemize}";
  print "";
  itemizing = 0;
}

## quoting stops when item appears
## or not indented any more
quoting && (inItem || !/^ /) {
  print "\\end{quotation}";
  print ""
  quoting = 0;
}

## empty lines do not count
/^ *$/ {
  next;
}

/`/ {
  line = $0;
  sub(/`/, "\\texttt{", line);
  sub(/`/, "}", line);
  $0 = line
}

/^==/ {
  sub(/^==/, "\\subsection*{");
  sub(/==/, "}");
  print;
  next;
}

/^=/ {
  sub(/^=/, "\\section*{");
  sub(/=/, "}");
  print;
  print "";
  print "\\label{f0}";
  next;
}

quoting && /^ / {
  sub(/^ /, "");
  print;
  next;
}

!itemizing && inItem {
  print "\\begin{itemize}";
  itemizing = 1;
}

itemizing && inItem {
  sub(/^   *\*/, "\\item");
  print;
  print "";
  next;
}

/^ / {
  print ""
  print "\\begin{quotation}";
  quoting = 1;
  sub(/^ /, "");
  print;
  next;
}

{ print }

BEGIN {
  print "%TCIDATA{Version=5.00.0.2606}";
  print "%TCIDATA{LaTeXparent=0,0,classes.tex}";
  print "                      ";
  print "";
  print "%%%%% BEGINNING OF DOCUMENT BODY %%%%%";
  print "% html: Beginning of file: `clean.html'";
  print "% DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\"";
  print "%  This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. ";
  print "";
  quoting = 0;
  itemizing = 0;
  coding = 0;
}

END {
  print "% html: End of file: `clean.html'";
  print "";
  print "%%%%% END OF DOCUMENT BODY %%%%%";
  print "% In the future, we might want to put some additional data here, such";
  print "% as when the documentation was converted from wiki to TeX.";
  print "%";
}

## HANDLE CODING
## make sure that control sequences are always at the front
/ *{{{/ { sub(/ */, "") }
/ *}}}/ { sub(/ */, "") }

!coding && /^{{{/ {

  if (quoting) {
    print "\\end{quotation}";
    print ""
    quoting = 0;
  }

  print "\\begin{verbatim}";
  coding = 1;
  next;
}

coding && /^}}}/ {
  print "\\end{verbatim}";
  coding = 0;
  next;
}

coding {
  sub(/^/, "{\\small ");
  print;
  print "}";
  next;
}

## NO CODING BELOW THIS POINT
coding { print "###code???####"; next; }

## pragmas do not count
/^\#labels/ {
  next;
}

## make sure that control sequences are always at the front
/^ *=/ { sub(/^ *=/, "=") }

## check itemness
{ inItem = 0 }
/^   *\*/ {
  inItem = 1
  ## print " ########## ITEM"
}

itemizing && !inItem {
  print "\\end{itemize}";
  itemizing = 0;
}

## quoting stops when item appears
## or not indented any more
quoting && (inItem || !/^ /) {
  print "\\end{quotation}";
  print ""
  quoting = 0;
}

## empty lines do not count
/^ *$/ {
  next;
}

/`/ {
  line = $0;
  sub(/`/, "\\texttt{", line);
  sub(/`/, "}", line);
  $0 = line
}

/^==/ {
  sub(/^==/, "\\subsection*{");
  sub(/==/, "}");
  print;
  next;
}

/^=/ {
  sub(/^=/, "\\section*{");
  sub(/=/, "}");
  print;
  print "";
  print "\\label{f0}";
  next;
}

quoting && /^ / {
  sub(/^ /, "");
  print;
  next;
}

!itemizing && inItem {
  print "\\begin{itemize}";
  itemizing = 1;
}

itemizing && inItem {
  sub(/^   *\*/, "\\item");
  print;
  print "";
  next;
}

/^ / {
  print ""
  print "\\begin{quotation}";
  quoting = 1;
  sub(/^ /, "");
  print;
  next;
}

{ print }


Friendly hint in sanitycheck/filter as for why a core file may be missing

#!/usr/bin/awk -f

BEGIN{
    succeeded=0;
}
/<blocked_tests>/ { counting_blocks=1; next }
/<\/blocked_tests/ { counting_blocks=0; next }
counting_blocks==1 && /<test name=.*\"\/>/  {
    if (nodeps) {
        unexpected_blocked[unexpected_block_count++]=$0;
    }
    else
    {
        block_count++;
    }
    next;
}
/<test name=.*should_fail.*result=\"FAILED\">/,/<\/test>/ {
    if ($0 ~ /<\/test>/)
    {
        if (files_left)
        {
            if (testname !~ /left_behind/)
            {
                unexpected_files_behind[unexpected_files_behind_count++]=testname;
            }
        }
        else
        {
            if (testname ~ /left_behind/)
            {
                unexpected_clean[unexpected_clean_count++]=testname;
                if (testname ~ /core/) {
                    core_msg="Is your ulimit setting preventing core dumps?";
                }
            }
        }
        files_left=0;
        testname="";
        next;
    }
    if ($0 ~ /<test name/) {
        testname=$0;
        ++counted_failures;
        if ($1 ~ /should_not_run/ && !nodeps) {
            unexpected_run[unexpected_run_count++]=$0;
        }
        next;
    }
}
/<test name=.*should_fail.*result=\"OK\"/ {
    unexpected_success[failed_ok++]=$0; next;
}
/<test name=.*succ.*FAILED/ {
    unexpected_fail[ok_fail++]=$0; next;
}
/<test name=.*succ.*OK\"\/>/ {
    ++counted_succeeded;
    if (($0 ~ /should_not_run/) && !nodeps) {
        unexpected_run[unexpected_run_count++]=$0;
        echo "apa"
    }
    next;
}
/<test name=.*succ.*OK\">/,/<\/test>/ {
    if ($0 ~ /<\/test>/) {
        if (files_left) {
            unexpected_files_behind[unexpected_files_behind_count++]=testname;
        } else  {
            if (testname ~ /left_behind/) {
                unexpected_clean[unexpected_clean_count++]=testname;
                if (testname ~ /core/) {
                    core_msg="Is your ulimit setting preventing core dumps?";
                }
            }
        }
        files_left=0;
        testname="";
        next;
    }
    if ($0 ~ /<test name/) {
        testname=$0;
        ++counted_succeeded;
        if (($0 ~ /should_not_run/) && !nodeps) {
            echo "apa"
            unexpected_run[unexpected_run_count++]=$0;
        }
        next;
    }
}
/<registered_test_cases>/ {
    stat_registered=gensub(/<\/?[a-z_]*>/, "", "g") + 0; next;
}
/<run_test_cases>/ {
    stat_run=gensub(/<\/?[a-z_]*>/, "", "g") + 0; next;
}
/<failed_test_cases>/ {
    stat_failed=gensub(/<\/?[a-z_]*>/, "", "g") + 0; next;
}
/nonempty_dir/ {
    split($0, parts, /"/);
    dirname=parts[2];
    if (testname!="")
    {
        files_left=1
    }
}
END {
    report=0;
    if (registered != stat_registered) {
        printf("ERROR: Expected %d registered tests and found %d in stats\n",
               registered, stat_registered);
        report=1;
    }
    if (run != stat_run) {
        printf("ERROR: Expected %d run tests and found %d in stats\n",
               run, stat_run);
        report=1;
    }
    if (failed != stat_failed) {
        printf("ERROR: Expected %d failed and %d found in stats\n",
               failed, stat_failed);
        report=1
    }
    if (stat_failed != counted_failures) {
        printf("ERROR stats:%d failed and %d found in log\n",
               stat_failed, counted_failures);
        report=1;
    }
    if (rv != stat_failed) {
        printf("ERROR return value was %d end %d tests failed\n",
               rv, stat_failed);
        report=1
    }
    if (verbose && (stat_run - stat_failed != counted_succeeded)) {
        printf("ERROR stats:%d successful, and %d found in log\n",
               stat_run - stat_failed, counted_succeeded);
        report=1;
    }
    if (!verbose && counted_succeeded) {
        printf("ERROR: OK tests printed without verbose flag\n");
        report=1;
    }
    if (blocked != block_count)
    {
        printf("ERROR: Expected %d blocked tests and found %d\n",
               blocked, block_count)
        report=1;
    }
    if (failed_ok) {
        report=1;
        print "ERROR: The following tests shouldn't have succeeded:";
        for (a in unexpected_success)
        {
            reason=""
            line=unexpected_success[a]
            print line reason
        }
    }
    if (ok_failed) {
        report=1;
        print "ERROR: The following tests shouldn't have failed:"
        for (a in unexpected_fail)
        {
            print unexpected_fail[a];
        }
    }
    if (unexpected_run_count) {
        report=1;
        print "ERROR: The following tests shouldn't have run:"
        for (a in unexpected_run)
        {
            print unexpected_run[a];
        }
    }
    if (unexpected_blocked_count) {
        report=1;
        print "ERROR: The following tests shouldn't have been blocekd:"
        for (a in unexpecteded_block)
        {
            print unexpected_blocked[a]
        }
    }
    if (unexpected_files_behind_count) {
        report=1;
        print "ERROR: The following tests shouldn't have left files behind:"
        for (a in unexpected_files_behind)
        {
            print unexpected_files_behind[a];
        }
    }
    if (unexpected_clean_count) {
        report=1;
        print "ERROR: The following files should've left files behind:"
        for (a in unexpected_clean)
        {
            print unexpected_clean[a];
        }
        print "    " core_msg
    }
    if (report)
    {
        if (dirname != "")
        {
            print dirname " contains files left behind by test cases";
        }
    }
    else
    {
        if (dirname != "") {
            system("rm -r " dirname);
        }

    }
    exit(report);
}

#!/usr/bin/awk -f

BEGIN{
    succeeded=0;
}
/<blocked_tests>/ { counting_blocks=1; next }
/<\/blocked_tests/ { counting_blocks=0; next }
counting_blocks==1 && /<test name=.*\"\/>/  {
    if (nodeps) {
        unexpected_blocked[unexpected_block_count++]=$0;
    }
    else
    {
        block_count++;
    }
    next;
}
/<test name=.*should_fail.*result=\"FAILED\">/,/<\/test>/ {
    if ($0 ~ /<\/test>/)
    {
        if (files_left)
        {
            if (testname !~ /left_behind/)
            {
                unexpected_files_behind[unexpected_files_behind_count++]=testname;
            }
        }
        else
        {
            if (testname ~ /left_behind/)
            {
                unexpected_clean[unexpected_clean_count++]=testname;
            }
        }
        files_left=0;
        testname="";
        next;
    }
    if ($0 ~ /<test name/) {
        testname=$0;
        ++counted_failures;
        if ($1 ~ /should_not_run/ && !nodeps) {
            unexpected_run[unexpected_run_count++]=$0;
        }
        next;
    }
}
/<test name=.*should_fail.*result=\"OK\"/ {
    unexpected_success[failed_ok++]=$0; next;
}
/<test name=.*succ.*FAILED/ {
    unexpected_fail[ok_fail++]=$0; next;
}
/<test name=.*succ.*OK\"\/>/ {
    ++counted_succeeded;
    if (($0 ~ /should_not_run/) && !nodeps) {
        unexpected_run[unexpected_run_count++]=$0;
        echo "apa"
    }
    next;
}
/<test name=.*succ.*OK\">/,/<\/test>/ {
    if ($0 ~ /<\/test>/) {
        if (files_left)
        {
            unexpected_files_behind[unexpected_files_behind_count++]=testname;
        }
        else
        {
            if (testname ~ /left_behind/)
            {
                unexpected_clean[unexpected_clean_count++]=testname;
            }
        }
        files_left=0;
        testname="";
        next;
    }
    if ($0 ~ /<test name/) {
        testname=$0;
        ++counted_succeeded;
        if (($0 ~ /should_not_run/) && !nodeps) {
            echo "apa"
            unexpected_run[unexpected_run_count++]=$0;
        }
        next;
    }
}
/<registered_test_cases>/ {
    stat_registered=gensub(/<\/?[a-z_]*>/, "", "g") + 0; next;
}
/<run_test_cases>/ {
    stat_run=gensub(/<\/?[a-z_]*>/, "", "g") + 0; next;
}
/<failed_test_cases>/ {
    stat_failed=gensub(/<\/?[a-z_]*>/, "", "g") + 0; next;
}
/nonempty_dir/ {
    split($0, parts, /"/);
    dirname=parts[2];
    if (testname!="")
    {
        files_left=1
    }
}
END {
    report=0;
    if (registered != stat_registered) {
        printf("ERROR: Expected %d registered tests and found %d in stats\n",
               registered, stat_registered);
        report=1;
    }
    if (run != stat_run) {
        printf("ERROR: Expected %d run tests and found %d in stats\n",
               run, stat_run);
        report=1;
    }
    if (failed != stat_failed) {
        printf("ERROR: Expected %d failed and %d found in stats\n",
               failed, stat_failed);
        report=1
    }
    if (stat_failed != counted_failures) {
        printf("ERROR stats:%d failed and %d found in log\n",
               stat_failed, counted_failures);
        report=1;
    }
    if (rv != stat_failed) {
        printf("ERROR return value was %d end %d tests failed\n",
               rv, stat_failed);
        report=1
    }
    if (verbose && (stat_run - stat_failed != counted_succeeded)) {
        printf("ERROR stats:%d successful, and %d found in log\n",
               stat_run - stat_failed, counted_succeeded);
        report=1;
    }
    if (!verbose && counted_succeeded) {
        printf("ERROR: OK tests printed without verbose flag\n");
        report=1;
    }
    if (blocked != block_count)
    {
        printf("ERROR: Expected %d blocked tests and found %d\n",
               blocked, block_count)
        report=1;
    }
    if (failed_ok) {
        report=1;
        print "ERROR: The following tests shouldn't have succeeded:";
        for (a in unexpected_success)
        {
            reason=""
            line=unexpected_success[a]
            if (line ~ /core/) {
                reason="\n     (is your ulimit setting preventing core dumps?)"
            }
            print line reason
        }
    }
    if (ok_failed) {
        report=1;
        print "ERROR: The following tests shouldn't have failed:"
        for (a in unexpected_fail)
        {
            print unexpected_fail[a];
        }
    }
    if (unexpected_run_count) {
        report=1;
        print "ERROR: The following tests shouldn't have run:"
        for (a in unexpected_run)
        {
            print unexpected_run[a];
        }
    }
    if (unexpected_blocked_count) {
        report=1;
        print "ERROR: The following tests shouldn't have been blocekd:"
        for (a in unexpecteded_block)
        {
            print unexpected_blocked[a]
        }
    }
    if (unexpected_files_behind_count) {
        report=1;
        print "ERROR: The following tests shouldn't have left files behind:"
        for (a in unexpected_files_behind)
        {
            print unexpected_files_behind[a];
        }
    }
    if (unexpected_clean_count) {
        report=1;
        print "ERROR: The following files should've left files behind:"
        for (a in unexpected_clean)
        {
            print unexpected_clean[a];
        }
    }
    if (report)
    {
        if (dirname != "")
        {
            print dirname " contains files left behind by test cases";
        }
    }
    else
    {
        if (dirname != "") {
            system("rm -r " dirname);
        }

    }
    exit(report);
}