build.awk: use ManTemplate to generate ManMarkdown

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/Languages.awk"
@include "metainfo.awk"

function readFrom(file,    line, text) {
    if (!file) file = "/dev/stdin"
    text = NULLSTR
    while (getline line < file)
        text = (text ? text "\n" : NULLSTR) line
    return text
}

function writeTo(text, file) {
    if (!file) file = "/dev/stdout"
    print text > file
}

function getOutput(command,    content, line) {
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

function init() {
    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    HomebrewFormula      = PagesPath "translate-shell.rb"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    rows = int(length(Locale) / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = group[1]
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language "_language" ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    print "***" length(Locale) "*** *languages in total. "              \
        "Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = group[1]
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language "_language", getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function build(target, type,    group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans
        print > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        if (fileExists(EntryPoint))
            while (getline line < EntryPoint) {
                match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
                if (RSTART) {
                    # Include file
                    if (fileExists(group[1] ".awk"))
                        while (getline inline < (group[1] ".awk"))
                            if (inline = squeeze(inline))
                                print inline > Trans # effective LOC
                } else {
                    if (line && line !~ /^[[:space:]]*#!/) {
                        # Remove preceding spaces
                        gsub(/^[[:space:]]+/, "", line)
                        print line > Trans
                    }
                }
            }
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "export TRANS_ABSPATH=$(cd \"$(dirname \"$0\")\" && pwd)/$(basename \"$0\")" > Trans

        print "export COLUMNS=$(tput cols)" > Trans

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp

        if (fileExists(EntryPoint))
            while (getline line < EntryPoint) {
                match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
                if (RSTART) {
                    # Include file
                    if (fileExists(group[1] ".awk"))
                        while (getline inline < (group[1] ".awk"))
                            if (inline = squeeze(inline, 1))
                                print inline > TransAwk
                } else {
                    if (temp == "Darwin" && line == "#!/usr/bin/gawk -f")
                        # OS X: gawk not in /usr/bin, use a better shebang
                        print "#!/usr/bin/env gawk -f" > TransAwk
                    else
                        print line > TransAwk
                }
            }

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/translate-shell.rb ...")
    # Update gh-pages/translate-shell.rb
    ("sha1sum " PagesPath "translate-shell.tar.gz") | getline temp
    split(temp, group)
    sha1 = group[1]
    text = readFrom(HomebrewFormula ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, HomebrewFormula)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/Languages.awk"
@include "metainfo.awk"

function readFrom(file,    line, text) {
    if (!file) file = "/dev/stdin"
    text = NULLSTR
    while (getline line < file)
        text = (text ? text "\n" : NULLSTR) line
    return text
}

function writeTo(text, file) {
    if (!file) file = "/dev/stdout"
    print text > file
}

function getOutput(command,    content, line) {
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

function init() {
    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManSource            = Man ".md"
    ManTemplate          = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    HomebrewFormula      = PagesPath "translate-shell.rb"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man() {
    if (fileExists(ManTemplate))
        system("pandoc -s -t html --toc --toc-depth 1 --template " ManTemplate " " ManSource " -o " ManHtml)
    return system("pandoc -s -t man " ManSource " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    rows = int(length(Locale) / 3) + 1
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = group[1]
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language "_language" ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    print "***" length(Locale) "*** *languages in total. "              \
        "Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = group[1]
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language "_language", getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function build(target, type,    group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans
        print > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        if (fileExists(EntryPoint))
            while (getline line < EntryPoint) {
                match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
                if (RSTART) {
                    # Include file
                    if (fileExists(group[1] ".awk"))
                        while (getline inline < (group[1] ".awk"))
                            if (inline = squeeze(inline))
                                print inline > Trans # effective LOC
                } else {
                    if (line && line !~ /^[[:space:]]*#!/) {
                        # Remove preceding spaces
                        gsub(/^[[:space:]]+/, "", line)
                        print line > Trans
                    }
                }
            }
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "export TRANS_ABSPATH=$(cd \"$(dirname \"$0\")\" && pwd)/$(basename \"$0\")" > Trans

        print "export COLUMNS=$(tput cols)" > Trans

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp

        if (fileExists(EntryPoint))
            while (getline line < EntryPoint) {
                match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
                if (RSTART) {
                    # Include file
                    if (fileExists(group[1] ".awk"))
                        while (getline inline < (group[1] ".awk"))
                            if (inline = squeeze(inline, 1))
                                print inline > TransAwk
                } else {
                    if (temp == "Darwin" && line == "#!/usr/bin/gawk -f")
                        # OS X: gawk not in /usr/bin, use a better shebang
                        print "#!/usr/bin/env gawk -f" > TransAwk
                    else
                        print line > TransAwk
                }
            }

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/translate-shell.rb ...")
    # Update gh-pages/translate-shell.rb
    ("sha1sum " PagesPath "translate-shell.tar.gz") | getline temp
    split(temp, group)
    sha1 = group[1]
    text = readFrom(HomebrewFormula ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, HomebrewFormula)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}


fix: sanitize_text() may create trailing whitespaces that must be removed

#
# Toolbox of GNU Awk functions.
#

#
# Return the string of a shell variable definition named `name` with quoted value `value`".
#
function to_shell_variable(name, value) {
  return name "='" escape_shell_quoted_text(value) "'"
}

#
# Escape a quoted text by adding backspaces behind simple quotes.
#
function escape_shell_quoted_text(text) {
  # escape simple quotes
  gsub(/'/, "\\'", text);
  return text;
}

#
# Sanitize a string by removing non ascii chars and truncating multiple spaces into a single one.
#
function sanitize_text(text) {
  # remove non-ascii characters
  # this set may evolve with time and only reflects the set of characters required for the
  # set of results we have so far.
  gsub(/[^[\x00-\x7F]]|[\(\)]/, "", text);
  gsub(/[[:space:]]{2,}/, " ", text);
  return trim(text);
}

#
# Trim functions
#
function ltrim(s) { sub(/^[ \t\r\n]+/, "", s); return s }
function rtrim(s) { sub(/[ \t\r\n]+$/, "", s); return s }
function trim(s)  { return rtrim(ltrim(s)); }

#
# Return the text of current row starting from the nth column `c`.
#
function text_from_nth_col(c) {
  for (i = c - 1; i >= 1; --i)
    $i="";
  return $0;
}

#
# Setter of dynamically-named variable (aka indirect reference).
# Users of this functions are required to implement a setter function for each variable
# named `set_<variable>(value)`.
# gawk does not support dynamically name variables, but it does support indirect *function*
# calls. So the trick here is to call a dedicated setter of the variable.
#
function set(variable, value) {
  call="set_" variable
  @call(value)
}

#
# Toolbox of GNU Awk functions.
#

#
# Return the string of a shell variable definition named `name` with quoted value `value`".
#
function to_shell_variable(name, value) {
  return name "='" escape_shell_quoted_text(value) "'"
}

#
# Escape a quoted text by adding backspaces behind simple quotes.
#
function escape_shell_quoted_text(text) {
  # escape simple quotes
  gsub(/'/, "\\'", text);
  return text;
}

#
# Sanitize a string by removing non ascii chars and truncating multiple spaces into a single one.
#
function sanitize_text(text) {
  # remove non-ascii characters
  # this set may evolve with time and only reflects the set of characters required for the
  # set of results we have so far.
  gsub(/[^[\x00-\x7F]]|[\(\)]/, "", text);
  gsub(/[[:space:]]{2,}/, " ", text);
  return text;
}

#
# Trim functions
#
function ltrim(s) { sub(/^[ \t\r\n]+/, "", s); return s }
function rtrim(s) { sub(/[ \t\r\n]+$/, "", s); return s }
function trim(s)  { return rtrim(ltrim(s)); }

#
# Return the text of current row starting from the nth column `c`.
#
function text_from_nth_col(c) {
  for (i = c - 1; i >= 1; --i)
    $i="";
  return $0;
}

#
# Setter of dynamically-named variable (aka indirect reference).
# Users of this functions are required to implement a setter function for each variable
# named `set_<variable>(value)`.
# gawk does not support dynamically name variables, but it does support indirect *function*
# calls. So the trick here is to call a dedicated setter of the variable.
#
function set(variable, value) {
  call="set_" variable
  @call(value)
}


Scale also the job counts

Scale also the job counts

BEGIN {
  year=strftime("%Y");
  starttime=0;
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

# divide and rund up anything that is not an onteger
# e.g. 1.1 becomes 2
function updiv(x,y) {
  t=1.0*x/y;
  dstr=sprintf("%d",t+0.99);
  return int(dstr);
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,2,length(shadow)-2)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID: /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }

  if (starttime>0) {
    joblength=ended-starttime;
  } else {
    # protect from incomplete data
    joblength=0;
  }

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}

/^===NewFile===/{
  if (starttime>0) {
    print "Termination event for job",jid,"from",shadow,"not found"
  }
  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}
########################################################################
END {
  tj=updiv(ended_jobs,parallelism);

  #since it is rounded, the sum could exceed tj
  tcnt=0;
  tgnz=updiv(goodjobsNZ,parallelism);tcnt=tcnt+tgnz;
  while (tcnt>tj) {tgnz=tgnz-1; tcnt=tcnt-1;}
  tbjs=updiv(badjobsSignal,parallelism);tcnt=tcnt+tbjs;
  while (tcnt>tj) {tbjs=tbjs-1; tcnt=tcnt-1;}
  tbjo=updiv(badjobsOther,parallelism);tcnt=tcnt+tbjo;
  while (tcnt>tj) {tbjo=tbjo-1; tcnt=tcnt-1;}

  # if needed, penalize the "good jobs" the most
  # we mostly want to monitor failures
  tgz=updiv(goodjobsZ,parallelism); tcnt=tcnt+tgz;
  while (tcnt>tj) {tgz=tgz-1; tcnt=tcnt-1;}
  # please notice that the percentages will still be accurate

  print "=================";
  print "Total jobs",tj,"utilization",int(total_used/parallelism);
  print "Total goodZ jobs",tgz, " (" percent(goodjobsZ,ended_jobs) "%)","utilization",int(goodputZ/parallelism) " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",tgnz, " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",int(goodputNZ/parallelism) " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",tbjs, " (" percent(badjobsSignal,ended_jobs) "%)","utilization",int(badputSignal/parallelism) " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",tbjo, " (" percent(badjobsOther,ended_jobs) "%)","utilization",int(badputOther/parallelism) " (" percent(badputOther,total_used) "%)";
}

BEGIN {
  year=strftime("%Y");
  starttime=0;
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,2,length(shadow)-2)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID: /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }

  if (starttime>0) {
    joblength=ended-starttime;
  } else {
    # protect from incomplete data
    joblength=0;
  }

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}

/^===NewFile===/{
  if (starttime>0) {
    print "Termination event for job",jid,"from",shadow,"not found"
  }
  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
  starttime=0;
}
########################################################################
END {
  print "=================";
  print "Total jobs",ended_jobs,"utilization",int(total_used/parallelism);
  print "Total goodZ jobs",goodjobsZ " (" percent(goodjobsZ,ended_jobs) "%)","utilization",int(goodputZ/parallelism) " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",goodjobsNZ " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",int(goodputNZ/parallelism) " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",badjobsSignal " (" percent(badjobsSignal,ended_jobs) "%)","utilization",int(badputSignal/parallelism) " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",badjobsOther " (" percent(badjobsOther,ended_jobs) "%)","utilization",int(badputOther/parallelism) " (" percent(badputOther,total_used) "%)";
}


added bibo prefix.

added bibo prefix.

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [          \n",             comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",                 comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];                            \n",               comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}


Handle mscorwks_unixexports.src with better regexp operator for compatibility

ver3:
According to the gawk manual, The '\S' operator (as shorthand for '[^[:space:]]')
matches any character that is not whitespace. However, '[ \t]' is better than
'\S' for Mac OSX.
(source - https://www.gnu.org/software/gawk/manual/html_node/GNU-Regexp-Operators.html )

ver2:
With ver1, the build-break is caused by different awk versions (by @jkotas).
(Revert PR - https://github.com/dotnet/coreclr/pull/4005)
To resolve the limitation of the ver1, @janvorli proposed that the
\S expression is better than [:space:] for the most compatible solution.

There are a lot of different AWK software (e.g. awk, gawk, mawk, nawk, etc)
and versions among the popular Linux distributions. This patch is a simply
way to handle 'mscorwks_unixexports.src' file with \S operator for more
compatibility amon the different awk versions.

ver1:
It's must be modified by gawk implementation that check it exactly
as a plain character (e.g., space, alnum) within the bracket expression
at ./coreclr/generateversionscript.awk. The [[:space:]] is more appropriate than
the [:space:]]. (PR - https://github.com/dotnet/coreclr/pull/3943)

Signed-off-by: Geunsik Lim <acd8f4539166019ed6425de5f67e6d834f46e5a4@samsung.com>
Signed-off-by: MyungJoo Ham <84146178f7c201fa3079bfc93fccba71d81dffe3@samsung.com>
Signed-off-by: Prajwal A N <b9045b1f56dab106e820d6cda13049bcd6c9e3af@samsung.com>

BEGIN {
	print "V1.0 {";
	print "    global:";
}
{
	# Remove the CR character in case the sources are mapped from
	# a Windows share and contain CRLF line endings
	gsub(/\r/,"", $0);

	# Skip empty lines and comment lines starting with semicolon
	if (NF && !match($0, /^[ \t]*;/))
	{
		print "        "  $0 ";";
	}
}
END {
	print "    local: *;"
	print "};";
}

BEGIN {
	print "V1.0 {";
	print "    global:";
}
{
	# Remove the CR character in case the sources are mapped from
	# a Windows share and contain CRLF line endings
	gsub(/\r/,"", $0);

	# Skip empty lines and comment lines starting with semicolon
	if (NF && !match($0, /^[:space:]*;/))
	{
		print "        "  $0 ";";
	}
}
END {
	print "    local: *;"
	print "};";
}


port.dispatch: disable ENVIRON["TERMINAL"]... something else should happen here, but what?

# pick a port based on -f / attr.from or a keyword provided as $0

# port.dispatch routes calls triggered by applications to what the following rules say.
# it's allowed to ignore the input and only handle the event itself.
# let's use the string "dummy" as argument to be ignored,
# to improve readability of the call configured in the application.

# todo: we could do without by better employing %notation to refer to nested instances' ctx:
@include "ctx.awk"

BEGIN {
    DEFAULT_MENU["menu"] = "default"
    MOUSE_MENU["menu"] = "9menu"
}

function menucolors(fg, bg, selfg, selbg,    s) {
    export("menu_fg", fg)
    export("menu_bg", bg)
    export("menu_sel_fg", selfg)
    export("menu_sel_bg", selbg)
}
# todo: is putting subevent names in $0 a good idea? i don't think so...
attr["from"] == "wm" && $0 == "title1" {
    menucolors("#ffffff", "#006699", "#ffffff", "#ff7f00")
    run(nest(get("ctx.xprop.wm_name"), "wm_name", MOUSE_MENU))
}
attr["from"] == "wm" && $0 == "title2" {
    run(nest(get("ctx.sel"), "open", MOUSE_MENU))
}
attr["from"] == "wm" && $0 == "title3" {
    menucolors("#ffffff", "#ff7f00", "#000000", "#ffffff")
    run(nest(get("ctx.xwin"), "wm", MOUSE_MENU))
}

attr["from"] == "wm" && $0 == "status1" {
    run("~/.teslawm/menu.sh ~/.teslawm/status3")
}

# this one expects a substring of the mailclient's window in $0 (sorry):
attr["from"] == "mail-incoming" {
    run("wmu_title.sh " Q($0))
}

# we get a file as argument:
attr["from"] == "BROWSER" ||
attr["from"] == "EDITOR" ||
# attr["from"] == "TERMINAL" ||
attr["from"] == "irc-user" ||
attr["from"] == "gnome-desktop-menu" {
    run(nest($0, "open", DEFAULT_MENU))
}
attr["from"] == "rox-mime" ||
attr["from"] == "claws-mime" ||
attr["from"] == "mail-browser" ||
attr["from"] == "mail-editor" ||
attr["from"] == "irc-url" ||
attr["from"] == "gnome-desktop" {
    run(nest($0, "open"))
}

# diff client
func aimplode(    r) {
    r = ""
    for(i = 1; i <= NF; ++i)
	r = r " " $i
    return trim(r)
}
attr["from"] == "hg-diff" {
    run(nest(aimplode(), "diff", DEFAULT_MENU))
}

# download events:
attr["from"] == "download-prepare" {
    # nothing yet
}
attr["from"] == "download-waiting" {
    # nothing yet
}
attr["from"] == "download-success" {
    history(get("download_file"))
    # run(nest(get("download_file"), "open"))
}
REQ_VERBOSE && attr["from"] == "download-success" {
    print "\nSUCCESS: download finished"
    print "url: " attr["download_url"]
    print "file: " attr["download_file"]
    printf("%s", "hit [ENTER] to continue: ")
    system("read") # todo: erm...?!?
}
attr["from"] == "download-error" {
    print "\nERROR: download failed!"
    print "reason: " attr["download_error"]
    print "exitcode: " attr["exitcode"]
    print "url: " attr["download_url"]
    print "debug: " $0
    printf("%s", "hit [ENTER] to continue: ")
    system("read") # todo: erm...?!?
}

func umount_menu(    f, s, l, m) {
    f = "/proc/mounts"
    s = ""
    while((getline l < f) > 0) {
	split(l, m)
	if(m[2] ~ /^\/(|boot|dev|proc|run|sys|tmp)$/ || m[2] ~ /^\/(dev|var|sys)/)
	    continue
	s = s "\n" m[2]
    }
    close(f)
    c = btick(nest(s, "menu"))
    if(c)
	runHook(nest(c, "umount"))
}
attr["from"] == "umount-menu" {
    run("@umount_menu")
}

# pick a port based on -f / attr.from or a keyword provided as $0

# port.dispatch routes calls triggered by applications to what the following rules say.
# it's allowed to ignore the input and only handle the event itself.
# let's use the string "dummy" as argument to be ignored,
# to improve readability of the call configured in the application.

# todo: we could do without by better employing %notation to refer to nested instances' ctx:
@include "ctx.awk"

BEGIN {
    DEFAULT_MENU["menu"] = "default"
    MOUSE_MENU["menu"] = "9menu"
}

function menucolors(fg, bg, selfg, selbg,    s) {
    export("menu_fg", fg)
    export("menu_bg", bg)
    export("menu_sel_fg", selfg)
    export("menu_sel_bg", selbg)
}
# todo: is putting subevent names in $0 a good idea? i don't think so...
attr["from"] == "wm" && $0 == "title1" {
    menucolors("#ffffff", "#006699", "#ffffff", "#ff7f00")
    run(nest(get("ctx.xprop.wm_name"), "wm_name", MOUSE_MENU))
}
attr["from"] == "wm" && $0 == "title2" {
    run(nest(get("ctx.sel"), "open", MOUSE_MENU))
}
attr["from"] == "wm" && $0 == "title3" {
    menucolors("#ffffff", "#ff7f00", "#000000", "#ffffff")
    run(nest(get("ctx.xwin"), "wm", MOUSE_MENU))
}

attr["from"] == "wm" && $0 == "status1" {
    run("~/.teslawm/menu.sh ~/.teslawm/status3")
}

# this one expects a substring of the mailclient's window in $0 (sorry):
attr["from"] == "mail-incoming" {
    run("wmu_title.sh " Q($0))
}

# we get a file as argument:
attr["from"] == "BROWSER" ||
attr["from"] == "EDITOR" ||
attr["from"] == "TERMINAL" ||
attr["from"] == "irc-user" ||
attr["from"] == "gnome-desktop-menu" {
    run(nest($0, "open", DEFAULT_MENU))
}
attr["from"] == "rox-mime" ||
attr["from"] == "claws-mime" ||
attr["from"] == "mail-browser" ||
attr["from"] == "mail-editor" ||
attr["from"] == "irc-url" ||
attr["from"] == "gnome-desktop" {
    run(nest($0, "open"))
}

# diff client
func aimplode(    r) {
    r = ""
    for(i = 1; i <= NF; ++i)
	r = r " " $i
    return trim(r)
}
attr["from"] == "hg-diff" {
    run(nest(aimplode(), "diff", DEFAULT_MENU))
}

# download events:
attr["from"] == "download-prepare" {
    # nothing yet
}
attr["from"] == "download-waiting" {
    # nothing yet
}
attr["from"] == "download-success" {
    history(get("download_file"))
    # run(nest(get("download_file"), "open"))
}
REQ_VERBOSE && attr["from"] == "download-success" {
    print "\nSUCCESS: download finished"
    print "url: " attr["download_url"]
    print "file: " attr["download_file"]
    printf("%s", "hit [ENTER] to continue: ")
    system("read") # todo: erm...?!?
}
attr["from"] == "download-error" {
    print "\nERROR: download failed!"
    print "reason: " attr["download_error"]
    print "exitcode: " attr["exitcode"]
    print "url: " attr["download_url"]
    print "debug: " $0
    printf("%s", "hit [ENTER] to continue: ")
    system("read") # todo: erm...?!?
}

func umount_menu(    f, s, l, m) {
    f = "/proc/mounts"
    s = ""
    while((getline l < f) > 0) {
	split(l, m)
	if(m[2] ~ /^\/(|boot|dev|proc|run|sys|tmp)$/ || m[2] ~ /^\/(dev|var|sys)/)
	    continue
	s = s "\n" m[2]
    }
    close(f)
    c = btick(nest(s, "menu"))
    if(c)
	runHook(nest(c, "umount"))
}
attr["from"] == "umount-menu" {
    run("@umount_menu")
}


changed Tool to Thing in default template

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix doap:          <http://usefulinc.com/ns/doap#> ."
      print "@prefix qb:            <http://purl.org/linked-data/cube#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [\n",                        comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",           comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];\n",                                            comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:domain_template \"thing_[r]\";\n");
                                    printf("      #   conversion:domain_name     \"Thing\";\n");
                                    printf("      #];\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:class_name \"Thing\";\n");
                                    printf("      #   conversion:subclass_of <http://purl.org/...>;\n");
                                    printf("      #];\n");
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v cellDelimiter
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   ALWAYS_SHOW_CONVERSION_PROCESS = 1; # Added back in to gather empirical results quantifying "effort" to create e1 params.
   showConversionProcess = ALWAYS_SHOW_CONVERSION_PROCESS + length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   #FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   # TODO: '|' has no length? --\/
          DELIMTER  = length(cellDelimiter) ? cellDelimiter : "	" # <--- that's a tab character.
   commentDELIMITER = length(cellDelimiter) ? ""            : "#";
   #print "#AWK: "length(cellDelimiter)" length cellDelimiter:",cellDelimiter," DELIMITER: ",DELIMITER," comment: ",commentDELIMITER

   if(length(showConversionProcess)) {
      print "@prefix rdf:           <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:          <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:          <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:           <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:           <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:          <http://purl.org/vocab/vann/> ."
      print "@prefix skos:          <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:          <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:           <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:      <http://www.geonames.org/ontology#> ."
      print "@prefix geonamesid:    <http://sws.geonames.org/> ."
      print "@prefix govtrackusgov: <http://www.rdfabout.com/rdf/usgov/geo/us/> ."
      print "@prefix dbpedia:       <http://dbpedia.org/resource/> ."
      print "@prefix con:           <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:           <http://purl.oclc.org/NET/muo/muo#> ."
      print "@prefix vs:            <http://www.w3.org/2003/06/sw-vocab-status/ns#> ."
      print "@prefix frbr:          <http://purl.org/vocab/frbr/core#> ."
      print "@prefix bibo:          <http://purl.org/ontology/bibo/> ."
      print "@prefix doap:          <http://usefulinc.com/ns/doap#> ."
      print "@prefix qb:            <http://purl.org/linked-data/cube#> ."
      print "@prefix dgtwc:         <http://data-gov.tw.rpi.edu/2009/data-gov-twc.rdf#> ."
   }
   print "@prefix dcterms:       <http://purl.org/dc/terms/> ."
   print "@prefix void:          <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:         <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:          <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:          <http://xmlns.com/foaf/0.1/> ."
   print "@prefix ov:            <http://open.vocab.org/terms/> ."
   print "@prefix conversion:    <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #### NOTE: brought this back for a place for implicit bundles to be named, HOWEVER, the dataset is NOT being named within this:
   printf("@prefix :              <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

   #
   # Describe the creator.
   #
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
                                     print
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s#%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s#%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }

   if( length(conversionID)) {
                                    printf("\n");
                                    printf("#:a_bundle\n");
                                    printf("#   a conversion:ImplicitBundle;\n");
                                    printf("#   conversion:property_name \"a_property\"; # Can also be a URI, e.g. dcterms:title.\n");
                                    printf("#   #conversion:type_name     \"My Class\";   # Can also be a URI, e.g. foaf:Person.\n");
                                    printf("#.\n");
   }
   #
   # Describe the dataset.
   #
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;\n"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
                                     print ""
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(!length(conversionID))        printf("      conversion:conversion_identifier \"raw\";\n");
   if( length(conversionID))        printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);

   #
   # Authorship description.
   #
                                     print
   if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s#%s>;\n",machine_uri,whoami); # NOTE: implemented in bin/util/user-account.sh, too.
   }else if(length(person_uri) && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n\n",nowXSD);
   }

   #
   # Structural enhancement parameters.
   #
   if( cellDelimiter != "\t" ) {
                                    printf("      conversion:delimits_cell \"%s\";\n",cellDelimiter);
                                    printf("      #conversion:delimits_cell \"	\"; # tab\n");
   }else {
                                    printf("      conversion:delimits_cell \"	\"; # tab\n");
   }
                                    printf("      #conversion:delimits_cell \"|\";   # pipe\n");
                                    printf("      #conversion:delimits_cell \",\";   # comma\n");
                                    print
   if(length(header)) {
                                    # Include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n\n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   comment_out = length(interpretAsNull) && length(conversionID) ? "" : "#"; # We want to put in the template so it is easy to uncomment.
                                    printf("      %sconversion:interpret [\n",                        comment_out);
                                    printf("      %s   conversion:symbol        \"%s\";\n",           comment_out,interpretAsNull);
                                    printf("      %s   conversion:interpretation conversion:null; \n",comment_out);
                                    printf("      %s];\n",                                            comment_out);
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:domain_template \"tool_[r]\";\n");
                                    printf("      #   conversion:domain_name     \"Tool\";\n");
                                    printf("      #];\n");
                                    printf("      #conversion:enhance [\n");
                                    printf("      #   conversion:class_name \"Tool\";\n");
                                    printf("      #   conversion:subclass_of <http://purl.org/...>;\n");
                                    printf("      #];\n");
}

{ # length(conversionID) {
   cellValue=$0;

   # If we know there is no header, we can give an example value.
   headerOrExample = (length(header) && header <= 0) ? "conversion:eg " : "ov:csvHeader  ";

   print "      conversion:enhance ["
   printf("         ov:csvCol          %s;\n",NR);
   printf("         %s     \"%s\";\n",headerOrExample,cellValue);
   if(length(conversionID)) {
      # NOTE: This MUST NOT be added to the raw interpretation parameters,
      #       otherwise columns will collapse together prematurely (with human approval).
      printf("         #conversion:label   \"%s\";\n",cellValue);
   }
   printf("         conversion:comment \"\";\n");
   printf("         conversion:range   %s:Literal;\n",RDFS); # Either 'rdfs' or 'todo' (for raw and e*, respectively)
   if(length(conversionID)) {
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
   }
   print "      ];"
}

END {
   print "      #conversion:enhance [            "
   print "      #   ov:csvRow 3,4,5;             "
   print "      #   a conversion:ExampleResource;"
   print "      #];                              "

   print "   ];"
   printf(".");
}