add PLTokenizer (initial)

add PLTokenizer (initial)

#!/usr/bin/gawk -f

# Return an element if it belongs to the array;
# Otherwise, return a null string.
function belongsTo(element, array,
                   ####
                   i) {
    for (i in array)
        if (element == array[i]) return element
    return ""
}

# Return one of the substrings if the string starts with it;
# Otherwise, return a null string.
function startsWithAny(string, substrings,
                       ####
                       i) {
    for (i in substrings)
        if (index(string, substrings[i]) == 1) return substrings[i]
    return ""
}

# Return one of the patterns if the string matches this pattern at the beginning;
# Otherwise, return a null string.
function matchesAny(string, patterns,
                    ####
                    i) {
    for (i in patterns)
        if (string ~ "^" patterns[i]) return patterns[i]
    return ""
}

# Tokenize a string.
function plTokenize(returnTokens, string,
                    delimiters,
                    newlines,
                    quotes,
                    escapeChars,
                    leftBlockComments,
                    rightBlockComments,
                    lineComments,
                    reservedOperators,
                    reservedPatterns,
                    ####
                    blockCommenting,
                    c,
                    currentToken,
                    escaping,
                    i,
                    lineCommenting,
                    p,
                    quoting,
                    r,
                    s,
                    tempGroup,
                    tempPattern,
                    tempString) {
    # Default parameters
    if (!delimiters[0]) {
        delimiters[0] = " "  # whitespace
        delimiters[1] = "\t" # horizontal tab
        delimiters[2] = "\v" # vertical tab
    }
    if (!newlines[0]) {
        newlines[0] = "\n" # line feed
        newlines[1] = "\r" # carriage return
    }
    if (!quotes[0]) {
        quotes[0] = "\"" # double quote
    }
    if (!escapeChars[0]) {
        escapeChars[0] = "\\" # backslash
    }
    if (!leftBlockComments[0]) {
        leftBlockComments[0] = "#|" # Lisp-style extended comment (open)
        leftBlockComments[1] = "/*" # C-style comment (open)
        leftBlockComments[2] = "(*" # ML-style comment (open)
    }
    if (!rightBlockComments[0]) {
        rightBlockComments[0] = "|#" # Lisp-style extended comment (close)
        rightBlockComments[1] = "*/" # C-style comment (close)
        rightBlockComments[2] = "*)" # ML-style comment (close)
    }
    if (!lineComments[0]) {
        lineComments[0] = ";"  # Lisp-style line comment
        lineComments[1] = "//" # C++-style line comment
        lineComments[2] = "#"  # hash comment
    }
    if (!reservedOperators[0]) {
        reservedOperators[0] = "(" #  left parenthesis
        reservedOperators[1] = ")" # right parenthesis
        reservedOperators[2] = "[" #  left bracket
        reservedOperators[3] = "]" # right bracket
        reservedOperators[4] = "{" #  left brace
        reservedOperators[5] = "}" # right brace
        reservedOperators[6] = "," # comma
    }
    if (!reservedPatterns[0]) {
        reservedPatterns[0] = "[+-]?((0|[1-9][0-9]*)|[.][0-9]*|(0|[1-9][0-9]*)[.][0-9]*)([Ee][+-]?[0-9]+)?" # numeric literal (scientific notation possible)
        reservedPatterns[1] = "[+-]?0[0-7]+([.][0-7]*)?" # numeric literal (octal)
        reservedPatterns[2] = "[+-]?0[Xx][0-9A-Fa-f]+([.][0-9A-Fa-f]*)?" # numeric literal (hexadecimal)
    }

    split(string, s, "")
    currentToken = ""
    quoting = escaping = blockCommenting = lineCommenting = 0
    p = 0
    i = 1
    while (i <= length(s)) {
        c = s[i]
        r = substr(string, i)

        if (blockCommenting) {
            if (tempString = startsWithAny(r, rightBlockComments))
                blockCommenting = 0 # block comment ends

            i++

        } else if (lineCommenting) {
            if (belongsTo(c, newlines))
                lineCommenting = 0 # line comment ends

            i++

        } else if (quoting) {
            currentToken = currentToken c

            if (escaping) {
                escaping = 0 # escape ends

            } else {
                if (belongsTo(c, quotes)) {
                    # Finish the current token
                    if (currentToken) {
                        returnTokens[p++] = currentToken
                        currentToken = ""
                    }

                    quoting = 0 # quotation ends

                } else if (belongsTo(c, escapeChars)) {
                    escaping = 1 # escape begins

                } else {
                    # Continue
                }
            }

            i++

        } else {
            if (belongsTo(c, delimiters) || belongsTo(c, newlines)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                i++

            } else if (belongsTo(c, quotes)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                }

                currentToken = c

                quoting = 1 # quotation begins

                i++

            } else if (tempString = startsWithAny(r, leftBlockComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                blockCommenting = 1 # block comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, lineComments)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                lineCommenting = 1 # line comment begins

                i += length(tempString)

            } else if (tempString = startsWithAny(r, reservedOperators)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                returnTokens[p++] = tempString

                i += length(tempString)

            } else if (tempPattern = matchesAny(r, reservedPatterns)) {
                # Finish the current token
                if (currentToken) {
                    returnTokens[p++] = currentToken
                    currentToken = ""
                }

                # Reserve token
                match(r, "^" tempPattern, tempGroup)
                returnTokens[p++] = tempGroup[0]

                i += length(tempGroup[0])

            } else {
                # Continue with the current token
                currentToken = currentToken c

                i++
            }
        }
    }

    # Finish the last token
    if (currentToken)
        returnTokens[p++] = currentToken

}


s_split(url, br, "/")

s_split(url, br, "/")

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/svnpenn/sage.git/'


Add for updating the config file

Add for updating the config file

git-svn-id: 244fd0a71b5b35931765a3ae1f99e3319f3fb8ab@822470 13f79535-47bb-0310-9956-ffa450edef68

# Invoke as awk addloadexample.awk

BEGIN {
  lms = 0;
}

tolower($0) ~ /^[# \t]*loadmodule[ \t]/ {
  if ( $2 == MODULE "_module" ) {
    print "LoadModule " MODULE "_module " LIBPATH "/mod_" MODULE DSO;
    lms = 2;
    next;
  }
  # test $3 since # LoadModule is split into two tokens
  else if ( $3 == MODULE "_module" ) {
    print $1 "LoadModule " MODULE "_module " LIBPATH "/mod_" MODULE DSO;
    lms = 2;
    next;
  }
  else if ( ! lms ) lms = 1;
}

$0 ~ /^[ \t]*$/ && lms == 1 {
  print "LoadModule " MODULE "_module " LIBPATH "/mod_" MODULE DSO;
  lms = 2;
}

tolower($0) ~ /^[# \t]*include[ \t]/ && $NF == EXAMPLECONF {
  lms = 3;
}

{ print }

END {
  if ( lms < 3 ) {
    if ( ! /^[ \t]*$/ ) print "";
    if ( lms < 2 ) {
      print "LoadModule " MODULE "_module " LIBPATH "/mod_" MODULE DSO;
      print "";
    }
    if ( length(EXAMPLECONF) ) {
      print "# Example mod_" MODULE " configuration";
      print "#Include " EXAMPLECONF "\n";
    }
  }
}

error: pathspec 'build/addloadexample.awk' did not match any file(s) known to git


Added more files for NDB Cluster

Added more files for NDB Cluster

BEGIN{
    where=0;
    n_hosts=0;
    n_api=0;
    n_ndb=0;
    n_mgm=0;
    n_ports=0;
}
/COMPUTERS/ {
    where=1;
}
/\[[ \t]*COMPUTER[ \t]*\]/ {
    where=1;
}
/PROCESSES/ {
    where=2;
}
/Type: MGMT/ {
    if(where!=1){
	where=2;
	n_mgm++;
    }
}
/\[[ \t]*MGM[ \t]*\]/ {
    where=2;
    n_mgm++;
}
/Type: DB/ {
    if(where!=1){
	where=3;
	n_ndb++;
    }
}
/\[[ \t]*DB[ \t]*\]/ {
    where=3;
    n_ndb++;
}
/Type: API/ {
    if(where!=1){
	where=4;
	n_api++;
    }
}
/\[[ \t]*API[ \t]*\]/ {
    where=4;
    n_api++;
}
/HostName:/ {
    host_names[host_ids[n_hosts]]=$2;
}

/FileSystemPath:/ {
  if (where==3){
    ndb_fs[ndb_ids[n_ndb]]=$2;
  }
}

/Id:/{
    if(where==1){
	n_hosts++;
	host_ids[n_hosts]=$2;
    }
    if(where==2){
	mgm_ids[n_mgm]=$2;
    }
    if(where==3){
	ndb_ids[n_ndb]=$2;
    }
    if(where==4){
	api_ids[n_api]=$2;
    }
}
/ExecuteOnComputer:/{
    if(where==2){
	mgm_hosts[mgm_ids[n_mgm]]=host_names[$2];
    }
    if(where==3){
	ndb_hosts[ndb_ids[n_ndb]]=host_names[$2];
    }
    if(where==4){
	api_hosts[api_ids[n_api]]=host_names[$2];
    }
}
END {
    for(i=1; i<=n_mgm; i++){
	printf("mgm_%d=%s\n", mgm_ids[i], mgm_hosts[mgm_ids[i]]);
    }
    for(i=1; i<=n_ndb; i++){
	printf("ndb_%d=%s\n", ndb_ids[i], ndb_hosts[ndb_ids[i]]);
	printf("ndbfs_%d=%s\n", ndb_ids[i], ndb_fs[ndb_ids[i]]);
    }
    for(i=1; i<=n_api; i++){
	printf("api_%d=%s\n", api_ids[i], api_hosts[api_ids[i]]);
    }
    printf("mgm_nodes=%d\n", n_mgm);
    printf("ndb_nodes=%d\n", n_ndb);
    printf("api_nodes=%d\n", n_api);
}

error: pathspec 'ndb/home/bin/parseConfigFile.awk' did not match any file(s) known to git


tests: add example awk file  Changes to be committed: 	new file:   tests/syntax-highlighting/file.awk

#!/usr/bin/awk -f
# This is a comment line.

# Special patterns:
BEGIN
END

# Field variables:
xxx$0xxx
xx$NFxxx

# String and numeric contansts:
string = "Hello, World!"
number = 42

# Patterns:
/\.'regexpr#"/ { print $0 }
NR == 1, NR == 5 { printf("%s\n", $0) }

# Statements:
print string, number

error: pathspec 'tests/syntax-highlighting/file.awk' did not match any file(s) known to git


cleaner comment

cleaner comment

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/szilvajuhos/CAW.git/'


add Jon's atmosphere converter

add Jon's atmosphere converter

error: pathspec 'apogee/modelspec/scripts/makemoogmodel.awk' did not match any file(s) known to git


contrib/slurm: add awk script to extract crash reports

Add awk script to extract crash reports from Scalaris log files. Simple grep
does not work because crash reports can expand over mutliple lines.

# Get all crashreports including reports spreading multiple lines

BEGIN {
    crashreport=0
    timestamp_pattern="^20[[:digit:]]{2}-[[:digit:]]{2}-[[:digit:]]{2}"
}

# FILENAME == last_filename { last_filename=FILENAME}
# FILENAME != last_filename { last_filename=FILENAME; print FILENAME}

crashreport == 1 && $0 ~ timestamp_pattern { crashreport = 0; print ""} # leave crash report
crashreport == 0 && /FD/ { crashreport = 1; print $0 } # enter crash report
crashreport == 1 && $0 !~ timestamp_pattern { print "\t", $0 } # during crash report

error: pathspec 'contrib/slurm/util/crashreport.awk' did not match any file(s) known to git


Add linked-sensegram.awk

Add linked-sensegram.awk

error: pathspec 'misc/linked-sensegram.awk' did not match any file(s) known to git


LanguageHelper: scriptName(): add Mong

LanguageHelper: scriptName(): add Mong

####################################################################
# LanguageHelper.awk                                               #
####################################################################

# Get locale key by language code or alias.
function getCode(code,    group) {
    if (code == "auto" || code in Locale)
        return code
    else if (code in LocaleAlias)
        return LocaleAlias[code]
    else if (tolower(code) in LocaleAlias)
        return LocaleAlias[tolower(code)]

    # Remove unidentified region or script code
    match(code, /^([[:alpha:]][[:alpha:]][[:alpha:]]?)-(.*)$/, group)
    if (group[1])
        return group[1]

    return # return nothing if not found
}

# Return the name of a language.
function getName(code) {
    return Locale[getCode(code)]["name"]
}

# Return all the names of a language, separated by "/"
function getNames(code) {
    if ("name2" in Locale[getCode(code)])
        return Locale[getCode(code)]["name"] " / " Locale[getCode(code)]["name2"]
    else
        return Locale[getCode(code)]["name"]
}

# Return the endonym of a language.
function getEndonym(code) {
    return Locale[getCode(code)]["endonym"]
}

# Return the string for displaying the endonym of a language.
function getDisplay(code) {
    return Locale[getCode(code)]["display"]
}

# Return formatted text for "translations of".
function showTranslationsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["translations-of"]
    if (!fmt) fmt = Locale["en"]["translations-of"]
    return sprintf(fmt, text)
}

# Return formatted text for "definitions of".
function showDefinitionsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["definitions-of"]
    if (!fmt) fmt = Locale["en"]["definitions-of"]
    return sprintf(fmt, text)
}

# Return a string of "synonyms".
function showSynonyms(code,    tmp) {
    tmp = Locale[getCode(code)]["synonyms"]
    if (!tmp) tmp = Locale["en"]["synonyms"]
    return tmp
}

# Return a string of "examples".
function showExamples(code,    tmp) {
    tmp = Locale[getCode(code)]["examples"]
    if (!tmp) tmp = Locale["en"]["examples"]
    return tmp
}

# Return a string of "see also".
function showSeeAlso(code,    tmp) {
    tmp = Locale[getCode(code)]["see-also"]
    if (!tmp) tmp = Locale["en"]["see-also"]
    return tmp
}

# Return the family of a language.
function getFamily(code) {
    return Locale[getCode(code)]["family"]
}

# Return the branch of a language.
function getBranch(code) {
    return Locale[getCode(code)]["branch"]
}

# Return the ISO 639-3 code of a language.
function getISO(code) {
    return Locale[getCode(code)]["iso"]
}

# Return the Glottocode of a language.
function getGlotto(code) {
    return Locale[getCode(code)]["glotto"]
}

# Return the ISO 15924 script code of a language.
function getScript(code) {
    return Locale[getCode(code)]["script"]
}

# Return 1 if a language is R-to-L; otherwise return 0.
function isRTL(code) {
    return Locale[getCode(code)]["rtl"] ? 1 : 0
}

# Return 1 if Google provides dictionary data for a language; otherwise return 0.
function hasDictionary(code) {
    return Locale[getCode(code)]["dictionary"] ? 1 : 0
}

# Comparator using getName().
function compName(i1, v1, i2, v2) {
    if (getName(i1) < getName(i2))
        return -1
    else
        return (getName(i1) != getName(i2))
}

# Return the name of script (writing system).
# See: <https://en.wikipedia.org/wiki/ISO_15924#List_of_codes>
#      <http://unicode.org/iso15924/iso15924-codes.html>
function scriptName(code) {
    switch (code) {
    case "Arab": return "Arabic"
    case "Armn": return "Armenian"
    case "Beng": return "Bengali"
    case "Cans": return "Canadian Aboriginal Syllabics"
    case "Cher": return "Cherokee"
    case "Cyrl": return "Cyrillic"
    case "Deva": return "Devanagari"
    case "Ethi": return "Ethiopic (Geʻez)"
    case "Geor": return "Georgian (Mkhedruli)"
    case "Grek": return "Greek"
    case "Gujr": return "Gujarati"
    case "Guru": return "Gurmukhi"
    case "Hani": return "Han"
    case "Hans": return "Han (Simplified)"
    case "Hant": return "Han (Traditional)"
    case "Hebr": return "Hebrew"
    case "Jpan": return "Japanese (Han + Hiragana + Katakana)"
    case "Khmr": return "Khmer"
    case "Knda": return "Kannada"
    case "Kore": return "Korean (Hangul + Han)"
    case "Laoo": return "Lao"
    case "Latn": return "Latin"
    case "Mlym": return "Malayalam"
    case "Mong": return "Mongolian"
    case "Mtei": return "Meitei Mayek"
    case "Mymr": return "Myanmar"
    case "Orya": return "Oriya"
    case "Sinh": return "Sinhala"
    case "Taml": return "Tamil"
    case "Telu": return "Telugu"
    case "Thaa": return "Thaana"
    case "Thai": return "Thai"
    case "Tibt": return "Tibetan"
    default: return "Unknown"
    }
}

# Return the regions that a language is spoken in, as an English string.
function getSpokenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["spoken-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return the regions that a script is written in, as an English string.
function getWrittenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["written-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return the extra description of a language.
function getDescription(code) {
    return Locale[getCode(code)]["description"]
}

# Return 1 if a language is supported by Google; otherwise return 0.
function isSupportedByGoogle(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "google") return 1
    }
    return 0
}

# Return 1 if a language is supported by Bing; otherwise return 0.
function isSupportedByBing(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "bing") return 1
    }
    return 0
}

# Return detailed information of a language as a string.
function getDetails(code,    article, desc, group, iso, name, names, script, writing) {
    if (code == "auto" || !getCode(code)) {
        e("[ERROR] Language not found: " code "\n"                      \
          "        Run '-reference / -R' to see a list of available languages.")
        exit 1
    }

    script = scriptName(getScript(code))
    if (isRTL(code)) script = script " (R-to-L)"

    split(getISO(code), group, "-")
    iso = group[1]

    name = getName(code)
    names = getNames(code)
    if (match(name, /\(.*\)/)) {
        writing = substr(name, match(name, /\(.*\)/) + 1)
        writing = substr(writing, 1, length(writing) - 1)
        name = substr(name, 1, match(name, /\(.*\)/) - 2)
    }

    if (getBranch(code)) {
        article = match(tolower(getBranch(code)), /^[aeiou]/) ? "an" : "a"
        if (iso == "eng")
            desc = sprintf("%s is %s %s language spoken %s.",
                           names, article, getBranch(code), getSpokenIn(code))
        else
            desc = sprintf("%s is %s %s language spoken mainly in %s.",
                           names, article, getBranch(code), getSpokenIn(code))
    } else if (getFamily(code) == NULLSTR || tolower(getFamily(code)) == "language isolate")
        desc = sprintf("%s is a language spoken mainly in %s.", names, getSpokenIn(code))
    else if (tolower(getFamily(code)) == "constructed language")
        desc = sprintf("%s is %s.", names, getDescription(code))
    else
        desc = sprintf("%s is a language of the %s family, spoken mainly in %s.",
                       names, getFamily(code), getSpokenIn(code))
    if (writing && getWrittenIn(code))
        desc = desc sprintf(" The %s writing system is officially used in %s.",
                            tolower(writing), getWrittenIn(code))

    return ansi("bold", sprintf("%s\n", getDisplay(code)))              \
        sprintf("%-22s%s\n", "Name", ansi("bold", names))               \
        sprintf("%-22s%s\n", "Family", ansi("bold", getFamily(code)))   \
        sprintf("%-22s%s\n", "Writing system", ansi("bold", script))    \
        sprintf("%-22s%s\n", "Code", ansi("bold", getCode(code)))       \
        sprintf("%-22s%s\n", "ISO 639-3", ansi("bold", iso))            \
        sprintf("%-22s%s\n", "SIL", ansi("bold", "https://iso639-3.sil.org/code/" iso)) \
        sprintf("%-22s%s\n", "Glottolog", getGlotto(code) ?
                ansi("bold", "https://glottolog.org/resource/languoid/id/" getGlotto(code)) : "") \
        sprintf("%-22s%s\n", "Wikipedia", ansi("bold", "https://en.wikipedia.org/wiki/ISO_639:" iso)) \
        (Locale[getCode(code)]["supported-by"] ? # FIXME
        sprintf("%-22s%s\n", "Translator support", sprintf("Google [%s]    Bing [%s]",
                isSupportedByGoogle(code) ? "✔" : "✘", isSupportedByBing(code) ? "✔" : "✘")) : "") \
        (Locale[getCode(code)]["spoken-in"] ? # FIXME
        ansi("bold", sprintf("\n%s", desc)) : "")
}

# Add /slashes/ for IPA phonemic notations and (parentheses) for others.
# Parameters:
#     code
function showPhonetics(phonetics, code) {
    if (code && getCode(code) == "en")
        return "/" phonetics "/"
    else
        return "(" phonetics ")"
}

# Convert a logical string to visual; don't right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
function show(text, code,    command, temp) {
    if (!code || isRTL(code)) {
        if (Cache[text][0])
            return Cache[text][0]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDiNoPad) {
                command = "echo " parameterize(text) PIPE BiDiNoPad
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][0] = temp
        }
    } else
        return text
}

# Convert a logical string to visual and right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
#     width: ignore to use default width for padding
function s(text, code, width,    command, temp) {
    if (!code || isRTL(code)) {
        if (!width) width = Option["width"]
        if (Cache[text][width])
            return Cache[text][width]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDi) {
                command = "echo " parameterize(text) PIPE sprintf(BiDi, width)
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][width] = temp
        }
    } else
        return text
}

# Convert a logical string to visual with a certain level of indentation.
# Parameters:
#     level: level of indentation
#     code: ignore to apply left indentation
#     width: ignore to use default width for padding
function ins(level, text, code, width,    i, temp) {
    if (code && isRTL(code)) {
        if (!width) width = Option["width"]
        return s(text, code, width - Option["indent"] * level)
    } else
        return replicate(" ", Option["indent"] * level) text
}

# Parse a POSIX locale identifier and return the language code;
# Identified by both language identifier and region identifier.
# Parameters:
#     lang = [language[_territory][.codeset][@modifier]]
# See: <https://en.wikipedia.org/wiki/Locale>
function parseLang(lang,    code, group) {
    match(lang, /^([a-z][a-z][a-z]?)(_|$)/, group)
    code = getCode(group[1])

    # Detect region identifier
    ## Regions using Chinese Simplified: China, Singapore
    if (lang ~ /^zh_(CN|SG)/) code = "zh-CN"
    ## Regions using Chinese Traditional: Taiwan, Hong Kong
    else if (lang ~ /^zh_(TW|HK)/) code = "zh-TW"

    # FIXME: handle unrecognized language code
    if (!code) code = "en"

    return code
}

# Initialize `UserLang`.
function initUserLang(    lang, utf) {
    if (lang = ENVIRON["LC_ALL"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (lang = ENVIRON["LANG"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (!UserLocale) {
        UserLocale = "en_US.UTF-8"
        utf = 1
    }
    if (!utf)
        w("[WARNING] Your locale codeset (" UserLocale ") is not UTF-8.")

    UserLang = parseLang(UserLocale)
}

####################################################################
# LanguageHelper.awk                                               #
####################################################################

# Get locale key by language code or alias.
function getCode(code,    group) {
    if (code == "auto" || code in Locale)
        return code
    else if (code in LocaleAlias)
        return LocaleAlias[code]
    else if (tolower(code) in LocaleAlias)
        return LocaleAlias[tolower(code)]

    # Remove unidentified region or script code
    match(code, /^([[:alpha:]][[:alpha:]][[:alpha:]]?)-(.*)$/, group)
    if (group[1])
        return group[1]

    return # return nothing if not found
}

# Return the name of a language.
function getName(code) {
    return Locale[getCode(code)]["name"]
}

# Return all the names of a language, separated by "/"
function getNames(code) {
    if ("name2" in Locale[getCode(code)])
        return Locale[getCode(code)]["name"] " / " Locale[getCode(code)]["name2"]
    else
        return Locale[getCode(code)]["name"]
}

# Return the endonym of a language.
function getEndonym(code) {
    return Locale[getCode(code)]["endonym"]
}

# Return the string for displaying the endonym of a language.
function getDisplay(code) {
    return Locale[getCode(code)]["display"]
}

# Return formatted text for "translations of".
function showTranslationsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["translations-of"]
    if (!fmt) fmt = Locale["en"]["translations-of"]
    return sprintf(fmt, text)
}

# Return formatted text for "definitions of".
function showDefinitionsOf(code, text,    fmt) {
    fmt = Locale[getCode(code)]["definitions-of"]
    if (!fmt) fmt = Locale["en"]["definitions-of"]
    return sprintf(fmt, text)
}

# Return a string of "synonyms".
function showSynonyms(code,    tmp) {
    tmp = Locale[getCode(code)]["synonyms"]
    if (!tmp) tmp = Locale["en"]["synonyms"]
    return tmp
}

# Return a string of "examples".
function showExamples(code,    tmp) {
    tmp = Locale[getCode(code)]["examples"]
    if (!tmp) tmp = Locale["en"]["examples"]
    return tmp
}

# Return a string of "see also".
function showSeeAlso(code,    tmp) {
    tmp = Locale[getCode(code)]["see-also"]
    if (!tmp) tmp = Locale["en"]["see-also"]
    return tmp
}

# Return the family of a language.
function getFamily(code) {
    return Locale[getCode(code)]["family"]
}

# Return the branch of a language.
function getBranch(code) {
    return Locale[getCode(code)]["branch"]
}

# Return the ISO 639-3 code of a language.
function getISO(code) {
    return Locale[getCode(code)]["iso"]
}

# Return the Glottocode of a language.
function getGlotto(code) {
    return Locale[getCode(code)]["glotto"]
}

# Return the ISO 15924 script code of a language.
function getScript(code) {
    return Locale[getCode(code)]["script"]
}

# Return 1 if a language is R-to-L; otherwise return 0.
function isRTL(code) {
    return Locale[getCode(code)]["rtl"] ? 1 : 0
}

# Return 1 if Google provides dictionary data for a language; otherwise return 0.
function hasDictionary(code) {
    return Locale[getCode(code)]["dictionary"] ? 1 : 0
}

# Comparator using getName().
function compName(i1, v1, i2, v2) {
    if (getName(i1) < getName(i2))
        return -1
    else
        return (getName(i1) != getName(i2))
}

# Return the name of script (writing system).
# See: <https://en.wikipedia.org/wiki/ISO_15924#List_of_codes>
#      <http://unicode.org/iso15924/iso15924-codes.html>
function scriptName(code) {
    switch (code) {
    case "Arab": return "Arabic"
    case "Armn": return "Armenian"
    case "Beng": return "Bengali"
    case "Cans": return "Canadian Aboriginal Syllabics"
    case "Cher": return "Cherokee"
    case "Cyrl": return "Cyrillic"
    case "Deva": return "Devanagari"
    case "Ethi": return "Ethiopic (Geʻez)"
    case "Geor": return "Georgian (Mkhedruli)"
    case "Grek": return "Greek"
    case "Gujr": return "Gujarati"
    case "Guru": return "Gurmukhi"
    case "Hani": return "Han"
    case "Hans": return "Han (Simplified)"
    case "Hant": return "Han (Traditional)"
    case "Hebr": return "Hebrew"
    case "Jpan": return "Japanese (Han + Hiragana + Katakana)"
    case "Khmr": return "Khmer"
    case "Knda": return "Kannada"
    case "Kore": return "Korean (Hangul + Han)"
    case "Laoo": return "Lao"
    case "Latn": return "Latin"
    case "Mtei": return "Meitei Mayek"
    case "Mlym": return "Malayalam"
    case "Mymr": return "Myanmar"
    case "Orya": return "Oriya"
    case "Sinh": return "Sinhala"
    case "Taml": return "Tamil"
    case "Telu": return "Telugu"
    case "Thaa": return "Thaana"
    case "Thai": return "Thai"
    case "Tibt": return "Tibetan"
    default: return "Unknown"
    }
}

# Return the regions that a language is spoken in, as an English string.
function getSpokenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["spoken-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return the regions that a script is written in, as an English string.
function getWrittenIn(code,    i, j, r, regions, str) {
    r = NULLSTR
    str = Locale[getCode(code)]["written-in"]
    if (str) {
        split(str, regions, /\s?;\s?/)
        j = 0
        for (i in regions) {
            r = r regions[i]
            j++
            if (j < length(regions) - 1)
                r = r ", "
            else if (j == length(regions) - 1)
                r = r " and "
        }
    }
    return r
}

# Return the extra description of a language.
function getDescription(code) {
    return Locale[getCode(code)]["description"]
}

# Return 1 if a language is supported by Google; otherwise return 0.
function isSupportedByGoogle(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "google") return 1
    }
    return 0
}

# Return 1 if a language is supported by Bing; otherwise return 0.
function isSupportedByBing(code,    engines, i, str) {
    str = Locale[getCode(code)]["supported-by"]
    if (str) {
        split(str, engines, /\s?;\s?/)
        for (i in engines)
            if (engines[i] == "bing") return 1
    }
    return 0
}

# Return detailed information of a language as a string.
function getDetails(code,    article, desc, group, iso, name, names, script, writing) {
    if (code == "auto" || !getCode(code)) {
        e("[ERROR] Language not found: " code "\n"                      \
          "        Run '-reference / -R' to see a list of available languages.")
        exit 1
    }

    script = scriptName(getScript(code))
    if (isRTL(code)) script = script " (R-to-L)"

    split(getISO(code), group, "-")
    iso = group[1]

    name = getName(code)
    names = getNames(code)
    if (match(name, /\(.*\)/)) {
        writing = substr(name, match(name, /\(.*\)/) + 1)
        writing = substr(writing, 1, length(writing) - 1)
        name = substr(name, 1, match(name, /\(.*\)/) - 2)
    }

    if (getBranch(code)) {
        article = match(tolower(getBranch(code)), /^[aeiou]/) ? "an" : "a"
        if (iso == "eng")
            desc = sprintf("%s is %s %s language spoken %s.",
                           names, article, getBranch(code), getSpokenIn(code))
        else
            desc = sprintf("%s is %s %s language spoken mainly in %s.",
                           names, article, getBranch(code), getSpokenIn(code))
    } else if (getFamily(code) == NULLSTR || tolower(getFamily(code)) == "language isolate")
        desc = sprintf("%s is a language spoken mainly in %s.", names, getSpokenIn(code))
    else if (tolower(getFamily(code)) == "constructed language")
        desc = sprintf("%s is %s.", names, getDescription(code))
    else
        desc = sprintf("%s is a language of the %s family, spoken mainly in %s.",
                       names, getFamily(code), getSpokenIn(code))
    if (writing && getWrittenIn(code))
        desc = desc sprintf(" The %s writing system is officially used in %s.",
                            tolower(writing), getWrittenIn(code))

    return ansi("bold", sprintf("%s\n", getDisplay(code)))              \
        sprintf("%-22s%s\n", "Name", ansi("bold", names))               \
        sprintf("%-22s%s\n", "Family", ansi("bold", getFamily(code)))   \
        sprintf("%-22s%s\n", "Writing system", ansi("bold", script))    \
        sprintf("%-22s%s\n", "Code", ansi("bold", getCode(code)))       \
        sprintf("%-22s%s\n", "ISO 639-3", ansi("bold", iso))            \
        sprintf("%-22s%s\n", "SIL", ansi("bold", "https://iso639-3.sil.org/code/" iso)) \
        sprintf("%-22s%s\n", "Glottolog", getGlotto(code) ?
                ansi("bold", "https://glottolog.org/resource/languoid/id/" getGlotto(code)) : "") \
        sprintf("%-22s%s\n", "Wikipedia", ansi("bold", "https://en.wikipedia.org/wiki/ISO_639:" iso)) \
        (Locale[getCode(code)]["supported-by"] ? # FIXME
        sprintf("%-22s%s\n", "Translator support", sprintf("Google [%s]    Bing [%s]",
                isSupportedByGoogle(code) ? "✔" : "✘", isSupportedByBing(code) ? "✔" : "✘")) : "") \
        (Locale[getCode(code)]["spoken-in"] ? # FIXME
        ansi("bold", sprintf("\n%s", desc)) : "")
}

# Add /slashes/ for IPA phonemic notations and (parentheses) for others.
# Parameters:
#     code
function showPhonetics(phonetics, code) {
    if (code && getCode(code) == "en")
        return "/" phonetics "/"
    else
        return "(" phonetics ")"
}

# Convert a logical string to visual; don't right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
function show(text, code,    command, temp) {
    if (!code || isRTL(code)) {
        if (Cache[text][0])
            return Cache[text][0]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDiNoPad) {
                command = "echo " parameterize(text) PIPE BiDiNoPad
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][0] = temp
        }
    } else
        return text
}

# Convert a logical string to visual and right justify RTL lines.
# Parameters:
#     code: ignore to apply bidirectional algorithm on every string
#     width: ignore to use default width for padding
function s(text, code, width,    command, temp) {
    if (!code || isRTL(code)) {
        if (!width) width = Option["width"]
        if (Cache[text][width])
            return Cache[text][width]
        else {
            if ((FriBidi || (code && isRTL(code))) && BiDi) {
                command = "echo " parameterize(text) PIPE sprintf(BiDi, width)
                command | getline temp
                close(command)
            } else # non-RTL language, or FriBidi not installed
                temp = text
            return Cache[text][width] = temp
        }
    } else
        return text
}

# Convert a logical string to visual with a certain level of indentation.
# Parameters:
#     level: level of indentation
#     code: ignore to apply left indentation
#     width: ignore to use default width for padding
function ins(level, text, code, width,    i, temp) {
    if (code && isRTL(code)) {
        if (!width) width = Option["width"]
        return s(text, code, width - Option["indent"] * level)
    } else
        return replicate(" ", Option["indent"] * level) text
}

# Parse a POSIX locale identifier and return the language code;
# Identified by both language identifier and region identifier.
# Parameters:
#     lang = [language[_territory][.codeset][@modifier]]
# See: <https://en.wikipedia.org/wiki/Locale>
function parseLang(lang,    code, group) {
    match(lang, /^([a-z][a-z][a-z]?)(_|$)/, group)
    code = getCode(group[1])

    # Detect region identifier
    ## Regions using Chinese Simplified: China, Singapore
    if (lang ~ /^zh_(CN|SG)/) code = "zh-CN"
    ## Regions using Chinese Traditional: Taiwan, Hong Kong
    else if (lang ~ /^zh_(TW|HK)/) code = "zh-TW"

    # FIXME: handle unrecognized language code
    if (!code) code = "en"

    return code
}

# Initialize `UserLang`.
function initUserLang(    lang, utf) {
    if (lang = ENVIRON["LC_ALL"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (lang = ENVIRON["LANG"]) {
        if (!UserLocale) UserLocale = lang
        utf = utf || tolower(lang) ~ /utf-?8$/
    }
    if (!UserLocale) {
        UserLocale = "en_US.UTF-8"
        utf = 1
    }
    if (!utf)
        w("[WARNING] Your locale codeset (" UserLocale ") is not UTF-8.")

    UserLang = parseLang(UserLocale)
}


Parse Condor StarterLog

Parse Condor StarterLog

BEGIN {
  year=strftime("%Y");
  ended_jobs=0;
  total_used=0;
  goodputZ=0;
  goodputNZ=0;
  badputSignal=0;
  badputOther=0;
  goodjobsZ=0;
  goodjobsNZ=0;
  badjobsSignal=0;
  badjobsOther=0;
}

# year is a 4 digit string or number
# date is month/day
# time is hour:minute:seconds
function convert_date(year,date,time) {
  split(date,datearr,"/");
  split(time,timearr,":");
  timestr=sprintf("%04d %02d %02d %02d %02d %02d",year,datearr[1],datearr[2],timearr[1],timearr[2],timearr[3]);
  return mktime(timestr)
}

function percent(a,b) {
  if (b==0) {
    return 0;
  } else {
    return a*100/b;
  }
}

########################################################################
/ Communicating with shadow /{
  # 7/17 10:42:20 (pid:26855) Communicating with shadow <131.225.204.208:34839>
  #
  for (i=1; i<NF; i++) {
    if ($i=="Communicating") {
      changenr=i;
      break;
    }
  }
  shadow=$(changenr+3);
  shadow=substr(shadow,3,length(shadow)-4)
  #print  $1,$2,"Shadow:",shadow
}
/ Starting .* job with ID /{
  # 7/17 10:42:21 (pid:26855) Starting a VANILLA universe job with ID: 40.0
  #
  for (i=1; i<NF; i++) {
    if ($i=="ID:") {
      changenr=i;
      break;
    }
  }
  jid=$(changenr+1);
  #print  $1,$2,"Jid:",jid
}
/ Create_Process succeeded, /{
  # 7/17 10:42:21 (pid:26855) Create_Process succeeded, pid=26858
  #
  for (i=1; i<NF; i++) {
    if ($i=="Create_Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  split(pidstr,pidarr,"=");
  pid=pidarr[2];
  #print  $1,$2,"PID:",pid

  starttime=convert_date(year,$1,$2);
  print $1,$2,"Starting job",jid,"from",shadow
}
/ Process exited, /{
  # 7/17 10:50:31 (pid:26855) Process exited, pid=26858, signal=9
  # 7/17 11:23:04 (pid:23112) Process exited, pid=23114, status=0
  #
  for (i=1; i<NF; i++) {
    if ($i=="Process") {
      changenr=i;
      break;
    }
  }
  pidstr=$(changenr+2);
  reasonstr=$(changenr+3);

  split(reasonstr,reasonarr,"=");

  ended=convert_date(year,$1,$2);
  if (ended<starttime) { # around new year
    ended=convert_date(year+1,$1,$2);
  }
  joblength=ended-starttime;

  ended_jobs++;
  total_used+=joblength;

  print $1,$2,"Terminated job",jid,"from",shadow,reasonarr[1],reasonarr[2],"duration",joblength
  if (reasonarr[1]=="status") {
    if (reasonarr[2]=="0") {
      goodputZ+=joblength;
      goodjobsZ++;
    } else {
      goodputNZ+=joblength;
      goodjobsNZ++;
    }
  } else if (reasonarr[1]=="signal") {
    badputSignal+=joblength;
    badjobsSignal++
  } else {
    badputOther+=joblength;
    babjobsOther++;
  }

  # cleanup, so it is not reused by accident
  jid="";
  shadow="";
}

########################################################################
END {
  print "=================";
  print "Total jobs",ended_jobs,"utilization",total_used;
  print "Total goodZ jobs",goodjobsZ " (" percent(goodjobsZ,ended_jobs) "%)","utilization",goodputZ " (" percent(goodputZ,total_used) "%)";
  print "Total goodNZ jobs",goodjobsNZ " (" percent(goodjobsNZ,ended_jobs) "%)","utilization",goodputNZ " (" percent(goodputNZ,total_used) "%)";
  print "Total badSignal jobs",badjobsSignal " (" percent(badjobsSignal,ended_jobs) "%)","utilization",badputSignal " (" percent(badputSignal,total_used) "%)";
  print "Total badOther jobs",badjobsOther " (" percent(badjobsOther,ended_jobs) "%)","utilization",badputOther " (" percent(badputOther,total_used) "%)";
}

error: pathspec 'creation/web_base/parse_starterlog.awk' did not match any file(s) known to git