Add missing FPAT from last commit

Add missing FPAT from last commit

Forgot to add the FPAT print out in the last commit.

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
}

# RENDER VALUE CHECKS
$1 == "field" {
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }

    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
}

# RENDER VALUE CHECKS
$1 == "field" {
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }

    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


add title, predicate prefix, exceptions, notes

#! /usr/bin/awk -f

#  Reduce the subject and object of RDF triples (ntriples format)
# down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier and
# the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
# labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
# a handy way to differentiate uri for subjects and objects
# which may belong to a "structural" ontology as opposed to
# a data uri.

#  Perhaps an improvement would be to to also
# express subject and objects as individual curies (ala predicates)
# iff their namespace is also used by a predicate.

##########################################################
# remove first and last chars from input
function trim(str){
	return substr(str,2,length(str)-2)
}

# remove final element in slashed path
# this leaves the namespace of the removed identifier
function stripid(uri){
	l = match(uri,/.*\//)  # side effect is RLENGTH
	if(l<=0)
		return uri
	else
		return substr(uri,1,RLENGTH-1)
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	if(uri in prefix)
		return prefix[uri]
	else
		for(ex in exception){
			start = match(uri, ex)
			if(0 < start)
				return exception[substr(uri, start, RLENGTH)]
		}
		return simplify(uri)
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# Monarchinitiave's dipper ingest files have cases
	# which escape the general rules
	# in everything
	exception["http://www.w3.org/2002/07"]="OWL_"
	exception["http://www.w3.org/1999/02"]="RDF"
	exception["http://www.w3.org/2000/01"]="RDFS"
	# in go,mgi,wormbase
	exception["http://dx.doi.org"]="DOI"
	# in mgi
	exception["https://www.mousephenotype.org"]="IMPC_"
	# in panther
	exception["http://identifiers.org/wormbase"]="WormBase"
	# just until skolemized bnodes get in the curie map?
	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
	# till all non httpS: are purged
	exception["http://monarchinitiave.org"]="MONARCH"
}

# main loop
# parse and stash the curie file (first file)
(FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ {
	split($0,a,"'")
	prefix[stripid(a[4])]=a[2]
}
# process the ntriple file(s)  which are not the first file
### case when subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	q = final(trim($2))
	p = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist[s " -> " o " [label=\"" p ":" q]++
}
### case when the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	q = final(trim($2))
	p = contract(stripid(trim($2)))
	### not a uri
	o = "LITERAL"
	edgelist[s " -> " o " [label=\"" p":"q ]++
	nodelist[o " [shape=record];"]++
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist) print edge " (" edgelist[edge] ")\"];"
	for(node in nodelist) print node
	print "labelloc=\"t\";"
	title = final(FILENAME)
	now = systime()
	# return date(1)-style output
	datestamp = strftime("%Y%m%d", now)
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}

#! /usr/bin/awk -f

# remove first and last chars
function trim(str){
	return substr(str,2,length(str)-2)
}

# remove final element in slashed path
function stripid(uri){
	l = match(uri,/.*\//)  # side effect is RLENGTH
	if(l<=0)
		return uri
	else
		return substr(uri,1,RLENGTH -1)
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	if(uri in prefix)
		return prefix[uri]
	else
		for(ex in exception){
			start = match(uri, ex)
			if(0 < start)
				return exception[substr(uri, start, RLENGTH)]
		}
		return simplify(uri)
}

# get the final (fragment identifier) portion of a slashed path
function final(uri){
	split(trim(uri),b,"/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# Monarchinitiave's dipper ingest files have cases
	# which escape the general rules
	# in everything
	exception["http://www.w3.org/2002/07"]="OWL_"
	# in go,mgi,wormbase
	exception["http://dx.doi.org"]="DOI"
	# in mgi
	exception["https://www.mousephenotype.org"]="IMPC_"
	# in panther
	exception["http://identifiers.org/wormbase"]="WormBase"
	# just until skolemized bnodes get in the curie map?
	exception["https://monarchinitiave.org/.well-known/genid"]="BNODE"
}

# main loop
# parse and stash the curie file (first file)
(FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ {
	split($0,a,"'")
	prefix[stripid(a[4])]=a[2]
}
# process the ntriple file(s)  which are not the first file
### case when subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final($2)
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist[s " -> " o " [label=\"" p ]++
}
### case when the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*" \.$/ {
	### Subject (uri)
	s = contract(stripid(trim($1)))
	### Predicate (uri)
	p = final($2)
	### not a uri
	o = "LITERAL"
	edgelist[s " -> " o " [label=\"" p ]++
	nodelist[o " [shape=record];"]++
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist) print edge " (" edgelist[edge] ")\"];"
	for(node in nodelist) print node
	print "}"
}


updates table related meta to be file based rules

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action == \"validate\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
}

$1 == "option" {
    options[$2]=$3;
}

$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
}

$1 == "file" {
    file_options[$2]=$3;
}

$1 == "field" {
    defaults["mode"]="append"
    defaults["summary"]="true"
    defaults["dcat"]="warning"
    defaults["dvalnum"]=-9999
    defaults["dvalstr"]="NA"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
    rule_type=$2
    split($3,params," ")
    field=params[1]
    cat=params[3]
    field_index=header_index[field]
    test=""
    msg=""

    if (rule_type == "type") {
        type=params[2]
        if (type == "numeric"){
            test=field " && !is_numeric(" field ")"
            msg="Field " field " in \" FILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        test=field " == \"\""
        msg="Field " field " in \" FILENAME \" line \" NR \" is required"
    } else if (rule_type == "unique") {
        test="!is_unique(" field ")"
        msg="Field " field " in \" FILENAME \" line \" NR \" is a duplicate and should be unique"
    } else if (rule_type ~ /^min|max$/) {
        comparator=$2 == "max" ? ">" : "<"
        limit=params[2]
        term=$2 == "max" ? "less" : "greater"

        test=field " " comparator " " limit
        msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
    } else if (rule_type == "pattern") {
        pattern=params[2]

        test=field " != \"\" && " field " !~ " pattern
        msg=field " in line \" NR \" should match the following pattern " pattern " and was \" " field " \" "
    }

    if (!options["mode"]) options["mode"] = "text"
    if (!options["dcat"]) options["dcat"] = "na"
    if (!cat) cat = options["dcat"]
    handler_prefix="{ log_err(\"" cat "\"); "
    if (options["mode"] == "text") {
        err_handler= handler_prefix "print \"" msg "\" RS $0 RS; }"
    } else if (options["mode"] == "wrap") {
        acc="$" field_index
        err_handler= handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"
    } else if (options["mode"] == "append") {
        err_handler= handler_prefix "print $0 FS \"" msg "\" }"
    }

    if (rule_type !~ /^(none|colmap)$/) { print "action == \"validate\" && NR > 1 && " test " " err_handler " " }
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]

    if (rule_type ~ /^(min|max)$/ || (rule_type == "type" && params[2] == "numeric")) {
        types[field] = "numeric"
    } else if (rule_type != "colmap") {
        types[field] = "text"
    }

    if(rule_type == "colmap") mappings[field]=params[2]
}

END {
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in types) {
        type=types[field]
        field_index=header_index[field]
        dval= type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    gsub(\"Range\", \"rangeno\");"
    print "    print \"COPY " file_options["table"] " (\" $0 \") FROM stdin;\""
    print "}"

    print "action == \"insert\" && NR > 1 {"
    print "    gsub(\",\", \"\t\");"
    print "    print;"
    print "}"

    print "action == \"table\" && NR == 1 {"

    fields=""
    l=length(headers)
    for (i in headers) {
        c++
        field=headers[i]
        type=types[field]
        if (length(mappings) && mappings[field]) field=mappings[field]
        fields=fields field " " type
        if (c < l) fields=fields ","
    }

    if(file_options["pkey"]) {
        cols=file_options["pkey"]
        gsub(":", ",", cols)
        fields=fields ", PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable=ref[1]
        fcols=ref[2]
        gsub(":", ",", fcols)
        fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
    }

    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }" RS

    print "# la fin"
    print "END {"
    print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action == \"validate\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
}

$1 == "option" {
    options[$2]=$3;
}

$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 { headers=\"" $3 "\"; if (!are_headers_valid(headers)) print \"invalid headers\" }"
}

$1 == "field" {
    defaults["mode"]="append"
    defaults["summary"]="true"
    defaults["dcat"]="warning"
    defaults["dvalnum"]=-9999
    defaults["dvalstr"]="NA"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
    rule_type=$2
    split($3,params," ")
    field=params[1]
    cat=params[3]
    field_index=header_index[field]
    test=""
    msg=""

    if (rule_type == "type") {
        type=params[2]
        if (type == "numeric"){
            test=field " && !is_numeric(" field ")"
            msg="Field " field " in \" FILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        test=field " == \"\""
        msg="Field " field " in \" FILENAME \" line \" NR \" is required"
    } else if (rule_type == "unique") {
        test="!is_unique(" field ")"
        msg="Field " field " in \" FILENAME \" line \" NR \" is a duplicate and should be unique"
    } else if (rule_type ~ /^min|max$/) {
        comparator=$2 == "max" ? ">" : "<"
        limit=params[2]
        term=$2 == "max" ? "less" : "greater"

        test=field " " comparator " " limit
        msg=field " in line \" NR \" should be " term " than " limit " and was \" " field " \" "
    } else if (rule_type == "pattern") {
        pattern=params[2]

        test=field " != \"\" && " field " !~ " pattern
        msg=field " in line \" NR \" should match the following pattern " pattern " and was \" " field " \" "
    }

    if (!options["mode"]) options["mode"] = "text"
    if (!options["dcat"]) options["dcat"] = "na"
    if (!cat) cat = options["dcat"]
    handler_prefix="{ log_err(\"" cat "\"); "
    if (options["mode"] == "text") {
        err_handler= handler_prefix "print \"" msg "\" RS $0 RS; }"
    } else if (options["mode"] == "wrap") {
        acc="$" field_index
        err_handler= handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"
    } else if (options["mode"] == "append") {
        err_handler= handler_prefix "print $0 FS \"" msg "\" }"
    }

    if (rule_type !~ /^(none|colmap)$/) { print "action == \"validate\" && NR > 1 && " test " " err_handler " " }
}

$1 == "field" {
    rule_type=$2
    split($3,params," ")
    field=params[1]

    if (rule_type ~ /^(min|max)$/ || (rule_type == "type" && params[2] == "numeric")) {
        types[field] = "numeric"
    } else if (rule_type != "colmap") {
        types[field] = "text"
    }

    if(rule_type == "colmap") mappings[field]=params[2]
}

END {
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in types) {
        type=types[field]
        field_index=header_index[field]
        dval= type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    gsub(\"Range\", \"rangeno\");"
    print "    print \"COPY " options["table"] " (\" $0 \") FROM stdin;\""
    print "}"

    print "action == \"insert\" && NR > 1 {"
    print "    gsub(\",\", \"\t\");"
    print "    print;"
    print "}"

    print "action == \"table\" && NR == 1 {"

    fields=""
    l=length(headers)
    for (i in headers) {
        c++
        field=headers[i]
        type=types[field]
        if (length(mappings) && mappings[field]) field=mappings[field]
        fields=fields field " " type
        if (c < l) fields=fields ","
    }

    if(options["pkey"]) {
        cols=options["pkey"]
        gsub(":", ",", cols)
        fields=fields ", PRIMARY KEY (" cols ") "
    }
    if(options["fkey"]) {
        split(options["fkey"], ref, " ")
        ftable=ref[1]
        fcols=ref[2]
        gsub(":", ",", fcols)
        fields=fields ", FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
    }

    print "     print \"CREATE TABLE IF NOT EXISTS " options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }" RS

    print "# la fin"
    print "END {"
    print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


dtx_check.gawk: Fix false-positives in DTX check

Previously, if ONSET happened exactly at the place where next SID FIRST
or UPDATE should be it was incorrectly detected as error. Similarly,
continuos FACCH was misinterpreted as error.

Change-Id: I43fdbceea0dbdb0833c3b1cf0fc3b825803ed30e

#!/usr/bin/gawk -f

# Expected input format: FN TYPE

BEGIN {
	DELTA = 0
	ERR = 0
	FORCE = 0
	FN = 0
	SILENCE = 0
	TYPE = ""
	CHK = ""
	U_MAX = 8 * 20 + 120 / 26
	U_MIN = 8 * 20 - 120 / 26
	F_MAX = 3 * 20 + 120 / 26
	F_MIN = 3 * 20 - 120 / 26
}

{
	if (NR > 2) { # we have data from previous record to compare to
		DELTA = ($1 - FN) * 120 / 26
		CHK = "OK"
		if ("FACCH" == $2 && "ONSET" == TYPE) { # ONSET due to FACCH is NOT a talkspurt
			SILENCE = 1
		}
		if (("UPDATE" == TYPE || "FIRST" == TYPE) && ("FACCH" == $2 || "SPEECH" == $2)) { # check for missing ONSET:
				CHK = "FAIL: missing ONSET (" $2 ") after " TYPE "."
				ERR++
		}
		if ("SID_P1" == $2) {
			CHK = "FAIL: regular AMR payload with FT SID and STI=0 (should be either pyaload Update or STI=1)."
			ERR++
		}
		if ("FORCED_FIRST" == $2 || "FORCED_NODATA" == $2 || "FORCED_F_P2" == $2 || "FORCED_F_INH" == $2 || "FORCED_U_INH" == $2) {
			CHK = "FAIL: event " $2 " inserted by DSP."
			FORCE++
			ERR++
		}
		if ("FIRST_P2" != $2 && "FIRST_P1" == TYPE) {
			CHK = "FAIL: " TYPE " followed by " $2 " instead of P2."
			ERR++
		}
		if ("OK" == CHK && "ONSET" != $2) { # check inter-SID distances:
			if ("UPDATE" == TYPE) {
				if (DELTA > U_MAX) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too big " DELTA "ms > " U_MAX "ms."
					ERR++
				}
				if ("UPDATE" == $2 && DELTA < U_MIN) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too small " DELTA "ms < " U_MIN "ms."
					ERR++
				}
			}
			if ("FIRST" == TYPE) {
				if (DELTA > F_MAX) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID FIRST (@" FN ") too big " DELTA "ms > " F_MAX "ms."
					ERR++
				}
				if ("UPDATE" == $2 && DELTA < F_MIN) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too small " DELTA "ms < " F_MIN "ms."
					ERR++
				}
			}
		}
		if ("FACCH" == TYPE && "FIRST" != $2 && "FACCH" != $2 && 1 == SILENCE) { # check FACCH handling
			CHK = "FAIL: incorrect silence resume with " $2 " after FACCH."
			ERR++
		}
	}
	if ("SPEECH" == $2 || "ONSET" == $2) { # talkspurt
		SILENCE = 0
	}
	if ("UPDATE" == $2 || "FIRST" == $2) { # silence
		SILENCE = 1
	}
	print $1, $2, CHK
	if ($2 != "EMPTY") { # skip over EMPTY records
		TYPE = $2
		FN = $1
	}
}

END {
	print "Check completed: found " ERR " errors (" FORCE " events inserted by DSP) in " NR " records."
}

#!/usr/bin/gawk -f

# Expected input format: FN TYPE

BEGIN {
	DELTA = 0
	ERR = 0
	FORCE = 0
	FN = 0
	SILENCE = 0
	TYPE = ""
	CHK = ""
	U_MAX = 8 * 20 + 120 / 26
	U_MIN = 8 * 20 - 120 / 26
	F_MAX = 3 * 20 + 120 / 26
	F_MIN = 3 * 20 - 120 / 26
}

{
	if (NR > 2) { # we have data from previous record to compare to
		DELTA = ($1 - FN) * 120 / 26
		CHK = "OK"
		if ("FACCH" == $2 && "ONSET" == TYPE) { # ONSET due to FACCH is NOT a talkspurt
			SILENCE = 1
		}
		if (("UPDATE" == TYPE || "FIRST" == TYPE) && ("FACCH" == $2 || "SPEECH" == $2)) { # check for missing ONSET:
				CHK = "FAIL: missing ONSET (" $2 ") after " TYPE "."
				ERR++
		}
		if ("SID_P1" == $2) {
			CHK = "FAIL: regular AMR payload with FT SID and STI=0 (should be either pyaload Update or STI=1)."
			ERR++
		}
		if ("FORCED_FIRST" == $2 || "FORCED_NODATA" == $2 || "FORCED_F_P2" == $2 || "FORCED_F_INH" == $2 || "FORCED_U_INH" == $2) {
			CHK = "FAIL: event " $2 " inserted by DSP."
			FORCE++
			ERR++
		}
		if ("FIRST_P2" != $2 && "FIRST_P1" == TYPE) {
			CHK = "FAIL: " TYPE " followed by " $2 " instead of P2."
			ERR++
		}
		if ("OK" == CHK) { # check inter-SID distances:
			if ("UPDATE" == TYPE) {
				if (DELTA > U_MAX) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too big " DELTA "ms > " U_MAX "ms."
					ERR++
				}
				if ("UPDATE" == $2 && DELTA < U_MIN) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too small " DELTA "ms < " U_MIN "ms."
					ERR++
				}
			}
			if ("FIRST" == TYPE) {
				if (DELTA > F_MAX) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID FIRST (@" FN ") too big " DELTA "ms > " F_MAX "ms."
					ERR++
				}
				if ("UPDATE" == $2 && DELTA < F_MIN) {
					CHK = "FAIL: delta (" $1 - FN "fn) from previous SID UPDATE (@" FN ") too small " DELTA "ms < " F_MIN "ms."
					ERR++
				}
			}
		}
		if ("FACCH" == TYPE && "FIRST" != $2 && 1 == SILENCE) { # check FACCH handling
			CHK = "FAIL: incorrect silence resume after FACCH."
			ERR++
		}
	}
	if ("SPEECH" == $2 || "ONSET" == $2) { # talkspurt
		SILENCE = 0
	}
	if ("UPDATE" == $2 || "FIRST" == $2) { # silence
		SILENCE = 1
	}
	print $1, $2, CHK
	if ($2 != "EMPTY") { # skip over EMPTY records
		TYPE = $2
		FN = $1
	}
}

END {
	print "Check completed: found " ERR " errors (" FORCE " events inserted by DSP) in " NR " records."
}


Add support for commenting the Python constants. (dm)

git-svn-id: 30a5f035a20f1bc647618dbad7eea2a951b61b7c@2829 91a5dbb7-01b9-0310-9b5f-b28072856b6e

###############################################################################
# BRLTTY - A background process providing access to the console screen (when in
#          text mode) for a blind person using a refreshable braille display.
#
# Copyright (C) 1995-2006 by The BRLTTY Developers.
#
# BRLTTY comes with ABSOLUTELY NO WARRANTY.
#
# This is free software, placed under the terms of the
# GNU General Public License, as published by the Free Software
# Foundation.  Please see the file COPYING for details.
#
# Web Page: http://mielke.cc/brltty/
#
# This software is maintained by Dave Mielke <dave@mielke.cc>.
###############################################################################

function brlCommand(name, symbol, value, help) {
  writeCommandDefinition(name, value, help)
}

function brlBlock(name, symbol, value, help) {
  if (name == "PASSCHAR") return
  if (name == "PASSKEY") return

  if (value ~ /^0[xX][0-9a-fA-F]+00$/) {
    writeCommandDefinition(name, tolower(value) "00", help)
  }
}

function brlKey(name, symbol, value, help) {
}

function brlFlag(name, symbol, value, help) {
  if (value ~ /^0[xX][0-9a-fA-F]+0000$/) {
    value = tolower(substr(value, 1, length(value)-4))
    if (name ~ /^CHAR_/) {
      name = substr(name, 6)
    } else {
      value = value "00"
    }
  } else if (value ~ /^\(/) {
    gsub("BRL_FLG_", "KEY_FLG_", value)
  } else {
    return
  }
  writePythonAssignment("KEY_FLG_" name, value, help)
}

function brlDot(number, symbol, value, help) {
  writePythonAssignment("DOT" number, value, help)
}

function apiConstant(name, symbol, value, help) {
  writePythonAssignment(name, value, help)
}

function apiMask(name, symbol, value, help) {
  writePythonAssignment("KEY_" name, "c_brlapi." symbol, help)
}

function apiShift(name, symbol, value, help) {
  value = hexadecimalValue(value)
  writePythonAssignment("KEY_" name, value, help)
}

function apiType(name, symbol, value, help) {
  value = hexadecimalValue(value)
  writePythonAssignment("KEY_TYPE_" name, value, help)
}

function apiKey(name, symbol, value, help) {
  value = hexadecimalValue(value)

  if (name == "FUNCTION") {
    for (i=0; i<35; ++i) {
      key = "F" (i+1)
      writeKeyDefinition(key, "(" value " + " i ")", "the " key " key")
    }
  } else {
    writeKeyDefinition(name, value, help)
  }
}

function writeCommandDefinition(name, value, help) {
  writePythonAssignment("KEY_CMD_" name, value, help)
}

function writeKeyDefinition(name, value, help) {
  writePythonAssignment("KEY_SYM_" name, value, help)
}

function writePythonAssignment(name, value, help) {
  if (length(help) > 0) print "## " help
  print name " = " value
}

function hexadecimalValue(value) {
  value = tolower(value)
  return value
}

###############################################################################
# BRLTTY - A background process providing access to the console screen (when in
#          text mode) for a blind person using a refreshable braille display.
#
# Copyright (C) 1995-2006 by The BRLTTY Developers.
#
# BRLTTY comes with ABSOLUTELY NO WARRANTY.
#
# This is free software, placed under the terms of the
# GNU General Public License, as published by the Free Software
# Foundation.  Please see the file COPYING for details.
#
# Web Page: http://mielke.cc/brltty/
#
# This software is maintained by Dave Mielke <dave@mielke.cc>.
###############################################################################

function brlCommand(name, symbol, value, help) {
  writeCommandDefinition(name, value)
}

function brlBlock(name, symbol, value, help) {
  if (name == "PASSCHAR") return
  if (name == "PASSKEY") return

  if (value ~ /^0[xX][0-9a-fA-F]+00$/) {
    writeCommandDefinition(name, tolower(value) "00")
  }
}

function brlKey(name, symbol, value, help) {
}

function brlFlag(name, symbol, value, help) {
  if (value ~ /^0[xX][0-9a-fA-F]+0000$/) {
    value = tolower(substr(value, 1, length(value)-4))
    if (name ~ /^CHAR_/) {
      name = substr(name, 6)
    } else {
      value = value "00"
    }
  } else if (value ~ /^\(/) {
    gsub("BRL_FLG_", "KEY_FLG_", value)
  } else {
    return
  }
  writePythonAssignment("KEY_FLG_" name, value)
}

function brlDot(number, symbol, value, help) {
  writePythonAssignment("DOT" number, value)
}

function apiConstant(name, symbol, value, help) {
  writePythonAssignment(name, value)
}

function apiMask(name, symbol, value, help) {
  writePythonAssignment("KEY_" name, "c_brlapi." symbol)
}

function apiShift(name, symbol, value, help) {
  value = hexadecimalValue(value)
  writePythonAssignment("KEY_" name, value)
}

function apiType(name, symbol, value, help) {
  value = hexadecimalValue(value)
  writePythonAssignment("KEY_TYPE_" name, value)
}

function apiKey(name, symbol, value, help) {
  value = hexadecimalValue(value)

  if (name == "FUNCTION") {
    for (i=0; i<35; ++i) {
      writeKeyDefinition("F" (i+1), "(" value " + " i ")")
    }
  } else {
    writeKeyDefinition(name, value)
  }
}

function writeCommandDefinition(name, value) {
  writePythonAssignment("KEY_CMD_" name, value)
}

function writeKeyDefinition(name, value) {
  writePythonAssignment("KEY_SYM_" name, value)
}

function writePythonAssignment(name, value) {
  print name " = " value
}

function hexadecimalValue(value) {
  value = tolower(value)
  return value
}


build.awk: update readme generator

build.awk: update readme generator

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/LanguageData.awk"
@include "include/LanguageHelper.awk"
@include "metainfo.awk"

function init() {
    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -f markdown-smart -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -f markdown-smart -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, iso, j, num, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getISO(cols[j][i]), group, "-")
                iso = group[1]

                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/ISO_639:" iso \
                    ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    #print "***" length(Locale) "*** *languages in total. "
    print "*Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "*Version: [English](https://github.com/soimort/translate-shell/wiki/Languages) " \
        "| [Chinese Simplified](https://github.com/soimort/translate-shell/wiki/Languages-%28%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87%29)*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        # Ignore unstable languages and non-languages
        if (Locale[code]["support"] == "unstable") continue
        if (Locale[code]["status"] == "non-language") continue

        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = length(group) == 1 ? group[1] "_language" :
            group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language, getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function readSqueezed(fileName, squeezed,    group, line, ret) {
    if (fileName ~ /\*$/ || fileName ~ /_$/) # glob simulation
        return readSqueezed(fileName ".awk", squeezed)

    ret = NULLSTR
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (group[1] ~ /\.awk$/)
                    append(Includes, group[1])

                if (ret) ret = ret RS
                ret = ret readSqueezed(group[1], squeezed)
            } else if (!squeezed || line = squeeze(line)) { # effective LOC
                if (ret) ret = ret RS
                ret = ret line
            }
        }
    return ret
}

function build(target, type,    i, group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans

        if (fileExists("DISCLAIMER")) {
            print "#" > Trans
            while (getline line < "DISCLAIMER")
                print "# " line > Trans
            print "#" > Trans
        }

        print "export TRANS_ENTRY=\"$0\"" > Trans
        print "if [[ ! $LANG =~ (UTF|utf)-?8$ ]]; then export LANG=en_US.UTF-8; fi" > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        print readSqueezed(EntryPoint, TRUE) > Trans
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline

        # Rebuild EntryScript
        print "#!/bin/sh" > EntryScript
        print "export TRANS_DIR=`dirname $0`" > EntryScript
        print "gawk \\" > EntryScript
        for (i = 0; i < length(Includes) - 1; i++)
            print "-i \"${TRANS_DIR}/" Includes[i] "\" \\" > EntryScript
        print "-f \"${TRANS_DIR}/" Includes[i] "\" -- \"$@\"" > EntryScript
        ("chmod +x " parameterize(EntryScript)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp
        print (temp == "Darwin" ?
               "#!/usr/bin/env gawk -f" : # macOS
               "#!/usr/bin/gawk -f") > TransAwk

        print readSqueezed(EntryPoint, TRUE) > TransAwk

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}

#!/usr/bin/gawk -f

# Not all 4.x versions of gawk can handle @include without ".awk" extension
# But the build.awk script and the single build should support gawk 4.0+.
@include "include/Commons.awk"
@include "include/Utils.awk"
@include "include/LanguageData.awk"
@include "include/LanguageHelper.awk"
@include "metainfo.awk"

function init() {
    BuildPath            = "build/"
    Trans                = BuildPath Command
    TransAwk             = Trans ".awk"

    ManPath              = "man/"
    Man                  = ManPath Command ".1"
    ManTemplate          = Man ".template.md"
    ManMarkdown          = Man ".md"
    ManHtmlTemplate      = Man ".template.html"
    ManHtml              = Man ".html"

    PagesPath            = "gh-pages/"
    BadgeDownload        = PagesPath "images/badge-download"
    BadgeRelease         = PagesPath "images/badge-release"
    Index                = PagesPath "index.md"

    ReadmePath           = "./"
    ReadmeTemplate       = ReadmePath "README.template.md"
    Readme               = ReadmePath "README.md"

    WikiPath             = "wiki/"
    WikiHome             = WikiPath "Home.md"
    WikiLanguages        = WikiPath "Languages.md"
    WikiLanguagesHtml    = WikiLanguages ".html"

    RegistryPath         = "registry/"
    MainRegistryTemplate = RegistryPath "index.template.trans"
    MainRegistry         = RegistryPath "index.trans"
}

function man(    text) {
    text = readFrom(ManTemplate)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$ReleaseDate\$/, ReleaseDate, text)
    writeTo(text, ManMarkdown)

    if (fileExists(ManHtmlTemplate))
        system("pandoc -s -f markdown-smart -t html --toc --toc-depth 1 --template " ManHtmlTemplate " " ManMarkdown " -o " ManHtml)
    return system("pandoc -s -f markdown-smart -t man " ManMarkdown " -o " Man)
}

function readme(    code, col, cols, content, group, i, j, num, language, r, rows, text) {
    text = readFrom(ReadmeTemplate)

    content = getOutput("gawk -f translate.awk -- -no-ansi -h")
    gsub(/\$usage\$/, content, text)

    initBiDi(); initLocale()
    # number of language codes with stable support
    num = 0
    for (code in Locale)
        if (Locale[code]["support"] != "unstable")
            num++
    rows = int(num / 3) + (num % 3 ? 1 : 0)
    cols[0][0] = cols[1][0] = cols[2][0] = NULLSTR
    i = 0
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "compName"
    for (code in Locale) {
        # Ignore unstable languages
        if (Locale[code]["support"] == "unstable") continue

        col = int(i / rows)
        append(cols[col], code)
        i++
    }
    PROCINFO["sorted_in"] = saveSortedIn
    r = "| Language | Code | Language | Code | Language | Code |" RS \
        "| :------: | :--: | :------: | :--: | :------: | :--: |" RS
    for (i = 0; i < rows; i++) {
        r = r "| "
        for (j = 0; j < 3; j++)
            if (cols[j][i]) {
                split(getName(cols[j][i]), group, " ")
                language = length(group) == 1 ? group[1] "_language" : join(group, "_")
                r = r "**[" getName(cols[j][i]) "](" "http://en.wikipedia.org/wiki/" language ")**
 **" getEndonym(cols[j][i]) "** | **`" cols[j][i] "`** | "
            }
        r = r RS
    }
    gsub(/\$code-list\$/, r, text)

    content = readFrom(WikiHome)
    gsub(/\$wiki-home\$/, content, text)

    writeTo(text, Readme)
    return 0
}

function wiki(    code, group, iso, language, saveSortedIn) {
    initBiDi(); initLocale()

    #print "***" length(Locale) "*** *languages in total. "
    print "*Generated from the source code of Translate Shell " Version ".*\n" > WikiLanguages
    print "*Version: [English](https://github.com/soimort/translate-shell/wiki/Languages) " \
        "| [Chinese Simplified](https://github.com/soimort/translate-shell/wiki/Languages-%28%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87%29)*\n" > WikiLanguages
    print "| Code | Name | Family | [Writing system](https://github.com/soimort/translate-shell/wiki/Writing-Systems-and-Fonts) | Is [RTL](http://en.wikipedia.org/wiki/Right-to-left)? | Has dictionary? |" > WikiLanguages
    print "| :--: | ---: | -----: | :------------: | :---------------------------------------------------: | :-------------: |" > WikiLanguages
    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (code in Locale) {
        # Ignore unstable languages and non-languages
        if (Locale[code]["support"] == "unstable") continue
        if (Locale[code]["status"] == "non-language") continue

        split(getISO(code), group, "-")
        iso = group[1]
        split(getName(code), group, " ")
        language = length(group) == 1 ? group[1] "_language" :
            group[2] ~ /^\(.*\)$/ ? group[1] "_language" : join(group, "_")
        print sprintf("| **`%s`**
 [`%s`](%s) | **[%s](%s)**
 **%s** | %s | `%s` | %s | %s |",
                      getCode(code), iso, "http://www.ethnologue.com/language/" iso,
                      getName(code), "http://en.wikipedia.org/wiki/" language, getEndonym(code),
                      getFamily(code), getScript(code),
                      isRTL(code) ? "✓" : NULLSTR,
                      hasDictionary(code) ? "✓" : NULLSTR) > WikiLanguages
    }
    PROCINFO["sorted_in"] = saveSortedIn

    return system("pandoc -s -t html " WikiLanguages " -o " WikiLanguagesHtml)
}

function doc() {
    man()
    readme()
    wiki()
    return 0
}

function readSqueezed(fileName, squeezed,    group, line, ret) {
    if (fileName ~ /\*$/ || fileName ~ /_$/) # glob simulation
        return readSqueezed(fileName ".awk", squeezed)

    ret = NULLSTR
    if (fileExists(fileName))
        while (getline line < fileName) {
            match(line, /^[[:space:]]*@include[[:space:]]*"(.*)"$/, group)
            if (RSTART) { # @include
                if (group[1] ~ /\.awk$/)
                    append(Includes, group[1])

                if (ret) ret = ret RS
                ret = ret readSqueezed(group[1], squeezed)
            } else if (!squeezed || line = squeeze(line)) { # effective LOC
                if (ret) ret = ret RS
                ret = ret line
            }
        }
    return ret
}

function build(target, type,    i, group, inline, line, temp) {
    # Default target: bash
    if (!target) target = "bash"

    ("mkdir -p " parameterize(BuildPath)) | getline

    if (target == "bash" || target == "zsh") {

        print "#!/usr/bin/env " target > Trans

        if (fileExists("DISCLAIMER")) {
            print "#" > Trans
            while (getline line < "DISCLAIMER")
                print "# " line > Trans
            print "#" > Trans
        }

        print "export TRANS_ENTRY=\"$0\"" > Trans
        print "if [[ ! $LANG =~ (UTF|utf)-?8$ ]]; then export LANG=en_US.UTF-8; fi" > Trans

        print "read -r -d '' TRANS_PROGRAM << 'EOF'" > Trans
        print readSqueezed(EntryPoint, TRUE) > Trans
        print "EOF" > Trans

        print "read -r -d '' TRANS_MANPAGE << 'EOF'" > Trans
        if (fileExists(Man))
            while (getline line < Man)
                print line > Trans
        print "EOF" > Trans
        print "export TRANS_MANPAGE" > Trans

        if (type == "release")
            print "export TRANS_BUILD=release" temp > Trans
        else {
            temp = getGitHead()
            if (temp)
                print "export TRANS_BUILD=git:" temp > Trans
        }

        print "gawk -f <(echo -E \"$TRANS_PROGRAM\") - \"$@\"" > Trans

        ("chmod +x " parameterize(Trans)) | getline

        # Rebuild EntryScript
        print "#!/bin/sh" > EntryScript
        print "export TRANS_DIR=`dirname $0`" > EntryScript
        print "gawk \\" > EntryScript
        for (i = 0; i < length(Includes) - 1; i++)
            print "-i \"${TRANS_DIR}/" Includes[i] "\" \\" > EntryScript
        print "-f \"${TRANS_DIR}/" Includes[i] "\" -- \"$@\"" > EntryScript
        ("chmod +x " parameterize(EntryScript)) | getline
        return 0

    } else if (target == "awk" || target == "gawk") {

        "uname -s" | getline temp
        print (temp == "Darwin" ?
               "#!/usr/bin/env gawk -f" : # macOS
               "#!/usr/bin/gawk -f") > TransAwk

        print readSqueezed(EntryPoint, TRUE) > TransAwk

        ("chmod +x " parameterize(TransAwk)) | getline
        return 0

    } else {

        w("[FAILED] Unknown target: " ansi("underline", target))
        w("         Supported targets: "                                \
          ansi("underline", "bash") ", "                                \
          ansi("underline", "zsh") ", "                                 \
          ansi("underline", "gawk"))
        return 1

    }
}

function clean() {
    ("rm -f " BuildPath Command "*") | getline
    return 0
}

function release(    content, group, sha1, size, temp, text) {
    d("Updating registry ...")
    # Update registry
    text = readFrom(MainRegistryTemplate)
    gsub(/\$Version\$/, Version, text)
    writeTo(text, MainRegistry)

    d("Updating gh-pages/images ...")
    # Update gh-pages/images/badge-release
    text = readFrom(BadgeRelease ".temp")
    gsub(/\$Version\$/, Version, text)
    writeTo(text, BadgeRelease)
    system("save-to-png " BadgeRelease)
    # Update gh-pages/images/badge-download
    ("wc -c " Trans) | getline temp
    split(temp, group)
    size = int(group[1] / 1000)
    text = readFrom(BadgeDownload ".temp")
    gsub(/\$Size\$/, size, text)
    writeTo(text, BadgeDownload)
    system("save-to-png " BadgeDownload)

    d("Updating gh-pages/index.md ...")
    # Update gh-pages/index.md
    ("sha1sum " Trans) | getline temp
    split(temp, group)
    sha1 = group[1]
    content = readFrom(Readme)
    text = readFrom(Index ".temp")
    gsub(/\$sha1\$/, sha1, text)
    gsub(/\$Version\$/, Version, text)
    gsub(/\$readme\$/, content, text)
    writeTo(text, Index)

    return 0
}

function test() {
    return 0
}

BEGIN {
    init()

    pos = 0
    while (ARGV[++pos]) {
        # -target TARGET
        match(ARGV[pos], /^--?target(=(.*)?)?$/, group)
        if (RSTART) {
            target = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # -type TYPE
        match(ARGV[pos], /^--?type(=(.*)?)?$/, group)
        if (RSTART) {
            type = tolower(group[2] ? group[2] : ARGV[++pos])
            continue
        }

        # TASK
        match(ARGV[pos], /^[^\-]/, group)
        if (RSTART) {
            append(tasks, ARGV[pos])
            continue
        }
    }

    # Default task: build
    if (!anything(tasks)) tasks[0] = "build"

    for (i = 0; i < length(tasks); i++) {
        task = tasks[i]
        status = 0
        switch (task) {

        case "man":
            status = man()
            break

        case "readme":
            status = readme()
            break

        case "wiki":
            status = wiki()
            break

        case "doc":
            status = doc()
            break

        case "build":
            status = build(target, type)
            break

        case "clean":
            status = clean()
            break

        case "release":
            status = release()
            break

        case "test":
            status = test()
            break

        default: # unknown task
            status = -1
        }

        if (status == 0) {
            d("[OK] Task " ansi("bold", task) " completed.")
        } else if (status < 0) {
            w("[FAILED] Unknown task: " ansi("bold", task))
            exit 1
        } else {
            w("[FAILED] Task " ansi("bold", task) " failed.")
            exit 1
        }
    }
}


Fix bug. Strings are 1 based

Fix bug. Strings are 1 based

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@17659 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# input: publisher\tplugin\tcontract_year\tyear\tstatus

BEGIN {
  FS="\t"
  pn = 0
}

{
# add a loop to add line only if first 4 char or last 4 char of year are gt or eq to contract
if ((substr($4,1,4) >= $3) || ((length($4) == 9) && (substr($4,6,4) >= $3))) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
  if (!(($1,lp2,$3,$4) in b)) {
    p[pn] = $1
    n[pn] = lp2
#    n[pn] = $2
    t[pn] = $3
    d[pn] = $4
    pn++
  }
  b[$1,lp2,$3,$4]++
  c[$1,lp2,$3,$4,$5]++
  x[$5]++
  tt++
}
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  printf "Publisher\tPlugin\tContr\tYear\tTotal"
  for (j = 0 ; j < scn ; j++) {
    printf "\t%s", sc[j]
  }
  printf "\n"

  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], d[i], b[p[i],n[i],t[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      printf "\t%d", c[p[i],n[i],t[i],d[i],s[j]]
    }
    printf "\n"
  }
    printf "Publisher\tPlugin\tContr\tYear\t%d", tt
    for (j = 0 ; j < sn ; j++) {
    	printf "\t%d", x[s[j]]
    }

  printf "\n"
}

#!/usr/bin/awk -f
# input: publisher\tplugin\tcontract_year\tyear\tstatus

BEGIN {
  FS="\t"
  pn = 0
}

{
# add a loop to add line only if first 4 char or last 4 char of year are gt or eq to contract
if ((substr($4,0,4) >= $3) || ((length($4) == 9) && (substr($4,6,4) >= $3))) {
    nn = split($2,na,/\./)
    lp2 = na[nn]
  if (!(($1,lp2,$3,$4) in b)) {
    p[pn] = $1
    n[pn] = lp2
#    n[pn] = $2
    t[pn] = $3
    d[pn] = $4
    pn++
  }
  b[$1,lp2,$3,$4]++
  c[$1,lp2,$3,$4,$5]++
  x[$5]++
  tt++
}
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  printf "Publisher\tPlugin\tContr\tYear\tTotal"
  for (j = 0 ; j < scn ; j++) {
    printf "\t%s", sc[j]
  }
  printf "\n"

  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], d[i], b[p[i],n[i],t[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      printf "\t%d", c[p[i],n[i],t[i],d[i],s[j]]
    }
    printf "\n"
  }
    printf "Publisher\tPlugin\tContr\tYear\t%d", tt
    for (j = 0 ; j < sn ; j++) {
    	printf "\t%d", x[s[j]]
    }

  printf "\n"
}


Added commandline args options -n and -c to oppress output of barcode in the readname and comment, respectively

BEGIN{
    attach_in_readname = 1;
    attach_in_comment = 1;

    for (i = 1; i < ARGC; i++) {
        if (ARGV[i] == "-n")
            attach_in_readname = 0
        else if (ARGV[i] == "-c")
            attach_in_comment = 0
        else if (ARGV[i] ~ /^-./) {
            e = sprintf("%s: unrecognized option -- %c",
                    ARGV[0], substr(ARGV[i], 2, 1))
            print e > "/dev/stderr"
        } else
            break
        delete ARGV[i]
    }

}

{
    if (NR % 8 == 1 && NR > 1) {
        barcode = substr(read_pair[2], 1, 16);
        if (attach_in_readname) {
            initial = barcode"-1_";
        }

        if (attach_in_comment) {
            final = " BX:Z:"barcode"-1";
        }

        printf initial read_pair[1] final"\n"
        printf substr(read_pair[2], 17)"\n"
        printf read_pair[3]"\n"
        printf substr(read_pair[4], 17)"\n"
        printf initial read_pair[5] final"\n"
        printf read_pair[6]"\n"
        printf read_pair[7]"\n"
        printf read_pair[0]"\n"
    }
    read_pair[NR % 8] = $0
}

{
    if (NR % 8 == 1 && NR > 1) {
        barcode = substr(read_pair[2], 1, 16)
        printf read_pair[1]" BX:Z:"barcode"\n"
        printf substr(read_pair[2], 17)"\n"
        printf read_pair[3]"\n"
        printf substr(read_pair[4], 17)"\n"
        printf read_pair[5]" BX:Z:"barcode"\n"
        printf read_pair[6]"\n"
        printf read_pair[7]"\n"
        printf read_pair[0]"\n"
    }
    read_pair[NR % 8] = $0
}


LanguageData: update section T

LanguageData: update section T


Put local symbol suppression rule into most recent (e.g. last) version
block.

Approved by:	re (kensmith)


Utils: fix base64() on systems where echo does not accept option -n

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev" SUPERR
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev" SUPERR "| sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    }

    if (ENVIRON["TRANS_ENTRY"]) {
        command = Rlwrap " " ENVIRON["TRANS_ENTRY"] " "                 \
            parameterize("-no-rlwrap") # never fork rlwrap again!
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        command = Rlwrap " sh "                                         \
            parameterize(ENVIRON["TRANS_DIR"] "/" EntryScript)          \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            command = command " " parameterize(ARGV[i])

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    }

    params = ""
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            params = params " " parameterize(ARGV[i], "\"")
    if (ENVIRON["TRANS_ENTRY"]) {
        el = "(progn (setq explicit-shell-file-name \"" ENVIRON["TRANS_ENTRY"] "\") " \
            "(setq explicit-" Command "-args '(\"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        el = "(progn (setq explicit-shell-file-name \"" "sh" "\") "     \
            "(setq explicit-" "sh" "-args '(\"" ENVIRON["TRANS_DIR"] "/" EntryScript "\" \"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    command = Emacs " --eval " parameterize(el)

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["proxy"])
        command = command " --proxy " parameterize(Option["proxy"])
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Fetch the content of a URL. Return a null string if failed.
function curlPost(url, data, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["proxy"])
        command = command " --proxy " parameterize(Option["proxy"])
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " --request POST --data " parameterize(data)
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    # hexdump tricks:
    # (1) use -v (--no-squeezing)
    # (2) use "%u" (unsigned integers)
    command = "hexdump" " -v -e'1/1 \"%03u\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

# Base64 encode a string.
# NOTE: can only be ran once for each text! Build a cache.
function base64(text,    command, temp) {
    command = "echo -n " parameterize(text) PIPE "base64"
    ("bash -c " parameterize(command, "\"")) | getline temp
    return temp
}

# Print a Unicode-escaped string. (requires GNU Bash)
# NOTE: can only be ran once for each text! Build a cache.
function uprintf(text,    command, temp) {
    command = "echo -en " parameterize(text)
    ("bash -c " parameterize(command, "\"")) | getline temp
    return temp
}

####################################################################
# Utils.awk                                                        #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") "          \
          "appears to be too old.\n"                                    \
          "        You need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Detect external bidirectional algorithm utility (fribidi);
# Fallback to Unix `rev` if not found.
function initBiDi() {
    FriBidi = detectProgram("fribidi", "--version", 1)
    BiDiNoPad = FriBidi ? "fribidi --nopad" : "rev" SUPERR
    BiDi = FriBidi ? "fribidi --width %s" :
        "rev" SUPERR "| sed \"s/'/\\\\\\'/\" | xargs printf '%%s '"
}

# Detect external readline wrapper (rlwrap).
function initRlwrap() {
    Rlwrap = detectProgram("rlwrap", "--version")
}

# Detect emacs.
function initEmacs() {
    Emacs = detectProgram("emacs", "--version")
}

# Detect curl.
function initCurl() {
    Curl = detectProgram("curl", "--version")
}

# Log any value if debugging is enabled.
function l(value, name, inline, heredoc, valOnly, numSub, sortedIn) {
    if (Option["debug"]) {
        if (name)
            da(value, name, inline, heredoc, valOnly, numSub, sortedIn)
        else
            d(value)
    }
}

# Return a log message if debugging is enabled.
function m(string) {
    if (Option["debug"])
        return ansi("cyan", string) RS
}

# Return 1 if the first version is newer than the second; otherwise return 0.
function newerVersion(ver1, ver2,    i, group1, group2) {
    split(ver1, group1, ".")
    split(ver2, group2, ".")
    for (i = 1; i <= 4; i++) {
        if (group1[i] + 0 > group2[i] + 0)
            return 1
        else if (group1[i] + 0 < group2[i] + 0)
            return 0
    }
    return 0
}

# Fork a rlwrap process as the wrapper. Return non-zero if failed,
function rlwrapMe(    i, command) {
    initRlwrap()

    if (!Rlwrap) {
        l(">> not found: rlwrap")
        return 1
    }

    if (ENVIRON["TRANS_ENTRY"]) {
        command = Rlwrap " " ENVIRON["TRANS_ENTRY"] " "                 \
            parameterize("-no-rlwrap") # never fork rlwrap again!
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        command = Rlwrap " sh "                                         \
            parameterize(ENVIRON["TRANS_DIR"] "/" EntryScript)          \
            " - " parameterize("-no-rlwrap") # never fork rlwrap again!
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            command = command " " parameterize(ARGV[i])

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fork an emacs process as the front-end. Return non-zero if failed,
function emacsMe(    i, params, el, command) {
    initEmacs()

    if (!Emacs) {
        l(">> not found: emacs")
        return 1
    }

    params = ""
    for (i = 1; i < length(ARGV); i++)
        if (ARGV[i])
            params = params " " parameterize(ARGV[i], "\"")
    if (ENVIRON["TRANS_ENTRY"]) {
        el = "(progn (setq explicit-shell-file-name \"" ENVIRON["TRANS_ENTRY"] "\") " \
            "(setq explicit-" Command "-args '(\"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else if (fileExists(ENVIRON["TRANS_DIR"] "/" EntryScript)) {
        el = "(progn (setq explicit-shell-file-name \"" "sh" "\") "     \
            "(setq explicit-" "sh" "-args '(\"" ENVIRON["TRANS_DIR"] "/" EntryScript "\" \"-I\" \"-no-rlwrap\"" params ")) " \
            "(command-execute 'shell) (rename-buffer \"" Name "\"))"
    } else {
        l(">> not found: $TRANS_ENTRY or EntryPoint")
        return 1
    }
    command = Emacs " --eval " parameterize(el)

    l(">> forking: " command)
    if (!system(command)) {
        l(">> process exited with code 0")
        exit ExitCode
    } else {
        l(">> process exited with non-zero return code")
        return 1
    }
}

# Fetch the content of a URL. Return a null string if failed.
function curl(url, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["proxy"])
        command = command " --proxy " parameterize(Option["proxy"])
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Fetch the content of a URL. Return a null string if failed.
function curlPost(url, data, output,    command, content, line) {
    initCurl()

    if (!Curl) {
        l(">> not found: curl")
        w("[WARNING] curl is not found.")
        return NULLSTR
    }

    command = Curl " --location --silent"
    if (Option["proxy"])
        command = command " --proxy " parameterize(Option["proxy"])
    if (Option["user-agent"])
        command = command " --user-agent " parameterize(Option["user-agent"])
    command = command " --request POST --data " parameterize(data)
    command = command " " parameterize(url)
    if (output) {
        command = command " --output " parameterize(output)
        system(command)
        return NULLSTR
    }
    content = NULLSTR
    while ((command |& getline line) > 0)
        content = (content ? content "\n" : NULLSTR) line
    return content
}

# Dump a Unicode string into a byte array. Return the length of the array.
# NOTE: can only be ran once for each text! Build a cache.
function dump(text, group,    command, temp) {
    # hexdump tricks:
    # (1) use -v (--no-squeezing)
    # (2) use "%u" (unsigned integers)
    command = "hexdump" " -v -e'1/1 \"%03u\" \" \"'"
    ("echo " parameterize(text) PIPE command) | getline temp
    split(temp, group, " ")
    return length(group) - 1
}

# Base64 encode a string.
# NOTE: can only be ran once for each text! Build a cache.
function base64(text,    command, temp) {
    command = "base64"
    ("echo -n " parameterize(text) PIPE command) | getline temp
    return temp
}

# Print a Unicode-escaped string. (requires GNU Bash)
# NOTE: can only be ran once for each text! Build a cache.
function uprintf(text,    command, temp) {
    command = "echo -en " parameterize(text)
    ("bash -c " parameterize(command, "\"")) | getline temp
    return temp
}