Change column count err message

Change column count err message

see: https://trello.com/c/ShoI6Sr2/350-user-interprets-his-upload-violations

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">=" : "<="
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print RS \"INVALID HEADERS IN \" CSVFILENAME RS \"WAS: \" RS $0 RS \"EXPECTED:\" RS headers RS; exit 0; } }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column number"
    rule_type = "field-width-missmatch"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "text"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = "value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = "value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = "value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">=" : "<="
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = "value should be " term " than: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = "value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = "max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " than " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"COPY " file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    #print "    if (action == \"insert\") print \"\\\\.\""
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


interp_utils.awk	-- macro expansion added

function IU_init(config, interp, name,    key, work, n_ary, ary, i, v_pat, parms) {

	interp["name"] = name

	# get the scale type (linear (default) and  log10 so far
	key = name ".scale_type"
	work = config["_globals", key]
	if(work == "")
		interp["scale_type"] = "linear"
	else if(work == "linear" || work == "log")
		interp["scale_type"] = work
	else{
		print("ERROR: IU_init: unknown scale_type: %s\n", key) > "/dev/stderr"
		return 1
	}

	# get the values
	key = name ".values"
	work = config["_globals", key]
	if(work == ""){
		printf("ERROR: IU_init: no key named \"%s\" in config\n", key) > "/dev/stderr"
		return 1
	}
	n_ary = split(work, ary, "|")
	interp["nvalues"] = n_ary
	interp["is_grad"] = 0
	interp["v_len"] = 0	# set only for grad elts
	for(i = 1; i <= n_ary; i++){
		sub(/^[ \t]*/, "", ary[i])
		sub(/[ \t]*$/, "", ary[i])
		parms["grad"] = ary[i]
		if(index(ary[i], ":") != 0){
			interp["is_grad"] = 1
			v_pat = v_pat "g"
			parms["v_len"] = 0
			if(_IU_check_grad(config, parms)){
				printf("ERROR: IU_init: _IU_check_grad failed for value %d (%s)\n", i, ary[i]) > "/dev/stderr"
				return 1
			}else if(interp["v_len"] == 0){
				interp["v_len"] = parms["v_len"]
			}else if(parms["v_len"] != interp["v_len"]){
				printf("ERROR: IU_init: v_len (%d) for value %d (%s) differs from current v_len (%d)\n", parms["v_len"], i, ary[i], interp["v_len"]) > "/dev/stderr"
				return 1
			}
		}else{
			if(substr(ary[i], 1, 1) == "$"){
				work = config["_globals", substr(ary[i], 2)]
				if(work == ""){
					printf("ERROR: IU_init: macro %s not defined\n", ary[i]) > "/dev/stderr"
					return 1
				}else
					parms["grad"] = work
			}
			v_pat = v_pat "b"
		}
		interp["values", i] = parms["grad"]
	}

	# get the breaks
	key = name ".breaks"
	work = config["_globals", key]
	if(work == ""){
		printf("ERROR: IU_init: no key named \"%s\" in config\n", key) > "/dev/stderr"
		return 1
	}
	n_ary = split(work, ary, "|")
	interp["nbreaks"] = n_ary
	for(i = 1; i <= n_ary; i++){
		sub(/^[ \t]*/, "", ary[i])
		sub(/[ \t]*$/, "", ary[i])
		interp["breaks", i] = ary[i] + 0	# force numeric
	}
	if(!interp["is_grad"]){
		if(interp["nbreaks"] != interp["nvalues"] - 1){
			printf("ERROR: IU_init: nbreaks (%d) != nvalues - 1 (%d)\n", interp["nbreaks"], interp["nvalues"] - 1) > "/dev/stderr"
			return 1
		}
	}else if(interp["nbreaks"] == interp["nvalues"] + 1){
		if(!match(v_pat, /^gg*$/)){
			printf("ERROR: IU_init: nbreaks (%d) = nvalues + 1 (%d): all values must be gradients\n", interp["nbreaks"], interp["nvalues"] + 1) > "/dev/stderr"
			return 1
		}
		interp["v_ob_info"] = ""
	}else if(interp["nbreaks"] == interp["nvalues"]){
		if(!match(v_pat, /^bgg*$/) && !match(v_pat, /^gg*b$/)){
			printf("ERROR: IU_init: nbreaks (%d) = nvalues (%d): All but first or last, but not both values must grad\n", interp["nbreaks"], interp["nvalues"]) > "/dev/stderr"
			return 1
		}
		interp["v_ob_info"] = v_pat ~ /^b/ ? "^" : "$"
	}else if(interp["nbreaks"] == interp["nvalues"] - 1){
		printf("DEBUG: IU_init: nb = nv - 1: chk that we have v[1] = bin, v[2:$-1] = grad, v[$] = bin\n") > "/dev/stderr"
		if(!match(v_pat, /^bgg*b$/)){
			printf("ERROR: IU_init: nbreaks (%d) = nvalues - 1 (%d): all values but first and last must be grad\n", interp["nbreaks"], interp["nvalues"] - 1) > "/dev/stderr"
			return 1
		}
		interp["v_ob_info"] = "^$"
	}else{
		printf("ERROR: IU_init: bad nbreaks (%d), legal values are (%d, %d, %d)\n", interp["nbreaks"], interp["nvalues"] - 1, interp["nvalues"], interp["nvalues"] + 1) > "/dev/stderr"
		return 1
	}

	# check that breaks are strictly ascending, insures interp denoms are > 0
	for(i = 2; i <= interp["nbreaks"]; i++){
		if(interp["breaks", i] <= interp["breaks", i-1]){
			printf("ERROR: IU_init: breaks must be strictly ascending: breaks[%d] (%g) <= breaks[%d] (%g)\n", i-1, interp["breaks", i-1], i, interp["breaks", i]) > "/dev/stderr"
			return 1
		}
	}

	# check for any exceptions
	key = name ".exceptions"
	work = config["_globals", key]
	if(work != ""){
		if(_IU_parse_exceptions(config, interp, work))
			return 1
	}

	# init all counts to 0
	for(i = 0; i <= interp["nbreaks"] + 1; i++)
		interp["counts", i] = 0
	interp["tcounts"] = 0

	return 0
}

function IU_interpolate(interp, v,   ev, idx, work, n_ary, ary, i) {

	if("exceptions" in interp){
		ev = _IU_handle_exceptions(interp, v)
		if(ev != "")
			return ev
	}else if(interp["scale_type"] == "log"){
		if(v <= 0)
			return ""
	}

	idx = _IU_search(interp, v)
	interp["tcounts"]++
	interp["counts", idx]++
	if(!interp["is_grad"]){
		return interp["values", idx]
	}else{
		# nbreaks = nvalues - 1
		if(interp["v_ob_info"]  == ""){	# ^gg*$, take out of bounds values from first and last grad elemnt
			if(idx == 1){
				work = interp["values", 1]
				n_ary = split(work, ary, ":")
				return ary[1]
			}else if(idx == interp["nbreaks"] + 1){
				work = interp["values", interp["nvalues"]]
				n_ary = split(work, ary, ":")
				return ary[2]
			}else
				return _IU_interpolate_grad(interp, interp["values", idx - 1], v, interp["breaks", idx - 1], interp["breaks", idx])
		}else if(interp["v_ob_info"] == "^"){	# ^bgg*$, lower out of bounds specified, take higher ob from last grad element
			if(idx == 1)
				return interp["values", 1]
			else if(idx == interp["nbreaks"] + 1){
				work = interp["values", interp["nvalues"]]
				n_ary = split(work, ary, ":")
				return ary[2]
			}else
				return _IU_interpolate_grad(interp, interp["values", idx], v, interp["breaks", idx], interp["breaks", idx + 1])
		}else if(interp["v_ob_info"] == "$"){	# ^gg*b$, upper out of bounds specified, take lower ob from first grad element
			if(idx == 1){
				work = interp["values", 1]
				n_ary = split(work, ary, ":")
				return ary[1]
			}else if(idx == interp["nbreaks"] + 1)
				return interp["values", idx - 1]
			else
				return _IU_interpolate_grad(interp, interp["values", idx - 1], v, interp["breaks", idx - 1], interp["breaks", idx])
		}else{	# ^bgg*b$, lower & upper ob info specified
			if(idx == 1 || idx == interp["nbreaks"] + 1)
				return interp["values", idx]
			else
				return _IU_interpolate_grad(interp, interp["values", idx], v, interp["breaks", idx], interp["breaks", idx + 1])
		}
	}
}

function IU_dump(file, interp,   i, keys, nk) {

	printf("interp = {\n") > file
	printf("\tname          = %s\n", interp["name"]) > file
	printf("\tscale_type    = %s\n", interp["scale_type"]) > file
	printf("\tis_grad       = %d\n", interp["is_grad"]) > file
	printf("\tv_ob_info     = %s\n", interp["v_ob_info"]) > file
	printf("\tv_len         = %d\n", interp["v_len"]) > file
	printf("\tnvalues       = %d\n", interp["nvalues"]) > file
	printf("\tvalues        = %s", interp["values", 1]) > file
	for(i = 2; i <= interp["nvalues"]; i++)
		printf(" | %s", interp["values", i]) > file
	printf("\n") > file
	printf("\tnbreaks       = %d\n", interp["nbreaks"]) > file
	printf("\tbreaks        = %s", interp["breaks", 1]) > file
	for(i = 2; i <= interp["nbreaks"]; i++)
		printf(" | %s", interp["breaks", i]) > file
	printf("\n") > file
	printf("\ttcounts       = %d\n", interp["tcounts"]) > file
	printf("\tcounts        = %d", interp["counts", 1]) > file
	for(i = 2; i <= interp["nbreaks"] + 1; i++)
		printf(" | %d", interp["counts", i]) > file
	printf("\n") > file
	if(!("exceptions" in interp))
		printf("\texceptions    =\n") > file
	else{
		printf("\texceptions    = %d {\n", interp["nexceptions"]) > file
		for(i = 1; i <= interp["nexceptions"]; i++)
			printf("\t\t%d = %s | %s | %s\n", i, interp["exceptions", i, "op"], interp["exceptions", i, "opnd"], interp["exceptions", i, "value"]) > file
		printf("\t}\n") > file
		printf("\tecounts       = %d", interp["ecounts", 1]) > file
		for(i = 2; i <= interp["nexceptions"]; i++)
			printf(" | %d", interp["ecounts", i]) > file
		printf("\n") > file
	}

	printf("}\n") > file
}

function _IU_check_grad(config, parms,    v_len, n_ary, ary, i, work, n_ary2, ary2, j, xgrad, xval) {

	parms["v_len"] = 0
	n_ary = split(parms["grad"], ary, ":")
	if(n_ary != 2){
		printf("ERROR: _IU_check_grad: grad %s has %d fields, must have %d\n", grad, n_ary, 2) > "/dev/stderr"
		return 1
	}
	xgrad = ""
	for(i = 1; i <= n_ary; i++){
		n_ary2 = split(ary[i], ary2, ",")
		if(parms["v_len"] == 0)
			parms["v_len"] = n_ary2
		else if(n_ary2 != parms["v_len"]){
			printf("ERROR: _IU_check_grad: vector %s has %d elements, must have %d\n", ary[i], n_ary2, parms["v_len"]) > "/dev/stderr"
			return 1
		}
		if(substr(ary[i], 1, 1) == "$"){
			xval = config["_globals", substr(ary[i], 2)]
			if(xval == ""){
				printf("ERROR: _IU_check_grad: macro %s is not defined\n", ary[i]) > "/dev/stderr"
				return 1
			}
			xgrad = xgrad (i > 1 ? ":" : "") xval
		}
	}
	parms["grad"] = xgrad

	return 0
}

function _IU_parse_exceptions(config, interp, str,   err, n_ary, ary, i, work, colon, cond, op, opnd, value, xvalue) {

	err = 0
	interp["exceptions"] = str
	n_ary = split(str, ary, "|")
	for(i = 1; i <= n_ary; i++){
		sub(/^  */, "", ary[i])
		sub(/  *$/, "", ary[i])
	}
	interp["nexceptions"] = n_ary
	for(i = 1; i <= n_ary; i++){
		colon = index(ary[i], ":")
		if(colon == 0){
			printf("ERROR: _IU_parse_exception: no colon: %s\n", ary[i]) > "/dev/stderr"
			return 1
		}
		cond = substr(ary[i], 1, colon - 1)
		sub(/^  */, "", cond)
		sub(/  *$/, "", cond)
		if(cond == ""){
			printf("ERROR: _IU_parse_exception: empty condition: %s\n", ary[i]) > "/dev/stderr"
			return 1
		}
		# op can be <=, <, ==, !=, >=, >
		op = substr(cond, 1, 2)
		if(op == "<=" || op == "==" || op == "!=" || op == ">="){
			opnd = substr(cond, 3)
		}else{
			op = substr(op, 1, 1)
			if(op == "<" || op == ">")
				opnd = substr(cond, 2)
			else{
				printf("ERROR: _IU_parse_exception: unknown operator \"%s\"\n", cond) > "/dev/stderr"
				return 1
			}
		}
		sub(/^  */, "", opnd)
		sub(/  *$/, "", opnd)
		if(cond == ""){
			print("ERROR: _IU_parse_exception: exception %d: %s: empty operand\n", i, ary[i]) > "/dev/stderr"
			return 1
		}

		value = substr(ary[i], colon + 1)
		sub(/^  */, "", value)
		sub(/  *$/, "", value)
		# TODO? value can be empty, is this OK?
		if(value == ""){
			printf("ERROR: _IU_parse_exception: exception %d: %s: empty value\n", i, ary[i]) > "/dev/stderr"
			return 1
		}else if(substr(value, 1, 1) == "$"){
			xvalue = config["_globals", substr(value, 2)]
			if(xvalue == ""){
				printf("ERROR: _IU_parse_exception: macro %s is not defined\n", value) > "/dev/stderr"
				return 1
			}
			value = xvalue
		}

		interp["exceptions", i, "op"] = op
		interp["exceptions", i, "opnd"] = opnd + 0 # force numeric
		interp["exceptions", i, "value"] = value
		interp["ecounts", i] = 0
	}
	return err
}

function _IU_handle_exceptions(interp, v,   i, hit) {

	for(i = 1; i <= interp["nexceptions"]; i++){
		hit = ""
		if(interp["exceptions", i, "op"] == "<"){
			if(v < interp["exceptions", i, "opnd"])
				hit = "<"
		}else if(interp["exceptions", i, "op"] == "<="){
			if(v <= interp["exceptions", i, "opnd"])
				hit = "<="
		}else if(interp["exceptions", i, "op"] == "=="){
			if(v == interp["exceptions", i, "opnd"])
				hit = "=="
		}else if(interp["exceptions", i, "op"] == "!="){
			if(v != interp["exceptions", i, "opnd"])
				hit = "!="
		}else if(interp["exceptions", i, "op"] == ">="){
			if(v >= interp["exceptions", i, "opnd"])
				hit = ">="
		}else if(v >= interp["exceptions", i, "opnd"])
			hit = ">"
		if(hit){
			interp["ecounts", i]++
			interp["tcounts"]++
			return interp["exceptions", i, "value"]
		}
	}

	return ""
}

function _IU_search(interp, v,   i, j, k, cv) {

	# deal with out of range values here to simplify the binary interval search
	if(v <= interp["breaks", 1])
		return 1
	else if(v > interp["breaks", interp["nbreaks"]])
		return interp["nbreaks"] + 1

	# binary search on intervals
	i = 1 ; j = interp["nbreaks"];
	for( ; i <= j ; ){
		k = int((i+j)/2)
		if(v == interp["breaks", k]){
			return k
		}else if(v < interp["breaks", k]){
			if(v > interp["breaks", k-1])
				return k
			j = k - 1
		}else{
			if(v <= interp["breaks", k+1])
				return k + 1
			i = k + 1
		}
	}
	# should never get here!
	return 0
}

function _IU_interpolate_grad(interp, grad, v, v_min, v_max,   n_ary, ary, n_ary2, ary2, i, g_min, g_max, f, rstr) {

	# grad is well formed or IU_init() would have failed
	n_ary = split(grad, ary, ":")
	n_ary2 = split(ary[1], ary2, ",")
	for(i = 1; i <= n_ary2; i++)
		g_min[i] = ary2[i] + 0	# force numeric
	n_ary2 = split(ary[2], ary2, ",")
	for(i = 1; i <= n_ary2; i++)
		g_max[i] = ary2[i] + 0	# force numeric

	# other scales may be possible but for now
	if(interp["scale_type"] == "log"){
		log10 = log(10)
		v = log(v)/log10
		v_min = log(v_min)/log10
		v_max = log(v_max)/log10
	}

	f = (v - v_min) / (v_max - v_min)	# safe b/c IU_init() checked that v_max > v_min
	rstr = ""
	for(i = 1; i <= n_ary2; i++){
		rstr = rstr ((i > 1) ? "," : "") sprintf("%g", g_min[i] + f * (g_max[i] - g_min[i]))
	}
	return rstr
}

function IU_init(config, interp, name,    key, work, n_ary, ary, i, v_pat, v_len) {

	interp["name"] = name

	# get the scale type (linear (default) and  log10 so far
	key = name ".scale_type"
	work = config["_globals", key]
	if(work == "")
		interp["scale_type"] = "linear"
	else if(work == "linear" || work == "log")
		interp["scale_type"] = work
	else{
		print("ERROR: IU_init: unknown scale_type: %s\n", key) > "/dev/stderr"
		return 1
	}

	# get the values
	key = name ".values"
	work = config["_globals", key]
	if(work == ""){
		printf("ERROR: IU_init: no key named \"%s\" in config\n", key) > "/dev/stderr"
		return 1
	}
	n_ary = split(work, ary, "|")
	interp["nvalues"] = n_ary
	interp["is_grad"] = 0
	interp["v_len"] = 0	# set only for grad elts
	for(i = 1; i <= n_ary; i++){
		sub(/^[ \t]*/, "", ary[i])
		sub(/[ \t]*$/, "", ary[i])
		if(index(ary[i], ":") != 0){
			interp["is_grad"] = 1
			v_pat = v_pat "g"
			v_len = _IU_check_grad(ary[i])
			if(v_len == 0){
				printf("ERROR: IU_init: _IU_check_grad failed for value %d (%s)\n", i, ary[i]) > "/dev/stderr"
				return 1
			}else if(interp["v_len"] == 0){
				interp["v_len"] = v_len
			}else if(v_len != interp["v_len"]){
				printf("ERROR: IU_init: v_len (%d) for value %d (%s) differs from current v_len (%d)\n", v_len, i, ary[i], interp["v_len"]) > "/dev/stderr"
				return 1
			}
		}else
			v_pat = v_pat "b"
		interp["values", i] = ary[i]
	}

	# get the breaks
	key = name ".breaks"
	work = config["_globals", key]
	if(work == ""){
		printf("ERROR: IU_init: no key named \"%s\" in config\n", key) > "/dev/stderr"
		return 1
	}
	n_ary = split(work, ary, "|")
	interp["nbreaks"] = n_ary
	for(i = 1; i <= n_ary; i++){
		sub(/^[ \t]*/, "", ary[i])
		sub(/[ \t]*$/, "", ary[i])
		interp["breaks", i] = ary[i] + 0	# force numeric
	}
	if(!interp["is_grad"]){
		if(interp["nbreaks"] != interp["nvalues"] - 1){
			printf("ERROR: IU_init: nbreaks (%d) != nvalues - 1 (%d)\n", interp["nbreaks"], interp["nvalues"] - 1) > "/dev/stderr"
			return 1
		}
	}else if(interp["nbreaks"] == interp["nvalues"] + 1){
		if(!match(v_pat, /^gg*$/)){
			printf("ERROR: IU_init: nbreaks (%d) = nvalues + 1 (%d): all values must be gradients\n", interp["nbreaks"], interp["nvalues"] + 1) > "/dev/stderr"
			return 1
		}
		interp["v_ob_info"] = ""
	}else if(interp["nbreaks"] == interp["nvalues"]){
		if(!match(v_pat, /^bgg*$/) && !match(v_pat, /^gg*b$/)){
			printf("ERROR: IU_init: nbreaks (%d) = nvalues (%d): All but first or last, but not both values must grad\n", interp["nbreaks"], interp["nvalues"]) > "/dev/stderr"
			return 1
		}
		interp["v_ob_info"] = v_pat ~ /^b/ ? "^" : "$"
	}else if(interp["nbreaks"] == interp["nvalues"] - 1){
		printf("DEBUG: IU_init: nb = nv - 1: chk that we have v[1] = bin, v[2:$-1] = grad, v[$] = bin\n") > "/dev/stderr"
		if(!match(v_pat, /^bgg*b$/)){
			printf("ERROR: IU_init: nbreaks (%d) = nvalues - 1 (%d): all values but first and last must be grad\n", interp["nbreaks"], interp["nvalues"] - 1) > "/dev/stderr"
			return 1
		}
		interp["v_ob_info"] = "^$"
	}else{
		printf("ERROR: IU_init: bad nbreaks (%d), legal values are (%d, %d, %d)\n", interp["nbreaks"], interp["nvalues"] - 1, interp["nvalues"], interp["nvalues"] + 1) > "/dev/stderr"
		return 1
	}

	# check that breaks are strictly ascending, insures interp denoms are > 0
	for(i = 2; i <= interp["nbreaks"]; i++){
		if(interp["breaks", i] <= interp["breaks", i-1]){
			printf("ERROR: IU_init: breaks must be strictly ascending: breaks[%d] (%g) <= breaks[%d] (%g)\n", i-1, interp["breaks", i-1], i, interp["breaks", i]) > "/dev/stderr"
			return 1
		}
	}

	# check for any exceptions
	key = name ".exceptions"
	work = config["_globals", key]
	if(work != ""){
		if(_IU_parse_exceptions(interp, work))
			return 1
	}

	# init all counts to 0
	for(i = 0; i <= interp["nbreaks"] + 1; i++)
		interp["counts", i] = 0
	interp["tcounts"] = 0

	return 0
}

function IU_interpolate(interp, v,   ev, idx, work, n_ary, ary, i) {

	if("exceptions" in interp){
		ev = _IU_handle_exceptions(interp, v)
		if(ev != "")
			return ev
	}else if(interp["scale_type"] == "log"){
		if(v <= 0)
			return ""
	}

	idx = _IU_search(interp, v)
	interp["tcounts"]++
	interp["counts", idx]++
	if(!interp["is_grad"]){
		return interp["values", idx]
	}else{
		# nbreaks = nvalues - 1
		if(interp["v_ob_info"]  == ""){	# ^gg*$, take out of bounds values from first and last grad elemnt
			if(idx == 1){
				work = interp["values", 1]
				n_ary = split(work, ary, ":")
				return ary[1]
			}else if(idx == interp["nbreaks"] + 1){
				work = interp["values", interp["nvalues"]]
				n_ary = split(work, ary, ":")
				return ary[2]
			}else
				return _IU_interpolate_grad(interp, interp["values", idx - 1], v, interp["breaks", idx - 1], interp["breaks", idx])
		}else if(interp["v_ob_info"] == "^"){	# ^bgg*$, lower out of bounds specified, take higher ob from last grad element
			if(idx == 1)
				return interp["values", 1]
			else if(idx == interp["nbreaks"] + 1){
				work = interp["values", interp["nvalues"]]
				n_ary = split(work, ary, ":")
				return ary[2]
			}else
				return _IU_interpolate_grad(interp, interp["values", idx], v, interp["breaks", idx], interp["breaks", idx + 1])
		}else if(interp["v_ob_info"] == "$"){	# ^gg*b$, upper out of bounds specified, take lower ob from first grad element
			if(idx == 1){
				work = interp["values", 1]
				n_ary = split(work, ary, ":")
				return ary[1]
			}else if(idx == interp["nbreaks"] + 1)
				return interp["values", idx - 1]
			else
				return _IU_interpolate_grad(interp, interp["values", idx - 1], v, interp["breaks", idx - 1], interp["breaks", idx])
		}else{	# ^bgg*b$, lower & upper ob info specified
			if(idx == 1 || idx == interp["nbreaks"] + 1)
				return interp["values", idx]
			else
				return _IU_interpolate_grad(interp, interp["values", idx], v, interp["breaks", idx], interp["breaks", idx + 1])
		}
	}
}

function IU_dump(file, interp,   i, keys, nk) {

	printf("interp = {\n") > file
	printf("\tname          = %s\n", interp["name"]) > file
	printf("\tscale_type    = %s\n", interp["scale_type"]) > file
	printf("\tis_grad       = %d\n", interp["is_grad"]) > file
	printf("\tv_ob_info     = %s\n", interp["v_ob_info"]) > file
	printf("\tv_len         = %d\n", interp["v_len"]) > file
	printf("\tnvalues       = %d\n", interp["nvalues"]) > file
	printf("\tvalues        = %s", interp["values", 1]) > file
	for(i = 2; i <= interp["nvalues"]; i++)
		printf(" | %s", interp["values", i]) > file
	printf("\n") > file
	printf("\tnbreaks       = %d\n", interp["nbreaks"]) > file
	printf("\tbreaks        = %s", interp["breaks", 1]) > file
	for(i = 2; i <= interp["nbreaks"]; i++)
		printf(" | %s", interp["breaks", i]) > file
	printf("\n") > file
	printf("\ttcounts       = %d\n", interp["tcounts"]) > file
	printf("\tcounts        = %d", interp["counts", 1]) > file
	for(i = 2; i <= interp["nbreaks"] + 1; i++)
		printf(" | %d", interp["counts", i]) > file
	printf("\n") > file
	if(!("exceptions" in interp))
		printf("\texceptions    =\n") > file
	else{
		printf("\texceptions    = %d {\n", interp["nexceptions"]) > file
		for(i = 1; i <= interp["nexceptions"]; i++)
			printf("\t\t%d = %s | %s | %s\n", i, interp["exceptions", i, "op"], interp["exceptions", i, "opnd"], interp["exceptions", i, "value"]) > file
		printf("\t}\n") > file
		printf("\tecounts       = %d", interp["ecounts", 1]) > file
		for(i = 2; i <= interp["nexceptions"]; i++)
			printf(" | %d", interp["ecounts", i]) > file
		printf("\n") > file
	}

	printf("}\n") > file
}

function _IU_check_grad(grad,   v_len, n_ary, ary, i, n_ary2, ary2, j) {

	v_len = 0
	n_ary = split(grad, ary, ":")
	if(n_ary != 2){
		printf("ERROR: _IU_check_grad: grad %s has %d fields, must have %d\n", grad, n_ary, 2) > "/dev/stderr"
		return 0
	}
	for(i = 1; i <= n_ary; i++){
		n_ary2 = split(ary[i], ary2, ",")
		if(v_len == 0)
			v_len = n_ary2
		else if(n_ary2 != v_len){
			printf("ERROR: _IU_check_grad: vector %s has %d elements, must have %d\n", ary[i], n_ary2, v_len) > "/dev/stderr"
			return 0
		}
	}

	return v_len
}

function _IU_parse_exceptions(interp, str,   err, n_ary, ary, i, colon, cond, op, opnd, value) {

	err = 0
	interp["exceptions"] = str
	n_ary = split(str, ary, "|")
	for(i = 1; i <= n_ary; i++){
		sub(/^  */, "", ary[i])
		sub(/  *$/, "", ary[i])
	}
	interp["nexceptions"] = n_ary
	for(i = 1; i <= n_ary; i++){
		colon = index(ary[i], ":")
		if(colon == 0){
			printf("ERROR: _IU_parse_exception: no colon: %s\n", ary[i]) > "/dev/stderr"
			return 1
		}
		cond = substr(ary[i], 1, colon - 1)
		sub(/^  */, "", cond)
		sub(/  *$/, "", cond)
		if(cond == ""){
			printf("ERROR: _IU_parse_exception: empty condition: %s\n", ary[i]) > "/dev/stderr"
			return 1
		}
		# op can be <=, <, ==, !=, >=, >
		op = substr(cond, 1, 2)
		if(op == "<=" || op == "==" || op == "!=" || op == ">="){
			opnd = substr(cond, 3)
		}else{
			op = substr(op, 1, 1)
			if(op == "<" || op == ">")
				opnd = substr(cond, 2)
			else{
				printf("ERROR: _IU_parse_exception: unknown operator \"%s\"\n", cond) > "/dev/stderr"
				return 1
			}
		}
		sub(/^  */, "", opnd)
		sub(/  *$/, "", opnd)
		if(cond == ""){
			print("ERROR: _IU_parse_exception: exception %d: %s: empty operand %s\n", i, ary[i]) > "/dev/stderr"
			return 1
		}

		value = substr(ary[i], colon + 1)
		sub(/^  */, "", value)
		sub(/  *$/, "", value)
		# TODO? value can be empty, is this OK?

		interp["exceptions", i, "op"] = op
		interp["exceptions", i, "opnd"] = opnd + 0 # force numeric
		interp["exceptions", i, "value"] = value
		interp["ecounts", i] = 0
	}
	return err
}

function _IU_handle_exceptions(interp, v,   i, hit) {

	for(i = 1; i <= interp["nexceptions"]; i++){
		hit = ""
		if(interp["exceptions", i, "op"] == "<"){
			if(v < interp["exceptions", i, "opnd"])
				hit = "<"
		}else if(interp["exceptions", i, "op"] == "<="){
			if(v <= interp["exceptions", i, "opnd"])
				hit = "<="
		}else if(interp["exceptions", i, "op"] == "=="){
			if(v == interp["exceptions", i, "opnd"])
				hit = "=="
		}else if(interp["exceptions", i, "op"] == "!="){
			if(v != interp["exceptions", i, "opnd"])
				hit = "!="
		}else if(interp["exceptions", i, "op"] == ">="){
			if(v >= interp["exceptions", i, "opnd"])
				hit = ">="
		}else if(v >= interp["exceptions", i, "opnd"])
			hit = ">"
		if(hit){
			interp["ecounts", i]++
			interp["tcounts"]++
			return interp["exceptions", i, "value"]
		}
	}

	return ""
}

function _IU_search(interp, v,   i, j, k, cv) {

	# deal with out of range values here to simplify the binary interval search
	if(v <= interp["breaks", 1])
		return 1
	else if(v > interp["breaks", interp["nbreaks"]])
		return interp["nbreaks"] + 1

	# binary search on intervals
	i = 1 ; j = interp["nbreaks"];
	for( ; i <= j ; ){
		k = int((i+j)/2)
		if(v == interp["breaks", k]){
			return k
		}else if(v < interp["breaks", k]){
			if(v > interp["breaks", k-1])
				return k
			j = k - 1
		}else{
			if(v <= interp["breaks", k+1])
				return k + 1
			i = k + 1
		}
	}
	# should never get here!
	return 0
}

function _IU_interpolate_grad(interp, grad, v, v_min, v_max,   n_ary, ary, n_ary2, ary2, i, g_min, g_max, f, rstr) {

	# grad is well formed or IU_init() would have failed
	n_ary = split(grad, ary, ":")
	n_ary2 = split(ary[1], ary2, ",")
	for(i = 1; i <= n_ary2; i++)
		g_min[i] = ary2[i] + 0	# force numeric
	n_ary2 = split(ary[2], ary2, ",")
	for(i = 1; i <= n_ary2; i++)
		g_max[i] = ary2[i] + 0	# force numeric

	# other scales may be possible but for now
	if(interp["scale_type"] == "log"){
		log10 = log(10)
		v = log(v)/log10
		v_min = log(v_min)/log10
		v_max = log(v_max)/log10
	}

	f = (v - v_min) / (v_max - v_min)	# safe b/c IU_init() checked that v_max > v_min
	rstr = ""
	for(i = 1; i <= n_ary2; i++){
		rstr = rstr ((i > 1) ? "," : "") sprintf("%g", g_min[i] + f * (g_max[i] - g_min[i]))
	}
	return rstr
}


pushing

pushing

# vim: ft=awk:tabstop=2:softtabstop=2:shiftwidth=2:expandtab:cindent:formatoptions+=cro

# I lie tradidd lilights
#
# - when they are green
# - adas
#
@include "lib"

END {
  print ""
  for(w in W)
    printf("%10s ",Name[w])
  print "\n"report(Data); tree(Data,NF-2,NF)
}

function tree(rows,max,k,   pre,ranges,r,c,use,sorted) {
  array(ranges)
  for(r in rows)
    for(c in Data[r])
      if (c < max)
        kept(r,c, Data[r][c], Data[r][k], ranges)
  use = best(ranges,sorted)
  for(r=length(sorted); r>=1; r--)
    if (sorted[r].column == use)
        kids(use, sorted[r],max,k,pre)
}
# Somethings to note:
#
# - asda
# - sdsd
function kids(use,range,max,k,pre,    txt) {
   if (length( range.rows ) > length(Data)^0.5) {
     print  report(range.rows) pre Name[use] " = " range.value
     tree(range.rows,max,k,"|__ " pre)  }
}
function range(i,c,x) {
  num(i)
  i.column = c
  i.value  = x
  has(i,"rows")
}
function kept(r,c,x,y,ranges,    key) {
  if (x=="?") return 0
  key = c "=" x
  if (! (key in ranges))
    hAS(ranges, key, "range",c,x)
  ninc(ranges[key],y)
  ranges[key].rows[r]
}
function best(ranges,sorted,  n) {
  n = musort(ranges,sorted)
  return sorted[n].column
}

function report(rows,   w,n,r,txt) {
  for(w in W) {
    num(n)
    for(r in rows)
      ninc(n,Data[r][w])
    txt = txt sprintf("%10.2f ",n.mu)
  }
  return txt "\t"
}


2015-03-12: parsegaspriceshtml.awk: Added substitution for Marathon pictures.

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

/
<div class=\"p","")
   sub("^.*

.*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   sub("^[ ]*","",address[i,ai[i]])
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && /<a href="/ {
   gsub("","Marathon")
   gsub("","Duchess")
   gsub("","Thorntons")
   gsub("","Thorntons")
   gsub("","Certified")
   gsub("","Marathon")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.* ",",")
   sub("<.*$","")
   split($0,foo,",")
   reporter[i]=foo[1]
   time[i]=foo[2]
   timeAgo[i]=foo[3]
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

/
<div class=\"p","")
   sub("^.*

.*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   sub("^[ ]*","",address[i,ai[i]])
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && /<a href="/ {
   gsub("","Marathon")
   gsub("","Duchess")
   gsub("","Thorntons")
   gsub("","Thorntons")
   gsub("","Certified")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.* ",",")
   sub("<.*$","")
   split($0,foo,",")
   reporter[i]=foo[1]
   time[i]=foo[2]
   timeAgo[i]=foo[3]
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}


Revert "Update awk file to match rupa's z (#42)"

This reverts commit 6962f589a495b80e35ed9c2678b9fd53cd670afa.


CWS swlayoutrefactoring: last portion of m100 merged


Main: escape embedded bracket in regexp (fix #55 for gawk 4.1.2)

####################################################################
# Main.awk                                                         #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") appears to be too old.\nYou need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Pre-initialization (before option parsing).
function preInit() {
    initGawk()          #<< AnsiCode

    # Languages
    initBiDi()
    initLocale()
    initLocaleDisplay() #<< Locale, BiDi
    initUserLang()      #<< Locale

    RS = "\n"

    ExitCode = 0

    Option["debug"] = 0

    Option["verbose"] = 1
    Option["show-original"] = 1
    Option["show-original-phonetics"] = 1
    Option["show-translation"] = 1
    Option["show-translation-phonetics"] = 1
    Option["show-prompt-message"] = 1
    Option["show-languages"] = 1
    Option["show-original-dictionary"] = 0
    Option["show-dictionary"] = 1
    Option["show-alternatives"] = 1

    Option["width"] = ENVIRON["COLUMNS"] ? ENVIRON["COLUMNS"] : 64
    Option["indent"] = 4

    Option["browser"] = ENVIRON["BROWSER"]

    Option["play"] = 0
    Option["player"] = ENVIRON["PLAYER"]

    Option["proxy"] = ENVIRON["HTTP_PROXY"] ? ENVIRON["HTTP_PROXY"] : ENVIRON["http_proxy"]

    Option["interactive"] = 0
    Option["no-rlwrap"] = 0
    Option["emacs"] = 0
    Option["prompt"] = ENVIRON["TRANS_PS"] ? ENVIRON["TRANS_PS"] : "%s>"
    Option["prompt-color"] = ENVIRON["TRANS_PS_COLOR"] ? ENVIRON["TRANS_PS_COLOR"] : "default"

    Option["input"] = ""
    Option["output"] = "/dev/stdout"

    Option["hl"] = ENVIRON["HOME_LANG"] ? ENVIRON["HOME_LANG"] : UserLang
    Option["sl"] = ENVIRON["SOURCE_LANG"] ? ENVIRON["SOURCE_LANG"] : "auto"
    Option["tl"][1] = ENVIRON["TARGET_LANG"] ? ENVIRON["TARGET_LANG"] : UserLang
}

# Post-initialization (after option parsing).
function postInit() {
    # Translate
    initHttpService()
}

# Main entry point.
BEGIN {
    preInit()

    pos = 0
    while (ARGV[++pos]) {
        # -, -no-op
        match(ARGV[pos], /^-(-?no-op)?$/)
        if (RSTART) continue

        # -V, -version
        match(ARGV[pos], /^--?(vers(i(on?)?)?|V)$/)
        if (RSTART) {
            print getVersion()
            print
            printf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])
            printf("%s\n", FriBidi ? FriBidi : "fribidi (GNU FriBidi) [NOT INSTALLED]")
            printf("%-22s%s\n", "User Language", Locale[getCode(UserLang)]["name"] " (" show(Locale[getCode(UserLang)]["endonym"]) ")")
            exit
        }

        # -H, -h, -help
        match(ARGV[pos], /^--?(h(e(lp?)?)?|H)$/)
        if (RSTART) {
            print getHelp()
            exit
        }

        # -M, -m, -manual
        match(ARGV[pos], /^--?(m(a(n(u(al?)?)?)?)?|M)$/)
        if (RSTART) {
            if (ENVIRON["TRANS_MANPAGE"])
                system("echo -E \"${TRANS_MANPAGE}\" | " \
                       "groff -Wall -mtty-char -mandoc -Tutf8 -rLL=${COLUMNS}n -rLT=${COLUMNS}n | " \
                       (system("most 2>/dev/null") ?
                        "less -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\"" :
                        "most -Cs"))
            else
                print getHelp()
            exit
        }

        # -r, -reference
        match(ARGV[pos], /^--?r(e(f(e(r(e(n(ce?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            print getReference("endonym")
            exit
        }

        # -R, -reference-english
        match(ARGV[pos], /^--?(reference-(e(n(g(l(i(sh?)?)?)?)?)?)?|R)$/)
        if (RSTART) {
            print getReference("name")
            exit
        }

        # -D, -debug
        match(ARGV[pos], /^--?(debug|D)$/)
        if (RSTART) {
            Option["debug"] = 1
            continue
        }

        # -v, -verbose
        match(ARGV[pos], /^--?v(e(r(b(o(se?)?)?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 1 # default value
            continue
        }

        # -b, -brief
        match(ARGV[pos], /^--?b(r(i(ef?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 0
            continue
        }

        # -show-original [yes|no]
        match(ARGV[pos], /^--?show-original(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-phonetics [yes|no]
        match(ARGV[pos], /^--?show-original-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation [yes|no]
        match(ARGV[pos], /^--?show-translation(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation-phonetics [yes|no]
        match(ARGV[pos], /^--?show-translation-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-prompt-message [yes|no]
        match(ARGV[pos], /^--?show-prompt-message(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-prompt-message"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-languages [yes|no]
        match(ARGV[pos], /^--?show-languages(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-languages"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-dictionary [yes|no]
        match(ARGV[pos], /^--?show-original-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-dictionary [yes|no]
        match(ARGV[pos], /^--?show-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-alternatives [yes|no]
        match(ARGV[pos], /^--?show-alternatives(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-alternatives"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -no-ansi
        match(ARGV[pos], /^--?no-ansi/)
        if (RSTART) {
            Option["no-ansi"] = 1
            continue
        }

        # -w [num], -width [num]
        match(ARGV[pos], /^--?w(i(d(th?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["width"] = group[4] ?
                (group[5] ? group[5] : Option["width"]) :
                ARGV[++pos]
            continue
        }

        # -indent [num]
        match(ARGV[pos], /^--?indent(=(.*)?)?$/, group)
        if (RSTART) {
            Option["indent"] = group[1] ?
                (group[2] ? group[2] : Option["indent"]) :
                ARGV[++pos]
            continue
        }

        # -browser [program]
        match(ARGV[pos], /^--?browser(=(.*)?)?$/, group)
        if (RSTART) {
            Option["browser"] = group[1] ?
                (group[2] ? group[2] : Option["browser"]) :
                ARGV[++pos]
            continue
        }

        # -p, -play
        match(ARGV[pos], /^--?p(l(ay?)?)?$/)
        if (RSTART) {
            Option["play"] = 1
            continue
        }

        # -player [program]
        match(ARGV[pos], /^--?player(=(.*)?)?$/, group)
        if (RSTART) {
            Option["play"] = 1
            Option["player"] = group[1] ?
                (group[2] ? group[2] : Option["player"]) :
                ARGV[++pos]
            continue
        }

        # -x [proxy], -proxy [proxy]
        match(ARGV[pos], /^--?(proxy|x)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["proxy"] = group[2] ?
                (group[3] ? group[3] : Option["proxy"]) :
                ARGV[++pos]
            continue
        }

        # -I, -interactive
        match(ARGV[pos], /^--?(int(e(r(a(c(t(i(ve?)?)?)?)?)?)?)?|I)$/)
        if (RSTART) {
            Option["interactive"] = 1
            continue
        }

        # -no-rlwrap
        match(ARGV[pos], /^--?no-rlwrap/)
        if (RSTART) {
            Option["no-rlwrap"] = 1
            continue
        }

        # -E, -emacs
        match(ARGV[pos], /^--?(emacs|E)$/)
        if (RSTART) {
            Option["emacs"] = 1
            continue
        }

        # -prompt [prompt_string]
        match(ARGV[pos], /^--?prompt(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt"] = group[1] ?
                (group[2] ? group[2] : Option["prompt"]) :
                ARGV[++pos]
            continue
        }

        # -prompt-color [color_code]
        match(ARGV[pos], /^--?prompt-color(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt-color"] = group[1] ?
                (group[2] ? group[2] : Option["prompt-color"]) :
                ARGV[++pos]
            continue
        }

        # -i [file], -input [file]
        match(ARGV[pos], /^--?i(n(p(ut?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["input"] = group[4] ?
                (group[5] ? group[5] : Option["input"]) :
                ARGV[++pos]
            continue
        }

        # -o [file], -output [file]
        match(ARGV[pos], /^--?o(u(t(p(ut?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["output"] = group[5] ?
                (group[6] ? group[6] : Option["output"]) :
                ARGV[++pos]
            continue
        }

        # -l [code], -lang [code]
        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["hl"] = group[3] ?
                (group[4] ? group[4] : Option["hl"]) :
                ARGV[++pos]
            continue
        }

        # -s [code], -source [code]
        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["sl"] = group[5] ?
                (group[6] ? group[6] : Option["sl"]) :
                ARGV[++pos]
            continue
        }

        # -t [codes], -target [codes]
        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            if (group[5]) {
                if (group[6]) split(group[6], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # Shortcut format
        # '[code]:[code]+...' or '[code]=[code]+...'
        match(ARGV[pos], /^[{(\[]?([[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?(:|=)((@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?\+)*(@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?)[})\]]?$/, group)
        if (RSTART) {
            if (group[1]) Option["sl"] = group[1]
            if (group[4]) split(group[4], Option["tl"], "+")
            continue
        }

        # --
        match(ARGV[pos], /^--$/)
        if (RSTART) {
            ++pos # skip the end-of-options option
            break # no more option from here
        }

        break # no more option from here
    }

    # Option parsing finished
    postInit()

    if (Option["interactive"] && !Option["no-rlwrap"]) {
        # Interactive mode
        initRlwrap() # initialize Rlwrap

        if (Rlwrap && (ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
            command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                           "\"${TRANS_PROGRAM}\"" :
                                           "-f " EntryPoint) " -" \
                " -no-rlwrap" # be careful - never fork Rlwrap recursively!
            for (i = 1; i < length(ARGV); i++)
                if (ARGV[i])
                    command = command " " parameterize(ARGV[i])

            if (!system(command))
                exit # child process finished, exit
            else
                ; # skip
        } else
            ; # skip

    } else if (!Option["interactive"] && !Option["no-rlwrap"] && Option["emacs"]) {
        # Emacs interface
        Emacs = "emacs"

        if (ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint)) {
            params = ""
            for (i = 1; i < length(ARGV); i++)
                if (ARGV[i])
                    params = params " " (parameterize(ARGV[i], "\""))
            if (ENVIRON["TRANS_PROGRAM"]) {
                el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
                    "(setq explicit-shell-file-name \"" Gawk "\") " \
                    "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
                    "(command-execute 'shell) (rename-buffer \"" Name "\"))"
            } else {
                el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
                    "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
                    "(command-execute 'shell) (rename-buffer \"" Name "\"))"
            }
            command = Emacs " --eval " parameterize(el)

            if (!system(command))
                exit # child process finished, exit
            else
                Option["interactive"] = 1 # skip
        } else
            Option["interactive"] = 1 # skip
    }

    if (Option["play"]) {
        # Initialize audio player or speech synthesizer
        if (!Option["player"]) {
            initAudioPlayer()
            Option["player"] = AudioPlayer ? AudioPlayer : Option["player"]
            if (!Option["player"])
                initSpeechSynthesizer()
        }

        if (!Option["player"] && !SpeechSynthesizer) {
            w("[WARNING] No available audio player or speech synthesizer is found.")
            Option["play"] = 0
        }
    }

    if (Option["interactive"]) {
        print AnsiCode["bold"] AnsiCode[tolower(Option["prompt-color"])] getVersion() AnsiCode[0] > "/dev/stderr"
        print AnsiCode[tolower(Option["prompt-color"])] "(:q to quit)" AnsiCode[0] > "/dev/stderr"
    }

    # Initialize browser
    if (!Option["browser"]) {
        "xdg-mime query default text/html 2>/dev/null" |& getline Option["browser"]
        match(Option["browser"], "(.*).desktop$", group)
        Option["browser"] = group[1]
    }

    # Disable ANSI SGR (Select Graphic Rendition) codes if required
    if (Option["no-ansi"])
        delete AnsiCode

    if (pos < ARGC) {
        # More parameters

        # Translate the remaining parameters
        for (i = pos; i < ARGC; i++) {
            # Verbose mode: separator between sources
            if (Option["verbose"] && i > pos)
                print replicate("═", Option["width"])

            translate(ARGV[i], 1) # inline mode
        }

        # If input not specified, we're done
    } else {
        # No more parameter besides options

        # If input not specified, use stdin
        if (!Option["input"]) Option["input"] = "/dev/stdin"
    }

    # If input specified, start translating
    if (Option["input"])
        translateMain()

    exit ExitCode
}

####################################################################
# Main.awk                                                         #
####################################################################

# Detect gawk version.
function initGawk(    group) {
    Gawk = "gawk"
    GawkVersion = PROCINFO["version"]

    split(PROCINFO["version"], group, ".")
    if (group[1] < 4) {
        e("[ERROR] Oops! Your gawk (version " GawkVersion ") appears to be too old.\nYou need at least gawk 4.0.0 to run this program.")
        exit 1
    }
}

# Pre-initialization (before option parsing).
function preInit() {
    initGawk()          #<< AnsiCode

    # Languages
    initBiDi()
    initLocale()
    initLocaleDisplay() #<< Locale, BiDi
    initUserLang()      #<< Locale

    RS = "\n"

    ExitCode = 0

    Option["debug"] = 0

    Option["verbose"] = 1
    Option["show-original"] = 1
    Option["show-original-phonetics"] = 1
    Option["show-translation"] = 1
    Option["show-translation-phonetics"] = 1
    Option["show-prompt-message"] = 1
    Option["show-languages"] = 1
    Option["show-original-dictionary"] = 0
    Option["show-dictionary"] = 1
    Option["show-alternatives"] = 1

    Option["width"] = ENVIRON["COLUMNS"] ? ENVIRON["COLUMNS"] : 64
    Option["indent"] = 4

    Option["browser"] = ENVIRON["BROWSER"]

    Option["play"] = 0
    Option["player"] = ENVIRON["PLAYER"]

    Option["proxy"] = ENVIRON["HTTP_PROXY"] ? ENVIRON["HTTP_PROXY"] : ENVIRON["http_proxy"]

    Option["interactive"] = 0
    Option["no-rlwrap"] = 0
    Option["emacs"] = 0
    Option["prompt"] = ENVIRON["TRANS_PS"] ? ENVIRON["TRANS_PS"] : "%s>"
    Option["prompt-color"] = ENVIRON["TRANS_PS_COLOR"] ? ENVIRON["TRANS_PS_COLOR"] : "default"

    Option["input"] = ""
    Option["output"] = "/dev/stdout"

    Option["hl"] = ENVIRON["HOME_LANG"] ? ENVIRON["HOME_LANG"] : UserLang
    Option["sl"] = ENVIRON["SOURCE_LANG"] ? ENVIRON["SOURCE_LANG"] : "auto"
    Option["tl"][1] = ENVIRON["TARGET_LANG"] ? ENVIRON["TARGET_LANG"] : UserLang
}

# Post-initialization (after option parsing).
function postInit() {
    # Translate
    initHttpService()
}

# Main entry point.
BEGIN {
    preInit()

    pos = 0
    while (ARGV[++pos]) {
        # -, -no-op
        match(ARGV[pos], /^-(-?no-op)?$/)
        if (RSTART) continue

        # -V, -version
        match(ARGV[pos], /^--?(vers(i(on?)?)?|V)$/)
        if (RSTART) {
            print getVersion()
            print
            printf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])
            printf("%s\n", FriBidi ? FriBidi : "fribidi (GNU FriBidi) [NOT INSTALLED]")
            printf("%-22s%s\n", "User Language", Locale[getCode(UserLang)]["name"] " (" show(Locale[getCode(UserLang)]["endonym"]) ")")
            exit
        }

        # -H, -h, -help
        match(ARGV[pos], /^--?(h(e(lp?)?)?|H)$/)
        if (RSTART) {
            print getHelp()
            exit
        }

        # -M, -m, -manual
        match(ARGV[pos], /^--?(m(a(n(u(al?)?)?)?)?|M)$/)
        if (RSTART) {
            if (ENVIRON["TRANS_MANPAGE"])
                system("echo -E \"${TRANS_MANPAGE}\" | " \
                       "groff -Wall -mtty-char -mandoc -Tutf8 -rLL=${COLUMNS}n -rLT=${COLUMNS}n | " \
                       (system("most 2>/dev/null") ?
                        "less -s -P\"\\ \\Manual page " Command "(1) line %lt (press h for help or q to quit)\"" :
                        "most -Cs"))
            else
                print getHelp()
            exit
        }

        # -r, -reference
        match(ARGV[pos], /^--?r(e(f(e(r(e(n(ce?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            print getReference("endonym")
            exit
        }

        # -R, -reference-english
        match(ARGV[pos], /^--?(reference-(e(n(g(l(i(sh?)?)?)?)?)?)?|R)$/)
        if (RSTART) {
            print getReference("name")
            exit
        }

        # -D, -debug
        match(ARGV[pos], /^--?(debug|D)$/)
        if (RSTART) {
            Option["debug"] = 1
            continue
        }

        # -v, -verbose
        match(ARGV[pos], /^--?v(e(r(b(o(se?)?)?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 1 # default value
            continue
        }

        # -b, -brief
        match(ARGV[pos], /^--?b(r(i(ef?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 0
            continue
        }

        # -show-original [yes|no]
        match(ARGV[pos], /^--?show-original(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-phonetics [yes|no]
        match(ARGV[pos], /^--?show-original-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation [yes|no]
        match(ARGV[pos], /^--?show-translation(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation-phonetics [yes|no]
        match(ARGV[pos], /^--?show-translation-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-prompt-message [yes|no]
        match(ARGV[pos], /^--?show-prompt-message(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-prompt-message"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-languages [yes|no]
        match(ARGV[pos], /^--?show-languages(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-languages"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-dictionary [yes|no]
        match(ARGV[pos], /^--?show-original-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-dictionary [yes|no]
        match(ARGV[pos], /^--?show-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-alternatives [yes|no]
        match(ARGV[pos], /^--?show-alternatives(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-alternatives"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -no-ansi
        match(ARGV[pos], /^--?no-ansi/)
        if (RSTART) {
            Option["no-ansi"] = 1
            continue
        }

        # -w [num], -width [num]
        match(ARGV[pos], /^--?w(i(d(th?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["width"] = group[4] ?
                (group[5] ? group[5] : Option["width"]) :
                ARGV[++pos]
            continue
        }

        # -indent [num]
        match(ARGV[pos], /^--?indent(=(.*)?)?$/, group)
        if (RSTART) {
            Option["indent"] = group[1] ?
                (group[2] ? group[2] : Option["indent"]) :
                ARGV[++pos]
            continue
        }

        # -browser [program]
        match(ARGV[pos], /^--?browser(=(.*)?)?$/, group)
        if (RSTART) {
            Option["browser"] = group[1] ?
                (group[2] ? group[2] : Option["browser"]) :
                ARGV[++pos]
            continue
        }

        # -p, -play
        match(ARGV[pos], /^--?p(l(ay?)?)?$/)
        if (RSTART) {
            Option["play"] = 1
            continue
        }

        # -player [program]
        match(ARGV[pos], /^--?player(=(.*)?)?$/, group)
        if (RSTART) {
            Option["play"] = 1
            Option["player"] = group[1] ?
                (group[2] ? group[2] : Option["player"]) :
                ARGV[++pos]
            continue
        }

        # -x [proxy], -proxy [proxy]
        match(ARGV[pos], /^--?(proxy|x)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["proxy"] = group[2] ?
                (group[3] ? group[3] : Option["proxy"]) :
                ARGV[++pos]
            continue
        }

        # -I, -interactive
        match(ARGV[pos], /^--?(int(e(r(a(c(t(i(ve?)?)?)?)?)?)?)?|I)$/)
        if (RSTART) {
            Option["interactive"] = 1
            continue
        }

        # -no-rlwrap
        match(ARGV[pos], /^--?no-rlwrap/)
        if (RSTART) {
            Option["no-rlwrap"] = 1
            continue
        }

        # -E, -emacs
        match(ARGV[pos], /^--?(emacs|E)$/)
        if (RSTART) {
            Option["emacs"] = 1
            continue
        }

        # -prompt [prompt_string]
        match(ARGV[pos], /^--?prompt(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt"] = group[1] ?
                (group[2] ? group[2] : Option["prompt"]) :
                ARGV[++pos]
            continue
        }

        # -prompt-color [color_code]
        match(ARGV[pos], /^--?prompt-color(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt-color"] = group[1] ?
                (group[2] ? group[2] : Option["prompt-color"]) :
                ARGV[++pos]
            continue
        }

        # -i [file], -input [file]
        match(ARGV[pos], /^--?i(n(p(ut?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["input"] = group[4] ?
                (group[5] ? group[5] : Option["input"]) :
                ARGV[++pos]
            continue
        }

        # -o [file], -output [file]
        match(ARGV[pos], /^--?o(u(t(p(ut?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["output"] = group[5] ?
                (group[6] ? group[6] : Option["output"]) :
                ARGV[++pos]
            continue
        }

        # -l [code], -lang [code]
        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["hl"] = group[3] ?
                (group[4] ? group[4] : Option["hl"]) :
                ARGV[++pos]
            continue
        }

        # -s [code], -source [code]
        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["sl"] = group[5] ?
                (group[6] ? group[6] : Option["sl"]) :
                ARGV[++pos]
            continue
        }

        # -t [codes], -target [codes]
        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            if (group[5]) {
                if (group[6]) split(group[6], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # Shortcut format
        # '[code]:[code]+...' or '[code]=[code]+...'
        match(ARGV[pos], /^[{([]?([[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?(:|=)((@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?\+)*(@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?)[})\]]?$/, group)
        if (RSTART) {
            if (group[1]) Option["sl"] = group[1]
            if (group[4]) split(group[4], Option["tl"], "+")
            continue
        }

        # --
        match(ARGV[pos], /^--$/)
        if (RSTART) {
            ++pos # skip the end-of-options option
            break # no more option from here
        }

        break # no more option from here
    }

    # Option parsing finished
    postInit()

    if (Option["interactive"] && !Option["no-rlwrap"]) {
        # Interactive mode
        initRlwrap() # initialize Rlwrap

        if (Rlwrap && (ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint))) {
            command = Rlwrap " " Gawk " " (ENVIRON["TRANS_PROGRAM"] ?
                                           "\"${TRANS_PROGRAM}\"" :
                                           "-f " EntryPoint) " -" \
                " -no-rlwrap" # be careful - never fork Rlwrap recursively!
            for (i = 1; i < length(ARGV); i++)
                if (ARGV[i])
                    command = command " " parameterize(ARGV[i])

            if (!system(command))
                exit # child process finished, exit
            else
                ; # skip
        } else
            ; # skip

    } else if (!Option["interactive"] && !Option["no-rlwrap"] && Option["emacs"]) {
        # Emacs interface
        Emacs = "emacs"

        if (ENVIRON["TRANS_PROGRAM"] || fileExists(EntryPoint)) {
            params = ""
            for (i = 1; i < length(ARGV); i++)
                if (ARGV[i])
                    params = params " " (parameterize(ARGV[i], "\""))
            if (ENVIRON["TRANS_PROGRAM"]) {
                el = "(progn (setq trans-program (getenv \"TRANS_PROGRAM\")) " \
                    "(setq explicit-shell-file-name \"" Gawk "\") " \
                    "(setq explicit-" Gawk "-args (cons trans-program '(\"-\" \"-I\" \"-no-rlwrap\"" params "))) " \
                    "(command-execute 'shell) (rename-buffer \"" Name "\"))"
            } else {
                el = "(progn (setq explicit-shell-file-name \"" Gawk "\") " \
                    "(setq explicit-" Gawk "-args '(\"-f\" \"" EntryPoint "\" \"--\" \"-I\" \"-no-rlwrap\"" params ")) " \
                    "(command-execute 'shell) (rename-buffer \"" Name "\"))"
            }
            command = Emacs " --eval " parameterize(el)

            if (!system(command))
                exit # child process finished, exit
            else
                Option["interactive"] = 1 # skip
        } else
            Option["interactive"] = 1 # skip
    }

    if (Option["play"]) {
        # Initialize audio player or speech synthesizer
        if (!Option["player"]) {
            initAudioPlayer()
            Option["player"] = AudioPlayer ? AudioPlayer : Option["player"]
            if (!Option["player"])
                initSpeechSynthesizer()
        }

        if (!Option["player"] && !SpeechSynthesizer) {
            w("[WARNING] No available audio player or speech synthesizer is found.")
            Option["play"] = 0
        }
    }

    if (Option["interactive"]) {
        print AnsiCode["bold"] AnsiCode[tolower(Option["prompt-color"])] getVersion() AnsiCode[0] > "/dev/stderr"
        print AnsiCode[tolower(Option["prompt-color"])] "(:q to quit)" AnsiCode[0] > "/dev/stderr"
    }

    # Initialize browser
    if (!Option["browser"]) {
        "xdg-mime query default text/html 2>/dev/null" |& getline Option["browser"]
        match(Option["browser"], "(.*).desktop$", group)
        Option["browser"] = group[1]
    }

    # Disable ANSI SGR (Select Graphic Rendition) codes if required
    if (Option["no-ansi"])
        delete AnsiCode

    if (pos < ARGC) {
        # More parameters

        # Translate the remaining parameters
        for (i = pos; i < ARGC; i++) {
            # Verbose mode: separator between sources
            if (Option["verbose"] && i > pos)
                print replicate("═", Option["width"])

            translate(ARGV[i], 1) # inline mode
        }

        # If input not specified, we're done
    } else {
        # No more parameter besides options

        # If input not specified, use stdin
        if (!Option["input"]) Option["input"] = "/dev/stdin"
    }

    # If input specified, start translating
    if (Option["input"])
        translateMain()

    exit ExitCode
}