Allow empty titles

Allow empty titles

Useful to order sections without have a title.

# end of paragraph if we hit '*/'
$0 ~ "*/" {
	p = 0;
}

# if in paragraph and no section is announced,
# concatenate
p && $0 !~ "[ /]* SECTION: " {
	cl[section] = cl[section]  gensub(/ \* ?/, "", "1", $0) "\n";
}

# section announcement
$0 ~ "[ /]* SECTION: " {
	section = $3;
	sl[len++] = section;
	if ($4) {
		tl[section] = gensub(/[\t ]*\/?\* SECTION: [^ ]+ +/, "", "1", $0);
	} else {
		tl[section] = "";
	}
	p = 1;
}

# sort sections, underline titles, print
END {
	asort(sl);
	for (i in sl) {
		section = sl[i]
		print(tl[section]);
		a = section
		c = gsub(/\./, "", a);
		if (c == 0)
			r = "-";
		else if (c == 1)
			r = "~"
		else if (c == 2)
			r = "."
		else
			r = "."
		print(gensub(/./, r, "g", tl[section]));
		print(cl[section]);
	}
}

# end of paragraph if we hit '*/'
$0 ~ "*/" {
	p = 0;
}

# if in paragraph and no section is announced,
# concatenate
p && $0 !~ "[ /]* SECTION: " {
	cl[section] = cl[section]  gensub(/ \* ?/, "", "1", $0) "\n";
}

# section announcement
$0 ~ "[ /]* SECTION: " {
	section = $3;
	sl[len++] = section;
	tl[section] = gensub(/[\t ]*\/?\* SECTION: [^ ]+ +/, "", "1", $0);
	p = 1;
}

# sort sections, underline titles, print
END {
	asort(sl);
	for (i in sl) {
		section = sl[i]
		print(tl[section]);
		a = section
		c = gsub(/\./, "", a);
		if (c == 0)
			r = "-";
		else if (c == 1)
			r = "~"
		else if (c == 2)
			r = "."
		else
			r = "."
		print(gensub(/./, r, "g", tl[section]));
		print(cl[section]);
	}
}


Fix number/int check to be more accurate

see: https://trello.com/c/t9k52j93/517-as-a-developer-i-will-ensure-integers-do-not-allow-decimals-during-import-this-will-ensure-that-decimals-do-not-cause-issue-load

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_integer(x) { return x ~ /^[0-9]+$/ }"
    print "function is_number(x) { return x ~ /^[0-9]+.[0-9]+$/ }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print  NR FS CSVFILENAME FS \"header\" FS \"invalid-header\" FS \"error\" FS  \"headers are invalid \"; exit 0;} }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "violation"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type == "integer") {
            cat       = "error"
            test      = field " && !is_integer(" field ")"
            mini_msg  = field " should be an integer"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be an integer but was \" " field " \" "
        } else if (type == "number") {
            cat       = "error"
            test      = field " && !is_number(" field ")"
            mini_msg  = field " should be a number"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a number but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = field " value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = field " value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = field " value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = field " value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = field " value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = field " max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        } else if (options["mode"] == "violation") {
            key=file_options["pkey"]
            gsub(/\|/, " \"-\" ", key)
            err_handler = handler_prefix "print  NR FS CSVFILENAME FS " key " FS \"" rule_type "\" FS \"" cat "\" FS  \"" mini_msg "\" FS \"" field "\"}"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    print \"COPY \" schema \"" file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    print "    if (action == \"insert\") print \"\\\\.\" RS"
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_numeric(x){ return(x==x+0) }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print  NR FS CSVFILENAME FS \"header\" FS \"invalid-header\" FS \"error\" FS  \"headers are invalid \"; exit 0;} }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "violation"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type ~ /^integer|number$/) {
            cat       = "error"
            test      = field " && !is_numeric(" field ")"
            mini_msg  = field " should be numeric but was a string"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a numeric but was \" " field " \" "
        }

    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = field " value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = field " value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = field " value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = field " value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = field " value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = field " max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        } else if (options["mode"] == "violation") {
            key=file_options["pkey"]
            gsub(/\|/, " \"-\" ", key)
            err_handler = handler_prefix "print  NR FS CSVFILENAME FS " key " FS \"" rule_type "\" FS \"" cat "\" FS  \"" mini_msg "\" FS \"" field "\"}"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    print \"COPY \" schema \"" file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    print "    if (action == \"insert\") print \"\\\\.\" RS"
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


feature: Add new prefixes: feature, refactor and update

#!/usr/bin/awk -f
BEGIN {
	# Determines whether to print 'Unreleased' banner at top
	UNRELEASED_COMMITS = 1
	# Prefixes that determine whether a commit will be printed
	CHANGELOG_REGEX = "^(changelog|fix|docs|chore|feat|feature|refactor|update): "
	FS="|"
	while ("git log --pretty='%D|%s|%H'" | getline) {
		IS_GIT_TAG = length($1) && match($1, /tag:/)
		if (IS_GIT_TAG) {
			UNRELEASED_COMMITS = 0
			# Cut out text up to tag
			sub(/.*tag: /, "", $1)
			# Cut out text after tag
			sub(/,.*/, "", $1)
			print $1
		} else {
			if ( UNRELEASED_COMMITS ) {
				print "Unreleased"
				UNRELEASED_COMMITS = 0
			}
			if ( match($2, CHANGELOG_REGEX) ) {
				sub(CHANGELOG_REGEX, "", $2)
				printf("\t- %s\n", $2)
			}
		}

	}
}

#!/usr/bin/awk -f
BEGIN {
	# Determines whether to print 'Unreleased' banner at top
	UNRELEASED_COMMITS = 1
	# Prefixes that determine whether a commit will be printed
	CHANGELOG_REGEX = "^(changelog|fix|docs|chore|feat): "
	FS="|"
	while ("git log --pretty='%D|%s|%H'" | getline) {
		IS_GIT_TAG = length($1) && match($1, /tag:/)
		if (IS_GIT_TAG) {
			UNRELEASED_COMMITS = 0
			# Cut out text up to tag
			sub(/.*tag: /, "", $1)
			# Cut out text after tag
			sub(/,.*/, "", $1)
			print $1
		} else {
			if ( UNRELEASED_COMMITS ) {
				print "Unreleased"
				UNRELEASED_COMMITS = 0
			}
			if ( match($2, CHANGELOG_REGEX) ) {
				sub(CHANGELOG_REGEX, "", $2)
				printf("\t- %s\n", $2)
			}
		}

	}
}


Clarified duration file output

Clarified duration file output

# Total the duration of all episodes in every series
#
# INVOCATION:
#       awk -v DURATION_FILE=$DURATION_FILE -f calculateMHzDurations.awk $EPISODE_INFO_FILE
#
# INPUT:
#  1 \t =HYPERLINK("https://mhzchoice.vhx.tv/detective-montalbano/season:1/videos/the-snack-thief",\
#  \t \t "Detective Montalbano, S01E01, Montalbano: The Snack Thief") \t '1:45:05 An elderly \
#  import-export merchant is found murdered and a young woman who may have been mixed up with \
#  him disappears the same day. Directed by Alberto Sironi, 1999.
#
# OUTPUT:
#       $DURATION_FILE with one total duration of a series per line

# Extract the duration
BEGIN {
    FS = "\t"
}

{
    sub (/\'/,"",$5)
    series = $1
    if (split ($5, tm, ":") == 3) {
        secs[series] += tm[3]
        mins[series] += tm[2] + int(secs[series] / 60)
        hrs[series] += tm[1] + int(mins[series] / 60)
        secs[series] %= 60; mins[series] %= 60
    } else {
        secs[series] += tm[2]
        mins[series] += tm[1] + int(secs[series] / 60)
        hrs[series] += int(mins[series] / 60)
        secs[series] %= 60; mins[series] %= 60
    }
}

END {
    for ( i = 1; i <= series; i++ )
        printf "%d:%02d:%02d\n", hrs[i], mins[i], secs[i] >> DURATION_FILE
}

# Total the duration of all episodes in every series
#
# INVOCATION:
#       awk -v DURATION_FILE=$DURATION_FILE -f calculateMHzDurations.awk $EPISODE_INFO_FILE
#
# INPUT:
#  1 \t =HYPERLINK("https://mhzchoice.vhx.tv/detective-montalbano/season:1/videos/the-snack-thief",\
#  \t \t "Detective Montalbano, S01E01, Montalbano: The Snack Thief") \t '1:45:05 An elderly \
#  import-export merchant is found murdered and a young woman who may have been mixed up with \
#  him disappears the same day. Directed by Alberto Sironi, 1999.
#
# OUTPUT:
#       $DURATION_FILE

# Extract the duration
BEGIN {
    FS = "\t"
}

{
    sub (/\'/,"",$5)
    series = $1
    if (split ($5, tm, ":") == 3) {
        secs[series] += tm[3]
        mins[series] += tm[2] + int(secs[series] / 60)
        hrs[series] += tm[1] + int(mins[series] / 60)
        secs[series] %= 60; mins[series] %= 60
    } else {
        secs[series] += tm[2]
        mins[series] += tm[1] + int(secs[series] / 60)
        hrs[series] += int(mins[series] / 60)
        secs[series] %= 60; mins[series] %= 60
    }
}

END {
    for ( i = 1; i <= series; i++ )
        printf "%d:%02d:%02d\n", hrs[i], mins[i], secs[i] >> DURATION_FILE
}


r5301/sofa-dev : FIX: make filter-qmake-pro.awk script more robust

#!/usr/bin/awk -f
# This script removes parts flagged by SOFA_DEV inside qmake project files
# The file with the relevant parts removed is output to the standard output,
# while files mentionned inside removed parts are output to the standard error.
# This can be used to remove then from the repository by svn rm'ing them.

# If you never used awk, a good introduction is available at http://www.cs.hmc.edu/qref/awk.html

# init
BEGIN {
    infilelist=0;
    blk_before=0;
    blk_after=0;
    blk_dev=0;
}

{ blk_before=blk_after; }

/[{}]/ {
  split($0,a,"#")
  if (a[1]!="") {
    split(a[1],beg,"{")
    split(a[1],end,"}")
    blk_after = blk_before + length(beg) - length(end)
  }
}

END {
    if (blk_after != 0) print "# ERROR: unmatched brackets"
}

# Match blocks conditionnally included depending on the SOFA_DEV flag
# Note that we now count the brackets in the source file to find the
# SOFA_DEV closing one instead of relying on the presence of a comment

blk_dev==0 && /contains[ \t]*\([ \t]*DEFINES[ \t]*,[ \t]*SOFA_DEV[ \t]*\)[ \t]*{/ {
    blk_dev=blk_after;
    infilelist=0;
}

blk_dev>0 && blk_after < blk_dev {
    blk_dev=0;
    next;
}

# /contains[ \t]*\([ \t]*DEFINES[ \t]*,[ \t]*SOFA_DEV[ \t]*\)[ \t]*{/,/[ \t]*}.*SOFA_DEV/ {
blk_dev>0 {
    # look for a filename
    # we assume files and directories are listed inside variables containing SUBDIRS, SOURCES or HEADERS
    f0=1;
    if (infilelist==0) {
	if (($1~"SUBDIRS" || $1~"SOURCES" || $1~"HEADERS") && ($2=="=" || $2=="+=")) {
#	    print "infilelist " $1
	    infilelist=1;
	    f0=3; # first file is in 3rd field
	}
    }
    if (infilelist) {
	for(f=f0;f<=NF && !(f==NF && $f~/^[:space:]*\\[:space:]*$/);f++) {
	    fname=$f
	    gsub(/[\r\n]/,"",fname);
	    gsub(/\\[:space:]*$/,"",fname);
	    if (fname != "#") print fname > "/dev/stderr";
	}
	if (f>NF && (NF==0 || $NF!~/\\[:space:]*$/)) { # no "\\" is put at the end of the line -> end of list
#	    print "end"
	    infilelist=0;
	}
    }
    next;
}

# Match lines mentionning SOFA_DEV, such as the point where it is defined
/SOFA_DEV/ { next; }

# other: simply print the line
#{ print blk_before "<" $0 ">" blk_after; }
{ print }

#!/usr/bin/awk -f
# This script removes parts flagged by SOFA_DEV inside qmake project files
# The file with the relevant parts removed is output to the standard output,
# while files mentionned inside removed parts are output to the standard error.
# This can be used to remove then from the repository by svn rm'ing them.

# If you never used awk, a good introduction is available at http://www.cs.hmc.edu/qref/awk.html

# init
BEGIN {
    infilelist=0;
}

# Match blocks conditionnally included depending on the SOFA_DEV flag

/contains[ \t]*\([ \t]*DEFINES[ \t]*,[ \t]*SOFA_DEV[ \t]*\)[ \t]*{/,/[ \t]*}.*SOFA_DEV/ {
#    for(f=1;f<=NF && !(f==NF && $f~/\\[:space:]*/);f++)
#        print f " -> \"" $f "\""
    # look for a filename
    # we assume files and directories are listed inside variables containing SUBDIRS, SOURCES or HEADERS
    f0=1;
    if (infilelist==0) {
	if (($1~"SUBDIRS" || $1~"SOURCES" || $1~"HEADERS") && ($2=="=" || $2=="+=")) {
#	    print "infilelist " $1
	    infilelist=1;
	    f0=3; # first file is in 3rd field
	}
    }
    if (infilelist) {
	for(f=f0;f<=NF && !(f==NF && $f~/\\[:space:]*/);f++) {
	    fname=$f
	    gsub(/[\r\n]/,"",fname);
	    if (fname != "#") print fname > "/dev/stderr";
	}
	if (f>NF) { # no "\\" is put at the end of the line -> end of list
#	    print "end"
	    infilelist=0;
	}
    }
    next;
}

# Match lines mentionning SOFA_DEV, such as the point where it is defined
/SOFA_DEV/ { next; }

# other: simply print the line
{ print; }


wireless: fix comments in genregdb.awk

wireless: fix comments in genregdb.awk

Apparently some awk versions choke on C-style comments -- who knew? :-)

Signed-off-by: John W. Linville <2a53bac7a5d324865ef46ec4c38b2c0fba1456b4@tuxdriver.com>
Signed-off-by: David S. Miller <fe08d3c717adf2ae63592e4c9aec6e3e404d8e3e@davemloft.net>

#!/usr/bin/awk -f
#
# genregdb.awk -- generate regdb.c from db.txt
#
# Actually, it reads from stdin (presumed to be db.txt) and writes
# to stdout (presumed to be regdb.c), but close enough...
#
# Copyright 2009 John W. Linville <linville@tuxdriver.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#

BEGIN {
	active = 0
	rules = 0;
	print "/*"
	print " * DO NOT EDIT -- file generated from data in db.txt"
	print " */"
	print ""
	print "#include <linux/nl80211.h>"
	print "#include <net/cfg80211.h>"
	print ""
	regdb = "const struct ieee80211_regdomain *reg_regdb[] = {\n"
}

/^[ \t]*#/ {
	# Ignore
}

!active && /^[ \t]*$/ {
	# Ignore
}

!active && /country/ {
	country=$2
	sub(/:/, "", country)
	printf "static const struct ieee80211_regdomain regdom_%s = {\n", country
	printf "\t.alpha2 = \"%s\",\n", country
	printf "\t.reg_rules = {\n"
	active = 1
	regdb = regdb "\t&regdom_" country ",\n"
}

active && /^[ \t]*\(/ {
	start = $1
	sub(/\(/, "", start)
	end = $3
	bw = $5
	sub(/\),/, "", bw)
	gain = $6
	sub(/\(/, "", gain)
	sub(/,/, "", gain)
	power = $7
	sub(/\)/, "", power)
	sub(/,/, "", power)
	# power might be in mW...
	units = $8
	sub(/\)/, "", units)
	sub(/,/, "", units)
	if (units == "mW") {
		if (power == 100) {
			power = 20
		} else if (power == 200) {
			power = 23
		} else if (power == 500) {
			power = 27
		} else if (power == 1000) {
			power = 30
		} else {
			print "Unknown power value in database!"
		}
	}
	flagstr = ""
	for (i=8; i<=NF; i++)
		flagstr = flagstr $i
	split(flagstr, flagarray, ",")
	flags = ""
	for (arg in flagarray) {
		if (flagarray[arg] == "NO-OFDM") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_OFDM | "
		} else if (flagarray[arg] == "NO-CCK") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_CCK | "
		} else if (flagarray[arg] == "NO-INDOOR") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_INDOOR | "
		} else if (flagarray[arg] == "NO-OUTDOOR") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_OUTDOOR | "
		} else if (flagarray[arg] == "DFS") {
			flags = flags "\n\t\t\tNL80211_RRF_DFS | "
		} else if (flagarray[arg] == "PTP-ONLY") {
			flags = flags "\n\t\t\tNL80211_RRF_PTP_ONLY | "
		} else if (flagarray[arg] == "PTMP-ONLY") {
			flags = flags "\n\t\t\tNL80211_RRF_PTMP_ONLY | "
		} else if (flagarray[arg] == "PASSIVE-SCAN") {
			flags = flags "\n\t\t\tNL80211_RRF_PASSIVE_SCAN | "
		} else if (flagarray[arg] == "NO-IBSS") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_IBSS | "
		}
	}
	flags = flags "0"
	printf "\t\tREG_RULE(%d, %d, %d, %d, %d, %s),\n", start, end, bw, gain, power, flags
	rules++
}

active && /^[ \t]*$/ {
	active = 0
	printf "\t},\n"
	printf "\t.n_reg_rules = %d\n", rules
	printf "};\n\n"
	rules = 0;
}

END {
	print regdb "};"
	print ""
	print "int reg_regdb_size = ARRAY_SIZE(reg_regdb);"
}

#!/usr/bin/awk -f
#
# genregdb.awk -- generate regdb.c from db.txt
#
# Actually, it reads from stdin (presumed to be db.txt) and writes
# to stdout (presumed to be regdb.c), but close enough...
#
# Copyright 2009 John W. Linville <linville@tuxdriver.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#

BEGIN {
	active = 0
	rules = 0;
	print "/*"
	print " * DO NOT EDIT -- file generated from data in db.txt"
	print " */"
	print ""
	print "#include <linux/nl80211.h>"
	print "#include <net/cfg80211.h>"
	print ""
	regdb = "const struct ieee80211_regdomain *reg_regdb[] = {\n"
}

/^[ \t]*#/ {
	/* Ignore */
}

!active && /^[ \t]*$/ {
	/* Ignore */
}

!active && /country/ {
	country=$2
	sub(/:/, "", country)
	printf "static const struct ieee80211_regdomain regdom_%s = {\n", country
	printf "\t.alpha2 = \"%s\",\n", country
	printf "\t.reg_rules = {\n"
	active = 1
	regdb = regdb "\t&regdom_" country ",\n"
}

active && /^[ \t]*\(/ {
	start = $1
	sub(/\(/, "", start)
	end = $3
	bw = $5
	sub(/\),/, "", bw)
	gain = $6
	sub(/\(/, "", gain)
	sub(/,/, "", gain)
	power = $7
	sub(/\)/, "", power)
	sub(/,/, "", power)
	# power might be in mW...
	units = $8
	sub(/\)/, "", units)
	sub(/,/, "", units)
	if (units == "mW") {
		if (power == 100) {
			power = 20
		} else if (power == 200) {
			power = 23
		} else if (power == 500) {
			power = 27
		} else if (power == 1000) {
			power = 30
		} else {
			print "Unknown power value in database!"
		}
	}
	flagstr = ""
	for (i=8; i<=NF; i++)
		flagstr = flagstr $i
	split(flagstr, flagarray, ",")
	flags = ""
	for (arg in flagarray) {
		if (flagarray[arg] == "NO-OFDM") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_OFDM | "
		} else if (flagarray[arg] == "NO-CCK") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_CCK | "
		} else if (flagarray[arg] == "NO-INDOOR") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_INDOOR | "
		} else if (flagarray[arg] == "NO-OUTDOOR") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_OUTDOOR | "
		} else if (flagarray[arg] == "DFS") {
			flags = flags "\n\t\t\tNL80211_RRF_DFS | "
		} else if (flagarray[arg] == "PTP-ONLY") {
			flags = flags "\n\t\t\tNL80211_RRF_PTP_ONLY | "
		} else if (flagarray[arg] == "PTMP-ONLY") {
			flags = flags "\n\t\t\tNL80211_RRF_PTMP_ONLY | "
		} else if (flagarray[arg] == "PASSIVE-SCAN") {
			flags = flags "\n\t\t\tNL80211_RRF_PASSIVE_SCAN | "
		} else if (flagarray[arg] == "NO-IBSS") {
			flags = flags "\n\t\t\tNL80211_RRF_NO_IBSS | "
		}
	}
	flags = flags "0"
	printf "\t\tREG_RULE(%d, %d, %d, %d, %d, %s),\n", start, end, bw, gain, power, flags
	rules++
}

active && /^[ \t]*$/ {
	active = 0
	printf "\t},\n"
	printf "\t.n_reg_rules = %d\n", rules
	printf "};\n\n"
	rules = 0;
}

END {
	print regdb "};"
	print ""
	print "int reg_regdb_size = ARRAY_SIZE(reg_regdb);"
}


Debug msg for restart parameters.

Debug msg for restart parameters.

# translate flatzinc format to python/Numberjack

BEGIN {
	MAXINT = 10000000;
	print "#! /usr/bin/env python"
	print "import time"
	print "from Numberjack import *"
	system("cat " MZNNJ_DIR "/fzn2py.py");
	print "";
	print "def get_model():"
	print "    model = Model()";
	parameter = 1;
	error = 0;
}

# { print $0 } # for debugging purposes only

/::_output_var/ {
	gsub("::_output_var","");
	$0 = $0 " ::_output_";
}

/::_output_array/ {
	regexp = "::_output_array[(][[](range[(][0-9]+,1[+][0-9]+[)][ ,]*)+[]][)]";
	if (match($0,regexp)) {
	  array = substr($0, RSTART, RLENGTH);
	  sub(regexp,"");
	  gsub("[[]","",array);
	  gsub("[]]","",array);
	  gsub(" ","",array);
	  gsub("range[(]","",array);
	  gsub(",1[+]","..",array);
	  gsub("[)]","",array);
	  nbdim = array;
	  gsub("[^,]","",nbdim);
	  dim = length(nbdim)+1;
	  sub("::_output_array", "array" dim "d",array);
	  $0 = $0 " ::_output_" array;
	} else {
		print "ERROR WRONG OUTPUT ARRAY DEFINITION",$0;
		error = 3;
		exit(2);
	}
}

/^var / {
	parameter = 0;
}

/ of var / {
	parameter = 0;
}

parameter {
	sub(".*: ","");
	if ($2 == "=") print "    " $0; # it might be a predicate instead if no =
}

/^var bool:/{
	print "    " $3 " = Variable('" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^var int:/{
	print "    " $3 " = Variable(-" MAXINT "," MAXINT ",'" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^var range[(][-]*[0-9]+,1[+][-]*[0-9]+[)]:/{
	sub("range[(]","",$2);
	sub("[)]:","",$2);
	sub(",1[+]",",",$2);
	print "    " $3 " = Variable(" $2 ",'" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^var {[-]*[0-9]+(,[-]*[0-9]+)*}:/{
	sub("{","[",$2);
	sub("}:","]",$2);
	print "    " $3 " = Variable(" $2 ",'" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var bool:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	name = $6;
	print "    " name " = VarArray(" isup ",'" name "')";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var int:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	name = $6;
	print "    " name " = VarArray(" isup ",-" MAXINT "," MAXINT ",'" name "')";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var range[(][-]*[0-9]+,1[+][-]*[0-9]+[)]:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	sub("range[(]","",$5);
	sub("[)]:","",$5);
	sub(",1[+]",",",$5);
	name = $6;
	print "    " name " = VarArray(" isup "," $5 ",'" name "')";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var {[-]*[0-9]+(,[-]*[0-9]+)*}:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	sub("{","[",$5);
	sub("}:","]",$5);
	name = $6;
	print "    " name " = [Variable(" $5 ",'" name "_" i "') for i in range(" isup ")]";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^constraint /{
	$1 = "    model.add(";
    	$NF = $NF " )";
	print $0;
}

/^solve / && / minimize /{
	print "    model.add(Minimize(" $NF "))";
	objective = $NF;
}

/^solve / && / maximize /{
	print "    model.add(Maximize(" $NF "))";
	objective = $NF;
}

END {
	if (!error) {
	output_vars = "";
	if(objective) output_vars = objective;

	n = asorti(output,varnames);
	for (i=1; i<=n; i++) {
		e = varnames[i];
		if(e != objective) {
			if(length(output_vars) > 0) output_vars = output_vars ", ";
			output_vars = output_vars e;
		}
	}
	print "    output_vars = (" output_vars ")";
	print "    return model, output_vars";

	print "";
	print "model, output_vars = get_model()";
	if(output_vars) print output_vars " = output_vars";
	print "solvers = ['Mistral', 'SCIP', 'MiniSat', 'Toulbar2', 'Gurobi']";
	print "default = dict([('solver', 'Mistral'), ('verbose', 0), ('tcutoff', 900), ('var', 'DomainOverWDegree'), ('val', 'Lex'), ('rand', 2), ('threads', 1), ('restart', GEOMETRIC), ('base', 256), ('factor', 1.3)])";
	print "param = input(default)";
	print "solver = model.load(param['solver'])";
	print "solver.setVerbosity(param['verbose'])";
	print "solver.setTimeLimit(param['tcutoff'] - int(time.clock()+0.5))";
	print "solver.setHeuristic(param['var'], param['val'], param['rand'])";
	print "if param['solver'] == 'Gurobi':";
	print "    solver.setThreadCount(param['threads'])";
	print "if param['solver'] == 'Mistral':";
	print "    # LUBY, GEOMETRIC = 0, 1"
	print "    print '%% running Mistral with restarts: %d %d %.1f' % (param['restart'], param['base'], param['factor'])";
	#print "    solver.solveAndRestart(GEOMETRIC, 256, 1.3)";
	print "    solver.solveAndRestart(param['restart'], param['base'], param['factor'])";
	print "else:";
	print "    solver.solve()";
	print "if solver.is_sat():"
	for (i=1; i<=n; i++) {
		e = varnames[i];
		if (output[e]==1) {
			if (e != objective && e != "objective" && e != "obj") print "    print '" e " = '," e ".get_value(),';'";
			else print "    print '" e " = ', (solver.getOptimum() if param['solver'] == 'Toulbar2' else " e ".get_value()),';'";
		} else {
			print "    print '" e " = " outputstring[e] ",',str(" e "),');'";
		}
	}
	print "    print '----------'";
	if (objective){
		print "    if solver.is_opt():"
		print "        print '=========='"
	} else {
		print "    print '=========='"
	}
	print "elif solver.is_unsat():"
	print "    print '=====UNSATISFIABLE====='"
    print "else:"
	print "    print '=====UNKNOWN====='"
	print "print '% SolveTime', solver.getTime()";
	print "print '% Nodes', solver.getNodes()";
	print "print '% Failures', solver.getFailures()";
	if (objective) print "if solver.is_sat(): print '% Objective', (solver.getOptimum() if param['solver'] == 'Toulbar2' else " objective ".get_value())";
	}
}

# translate flatzinc format to python/Numberjack

BEGIN {
	MAXINT = 10000000;
	print "#! /usr/bin/env python"
	print "import time"
	print "from Numberjack import *"
	system("cat " MZNNJ_DIR "/fzn2py.py");
	print "";
	print "def get_model():"
	print "    model = Model()";
	parameter = 1;
	error = 0;
}

# { print $0 } # for debugging purposes only

/::_output_var/ {
	gsub("::_output_var","");
	$0 = $0 " ::_output_";
}

/::_output_array/ {
	regexp = "::_output_array[(][[](range[(][0-9]+,1[+][0-9]+[)][ ,]*)+[]][)]";
	if (match($0,regexp)) {
	  array = substr($0, RSTART, RLENGTH);
	  sub(regexp,"");
	  gsub("[[]","",array);
	  gsub("[]]","",array);
	  gsub(" ","",array);
	  gsub("range[(]","",array);
	  gsub(",1[+]","..",array);
	  gsub("[)]","",array);
	  nbdim = array;
	  gsub("[^,]","",nbdim);
	  dim = length(nbdim)+1;
	  sub("::_output_array", "array" dim "d",array);
	  $0 = $0 " ::_output_" array;
	} else {
		print "ERROR WRONG OUTPUT ARRAY DEFINITION",$0;
		error = 3;
		exit(2);
	}
}

/^var / {
	parameter = 0;
}

/ of var / {
	parameter = 0;
}

parameter {
	sub(".*: ","");
	if ($2 == "=") print "    " $0; # it might be a predicate instead if no =
}

/^var bool:/{
	print "    " $3 " = Variable('" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^var int:/{
	print "    " $3 " = Variable(-" MAXINT "," MAXINT ",'" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^var range[(][-]*[0-9]+,1[+][-]*[0-9]+[)]:/{
	sub("range[(]","",$2);
	sub("[)]:","",$2);
	sub(",1[+]",",",$2);
	print "    " $3 " = Variable(" $2 ",'" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^var {[-]*[0-9]+(,[-]*[0-9]+)*}:/{
	sub("{","[",$2);
	sub("}:","]",$2);
	print "    " $3 " = Variable(" $2 ",'" $3 "')";
	if (match($0,"::_output_")) output[$3] = 1;
	if ($4 == "=") print "    model.add(" $3 " == " $5 ")";
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var bool:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	name = $6;
	print "    " name " = VarArray(" isup ",'" name "')";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var int:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	name = $6;
	print "    " name " = VarArray(" isup ",-" MAXINT "," MAXINT ",'" name "')";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var range[(][-]*[0-9]+,1[+][-]*[0-9]+[)]:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	sub("range[(]","",$5);
	sub("[)]:","",$5);
	sub(",1[+]",",",$5);
	name = $6;
	print "    " name " = VarArray(" isup "," $5 ",'" name "')";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^array [[]range[(][-]*[0-9]+,1[+][-]*[0-9]+[)][]] of var {[-]*[0-9]+(,[-]*[0-9]+)*}:/{
	sub("[[]range[(]","",$2);
	sub("[)][]]","",$2);
	sub(",1[+]",",",$2);
	iinf = $2;
	sub(",.*","",iinf);
	if (iinf != 1) {
		print "ERROR ASSUMING MINIMUM INDEX VALUE EQUAL TO ONE IN ARRAY DEFINITION",$0;
		error = 1;
		exit(1);
	}
	isup = $2;
	sub(".*,","",isup);
	if (isup < iinf) {
		print "ERROR WRONG RANGE INDEX VALUE IN ARRAY DEFINITION",$0;
		error = 2;
		exit(2);
	}
	sub("{","[",$5);
	sub("}:","]",$5);
	name = $6;
	print "    " name " = [Variable(" $5 ",'" name "_" i "') for i in range(" isup ")]";
	if (match($0,"::_output_")) {
		output[ name ] = isup;
		outputstring[ name ] = substr($0, RSTART+RLENGTH);
	}
	if ($7 == "=") {
		sub(".*= ","");
		sub("::_output_.*","");
		gsub(" ","");
		for (i=0; i<isup; i++) {
			print "    model.add(" name "[" i "] == " $0 "[" i "])";
		}
	}
}

/^constraint /{
	$1 = "    model.add(";
    	$NF = $NF " )";
	print $0;
}

/^solve / && / minimize /{
	print "    model.add(Minimize(" $NF "))";
	objective = $NF;
}

/^solve / && / maximize /{
	print "    model.add(Maximize(" $NF "))";
	objective = $NF;
}

END {
	if (!error) {
	output_vars = "";
	if(objective) output_vars = objective;

	n = asorti(output,varnames);
	for (i=1; i<=n; i++) {
		e = varnames[i];
		if(e != objective) {
			if(length(output_vars) > 0) output_vars = output_vars ", ";
			output_vars = output_vars e;
		}
	}
	print "    output_vars = (" output_vars ")";
	print "    return model, output_vars";

	print "";
	print "model, output_vars = get_model()";
	if(output_vars) print output_vars " = output_vars";
	print "solvers = ['Mistral', 'SCIP', 'MiniSat', 'Toulbar2', 'Gurobi']";
	print "default = dict([('solver', 'Mistral'), ('verbose', 0), ('tcutoff', 900), ('var', 'DomainOverWDegree'), ('val', 'Lex'), ('rand', 2), ('threads', 1), ('restart', GEOMETRIC), ('base', 256), ('factor', 1.3)])";
	print "param = input(default)";
	print "solver = model.load(param['solver'])";
	print "solver.setVerbosity(param['verbose'])";
	print "solver.setTimeLimit(param['tcutoff'] - int(time.clock()+0.5))";
	print "solver.setHeuristic(param['var'], param['val'], param['rand'])";
	print "if param['solver'] == 'Gurobi':";
	print "    solver.setThreadCount(param['threads'])";
	print "if param['solver'] == 'Mistral':";
	print "    # LUBY, GEOMETRIC = 0, 1"
	print "    # print 'running %d %d %.1f' % (param['restart'], param['base'], param['factor'])";
	#print "    solver.solveAndRestart(GEOMETRIC, 256, 1.3)";
	print "    solver.solveAndRestart(param['restart'], param['base'], param['factor'])";
	print "else:";
	print "    solver.solve()";
	print "if solver.is_sat():"
	for (i=1; i<=n; i++) {
		e = varnames[i];
		if (output[e]==1) {
			if (e != objective && e != "objective" && e != "obj") print "    print '" e " = '," e ".get_value(),';'";
			else print "    print '" e " = ', (solver.getOptimum() if param['solver'] == 'Toulbar2' else " e ".get_value()),';'";
		} else {
			print "    print '" e " = " outputstring[e] ",',str(" e "),');'";
		}
	}
	print "    print '----------'";
	if (objective){
		print "    if solver.is_opt():"
		print "        print '=========='"
	} else {
		print "    print '=========='"
	}
	print "elif solver.is_unsat():"
	print "    print '=====UNSATISFIABLE====='"
    print "else:"
	print "    print '=====UNKNOWN====='"
	print "print '% SolveTime', solver.getTime()";
	print "print '% Nodes', solver.getNodes()";
	print "print '% Failures', solver.getFailures()";
	if (objective) print "if solver.is_sat(): print '% Objective', (solver.getOptimum() if param['solver'] == 'Toulbar2' else " objective ".get_value())";
	}
}


cleaning up header2params2.awk; relics from header2params.awk

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) { # NOTE: implemented in bin/util/user-account-cite.sh, too.
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami);
   }else if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # TODO: same as above.
   }else if(length(person_uri)  && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n",nowXSD);
   }
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   } # END if(showConversionProcess > 0)
}

length(conversionID) { #NR == 1 && length(conversionID) {
      label=$0;
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS); # this is either 'rdfs' or 'todo' (raw and e1, respectively)
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}

END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}

# header2params2.sh
#
# new version of headers2params.sh, processing by line instead of parsing the first line.
#
# Used by $CSV2RDF4LOD_HOME/bin/convert.sh
#
# Parameters:
# -v surrogate
# -v sourceID
# -v datasetID
# -v datasetVersion
# -v subjectDiscriminator
# -v conversionID
#
# -v header
# -v dataStart
# -v dataEnd
#
# -v onlyIfCol
# -v repeatAboveIfEmptyCol
# -v interpretAsNull
#
# -v whoami
# -v machine_uri
# -v person_uri

BEGIN {
   showConversionProcess = length(conversionID) + length(subjectDiscriminator) + length(header) + length(dataStart) + length(interpretAsNull) + length(dataEnd);
   FS=","
   STEP = length(conversionID) ? sprintf("enhancement/%s",conversionID) : "raw";
   TYPE = length(conversionID) ? "Enhancement" : "Raw";
   if(length(conversionID)) {
      print "@prefix rdf:        <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
      print "@prefix rdfs:       <http://www.w3.org/2000/01/rdf-schema#> ."
   }
   RDFS="rdfs"
   if(length(conversionID)) {
      print "@prefix todo:       <http://www.w3.org/2000/01/rdf-schema#> ."
      RDFS="todo"
   }
   print "@prefix xsd:        <http://www.w3.org/2001/XMLSchema#> ."
   if(length(conversionID)) {
      print "@prefix owl:        <http://www.w3.org/2002/07/owl#> ."
      print "@prefix vann:       <http://purl.org/vocab/vann/> ."
      print "@prefix skos:       <http://www.w3.org/2004/02/skos/core#> ."
      print "@prefix time:       <http://www.w3.org/2006/time#> ."
      print "@prefix wgs:        <http://www.w3.org/2003/01/geo/wgs84_pos#> ."
      print "@prefix geonames:   <http://www.geonames.org/ontology#> ."
      print "@prefix dbpedia:    <http://dbpedia.org/resource/> ."
      print "@prefix con:        <http://www.w3.org/2000/10/swap/pim/contact#> ."
      print "@prefix muo:        <http://purl.oclc.org/NET/muo/muo#> ."
   }
   print "@prefix dcterms:    <http://purl.org/dc/terms/> ."
   print "@prefix void:       <http://rdfs.org/ns/void#> ."
   print "@prefix scovo:      <http://purl.org/NET/scovo#> ."
   print "@prefix sioc:       <http://rdfs.org/sioc/ns#> ."
   print "@prefix foaf:       <http://xmlns.com/foaf/0.1/> ."
   #if(showConversionProcess>0) {
      print "@prefix ov:         <http://open.vocab.org/terms/> ."
   #}
   print "@prefix conversion: <http://purl.org/twc/vocab/conversion/> ."

   # Converter produces URIs for the LayerDatasets:
   #
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/raw>
   # <http://logd.tw.rpi.edu/source/nitrd-gov/dataset/fedRDnetIT/version/2011-Jan-27/conversion/enhancement/1>
   #
   # To make the params more connected to the datasets, we're replacing this:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/params/%s/> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
   # with this:
   # Still not right; moved down to subject:
   #printf("@prefix :           <%s/source/%s/dataset/%s/version/%s/conversion/%s> .\n",surrogate,sourceID,datasetID,datasetVersion,STEP);

                                     print
   # NOTE: user account and person implemented in bin/util/user-account.sh, too.
   if( length(machine_uri) && length(whoami) ) {
      if(length(person_uri)) {
                                    printf("<%s> foaf:holdsAccount <%s%s> .\n",person_uri,   machine_uri,whoami);
      }
                                    printf("<%s%s>\n   a foaf:OnlineAccount;\n   foaf:accountName \"%s\";\n",machine_uri,whoami,   whoami);
                                    printf("   dcterms:isPartOf <%s>;\n",machine_uri);
      if(length(person_uri)) {
                                    printf("   sioc:account_of  <%s>;\n",person_uri);
      }
                                    printf(".\n");
   }else if(length(person_uri)&&length(whoami)) {
                                    printf("<%s> dcterms:identifier \"%s\" .\n",person_uri,whoami);
   }
                                     print
                                    printf("<%s/source/%s/dataset/%s/version/%s/conversion/%s>\n",surrogate,sourceID,datasetID,datasetVersion,STEP);
                                     print "   a conversion:LayerDataset, void:Dataset;"
                                    printf("   conversion:base_uri           \"%s\"^^xsd:anyURI;\n",surrogate);
                                    printf("   conversion:source_identifier  \"%s\";\n",sourceID);
                                    printf("   conversion:dataset_identifier \"%s\";\n",datasetID);
                                    printf("   conversion:version_identifier \"%s\";\n",datasetVersion);
                                    #printf("   conversion:dataset_version    \"%s\"; # DEPRECATED in favor of version_identifier\n",datasetVersion);
   if(showConversionProcess > 0) {
                                     print "   conversion:conversion_process ["
                                    printf("      a conversion:%sConversionProcess;\n",TYPE);
   if(length(conversionID))         printf("      conversion:enhancement_identifier \"%s\";\n",conversionID);
   if(length(subjectDiscriminator)) printf("      conversion:subject_discriminator  \"%s\";\n",subjectDiscriminator);
                                     print
   if(length(person_uri)&&length(machine_uri)&&length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami);
   }else if(length(machine_uri) && length(whoami)) {
                                    printf("      dcterms:creator <%s%s>;\n",machine_uri,whoami); # TODO: same as above.
   }else if(length(person_uri)  && length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\";\n",whoami);
                                    printf("                        sioc:account_of <%s> ];\n",person_uri);
   }else if(length(whoami)) {
                                    printf("      dcterms:creator [ a foaf:OnlineAccount; foaf:accountName \"%s\" ];\n",whoami);
   }
   if(length(nowXSD)) {
                                    printf("      dcterms:created \"%s\"^^xsd:dateTime;\n",nowXSD);
   }
   if(length(header)) {
   # TODO: include header info even if just raw.
                                    printf("      conversion:enhance [      \n");
                                    printf("         ov:csvRow %s;\n",header);
                                    printf("         a conversion:HeaderRow;\n");
                                    printf("      ];                        \n");
   }
   if(length(dataStart)) {
                                    printf("      conversion:enhance [          \n");
                                    printf("         ov:csvRow %s;\n",dataStart);
                                    printf("         a conversion:DataStartRow; \n");
                                    printf("      ];                            \n");
   }
   if(length(interpretAsNull)) {
                                    printf("      conversion:interpret [          \n");
                                    printf("         conversion:symbol \"%s\";\n",interpretAsNull);
                                    printf("         conversion:intepretation conversion:null; \n");
                                    printf("      ];                            \n");
   }
   if(length(dataEnd)) {
                                    printf("      conversion:enhance [        \n");
                                    printf("         ov:csvRow %s;\n",dataEnd);
                                    printf("         a conversion:DataEndRow; \n");
                                    printf("      ];                          \n");
   }
   } # END if(showConversionProcess > 0)
}
#NR == 1 && length(conversionID) {
length(conversionID) {
#   for(i=1;i<=NF;i++) {
#     label=$i;
      label=$0;
#     gsub(/"/,"",label);
      (length(label)>0) ? sprintf("\n         ov:csvHeader      \"%s\";",label) : "";
      print "      conversion:enhance ["
      printf("         ov:csvCol         %s;\n",NR)
      printf("         ov:csvHeader     \"%s\";\n",label) # TODO: if HeaderRow == 0, s/csvHeader/eg/
      if(length(conversionID)) {
         printf("         conversion:label \"%s\";\n",label);
         printf("         conversion:comment \"\";\n");
      }
      printf("         conversion:range  %s:Literal;\n",RDFS); # this is either 'rdfs' or 'todo' (raw and e1, respectively)
      if( length(onlyIfCol) && onlyIfCol == i ) {
         print "         a conversion:Only_if_column;"
      }
      if( length(repeatAboveIfEmptyCol) && repeatAboveIfEmptyCol == NR ) {
         print "         a conversion:Repeat_previous_if_empty_column;"
      }
      print "      ];"
   #}
}
END {
   if (showConversionProcess > 0) print "   ];"
   printf(".");
}