pass on less three or less

pass on less three or less

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology namespace they are from. (curi prefix )
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
    return substr(str,2,length(str)-2)
}

# break URI into tokens to use a a path in
# multi-dim array pointing to a curie-prefix
# base URI include "!#&-=?_"
# which are all also present in localID as well
function tokenize (str){
    # b/c found in local IDs leave .:
    gsub(/[!&:#=?/_-]+/, SUBSEP, str)
    # discard the first token it is only low info content  http|https|ftp
    # discard any empty trailing tokens
    tail = match(str, ".*" SUBSEP)  # side effects RSTART & RLENGTH
    # print "tail = " tail " tail length = " RLENGTH
    finish = 0
    start = index(str, SUBSEP) + 1
    if (tail > 0)
        finish = RLENGTH - start
    else
        finish = length(str) - start + 1
    return substr(str, start, finish)
}

# remove final token from token path delimited with SUBSEP
# this ends up matching chars included in localID
# ._-;():%#,?=@*\&+   (yea ... nope)

function detail(key,   start) {
    start = match(key, ".+" SUBSEP)
    if (start == 1)
        return substr(key, 1, RLENGTH -1)
    else
        return key
}

# for output, keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
    gsub(/^_*|_*$/, "",str)
    gsub(/__*/,"_",str)
    return str
}

# Replace IRI with CuriePrefix when possible
# Otherwise whinge and return a gv printable version of the original
function contract(uri,  u){
    u = tokenize(uri)

    # sometimes uri is already correct as is
    # if (index(u, SUBSEP) > 0)
    #    u = detail(u)
    # if (u == "") {
    #    printf("ERROR contracting %s\n", uri) > "/dev/stderr"
    #    printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
    #    printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
    #    # exit(1)  # while testing
    # }

    # shorten till longest uri in curi map is found (or not)
    while(!(u in prefix) && index(u, SUBSEP)>0)
        u = detail(u)

    if(u in prefix)
        return prefix[u]
    else{
        # printf("WARN  %s\n", uri) > "/dev/stderr"  # too many
        # printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
        warn[uri]++
        return simplify(uri)
    }
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
    split(uri, b, "/")
    p = b[length(b)]
    anchor = match(p, "#")
    if(anchor > 0)
        p = substr(p, anchor+1)
    return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123
# is this still needed circa 2020?
function deoboify(curie,  a){
    split(curie, a , ":")
    if(match(a[2], a[1]"_") > 0){
        sub("_", ":", a[2]);
        curie = a[2]
    }
    return curie
}

BEGIN{
    # exceptions
    prefix["BNODE"]="BNODE"  # is a fixed point
    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
    ############################################################
    # not expected to be added to curie map
    # in HPOA
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"

    ############################################################
    # Often re-visit whether these exceptions are still necessary
    # they may have been moved into the curie prefix mapping file
    # or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

    # in mgi/impc
    prefix[tokenize("https://www.mousephenotype.org/")]="IMPC"
    # in IMPC (not httpS  --- hmm all three IRI exist)
    # note in curie_map IMPC is:
    # http://www.mousephenotype.org/data/genes/

    # in panther
    # prefix[tokenize("http://identifiers.org/wormbase/"])="WormBase"
    # note in curie_map WormBase is
    # 'https://www.wormbase.org/get?name='

    # various, transient?
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
    # kegg
    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/")]="KEGG-img"
    # in EOM
    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
    # prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS

    # playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
    # and their IRI are unroutable, i.e in same machine (or local net)
    # this is motivated in part by not wanting to publish
    # urls to our own servers which go nowhere.
    # prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
    # prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
    # prefix["http://0.0.0.0/"]="LCL"                              # ID halfway house

    # LCL's obvious down side is zero options for global collisions
    # best you can do is go the uuid which is ugly, or
    # belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# convert the base url to a path in a multi-dimensional array
# (FNR == NR) && /^'[fht]\+tp[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
    split($0, arr, "'")
    if(arr[2]=="")
        arr[2]="BASE"
    prefix[tokenize(arr[4])]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    o = contract(trim($3))
    edgelist[s,ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
    s = contract(trim($1))
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist[s, ns ":" p, "LITERAL"]++
    done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
    # once our bnode syntax is uniform we may want more info out of it
    # s = contract(($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    ### Object (like subject)
    o = contract(trim($3))
    edgelist["BNODE",ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    #o = contract($3))
    edgelist[s,ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
    # s = contract($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    # o = contract($3)
    edgelist["BNODE",ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {  # "balance quote for wonky hiliter
    # s = contract($1)
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist["BNODE", ns ":" p, "LITERAL"]++
    done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
    printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
    # output the graphviz directed graph
    print "digraph {"
    print "rankdir=LR;"
    print "charset=\"utf-8\";"
    # sort the edges to facilitate future inter dot comparisons
    n=asorti(edgelist, sortlist)
    for (i=1; i<=n; i++) {
        edge=sortlist[i]
        split(edge ,spo, SUBSEP);
        print simplify(spo[1]) " -> " simplify(spo[3]) \
        " [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
    }
    print "LITERAL [shape=record];"
    print "labelloc=\"t\";"
    title = final(FILENAME)
    datestamp = strftime("%Y%m%d", systime())
    print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
    print "}"

    ######################################################
    # dump a summary of significant numbers of unresolved-curi uri  to stderr
    # FNR gives at least half the number ot URI attempted to resolve for %s
    # FILENAME gives filename we know where they were generated
    # have warn[uri] = count
    toofew = 3

    n = asorti(warn, wsort, "@val_num_desc")
    for(i=1;i<=n;i++){
        if(warn[wsort[i]] > toofew)
            printf("%s\t%s\t%i\n", FILENAME, wsort[i], warn[wsort[i]]) > "/dev/stderr"
        else
            i = n + 1
    }
}

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology namespace they are from. (curi prefix )
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
    return substr(str,2,length(str)-2)
}

# break URI into tokens to use a a path in
# multi-dim array pointing to a curie-prefix
# base URI include "!#&-=?_"
# which are all also present in localID as well
function tokenize (str){
    # b/c found in local IDs leave .:
    gsub(/[!&:#=?/_-]+/, SUBSEP, str)
    # discard the first token it is only low info content  http|https|ftp
    # discard any empty trailing tokens
    tail = match(str, ".*" SUBSEP)  # side effects RSTART & RLENGTH
    # print "tail = " tail " tail length = " RLENGTH
    finish = 0
    start = index(str, SUBSEP) + 1
    if (tail > 0)
        finish = RLENGTH - start
    else
        finish = length(str) - start + 1
    return substr(str, start, finish)
}

# remove final token from token path delimited with SUBSEP
# this ends up matching chars included in localID
# ._-;():%#,?=@*\&+   (yea ... nope)

function detail(key,   start) {
    start = match(key, ".+" SUBSEP)
    if (start == 1)
        return substr(key, 1, RLENGTH -1)
    else
        return key
}

# for output, keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
    gsub(/^_*|_*$/, "",str)
    gsub(/__*/,"_",str)
    return str
}

# Replace IRI with CuriePrefix when possible
# Otherwise whinge and return a gv printable version of the original
function contract(uri,  u){
    u = tokenize(uri)

    # sometimes uri is already correct as is
    # if (index(u, SUBSEP) > 0)
    #    u = detail(u)
    # if (u == "") {
    #    printf("ERROR contracting %s\n", uri) > "/dev/stderr"
    #    printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
    #    printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
    #    # exit(1)  # while testing
    # }

    # shorten till longest uri in curi map is found (or not)
    while(!(u in prefix) && index(u, SUBSEP)>0)
        u = detail(u)

    if(u in prefix)
        return prefix[u]
    else{
        # printf("WARN  %s\n", uri) > "/dev/stderr"  # too many
        # printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
        warn[uri]++
        return simplify(uri)
    }
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
    split(uri, b, "/")
    p = b[length(b)]
    anchor = match(p, "#")
    if(anchor > 0)
        p = substr(p, anchor+1)
    return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123
# is this still needed circa 2020?
function deoboify(curie,  a){
    split(curie, a , ":")
    if(match(a[2], a[1]"_") > 0){
        sub("_", ":", a[2]);
        curie = a[2]
    }
    return curie
}

BEGIN{
    # exceptions
    prefix["BNODE"]="BNODE"  # is a fixed point
    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
    ############################################################
    # not expected to be added to curie map
    # in HPOA
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"

    ############################################################
    # Often re-visit whether these exceptions are still necessary
    # they may have been moved into the curie prefix mapping file
    # or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

    # in mgi/impc
    prefix[tokenize("https://www.mousephenotype.org/")]="IMPC"
    # in IMPC (not httpS  --- hmm all three IRI exist)
    # note in curie_map IMPC is:
    # http://www.mousephenotype.org/data/genes/

    # in panther
    # prefix[tokenize("http://identifiers.org/wormbase/"])="WormBase"
    # note in curie_map WormBase is
    # 'https://www.wormbase.org/get?name='

    # various, transient?
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
    # kegg
    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/")]="KEGG-img"
    # in EOM
    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
    # prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS

    # playing with the idea of a LOCAL identifier
    # not the same as a bnode in that tools won't rewrite identifier
    # and their IRI are unroutable, i.e in same machine (or local net)
    # this is motivated in part by not wanting to publish
    # urls to our own servers which go nowhere.
    # prefix["https://127.0.0.1/.well-known/genid"]="BNODE"  # exists, phase out
    # prefix["http://0.0.0.0/.well-known/genid"]="BNODE"     # prefer
    # prefix["http://0.0.0.0/"]="LCL"                              # ID halfway house

    # LCL's obvious down side is zero options for global collisions
    # best you can do is go the uuid which is ugly, or
    # belive your obscure but readable string _is_ a special snowflake
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# a colon, whitespace
# then a tic delimited url
# convert the base url to a path in a multi-dimensional array
# (FNR == NR) && /^'[fht]\+tp[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
    split($0, arr, "'")
    if(arr[2]=="")
        arr[2]="BASE"
    prefix[tokenize(arr[4])]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]+> <[^>]+> \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    o = contract(trim($3))
    edgelist[s,ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
    s = contract(trim($1))
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist[s, ns ":" p, "LITERAL"]++
    done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
    # once our bnode syntax is uniform we may want more info out of it
    # s = contract(($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    ### Object (like subject)
    o = contract(trim($3))
    edgelist["BNODE",ns ":" p,o]++
    done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
    s = contract(trim($1))
    p =  final(trim($2))
    ns = contract(trim($2))
    #o = contract($3))
    edgelist[s,ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
    # s = contract($1)
    p =  final(trim($2))
    ns = contract(trim($2))
    # o = contract($3)
    edgelist["BNODE",ns ":" p,"BNODE"]++
    done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {  # "balance quote for wonky hiliter
    # s = contract($1)
    p = final(trim($2))
    ns = contract(trim($2))
    edgelist["BNODE", ns ":" p, "LITERAL"]++
    done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
    printf("ERROR? in %s at line %i:  %s\n",FILENAME,FNR,$0) > "/dev/stderr"
}

# output graphviz dot file, include edge counts
END{
    # output the graphviz directed graph
    print "digraph {"
    print "rankdir=LR;"
    print "charset=\"utf-8\";"
    # sort the edges to facilitate future inter dot comparisons
    n=asorti(edgelist, sortlist)
    for (i=1; i<=n; i++) {
        edge=sortlist[i]
        split(edge ,spo, SUBSEP);
        print simplify(spo[1]) " -> " simplify(spo[3]) \
        " [label=\"" deoboify(spo[2]) " (" edgelist[edge] ")\"];"
    }
    print "LITERAL [shape=record];"
    print "labelloc=\"t\";"
    title = final(FILENAME)
    datestamp = strftime("%Y%m%d", systime())
    print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
    print "}"

    ######################################################
    # dump a summary of significant numbers of unresolved-curi uri  to stderr
    # FNR gives at least half the number ot URI attempted to resolve for %s
    # FILENAME gives filename we know where they were generated
    # have warn[uri] = count
    toofew = 3

    n = asorti(warn, wsort, "@val_num_desc")
    for(i=1;i<=n;i++){
        if(warn[wsort[i]] >= toofew)
            printf("%s\t%s\t%i\n", FILENAME, wsort[i], warn[wsort[i]]) > "/dev/stderr"
        else
            i = n + 1
    }
}


Do not check nisas

Do not check nisas

#!/usr/bin/awk -f
BEGIN {
    FS = "\t";

    URN           = "urn:swn:";

    LABEL         = "<http://www.w3.org/2000/01/rdf-schema#label>";
    TYPE          = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";

    CONCEPT       = "<http://www.w3.org/2004/02/skos/core#Concept>";
    LEXICALSENSE  = "<http://lemon-model.net/lemon#LexicalSense>";
    WORD          = "<http://lemon-model.net/lemon#Word>";
    FORM          = "<http://lemon-model.net/lemon#Form>";

    WRITTENREP    = "<http://lemon-model.net/lemon#writtenRep>";
    CANONICALFORM = "<http://lemon-model.net/lemon#canonicalForm>";
    BROADER       = "<http://lemon-model.net/lemon#broader>";
    NARROWER      = "<http://lemon-model.net/lemon#narrower>";
    SENSE         = "<http://lemon-model.net/lemon#sense>";
    ISSENSEOF     = "<http://lemon-model.net/lemon#isSenseOf>";
    REFERENCE     = "<http://lemon-model.net/lemon#reference>";
    ISREFERENCEOF = "<http://lemon-model.net/lemon#isReferenceOf>";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0) next;

    print "<" URN "c" $1 ">", TYPE,  CONCEPT,            ".";
    print "<" URN "c" $1 ">", LABEL, "\"synset" $1 "\"", ".";

    for (sense_id in senses) {
        sense = senses[sense_id];

        sub(":.*$", "", sense);

        if (!(sense in sindex)) {
            sindex[sense] = scount++;

            word = sense;
            gsub("_", " ",  word);
            sub("#.*$", "", word);

            if (!(word in windex)) {
                windex[word] = wcount++;

                print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
            }

            print "<" URN "s" sindex[sense] ">", TYPE,          LEXICALSENSE,                  ".";
            print "<" URN "s" sindex[sense] ">", LABEL,         "\"" sense "\"",               ".";
            print "<" URN "s" sindex[sense] ">", ISSENSEOF,     "<" URN "w" windex[word] ">",  ".";
            print "<" URN "w" windex[word]  ">", SENSE,         "<" URN "s" sindex[sense] ">", ".";
            print "<" URN "s" sindex[sense] ">", REFERENCE,     "<" URN "c" $1 ">",            ".";
            print "<" URN "c" $1 ">",            ISREFERENCEOF, "<" URN "s" sindex[sense] ">", ".";
        }

        for (isa_id in isas) {
            isa = isas[isa_id];

            sub(":.*$", "", isa);

            if (!(isa in sindex)) {
                sindex[isa] = scount++;

                word = isa;
                gsub("_", " ",  word);
                sub("#.*$", "", word);

                if (!(word in windex)) {
                    windex[word] = wcount++;

                    print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                    print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                    print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                    print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
                }

                print "<" URN "s" sindex[isa] ">",  TYPE,      LEXICALSENSE,                 ".";
                print "<" URN "s" sindex[isa] ">",  LABEL,     "\"" isa "\"",                ".";
                print "<" URN "s" sindex[isa] ">",  ISSENSEOF, "<" URN "w" windex[word] ">", ".";
                print "<" URN "w" windex[word] ">", SENSE,     "<" URN "s" sindex[isa] ">",  ".";
            }

            print "<" URN "s" sindex[sense] ">", BROADER,  "<" URN "s" sindex[isa] ">",   ".";
            print "<" URN "s" sindex[isa] ">",   NARROWER, "<" URN "s" sindex[sense] ">", ".";
        }
    }
}

#!/usr/bin/awk -f
BEGIN {
    FS = "\t";

    URN           = "urn:swn:";

    LABEL         = "<http://www.w3.org/2000/01/rdf-schema#label>";
    TYPE          = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";

    CONCEPT       = "<http://www.w3.org/2004/02/skos/core#Concept>";
    LEXICALSENSE  = "<http://lemon-model.net/lemon#LexicalSense>";
    WORD          = "<http://lemon-model.net/lemon#Word>";
    FORM          = "<http://lemon-model.net/lemon#Form>";

    WRITTENREP    = "<http://lemon-model.net/lemon#writtenRep>";
    CANONICALFORM = "<http://lemon-model.net/lemon#canonicalForm>";
    BROADER       = "<http://lemon-model.net/lemon#broader>";
    NARROWER      = "<http://lemon-model.net/lemon#narrower>";
    SENSE         = "<http://lemon-model.net/lemon#sense>";
    ISSENSEOF     = "<http://lemon-model.net/lemon#isSenseOf>";
    REFERENCE     = "<http://lemon-model.net/lemon#reference>";
    ISREFERENCEOF = "<http://lemon-model.net/lemon#isReferenceOf>";
}
{
    nsenses = split($3, senses, ", ");
    nisas   = split($5, isas,   ", ");

    if (nsenses == 0 || nisas == 0) next;

    print "<" URN "c" $1 ">", TYPE,  CONCEPT,            ".";
    print "<" URN "c" $1 ">", LABEL, "\"synset" $1 "\"", ".";

    for (sense_id in senses) {
        sense = senses[sense_id];

        sub(":.*$", "", sense);

        if (!(sense in sindex)) {
            sindex[sense] = scount++;

            word = sense;
            gsub("_", " ",  word);
            sub("#.*$", "", word);

            if (!(word in windex)) {
                windex[word] = wcount++;

                print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
            }

            print "<" URN "s" sindex[sense] ">", TYPE,          LEXICALSENSE,                  ".";
            print "<" URN "s" sindex[sense] ">", LABEL,         "\"" sense "\"",               ".";
            print "<" URN "s" sindex[sense] ">", ISSENSEOF,     "<" URN "w" windex[word] ">",  ".";
            print "<" URN "w" windex[word]  ">", SENSE,         "<" URN "s" sindex[sense] ">", ".";
            print "<" URN "s" sindex[sense] ">", REFERENCE,     "<" URN "c" $1 ">",            ".";
            print "<" URN "c" $1 ">",            ISREFERENCEOF, "<" URN "s" sindex[sense] ">", ".";
        }

        for (isa_id in isas) {
            isa = isas[isa_id];

            sub(":.*$", "", isa);

            if (!(isa in sindex)) {
                sindex[isa] = scount++;

                word = isa;
                gsub("_", " ",  word);
                sub("#.*$", "", word);

                if (!(word in windex)) {
                    windex[word] = wcount++;

                    print "<" URN "w" windex[word] ">", TYPE,          WORD,                         ".";
                    print "<" URN "w" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", TYPE,          FORM,                         ".";
                    print "<" URN "f" windex[word] ">", LABEL,         "\"" word "\"",               ".";
                    print "<" URN "f" windex[word] ">", WRITTENREP,    "\"" word "\"",               ".";
                    print "<" URN "w" windex[word] ">", CANONICALFORM, "<" URN "f" windex[word] ">", ".";
                }

                print "<" URN "s" sindex[isa] ">",  TYPE,      LEXICALSENSE,                 ".";
                print "<" URN "s" sindex[isa] ">",  LABEL,     "\"" isa "\"",                ".";
                print "<" URN "s" sindex[isa] ">",  ISSENSEOF, "<" URN "w" windex[word] ">", ".";
                print "<" URN "w" windex[word] ">", SENSE,     "<" URN "s" sindex[isa] ">",  ".";
            }

            print "<" URN "s" sindex[sense] ">", BROADER,  "<" URN "s" sindex[isa] ">",   ".";
            print "<" URN "s" sindex[isa] ">",   NARROWER, "<" URN "s" sindex[sense] ">", ".";
        }
    }
}


Removed renamed file

Removed renamed file

#! /bin/bash -

#########################
#
# Name: rewrite-to-bash-case.bash
#
# Purpose: Rewrite CMIP5-tables to bash case selection
#
# Usage: ./rewrite-to-bash-case.bash
#
# Revision history: 2016-02-25  --  Script created, Martin Evaldsson, Rossby Centre
#
# Contact persons:  martin.evaldsson@smhi.se
#
########################


bash: more generic name for count

bash: more generic name for count

min == "" {min=max=$1}
$1 < min  {min = $1}
$1 > max  {max = $1}
          {sum+=$1; sumsq+=$1*$1}
END {
  print "count: ", NR;
  print "min:   ", min;
  print "max:   ", max;
  print "sum:   ", sum;
  print "mean:  ", sum/NR;
  print "stddev:", sqrt(sumsq/NR - (sum/NR)**2)
}

min == "" {min=max=$1}
$1 < min  {min = $1}
$1 > max  {max = $1}
          {sum+=$1; sumsq+=$1*$1}
END {
  print "lines: ", NR;
  print "min:   ", min;
  print "max:   ", max;
  print "sum:   ", sum;
  print "mean:  ", sum/NR;
  print "stddev:", sqrt(sumsq/NR - (sum/NR)**2)
}


win32/version.awk: fixed whitespace

win32/version.awk: fixed whitespace

darcs-hash:20081127232850-db55f-a65203bc70bccd003a74acf8fe55d78d4f639b41.gz

#! /bin/gawk
#
# This script extracts the version information from configure.ac
# and re-generates win32/config.h and
# libsofia-sip-ua/features/sofia_sip_features.h
#
# --------------------------------------------------------------------
#
# This file is part of the Sofia-SIP package
#
# Copyright (C) 2005 Nokia Corporation.
#
# Contact: Pekka Pessi <pekka.pessi@nokia.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
#
# --------------------------------------------------------------------
#
# Contributor(s): Pekka.Pessi@nokia.com.
#
# Created: Wed Jan 25 15:57:10 2006 ppessi
#

BEGIN { IN=1; OUT=0; }

IN && /^AC_INIT/ { version=$2; gsub(/[\]\[)]/, "", version); }

OUT && /@[A-Z_]+@/ {
  gsub(/@PACKAGE_VERSION@/, version);
  gsub(/@PACKAGE_BUGREPORT@/, "sofia-sip-devel@lists.sourceforge.net");
  gsub(/@PACKAGE_NAME@/, "sofia-sip");
  gsub(/@PACKAGE@/, "sofia-sip");
  gsub(/@PACKAGE_STRING@/, "sofia-sip");
  gsub(/@PACKAGE_TARNAME@/, "sofia-sip");
}

OUT { print; }


Rearranged columns in outfile reports

Rearranged columns in outfile reports

# Print the processed "Episode" lines from a WebScraper csv file saved in tsv format

# INVOCATION:
#       awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getBBoxEpisodesFrom-webscraper.awk $EPISODES_SORTED_FILE >$EPISODES_SPREADSHEET_FILE

# Field numbers
#    1 web-scraper-start-url  2 SsnURL     3 SsnURL-href    4 URL        5 Program_Title
#    6 Episode_Title          7 Year       8 Duration       9 Rating    10 Description

BEGIN {
    FS="\t"
    OFS="\t"
    print "Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription"
    # For debugging variations in seasons and episodes format
    str = "date +%y%m%d.%H%M%S"
    str | getline longdate
    close str
    for ( i = 1; i <= 4; i++ ) {
        outfile = "BBox-scrapes/c" i "-" longdate ".csv"
        print "Type\t#\tSortkey\tURL\tshowTitle\tepisodeTitle\tYear(s)" > outfile
    }
}

{
    for ( i = 1; i <= NF; i++ ) {
        if ($i == "null")
            $i = ""
    }
}

/\/us\/movie\/|\/us\/episode\// {
    baseURL = $1
    URL = $4
    showTitle = $5
    episodeTitle = $6
    Years = $7
    Duration = $8
    Rating = $9
    Description = $10
    showtype = "S"

    shortURL = URL
    sub (/^\/us\/episode\//,"",shortURL)
    episodeSeason = ""
    episodeNumber = ""
    # Season 3, Episode 8
    if (match (episodeTitle, /^Season [[:digit:]]+|^Series [[:digit:]]+/))
        episodeSeason = substr(episodeTitle,8,RLENGTH-7)

    # Extract sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_S[[:digit:]]*_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Days_S3_E8_23851
        seasonNumber = substr(fld[nflds-2], 2)
        episodeNumber = substr(fld[nflds-1], 2)
        if (baseURL ~ /The_Edinburgh_Show_23924$/) {
            episodeSeason = Years
        }
        # Give precedence to season number in episodeTitle over season number in URL
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            revisedSeasons1 += 1
            printf ("==> Fixed c1 season: %02d-%02d\t%s\t%s\t%s\n", \
                    seasonNumber, episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c1Num += 1
        outfile = "BBox-scrapes/c1-" longdate ".csv"
        printf ("c1\t%d\t%s\t%s\t%s\t%s\t%s\n",c1Num,sortkey,showTitle,shortURL,episodeTitle,Years) >> outfile
    } else if (URL ~ /_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Hours_E1_26129
        revisedSeasons2 += 1
        episodeSeason = Years
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            printf ("==> Fixed c2 season: %02d-%02d\t%s\t%s\t%s\n", \
                    seasonNumber, episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c2Num += 1
        outfile = "BBox-scrapes/c2-" longdate ".csv"
        printf ("c2\t%d\t%s\t%s\t%s\t%s\t%s\n",c2Num,sortkey,showTitle,shortURL,episodeTitle,Years) >> outfile
    } else if (URL ~ /Episode_[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/Episode_5_26089
        revisedSeasons3 += 1
        seasonNumber = Years
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            printf ("==> Fixed c3 season: %02d-%02d\t%s\t%s\t%s\n", \
                    seasonNumber, episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        episodeNumber = substr(fld[nflds-1], 1)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c3Num += 1
        outfile = "BBox-scrapes/c3-" longdate ".csv"
        printf ("c3\t%d\t%s\t%s\t%s\t%s\t%s\n",c3Num,sortkey,showTitle,shortURL,episodeTitle,Years) >> outfile
    } else {
        revisedSeasons4 += 1
        URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "E"
        # /us/episode/Orkneys_Stone_Age_Temple_7144
        # A History of Ancient Britain Special, Orkney's Stone Age Temple
        sortkey = sprintf ("%s%05d", showtype, fld[nflds])
        if (episodeSeason != "") {
            showtype = "S"
            sortkey = sprintf ("%s%02dE%05d", showtype, episodeSeason, 99999 - fld[nflds])
            printf ("==> Fixed c4 season: null-%02d\t%s\t%s\t%s\n", \
                    episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
        }
        c4Num += 1
        outfile = "BBox-scrapes/c4-" longdate ".csv"
        printf ("c4\t%d\t%s\t%s\t%s\t%s\t%s\n",c4Num,sortkey,showTitle,shortURL,episodeTitle,Years) >> outfile
    }

    # Convert duration from minutes to HMS
    sub( / min/,"",Duration)
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (HMS == "00:00:00")
        HMS = ""

    # Special processing for some shows is needed in all getBBox*From-webscraper.awk scripts.
    # "Changed" message and counting revisedTitles are only in getBBoxProgramsFrom-webscraper.awk
    # so they are not repeated for each episode and each season.
    if (baseURL ~ /Maigret_15974$/) {
        showTitle = "Maigret (2016)"
    }
    if (baseURL ~ /Maigret_\(1992\)_15928$/) {
        showTitle = "Maigret (1992)"
    }
    if (baseURL ~ /Porridge_9509$/) {
        showTitle = "Porridge (1974-1977)"
    }
    if (baseURL ~ /Porridge_14747$/) {
        showTitle = "Porridge (2016-2017)"
    }

    # Get rid of redundant "Series #" or "Series #" from episodeTitle
    if (match (episodeTitle, /^Season [[:digit:]]*, |^Series [[:digit:]]*, /))
        episodeTitle = substr(episodeTitle,RLENGTH+1)

    # Get rid of initial/trailing quotes in showTitle
    # "BBC Three's ""Murdered By..."" Collection Trailer"
    if (showTitle ~ /^".*"$/) {
        revisedTitles += 1
        printf ("==> Fixed quoted title: www.britbox.com%s\n", URL) >> ERROR_FILE
        # printf ("    %s\n", showTitle) >> ERROR_FILE
        sub (/^"/,"",showTitle)
        sub (/"$/,"",showTitle)
        # printf ("    %s\n", showTitle) >> ERROR_FILE
    }

    # Non-existent fields in movies or episodes
    if (URL ~ /^\/us\/movie\//) {
        depth = "1"
        numSeasons = 1
        numEpisodes = 1
        fullTitle = showTitle
    } else {
        depth = "2"
        numSeasons = ""
        numEpisodes = ""
        fullTitle = showTitle ", " sortkey ", " episodeTitle
    }

    # Fix two consecutive double quotes in showTitle, but leave in fullTitle
    gsub ("\"\"","\"",showTitle)

    # Fix HEX quotes in Description
    gsub ("\xc2\x91","\"",Description)
    gsub ("\xc2\x92","\"",Description)

    # Create spreadsheet row in common format
    savedLine = sprintf \
        ("%s (%s) %s %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
         showTitle, depth, Years, sortkey, URL, fullTitle, numSeasons, numEpisodes, \
         HMS, Years, Rating, Description)

    # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
    sub (/^'/,"",savedLine)

    print savedLine
    next
}

/\/us\/show\// {
    # Shouldn't get here
    badEpisodes += 1
    URL = $4
    nflds = split (URL,URLflds,"/")
    printf ("==> Upstream error in %s\n", FILENAME)  >> ERROR_FILE
    print "==> Found /us/show/ in episodes: " URLflds[nflds]  >> ERROR_FILE
}

END {
    printf ("In getBBoxEpisodesFrom-webscraper.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons1 > 0 ) {
        revisedSeasons1 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c1 %s revised in %s\n", revisedSeasons1, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons2 > 0 ) {
        revisedSeasons2 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c2 %s revised in %s\n", revisedSeasons2, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons3 > 0 ) {
        revisedSeasons3 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c3 %s revised in %s\n", revisedSeasons3, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons4 > 0 ) {
        revisedSeasons4 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c4 %s revised in %s\n", revisedSeasons4, field, FILENAME) > "/dev/stderr"
    }
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("    %2d extra /show/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}

# Print the processed "Episode" lines from a WebScraper csv file saved in tsv format

# INVOCATION:
#       awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getBBoxEpisodesFrom-webscraper.awk $EPISODES_SORTED_FILE >$EPISODES_SPREADSHEET_FILE

# Field numbers
#    1 web-scraper-start-url  2 SsnURL     3 SsnURL-href    4 URL        5 Program_Title
#    6 Episode_Title          7 Year       8 Duration       9 Rating    10 Description

BEGIN {
    FS="\t"
    OFS="\t"
    print "Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription"
    # For debugging variations in seasons and episodes format
    str = "date +%y%m%d.%H%M%S"
    str | getline longdate
    close str
    for ( i = 1; i <= 4; i++ ) {
        outfile = "BBox-scrapes/c" i "-" longdate ".csv"
        print "Type\t#\tSortkey\tURL\tshowTitle\tepisodeTitle\tYear(s)" > outfile
    }
}

{
    for ( i = 1; i <= NF; i++ ) {
        if ($i == "null")
            $i = ""
    }
}

/\/us\/movie\/|\/us\/episode\// {
    baseURL = $1
    URL = $4
    showTitle = $5
    episodeTitle = $6
    Years = $7
    Duration = $8
    Rating = $9
    Description = $10
    showtype = "S"

    shortURL = URL
    sub (/^\/us\/episode\//,"",shortURL)
    episodeSeason = ""
    episodeNumber = ""
    # Season 3, Episode 8
    if (match (episodeTitle, /^Season [[:digit:]]+|^Series [[:digit:]]+/))
        episodeSeason = substr(episodeTitle,8,RLENGTH-7)

    # Extract sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_S[[:digit:]]*_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Days_S3_E8_23851
        seasonNumber = substr(fld[nflds-2], 2)
        episodeNumber = substr(fld[nflds-1], 2)
        if (baseURL ~ /The_Edinburgh_Show_23924$/) {
            episodeSeason = Years
        }
        # Give precedence to season number in episodeTitle over season number in URL
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            revisedSeasons1 += 1
            printf ("==> Fixed c1 season: %02d-%02d\t%s\t%s\t%s\n", \
                    seasonNumber, episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c1Num += 1
        outfile = "BBox-scrapes/c1-" longdate ".csv"
        printf ("c1\t%d\t%s\t%s\t%s\t%s\t%s\n",c1Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Hours_E1_26129
        revisedSeasons2 += 1
        episodeSeason = Years
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            printf ("==> Fixed c2 season: %02d-%02d\t%s\t%s\t%s\n", \
                    seasonNumber, episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c2Num += 1
        outfile = "BBox-scrapes/c2-" longdate ".csv"
        printf ("c2\t%d\t%s\t%s\t%s\t%s\t%s\n",c2Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /Episode_[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/Episode_5_26089
        revisedSeasons3 += 1
        seasonNumber = Years
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            printf ("==> Fixed c3 season: %02d-%02d\t%s\t%s\t%s\n", \
                    seasonNumber, episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        episodeNumber = substr(fld[nflds-1], 1)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c3Num += 1
        outfile = "BBox-scrapes/c3-" longdate ".csv"
        printf ("c3\t%d\t%s\t%s\t%s\t%s\t%s\n",c3Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else {
        revisedSeasons4 += 1
        URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "E"
        # /us/episode/Orkneys_Stone_Age_Temple_7144
        # A History of Ancient Britain Special, Orkney's Stone Age Temple
        sortkey = sprintf ("%s%05d", showtype, fld[nflds])
        if (episodeSeason != "") {
            showtype = "S"
            sortkey = sprintf ("%s%02dE%05d", showtype, episodeSeason, fld[nflds])
            printf ("==> Fixed c4 season: null-%02d\t%s\t%s\t%s\n", \
                    episodeSeason, showTitle, shortURL, episodeTitle) >> ERROR_FILE
        }
        c4Num += 1
        outfile = "BBox-scrapes/c4-" longdate ".csv"
        printf ("c4\t%d\t%s\t%s\t%s\t%s\t%s\n",c4Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    }

    # Convert duration from minutes to HMS
    sub( / min/,"",Duration)
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (HMS == "00:00:00")
        HMS = ""

    # Special processing for some shows is needed in all getBBox*From-webscraper.awk scripts.
    # "Changed" message and counting revisedTitles are only in getBBoxProgramsFrom-webscraper.awk
    # so they are not repeated for each episode and each season.
    if (baseURL ~ /Maigret_15974$/) {
        showTitle = "Maigret (2016)"
    }
    if (baseURL ~ /Maigret_\(1992\)_15928$/) {
        showTitle = "Maigret (1992)"
    }
    if (baseURL ~ /Porridge_9509$/) {
        showTitle = "Porridge (1974-1977)"
    }
    if (baseURL ~ /Porridge_14747$/) {
        showTitle = "Porridge (2016-2017)"
    }

    # Get rid of redundant "Series #" or "Series #" from episodeTitle
    if (match (episodeTitle, /^Season [[:digit:]]*, |^Series [[:digit:]]*, /))
        episodeTitle = substr(episodeTitle,RLENGTH+1)

    # Get rid of initial/trailing quotes in showTitle
    # "BBC Three's ""Murdered By..."" Collection Trailer"
    if (showTitle ~ /^".*"$/) {
        revisedTitles += 1
        printf ("==> Fixed quoted title: www.britbox.com%s\n", URL) >> ERROR_FILE
        # printf ("    %s\n", showTitle) >> ERROR_FILE
        sub (/^"/,"",showTitle)
        sub (/"$/,"",showTitle)
        # printf ("    %s\n", showTitle) >> ERROR_FILE
    }

    # Non-existent fields in movies or episodes
    if (URL ~ /^\/us\/movie\//) {
        depth = "1"
        numSeasons = 1
        numEpisodes = 1
        fullTitle = showTitle
    } else {
        depth = "2"
        numSeasons = ""
        numEpisodes = ""
        fullTitle = showTitle ", " sortkey ", " episodeTitle
    }

    # Fix two consecutive double quotes in showTitle, but leave in fullTitle
    gsub ("\"\"","\"",showTitle)

    # Fix HEX quotes in Description
    gsub ("\xc2\x91","\"",Description)
    gsub ("\xc2\x92","\"",Description)

    # Create spreadsheet row in common format
    savedLine = sprintf \
        ("%s (%s) %s %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
         showTitle, depth, Years, sortkey, URL, fullTitle, numSeasons, numEpisodes, \
         HMS, Years, Rating, Description)

    # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
    sub (/^'/,"",savedLine)

    print savedLine
    next
}

/\/us\/show\// {
    # Shouldn't get here
    badEpisodes += 1
    URL = $4
    nflds = split (URL,URLflds,"/")
    printf ("==> Upstream error in %s\n", FILENAME)  >> ERROR_FILE
    print "==> Found /us/show/ in episodes: " URLflds[nflds]  >> ERROR_FILE
}

END {
    printf ("In getBBoxEpisodesFrom-webscraper.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons1 > 0 ) {
        revisedSeasons1 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c1 %s revised in %s\n", revisedSeasons1, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons2 > 0 ) {
        revisedSeasons2 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c2 %s revised in %s\n", revisedSeasons2, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons3 > 0 ) {
        revisedSeasons3 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c3 %s revised in %s\n", revisedSeasons3, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons4 > 0 ) {
        revisedSeasons4 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c4 %s revised in %s\n", revisedSeasons4, field, FILENAME) > "/dev/stderr"
    }
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("    %2d extra /show/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}


Main: update

Main: update

####################################################################
# Main.awk                                                         #
####################################################################

# Initialization.
function init() {
    initGawk()

    initBiDi()
    initLocale()
    initLocaleDisplay() #<< Locale, BiDi
    initUserLang()      #<< Locale

    RS = "\n"

    ExitCode = 0

    Option["debug"] = 0

    Option["verbose"] = 1
    Option["show-original"] = 1
    Option["show-original-phonetics"] = 1
    Option["show-translation"] = 1
    Option["show-translation-phonetics"] = 1
    Option["show-prompt-message"] = 1
    Option["show-languages"] = 1
    Option["show-original-dictionary"] = 0
    Option["show-dictionary"] = 1
    Option["show-alternatives"] = 1

    Option["no-ansi"] = 0

    Option["width"] = ENVIRON["COLUMNS"] ? ENVIRON["COLUMNS"] : 64
    Option["indent"] = 4

    Option["view"] = 0
    Option["pager"] = ENVIRON["PAGER"]

    Option["browser"] = ENVIRON["BROWSER"]

    Option["play"] = 0
    Option["player"] = ENVIRON["PLAYER"]

    Option["proxy"] = ENVIRON["HTTP_PROXY"] ? ENVIRON["HTTP_PROXY"] : ENVIRON["http_proxy"]
    Option["user-agent"] = ENVIRON["USER_AGENT"]

    Option["interactive"] = 0
    Option["no-rlwrap"] = 0
    Option["emacs"] = 0
    Option["prompt"] = ENVIRON["TRANS_PS"] ? ENVIRON["TRANS_PS"] : "%s>"
    Option["prompt-color"] = ENVIRON["TRANS_PS_COLOR"] ? ENVIRON["TRANS_PS_COLOR"] : "default"

    Option["input"] = NULLSTR
    Option["output"] = STDOUT

    Option["hl"] = ENVIRON["HOME_LANG"] ? ENVIRON["HOME_LANG"] : UserLang
    Option["sl"] = ENVIRON["SOURCE_LANG"] ? ENVIRON["SOURCE_LANG"] : "auto"
    Option["tl"][1] = ENVIRON["TARGET_LANG"] ? ENVIRON["TARGET_LANG"] : UserLang

    Option["theme"] = "default"
}

# Initialization script.
function initScript(    file, line, script, temp) {
    if (belongsTo("-no-init", ARGV) || belongsTo("--no-init", ARGV))
        return

    # Find the initialization file
    file = ".trans"
    if (!fileExists(file)) {
        file = ENVIRON["HOME"] "/.translate-shell/init.trans"
        if (!fileExists(file)) {
            file = "/etc/translate-shell"
            if (!fileExists(file)) return
        }
    }

    script = NULLSTR
    while (getline line < file)
        script = script "\n" line
    loadOptions(script)

    # Hack: Option["tl"] must be an array
    if (!isarray(Option["tl"])) {
        temp = Option["tl"]
        delete Option["tl"]
        Option["tl"][1] = temp
    }
}

# Theme initialization.
function initTheme() {
    # Set theme
    setTheme()

    # Disable ANSI escape codes if required
    if (Option["no-ansi"])
        delete AnsiCode
}

# Miscellany initialization.
function initMisc(    group) {
    initHttpService()

    # Initialize pager
    if (Option["view"]) {
        if (!Option["pager"]) {
            initPager()
            Option["pager"] = Pager
        }

        if (!Option["pager"]) {
            w("[WARNING] No available terminal pager.")
            Option["view"] = 0
        }
    }

    # Initialize browser
    if (!Option["browser"]) {
        "xdg-mime query default text/html" SUPERR |& getline Option["browser"]
        match(Option["browser"], "(.*).desktop$", group)
        Option["browser"] = group[1]
    }

    # Initialize audio player or speech synthesizer
    if (Option["play"]) {
        if (!Option["player"]) {
            initAudioPlayer()
            Option["player"] = AudioPlayer ? AudioPlayer : Option["player"]
            if (!Option["player"])
                initSpeechSynthesizer()
        }

        if (!Option["player"] && !SpeechSynthesizer) {
            w("[WARNING] No available audio player or speech synthesizer.")
            Option["play"] = 0
        }
    }
}

# Main entry point.
BEGIN {
    init()
    initScript()

    # Option parsing
    pos = 0
    while (ARGV[++pos]) {
        # -, -no-op
        match(ARGV[pos], /^-(-?no-op)?$/)
        if (RSTART) continue

        # -V, -version
        match(ARGV[pos], /^--?(vers(i(on?)?)?|V)$/)
        if (RSTART) {
            print getVersion()
            print
            printf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])
            printf("%s\n", FriBidi ? FriBidi : "fribidi (GNU FriBidi) [NOT INSTALLED]")
            printf("%-22s%s\n", "User Language", getName(UserLang) " (" getDisplay(UserLang) ")")
            exit
        }

        # -H, -h, -help
        match(ARGV[pos], /^--?(h(e(lp?)?)?|H)$/)
        if (RSTART) {
            print getHelp()
            exit
        }

        # -M, -m, -manual
        match(ARGV[pos], /^--?(m(a(n(u(al?)?)?)?)?|M)$/)
        if (RSTART) {
            man()
            exit
        }

        # -r, -reference
        match(ARGV[pos], /^--?r(e(f(e(r(e(n(ce?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            print getReference("endonym")
            exit
        }

        # -R, -reference-english
        match(ARGV[pos], /^--?(reference-(e(n(g(l(i(sh?)?)?)?)?)?)?|R)$/)
        if (RSTART) {
            print getReference("name")
            exit
        }

        # -D, -debug
        match(ARGV[pos], /^--?(debug|D)$/)
        if (RSTART) {
            Option["debug"] = 1
            continue
        }

        # -verbose
        match(ARGV[pos], /^--?verbose$/)
        if (RSTART) {
            Option["verbose"] = 1 # default value
            continue
        }

        # -d, -dictionary
        match(ARGV[pos], /^--?d(i(c(t(i(o(n(a(ry?)?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            Option["show-original-dictionary"] = 1
            Option["show-dictionary"] = 0
            Option["show-alternatives"] = 0
            continue
        }

        # -b, -brief
        match(ARGV[pos], /^--?b(r(i(ef?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 0
            continue
        }

        # -show-original [yes|no]
        match(ARGV[pos], /^--?show-original(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-phonetics [yes|no]
        match(ARGV[pos], /^--?show-original-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation [yes|no]
        match(ARGV[pos], /^--?show-translation(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation-phonetics [yes|no]
        match(ARGV[pos], /^--?show-translation-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-prompt-message [yes|no]
        match(ARGV[pos], /^--?show-prompt-message(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-prompt-message"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-languages [yes|no]
        match(ARGV[pos], /^--?show-languages(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-languages"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-dictionary [yes|no]
        match(ARGV[pos], /^--?show-original-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-dictionary [yes|no]
        match(ARGV[pos], /^--?show-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-alternatives [yes|no]
        match(ARGV[pos], /^--?show-alternatives(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-alternatives"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -theme [theme]
        match(ARGV[pos], /^--?theme(=(.*)?)?$/, group)
        if (RSTART) {
            Option["theme"] = group[1] ?
                (group[2] ? group[2] : Option["theme"]) :
                ARGV[++pos]
            continue
        }

        # -no-ansi
        match(ARGV[pos], /^--?no-ansi$/)
        if (RSTART) {
            Option["no-ansi"] = 1
            continue
        }

        # -w [num], -width [num]
        match(ARGV[pos], /^--?w(i(d(th?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["width"] = group[4] ?
                (group[5] ? group[5] : Option["width"]) :
                ARGV[++pos]
            continue
        }

        # -indent [num]
        match(ARGV[pos], /^--?indent(=(.*)?)?$/, group)
        if (RSTART) {
            Option["indent"] = group[1] ?
                (group[2] ? group[2] : Option["indent"]) :
                ARGV[++pos]
            continue
        }

        # -v, -view
        match(ARGV[pos], /^--?v(i(ew?)?)?$/)
        if (RSTART) {
            Option["view"] = 1
            continue
        }

        # -pager [program]
        match(ARGV[pos], /^--?pager(=(.*)?)?$/, group)
        if (RSTART) {
            Option["view"] = 1
            Option["pager"] = group[1] ?
                (group[2] ? group[2] : Option["pager"]) :
                ARGV[++pos]
            continue
        }

        # -browser [program]
        match(ARGV[pos], /^--?browser(=(.*)?)?$/, group)
        if (RSTART) {
            Option["browser"] = group[1] ?
                (group[2] ? group[2] : Option["browser"]) :
                ARGV[++pos]
            continue
        }

        # -p, -play
        match(ARGV[pos], /^--?p(l(ay?)?)?$/)
        if (RSTART) {
            Option["play"] = 1
            continue
        }

        # -player [program]
        match(ARGV[pos], /^--?player(=(.*)?)?$/, group)
        if (RSTART) {
            Option["play"] = 1
            Option["player"] = group[1] ?
                (group[2] ? group[2] : Option["player"]) :
                ARGV[++pos]
            continue
        }

        # -x [proxy], -proxy [proxy]
        match(ARGV[pos], /^--?(proxy|x)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["proxy"] = group[2] ?
                (group[3] ? group[3] : Option["proxy"]) :
                ARGV[++pos]
            continue
        }

        # -u [agent], -user-agent [agent]
        match(ARGV[pos], /^--?(user-agent|u)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["user-agent"] = group[2] ?
                (group[3] ? group[3] : Option["user-agent"]) :
                ARGV[++pos]
            continue
        }

        # -I, -interactive, -shell
        match(ARGV[pos], /^--?(int(e(r(a(c(t(i(ve?)?)?)?)?)?)?)?|shell|I)$/)
        if (RSTART) {
            Option["interactive"] = 1
            continue
        }

        # -no-rlwrap
        match(ARGV[pos], /^--?no-rlwrap$/)
        if (RSTART) {
            Option["no-rlwrap"] = 1
            continue
        }

        # -E, -emacs
        match(ARGV[pos], /^--?(emacs|E)$/)
        if (RSTART) {
            Option["emacs"] = 1
            continue
        }

        # -prompt [prompt_string]
        match(ARGV[pos], /^--?prompt(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt"] = group[1] ?
                (group[2] ? group[2] : Option["prompt"]) :
                ARGV[++pos]
            continue
        }

        # -prompt-color [color_code]
        match(ARGV[pos], /^--?prompt-color(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt-color"] = group[1] ?
                (group[2] ? group[2] : Option["prompt-color"]) :
                ARGV[++pos]
            continue
        }

        # -i [file], -input [file]
        match(ARGV[pos], /^--?i(n(p(ut?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["input"] = group[4] ?
                (group[5] ? group[5] : Option["input"]) :
                ARGV[++pos]
            continue
        }

        # -o [file], -output [file]
        match(ARGV[pos], /^--?o(u(t(p(ut?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["output"] = group[5] ?
                (group[6] ? group[6] : Option["output"]) :
                ARGV[++pos]
            continue
        }

        # -l [code], -lang [code]
        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["hl"] = group[3] ?
                (group[4] ? group[4] : Option["hl"]) :
                ARGV[++pos]
            continue
        }

        # -s [code], -source [code]
        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["sl"] = group[5] ?
                (group[6] ? group[6] : Option["sl"]) :
                ARGV[++pos]
            continue
        }

        # -t [codes], -target [codes]
        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            if (group[5]) {
                if (group[6]) split(group[6], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # Shortcut format
        # '[code]:[code]+...' or '[code]=[code]+...'
        match(ARGV[pos], /^[{(\[]?([[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?(:|=)((@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?\+)*(@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?)[})\]]?$/, group)
        if (RSTART) {
            if (group[1]) Option["sl"] = group[1]
            if (group[4]) split(group[4], Option["tl"], "+")
            continue
        }

        # -no-init
        match(ARGV[pos], /^--?no-init/)
        if (RSTART) continue

        # --
        match(ARGV[pos], /^--$/)
        if (RSTART) {
            ++pos # skip the end-of-options option
            break # no more option from here
        }

        break # no more option from here
    }

    initTheme()

    if (Option["interactive"] && !Option["no-rlwrap"])
        rlwrapMe() # interactive mode
    else if (Option["emacs"] && !Option["interactive"] && !Option["no-rlwrap"])
        if (emacsMe()) # emacs front-end
            Option["interactive"] = 1 # fallback to interactive mode

    if (Option["interactive"])
        welcome()

    initMisc()

    if (pos < ARGC) {
        # More parameters

        # Translate remaining parameters
        for (i = pos; i < ARGC; i++) {
            # Verbose mode: separator between sources
            if (Option["verbose"] && i > pos)
                p(replicate(Option["chr-source-seperator"], Option["width"]))

            translate(ARGV[i], 1) # inline mode
        }

        # If input not specified, we're done
    } else {
        # No more parameter besides options

        # If input not specified, use stdin
        if (!Option["input"]) Option["input"] = STDIN
    }

    # If input specified, start translating
    if (Option["input"])
        translateMain()

    exit ExitCode
}

####################################################################
# Main.awk                                                         #
####################################################################

# Initialization #0. (prior to option parsing)
function init0() {
    initGawk()

    initBiDi()
    initLocale()
    initLocaleDisplay() #<< Locale, BiDi
    initUserLang()      #<< Locale

    RS = "\n"

    ExitCode = 0

    Option["debug"] = 0

    Option["verbose"] = 1
    Option["show-original"] = 1
    Option["show-original-phonetics"] = 1
    Option["show-translation"] = 1
    Option["show-translation-phonetics"] = 1
    Option["show-prompt-message"] = 1
    Option["show-languages"] = 1
    Option["show-original-dictionary"] = 0
    Option["show-dictionary"] = 1
    Option["show-alternatives"] = 1

    Option["no-ansi"] = 0

    Option["width"] = ENVIRON["COLUMNS"] ? ENVIRON["COLUMNS"] : 64
    Option["indent"] = 4

    Option["view"] = 0
    Option["pager"] = ENVIRON["PAGER"]

    Option["browser"] = ENVIRON["BROWSER"]

    Option["play"] = 0
    Option["player"] = ENVIRON["PLAYER"]

    Option["proxy"] = ENVIRON["HTTP_PROXY"] ? ENVIRON["HTTP_PROXY"] : ENVIRON["http_proxy"]
    Option["user-agent"] = ENVIRON["USER_AGENT"]

    Option["interactive"] = 0
    Option["no-rlwrap"] = 0
    Option["emacs"] = 0
    Option["prompt"] = ENVIRON["TRANS_PS"] ? ENVIRON["TRANS_PS"] : "%s>"
    Option["prompt-color"] = ENVIRON["TRANS_PS_COLOR"] ? ENVIRON["TRANS_PS_COLOR"] : "default"

    Option["input"] = NULLSTR
    Option["output"] = STDOUT

    Option["hl"] = ENVIRON["HOME_LANG"] ? ENVIRON["HOME_LANG"] : UserLang
    Option["sl"] = ENVIRON["SOURCE_LANG"] ? ENVIRON["SOURCE_LANG"] : "auto"
    Option["tl"][1] = ENVIRON["TARGET_LANG"] ? ENVIRON["TARGET_LANG"] : UserLang

    Option["theme"] = "default"
}

# Initialization script.
function initScript(    file, line, script, temp) {
    if (belongsTo("-no-init", ARGV) || belongsTo("--no-init", ARGV))
        return

    # Find the initialization file
    file = ".trans"
    if (!fileExists(file)) {
        file = ENVIRON["HOME"] "/.translate-shell/init.trans"
        if (!fileExists(file)) {
            file = "/etc/translate-shell"
            if (!fileExists(file)) return
        }
    }

    script = NULLSTR
    while (getline line < file)
        script = script "\n" line
    loadOptions(script)

    # Hack: Option["tl"] must be an array
    if (!isarray(Option["tl"])) {
        temp = Option["tl"]
        delete Option["tl"]
        Option["tl"][1] = temp
    }
}

# Initialization #2. (prior to interactive mode handling)
function init2() {
    # Set theme
    setTheme()

    # Disable ANSI escape codes if required
    if (Option["no-ansi"])
        delete AnsiCode
}

# Initialization #3.
function init3(    group) {
    # Translate.awk
    initHttpService()

    # Initialize pager
    if (Option["view"]) {
        if (!Option["pager"]) {
            initPager()
            Option["pager"] = Pager
        }

        if (!Option["pager"]) {
            w("[WARNING] No available terminal pager.")
            Option["view"] = 0
        }
    }

    # Initialize browser
    if (!Option["browser"]) {
        "xdg-mime query default text/html" SUPERR |& getline Option["browser"]
        match(Option["browser"], "(.*).desktop$", group)
        Option["browser"] = group[1]
    }

    # Initialize audio player or speech synthesizer
    if (Option["play"]) {
        if (!Option["player"]) {
            initAudioPlayer()
            Option["player"] = AudioPlayer ? AudioPlayer : Option["player"]
            if (!Option["player"])
                initSpeechSynthesizer()
        }

        if (!Option["player"] && !SpeechSynthesizer) {
            w("[WARNING] No available audio player or speech synthesizer.")
            Option["play"] = 0
        }
    }
}

# Main entry point.
BEGIN {
    init0() # initialization #0
    initScript() # initialization script

    # Option parsing
    pos = 0
    while (ARGV[++pos]) {
        # -, -no-op
        match(ARGV[pos], /^-(-?no-op)?$/)
        if (RSTART) continue

        # -V, -version
        match(ARGV[pos], /^--?(vers(i(on?)?)?|V)$/)
        if (RSTART) {
            print getVersion()
            print
            printf("%-22s%s\n", "gawk (GNU Awk)", PROCINFO["version"])
            printf("%s\n", FriBidi ? FriBidi : "fribidi (GNU FriBidi) [NOT INSTALLED]")
            printf("%-22s%s\n", "User Language", getName(UserLang) " (" getDisplay(UserLang) ")")
            exit
        }

        # -H, -h, -help
        match(ARGV[pos], /^--?(h(e(lp?)?)?|H)$/)
        if (RSTART) {
            print getHelp()
            exit
        }

        # -M, -m, -manual
        match(ARGV[pos], /^--?(m(a(n(u(al?)?)?)?)?|M)$/)
        if (RSTART) {
            man()
            exit
        }

        # -r, -reference
        match(ARGV[pos], /^--?r(e(f(e(r(e(n(ce?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            print getReference("endonym")
            exit
        }

        # -R, -reference-english
        match(ARGV[pos], /^--?(reference-(e(n(g(l(i(sh?)?)?)?)?)?)?|R)$/)
        if (RSTART) {
            print getReference("name")
            exit
        }

        # -D, -debug
        match(ARGV[pos], /^--?(debug|D)$/)
        if (RSTART) {
            Option["debug"] = 1
            continue
        }

        # -verbose
        match(ARGV[pos], /^--?verbose$/)
        if (RSTART) {
            Option["verbose"] = 1 # default value
            continue
        }

        # -d, -dictionary
        match(ARGV[pos], /^--?d(i(c(t(i(o(n(a(ry?)?)?)?)?)?)?)?)?$/)
        if (RSTART) {
            Option["show-original-dictionary"] = 1
            Option["show-dictionary"] = 0
            Option["show-alternatives"] = 0
            continue
        }

        # -b, -brief
        match(ARGV[pos], /^--?b(r(i(ef?)?)?)?$/)
        if (RSTART) {
            Option["verbose"] = 0
            continue
        }

        # -show-original [yes|no]
        match(ARGV[pos], /^--?show-original(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-phonetics [yes|no]
        match(ARGV[pos], /^--?show-original-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation [yes|no]
        match(ARGV[pos], /^--?show-translation(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-translation-phonetics [yes|no]
        match(ARGV[pos], /^--?show-translation-phonetics(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-translation-phonetics"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-prompt-message [yes|no]
        match(ARGV[pos], /^--?show-prompt-message(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-prompt-message"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-languages [yes|no]
        match(ARGV[pos], /^--?show-languages(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-languages"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-original-dictionary [yes|no]
        match(ARGV[pos], /^--?show-original-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-original-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-dictionary [yes|no]
        match(ARGV[pos], /^--?show-dictionary(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-dictionary"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -show-alternatives [yes|no]
        match(ARGV[pos], /^--?show-alternatives(=(.*)?)?$/, group)
        if (RSTART) {
            Option["show-alternatives"] = yn(group[1] ? group[2] : ARGV[++pos])
            continue
        }

        # -theme [theme]
        match(ARGV[pos], /^--?theme(=(.*)?)?$/, group)
        if (RSTART) {
            Option["theme"] = group[1] ?
                (group[2] ? group[2] : Option["theme"]) :
                ARGV[++pos]
            continue
        }

        # -no-ansi
        match(ARGV[pos], /^--?no-ansi$/)
        if (RSTART) {
            Option["no-ansi"] = 1
            continue
        }

        # -w [num], -width [num]
        match(ARGV[pos], /^--?w(i(d(th?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["width"] = group[4] ?
                (group[5] ? group[5] : Option["width"]) :
                ARGV[++pos]
            continue
        }

        # -indent [num]
        match(ARGV[pos], /^--?indent(=(.*)?)?$/, group)
        if (RSTART) {
            Option["indent"] = group[1] ?
                (group[2] ? group[2] : Option["indent"]) :
                ARGV[++pos]
            continue
        }

        # -v, -view
        match(ARGV[pos], /^--?v(i(ew?)?)?$/)
        if (RSTART) {
            Option["view"] = 1
            continue
        }

        # -pager [program]
        match(ARGV[pos], /^--?pager(=(.*)?)?$/, group)
        if (RSTART) {
            Option["view"] = 1
            Option["pager"] = group[1] ?
                (group[2] ? group[2] : Option["pager"]) :
                ARGV[++pos]
            continue
        }

        # -browser [program]
        match(ARGV[pos], /^--?browser(=(.*)?)?$/, group)
        if (RSTART) {
            Option["browser"] = group[1] ?
                (group[2] ? group[2] : Option["browser"]) :
                ARGV[++pos]
            continue
        }

        # -p, -play
        match(ARGV[pos], /^--?p(l(ay?)?)?$/)
        if (RSTART) {
            Option["play"] = 1
            continue
        }

        # -player [program]
        match(ARGV[pos], /^--?player(=(.*)?)?$/, group)
        if (RSTART) {
            Option["play"] = 1
            Option["player"] = group[1] ?
                (group[2] ? group[2] : Option["player"]) :
                ARGV[++pos]
            continue
        }

        # -x [proxy], -proxy [proxy]
        match(ARGV[pos], /^--?(proxy|x)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["proxy"] = group[2] ?
                (group[3] ? group[3] : Option["proxy"]) :
                ARGV[++pos]
            continue
        }

        # -u [agent], -user-agent [agent]
        match(ARGV[pos], /^--?(user-agent|u)(=(.*)?)?$/, group)
        if (RSTART) {
            Option["user-agent"] = group[2] ?
                (group[3] ? group[3] : Option["user-agent"]) :
                ARGV[++pos]
            continue
        }

        # -I, -interactive, -shell
        match(ARGV[pos], /^--?(int(e(r(a(c(t(i(ve?)?)?)?)?)?)?)?|shell|I)$/)
        if (RSTART) {
            Option["interactive"] = 1
            continue
        }

        # -no-rlwrap
        match(ARGV[pos], /^--?no-rlwrap$/)
        if (RSTART) {
            Option["no-rlwrap"] = 1
            continue
        }

        # -E, -emacs
        match(ARGV[pos], /^--?(emacs|E)$/)
        if (RSTART) {
            Option["emacs"] = 1
            continue
        }

        # -prompt [prompt_string]
        match(ARGV[pos], /^--?prompt(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt"] = group[1] ?
                (group[2] ? group[2] : Option["prompt"]) :
                ARGV[++pos]
            continue
        }

        # -prompt-color [color_code]
        match(ARGV[pos], /^--?prompt-color(=(.*)?)?$/, group)
        if (RSTART) {
            Option["prompt-color"] = group[1] ?
                (group[2] ? group[2] : Option["prompt-color"]) :
                ARGV[++pos]
            continue
        }

        # -i [file], -input [file]
        match(ARGV[pos], /^--?i(n(p(ut?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["input"] = group[4] ?
                (group[5] ? group[5] : Option["input"]) :
                ARGV[++pos]
            continue
        }

        # -o [file], -output [file]
        match(ARGV[pos], /^--?o(u(t(p(ut?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["output"] = group[5] ?
                (group[6] ? group[6] : Option["output"]) :
                ARGV[++pos]
            continue
        }

        # -l [code], -lang [code]
        match(ARGV[pos], /^--?l(a(ng?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["hl"] = group[3] ?
                (group[4] ? group[4] : Option["hl"]) :
                ARGV[++pos]
            continue
        }

        # -s [code], -source [code]
        match(ARGV[pos], /^--?s(o(u(r(ce?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            Option["sl"] = group[5] ?
                (group[6] ? group[6] : Option["sl"]) :
                ARGV[++pos]
            continue
        }

        # -t [codes], -target [codes]
        match(ARGV[pos], /^--?t(a(r(g(et?)?)?)?)?(=(.*)?)?$/, group)
        if (RSTART) {
            if (group[5]) {
                if (group[6]) split(group[6], Option["tl"], "+")
            } else
                split(ARGV[++pos], Option["tl"], "+")
            continue
        }

        # Shortcut format
        # '[code]:[code]+...' or '[code]=[code]+...'
        match(ARGV[pos], /^[{(\[]?([[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?(:|=)((@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?\+)*(@?[[:alpha:]][[:alpha:]][[:alpha:]]?(-[[:alpha:]][[:alpha:]])?)?)[})\]]?$/, group)
        if (RSTART) {
            if (group[1]) Option["sl"] = group[1]
            if (group[4]) split(group[4], Option["tl"], "+")
            continue
        }

        # -no-init
        match(ARGV[pos], /^--?no-init/)
        if (RSTART) continue

        # --
        match(ARGV[pos], /^--$/)
        if (RSTART) {
            ++pos # skip the end-of-options option
            break # no more option from here
        }

        break # no more option from here
    }

    init2() # initialization #2

    if (Option["interactive"] && !Option["no-rlwrap"])
        rlwrapMe() # interactive mode
    else if (Option["emacs"] && !Option["interactive"] && !Option["no-rlwrap"])
        if (emacsMe()) # emacs front-end
            Option["interactive"] = 1 # fallback to interactive mode

    if (Option["interactive"])
        welcome()

    init3() # initialization #3

    if (pos < ARGC) {
        # More parameters

        # Translate remaining parameters
        for (i = pos; i < ARGC; i++) {
            # Verbose mode: separator between sources
            if (Option["verbose"] && i > pos)
                p(replicate(Option["chr-source-seperator"], Option["width"]))

            translate(ARGV[i], 1) # inline mode
        }

        # If input not specified, we're done
    } else {
        # No more parameter besides options

        # If input not specified, use stdin
        if (!Option["input"]) Option["input"] = STDIN
    }

    # If input specified, start translating
    if (Option["input"])
        translateMain()

    exit ExitCode
}


[as] Add some debug information in generated files

BEGIN		{
		FS = "\t"
		printf "#include \"../../../inc/scc.h\"\n"\
		       "#include \"../../as.h\"\n"\
		       "#include \"../" family "/proc.h\"\n"
		nop = 0; nvar = 0

		rules = "target/" family "/rules.dat"
		while (getline < rules > 0) {
			regex[++nregs] = $1
			value[nregs] = $2
		}
		close "target/rules.awk"
}
		{sub(/#.*/,"")}

$6 !~ cpu	{next}

/^$/		{next}

		{
		if (opstart[$1] == 0) {
			opstart[$1] = nvar
			opnames[nop++] = $1
		}
		opcount[$1]++
		opargs[nvar] = $2
		opsize[nvar] = $3
		opbytes[nvar] = ($4 == "none") ? "" : $4
		opformat[nvar++] = $5
		formats[$5] = 1
}

END		{
		for (i in formats)
			printf "Format %s;\n", i

		printf "int nr_ins = %d;\n\n", nop
		print "struct ins instab[] = {"
		for (i = 0; i < nop; i++) {
			n = opnames[i]
			start = opstart[n]
			end = start + opcount[n]
			printf "\t{.str = \"%s\", .begin = %d, .end = %d},\n",
			       n, start, end | "sort"
		}
		close("sort")
		printf "};\n\n"

		print "struct op optab[] = {"
		for (i = 0; i < nvar; i++) {
			printf "\t/* %d */\n", i
			printf "\t{\n" \
			       "\t\t.size = %d,\n"\
			       "\t\t.format = %s,\n",
			       opsize[i], opformat[i]

			if (opbytes[i] != "")
				printf "\t\t.bytes = (unsigned char []) {%s},\n",
				        opbytes[i]

			a = str2args(opargs[i])
			if (a != "")
				printf "\t\t.args = (unsigned char []) {%s}\n", a

			print "\t},"
		}
		print "};"
}

function str2args(s, args, i, j, out, n, found)
{
	n = split(s, args, /,/)
	if (n == 0 || args[1] == "none")
		return ""
	for (i = 1; i <= n; i++) {
		a = args[i]
		found = 0
		for (j = 1; j <= nregs; j++) {
			if (match(a, regex[j])) {
				out = out value[j]
				found = 1
				break
			}
		}

		if (!found) {
			print FILENAME ":" NR ":" \
			      $0 ":wrong arg", a > "/dev/stderr"
			exit 1
		}

		a = substr(a, RLENGTH+1)
		if (a ~ /^\+$/) {
			return out "|AREP"
		} else if (a ~ /^\?$/) {
			return out "|AOPT"
		} else if (a != "") {
			print FILENAME ":" NR ":" \
			      $0 ": trailing chars: ", a > "/dev/stderr"
			exit 1
		}
		out = out ","
	}
	out = out "0"

	return out
}

BEGIN		{
		FS = "\t"
		printf "#include \"../../../inc/scc.h\"\n"\
		       "#include \"../../as.h\"\n"\
		       "#include \"../" family "/proc.h\"\n"
		nop = 0; nvar = 0

		rules = "target/" family "/rules.dat"
		while (getline < rules > 0) {
			regex[++nregs] = $1
			value[nregs] = $2
		}
		close "target/rules.awk"
}
		{sub(/#.*/,"")}

$6 !~ cpu	{next}

/^$/		{next}

		{
		if (opstart[$1] == 0) {
			opstart[$1] = nvar
			opnames[nop++] = $1
		}
		opcount[$1]++
		opargs[nvar] = $2
		opsize[nvar] = $3
		opbytes[nvar] = ($4 == "none") ? "" : $4
		opformat[nvar++] = $5
		formats[$5] = 1
}

END		{
		for (i in formats)
			printf "Format %s;\n", i

		printf "int nr_ins = %d;\n\n", nop
		print "struct ins instab[] = {"
		for (i = 0; i < nop; i++) {
			n = opnames[i]
			start = opstart[n]
			end = start + opcount[n]
			printf "\t{.str = \"%s\", .begin = %d, .end = %d},\n",
			       n, start, end | "sort"
		}
		close("sort")
		printf "};\n\n"

		print "struct op optab[] = {"
		for (i = 0; i < nvar; i++) {
			printf "\t{\n" \
			       "\t\t.size = %d,\n"\
			       "\t\t.format = %s,\n",
			       opsize[i], opformat[i]

			if (opbytes[i] != "")
				printf "\t\t.bytes = (unsigned char []) {%s},\n",
				        opbytes[i]

			a = str2args(opargs[i])
			if (a != "")
				printf "\t\t.args = (unsigned char []) {%s}\n", a

			print "\t},"
		}
		print "};"
}

function str2args(s, args, i, j, out, n, found)
{
	n = split(s, args, /,/)
	if (n == 0 || args[1] == "none")
		return ""
	for (i = 1; i <= n; i++) {
		a = args[i]
		found = 0
		for (j = 1; j <= nregs; j++) {
			if (match(a, regex[j])) {
				out = out value[j]
				found = 1
				break
			}
		}

		if (!found) {
			print FILENAME ":" NR ":" \
			      $0 ":wrong arg", a > "/dev/stderr"
			exit 1
		}

		a = substr(a, RLENGTH+1)
		if (a ~ /^\+$/) {
			return out "|AREP"
		} else if (a ~ /^\?$/) {
			return out "|AOPT"
		} else if (a != "") {
			print FILENAME ":" NR ":" \
			      $0 ": trailing chars: ", a > "/dev/stderr"
			exit 1
		}
		out = out ","
	}
	out = out "0"

	return out
}


genplotscript.awk has moved to scripts at top level

#!/usr/bin/awk -f
/\#t/
{
  printf("set data style linespoints\n");
  printf("plot");
  n = 0;
  for (i = 2; i != ARGC; i++)
  {
    for (j = 2; j != NF+1; j++)
    {
      if (ARGV[i] == $(j))
      {
        printf(" '");
	printf(ARGV[1]);
	printf("'");
	printf(" using 1:");
	printf(j);
	printf(" title '");
	printf($(j));
	printf("'");

	if (i != ARGC - 1)
	   printf(",");
	n++;
      }
    }
  }
  exit 0;
}