output edges in a sorted order, make gawk explicit

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
# down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier and
# the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
# labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
# a handy way to differentiate uri for subjects and objects
# which may belong to a "structural" ontology as opposed to
# a data uri.

#  Perhaps an improvement would be to also
# express subject and objects as individual curies (ala predicates)
# iff their namespace is also used by a predicate.

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:/))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/_"]="BNODE"
	# just until skolemized bnodes get in the curie map?
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	############################################################
	# revisit whether these exceptions are still necessary often
	# they may have been moved into the  prefix mapping file
	prefix["http://www.w3.org/1999/02/22-rdf-syntax-ns#"]="rdf"
	prefix["http://www.w3.org/2000/01/rdf-schema#"]="rdfs"
	prefix["http://www.w3.org/2002/07/owl#"]="owl"
	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in panther
	prefix["http://identifiers.org/wormbase"]="WormBase"
	# in IMPC
	prefix["http://www.mousephenotype.org"]="IMPC"
	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# till all non httpS: are purged
	prefix["http://monarchinitiative.org/"]="BASE"
	prefix["http://www.monarchinitiative.org/"]="BASE"
	prefix["http://monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://www.monarchinitiative.org/MONARCH"]=""
	prefix["http://www.monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://data.monarchinitiative.org/ttl"]="MonarchData"
	prefix["http://archive.monarchinitiative.org/ttl"]="MonarchArchive"
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, optional whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i  %s",FILENAME,FNR,$0) > "/dev/stderr"
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}

#! /usr/bin/awk -f

#  Reduce the subject and object of RDF triples (ntriples format)
# down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier and
# the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
# labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
# a handy way to differentiate uri for subjects and objects
# which may belong to a "structural" ontology as opposed to
# a data uri.

#  Perhaps an improvement would be to also
# express subject and objects as individual curies (ala predicates)
# iff their namespace is also used by a predicate.

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:/))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/_"]="BNODE"
	# just until skolemized bnodes get in the curie map?
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	############################################################
	# revisit whether these exceptions are still necessary often
	# they may have been moved into the  prefix mapping file
	prefix["http://www.w3.org/1999/02/22-rdf-syntax-ns#"]="rdf"
	prefix["http://www.w3.org/2000/01/rdf-schema#"]="rdfs"
	prefix["http://www.w3.org/2002/07/owl#"]="owl"
	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in panther
	prefix["http://identifiers.org/wormbase"]="WormBase"
	# in IMPC
	prefix["http://www.mousephenotype.org"]="IMPC"
	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# till all non httpS: are purged
	prefix["http://monarchinitiative.org/"]="BASE"
	prefix["http://www.monarchinitiative.org/"]="BASE"
	prefix["http://monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://www.monarchinitiative.org/MONARCH"]=""
	prefix["http://www.monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://data.monarchinitiative.org/ttl"]="MonarchData"
	prefix["http://archive.monarchinitiative.org/ttl"]="MonarchArchive"
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, optional whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i  %s",FILENAME,FNR,$0) > "/dev/stderr"
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	for(edge in edgelist){
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}


- Use 'options DEBUG_VFS_LOCKS' instead of the DEBUG_ALL_VFS_LOCKS    environment variable to enable the lock verifiction code.


nasty conversion:dataset_identifier replacement bug fixed.


Better changelog creator.

Better changelog creator.

BEGIN{
	revision_key=0
}

function gline()
{
	if(!getline)
		error("Unexpected EOF")
	return 1
}

function error(s)
{
	print s > "/dev/stderr"
	exit(1)
}

/^?/{next}

#Process a file entry
/^Working file:/{

	#Get the filename
	filename=$3

	#Find the tags
	while(gline() && !/^symbolic names:/);

	#Extract tags
	delete revision_to_tag
	while(gline() && ! /^keyword substitution:/)
	{
		tagname=substr($1, 1, length($1)-1)
		revision_to_tag[$2]=tagname
	}

	#Skip over the rest until the revisions
	while(gline() && !/^----------------/);

	#There is always at least one revision
	do
	{
		#The revision number
		gline()
		revision=$2

		#The commit line
		gline()
		commit_line=$0
		author=substr($6, 1, length($6)-1)

		#Store the date* 10 for extra precision
		date = $2" "$3
		gsub(/[^0-9]+/, " ", date)
		date=mktime(date)*10

		#The commit message
		message=""
		while(gline() && !/^[-=]+$/)
			message = message $0 "\n"

		#The commit line is the primary key.

		#Keep track of commits
		date_to_line[date]=commit_line
		line_to_message[commit_line]=message
		line_to_author[commit_line]=author
		line_to_files[commit_line]=line_to_files[commit_line] " " filename
		line_to_revision[commit_line]=line_to_revision[commit_line] " " revision

		#The last file commit which has a certain tag
		#happens just before the tag.
		if(tag_to_date[revision_to_tag[revision]] < date)
			tag_to_date[revision_to_tag[revision]] = date
	}
	while(!/^=+$/)
}

END{
	#Delete the null tag for untagged revisions
	delete tag_to_date[""]

	#The last
	for(i in tag_to_date)
	{
		#Put the tag in just after the last commit
		date = tag_to_date[i] + 1
		date_to_line[date]="hello"
		date_to_tag[date]=i
	}

	#Sort commits by date
	num_dates=asorti(date_to_line, sorted_dates)

	#Iterate through all commits, newest first
	for(i=num_dates; i > 0; i--)
	{
		date = sorted_dates[i]

		if(date in date_to_tag)
		{
			print "=========================================="
			print "Repository tagged as " date_to_tag[date]
			print "=========================================="
		}
		else
		{
			line = date_to_line[date]

			author=line_to_author[line]
			message=line_to_message[line]

			file_list=line_to_files[line]
			rev_list=line_to_revision[line]

			print ""
			#Long commits can get spread over several seconds.
			#Try to concatenate entries which differ by a few
			#seconds, but have the same commit messsage and
			#author
			#We allow 20 seconds
			concatenation=""
			num_concat=0
			for(j=i-1; j>0 &&
					   author==line_to_author[date_to_line[sorted_dates[j]]] &&
					   message==line_to_message[date_to_line[sorted_dates[j]]] &&
					   date - sorted_dates[j] < 20; j--)
			{
				#Reset i, so that the fragment appears once only.
				i=j

				#Append to the file and revision list
				file_list=file_list " " line_to_files[date_to_line[sorted_dates[j]]];
				rev_list=rev_list " " line_to_revision[date_to_line[sorted_dates[j]]];

				#Make a note of teh concatenations
				concatenation=concatenation ", " date - sorted_dates[j] "s"
				num_concat++
			}

			#Create a message based on concatenation
			if(num_concat)
				concatenation = " (Concatenated " num_concat+1 " entries at " substr(concatenation, 3) ")"

			#Leader line
			print strftime("%a %e %b, %Y %H:%M:%S", date/10) " author: " author concatenation

			#List files commit happened on
			numfiles=split(file_list, files);
			split(rev_list, revisions);

			for(j=1; j <= numfiles; j++)
				print "    File: " files[j] " (Rev. " revisions[j] ")"

			#Print the commit message
			print "\n" message "\n--------------"
		}
	}
}

BEGIN{
	key=0
}

#A log entry
/RCS file:/{
	getline
	sub(/^Working file: /,"")
	filename=$0

	do
		getline
	while(!/symbolic names/)

	delete sym

	getline
	for(;!/keyword substitution/;getline)
	{
		sub(/:/,"",$1)
		sym[$2]=$1
	}

	while(!/----------/)
		getline

	while(1)
	{
		getline
		revision=$2
		getline
		date=$2" "$3

		datestamp=date
		gsub(/[^0-9]+/, " ", datestamp)

		epoch=mktime(datestamp)
		date = strftime("%a %e %b, %Y %H:%M:%S", epoch)

		author = $6
		sub(/;$/, "", author)

		#Try to build a checkin stamp

		tag=sym[revision]

		#Find the most recent date for the tag (if present)
		if(tag != "" && alltags[tag] < datestamp)
			alltags[tag] = datestamp

		text=""
		#Get text
		while(getline && !/-----/ && !/======/)
			text=text (text == "" ? "" : "\n") $0

		#Now we have: filename revision date datestamp author tag text key
		authors[key] = author
		revisions[key] = revision
		tags[key] =  tag
		texts[key] = text
		files[key] = filename

		dates[datestamp] = date

		entries[datestamp] = entries[datestamp] " " key

		key++

		if(/========/)
			break
	}
}

END{
	#Put tags in to entries
	for(i in alltags)
		entries[alltags[i]] = "tag " i

	n = asorti(entries, se)

	for(i=n; i >0; i--)
	{
		ds = se[i]
		keylist = entries[ds]
		nk = split(keylist, keys)

		#Try to concatenate non-atomic commits
		concat=""
		nc=1
		for(;;)
		{
			ds1 = se[i-1]
			kl1 = entries[ds1]
			nk1 = split(kl1, k1)
			if(authors[keys[1]] == authors[k1[1]] && texts[keys[1]] == texts[k1[1]])
			{
				keylist = keylist " " kl1
				nk = split(keylist, keys)
				i--
				concat = concat ", "(ds1 - ds) "s"
				nc++
			}
			else
				break
		}

		if(concat != "")
			concat = "(Concatenated " nc " entries at 0s" concat ")"

		if(keys[1] == "tag")
		{
			print "====================================================================="
			print "Repository tagged as " keys[2]
			print "====================================================================="
			print ""
			print ""
		}
		else
		{
			print dates[ds] " author: " authors[keys[1]] " " concat

			for(j=1; j <= nk; j++)
			{
				k = keys[j]
				print "   File: " files[k] " (Rev. " revisions[k] ")"
			}

			print ""
			print texts[keys[1]]
			print ""
			print "-----------------------------------"
			print ""
		}
	}

}


Use color "0" (black) is no session information is found.

git-svn-id: 392d0e5ce652024192c571322d40dcc23d6d83b0@20 8a65f31f-a56f-4bd6-98e8-2755f64920ec

yLineSpace = 24;
  xHostSpace = 200;
  leftMargin = 100;
  rightMargin = 100;
  topMargin = 50;
  bottomMargin = 50;
  numTraces = numHosts;

  color[0]  = "black";
  color[1]  = "orange";
  color[2]  = "sienna";
  color[3]  = "red";
  color[4]  = "green";
  color[5]  = "purple";
  color[6]  = "chocolate";
  color[7]  = "olivedrab";
  color[8]  = "darkred";
  color[9]  = "darkslategrey";
  color[10] = "MidnightBlue";
  color[11] = "maroon";

  print "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>";
  print "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd\">";

  for(i=0;i<numHosts;i++) {
    lookup[hosts[i]] = i;
    printf "\n", hosts[i], i;
  }

  w = (numHosts-1) * xHostSpace + leftMargin + rightMargin;
  h = numLines * yLineSpace + topMargin + bottomMargin;

  printf "<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"%d\" height=\"%d\" viewBox=\"0 0 %d %d\">\n",w,h,w,h;

  insertStyleDefs();

  ystart = 60;
  yend = h;

  printf "<text x=\"%d\" y=\"%d\" class=\"label title-text\">%s</text>\n",
    (w/2),
    ystart-35,
    title;

  for(i=0;i<numTraces;i++) {

    if (label[i] == "")
      label[i] = hosts[i];

    printf "<text x=\"%d\" y=\"%d\" class=\"label host-text\">%s</text>\n",
      leftMargin+(i*xHostSpace),
      ystart-(15*(i%2)+2),
      label[i];

    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"element-line\" />\n", leftMargin+(i*xHostSpace), ystart, leftMargin+(i*xHostSpace), yend;
  }

  printf "   <map name=\"callflowmap\" id=\"callflowmap\">\n" > "imagemap";
}

func insertStyleDefs () {
  printf "<defs>\n\n</defs>\n";
}

func arrow(x,y,d,c) {

  printf "<polygon points=\"%d,%d %d,%d %d,%d %d,%d\" class=\"arrowhead\" style=\"fill: %s; stroke: %s;\"/>\n",
  x,y,
  x+5*d, y-3,
  x+3*d, y,
  x+5*d, y+3,
  color[c], color[c];
}

func line(x1,x2,y,output, c) {

  if (x1 == x2) {
    printf "<polyline points=\"%d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d\" fill=\"none\" class=\"traceline\"/>\n",
      x1,y-18+7,
      x1+5,y-18+7,
      x1+12,y-15+7,
      x1+15,y-10+7,
      x1+15,y-5+7,
      x1+12,y-2+7,
      x1+5,y+7,
      x1,y+7;
    arrow(x1,y+7,1);

    xtext = x1 + 18;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"/>\n", $1, x1, y-yLineSpace+2, x1+15, y+7+1, $1 >> "imagemap"

  } else if (x1<x2) {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,-1,c);

    xtext = x1 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"/>\n", $1, x1, y-yLineSpace+2, x2, y+1, $1 >> "imagemap"

  } else {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,1,c);

    xtext = x2 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"/>\n", $1, x2, y-yLineSpace+2, x1, y+1, $1 >> "imagemap"
  }

  printf "\n", $1;
  printf "<text x=\"%d\" y=\"%d\" class=\"pkt-text\">%s</text>\n", xtext, y-4, output;
  printf "\n";
}

{
  y = NR;
  y = y * yLineSpace + ystart;

  if($1 == "#")
  {
	 output = "";
	 for (i=2; i<=NF; i++)
	 {
		if ($i == "!")
		   break;
		output = output " " $i;
	 }

	 link = "";
	 if($i == "!")
	 {
		i++;
		for(;i<=NF;i++)
		{
		   link = link " " $i;
		}
	 }

	 if (link != "")
	 {
		printf("<text x=\"%d\" y=\"%d\" class=\"link\">%s</text>\n", 50, y, output);
		printf("    <area href=\"%s\" coords=\"%d,%d,%d,%d\"/>\n", link, 50, y-yLineSpace+2, w, y+1) >> "imagemap";
	 }
	 else
	 {
		printf("<text x=\"%d\" y=\"%d\" class=\"comment-text\">%s</text>\n", 50, y, output);
	 }
  }
  else
  {
	 if ($4 ~ "{([0-9]+)}") {
		c = $4
		gsub("{", "", c)
		gsub("}", "", c)

		# If over 11 colors, choose black color as default...
		# TODO: change color behavior
		if (strtonum(c)>11) {c="0"}
	 } else {
		c = 0
	 }
	 l1 = sprintf("%s:%s", $2,$3);
	 l2 = sprintf("%s:%s", $5,$6);
	 for (i=0; i<numHosts; i++)
	 {
	   if(l1 ~ hosts[i])
	   {
		 x1 = strtonum(lookup[hosts[i]]);
	   }
	   if(l2 ~ hosts[i])
	   {
		 x2 = strtonum(lookup[hosts[i]]);
	   }
	 }

	 x1 = x1 * xHostSpace + leftMargin;
	 x2 = x2 * xHostSpace + leftMargin;

	 ORS = "";
	 printf "<text x=\"%d\" y=\"%d\" class=\"pkt-text\">%d</text>\n", leftMargin/2, y, $1;

	 output = "";
	 for(i=7;i<=NF;i++) output = output " " $i;

	 gsub("SIP(/SDP|/XML|) *(Status|Request): *","", output);
	 gsub(", with session description *$"," w/SDP",output);
	if ((x1==x2) && (noAuto==1)){
		}
	else{
		line(x1,x2,y,output,c);
		}
  }
}

END {
  printf "   </map>\n" >> "imagemap";
  printf "</svg>\n";
}

yLineSpace = 24;
  xHostSpace = 200;
  leftMargin = 100;
  rightMargin = 100;
  topMargin = 50;
  bottomMargin = 50;
  numTraces = numHosts;

  color[0]  = "black";
  color[1]  = "orange";
  color[2]  = "sienna";
  color[3]  = "red";
  color[4]  = "green";
  color[5]  = "purple";
  color[6]  = "chocolate";
  color[7]  = "olivedrab";
  color[8]  = "darkred";
  color[9]  = "darkslategrey";
  color[10] = "MidnightBlue";
  color[11] = "maroon";

  print "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>";
  print "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd\">";

  for(i=0;i<numHosts;i++) {
    lookup[hosts[i]] = i;
    printf "\n", hosts[i], i;
  }

  w = (numHosts-1) * xHostSpace + leftMargin + rightMargin;
  h = numLines * yLineSpace + topMargin + bottomMargin;

  printf "<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"%d\" height=\"%d\" viewBox=\"0 0 %d %d\">\n",w,h,w,h;

  insertStyleDefs();

  ystart = 60;
  yend = h;

  printf "<text x=\"%d\" y=\"%d\" class=\"label title-text\">%s</text>\n",
    (w/2),
    ystart-35,
    title;

  for(i=0;i<numTraces;i++) {

    if (label[i] == "")
      label[i] = hosts[i];

    printf "<text x=\"%d\" y=\"%d\" class=\"label host-text\">%s</text>\n",
      leftMargin+(i*xHostSpace),
      ystart-(15*(i%2)+2),
      label[i];

    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"element-line\" />\n", leftMargin+(i*xHostSpace), ystart, leftMargin+(i*xHostSpace), yend;
  }

  printf "   <map name=\"callflowmap\" id=\"callflowmap\">\n" > "imagemap";
}

func insertStyleDefs () {
  printf "<defs>\n\n</defs>\n";
}

func arrow(x,y,d,c) {

  printf "<polygon points=\"%d,%d %d,%d %d,%d %d,%d\" class=\"arrowhead\" style=\"fill: %s; stroke: %s;\"/>\n",
  x,y,
  x+5*d, y-3,
  x+3*d, y,
  x+5*d, y+3,
  color[c], color[c];
}

func line(x1,x2,y,output, c) {

  if (x1 == x2) {
    printf "<polyline points=\"%d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d %d,%d\" fill=\"none\" class=\"traceline\"/>\n",
      x1,y-18+7,
      x1+5,y-18+7,
      x1+12,y-15+7,
      x1+15,y-10+7,
      x1+15,y-5+7,
      x1+12,y-2+7,
      x1+5,y+7,
      x1,y+7;
    arrow(x1,y+7,1);

    xtext = x1 + 18;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"/>\n", $1, x1, y-yLineSpace+2, x1+15, y+7+1, $1 >> "imagemap"

  } else if (x1<x2) {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,-1,c);

    xtext = x1 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"/>\n", $1, x1, y-yLineSpace+2, x2, y+1, $1 >> "imagemap"

  } else {
    printf "<line x1=\"%d\" y1=\"%d\" x2=\"%d\" y2=\"%d\" class=\"traceline\" style=\"stroke: %s;\"/>\n", x1, y, x2, y, color[c];
    arrow(x2,y,1,c);

    xtext = x2 + 10;

    printf "    <area href=\"frames/Frame%d.html\" coords=\"%d,%d,%d,%d\" alt=\"frame %d\"/>\n", $1, x2, y-yLineSpace+2, x1, y+1, $1 >> "imagemap"
  }

  printf "\n", $1;
  printf "<text x=\"%d\" y=\"%d\" class=\"pkt-text\">%s</text>\n", xtext, y-4, output;
  printf "\n";
}

{
  y = NR;
  y = y * yLineSpace + ystart;

  if($1 == "#")
  {
	 output = "";
	 for (i=2; i<=NF; i++)
	 {
		if ($i == "!")
		   break;
		output = output " " $i;
	 }

	 link = "";
	 if($i == "!")
	 {
		i++;
		for(;i<=NF;i++)
		{
		   link = link " " $i;
		}
	 }

	 if (link != "")
	 {
		printf("<text x=\"%d\" y=\"%d\" class=\"link\">%s</text>\n", 50, y, output);
		printf("    <area href=\"%s\" coords=\"%d,%d,%d,%d\"/>\n", link, 50, y-yLineSpace+2, w, y+1) >> "imagemap";
	 }
	 else
	 {
		printf("<text x=\"%d\" y=\"%d\" class=\"comment-text\">%s</text>\n", 50, y, output);
	 }
  }
  else
  {
	 if ($4 ~ "{([0-9]+)}") {
		c = $4
		gsub("{", "", c)
		gsub("}", "", c)

		# If over 11 colors, choose black color as default...
		# TODO: change color behavior
		if (strtonum(c)>11) {c="0"}
	 }
	 l1 = sprintf("%s:%s", $2,$3);
	 l2 = sprintf("%s:%s", $5,$6);
	 for (i=0; i<numHosts; i++)
	 {
	   if(l1 ~ hosts[i])
	   {
		 x1 = strtonum(lookup[hosts[i]]);
	   }
	   if(l2 ~ hosts[i])
	   {
		 x2 = strtonum(lookup[hosts[i]]);
	   }
	 }

	 x1 = x1 * xHostSpace + leftMargin;
	 x2 = x2 * xHostSpace + leftMargin;

	 ORS = "";
	 printf "<text x=\"%d\" y=\"%d\" class=\"pkt-text\">%d</text>\n", leftMargin/2, y, $1;

	 output = "";
	 for(i=7;i<=NF;i++) output = output " " $i;

	 gsub("SIP(/SDP|/XML|) *(Status|Request): *","", output);
	 gsub(", with session description *$"," w/SDP",output);
	if ((x1==x2) && (noAuto==1)){
		}
	else{
		line(x1,x2,y,output,c);
		}
  }
}

END {
  printf "   </map>\n" >> "imagemap";
  printf "</svg>\n";
}


include systm.h in vnode_if.c so that panic is defined when we're doing DEBUG_ALL_VFS_LOCKS.

#!/usr/bin/awk -f

#
# Copyright (c) 1992, 1993
#	The Regents of the University of California.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#	This product includes software developed by the University of
#	California, Berkeley and its contributors.
# 4. Neither the name of the University nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#	@(#)vnode_if.sh	8.1 (Berkeley) 6/10/93
# $FreeBSD$
#
# Script to produce VFS front-end sugar.
#
# usage: vnode_if.awk <srcfile> [-c | -h]
#	(where <srcfile> is currently /sys/kern/vnode_if.src)
#

function usage()
{
	print "usage: vnode_if.awk <srcfile> [-c|-h]";
	exit 1;
}

function die(msg, what)
{
	printf msg "\n", what > "/dev/stderr";
	exit 1;
}

function t_spc(type)
{
	# Append a space if the type is not a pointer
	return (type ~ /\*$/) ? type : type " ";
}

# These are just for convenience ...
function printc(s) {print s > cfile;}
function printh(s) {print s > hfile;}

function add_debug_code(name, arg)
{
	if (debug_all_vfs_locks && lockdata[name, arg, "Entry"]) {
		# Add assertions for locking
		if (lockdata[name, arg, "Entry"] == "L")
			printh("\tASSERT_VOP_LOCKED("arg", \""uname"\");");
		else if (lockdata[name, arg, "Entry"] == "U")
			printh("\tASSERT_VOP_UNLOCKED("arg", \""uname"\");");
		else if (0) {
			# XXX More checks!
		}
	}
}

function find_arg_with_type (type)
{
	for (jj = 0; jj < numargs; jj++) {
		if (types[jj] == type) {
			return "VOPARG_OFFSETOF(struct " \
			    name "_args,a_" args[jj] ")";
		}
	}

	return "VDESC_NO_OFFSET";
}

BEGIN{

# Process the command line
for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[ch]+$/ && arg !~ /\.src$/)
		usage();
	if (arg ~ /^-.*c/)
		cfile = "vnode_if.c";
	if (arg ~ /^-.*h/)
		hfile = "vnode_if.h";
	if (arg ~ /\.src$/)
		srcfile = arg;
}
ARGC = 1;

if (!cfile && !hfile)
	exit 0;

if (!srcfile)
	usage();

debug_all_vfs_locks = (ENVIRON["DEBUG_ALL_VFS_LOCKS"] ~ /[Yy][Ee][Ss]/);

common_head = \
    "/*\n" \
    " * This file is produced automatically.\n" \
    " * Do not modify anything in here by hand.\n" \
    " *\n" \
    " * Created from $FreeBSD$\n" \
    " */\n" \
    "\n";

if (hfile)
	printh(common_head "extern struct vnodeop_desc vop_default_desc;");

if (cfile) {
	printc(common_head \
	    "#include <sys/param.h>\n" \
	    "#include <sys/systm.h>\n" \
	    "#include <sys/vnode.h>\n" \
	    "\n" \
	    "struct vnodeop_desc vop_default_desc = {\n" \
	    "	1,\t\t\t/* special case, vop_default => 1 */\n" \
	    "	\"default\",\n" \
	    "	0,\n" \
	    "	NULL,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	NULL,\n" \
	    "};\n");
}

while ((getline < srcfile) > 0) {
	if (NF == 0)
		continue;
	if ($1 ~ /^#/) {
		if (NF != 6  ||  $1 != "#%"  || \
		    $2 !~ /^[a-z]+$/  ||  $3 !~ /^[a-z]+$/  || \
		    $4 !~ /^.$/  ||  $5 !~ /^.$/  ||  $6 !~ /^.$/)
			continue;
		lockdata["vop_" $2, $3, "Entry"] = $4;
		lockdata["vop_" $2, $3, "OK"]    = $5;
		lockdata["vop_" $2, $3, "Error"] = $6;
		continue;
	}

	# Get the function name.
	name = $1;
	uname = toupper(name);
	# Get the function arguments.
	for (numargs = 0; ; ++numargs) {
		if ((getline < srcfile) <= 0) {
			die("Unable to read through the arguments for \"%s\"",
			    name);
		}
		if ($1 ~ /^\};/)
			break;

		# Delete comments, if any.
		gsub (/\/\*.*\*\//, "");

		# Condense whitespace and delete leading/trailing space.
		gsub(/[[:space:]]+/, " ");
		sub(/^ /, "");
		sub(/ $/, "");

		# Pick off direction.
		if ($1 != "INOUT" && $1 != "IN" && $1 != "OUT")
			die("No IN/OUT direction for \"%s\".", $0);
		dirs[numargs] = $1;
		sub(/^[A-Z]* /, "");

		if ((reles[numargs] = $1) == "WILLRELE")
			sub(/^[A-Z]* /, "");
		else
			reles[numargs] = "WONTRELE";

		# kill trailing ;
		if (sub(/;$/, "") < 1)
			die("Missing end-of-line ; in \"%s\".", $0);

		# pick off variable name
		if ((argp = match($0, /[A-Za-z0-9_]+$/)) < 1)
			die("Missing var name \"a_foo\" in \"%s\".", $0);
		args[numargs] = substr($0, argp);
		$0 = substr($0, 1, argp - 1);

		# what is left must be type
		# remove trailing space (if any)
		sub(/ $/, "");
		types[numargs] = $0;
	}

	if (hfile) {
		# Print out the vop_F_args structure.
		printh("struct "name"_args {\n\tstruct vnodeop_desc *a_desc;");
		for (i = 0; i < numargs; ++i)
			printh("\t" t_spc(types[i]) "a_" args[i] ";");
		printh("};");

		# Print out extern declaration.
		printh("extern struct vnodeop_desc " name "_desc;");

		# Print out function.
		printh("static __inline int " uname "(");
		for (i = 0; i < numargs; ++i) {
			printh("\t" t_spc(types[i]) args[i] \
			    (i < numargs - 1 ? "," : ")"));
		}
		printh("{\n\tstruct " name "_args a;");
		printh("\tint rc;");
		printh("\ta.a_desc = VDESC(" name ");");
		for (i = 0; i < numargs; ++i)
			printh("\ta.a_" args[i] " = " args[i] ";");
		for (i = 0; i < numargs; ++i)
			add_debug_code(name, args[i]);
		printh("\trc = VCALL(" args[0] ", VOFFSET(" name "), &a);");
		printh("\treturn (rc);\n}");
	}

	if (cfile) {
		# Print out the vop_F_vp_offsets structure.  This all depends
		# on naming conventions and nothing else.
		printc("static int " name "_vp_offsets[] = {");
		# as a side effect, figure out the releflags
		releflags = "";
		vpnum = 0;
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode *") {
				printc("\tVOPARG_OFFSETOF(struct " name \
				    "_args,a_" args[i] "),");
				if (reles[i] == "WILLRELE") {
					releflags = releflags \
					    "|VDESC_VP" vpnum "_WILLRELE";
				}
				vpnum++;
			}
		}

		sub(/^\|/, "", releflags);
		printc("\tVDESC_NO_OFFSET");
		printc("};");

		# Print out the vnodeop_desc structure.
		printc("struct vnodeop_desc " name "_desc = {");
		# offset
		printc("\t0,");
		# printable name
		printc("\t\"" name "\",");
		# flags
		vppwillrele = "";
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode **" && \
			    reles[i] == "WILLRELE") {
				vppwillrele = "|VDESC_VPP_WILLRELE";
			}
		}

		if (!releflags)
			releflags = "0";
		printc("\t" releflags vppwillrele ",");

		# vp offsets
		printc("\t" name "_vp_offsets,");
		# vpp (if any)
		printc("\t" find_arg_with_type("struct vnode **") ",");
		# cred (if any)
		printc("\t" find_arg_with_type("struct ucred *") ",");
		# thread (if any)
		printc("\t" find_arg_with_type("struct thread *") ",");
		# componentname
		printc("\t" find_arg_with_type("struct componentname *") ",");
		# transport layer information
		printc("\tNULL,\n};\n");
	}
}

if (hfile)
	close(hfile);
if (cfile)
	close(cfile);
close(srcfile);

exit 0;

}

#!/usr/bin/awk -f

#
# Copyright (c) 1992, 1993
#	The Regents of the University of California.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#	This product includes software developed by the University of
#	California, Berkeley and its contributors.
# 4. Neither the name of the University nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#	@(#)vnode_if.sh	8.1 (Berkeley) 6/10/93
# $FreeBSD$
#
# Script to produce VFS front-end sugar.
#
# usage: vnode_if.awk <srcfile> [-c | -h]
#	(where <srcfile> is currently /sys/kern/vnode_if.src)
#

function usage()
{
	print "usage: vnode_if.awk <srcfile> [-c|-h]";
	exit 1;
}

function die(msg, what)
{
	printf msg "\n", what > "/dev/stderr";
	exit 1;
}

function t_spc(type)
{
	# Append a space if the type is not a pointer
	return (type ~ /\*$/) ? type : type " ";
}

# These are just for convenience ...
function printc(s) {print s > cfile;}
function printh(s) {print s > hfile;}

function add_debug_code(name, arg)
{
	if (debug_all_vfs_locks && lockdata[name, arg, "Entry"]) {
		# Add assertions for locking
		if (lockdata[name, arg, "Entry"] == "L")
			printh("\tASSERT_VOP_LOCKED("arg", \""uname"\");");
		else if (lockdata[name, arg, "Entry"] == "U")
			printh("\tASSERT_VOP_UNLOCKED("arg", \""uname"\");");
		else if (0) {
			# XXX More checks!
		}
	}
}

function find_arg_with_type (type)
{
	for (jj = 0; jj < numargs; jj++) {
		if (types[jj] == type) {
			return "VOPARG_OFFSETOF(struct " \
			    name "_args,a_" args[jj] ")";
		}
	}

	return "VDESC_NO_OFFSET";
}

BEGIN{

# Process the command line
for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[ch]+$/ && arg !~ /\.src$/)
		usage();
	if (arg ~ /^-.*c/)
		cfile = "vnode_if.c";
	if (arg ~ /^-.*h/)
		hfile = "vnode_if.h";
	if (arg ~ /\.src$/)
		srcfile = arg;
}
ARGC = 1;

if (!cfile && !hfile)
	exit 0;

if (!srcfile)
	usage();

debug_all_vfs_locks = (ENVIRON["DEBUG_ALL_VFS_LOCKS"] ~ /[Yy][Ee][Ss]/);

common_head = \
    "/*\n" \
    " * This file is produced automatically.\n" \
    " * Do not modify anything in here by hand.\n" \
    " *\n" \
    " * Created from $FreeBSD$\n" \
    " */\n" \
    "\n";

if (hfile)
	printh(common_head "extern struct vnodeop_desc vop_default_desc;");

if (cfile) {
	printc(common_head \
	    "#include <sys/param.h>\n" \
	    "#include <sys/vnode.h>\n" \
	    "\n" \
	    "struct vnodeop_desc vop_default_desc = {\n" \
	    "	1,\t\t\t/* special case, vop_default => 1 */\n" \
	    "	\"default\",\n" \
	    "	0,\n" \
	    "	NULL,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	VDESC_NO_OFFSET,\n" \
	    "	NULL,\n" \
	    "};\n");
}

while ((getline < srcfile) > 0) {
	if (NF == 0)
		continue;
	if ($1 ~ /^#/) {
		if (NF != 6  ||  $1 != "#%"  || \
		    $2 !~ /^[a-z]+$/  ||  $3 !~ /^[a-z]+$/  || \
		    $4 !~ /^.$/  ||  $5 !~ /^.$/  ||  $6 !~ /^.$/)
			continue;
		lockdata["vop_" $2, $3, "Entry"] = $4;
		lockdata["vop_" $2, $3, "OK"]    = $5;
		lockdata["vop_" $2, $3, "Error"] = $6;
		continue;
	}

	# Get the function name.
	name = $1;
	uname = toupper(name);
	# Get the function arguments.
	for (numargs = 0; ; ++numargs) {
		if ((getline < srcfile) <= 0) {
			die("Unable to read through the arguments for \"%s\"",
			    name);
		}
		if ($1 ~ /^\};/)
			break;

		# Delete comments, if any.
		gsub (/\/\*.*\*\//, "");

		# Condense whitespace and delete leading/trailing space.
		gsub(/[[:space:]]+/, " ");
		sub(/^ /, "");
		sub(/ $/, "");

		# Pick off direction.
		if ($1 != "INOUT" && $1 != "IN" && $1 != "OUT")
			die("No IN/OUT direction for \"%s\".", $0);
		dirs[numargs] = $1;
		sub(/^[A-Z]* /, "");

		if ((reles[numargs] = $1) == "WILLRELE")
			sub(/^[A-Z]* /, "");
		else
			reles[numargs] = "WONTRELE";

		# kill trailing ;
		if (sub(/;$/, "") < 1)
			die("Missing end-of-line ; in \"%s\".", $0);

		# pick off variable name
		if ((argp = match($0, /[A-Za-z0-9_]+$/)) < 1)
			die("Missing var name \"a_foo\" in \"%s\".", $0);
		args[numargs] = substr($0, argp);
		$0 = substr($0, 1, argp - 1);

		# what is left must be type
		# remove trailing space (if any)
		sub(/ $/, "");
		types[numargs] = $0;
	}

	if (hfile) {
		# Print out the vop_F_args structure.
		printh("struct "name"_args {\n\tstruct vnodeop_desc *a_desc;");
		for (i = 0; i < numargs; ++i)
			printh("\t" t_spc(types[i]) "a_" args[i] ";");
		printh("};");

		# Print out extern declaration.
		printh("extern struct vnodeop_desc " name "_desc;");

		# Print out function.
		printh("static __inline int " uname "(");
		for (i = 0; i < numargs; ++i) {
			printh("\t" t_spc(types[i]) args[i] \
			    (i < numargs - 1 ? "," : ")"));
		}
		printh("{\n\tstruct " name "_args a;");
		printh("\tint rc;");
		printh("\ta.a_desc = VDESC(" name ");");
		for (i = 0; i < numargs; ++i)
			printh("\ta.a_" args[i] " = " args[i] ";");
		for (i = 0; i < numargs; ++i)
			add_debug_code(name, args[i]);
		printh("\trc = VCALL(" args[0] ", VOFFSET(" name "), &a);");
		printh("\treturn (rc);\n}");
	}

	if (cfile) {
		# Print out the vop_F_vp_offsets structure.  This all depends
		# on naming conventions and nothing else.
		printc("static int " name "_vp_offsets[] = {");
		# as a side effect, figure out the releflags
		releflags = "";
		vpnum = 0;
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode *") {
				printc("\tVOPARG_OFFSETOF(struct " name \
				    "_args,a_" args[i] "),");
				if (reles[i] == "WILLRELE") {
					releflags = releflags \
					    "|VDESC_VP" vpnum "_WILLRELE";
				}
				vpnum++;
			}
		}

		sub(/^\|/, "", releflags);
		printc("\tVDESC_NO_OFFSET");
		printc("};");

		# Print out the vnodeop_desc structure.
		printc("struct vnodeop_desc " name "_desc = {");
		# offset
		printc("\t0,");
		# printable name
		printc("\t\"" name "\",");
		# flags
		vppwillrele = "";
		for (i = 0; i < numargs; i++) {
			if (types[i] == "struct vnode **" && \
			    reles[i] == "WILLRELE") {
				vppwillrele = "|VDESC_VPP_WILLRELE";
			}
		}

		if (!releflags)
			releflags = "0";
		printc("\t" releflags vppwillrele ",");

		# vp offsets
		printc("\t" name "_vp_offsets,");
		# vpp (if any)
		printc("\t" find_arg_with_type("struct vnode **") ",");
		# cred (if any)
		printc("\t" find_arg_with_type("struct ucred *") ",");
		# thread (if any)
		printc("\t" find_arg_with_type("struct thread *") ",");
		# componentname
		printc("\t" find_arg_with_type("struct componentname *") ",");
		# transport layer information
		printc("\tNULL,\n};\n");
	}
}

if (hfile)
	close(hfile);
if (cfile)
	close(cfile);
close(srcfile);

exit 0;

}


add a usage fx

add a usage fx

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier
#  and the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology as opposed to
# a data uri.

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
	print "usage: ntriple2dot.awk curie_map.yaml rdf.nt > rdf.dot"
}

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:/))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/_"]="BNODE"
	# just until skolemized bnodes get in the curie map?
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	############################################################
	# revisit whether these exceptions are still necessary often
	# they may have been moved into the  prefix mapping file
	prefix["http://www.w3.org/1999/02/22-rdf-syntax-ns#"]="rdf"
	prefix["http://www.w3.org/2000/01/rdf-schema#"]="rdfs"
	prefix["http://www.w3.org/2002/07/owl#"]="owl"
	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in panther
	prefix["http://identifiers.org/wormbase"]="WormBase"
	# in IMPC
	prefix["http://www.mousephenotype.org"]="IMPC"
	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# till all non httpS: are purged
	prefix["http://monarchinitiative.org/"]="BASE"
	prefix["http://www.monarchinitiative.org/"]="BASE"
	prefix["http://monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://www.monarchinitiative.org/MONARCH"]=""
	prefix["http://www.monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://data.monarchinitiative.org/ttl"]="MonarchData"
	prefix["http://archive.monarchinitiative.org/ttl"]="MonarchArchive"
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, optional whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i  %s",FILENAME,FNR,$0) > "/dev/stderr"
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}

#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their @prefix (or literal object class)
#  Reduce predicates to the specific identifier
#  and the ontology they are from.(curi form)
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology as opposed to
# a data uri.

#  Perhaps an improvement would be to also
#  express subject and objects as individual curies (ala predicates)
#  iff their namespace is also used by a predicate.

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

##########################################################
# remove first and last chars from input string <>
function trim(str){
	return substr(str,2,length(str)-2)
}

########################################################
# remove final(ish) element in paths with various delimiters
# this leaves the namespace of a removed identifier

# this may need to be called more than once since
# some identifiers may have embedded delimiters other uri use differently
function stripid(uri){
	# <_:blanknode> are not allowed in ntriples (but may happen anyway)
	if(0 < match(uri, /^_:/))
		return "BNODE"
	# perspective endpoints, choose the longest
	delim["_"]=0; delim["/"]=0;
	delim["="]=0; delim[":"]=0; delim["#"]=0;
	char=""; max=-1;
	for(c in delim){
		l = match(uri, char)  # side effect is RLENGTH
		if(l > max){
			char = c;
			delim[char] = RLENGTH
		}
	}
	if(max<=0 || char=="")
		# we don't know what it is, a literal perhaps or uri fragment
		return uri
	else
		return substr(uri,1,delim[char])  # the probably truncated uri
}

# keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective
function simplify(str){
	gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
	gsub(/^_+|_+$/, "",str)
	gsub(/__*/,"_",str)
	return str
}

# if possible, find a shorter form for the input
function contract(uri){
	u = uri
	# shorten till longest uri in curi map is found (or not)
	while(!(u in prefix) &&  length(u))
		u = stripid(substr(u,1,length(u)-1))
	if(u in prefix)
		return prefix[u]
	else{
		# printf("WARN  %s\n", uri) > "/dev/stderr"
		return simplify(uri)
	}
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri){
	split(uri, b, "/")
	p = b[length(b)]
	anchor = match(p, "#")
	if(anchor > 0)
		p = substr(p, anchor+1)
	return p
}

BEGIN{
	# exceptions
	prefix["BNODE"]="BNODE"  # is a fixed point
	prefix["https://monarchinitiative.org/_"]="BNODE"
	# just until skolemized bnodes get in the curie map?
	prefix["https://monarchinitiative.org/.well-known/genid"]="BNODE"
	############################################################
	# revisit whether these exceptions are still necessary often
	# they may have been moved into the  prefix mapping file
	prefix["http://www.w3.org/1999/02/22-rdf-syntax-ns#"]="rdf"
	prefix["http://www.w3.org/2000/01/rdf-schema#"]="rdfs"
	prefix["http://www.w3.org/2002/07/owl#"]="owl"
	# in mgi
	prefix["https://www.mousephenotype.org"]="IMPC"
	# in panther
	prefix["http://identifiers.org/wormbase"]="WormBase"
	# in IMPC
	prefix["http://www.mousephenotype.org"]="IMPC"
	# in GWAScatalog
	prefix["http://www.ncbi.nlm.nih.gov/SNP/"]="dbSNP"
	# till all non httpS: are purged
	prefix["http://monarchinitiative.org/"]="BASE"
	prefix["http://www.monarchinitiative.org/"]="BASE"
	prefix["http://monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://www.monarchinitiative.org/MONARCH"]=""
	prefix["http://www.monarchinitiative.org/MONARCH_"]="MONARCH"
	prefix["http://data.monarchinitiative.org/ttl"]="MonarchData"
	prefix["http://archive.monarchinitiative.org/ttl"]="MonarchArchive"
}

# main loop
# parse and stash the curie yaml file (first file)
# YAML format is tic delimited word (the curi prefix)
# optional whitespace, a colon, optional whitespace
# then a tic delimited url
# (FNR == NR) && /^'[^']*' *: 'http[^']*'.*/ { # loosing some?
(FNR == NR) && /^'.*/ {
	split($0, arr, "'")
	if(arr[2]=="")
		arr[2]="BASE"
	prefix[arr[4]]=arr[2]
}

# process the ntriple file(s)  which are not the first file

### subject predicate and object are all uri
(FNR != NR) && /^<[^>]*> <[^>]*> <[^>]*> \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	o = contract(stripid(trim($3)))
	edgelist[s,ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri but the object is a literal
(FNR != NR) && /^<[^>]*> <[^>]*> "[^"]*".*\.$/ {
	s = contract(stripid(trim($1)))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist[s, ns ":" p, "LITERAL"]++
	done[NR]=1
}
### subject is a bare blank node, predicate and object are uri
(FNR != NR) && /^_:[^ ]* <[^>]*> <[^>]*> \.$/ {
	# once our bnode syntax is uniform we may want more info out of it
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	### Object (like subject)
	o = contract(stripid(trim($3)))
	edgelist["BNODE",ns ":" p,o]++
	done[NR]=1
}
### subject & predicate are uri & object is a bare blank node
(FNR != NR) && /^<[^>]*> <[^>]*> _:[^ ]* \.$/ {
	s = contract(stripid(trim($1)))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	#o = contract(stripid($3))
	edgelist[s,ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject and object are a bare blank nodes, predicate is a uri
(FNR != NR) && /^_:[^ ]* <[^>]*> _:[^ ]* \.$/ {
	# s = contract(stripid($1))
	p =  final(trim($2))
	ns = contract(stripid(trim($2)))
	# o = contract(stripid($3))
	edgelist["BNODE",ns ":" p,"BNODE"]++
	done[NR]=1
}
### subject is a bare blank node, predicate is a uri & the object is a literal
(FNR != NR) && /^_:[^ ]* <[^>]*> "[^"]*".*\.$/ {
	# s = contract(stripid($1))
	p = final(trim($2))
	ns = contract(stripid(trim($2)))
	edgelist["BNODE", ns ":" p, "LITERAL"]++
	done[NR]=1
}

# print anything else so it makes itself known
(FNR != NR) && (done[NR]<1) && (NF>1){
	printf("ERROR? in %s at line %i  %s",FILENAME,FNR,$0) > "/dev/stderr"
}

# output dot file, include edge counts
END{
	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	# sort the edges to facilitate future inter dot comparisons
	n=asorti(edgelist, sortlist)
	for (i=1; i<=n; i++) {
		edge=sortlist[i]
		split(edge ,spo, SUBSEP);
		print simplify(spo[1]) " -> " simplify(spo[3]) \
		" [label=\"" spo[2] " (" edgelist[edge] ")\"];"
	}
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	title = final(FILENAME)
	datestamp = strftime("%Y%m%d", systime())
	print "label=\"" substr(title,1,length(title)-3) " (" datestamp ")\";"
	print "}"
}


fix for relation between p tag and other block elem

function parse_line(line,   ret) {
	if (line ~ /^==+/ || lines ~ /^--+/) {
		if (line ~ /^==+/ && (slide == "hamaji" || slide == "shinh" || slide == "simple" || slide == "s")) {
		} else {
			ret = ret "
"
		}
	} else if (line ~ /\*\*[^[:space:]].*[^[:space:]]\*\*/) {
		ret = ret parse_line(gensub(/\*\*([^[:space:]].*[^[:space:]])\*\*/, "\\1", "g", line))
	} else if (line ~ /__[^[:space:]].*[^[:space:]]__/) {
		ret = ret parse_line(gensub(/__([^[:space:]].*[^[:space:]])__/, "\\1", "g", line))
	} else if (line ~ /\*[^[:space:]*].*[^[:space:]*]\*/) {
		ret = ret parse_line(gensub(/\*([^[:space:]*].*[^[:space:]*])\*/, "\\1", "g", line))
	} else if (line ~ /_[^[:space:]_].*[^[:space:]_]_/) {
		ret = ret parse_line(gensub(/_([^[:space:]_].*[^[:space:]_])_/, "\\1", "g", line))
	} else if (line ~ /\[.+\]\(.+\)/) {
		ret = ret gensub(/\[(.+)\]\((.+)\)/, "\\1", "g", line)
	} else {
		gsub(/  $/, "
", line)
		ret = ret line
	}
	return ret
}

function tag(stag, body, opt) {
	return "<" stag " " opt ">" body "</" stag ">"
}

function parse_para(para,   lines, ispara, ret, begp, endp) {
	max_idx = split(para, lines)
	begp = ""
	endp = ""
	ispara = 0
	for (i = 1; i <= max_idx; i++) {
		if (lines[i + 1] ~ /^==+/) {
			if (slide == "hamaji" || slide == "shinh" || slide == "simple" || slide == "s") {
				headline = tag("h1", parse_line(lines[i])) "
"
				i++
			} else {
				ret = ret endp tag("h1", parse_line(lines[i]))
				ispara = 0
				endp = ""
			}
		} else if (lines[i + 1] ~ /^--+/) {
			if (slide == "hamaji" || slide == "shinh" || slide == "simple" || slide == "s") {
				ret = ret endp parse_line(lines[i])
				ispara = 0
				endp = ""
			} else {
				ret = ret endp tag("h2", parse_line(lines[i]))
				ispara = 0
				endp = ""
			}
		} else if (lines[i] ~ /^#[[:space:]]+[^[:space:]].*[^[:space:]][[:space:]]*$/) {
			ret = ret endp tag("h1", parse_line(gensub(/^#[[:space:]]+([^[:space:]].*[^[:space:]])[[:space:]]*$/, "\\1", "g", lines[i])))
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^##[[:space:]]+[^[:space:]].*[^[:space:]][[:space:]]*$/) {
			ret = ret endp tag("h2", parse_line(gensub(/^##[[:space:]]+([^[:space:]].*[^[:space:]])[[:space:]]*$/, "\\1", "g", lines[i])))
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^###[[:space:]]+[^[:space:]].*[^[:space:]][[:space:]]*$/) {
			ret = ret endp tag("h3", parse_line(gensub(/^###[[:space:]]+([^[:space:]].*[^[:space:]])[[:space:]]*$/, "\\1", "g", lines[i])))
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^####[[:space:]]+[^[:space:]].*[^[:space:]][[:space:]]*$/) {
			ret = ret endp tag("h4", parse_line(gensub(/^####[[:space:]]+([^[:space:]].*[^[:space:]])[[:space:]]*$/, "\\1", "g", lines[i])))
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^#####[[:space:]]+[^[:space:]].*[^[:space:]][[:space:]]*$/) {
			ret = ret endp tag("h5", parse_line(gensub(/^#####[[:space:]]+([^[:space:]].*[^[:space:]])[[:space:]]*$/, "\\1", "g", lines[i])))
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^######[[:space:]]+[^[:space:]].*[^[:space:]][[:space:]]*$/) {
			ret = ret endp tag("h6", parse_line(gensub(/^######[[:space:]]+([^[:space:]].*[^[:space:]])[[:space:]]*$/, "\\1", "g", lines[i])))
			ispara = 0
			endp = ""
		}else if (lines[i] ~ /^[[:space:]]*>/) {
			ret = ret endp "
"
			for (; lines[i] ~ /^[[:space:]]*>/; i++) {
				ret = ret parse_line(gensub(/^[[:space:]]*>(.*)/, "\\1", "g", lines[i]))
			}
			i--
			ret = ret "
"
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^[[:space:]]*(\*|-)[[:space:]]+[^[:space:]].*/) {
			ret = ret endp "
"
			for (; lines[i] ~ /^[[:space:]]*(\*|-)/; i++) {
				ret = ret tag("li", parse_line(gensub(/^[[:space:]]*(\*|-)[[:space:]]+([^[:space:]].*)/, "\\2", "g", lines[i])))
			}
			i--
			ret = ret "
"
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^[[:space:]]*[[:digit:]]+\.[[:space:]]+[^[:space:]].*/) {
			ret = ret endp "
"
			for (; lines[i] ~ /^[[:space:]]*[[:digit:]]+\.[[:space:]]+[^[:space:]].*/; i++) {
				ret = ret tag("li", parse_line(gensub(/^[[:space:]]*[[:digit:]]+\.[[:space:]]+([^[:space:]].*)/, "\\1", "g", lines[i])))
			}
			i--
			ret = ret "
"
			ispara = 0
			endp = ""
		} else if (lines[i] ~ /^```/) {
			ret = ret endp "<pre><code>"
			for (i++; lines[i] !~ /^```/; i++) {
				gsub(/>/, "\\>", lines[i])
				gsub(/</, "\\<", lines[i])
				ret = ret  lines[i] "\n"
			}
			ret = ret "
"
			ispara = 0
			endp = ""
		} else {
			if (ispara != 0) {
				ret = ret parse_line(lines[i])
			} else {
			ret = ret "
" parse_line(lines[i])
			ispara = 1
			endp = "
"
			}
		}
	}
	if (ispara != 0) {
		ret = ret "
"
		ispara = 0
		endp = ""
	}
	return ret
}

BEGIN {
	RS = ""
	FS = "\n"
	slide = "simple"
	filename = gensub(/(.*)\.md/, "\\1", "g", ARGV[1])
	"mkdir -p " filename | getline
	print ARGV[1] "-> ./" filename "/" filename ".html"
}

{
	if (slide == "hamaji" || slide == "shinh" || slide == "simple" || slide == "s") {
		body = parse_para($0)
		print "<!DOCTYPE html>" tag("html", tag("head","<meta charset=\"UTF-8\">") tag("body",
				tag("a", "<", "href = \"./" filename NR-1 ".html\"") \
				tag("a", ">", "href = \"./" filename NR+1 ".html\"") \
				tag("section", headline body) \
				)) >  "./" filename "/" filename NR ".html"
	} else {
		body = body tag("p", parse_para($0))
	}

}

END {
}


Remove special processing that BBox fixed

# Produce a raw data spreadsheet from fields in BritBox Catalog --  apple_catalogue_feed.xml
# The "showContentId" from "tv_show" items are needed to process "tv_season" and "tv_episode" items,
# so you should sort the catalog to ensure items are in the proper order.

# INVOCATION
#   awk -v ERRORS=$ERRORS -v IDS_SEASONS=$IDS_SEASONS -v IDS_EPISODES=$IDS_EPISODES \
#       -v RAW_TITLES=$RAW_TITLES -f getBBoxCatalogFromSitemap.awk $SORTED_SITEMAPS >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title         3 Seasons          4 Episodes         5 Duration      6 Genre
#     7 Year          8 Rating        9 Description     10 Content_Type    11 Content_ID   12 Entity_ID
#    13 Show_Type    14 Date_Type    15 Original_Date   16 Show_ID         17 Season_ID    18 Sn_#
#    19 Ep_#         20 1st_#        21 Last_#

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tGenre\tYear\tRating\tDescription\t")
    printf ("Content_Type\tContent_ID\tEntity_ID\tShow_Type\tDate_Type\tOriginal_Date\t")
    printf ("Show_ID\tSeason_ID\tSn_#\tEp_#\t1st_#\tLast_#\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    firstLineNum = NR
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType > "/dev/stderr"
    # print "contentId = " contentId > "/dev/stderr"
    #
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = ""
    numEpisodes = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title > "/dev/stderr"
    gsub (/&/,"\\&",title)
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    genre = toupper(substr(genre,1,1)) substr(genre,2)
    sub ("Sci_fi","Sci-Fi",genre)
    sub ("Special_interest","Special interest",genre)
    # print "genre = " genre > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description > "/dev/stderr"
    gsub (/&/,"\\&",description)
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType > "/dev/stderr"
    # print "originalDate = " originalDate > "/dev/stderr"
    # print "year = " year > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId > "/dev/stderr"
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    lastLineNum = NR

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERRORS
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERRORS
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> contentId = " contentId > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
        # print "---" > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERRORS
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERRORS
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> originalDate = " originalDate > "/dev/stderr"
        # print "==> contentId = " contentId > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
        # print "---" > "/dev/stderr"
    }

    if (EntityId == "") {
        missingEntityIds += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s at line %d\n", contentType, contentId,
                title, showContentId, firstLineNum) >> ERRORS
    }

    if (contentType == "movie") {
        totalMovies += 1
        # print "\ntv_movie" > "/dev/stderr"
        # Wish I didn't have to do this, but "movie" is too common to be in a key field
        contentType = "tv_movie"
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
    }

    if (contentType == "tv_show") {
        totalShows += 1
        # print "\ntv_show" > "/dev/stderr"
        showArray[totalShows] = contentId
        titleArray[totalShows] = title
        sortkey = sprintf ("%s (1) S%s %s", title, EntityId, contentId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
        #
        str = "grep -c " contentId " " IDS_SEASONS
        str | getline numSeasons
        close str
        #
        str = "grep -c " contentId " " IDS_EPISODES
        str | getline numEpisodes
        close str
    }

    if (contentType == "tv_season") {
        totalSeasons += 1
        # print "\ntv_season" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02d %s", showTitle, seasonNumber, showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02d %s", showTitle, seasonNumber, showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02d, %s", showTitle, seasonNumber, title)
    }

    if (contentType == "tv_episode") {
        totalEpisodes += 1
        # print "\ntv_episode" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                    showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02dE%03d, %s", showTitle, seasonNumber, episodeNumber, title)
    }

    # Generate a link that will lead to the show on Britbox
    # https://www.britbox.com/us/movie/A_Queen_Is_Crowned_13551
    # https://www.britbox.com/us/movie/_13551
    #
    # https://www.britbox.com/us/show/All_Creatures_Great_and_Small_23737
    # https://www.britbox.com/us/show/_23737
    #
    # https://www.britbox.com/us/season/All_Creatures_Great_and_Small_S2_23752
    # https://www.britbox.com/us/season/_23752
    #
    # https://www.britbox.com/us/episode/All_Creatures_Great_and_Small_S2_E2_23764
    # https://www.britbox.com/us/episode/_23764
    showType_URL = contentType "/"
    sub ("tv_","",showType_URL)
    full_URL = "https://www.britbox.com/us/" showType_URL EntityId
    fullTitle = "=HYPERLINK(\"" full_URL "\";\"" title "\")"
    # print "fullTitle = " fullTitle > "/dev/stderr"

    # A seasonless "show" should have blank rather than 0 seasons
    if (showType == "seasonless")
        numSeasons = ""
    # If an "episode" sortkey or fullTitle conains "S00E" delete the "S00" part
    sub ("S00E","E",sortkey)
    sub ("S00E","E",fullTitle)
    # Print everything except tv_seasons
    if (contentType != "tv_season")
        printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%06d\t%06d\n",
                sortkey, fullTitle, numSeasons, numEpisodes, duration, genre, year, rating, description,
                contentType, contentId, EntityId, showType, dateType, originalDate, showContentId,
                seasonContentId, seasonNumber, episodeNumber, firstLineNum, lastLineNum)
}

END {
    printf ("In getBBoxCatalogFromSitemap.awk\n") > "/dev/stderr"

    totalMovies == 1 ? pluralMovies = "movie" : pluralMovies = "movies"
    totalShows == 1 ? pluralShows = "show" : pluralShows = "shows"
    totalSeasons == 1 ? pluralSeasons = "season" : pluralSeasons = "seasons"
    totalEpisodes == 1 ? pluralEpisodes = "episode" : pluralEpisodes = "episodes"
    #
    printf ("    Processed %d %s, %d %s, %d %s, %d %s\n", totalMovies, pluralMovies, totalShows,
            pluralShows, totalSeasons, pluralSeasons, totalEpisodes, pluralEpisodes) > "/dev/stderr"

    if (missingEntityIds > 0 ) {
        missingEntityIds == 1 ? plural = "EntityId" : plural = "EntityIds"
        printf ("%8d missing %s in %s\n", missingEntityIds, plural, FILENAME) > "/dev/stderr"
    }

    if (missingShowContentIds > 0 ) {
        missingShowContentIds == 1 ? plural = "showContentId" : plural = "showContentIds"
        printf ("%8d missing %s in %s\n", missingShowContentIds, plural, FILENAME) > "/dev/stderr"
    }

    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? plural = "title" : plural = "titles"
        printf ("%8d %s revised in %s\n", revisedTitles, plural, FILENAME) > "/dev/stderr"
    }
}

# Produce a raw data spreadsheet from fields in BritBox Catalog --  apple_catalogue_feed.xml
# The "showContentId" from "tv_show" items are needed to process "tv_season" and "tv_episode" items,
# so you should sort the catalog to ensure items are in the proper order.

# INVOCATION
#   awk -v ERRORS=$ERRORS -v IDS_SEASONS=$IDS_SEASONS -v IDS_EPISODES=$IDS_EPISODES \
#       -v RAW_TITLES=$RAW_TITLES -f getBBoxCatalogFromSitemap.awk $SORTED_SITEMAPS >$CATALOG_SPREADSHEET

# Field numbers
#     1 Sortkey       2 Title         3 Seasons          4 Episodes         5 Duration      6 Genre
#     7 Year          8 Rating        9 Description     10 Content_Type    11 Content_ID   12 Entity_ID
#    13 Show_Type    14 Date_Type    15 Original_Date   16 Show_ID         17 Season_ID    18 Sn_#
#    19 Ep_#         20 1st_#        21 Last_#

BEGIN {
    # Print header
    printf ("Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tGenre\tYear\tRating\tDescription\t")
    printf ("Content_Type\tContent_ID\tEntity_ID\tShow_Type\tDate_Type\tOriginal_Date\t")
    printf ("Show_ID\tSeason_ID\tSn_#\tEp_#\t1st_#\tLast_#\n")
}

# Don't process credits
/<credit role=/,/<\/credit>/ {
    next
}

# Don't use Canadian ratings
/<rating systemCode="ca-tv">/ {
    next
}

# Don't use pubDate - it's too random to be useful
# <pubDate>2020-06-05T21:51:00Z</pubDate>
/<pubDate>/ {
    next
}

# Don't use "cover_artwork_horizontal"
# <artwork url="https://britbox-images-prod.s3-eu-west-1.amazonaws.com/3840x2160px/p05wv7gy.png" type="cover_artwork_horizontal" />
/<artwork url=.*type="cover_artwork_horizontal"/ {
    next
}

# Grab contentType & contentId
# <item contentType="tv_episode" contentId="p079sxm9">
/<item contentType="/ {
    firstLineNum = NR
    split ($0,fld,"\"")
    contentType = fld[2]
    contentId = fld[4]
    # print "contentType = " contentType > "/dev/stderr"
    # print "contentId = " contentId > "/dev/stderr"
    #
    # Make sure no fields will be carried over due to missing keys
    sortkey = ""
    title = ""
    EntityId = ""
    genre = ""
    rating = ""
    showType = ""
    duration = ""
    dateType = ""
    originalDate = ""
    showContentId = ""
    seasonContentId = ""
    seasonNumber = ""
    episodeNumber = ""
    description = ""
    # Generated fields
    year = ""
    # New fields that haven't been created yet
    numSeasons = ""
    numEpisodes = ""
}

# Grab title
# <title locale="en-US">Frequent Flyers</title>
/<title locale="en-US">/ {
    split ($0,fld,"[<>]")
    title = fld[3]
    # print "title = " title > "/dev/stderr"
    gsub (/&/,"\\&",title)
}

# Grab genre
# <genre>drama</genre>
/<genre>/ {
    split ($0,fld,"[<>]")
    genre = fld[3]
    genre = toupper(substr(genre,1,1)) substr(genre,2)
    sub ("Sci_fi","Sci-Fi",genre)
    sub ("Special_interest","Special interest",genre)
    # print "genre = " genre > "/dev/stderr"
}

# Grab description
# <description>Set in the beautiful Yorkshire Dales, ...word.</description>
/<description>/ {
    split ($0,fld,"[<>]")
    description = fld[3]
    # print "description = " description > "/dev/stderr"
    gsub (/&/,"\\&",description)
}

# Grab rating
# <rating systemCode="us-tv">TV-PG</rating>
/<rating systemCode=/ {
    split ($0,fld,"[<>]")
    rating = fld[3]
    # print "rating = " rating > "/dev/stderr"
}

# Grab type
# <type>series</type>
/<type>/ {
    split ($0,fld,"[<>]")
    showType = fld[3]
    # print "showType = " showType > "/dev/stderr"
}

# Grab duration
# <duration>3000</duration>
/<duration>/ {
    split ($0,fld,"[<>]")
    seconds = fld[3]
    secs = seconds % 60
    mins = int(seconds / 60 % 60)
    hrs = int(seconds / 3600)
    duration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (duration == "00:00:00")
        duration = ""
    # print "duration = " seconds " seconds = " duration > "/dev/stderr"
}

# Grab originalDate
# <originalAirDate>1978-03-25</originalAirDate>
# <originalPremiereDate>1978-01-08</originalPremiereDate>
# <originalReleaseDate>2017-12-25</originalReleaseDate>
/<original.*Date>/ {
    split ($0,fld,"[<>]")
    dateType = fld[2]
    sub (/original/,"",dateType)
    originalDate = fld[3]
    year = substr (originalDate,1,4)
    # print "dateType = " dateType > "/dev/stderr"
    # print "originalDate = " originalDate > "/dev/stderr"
    # print "year = " year > "/dev/stderr"
}

# Grab showContentId
# <showContentId>b008yjd9</showContentId>
/<showContentId>/ {
    split ($0,fld,"[<>]")
    showContentId = fld[3]
    # print "showContentId = " showContentId > "/dev/stderr"
}

# Grab seasonContentId
# <seasonContentId>p0318ps9</seasonContentId>
/<seasonContentId>/ {
    split ($0,fld,"[<>]")
    seasonContentId = fld[3]
    # print "seasonContentId = " seasonContentId > "/dev/stderr"
}

# Grab seasonNumber
# <seasonNumber>1</seasonNumber>
/<seasonNumber>/ {
    split ($0,fld,"[<>]")
    seasonNumber = fld[3]
    # print "seasonNumber = " seasonNumber > "/dev/stderr"
}

# Grab episodeNumber
# <episodeNumber>10</episodeNumber>
/<episodeNumber>/ {
    split ($0,fld,"[<>]")
    episodeNumber = fld[3]
    # print "episodeNumber = " episodeNumber > "/dev/stderr"
}

# Grab EntityId from artwork
# <artwork url="https://us.britbox.com/isl/api/v1/dataservice/ResizeImage/$value?Format='jpg'&Quality=45&ImageId='176236'&EntityType='Item'&EntityId='16103'&Width=1920&Height=1080&ResizeAction='fit'" type="tile_artwork" />
/<artwork url=.*EntityId='/ {
    sub (/.*EntityId='/,"")
    sub (/&apos.*/,"")
    EntityId = "_" $0
    # print "EntityId = " EntityId > "/dev/stderr"
}

# Do any special end of item processing, then print raw data spreadsheet row
/<\/item>/ {
    lastLineNum = NR

    # "Porridge" needs to be revised to avoid duplicate names
    if (title == "Porridge") {
        if (EntityId == "_9509") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (1974-1977)'\n", title) >> ERRORS
            title = "Porridge (1974-1977)"
        } else if (EntityId == "_14747") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Porridge (2016-2017)'\n", title) >> ERRORS
            title = "Porridge (2016-2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> contentId = " contentId > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
        # print "---" > "/dev/stderr"
    }

    # "A Midsummer Night's Dream" needs to be revised to avoid duplicate names
    if (title == "A Midsummer Night's Dream") {
        if (EntityId == "_26179") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (1981)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (1981)"
        } else if (EntityId == "_15999") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'A Midsummer Night\'s Dream (2016)'\n", title) >> ERRORS
            title = "A Midsummer Night's Dream (2016)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
    }

    # "Maigret" needs to be revised to clarify timeframe
    if (title ~ /^Maigret/ && contentType == "tv_show") {
        if (EntityId == "_15928") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (1992-1993)'\n", title) >> ERRORS
            title = "Maigret (1992-1993)"
        } else if (EntityId == "_15974") {
            revisedTitles += 1
            printf ("==> Changed title '%s' to 'Maigret (2016–2017)'\n", title) >> ERRORS
            title = "Maigret (2016–2017)"
        }
        # print "==> title = " title > "/dev/stderr"
        # print "==> originalDate = " originalDate > "/dev/stderr"
        # print "==> contentId = " contentId > "/dev/stderr"
        # print "==> EntityId = " EntityId > "/dev/stderr"
        # print "---" > "/dev/stderr"
    }

    if (EntityId == "") {
        missingEntityIds += 1
        printf ("==> Missing EntityId for %s %s '%s' in show %s at line %d\n", contentType, contentId,
                title, showContentId, firstLineNum) >> ERRORS
        # Special processing for tv_episode p07gnw9f in show p07gnpx0 - add missing EntityId
        if (contentId == "p07gnw9f") {
            EntityId = "_23842"
            printf ("==> Added EntityId %s for %s %s '%s' in show %s\n", EntityId, contentType, contentId,
                    title, showContentId) >> ERRORS
            # print "==> EntityId = " EntityId > "/dev/stderr"
        }
    }

    if (contentType == "movie") {
        totalMovies += 1
        # print "\ntv_movie" > "/dev/stderr"
        # Wish I didn't have to do this, but "movie" is too common to be in a key field
        contentType = "tv_movie"
        sortkey = sprintf ("%s (1) %s M%s", title, originalDate, EntityId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
    }

    if (contentType == "tv_show") {
        totalShows += 1
        # print "\ntv_show" > "/dev/stderr"
        showArray[totalShows] = contentId
        titleArray[totalShows] = title
        sortkey = sprintf ("%s (1) S%s %s", title, EntityId, contentId)
        # print "sortkey = " sortkey > "/dev/stderr"
        print title >> RAW_TITLES
        #
        str = "grep -c " contentId " " IDS_SEASONS
        str | getline numSeasons
        close str
        #
        str = "grep -c " contentId " " IDS_EPISODES
        str | getline numEpisodes
        close str
    }

    if (contentType == "tv_season") {
        totalSeasons += 1
        # print "\ntv_season" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02d %s", showTitle, seasonNumber, showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02d %s", showTitle, seasonNumber, showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02d, %s", showTitle, seasonNumber, title)
    }

    if (contentType == "tv_episode") {
        totalEpisodes += 1
        # print "\ntv_episode" > "/dev/stderr"
        for ( i = 1; i <= totalShows; i++ ) {
            if (showArray[i] == showContentId) {
                showTitle = titleArray[i]
                break
            }
        }
        sortkey = sprintf ("%s (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                showContentId)
        # Check that showContentId was actually found - if not, put parens around showTitle in sortkey
        if (showArray[i] != showContentId) {
            missingShowContentIds += 1
            printf ("==> Missing showContentId for %s %s '%s' in show %s at line %d\n", contentType,
                    contentId, title, showContentId, firstLineNum) >> ERRORS
            sortkey = sprintf ("(%s) (2) S%02dE%03d %s", showTitle, seasonNumber, episodeNumber,
                    showContentId)
        }
        # print "sortkey = " sortkey > "/dev/stderr"
        # Compose title
        title = sprintf ("%s, S%02dE%03d, %s", showTitle, seasonNumber, episodeNumber, title)
    }

    # Generate a link that will lead to the show on Britbox
    # https://www.britbox.com/us/movie/A_Queen_Is_Crowned_13551
    # https://www.britbox.com/us/movie/_13551
    #
    # https://www.britbox.com/us/show/All_Creatures_Great_and_Small_23737
    # https://www.britbox.com/us/show/_23737
    #
    # https://www.britbox.com/us/season/All_Creatures_Great_and_Small_S2_23752
    # https://www.britbox.com/us/season/_23752
    #
    # https://www.britbox.com/us/episode/All_Creatures_Great_and_Small_S2_E2_23764
    # https://www.britbox.com/us/episode/_23764
    showType_URL = contentType "/"
    sub ("tv_","",showType_URL)
    full_URL = "https://www.britbox.com/us/" showType_URL EntityId
    fullTitle = "=HYPERLINK(\"" full_URL "\";\"" title "\")"
    # print "fullTitle = " fullTitle > "/dev/stderr"

    # A seasonless "show" should have blank rather than 0 seasons
    if (showType == "seasonless")
        numSeasons = ""
    # If an "episode" sortkey or fullTitle conains "S00E" delete the "S00" part
    sub ("S00E","E",sortkey)
    sub ("S00E","E",fullTitle)
    # Print everything except tv_seasons
    if (contentType != "tv_season")
        printf ("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%06d\t%06d\n",
                sortkey, fullTitle, numSeasons, numEpisodes, duration, genre, year, rating, description,
                contentType, contentId, EntityId, showType, dateType, originalDate, showContentId,
                seasonContentId, seasonNumber, episodeNumber, firstLineNum, lastLineNum)
}

END {
    printf ("In getBBoxCatalogFromSitemap.awk\n") > "/dev/stderr"

    totalMovies == 1 ? pluralMovies = "movie" : pluralMovies = "movies"
    totalShows == 1 ? pluralShows = "show" : pluralShows = "shows"
    totalSeasons == 1 ? pluralSeasons = "season" : pluralSeasons = "seasons"
    totalEpisodes == 1 ? pluralEpisodes = "episode" : pluralEpisodes = "episodes"
    #
    printf ("    Processed %d %s, %d %s, %d %s, %d %s\n", totalMovies, pluralMovies, totalShows,
            pluralShows, totalSeasons, pluralSeasons, totalEpisodes, pluralEpisodes) > "/dev/stderr"

    if (missingEntityIds > 0 ) {
        missingEntityIds == 1 ? plural = "EntityId" : plural = "EntityIds"
        printf ("%8d missing %s in %s\n", missingEntityIds, plural, FILENAME) > "/dev/stderr"
    }

    if (missingShowContentIds > 0 ) {
        missingShowContentIds == 1 ? plural = "showContentId" : plural = "showContentIds"
        printf ("%8d missing %s in %s\n", missingShowContentIds, plural, FILENAME) > "/dev/stderr"
    }

    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? plural = "title" : plural = "titles"
        printf ("%8d %s revised in %s\n", revisedTitles, plural, FILENAME) > "/dev/stderr"
    }
}


Add totals to bottom line of output

Add totals to bottom line of output

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@16921 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# input: publisher\tcontract_year\tyear\tstatus

BEGIN {
  FS="\t"
  pn = 0
}

{
  if (!(($1,$2,$3) in b)) {
    p[pn] = $1
    t[pn] = $2
    d[pn] = $3
    pn++
  }
  b[$1,$2,$3]++
  c[$1,$2,$3,$4]++
  x[$4]++
  tt++
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  sn = 10

  printf "Publisher\tContract\tYear\tTotal"
  for (j = 0 ; j < sn ; j++) {
    printf "\t%s", s[j]
  }
  printf "\n"

  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], t[i], d[i], b[p[i],t[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      printf "\t%d", c[p[i],t[i],d[i],s[j]]
    }
    printf "\n"
  }
    printf "Publisher\tContract\tYear\t%d", tt
    for (j = 0 ; j < sn ; j++) {
    	printf "\t%d", x[s[j]]
    }

}

#!/usr/bin/awk -f
# input: publisher\tcontract_year\tyear\tstatus

BEGIN {
  FS="\t"
  pn = 0
}

{
  if (!(($1,$2,$3) in b)) {
    p[pn] = $1
    t[pn] = $2
    d[pn] = $3
    pn++
  }
  b[$1,$2,$3]++
  c[$1,$2,$3,$4]++
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  sn = 10

  printf "Publisher\tContract\tYear\tTotal"
  for (j = 0 ; j < sn ; j++) {
    printf "\t%s", s[j]
  }
  printf "\n"

  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], t[i], d[i], b[p[i],t[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      printf "\t%d", c[p[i],t[i],d[i],s[j]]
    }
    printf "\n"
  }
}