Add new script for gln testing report

Add new script for gln testing report

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@24405 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[tester],status

BEGIN {
  FS="\t"
  pn = 0
}

{
    nn = split($2,na,/\./)
    lp2 = na[nn]
    if (!(($1,lp2) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      r[pn] = $3
      pn++
    }
    b[$1,lp2]++
    c[$1,lp2,$4]++
    x[$4]++
    tt++
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%d", p[i], n[i], r[i], b[p[i],n[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

error: pathspec 'scripts/tdb/statuses_gln.awk' did not match any file(s) known to git


updated awk yamado parser for dating and abstract

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/orbifx/yamado.git/'


SCCS var.gen.awk 10/09/30 08:23:33: date and time created 10/09/30 08:23:33 by neo

#!/bin/awk
#This is a utility script to generate get-/set and conversion-functions for the
#variant data type pvar implemented in var.h
#
#I was lazy, had not the wish to write every of  this functions by hand... so
#I'm sorry for yet another generator script...

BEGIN								{
										FS = "[ \t;:]+"
										datatype = ""
										member = ""
										within_vargen = 0
										first = 0
										variants_cnt = 0
										on_vargen = 0

										SUBSEP = "#"

										print "/*"
										print " * WARNING:"
										print " * THIS FILE IS GENERATED - DO NOT EDIT MANUALLY - IT WILL GO AWAY!"
										print " */"
										print ""
										print "#include \"pbasis.h\""
										print ""
									}

END									{
										prev = ""
										for( i = 1; i <= variants_cnt; i++ )
										{
											if( prev == members[ variants[ i ] \
													SUBSEP "member" ] )
												continue

											if( with_conv )
												convfunc( variants[ i ] );

											prev = members[ variants[ i ] \
													SUBSEP "member" ]
										}

										if( with_conv )
											allconv_func()
									}

/[a-zA-Z_]+\*?[ \t]*[a-zA-Z_]+;/	{
										if( !within_vargen )
										{
											datatype = $2
											member = $3
											next
										}
									}

/vargen:/							{
										within_vargen = 1
										line = $0
										gsub( "/\\*", "", line )
										gsub( "\\*/", "", line )
										split( line, types, ":" )

										var_type = types[2]
										var_format = types[3]
										var_define = types[4]
										var_emptyval = types[5]

										if( var_define == "" )
											var_define = "PVAR_" \
												toupper( var_type )

										variants[ ++variants_cnt ] = datatype
										members[ datatype SUBSEP \
													"member" ] = member
										members[ datatype SUBSEP \
													"var_type" ] = var_type
										members[ datatype SUBSEP \
													"var_format" ] = var_format
										members[ datatype SUBSEP \
													"var_define" ] = var_define
										members[ datatype SUBSEP \
													"var_emptyval" ] = var_emptyval

										setcode = ""
										first = 1
										lastconvcode = ""
									}

/to [a-zA-Z_]+\*?:/					{
										if( !within_vargen )
											next

										convcode = trim( substr( $0, \
														index( $0, ":" ) + 1) )

										if( convcode == "(same)" )
											convcode = lastconvcode

										#print $3 SUBSEP datatype
										convert[ $3 SUBSEP datatype ] = convcode

										lastconvcode = convcode
									}

/set:/								{
										if( !within_vargen )
											next

										gsub( "set:", "", $0 )

										if( !first )
										{
											if( setcode != "" )
												setcode = setcode "\n"

											setcode = setcode "\t" trim( $0 )
										}

										first = 0
										next
									}

									{
										if( !within_vargen )
											next

										line = trim( $0 )
										if( substr( line, length( line ) \
												 - 1, 2 ) == "*/" )
										{
											within_vargen = 0

											if( with_set )
												setfunc()

											if( with_get )
												getfunc()
										}

										first = 0
									}

#--- Utility functions go here ---

function setfunc()
{
	print "/* -FUNCTION--------------------------------------------------------------------"
	print "	Function:		pvar_set_" var_type "()"
	print ""
	print "	Author:			Jan Max Meyer (generated with var.gen.awk)"
	print ""
	print "	Usage:			Sets a variant's " datatype " data value and type."
	print ""
	print "	Parameters:		pvar*	var	Pointer to pvar structure."
	print "					" datatype "	" member "	Value to be assigned."
	print ""
	print "	Returns:		" datatype "		The value of " member "."
	print ""
	print "	~~~ CHANGES & NOTES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "	Date:		Author:			Note:"
	print "----------------------------------------------------------------------------- */"

	print datatype " pvar_set_" var_type "( pvar* var, " \
				datatype " " member " )"
	print "{"
	print "	PROC( \"pvar_set_" var_type "\" );"
	print "	PARMS( \"var\", \"%p\", var );"
	print "	PARMS( \"" member "\", \"" var_format "\", " member " );"
	print ""
	print "	pvar_reset( var );"
	print " var->type &= ~0x0F;"
	print "	var->type |= " var_define ";"
	print "	var->val." member " = " member ";"

	if( setcode != "" )
		print setcode

	print ""
	print "	RETURN( var->val." member " );"
	print "}\n"
}

function getfunc()
{
	print "/* -FUNCTION--------------------------------------------------------------------"
	print "	Function:		pvar_get_" var_type "()"
	print ""
	print "	Author:			Jan Max Meyer (generated with var.gen.awk)"
	print ""
	print "	Usage:			Returns a variant's " datatype " data value and type."
	print "					If the variant exists in another data type, it will be"
	print "					converted, so use it carefully if data loss is not desired!"
	print ""
	print "	Parameters:		pvar*	var	Pointer to pvar structure."
	print ""
	print "	Returns:		" datatype "		The " datatype "-value of the variant."
	print ""
	print "	~~~ CHANGES & NOTES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "	Date:		Author:			Note:"
	print "----------------------------------------------------------------------------- */"

	print datatype " pvar_get_" var_type "( pvar* var )"
	print "{"
	print "	PROC( \"pvar_get_" var_type "\" );"
	print "	PARMS( \"var\", \"%p\", var );"
	print ""
	print "	if( pvar_type( var ) != " var_define ")"
	print "	{"
	print "		if( pvar_is_convertible( var ) )"
	print "		{"
	print "			MSG( \"Conversion allowed and required\" );"
	print "			if( pvar_convert( var, " var_define " ) != ERR_OK )"
	print "				RETURN( (" datatype ")" var_emptyval " );"
	print "		}"
	print "		else"
	print "			RETURN( (" datatype ")" var_emptyval " );"
	print "	}"
	print ""
	print "	RETURN( var->val." member " );"
	print "}\n"
}

function convfunc( type )
{
	for( ok in okdone )
		okdone[ ok ] = 0

	member = members[ type SUBSEP "member" ]
	var_type = members[ type SUBSEP "var_type" ]
	var_format = members[ type SUBSEP "var_format" ]
	var_define = members[ type SUBSEP "var_define" ]
	var_emptyval = members[ type SUBSEP "var_emptyval" ]

	print "/* -FUNCTION--------------------------------------------------------------------"
	print "	Function:		pvar_to_" var_type "()"
	print ""
	print "	Author:			Jan Max Meyer (generated with var.gen.awk)"
	print ""
	print "	Usage:			Converts a variant's current value into a " type " variable."

	if( var_type ~ /string/ )
		print "					The returned memory is allocated, and must be freed by the caller."

	print ""
	print "	Parameters:		pvar*	var	Pointer to pvar structure."
	print ""
	print "	Returns:		" type "		The " type "-value of the variant."

	if( var_type ~ /string/ )
		print "					Memory must be freed by caller!"

	print ""
	print "	~~~ CHANGES & NOTES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "	Date:		Author:			Note:"
	print "----------------------------------------------------------------------------- */"

	print type " pvar_to_" var_type "( pvar* var )"
	print "{"
	print "	PROC( \"pvar_to_" var_type "\" );"
	print "	PARMS( \"var\", \"%p\", var );"
	print ""

	print "	switch( pvar_type( var ) )"
	print "	{"

	for( j = 1; j <= variants_cnt; j++ )
	{
		if( okdone[ variants[j] ] )
			continue

		okdone[ variants[j] ] = 1
		print "		case " members[ variants[j] SUBSEP "var_define" ] ":"

		if( variants[j] == type )
		{
			print "			RETURN( var->val." member " );"
			continue
		}

		if( ( conv = convert[ type SUBSEP variants[j] ] ) == "" )
			conv = "var->val." members[ variants[j] SUBSEP "member" ]

		print "			RETURN( (" type ")" conv " );"
	}

	print "	}\n"
	print "	MSG( \"Can't convert this type!\" );"
	print "	RETURN( (" type ")" var_emptyval " );"
	print "}\n"
}

function allconv_func()
{
	for( ok in okdone )
		okdone[ ok ] = 0

	print "/* -FUNCTION--------------------------------------------------------------------"
	print "	Function:		pvar_convert()"
	print ""
	print "	Author:			Jan Max Meyer (generated with var.gen.awk)"
	print ""
	print "	Usage:			Converts a pvar-structure to any supported type."
	print ""
	print "	Parameters:		pvar*	var		Pointer to pvar structure."
	print "					pbyte	type	Type to which var should be converted to."
	print ""
	print "	Returns:		pint	ERR_OK	on success, else an ERR_-define."
	print ""
	print "	~~~ CHANGES & NOTES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "	Date:		Author:			Note:"
	print "----------------------------------------------------------------------------- */"

	print "pint pvar_convert( pvar* var, pbyte type )"
	print "{"
	print "	PROC( \"pvar_convert\" );"
	print "	PARMS( \"var\", \"%p\", var );"
	print "	PARMS( \"type\", \"%d\", type );"
	print ""
	print "	if( pvar_type( var ) == type )"
	print "		RETURN( ERR_OK );"
	print ""
	print "	switch( type )"
	print "	{"

	for( i = 1; i <= variants_cnt; i++ )
	{
		if( okdone[ variants[i] ] )
			continue

		okdone[ variants[i] ] = 1

		print "		case " members[ variants[i] SUBSEP "var_define" ] ":"
		print "			pvar_set_" members[ variants[i] SUBSEP "var_type" ] \
			  "( var, pvar_to_" members[ variants[i] SUBSEP "var_type" ] \
			  "( var ) );"
		print "			RETURN( ERR_OK );\n"
	}

	print "	}"
	print ""
	print "	RETURN( ERR_FAILURE );"
	print "}\n"
}

function ltrim( s )
{
	sub( /^[ \t\n]+/, "", s )
	return s
}

function rtrim( s )
{
	sub( /[ \t\n]+$/, "", s )
	return s
}

function trim( s )
{
	return rtrim( ltrim( s ) )
}

error: pathspec 'src/pbasis/var.gen.awk' did not match any file(s) known to git


file sep

file sep

#! /bin/awk -f

#find mean and variance of a column from a file
BEGIN {
OFS=","
}
#body
{
$1=$1 #rebuild the record using current OFS
print $0
}
END {
print "Done"
}

error: pathspec 'src/awk/fs.awk' did not match any file(s) known to git


* awk filter for logs

* awk filter for logs

#!/usr/bin/gawk

BEGIN {

	m_scope_print = 1;

	m_bg[0] = "\033[0;40m"; # blck
	m_bg[1] = "\033[0;41m"; # r
	m_bg[2] = "\033[0;42m"; # g
	m_bg[3] = "\033[0;43m"; # y
	m_bg[4] = "\033[0;44m"; # b
	m_bg[5] = "\033[0;45m"; # m
	m_bg[6] = "\033[0;46m"; # c
	m_bg[7] = "\033[0;47m"; # w
	m_bg_def = "\033[0;49m"; # default

    m_fg[0] = "\033[0;30m"; # blck
    m_fg[1] = "\033[0;31m"; # r
    m_fg[2] = "\033[0;32m"; # g
    m_fg[3] = "\033[0;33m"; # y
    m_fg[4] = "\033[0;34m"; # b
    m_fg[5] = "\033[0;35m"; # m
    m_fg[6] = "\033[0;36m"; # c
    m_fg[7] = "\033[0;37m"; # wht
    m_fg_def = "\033[0;39m"; # default

    m_colormap[0] = m_fg_def
    m_colormap[1] = m_fg[2]
    m_colormap[2] = m_fg[3]
    m_colormap[3] = m_fg[5]
    m_colormap[4] = m_fg[6]
    m_colormap[5] = m_fg[4]
    m_colormap[6] = m_fg[7]
    m_colormap[7] = m_bg[4]m_fg[3]
    m_colormap[8] = m_bg[4]m_fg[7]
    maxcolors = 9;

    m_threadid[0] = 0;
    m_last_thread = 0;

    m_scope_level = 0;
    m_indenttab[0]=0;

    FS="|";
	printf("%s%slog_filter.awk%s\n",m_bg_def, m_colormap[1], m_colormap[0]);
}

function indentText(thread_index, tmp)
{
    tmp = "";
    for(i = 0; i < m_indenttab[thread_index]; ++i)
    {
        tmp = tmp" ";
    }
    return tmp;
}

function incrIndent(thread_index)
{
    return ++m_indenttab[thread_index];
}

function decrIndent(thread_index)
{
    return --m_indenttab[thread_index];
}

function replaceText(text, tmp)
{
    tmp = "";
    gsub("ue::sys::threads::", "UE::", text)
    gsub("ue::ai::pathfind", "pf", text)
    gsub("virtual ", "V ", text)
	t = m_bg[2]"_"m_bg_def
    gsub("error ", t, text)
    gsub("###", "\033[0;1m###\033[0;22m", text)
    gsub("ERR", "\033[0;9m!!!\033[0;29m", text)
    return text;
}

{
# |date|pid|tid|file:line|fn|{pfn
#   1    2   3      4     5   6

	if (match($3, "[[:digit:]]+"))
	{
		found = -1;
		for (i=0; i<m_last_thread; ++i)
		{
			#printf("cmp%d [[ %s with %s ]]\n", i, $3, m_threadid[i]);
			if (m_threadid[i] == $3)
				found = i;
		}

		if (found == -1)
		{
			found = m_last_thread;
			m_threadid[m_last_thread] = $3;
			m_indenttab[m_last_thread] = 0;
			++m_last_thread;
		}

		#printf("colormap = %d for threadid=%d \n", m_colormap[found], m_threadid[found]);
		scope_detected = 0

		if (index($6,"}"))
		{
			decrIndent(found);
			scope_detected = 1;
		}
		if (index($6,"{"))
		{
			scope_detected = 1;
		}

		strout = "";

		fill = indentText(found);

		msg = replaceText($6);

		date = $1;
		gsub("[0-9]*-[a-zA-Z]*-[0-9]* ", "", date)

		if (scope_detected)
		{
			if (m_scope_print)
			{
				printf("%s%s|%02i|%s%s\n", m_colormap[found], date, found, fill, msg);
			}
		}
		else
			printf("%s%s|%02i|%s%s\n", m_colormap[found], date, found, fill, msg);

		if (index($6,"{"))
			incrIndent(found);
	}
	else
	{
		printf("* %s%s\n", m_colormap[0], $0);
	}

	fflush();
}

END {
	for (i=0; i<m_last_thread; ++i)
	{
		printf("thread map: %i -> %d (%x)\n", i, m_threadid[i], m_threadid[i]);
	}
}

error: pathspec 'tools/log_bbTT.awk' did not match any file(s) known to git


Added an auxiliary utility to ncheck 16 consecutive inumbers with debugfs.

# 2 january 2015
# usage: awk -f icheck16.awk diskimage
# enter one inumber at a time
# TODO the usual
BEGIN {
	diskimage = ARGV[1]
	ARGC--
}
{
	for (i = 0; i < 16; i++) {
		n = $1 + i
		c = sprintf("debugfs -R 'ncheck %d' '%s' 2>/dev/null", n, diskimage)
		system(c)
	}
}

error: pathspec 'ncheck16.awk' did not match any file(s) known to git


rollback

rollback

#!/usr/bin/awk -f
##########
#The MIT License (MIT)
#
# Copyright (c) 2015 Aiden Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
##########
# Takes a SAM file, looks for interesting abnormal chimeric reads.
# only those reads mapped to chromosomes 1-24 with MAPQ >= 10, and
# with at least a triple where the minimum distance >= 20Kb
# Juicer version 1.5
function abs(v) {
    return v<0?-v:v;
}
BEGIN{
  OFS="\t";
  mapq_thresh=-1;
  tottot = -1; # will count first non-group
}
{
  # input file is sorted by read name.  Look at read name to group
  # appropriately
  split($1,a,"/");
  if(a[1]==prev){
    # move on to next record.  look below this block for actions that occur
    # regardless.
    count++;
  }
  else {
    # deal with read pair group
    tottot++;
    printme = 1;
    j = 0;
    for (i in c) {
      split(c[i], tmp);
      mapped[j] = tmp[3] ~ /^[1-9,X,Y,M][0-9,T]?$/ || tmp[3] ~ /^chr[1-9,X,Y,M][0-9,T]?$/;
      name[j] = tmp[1];
      str[j] = tmp[2];
      chr[j] = tmp[3];
      pos[j] = tmp[4];
      m[j] = tmp[5];
      cig[j] = tmp[6];
      seq[j] = tmp[10];
      j++;
    }
    len = j;
    bigdist = 0;
    for (j=0; j<len; j++) {
        printme = printme && mapped[j] && (m[j] >= 10);
        for (k=j+1; k<len; k++) {
            if (abs(pos[j]-pos[k]) >= 20000) {
                bigdist++;
            }
        }
    }
#    printme = printme && (bigdist >= 3);
    if (printme) {
      for (j = 0; j < len; j++) {
          printf("%s %d %s %d %d %s %s ", name[j], str[j], chr[j], pos[j], m[j], cig[j], seq[j]);
      }
      printf("\n");
    }

    # reset variables
    delete c;
    count=1;
    prev=a[1];
  }
  # these happen no matter what, after the above processing
  c[count] = $0;
}
END {
    printme = 1;
    j = 0;
    for (i in c) {
      split(c[i], tmp);
      mapped[j] = tmp[3] ~ /^[1-9,X,Y,M][0-9,T]?$/ || tmp[3] ~ /^chr[1-9,X,Y,M][0-9,T]?$/;
      name[j] = tmp[1];
      str[j] = tmp[2];
      chr[j] = tmp[3];
      pos[j] = tmp[4];
      m[j] = tmp[5];
      cig[j] = tmp[6];
      seq[j] = tmp[10];
      j++;
    }
    len = j;
    bigdist = 0;
    for (j=0; j<len; j++) {
        printme = printme && mapped[j] && (m[j] >= 10);
        for (k=j+1; k<len; k++) {
            if (abs(pos[j]-pos[k]) >= 20000) {
                bigdist++;
            }
        }
    }
#    printme = printme && (bigdist >= 3);
    if (printme) {
      for (j = 0; j < len; j++) {
          printf("%s %d %s %d %d %s %s ", name[j], str[j], chr[j], pos[j], m[j], cig[j], seq[j]);
      }
      printf("\n");
    }
}

error: pathspec 'CPU/common/collisions.awk' did not match any file(s) known to git


if (s_length(pa[ch]) != 3) {

if (s_length(pa[ch]) != 3) {

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/svnpenn/sage.git/'


Add AWK script to generate a frequency table from a column of numbers

#!/usr/bin/env awk -f
#
# Generates a frequency table.
#
# Usage: frequency_table
#
# Input:
#   - a column of numbers
#
# Output:
#   - frequency table

{
	total += 1
	table[$1] += 1
}
END {
	for (v in table) {
		count = table[v]
		print v OFS count OFS count/total
	}
}

error: pathspec 'tools/awk/frequency_table.awk' did not match any file(s) known to git


Added script to filter out long stack traces.

#!/usr/bin/awk -f

BEGIN {
    PRINT=0
    BUFF=""
    LINES=0
}
/"[^"]*".*prio=[^ ]* tid=[^ ]* .*/ {
    if (LINES>30) {
	print BUFF
	PRINT=0
    }
    LINES=0
    BUFF=""
}
{
    if (LINES<2000) {
	BUFF=BUFF "\n" $0
	LINES=LINES+1
    }
}

END {
    if (PRINT) {
        print BUFF
    }
}

error: pathspec 'src/awk/longThreads.awk' did not match any file(s) known to git


Add Tobias Waldekranz's awesome little AWK script

Signed-off-by: Joachim Nilsson <583c295fd7602c168ad814279bbc3894ba65f5d6@gmail.com>

error: pathspec 'ipcalc.awk' did not match any file(s) known to git


A script that generates language model data from frame data, suitable
for gnuplot

git-svn-id: a8b04003a33e1d3e001b9d20391fa392a9f62d91@2457 94700074-3cef-4d97-a70e-9c8c206c02f5

# An awk/gawk script that extracts the language probabilities from an
# S4 results output file.  Output is 3 columns:
# 1 - frame number
# 2 - total lang probability applied
# 3 - this frames lang prob
# 4 - this frames insertion prob
$1 >= 1 && $1 < 100000 { sum += $4;  sum += $5; }
$1 >= 1 && $1 < 100000 { print $1, sum, $4, $5 }

error: pathspec 'scripts/extractLanguageProbs.awk' did not match any file(s) known to git


Create trace.awk

Create trace.awk

#提取trace文件中想要的信息
cat usr_1.csv | awk -F"," '{if($4=="Write") print "W",$5,$6;else print "R",$5,$6}'> ~/usr_1_ssd.csv

error: pathspec 'tools/trace.awk' did not match any file(s) known to git


Added sjCollapseSamples.awk script

Added sjCollapseSamples.awk script

# usage:
# awk -f sjCollapseSamples.awk /path/to/all/*/SJ.out.tab | sort -k1,1V -k2,2n -k3,3n > SJ.all
# output columns:
# 1-6 - same as in SJ.out.tab
# 7   - total number of unique mappers
# 8   - total number of multi-mappers
# 9   - max overhang
# 10  - number of samples the junction was detected in

BEGIN {
    OFS="\t";
}

{
    sj=$1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6;
    nSamples[sj]++;
    nU[sj]+=$7;
    nM[sj]+=$8;
    if (nO[sj]<$9) nO[sj]=$9;
};

END {
    for (sj in nSamples) print sj,nU[sj],nM[sj],nO[sj],nSamples[sj];
}

error: pathspec 'extras/scripts/sjCollapseSamples.awk' did not match any file(s) known to git


Fix Awk code avoiding division by zero error

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/vilugao/fibonacci.git/'


Initial revision

Initial revision

BEGIN		{
		FS = "|";
		seenproc = 0;
		CC="${CMD}"
		}
/^%/		{}
/^$/		{}
/^[a-z]/ && $3 !~ /.*NOTIMPLEMENTED.*/ {
		if(seenproc) {
			print "}"
			print "--EOF--"
			printf "if %s C_%s.c\n",CC,nam
			printf "then :\nelse exit 1\nfi\n"
			printf "rm -f C_%s.c\n",nam
		}
		seenproc = 1
		$1 = substr($1,1,index($1,"\t")-1);
		nam = $1
		printf "cat > C_%s.c << '--EOF--'\n",$1
		print "#include \"nopt.h\""
		printf "C_%s(",$1
		nparms = split($2,parms,":");
		for(p=1;p<nparms;p++) {
			if(p!=1) {
				printf ","
			}
			split(parms[p+1],a," ")
			printf a[1]
		}
		printf ")\n"
		if(nparms) {
			printf "\t%s",parms[1]
		}
		for(p=1;p<nparms;p++) {
			split(parms[p+1],a," ")
			printf " %s;\n",a[1]
			if(a[2]) {
				printf "\t%s%s%s",a[2],a[3],a[4]
			}
		}
		if($3) {
			printf "{\n\t%s\n",$3
		}
		else {
			printf "{\n"
		}
		}
/^	/	{
		print
		}
END		{
		if(seenproc) {
			print "}"
			print "--EOF--"
			printf "if %s C_%s.c\n",CC,nam
			printf "then :\nelse exit 1\nfi\n"
			printf "rm -f C_%s.c\n",nam
		}
		}

error: pathspec 'modules/src/em_opt/makefuns.awk' did not match any file(s) known to git


Add an awk script used in one-vs-the-rest classification example.

BEGIN{ FS="\t"; OFS="\t"; }
{
 	possible_labels=$1;
 	rowid=$2;
    label=$3;
    features=$4;

 	label_count = split(possible_labels, label_array, ",");

	for(i = 1; i <= label_count; i++) {
    	if (label_array[i] == label)
      		print rowid, label, 1, features;
      	else
      		print rowid, label_array[i], -1, features;
	}
}
END{}

error: pathspec 'scripts/misc/one-vs-rest.awk' did not match any file(s) known to git


2008-03-24  Martin Sebor  <sebor@roguewave.com>

	* bin/xcomp.awk (print_section): Prepended "../" to the log file name
	since build logs are stored in the parent directory of the one that
	contains the cross-build results.

git-svn-id: 8ce9432b95a2a271c6c279131eb1bb968a8ee7fb@640483 13f79535-47bb-0310-9956-ffa450edef68

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/pathscale/stdcxx.git/'


Added normalize.

Added normalize.

#!/usr/bin/awk -f

{
	for (i = 1; i <= NF; ++i) {
	    split ($i, fields, "=");
	    map[fields[1]] = fields[2];
	}

	user = map["user"];
	total = map["total"];

	printf ("system=%f\n", (total - user) / total);
}

error: pathspec 'examples/normalize.awk' did not match any file(s) known to git


gps.sh -> .awk

gps.sh -> .awk

#!/usr/bin/awk -f
#
# Copyright (c) 1995-2016 Cyrille Lefevre (Cyrille.Lefevre@laposte.net). All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

BEGIN {
	split("N S", nsa); split("E W", eoa)
	CONVFMT = "%04.1f"
	fmt = "%.8f"
	epsilon = .00000001
}
function round(l, n) { return +(+(l "e+" n) "e-" n) }
function l2dms(l, t,	n, d, m, s) {
	n = t[1+(l<0)]
	if (l < 0) l = -l
	d = int(l)
	l -= int(l)
	l *= 60
	m = int(l)
	l -= int(l)
	l *= 60
	s = round(l, 2)
	if (s == 60) { m++; s -= 60 }
	return d "°" m "'" s "''" n
}
function dms2l(n, d, m, s,	l, t) {
	return n * (d + (m / 60) + (s / 3600))
}
function l2l(l, a, m, n,	s, o, x, t) {
	s = o = l2dms(l, a)
	x = o ~ m ? n : -n
	gsub(/[^-.0-9]/, " ", o)
	split(o, t)
	return dms2l(x, t[1], t[2], t[3])
}
/./ {
	ns = /N/ ? 1 : -1
	eo = /[OW]/ ? -1 : 1
	gsub(/[^-.0-9]/, " ")
	if (NF != 6) {
		nss = l2dms($1, nsa)
		eos = l2dms($2, eoa)
		nsn = l2l($1, nsa, "N", 1)
		eon = l2l($2, eoa, "[OW]", -1)
		printf fmt " " fmt "\n", nsn, eon
		print nss, eos
	} else {
		nsn = dms2l(ns, $1, $2, $3)
		eon = dms2l(eo, $4, $5, $6)
		nss = l2dms(nsn, nsa)
		eos = l2dms(eon, eoa)
		printf fmt " " fmt "\n", nsn, eon
		print nss, eos
	}
}

# eof

error: pathspec 'bin/gps.awk' did not match any file(s) known to git


Added unbundle script for hub4

Added unbundle script for hub4

git-svn-id: a8b04003a33e1d3e001b9d20391fa392a9f62d91@1409 94700074-3cef-4d97-a70e-9c8c206c02f5

# This is an awk script that will take the hub4 eval99 distribution
# and generate a script that will unpack the distribution into a set
# of audio and batch files suitable for s4.
#

BEGIN {
    audioCount = 0;
    directory = "/lab/speech/sphinx4/data/hub4/eval99/supervised/";

    print "#!/bin/sh"
    print "#  This Script unbundles the hub4/eval99 (broadcast news)"
    print "#  into a set of raw audio and batch files suitable for "
    print "#  processing by sphinx."
    print "#  "
    print "#  This script is automatically generated by unbundleHub99.awk"
    print "#  "
    print "#  "
    print "#  Create the destination directory"
    print "mkdir -p " directory;
    print "rm -rf " directory "/*";

}

function displaySummary() {
    print "========================";
    print "Sphere   : "  sphereName;
    print "Audio    : " audioName;
    print "Speaker  : "  speaker;
    print "start    : "  startTime;
    print "end      : "  endTime;
    print "duration : " duration;
    print "class    : " speechClass;
    print "sox cmd  : " soxCommand;
    print "xcrpt    : " transcript;
}

function outputCommands() {
    print "";
    print "";
    print "echo Processing" audioName
    print soxCommand;
    print "echo " audioName " " transcript " >> " batchName;
    print "echo " audioName " " transcript " >> " allBatch;
}

function isGoodUtterance() {
    return speechClass != "";
}

$1 != ";;"  {
    audioCount++;
    source = $1;
    sphereName = $1 ".sph";
    speaker = $3;
    startTime = $4;
    endTime = $5
    duration = endTime - startTime;
    info = $6;

    split(info, infoArray, ",");
    speechClass = infoArray[2];
    transcript = "";
    for (i = 7; i <= NF; i++) {
        # markup is in parans, so drop that.
        if (index($i, "(")  == 0) {
            transcript = transcript " " $i;
        }
    }

    audioName = directory source "." audioCount "." speechClass ".raw";
    batchName = directory speechClass "_hub4.batch"
    allBatch = directory "all_hub4.batch"

    soxCommand =  "sox " sphereName " -r 16000 -s -w " audioName  \
        " trim " startTime " " duration

    if (isGoodUtterance()) {
        outputCommands();
    }
    # displaySummary();
}

error: pathspec 'scripts/unbundleHub4.awk' did not match any file(s) known to git


Created an AWK script to generate pipe separate keyword list

error: pathspec 'SupportDocs/keywordPipeGenerator.awk' did not match any file(s) known to git


Awk version of makeobjops.PL.
Note the invocation ordering is slightly different:
	awk -f makeobjops.awk foo.m -ch

Submitted by:	Oliver Fromme <olli@fromme.com>

#!/usr/bin/awk -f
#
# Copyright (c) 1992, 1993
#        The Regents of the University of California.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#        This product includes software developed by the University of
#        California, Berkeley and its contributors.
# 4. Neither the name of the University nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# From @(#)vnode_if.sh        8.1 (Berkeley) 6/10/93
# From @(#)makedevops.sh 1.1 1998/06/14 13:53:12 dfr Exp $
# From @(#)makedevops.sh ?.? 1998/10/05
# From src/sys/kern/makedevops.pl,v 1.12 1999/11/22 14:40:04 n_hibma Exp
# From src/sys/kern/makeobjops.pl,v 1.8 2001/11/16 02:02:42 joe Exp
#
# $FreeBSD$

#
#   Script to produce kobj front-end sugar.
#

function usage ()
{
	print "usage: makeobjops.awk <srcfile.m> [-d] [-p] [-l <nr>] [-c|-h]";
	print "where -c   produce only .c files";
	print "      -h   produce only .h files";
	print "      -p   use the path component in the source file for destination dir";
	print "      -l   set line width for output files [80]";
	print "      -d   switch on debugging";
	exit 1;
}

function warn (msg)
{
	print "makeobjops.awk:", msg > "/dev/stderr";
}

function warnsrc (msg)
{
	warn(src ":" lineno ": " msg);
}

function debug (msg)
{
	if (opt_d)
		warn(msg);
}

function die (msg)
{
	warn(msg);
	exit 1;
}

#   These are just for convenience ...
function printc(s) {if (opt_c) print s > ctmpfilename;}
function printh(s) {if (opt_h) print s > htmpfilename;}

#
#   If a line exceeds maxlength, split it into multiple
#   lines at commas.  Subsequent lines are indented by
#   the specified number of spaces.
#
#   In other words:  Lines are split by replacing ", "
#   by ",\n" plus indent spaces.
#

function format_line (line, maxlength, indent)
{
	rline = "";

	while (length(line) > maxlength) {
		#
		#   Find the rightmost ", " so that the part
		#   to the left of it is just within maxlength.
		#   If there is none, give up and leave it as-is.
		#
		if (!match(substr(line, 1, maxlength + 1), /^.*, /))
			break;
		rline = rline substr(line, 1, RLENGTH - 1) "\n";
		line = sprintf("%*s", indent, "") substr(line, RLENGTH + 1);
	}
	return rline line;
}

#
#   Join an array into a string.
#

function join (separator, array, num)
{
	_result = ""
	if (num) {
		while (num > 1)
			_result = separator array[num--] _result;
		_result = array[1] _result;
	}
	return _result;
}

#
#   Execute a system command and report if it failed.
#

function system_check (cmd)
{
	if ((rc = system(cmd)))
		warn(cmd " failed (" rc ")");
}

#
#   Handle "INTERFACE" line.
#

function handle_interface ()
{
	intname = $2;
	sub(/;$/, "", intname);
	if (intname !~ /^[a-z_][a-z0-9_]*$/) {
		debug($0);
		warnsrc("Invalid interface name '" intname "', use [a-z_][a-z0-9_]*");
		error = 1;
		return;
	}
	if (!/;[ 	]*$/)
		warnsrc("Semicolon missing at end of line, no problem");

	debug("Interface " intname);

	printh("#ifndef _" intname "_if_h_");
	printh("#define _" intname "_if_h_\n");
	printc("#include \"" intname "_if.h\"\n");
}

#
#   Handle "CODE" and "HEADER" sections.
#   Returns the code as-is.
#

function handle_code ()
{
	code = "\n";
	getline < src;
	indent = $0;
	sub(/[^	 ].*$/, "", indent);	# find the indent used
	while (!/^}/) {
		sub("^" indent, "");	# remove the indent
		code = code $0 "\n";
		getline < src;
		lineno++;;
	}
	return code;
}

#
#   Handle "METHOD" and "STATICMETHOD" sections.
#

function handle_method (static)
{
	#
	#   Get the return type and function name and delete that from
	#   the line. What is left is the possibly first function argument
	#   if it is on the same line.
	#
	if (!intname) {
		warnsrc("No interface name defined");
		error = 1;
		return;
	}
	sub(/^[^ 	]+[ 	]+/, "");
	ret = $0;
	sub(/[ 	]*\{.*$/, "", ret);
	name = ret;
	sub(/^.*[ 	]/, "", name);	# last element is name of method
	sub(/[ 	]+[^ 	]+$/, "", ret);	# return type
	debug("Method: name=" name " return type=" ret);

	sub(/^[^\{]*\{[	 ]*/, "");

	if (!name || !ret) {
		debug($0);
		warnsrc("Invalid method specification");
		error = 1;
		return;
	}

	if (name !~ /^[a-z_][a-z_0-9]*$/) {
		warnsrc("Invalid method name '" name "', use [a-z_][a-z0-9_]*");
		error = 1;
		return;
	}

	if (methods[name]) {
		warnsrc("Duplicate method name");
		error = 1;
		return;
	}
	methods[name] = name;

	line = $0;
	while (line !~ /\}/ && (getline < src) > 0) {
		line = line " " $0;
		lineno++
	}

	default = "";
	if (!match(line, /\};?/)) {
		warnsrc("Premature end of file");
		error = 1;
		return;
	}
	extra = substr(line, RSTART + RLENGTH);
	if (extra ~ /[	 ]*DEFAULT[ 	]*[a-zA-Z_][a-zA-Z_0-9]*[ 	]*;/) {
		default = extra;
		sub(/.*DEFAULT[	 ]*/, "", default);
		sub(/[; 	]+.*$/, "", default);
	}
	else if (extra && opt_d) {
		#   Warn about garbage at end of line.
		warnsrc("Ignored '" extra "'");
	}
	sub(/\};?.*$/, "", line);

	#
	#   Create a list of variables without the types prepended.
	#
	sub(/^[	 ]+/, "", line);	# remove leading ...
	sub(/[ 	]+$/, "", line);	# ... and trailing whitespace
	gsub(/[	 ]+/, " ", line);	# remove double spaces

	num_arguments = split(line, arguments, / *; */) - 1;
	delete varnames;		# list of varnames
	num_varnames = 0;
	for (i = 1; i <= num_arguments; i++) {
		if (!arguments[i])
			continue;	# skip argument if argument is empty
		num_ar = split(arguments[i], ar, /[* 	]+/);
		if (num_ar < 2) {	# only 1 word in argument?
			warnsrc("no type for '" arguments[i] "'");
			error = 1;
			return;
		}
		#   Last element is name of variable.
		varnames[++num_varnames] = ar[num_ar];
	}

	argument_list = join(", ", arguments, num_arguments);
	varname_list = join(", ", varnames, num_varnames);

	if (opt_d) {
		warn("Arguments: " argument_list);
		warn("Varnames: " varname_list);
	}

	mname = intname "_" name;	# method name
	umname = toupper(mname);	# uppercase method name

	firstvar = varnames[1];

	if (default == "")
		default = "0";

	# the method description
	printh("extern struct kobjop_desc " mname "_desc;");
	# the method typedef
	prototype = "typedef " ret " " mname "_t(";
	printh(format_line(prototype argument_list ");",
	    line_width, length(prototype)));

	# Print out the method desc
	printc("struct kobjop_desc " mname "_desc = {");
	printc("\t0, (kobjop_t) " default);
	printc("};\n");

	# Print out the method itself
	if (0) {		# haven't chosen the format yet
		printh("static __inline " ret " " umname "(" varname_list ")");
		printh("\t" join(";\n\t", arguments, num_arguments) ";");
	}
	else {
		prototype = "static __inline " ret " " umname "(";
		printh(format_line(prototype argument_list ")",
		    line_width, length(prototype)));
	}
	printh("{");
	printh("\tkobjop_t _m;");
	if (!static)
		firstvar = "((kobj_t)" firstvar ")";
	printh("\tKOBJOPLOOKUP(" firstvar "->ops," mname ");");
	retrn =  (ret != "void") ? "return " : "";
	printh("\t" retrn "((" mname "_t *) _m)(" varname_list ");");
	printh("}\n");
}

#
#   Begin of the main program.
#

BEGIN {

line_width = 80;
gerror = 0;

#
#   Process the command line.
#

num_files = 0;

for (i = 1; i < ARGC; i++) {
	if (ARGV[i] ~ /^-/) {
		#
		#   awk doesn't have getopt(), so we have to do it ourselves.
		#   This is a bit clumsy, but it works.
		#
		for (j = 2; j <= length(ARGV[i]); j++) {
			o = substr(ARGV[i], j, 1);
			if	(o == "c")	opt_c = 1;
			else if	(o == "h")	opt_h = 1;
			else if	(o == "p")	opt_p = 1;
			else if	(o == "d")	opt_d = 1;
			else if	(o == "l") {
				if (length(ARGV[i]) > j) {
					opt_l = substr(ARGV[i], j + 1);
					break;
				}
				else {
					if (++i < ARGC)
						opt_l = ARGV[i];
					else
						usage();
				}
			}
			else
				usage();
		}
	}
	else if (ARGV[i] ~ /\.m$/)
		filenames[num_files++] = ARGV[i];
	else
		usage();
}

if (!num_files || !(opt_c || opt_h))
	usage();

if (opt_p)
	debug("Will produce files in original not in current directory");

if (opt_l) {
	if (opt_l !~ /^[0-9]+$/ || opt_l < 1)
		die("Invalid line width '" opt_l "'");
	line_width = opt_l;
	debug("Line width set to " line_width);
}

for (i = 0; i < num_files; i++)
	debug("Filename: " filenames[i]);

for (file_i = 0; file_i < num_files; file_i++) {
	src = filenames[file_i];
	cfilename = hfilename = src;
	sub(/\.m$/, ".c", cfilename);
	sub(/\.m$/, ".h", hfilename);
	if (!opt_p) {
		sub(/^.*\//, "", cfilename);
		sub(/^.*\//, "", hfilename);
	}

	debug("Processing from " src " to " cfilename " / " hfilename);

	ctmpfilename = cfilename ".tmp";
	htmpfilename = hfilename ".tmp";

	common_head = \
	    "/*\n" \
	    " * This file is produced automatically.\n" \
	    " * Do not modify anything in here by hand.\n" \
	    " *\n" \
	    " * Created from source file\n" \
	    " *   " src "\n" \
	    " * with\n" \
	    " *   makeobjops.awk\n" \
	    " *\n" \
	    " * See the source file for legal information\n" \
	    " */\n";

	printc(common_head "\n" \
	    "#include <sys/param.h>\n" \
	    "#include <sys/queue.h>\n" \
	    "#include <sys/kernel.h>\n" \
	    "#include <sys/kobj.h>");

	printh(common_head);

	delete methods;		# clear list of methods
	intname = "";
	lineno = 0;
	error = 0;		# to signal clean up and gerror setting

	while (!error && (getline < src) > 0) {
		lineno++;

		#
		#   Take special notice of include directives.
		#
		if (/^#[ 	]*include[ 	]+["<][^">]+[">]/) {
			incld = $0;
			sub(/^#[ 	]*include[ 	]+/, "", incld);
			debug("Included file: " incld);
			printc("#include " incld);
		}

		sub(/#.*/, "");		# remove comments
		sub(/^[	 ]+/, "");	# remove leading ...
		sub(/[ 	]+$/, "");	# ... and trailing whitespace

		if (/^$/) {		# skip empty lines
		}
		else if (/^INTERFACE[ 	]+[^ 	;]*[ 	]*;?[ 	]*$/)
			handle_interface();
		else if (/^CODE[ 	]*{$/)
			printc(handle_code());
		else if (/^HEADER[	 ]*{$/)
			printh(handle_code());
		else if (/^METHOD/)
			handle_method(0);
		else if (/^STATICMETHOD/)
			handle_method(1);
		else {
			debug($0);
			warnsrc("Invalid line encountered");
			error = 1;
		}
	}

	#
	#   Print the final '#endif' in the header file.
	#
	printh("#endif /* _" intname "_if_h_ */");

	close (ctmpfilename);
	close (htmpfilename);

	if (error) {
		warn("Output skipped");
		system_check("rm -f " ctmpfilename " " htmpfilename);
		gerror = 1;
	}
	else {
		if (opt_c)
			system_check("mv " ctmpfilename " " cfilename);
		if (opt_h)
			system_check("mv " htmpfilename " " hfilename);
	}
}

exit gerror;

}

error: pathspec 'sys/tools/makeobjops.awk' did not match any file(s) known to git


Add script to convert old submission format to new format.

#!/usr/bin/env awk -f

{
    for (i = 1; i <= NF; i++) {
        if (i == 3) printf "-1 ";
        printf $i" "
    }
    printf "\n"
}

error: pathspec 'tools/data/submission_old_to_new.awk' did not match any file(s) known to git


add httpd.awk for http test

add httpd.awk for http test

#!/usr/bin/gawk -f
# httpd.awk
# original from http://dqn.sakusakutto.jp/2013/08/awk_http_server.html
BEGIN {
    port = "8080";
    http_service = "/inet/tcp/" port "/0/0";
    RS = ORS = "\r\n";

    for (;;) {
        if ((http_service |& getline) > 0) {
            print "HTTP/1.x 200 OK"         |& http_service;
            print "Content-type: text/html" |& http_service;
            print ""                        |& http_service;
            print "
Hello, world!
"  |& http_service;
        }
        close(http_service);
    }
}

error: pathspec 'httpd.awk' did not match any file(s) known to git


Modify script to get is all sorted out.

Modify script to get is all sorted out.

#!/usr/bin/awk -f
#tdbout -t publisher,plugin,publisher:info[tester],status,year *.tdb
#           #1        #2      #3                     #4    #5
#tdbout -t publisher,plugin,auid,publisher:info[tester],status,year *.tdb
#           #1        #2      #3   #4                     #5     #6

BEGIN {
  FS="\t"
  pn = 0
    #str = "date +%m-%d-%Y";
    #str | getline date;
    #close str;
}

{
  # add a loop to add line only if either [status is _not_ manifest] OR [journal_year (or journal end year) is gt or eq to 2010 && journal year (or journal end year) is lt or eq to the current year we are testing]
  current_year = 2016
  end_year = 0
  incontract = 0
  test_year = ""
  test_year = 2010
  if (length($6) > 3) {
    end_year = substr($6,length($6)-3,4)
  }
  #printf "%s\n", $0
  if (($5 != "manifest") || ((end_year >= test_year) && (end_year <= current_year))) {
    incontract = 1
  }

  if (incontract == 1) {
    #split up the plugin. nn=number of fields. na is the name of the array.
    nn = split($2,na,/\./)
    #save the last element into 1p2.
    lp2 = na[nn]
    p[pn] = $1        #publisher
    n[pn] = lp2       #plugin
    auid[pn] = $3     #auid
    status[pn] = $5   #status
    r[pn] = $4        #tester
    if (r[pn] == "") {
      if ((substr(p[pn],1,2) <= "Mz")) {
        r[pn] = "5"
      } else if (substr(p[pn],1,2) >= "NA") {
        r[pn] = "8"
      }
    }
    pn++
    x[$5]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "doNotProcess"
  s[11] = "doesNotExist"
  sn = 12

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "doNP"
  sc[11] = "dNoE"
  scn = 12

  #print out header
  printf "Publisher\tPlugin\tAUid\tT\tTotal\tStatus"
  #for (j = 0 ; j < scn ; j++) {
  #  if (x[s[j]] > 0) {
  #  printf "\t%s", sc[j]
  #  }
  #}
  printf "\n"

  #print out publisher, plugin, auid, tester, status
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t1", p[i], n[i], auid[i], r[i], status[i]
    #for (j = 0 ; j < sn ; j++) {
    #  if (x[s[j]] > 0){
#   #   if (status[i] == 0) {
    #  printf "\t.."
    #} else {
    #    printf "1"
    #  }
    #}
#    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tAUid\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  #printf "%s", date
  #printf "\n"
}

error: pathspec 'scripts/tdb/statuses_gln_auids.awk' did not match any file(s) known to git


Add crosscheck of EPISODES with SEASONS

Add crosscheck of EPISODES with SEASONS

# WebScraper has problems getting incomplete data.
#
# Crosscheck the info in EPISODES with the info in SEASONS. Count up episodes
# and compare with the number of episodes listed in the seasons file.

# For now these seem more likely a problem in scraping SEASONS than EPISODES
# so it could be these are false positives in generating spreadsheets.
# Problems listed in BBox_anomalies files are more likely to be real.

# INVOCATION
#    awk -f verifyBBoxInfoFrom-webscraper.awk BBox_episodeInfo-180421.123042.txt

/ movie / {
    title = $3
    numEpisodes = $5
    if (numEpisodes == 0)
        print
}

/ show / {
    numShows += 1
    showTitle[numShows] = $3
    shouldHave[numShows] = $5
    doesHave[numShows] = 0
    if (numEpisodes == 0)
        print
}

/^         / {
    epis  = NF-1
    if ($epis !~ /^[[:digit:]]*$/)
        print "==> Bad input line " NR "\n" $0
    else
        doesHave[numShows] += $epis
}

END {
    print ""
    for ( i = 1; i <= numShows; i++ ) {
        if (shouldHave[i] != doesHave[i]) {
            print "==> show  "showTitle[i] " has " doesHave[i] " instead of " \
                shouldHave[i] " episodes."
        }
    }
}

error: pathspec 'verifyBBoxInfoFrom-webscraper.awk' did not match any file(s) known to git


changelog: Added AWK script

changelog: Added AWK script

#!/usr/bin/awk -f
BEGIN {
	# Determines whether to print 'Unreleased' banner at top
	UNRELEASED_COMMITS = 1
	# Prefixes that determine whether a commit will be printed
	CHANGELOG_REGEX = "^(changelog|fix|docs|chore|feat): "
	FS="|"
	while ("git log --pretty='%D|%s|%H'" | getline) {
		IS_GIT_TAG = length($1) && match($1, /tag:/)
		if (IS_GIT_TAG) {
			UNRELEASED_COMMITS = 0
			# Cut out text up to tag
			sub(/.*tag: /, "", $1)
			# Cut out text after tag
			sub(/,.*/, "", $1)
			print $1
		} else {
			if ( UNRELEASED_COMMITS ) {
				print "Unreleased"
				UNRELEASED_COMMITS = 0
			}
			if ( match($2, CHANGELOG_REGEX) ) {
				sub(CHANGELOG_REGEX, "", $2)
				printf("\t- %s\n", $2)
			}
		}

	}
}

error: pathspec 'changelog.awk' did not match any file(s) known to git


Turn IMDB scrapes into usable data

Turn IMDB scrapes into usable data

BEGIN {
    FS = "\t"
}

{
    web_scraper_start_url = $1
    Title = $2
    Years = $3
    Cast = $4
    Directors = $5
    Writers = $6

    if (Cast == "" && Directors == "" && Writers == "")
        next

    if (NR == 1)
        next

    if (Cast != "") {
        gsub (/[[:blank:]]{3,99}\(uncredited\)/," (uncredited)",Cast)
        nflds1 = split (Cast,fld,/[[:blank:]]{3,99}/)
        type = "Actor"
        person = fld[1]
        role = fld[3]
        sub (/\302\240/,"",role)
        episodes = fld[4]
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    if (Directors != "") {
        nflds1 = split (Directors,fld,/[[:blank:]]{3,99}/)
        type = "Director"
        role = ""
        person = fld[1]
        episodes = fld[3]
        gsub (/[()]/,"",episodes)
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    if (Writers != "") {
        nflds1 = split (Writers,fld,/[[:blank:]]{3,99}/)
        type = "Writer"
        role = ""
        person = fld[1]
        episodes = fld[3]
        gsub (/[()]/,"",episodes)
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    printf ("%d - %s %s\t%s\t%s\t%s\t%s\t%s\n",nflds1,Title,Years,person,type,role,num_episodes,episode_years)

}

error: pathspec 'getFullcastFrom-imdb.awk' did not match any file(s) known to git


- added new symsets script

- added new symsets script

#!/usr/bin/awk -f
# =============================================================================
#
# @(#) $RCSfile: symsets.awk,v $ $Name:  $($Revision: 1.1.2.1 $) $Date: 2011-03-17 07:01:22 $
#
# -----------------------------------------------------------------------------
#
# Copyright (c) 2008-2011  Monavacon Limited <http://www.monavacon.com/>
# Copyright (c) 2001-2008  OpenSS7 Corporation <http://www.openss7.com/>
# Copyright (c) 1997-2001  Brian F. G. Bidulock <bidulock@openss7.org>
#
# All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation; version 3 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>, or write to
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# -----------------------------------------------------------------------------
#
# U.S. GOVERNMENT RESTRICTED RIGHTS.  If you are licensing this Software on
# behalf of the U.S. Government ("Government"), the following provisions apply
# to you.  If the Software is supplied by the Department of Defense ("DoD"), it
# is classified as "Commercial Computer Software" under paragraph 252.227-7014
# of the DoD Supplement to the Federal Acquisition Regulations ("DFARS") (or any
# successor regulations) and the Government is acquiring only the license rights
# granted herein (the license rights customarily provided to non-Government
# users).  If the Software is supplied to any unit or agency of the Government
# other than DoD, it is classified as "Restricted Computer Software" and the
# Government's rights in the Software are defined in paragraph 52.227-19 of the
# Federal Acquisition Regulations ("FAR") (or any successor regulations) or, in
# the cases of NASA, in paragraph 18.52.227-86 of the NASA Supplement to the FAR
# (or any successor regulations).
#
# -----------------------------------------------------------------------------
#
# Commercial licensing and support of this software is available from OpenSS7
# Corporation at a fee.  See http://www.openss7.com/
#
# -----------------------------------------------------------------------------
#
# Last Modified $Date: 2011-03-17 07:01:22 $ by $Author: brian $
#
# =============================================================================

function getline_command(cmd)
{
    cmd | getline; close(cmd); return $0
}
function date(format) {
    if (format) {
	return getline_command("date +\"" format "\"")
    } else {
	return getline_command("date -uIseconds")
    }
}
function year()
{
    return date("%Y")
}
function allyears(    this, last, sep, result)
{
    last = year(); sep = ""; result = ""
    for (this = 2009; this <= last; this++) {
	result = result sep this
	sep = ", "
    }
    return result
}
function print_info(string)
{
    if (values["quiet"] == 0 && values["verbose"] > 0) {
	print blu me ": I: " string std > stderr
	written[stderr] = 1
    }
}
function print_debug(string)
{
    if (values["debug"] > 0) {
	print mag me ": D: " string std > stderr
	written[stderr] = 1
    }
}
function print_error(string)
{
    print red me ": E: " string std > stderr
    written[stderr] = 1
}
function print_warn(string)
{
    if (values["quiet"] == 0 || values["verbose"] > 0 || values["debug"] > 0) {
	print org me ": W: " string std > stderr
	written[stderr] = 1
    }
}
function usage(output)
{
    if (values["quiet"])
	return
    print "\
symsets:\n\
  $Id: symsets.awk,v 1.1.2.1 2011-03-17 07:01:22 brian Exp $\n\
Usage:\n\
  symsets [options] [INPUT ...]\n\
  symsets -h\n\
  symsets -V\n\
  symsets -C\
" > output
    written[output] = 1
}
function help(output)
{
    if (values["quiet"])
	return
    usage(output)
    print "\
Arguments:\n\
  INPUT ...\n\
      Module.symvers files for which to generate symsets\n\
      [default: " defaults["inputs"] "] {" environs["inputs"] "}\n\
Options:\n\
  -s, --suse\n\
      output symsets in SuSE format\n\
  -r, --redhat\n\
      output symsets in RedHat format (default)\n\
  -k, --kversion KVERSION\n\
      kernel version KVERSION\n\
      [default: " defaults["kversion"] "] {" environs["kversion"] "}\n\
  -I, --inputs INPUTS\n\
      input Module.symvers files (space separated)\n\
      [default: " defaults["inputs"] "] {" environs["inputs"] "}\n\
  -o, --outfile OUTFILE\n\
      output file name OUTFILE\n\
      [default: " defaults["outfile"] "] {" environs["outfile"] "}\n\
  -n, --dryrun, --dry-run\n\
      don't perform the actions, just check them\n\
  -q, --quiet, --silent\n\
      suppress normal output\n\
  -D, --debug[=LEVEL]\n\
      increase or set debug level\n\
      [default: " defaults["debug"] "]\n\
  -v, --verbose[=LEVEL]\n\
      increase or set verbosity level\n\
      [default: " defaults["verbose"] "]\n\
  -h, --help\n\
      display this usage information and exit\n\
  -V, --version\n\
      display script version and exit\n\
  -C, --copying\n\
      display copying permissions and exit\n\
" > output
    written[output] = 1
}
function version(output)
{
    if (values["quiet"])
	return
    print "\
Version 2.1\n\
$Id: symsets.awk,v 1.1.2.1 2011-03-17 07:01:22 brian Exp $\n\
Copyright (c) 2008, " allyears() "  Monavacon Limited.\n\
Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008  OpenSS7 Corporation.\n\
Copyright (c) 1997, 1998, 1999, 2000, 2001  Brian F. G. Bidulock.\n\
\n\
All Rights Reserved.\n\
\n\
This is free software; see the source for copying conditions.  There is NO\n\
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
\n\
Distributed by OpenSS7 under GNU Affero General Public License Version 3,\n\
with conditions, incorporated herein by reference.\n\
\n\
See " me " -- --copying' for copying permissions.\
" > output
    written[output] = 1
}
function copying(output)
{
    if (values["quiet"])
	return
    print "\
--------------------------------------------------------------------------------\n\
$Id: symsets.awk,v 1.1.2.1 2011-03-17 07:01:22 brian Exp $\n\
--------------------------------------------------------------------------------\n\
Copyright (c) 2008, " allyears() "  Monavacon Limited.\n\
Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008  OpenSS7 Corporation.\n\
Copyright (c) 1997, 1998, 1999, 2000, 2001  Brian F. G. Bidulock.\n\
\n\
All Rights Reserved.\n\
--------------------------------------------------------------------------------\n\
This program is free software; you can  redistribute  it and/or modify  it under\n\
the terms  of the  GNU  Affero General Public  License as published  by the Free\n\
Software Foundation; version 3 of the License.\n\
\n\
This program is distributed in the hope that it will  be useful, but WITHOUT ANY\n\
WARRANTY; without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A\n\
PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details.\n\
\n\
You should have received a copy of the  GNU Affero General Public License  along\n\
with this program.   If not, see <http://www.gnu.org/licenses/>, or write to the\n\
Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.\n\
--------------------------------------------------------------------------------\n\
U.S. GOVERNMENT RESTRICTED RIGHTS.  If you are licensing this Software on behalf\n\
of the U.S. Government (\"Government\"), the following provisions apply to you. If\n\
the Software is supplied by the  Department of Defense (\"DoD\"), it is classified\n\
as \"Commercial  Computer  Software\"  under  paragraph  252.227-7014  of the  DoD\n\
Supplement  to the  Federal Acquisition Regulations  (\"DFARS\") (or any successor\n\
regulations) and the  Government  is acquiring  only the  license rights granted\n\
herein (the license rights customarily provided to non-Government users). If the\n\
Software is supplied to any unit or agency of the Government  other than DoD, it\n\
is  classified as  \"Restricted Computer Software\" and the Government's rights in\n\
the Software  are defined  in  paragraph 52.227-19  of the  Federal  Acquisition\n\
Regulations (\"FAR\")  (or any successor regulations) or, in the cases of NASA, in\n\
paragraph  18.52.227-86 of  the  NASA  Supplement  to the FAR (or any  successor\n\
regulations).\n\
--------------------------------------------------------------------------------\n\
Commercial  licensing  and  support of this  software is  available from OpenSS7\n\
Corporation at a fee.  See http://www.openss7.com/\n\
--------------------------------------------------------------------------------\
" > output
    written[output] = 1
}
function getopt_long(argc, argv, optstring, longopts, longindex,    pos, needarg, wantarg)
{
    option = ""; optval = ""; optarg = ""; pos = 0; needarg = 0; wantarg = 0
    if (optind == 0) { optind = 1; more = "" }
    while ((optind < argc) || (more != "")) {
	if (more) { arg = "-" more; more = "" }
	else { arg = argv[optind]; optind++ }
	if (arg ~ /^--?[a-zA-Z0-9]/) {
	    if (needarg) {
		print_error("option -" optval " requires an argument")
		usage(stderr)
		exit 2
	    }
	    if (wantarg) {
		more = substr(arg, 2)
		return optval
	    }
	    if (arg ~ /^--[a-zA-Z0-9][-_a-zA-Z0-9]*=.*$/) {
		option = arg; sub(/^--/, "", option); sub(/=.*$/, "", option)
		optarg = arg; sub(/^--([a-zA-Z0-9][-_a-zA-Z0-9]*)=/, "", optarg)
		if (!(option in longopts)) {
		    print_error("option --" option " not recognized")
		    usage(stderr)
		    exit 2
		}
		if (longopts[option] !~ /:/) {
		    print_error("option --" option " does not take an argument")
		    usage(stderr)
		    exit 2
		}
		optval = substr(longopts[option], 1, 1)
		return optval
	    }
	    if (arg ~ /^--[a-zA-Z0-9][-_a-zA-Z0-9]*$/) {
		option = arg; sub(/^--/, "", option)
		if (!(option in longopts)) {
		    print_error("option --" option " not recognized")
		    usage(stderr)
		    exit 2
		}
		if (longopts[option] ~ /::/) { wantarg = 1 } else
		if (longopts[option] ~ /:/ ) { needarg = 1 }
		optval = substr(longopts[option], 1, 1)
		for (option in longopts)
		    if (substr(longopts[option], 1, 1) == optval)
			break
		if (!needarg && !wantarg)
		    return optval
		continue
	    }
	    if (arg ~ /^-[a-zA-Z0-9]/) {
		optval = substr(arg, 2, 1)
		pos = index(optstring, optval)
		if (pos == 0) {
		    print_error("option -" optval " not recognized")
		    usage(stderr)
		    exit 2
		}
		if (substr(optstring, pos+1, 1) == ":") {
		    if (length(arg) > 2) {
			if (substr(optstring, pos+2, 1) == ":") {
			    more = substr(arg, 3)
			    if (more && more !~ /^[a-zA-Z0-9]/) {
				print_error("bad option sequence " arg)
				usage(stderr)
				exit 2
			    }
			} else
			    optarg = substr(arg, 3)
			for (option in longopts)
			    if (substr(longopts[option], 1, 1) == optval)
				break
			return optval
		    }
		    if (substr(optstring, pos+2, 1) == ":") { wantarg = 1 } else
		    if (substr(optstring, pos+1, 1) == ":") { needarg = 1 }
		    for (option in longopts)
			if (substr(longopts[option], 1, 1) == optval)
			    break
		    continue
		}
		if (length(arg) > 2) { more = substr(arg, 3) } else { more = "" }
		if (more && more !~ /^[a-zA-Z0-9]/) {
		    print_error("bad option sequence " arg)
		    usage(stderr)
		    exit 2
		}
		for (option in longopts)
		    if (substr(longopts[option], 1, 1) == optval)
			break
		return optval
	    }
	}
	if (arg == "--")
	    return -1
	if (needarg || wantarg) {
	    optarg = arg
	    return optval
	}
	optind--
	return -1
    }
    return -1
}
function system_command(cmd)
{
    print_debug(cmd); return system(cmd)
}
function missing(fnlp, expected) {
    print_error(fnlp " definition missing " (expected - NF) " fields")
}
function garbage(fnlp, maximum) {
    print_warn(fnlp " definition with " (NF - maximum) " garbage fields")
}
function unrecog(fnlp, line) {
    print_error(fnlp " unrecognized line: " line)
}
function read_inputs(inputs) {
    print_debug("reading inputs...")
    for (k in inputs) {
	input = inputs[k]
	print_debug("reading input " input)
	filename = input
	lineno = 0
	while ((getline < input) == 1) {
	    lineno++; fnl = filename ":" lineno; fnlp = fnl ": "
	    if (NF < 4) { missing(fnlp, 4); continue }
	    if (NF > 4) { garbage(fnlp, 4); NF = 4 }
	    if ($1~/^0x/) {
		gsub(/\//, "_")
		if ($3 in symsets) {
		    symsets[$3] = symsets[$3] " " $2
		    crcsets[$3] = crcsets[$3] $1
		} else {
		    symsets[$3] = $2
		    crcsets[$3] = $1
		}
		# Note that SuSE includes EXPORT type in the
		# kABI, whereas RedHat does not.
		if (values["suse"] == 1) {
		    syms[$2] = $1 "\t" $2 "\t" $3 "\t" $4
		} else {
		    syms[$2] = $1 "\t" $2 "\t" $3
		}
		#print $1 "\t" $2 "\t" $3
	    } else { unrecog(fnlp, $0); continue }
	}
	close(input)
    }
}
BEGIN {
    LINT = "yes"
    me = "symsets.awk"
    if (!("TERM" in ENVIRON)) ENVIRON["TERM"] = "dumb"
    if (ENVIRON["TERM"] == "dumb" || system("test -t 1 -a -t 2") != 0) {
	stdout = "/dev/stderr"; written[stdout] = 0
	stderr = "/dev/stderr"; written[stderr] = 0
	cr = ""; lf = "\n"
	blk = ""; hblk = ""
	red = ""; hred = ""
	grn = ""; hgrn = ""
	org = ""; horg = ""
	blu = ""; hblu = ""
	mag = ""; hmag = ""
	cyn = ""; hcyn = ""
	std = ""
    } else {
	stdout = "/dev/stdout"; written[stdout] = 0
	stderr = "/dev/stderr"; written[stderr] = 0
	cr = "\r"; lf = ""
	blk = "\033[0;30m"; hblk = "\033[1;30m"
	red = "\033[0;31m"; hred = "\033[1;31m"
	grn = "\033[0;32m"; hgrn = "\033[1;32m"
	org = "\033[0;33m"; horg = "\033[1;33m"
	blu = "\033[0;34m"; hblu = "\033[1;34m"
	mag = "\033[0;35m"; hmag = "\033[1;35m"
	cyn = "\033[0;36m"; hcyn = "\033[1;36m"
	std = "\033[m"
    }
    if ("kversion" in ENVIRON) { uname = ENVIRON["kversion"] }
    else                       { uname = getline_command("uname -r") }
    longopts["inputs"       ] = "I:" ; environs["inputs"        ] = "MODPOST_MODVER"  ; defaults["inputs"       ] = "Module.symvers"
    longopts["suse"         ] = "s"  ;                                                  defaults["suse"         ] = 0
    longopts["redhat"       ] = "r"  ;                                                  defaults["redhat"       ] = 1
    longopts["kversion"     ] = "k:" ; environs["kversion"      ] = "kversion"        ; defaults["kversion"     ] = uname
    longopts["outfile"      ] = "o:" ; environs["outfile"       ] = "MODPOST_SYMSETS" ; defaults["outfile"      ] = "symsets-" uname ".tar.gz"
    longopts["dryrun"       ] = "n"  ;                                                  defaults["dryrun"       ] = 0
    longopts["dry-run"      ] = "n"  ;                                                  defaults["dry-run"      ] = 0
    longopts["quiet"        ] = "q"  ;                                                  defaults["quiet"        ] = 0
    longopts["silent"       ] = "q"  ;                                                  defaults["silent"       ] = 0
    longopts["debug"        ] = "D::";                                                  defaults["debug"        ] = 0
    longopts["verbose"      ] = "v::";                                                  defaults["verbose"      ] = 0
    longopts["help"         ] = "h"  ;
    longopts["version"      ] = "V"  ;
    longopts["copying"      ] = "C"  ;
    # set mandatory defaults
    values["suse"         ] = defaults["suse"    ]
    values["redhat"       ] = defaults["redhat"  ]
    values["kversion"     ] = defaults["kversion"]
    values["inputs"       ] = defaults["inputs"  ]
    values["outfile"      ] = defaults["outfile" ]
    values["debug"        ] = defaults["debug"   ]
    values["verbose"      ] = defaults["verbose" ]
    values["quiet"        ] = defaults["quiet"   ]
    if ("V" in ENVIRON && ENVIRON["V"] == 1) {
	values["quiet"  ] = 0
	values["verbose"] = 2
    }
    optstring = "I:srk:o:nqD::v::hVC"
    optind = 0
    while (1) {
	c = getopt_long(ARGC, ARGV, optstring, longopts)
	if (c == -1) break
	else if (c ~ /[Iko]/)  { values[option] = optarg }
	else if (c ~ /[srnq]/) { values[option] = 1 }
	else if (c == "D")     { if (optarg) { values["debug"  ] = optarg } else { values["debug"  ]++ } }
	else if (c == "v")     { if (optarg) { values["verbose"] = optarg } else { values["verbose"]++ } }
	else if (c == "h")     { help(   stdout); exit 0 }
	else if (c == "V")     { version(stdout); exit 0 }
	else if (c == "C")     { copying(stdout); exit 0 }
	else                   { usage(  stderr); exit 2 }
    }
    if (optind < ARGC) {
	values["inputs"] = ARGV[optind]; optind++
	while (optind < ARGC) {
	    values["inputs"] = values["inputs"] " " ARGV[optind]
	    optind++
	}
    }
    for (i=1;ARGC>i;i++) { delete ARGV[i] }
    for (value in values) {
	if (!values[value] && (value in environs) && (environs[value] in ENVIRON) && ENVIRON[environs[value]]) {
	    print_debug("assigning value for " value " from environment " environs[value])
	    values[value] = ENVIRON[environs[value]]
	}
	if (!values[value] && (value in defaults) && defaults[value]) {
	    print_debug("assigning value for " value " from default " defaults[value])
	    values[value] = defaults[value]
	}
    }
    if (values["suse"] == 1) { values["redhat"] = 0 }
    if (!values["inputs"] || values["inputs"] == "-") {
	print_debug("no inputs, using " stdin)
	values["inputs"] = stdin
    }
    split(values["inputs"], inputs)
    read_inputs(inputs)
    directory = values["outfile"]
    sub(/\.tar\.gz/, "", directory)
    tarball = directory ".tar.gz"
    dirname  = getline_command("dirname "  directory)
    basename = getline_command("basename " directory)
    if (dirname == ".") { chngdir = "" } else { chngdir = " -C " dirname }
    system_command("mkdir -p " directory)
    if (system("test -f " tarball) == 0) {
	print_debug("unpacking tarball " tarball)
	system_command("tar -xzf " tarball chngdir)
    }

    print_debug("processing symbols")
    for (symset in symsets) {
	print_debug("processing symbol set " symset)
	if (values["redhat"] == 1) {
	    # note that RedHat does not protect the symbol name,
	    # the location of the symbol within the kernel, nor
	    # the export type in the kABI, whereas SuSE does.
	    $0 = getline_command("echo '" crcsets[symset] "' | sha1sum")
	    hash = $1
	    print_debug("hash is: '" hash "'")
	} else {
	    hash = "tmp"
	}
	print_debug("removing " directory "/" symset ".*")
	system_command("rm -f -- " directory "/" symset ".*")
	file = directory "/" symset "." hash
	print_debug("creating " file)
	n = split(symsets[symset],symbols)
	n = asort(symbols)
	for (i=1;i<=n;i++) {
	    sym = symbols[i]
	    print syms[sym] > file
	}
	close(file)
	if (hash == "tmp") {
	    hash = getline_command("md5sum " file)
	    hash = substr(hash, 1, 16)
	    print_debug("hash is: '" hash "'")
	    print_debug("moving " file " to " directory "/" symset "." hash)
	    system_command("mv -f -- " file " " directory "/" symset "." hash)
	}
    }
    print_debug("done processing symbols")
    print_debug("creating " tarball)
    system_command("tar -czf " tarball chngdir " " basename)
    print_debug("removing working directory " directory)
    system_command("rm -fr -- " directory)
    exit 0
}
END {
    if (written[stdout]) {
	close(stdout)
	written[stdout] = 0
    }
    if (written[stderr]) {
	close(stderr)
	written[stderr] = 0
    }
}

# =============================================================================
#
# $Log: symsets.awk,v $
# Revision 1.1.2.1  2011-03-17 07:01:22  brian
# - added new symsets script
#
#
# =============================================================================
# vim: ft=awk sw=4 nocin nosi fo+=tcqlorn

error: pathspec 'scripts/symsets.awk' did not match any file(s) known to git


Create parseOui.awk

Create parseOui.awk

#!/usr/bin/awk -f

#####
##
##      DATE: 2015-04-11
##    AUTHOR: Emile Mercier
##   PURPOSE: Parse oui.txt into a associative array
##            with the key being the entire manufacturer string.
##            for use during compiling randomac.c
##
##     NOTE: after running this on oui.txt, it becomes obvious that the data is
##           not normalized.
##
##           For example, look at this output:
##              2wire>--001FB3 00217C 0022A4 002351 002456 00253C 002650 28162E 34EF44 383BC8 3CEA4F
##              2wire inc>------60C397 94C150
##              2wire inc.>-----001D5A
##              2wire, inc>-----000D72 001288 00183F 0019E4 001AC4
##              2wire, inc.>----001495 001B5B 001EC7 00D09E
##
##           sloppy :(
##
##
##    USAGE: parseOui.awk oui.txt

# take action only the lines matching "base 16" in oui.txt
/base 16/ {
    count+=1;
    mfgString=""

    # store the base 16 mac prefix in an array named MacPrefix
    MacPrefix[count] = sprintf("%s", $1);

    # store the entire manufacturer name in the variable mfgString
    # I did this in awk but sed or cut in shell is what i'd rather do.
    # "from here to end of line" in awk is pretty ugly
    for (i=4; i<=NF; ++i) {
        # if condition is so that mfgString doesn't have a preceeding space
        if ( i == 4)
            mfgString = sprintf("%s",$i);
        else
            mfgString = sprintf("%s %s", mfgString, sprintf("%s",$i) );
    }

    # Store the manufacturer string "mfgString" in a associative array "Manuf"
    # as the "primary key"
    # each mac prefix associated with the literal manufacturer string
    # is then stored, space delimited, in it.
    # The if condition is so that first mac prefix entry isn't preceeded by a space
    if ( Manuf[ tolower(mfgString) ] == "" )
        Manuf[ tolower(mfgString) ] = sprintf("%s", MacPrefix[count]);
    else
        Manuf[ tolower(mfgString) ] = sprintf("%s %s",Manuf[ tolower(mfgString) ],MacPrefix[count]);
}

END{
   for (i in Manuf)
       printf( "%s\t%s\n", i, Manuf[i] );
}

error: pathspec 'parseOui.awk' did not match any file(s) known to git


perf.awk: logs performance meter

perf.awk: logs performance meter

error: pathspec 'awk/perf.awk' did not match any file(s) known to git


2008-03-14  Martin Sebor  <sebor@roguewave.com>

	* bin/xcomp.awk (get_gzlogfname): Replaced .txt suffix with
	the hopefully now correct .gz.txt.

git-svn-id: 8ce9432b95a2a271c6c279131eb1bb968a8ee7fb@637303 13f79535-47bb-0310-9956-ffa450edef68


Add parser for Z3's get-model output.

Add parser for Z3's get-model output.

Doesn't really parse it, it just reformats it into something less
rubbish.

error: pathspec 'z3_parse.awk' did not match any file(s) known to git


2008-04-15  Martin Sebor  <sebor@roguewave.com>

	* bin/xcomp.awk: Added comments describing how platform pattern
	matching works for the expected failures feature.
	Prepended 'X' to the signal name when the signal is expected
	to match the legend at the bottom of the page.

git-svn-id: 8ce9432b95a2a271c6c279131eb1bb968a8ee7fb@648374 13f79535-47bb-0310-9956-ffa450edef68


add trim.awk

add trim.awk

#!/usr/bin/awk -f

#+block usage

# ### `trim.awk`: trim blank lines
# ```
# usage: trim.awk FILENAME
# ```

# Trim blank lines in FILENAME

#+endblock

BEGIN {
    n = 0
}

{
    lines[n++] = $0
}

END {
    for (start=0; start<n && lines[start]~/^\s*$/; start++);
    for (end=n-1; end>=0 && lines[end]~/^\s*$/; end--);
    for (i=start; i<=end; i++) {
        print lines[i];
    }
}

error: pathspec 'lips/trim.awk' did not match any file(s) known to git


Added 99 Bottles Problem in AWK

Added 99 Bottles Problem in AWK

func bottle(){
  bottles = 99;
  while(bottles > 0){
    printf("%d bottles of beer on the wall, %d bottles of beer.\n",bottles,bottles);
    bottles--;
    print("Take one down and pass it around,",(bottles > 0) ?bottles:"no more" ,"bottles of beer on the wall.");
  }
  print("No more bottles of beer on the wall, no more bottles of beer.");
  print("Go to the store and buy some more, 99 bottles of beer on the wall.");
}

BEGIN {
    // run the program
	bottle();
}
END {

}

error: pathspec '99_Bottles_Problem/AWK/wasi0013/99Bottles.awk' did not match any file(s) known to git


reverted experimental mawk

reverted experimental mawk

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/orbifx/yamado.git/'


display more conventional date format in html pages


gnumake2: added missing processdelivered.awk script

#*************************************************************************
#
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# Copyright 2000, 2010 Oracle and/or its affiliates.
#
# OpenOffice.org - a multi-platform office productivity suite
#
# This file is part of OpenOffice.org.
#
# OpenOffice.org is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenOffice.org is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenOffice.org.  If not, see
# <http://www.openoffice.org/license.html>
# for a copy of the LGPLv3 License.
#
#*************************************************************************

BEGIN {
    RS=" "
    FS=":"
}

NF == 2 {
    print "COPY " $1 " " $2;
}

error: pathspec 'solenv/gbuild/processdelivered.awk' did not match any file(s) known to git


adding awk script to read winners from stdout.txt

error: pathspec 'Tensile/Source/winners.awk' did not match any file(s) known to git


Calculate durations

Calculate durations

# Total the duration of episodes in every series
# Update the number of episodes and the total show duration for any that are missing
# NOTE: Depends on file being sorted in reverse order

# INVOCATION:
#       grep -hv ^Sortkey $PROGRAMS_SPREADSHEET_FILE $EPISODES_SPREADSHEET_FILE | sort -f |
#           tail -r | awk -v ERROR_FILE=$ERROR_FILE -v DURATION_FILE=$DURATION_FILE \
#           -f calculateBBoxDurations.awk | tail -r >>$LONG_SPREADSHEET_FILE

# Field numbers
#    1 Sortkey    2 Title    3 Seasons    4 Episodes    5 Duration    6 Year(s)    7 Rating

BEGIN {
    FS = "\t"
    OFS = "\t"
}

# No processing on header and other lines unrelated to shows
$1 == "" || $1 == "Sortkey" {
    print
    next
}

# Accumulate total time on any line that has a valid duration
$5 != "" {
    # Check all durations for strict HH:MM:SS format
    if ($5 !~ /^[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}$/) {
        print "==> Bad duration " $5 " in " $0 >> ERROR_FILE
        print
        next
    } else {
        split ($5, tm, ":")
        totalTime[3] += tm[3]
        totalTime[2] += tm[2] + int(totalTime[3] / 60)
        totalTime[1] += tm[1] + int(totalTime[2] / 60)
        totalTime[3] %= 60; totalTime[2] %= 60
    }
}

# (2) indicates an episode, which should always have a valid duration
# Accumulate series time and episode count on any line that has a valid duration
$1 ~ / \(2\) / {
        secs += tm[3]
        mins += tm[2] + int(secs / 60)
        hrs += tm[1] + int(mins / 60)
        secs %= 60; mins %= 60
        episodes += 1
        print
}

# (1) indicates a show, which may or may not have a duration
# If it has no duration, update it from the running total
$1 ~ / \(1\) / {
    if ($5 == "") {
        $4 = episodes
        $5 = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
        secs = 0; mins = 0; hrs = 0;
        episodes = 0
    }
    print
}

END {
    printf ("%02d:%02d:%02d\n", totalTime[1], totalTime[2], totalTime[3]) >> DURATION_FILE
}

error: pathspec 'calculateBBoxDurations.awk' did not match any file(s) known to git


Little awk test

Little awk test

{
	if ($0 ~ /var pontuacao/ )
		print;
}

error: pathspec 'test2.awk' did not match any file(s) known to git


Add tool to populate options for configure

BEGIN { OFS = "=" }
$1 == "--bindir" { print $0,BINDIR }
$1 == "--datadir" { print $0,DATADIR }
$1 == "--datarootdir" { print $0,DATAROOTDIR }
$1 == "--docdir" { print $0,DOCDIR }
$1 == "--dvidir" { print $0,DVIDIR }
$1 == "--dynlibdir" { print $0,DYNLIBDIR }
$1 == "--htmldir" { print $0,HTMLDIR }
$1 == "--includedir" { print $0,INCLUDEDIR }
$1 == "--infodir" { print $0,INFODIR }
$1 == "--libdir" { print $0,LIBDIR }
$1 == "--libexecdir" { print $0,LIBEXECDIR }
$1 == "--localedir" { print $0,LOCALEDIR }
$1 == "--localstatedir" { print $0,LOCALSTATEDIR }
$1 == "--mandir" { print $0,MANDIR }
$1 == "--oldincludedir" { print $0,OLDINCLUDEDIR }
$1 == "--pdfdir" { print $0,PDFDIR }
$1 == "--psdir" { print $0,PSDIR }
$1 == "--sbindir" { print $0,SBINDIR }
$1 == "--sharedstatedir" { print $0,SHAREDSTATEDIR }
$1 == "--srcdir" { print $0,SRCDIR }
$1 == "--sysconfdir" { print $0,SYSCONFDIR }
$1 == "--sysdepdir" { print $0,SYSDEPDIR }

error: pathspec 'tool/configure.awk' did not match any file(s) known to git


END has to be last with Sun's awk

END has to be last with Sun's awk

git-svn-id: 193148810cf65385f3a195dbc9a8b504846da3a2@9772 ec53bebd-3082-4978-b11e-865c3cabbd6b

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/madscientist159/heimdal.git/'


MONDRIAN-LAGUNITAS: checkFile: Allow long string literal on line starting with '?' or ':'.

[git-p4: depot-paths = "//open/mondrian-release/lagunitas/": change = 14917]

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/citycloud-bigdata/mondrian.git/'


add correct_energy.awk

add correct_energy.awk

error: pathspec 'tool.hgas/correct_energy.awk' did not match any file(s) known to git


chinese block

chinese block

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/idevz/life.git/'


add tool.imbh/print_dumpparticle.awk

add tool.imbh/print_dumpparticle.awk

{
    printf("            dumpOneParticle(%10d, sph);\n", $1);
}

error: pathspec 'tool.imbh/print_dumpparticle.awk' did not match any file(s) known to git


make-barcodes.awk: New script for code creating barcodes from sequences.

BEGIN {
  FS = "\t"; OFS = "\t";
  if (! BAR_LEN) {
    BAR_LEN = 24;
  }
  if (! INVARIANT) {
    INVARIANT = 5;
  }
  HALF = int(BAR_LEN/2);
  # Fill an array mapping each character to its ASCII value.
  for (i = 0; i < 128; i++) {
    char = sprintf("%c", i);
    ORD[char] = i;
  }
}
function get_sum(seq) {
  sum = 0;
  for (i = 1; i <= length(seq); i++) {
    sum += ORD[substr(seq, i, 1)];
  }
  return sum;
}
{
  alpha = substr($2, 1, HALF);
  beta = substr($6, 1, HALF);
  alpha_sum = get_sum(alpha);
  beta_sum = get_sum(beta);
  if (alpha_sum < beta_sum) {
    barcode = alpha beta;
    order = "ab";
  } else {
    barcode = beta alpha;
    order = "ba";
  }
  read1 = substr($2, HALF + INVARIANT + 1);
  read2 = substr($6, HALF + INVARIANT + 1);
  # $1 = read1 name | $4 = read1 quality | $5 = read2 name | $8 = read2 quality
  print barcode, order, $1, read1, $4, $5, read2, $8;
}

error: pathspec 'make-barcodes.awk' did not match any file(s) known to git


Add awk solution to problem 16

Add awk solution to problem 16

#!/usr/bin/awk -f

{
    protein = ""
    flag = 0    # Get rid of line with label
    while (("curl -Ls http://www.uniprot.org/uniprot/"$1".fasta" | getline prot_line) > 0) {
        if (flag)
            # Append line to protein
            protein = protein prot_line
        else
            # We're on the first line - ignore label and set flag
            flag = 1
    }

    first = 1    # Print label only once
    startidx = 1
    do {
        where = match(substr(protein, startidx), /N[^P][ST][^P]/)
        if (where != 0) {
            if (first) {
                # Print label, start location string
                print $0
                loc_string = where + startidx - 1
                # Reset flag
                first = 0
            }
            else {
                # Append to location string
                loc_string = loc_string " " where + startidx - 1
            }
            # Update offset: start one after start of last match
            startidx += where
        }
    } while (where != 0)

    # Did we find something?
    if (loc_string)
        print loc_string
}

error: pathspec 'MPRT/MPRT.awk' did not match any file(s) known to git


Add a crosscheck for the SEASONS_SORTED_FILE

# Crosscheck the number of episodes of each show found by counting them (grep -c)
# in EPISODES_SORTED_FILE versus the number added up from SEASONS_SORTED_FILE
# Both numbers are found by processing a checkEpisodeInfo file
#
# For now these seem more likely a problem in scraping SEASONS_SORTED_FILE than EPISODES_SORTED_FILE
# so it could be these are false positives in generating spreadsheets. The case is stronger for
# problems listed in checkBBox_anomalies files.

# INVOCATION:
#    awk -f crosscheckInfo.awk checkEpisodeInfo-180415.185805.txt

/ movie / {
    title = $3
    numEpisodes = $5
    if (numEpisodes == 0)
        print
}

/ show / {
    numShows += 1
    showTitle[numShows] = $3
    shouldHave[numShows] = $5
    doesHave[numShows] = 0
    if (numEpisodes == 0)
        print
}

/^         / {
    epis  = NF-1
    if ($epis !~ /^[[:digit:]]*$/)
        print "==> Bad input line " NR "\n" $0
    else
        doesHave[numShows] += $epis
}

END {
    for ( i = 1; i <= numShows; i++ ) {
        if (shouldHave[i] != doesHave[i]) {
            print "==> show  "showTitle[i] " has " doesHave[i] " instead of " \
                shouldHave[i] " episodes."
        }
    }
}

error: pathspec 'crosscheckInfo.awk' did not match any file(s) known to git


tools: remove old updateAuthors.awk script

Using the new tools/update-authors.sh script so this is
redundant

PR-URL: https://github.com/iojs/io.js/pull/582
Reviewed-By: Ben Noordhuis <59bd0a3ff43b32849b319e645d4798d8a5d1e889@bnoordhuis.nl>

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/dreamllq/node.git/'


asplit.awk now 1-base numbering, for the need of kaldi's run.pl

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cybeliak/mispronounceableness.git/'


Add AWK script to merge comma-terminated lines

#!/usr/bin/awk -f
#
# AWK script to join multiple lines if the preceeding line ends with a comma (,).
#
# For example:
#
#   1,
#   2,
#   3
#
# Turns into:
#
#   1, 2, 3
#

/,$/ {
    ORS=""
    print $0
    do {
        getline
        print $0
    } while ($0 ~ /,$/)
    ORS="\n"
    print ""
}

error: pathspec 'mergelines.awk' did not match any file(s) known to git


Create signal_clean.awk

Create signal_clean.awk

A clean baseline reference for data processing comparisons

error: pathspec 'signal_clean.awk' did not match any file(s) known to git


Create signal_data.awk

Create signal_data.awk

A simple awk script for generating noisy signal data for testing the linear_regression code.

error: pathspec 'signal_data.awk' did not match any file(s) known to git


added missing file

added missing file

BEGIN {
	s = -1;
	if (table == "")
		table = "sequences";
	if (create_table == 1)
		printf("DROP TABLE IF EXISTS %s; CREATE TABLE %s ( otu VARCHAR(32) PRIMARY KEY, sequence LONGTEXT );", table, table);
}

/^>/ {
	++s;
	otu[s] = $1;
	sub(">", "", otu[s]);
	sequence[s] = "";
}
{
	if (substr($1, 1, 1) != ">")
		sequence[s] = sequence[s] $1;
}
END {
	for (i = 0; i <= s; ++i) {
		printf("INSERT INTO %s (otu, sequence) VALUES (\"%s\", \"%s\");\n", table, otu[i], sequence[i]);
	}
}

error: pathspec 'fa2sql.awk' did not match any file(s) known to git


Script for sampling N number of random reads from FASTQ files.

#!/usr/bin/awk -f
BEGIN{RS="@";
    ORS="";
    FS="\n"
    if(ARGC< 3){
	n=0;
	count=-1;}
    else{
	n=ARGV[2];
	count=0;}
    delete ARGV[2];
    srand();

    print "Usage:\n\tsampleReads.awk <INFILE.fastq> <NUMBER OF READS TO BE SAMPLED>\n\nIf no number is provided reads will be sampled randomly till the end of the inpuit file.\n\n";

}
{
    OUTF="subsam_" FILENAME;
    OUTF2="unsam_" FILENAME;
    if (rand() >= 0.5 && count < n  && NF > 1){
	print "@" $0 >> OUTF;
	if(n != 0)
	    count=count+1;
    }
    else if (NF > 1)
	print "@" $0 >> OUTF2;

}

error: pathspec 'sampleReads.awk' did not match any file(s) known to git


tools: introduce log analyzer

tools: introduce log analyzer

The included AWK script can analyze logs from BeeeOn Server in the current
configuration. It also handles two date-time formats used:

* testing: hh:mm:ss.iii
* production: YY-MM-DD hh:mm:ss.iii

The script tries to determine timing of certain operations (if the logs
contain enough information), counts unique clients, client requests, etc.

Signed-off-by: Jan Viktorin <3eab6ac54c005c4408f608556a6663b7f2abede7@fit.vutbr.cz>

#! /usr/bin/awk -f

function log_trim_left(line)
{
	sub(/^.+\[(Trace|Debug|Information|Notice|Warning|Error|Critical|Fatal)\] /, "", line)
	return line
}

function log_trim_right(line)
{
	sub(/\([^)]*:[0-9]*\)$/, "", line)
	return line
}

function log_extract_pid_tid(line)
{
	sub(/:\{/, "-", line)
	sub(/,.+$/, "", line)

	return line
}

function extract_message(line)
{
	line = log_trim_left(line)
	return log_trim_right(line)
}

function time2ms(time, part, result)
{
	split(time, part, "[:.]")

	result = part[1] * 60 * 60 * 1000
	result += part[2] * 60 * 1000
	result += part[3] * 1000
	result += part[4]

	return result
}

function time_stat(start, end)
{
	if (start == 0)
		return "??:??:??.???.." end " ?ms"

	return start ".." end " " (time2ms(end) - time2ms(start)) "ms"
}

function report_counts(title, counts)
{
	if (counts[0] > 0) {
		print title ":", counts[0]

		for (key in counts) {
			if (key == 0)
				continue

			print "*", key ":", counts[key]
		}
	}
}

function clear_array(array, first_zero) {
	for (key in array)
		delete array[key]

	if (first_zero)
		array[0] = 0
}

function count_array(array, key, count) {
	count = 0
	for (key in array)
		count += 1

	return count
}

function report_server_stats()
{
	report_counts("Criticals", criticals)
	report_counts("Fatals", fatals)

	report_counts("XML-UI clients", xmlui_clients)

	if (xmlui_requests[0] > 0) {
		print "Requests"

		for (key in xmlui_requests) {
			if (key == 0)
				continue

			split(key, multikey, SUBSEP)
			if (multikey[2] != "ip")
				continue

			ip = xmlui_requests[multikey[1], "ip"]
			start = xmlui_requests[multikey[1], "start"]
			end = xmlui_requests[multikey[1], "end"]
			info = xmlui_requests[multikey[1], "info"]

			print "*", ip ":", time_stat(start, end), info
		}
	}

	report_counts("REST-UI clients", restui_clients)

	if (count_array(restui_requests_total) > 0) {
		print "REST-UI Requests"
		for (key in restui_requests_total) {
			print "*", key ":", restui_requests_total[key]
		}
	}

	if (count_array(restui_requests_uri) > 0) {
		print "REST-UI URIs"
		for (key in restui_requests_uri) {
			print "*", key ":", restui_requests_uri[key]
		}
	}

	report_counts("GWS clients", gws_clients)

	if (transactions[0] > 0) {
		print "Transactions"

		for (key in transactions) {
			if (key == 0)
				continue

			split(key, multikey, SUBSEP)
			if (multikey[2] != "id")
				continue

			id = transactions[multikey[1], "id"]
			type = transactions[multikey[1], "type"]
			start = transactions[multikey[1], "start"]
			end = transactions[multikey[1], "end"]

			print "*", id ":", time_stat(start, end), type
		}
	}
}

function clear_stats()
{
	clear_array(fatals, 1)
	clear_array(criticals, 1)
	clear_array(xmlui_clients, 1)
	clear_array(xmlui_requests, 1)
	clear_array(restui_clients, 1)
	clear_array(restui_requests_total, 0)
	clear_array(restui_requests_uri, 0)
	clear_array(gws_clients, 1)
	clear_array(transactions, 1)
	clear_array(thread_pools, 1)

	types_count = 0
	enums_count = 0
	devs_count = 0
	queries_count = 0
	instances_count = 0
	rest_mappings = 0
}

BEGIN {
	clear_stats()
	HAS_DATE = 0
}

/ (19[0-9]{2}|20[0-9]{2})-[012][0-9]-[0-3][0-9] / {
	HAS_DATE = 1
}

!/ (19[0-9]{2}|20[0-9]{2})-[012][0-9]-[0-3][0-9] / {
	HAS_DATE = 0
}

/PermitAuthProvider/ && /SOME AUTHS/ {
	print
}

/PermitAuthProvider/ {
	next
}

/\[Fatal\]/ {
	_line = $0
	_msg = extract_message($0)

	if (fatals[_msg] == 0)
		fatals[0] += 1

	fatals[_msg] += 1
	if (fatals[_msg] == 1)
		print line
}

/\[Critical\]/ {
	_line = $0
	_msg = extract_message($0)

	if (criticals[_msg] == 0)
		criticals[0] += 1

	criticals[_msg] += 1
	if (criticals[_msg] == 1)
		print line
}

/^Application / && / version / {
	report_server_stats()
	clear_stats()

	if (HAS_DATE)
		_time = $2 " " $3
	else
		_time = $2

	print "# Server run report " _time
	print extract_message($0)
}
/^Application / && / Poco library / {
	print extract_message($0)
}

/IcuLocaleManager/ && /using ICU / {
	print extract_message($0)
}

/IcuLocaleManager/ && /resource dir: / {
	print extract_message($0)
}

/IcuLocaleManager/ && /detected locale: / {
	$0 = log_trim_left($0)
	print "locale: ", $3
}

/PocoDaoManager/ && /initialize database/ {
	print extract_message($0)
}

/PocoDaoManager/ && /Session / && /transact/ {
	print extract_message($0)
}

/PocoSqitchInitializer/ && /unsafe deploy / {
	print extract_message($0)
}

/PocoSimpleDBInitializer/ && /executed script / {
	print extract_message($0)
}

/TypeInfoProvider/ && /register type / {
	types_count += 1
}

/EnumInfoProvider/ && /register enum / {
	enums_count += 1
}

/DevicesSAXHandler/ && /Warning/ {
	print extract_message($0)
}

/DeviceInfoProvider/ && /register device / {
	devs_count += 1
}

/ loading query / {
	queries_count += 1
}

/DependencyInjector/ && /successfully created / {
	instances_count += 1
}

/RestRouter/ && /mapping / {
	rest_mappings += 1
}

/^Application/ && /starting runner/ {
	print extract_message($0)
}

/ creating thread pool / {
	$0 = log_trim_left($0)
	_curr = thread_pools[0] + 1
	thread_pools[0] = _curr
	thread_pools[_curr, "min"] += $5
	thread_pools[_curr, "max"] += $7
}

/LoopRunner/ && / started / && / loops / {
	print "Value types:   ", types_count
	print "Enum subtypes: ", enums_count
	print "Device types:  ", devs_count
	print "DB queries:    ", queries_count
	print "DI instances:  ", instances_count
	print "REST mappings: ", rest_mappings

	$0 = log_trim_left($0)
	print "Loops:         ", $2

	threads_min = 0
	threads_max = 0

	for (i = 0; i < thread_pools[0]; ++i) {
		threads_min += thread_pools[i + 1, "min"]
		threads_max += thread_pools[i + 1, "max"]
	}

	print "Thread pools:  ", thread_pools[0]
	print "Threads:       ", threads_min ".." threads_max
}

/XmlRequestHandlerFactory/ && / accepting client / {
	_id = log_extract_pid_tid($(3 + HAS_DATE))
	$0 = log_trim_left($0)

	split($3, _addr, ":")
	_ip = _addr[1]

	if (xmlui_clients[_ip] == 0)
		xmlui_clients[0] += 1

	xmlui_clients[_ip] += 1
	xmlui_requests_tmp[_id, "ip"] = _ip
}

/XmlRequestHandlerFactory/ && /document: <request/ {
	_id = log_extract_pid_tid($(3 + HAS_DATE))
	xmlui_requests_tmp[_id, "start"] = $(2 + HAS_DATE)

	_msg = extract_message($0)

	if (match(_msg, / ns="[^"]+" /) > 0) {
		_ns = substr(_msg, RSTART, RLENGTH)
		sub(/^.*="/, "", _ns)
		sub(/".*$/, "", _ns)
	}
	if (match(_msg, / type="[^"]+" /) > 0) {
		_type = substr(_msg, RSTART, RLENGTH)
		sub(/^.*="/, "", _type)
		sub(/".*$/, "", _type)
	}

	xmlui_requests_tmp[_id, "info"] = _ns ":" _type
}

/XmlRequestHandler/ && /<response/ {
	_id = log_extract_pid_tid($(3 + HAS_DATE))
	_curr = xmlui_requests[0] + 1
	xmlui_requests[0] = _curr

	xmlui_requests[_curr, "ip"] = xmlui_requests_tmp[_id, "ip"]
	delete xmlui_requests_tmp[_id, "ip"]
	xmlui_requests[_curr, "start"] = xmlui_requests_tmp[_id, "start"]
	delete xmlui_requests_tmp[_id, "start"]
	xmlui_requests[_curr, "info"] = xmlui_requests_tmp[_id, "info"]
	delete xmlui_requests_tmp[_id, "info"]

	xmlui_requests[_curr, "end"] = $(2 + HAS_DATE)
}

/PocoRestRequestFactory/ && /handling request/ {
	$0 = extract_message($0)
	restui_requests_total[$3] += 1
	restui_requests_uri[$4] += 1

	split($6, _addr, ":")
	_ip = _addr[1]

	if (restui_clients[_ip] == 0)
		restui_clients[0] += 1

	restui_clients[_ip] += 1
}

/WebSocketRequestHandler/ && /connection from / {
	$0 = log_trim_left($0)

	split($3, _addr, ":")
	_ip = _addr[1]

	if (gws_clients[_ip] == 0)
		gws_clients[0] += 1

	gws_clients[_ip] += 1
}

/ transaction / && / started / {
	_time = $(2 + HAS_DATE)
	_thread = log_extract_pid_tid($(3 + HAS_DATE))
	$0 = extract_message($0)
	_id = $2 "-" _thread
	transactions_tmp[_id, "start"] = _time
}

/ transaction / && (/ commit / || / rollback /) {
	_time = $(2 + HAS_DATE)
	_thread = log_extract_pid_tid($(3 + HAS_DATE))
	$0 = extract_message($0)
	_id = $2 "-" _thread
	_type = $3

	_curr = transactions[0] + 1
	transactions[0] = _curr

	transactions[_curr, "start"] = transactions_tmp[_id, "start"]
	delete transactions_tmp[_id, "start"]

	transactions[_curr, "end"] = _time
	transactions[_curr, "type"] = _type
	transactions[_curr, "id"] = _id
}

END {
	report_server_stats()
	clear_stats()
}

error: pathspec 'tools/log-analyze.awk' did not match any file(s) known to git