display clean-up

display clean-up

BEGIN {
    num_hosts = 0;
    num_subnets = 0;
    num_users = 0;
}
{
    if ($3 != "RECV" && $3 != "SENT") {
	if (num_hosts == 0) {
	    StartDate = $1;
	    StartTime = $2;
	} else {
	    EndDate = $1;
	    EndTime = $2;
	}
	if (AttemptedSends[$9] == 0 && AttemptedRecvs[$9] == 0) {
	    Hosts[num_hosts] = $9;
	    num_hosts++;
	}
	split($9, ip_tmp, ".");
	Subnet = sprintf("%s.%s.%s", ip_tmp[1], ip_tmp[2], ip_tmp[3]);
	if (AttemptedSends[Subnet] == 0 && AttemptedRecvs[Subnet] == 0) {
	    Subnets[num_subnets] = Subnet;
	    num_subnets++;
	}
	split($10, user_tmp, "\@");
	User = user_tmp[1];
	if (AttemptedSends[User] == 0 && AttemptedRecvs[User] == 0) {
	    Users[num_users] = User;
	    num_users++;
	}
	if ($3 == "S") {
	    AttemptedSends[$9]++;
	    AttemptedSends[Subnet]++;
	    AttemptedSends[User]++;
	    if ($4 != "F") {
		CompletedSends[$9]++;
		CompletedSends[Subnet]++;
		CompletedSends[User]++;
		BytesSent[$9] += $5;
		BytesSent[Subnet] += $5;
		BytesSent[User] += $5;
		TimeSending[$9] += $7;
		TimeSending[Subnet] += $7;
		TimeSending[User] += $7;
	    }
	} else {
	    AttemptedRecvs[$9]++;
	    AttemptedRecvs[Subnet]++;
	    AttemptedRecvs[User]++;
	    if ($4 != "F") {
		CompletedRecvs[$9]++;
		CompletedRecvs[Subnet]++;
		CompletedRecvs[User]++;
		BytesRecvd[$9] += $5;
		BytesRecvd[Subnet] += $5;
		BytesRecvd[User] += $5;
		TimeReceiving[$9] += $7;
		TimeReceiving[Subnet] += $7;
		TimeReceiving[User] += $7;
	    }
	}
    }
}
END {
    qsort(Hosts, 0, num_hosts-1);
    qsort(Subnets, 0, num_subnets-1);
    qsort(Users, 0, num_users-1);
    printf("\nCheckpoint Server Transfer Log Data from %s %s to %s %s\n\n",
	   StartDate, StartTime, EndDate, EndTime);
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Send", "Send", "Receive", "Receive", "Total", "Total");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Bandwidth", "Success", "Bandwidth", "Success", "Bandwidth",
	   "Success");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n\n",
	   "User/Host/Subnet", "(Mb/s)", "Rate", "(Mb/s)", "Rate", "(Mb/s)",
	   "Rate");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	Bytes = BytesSent[User] + BytesRecvd[User];
	Time = TimeSending[User] + TimeReceiving[User];
	Attempted = AttemptedSends[User] + AttemptedRecvs[User];
	Completed = CompletedSends[User] + CompletedRecvs[User];
	if (TimeSending[User] > 0 && TimeReceiving[User] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   User,
		   BytesSent[User]*8/TimeSending[User]/1024000,
		   CompletedSends[User]/AttemptedSends[User]*100,
		   BytesRecvd[User]*8/TimeReceiving[User]/1024000,
		   CompletedRecvs[User]/AttemptedRecvs[User]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    printf("\n");
    for (i=0; i < num_hosts; i++) {
	Host = Hosts[i];
	Bytes = BytesSent[Host] + BytesRecvd[Host];
	Time = TimeSending[Host] + TimeReceiving[Host];
	Attempted = AttemptedSends[Host] + AttemptedRecvs[Host];
	Completed = CompletedSends[Host] + CompletedRecvs[Host];
	if (TimeSending[Host] > 0 && TimeReceiving[Host] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Host,
		   BytesSent[Host]*8/TimeSending[Host]/1024000,
		   CompletedSends[Host]/AttemptedSends[Host]*100,
		   BytesRecvd[Host]*8/TimeReceiving[Host]/1024000,
		   CompletedRecvs[Host]/AttemptedRecvs[Host]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
	TotalBytesSent += BytesSent[Host];
	TotalBytesRecvd += BytesRecvd[Host];
	TotalTimeSending += TimeSending[Host];
	TotalTimeReceiving += TimeReceiving[Host];
	TotalAttemptedSends += AttemptedSends[Host];
	TotalAttemptedRecvs += AttemptedRecvs[Host];
	TotalCompletedSends += CompletedSends[Host];
	TotalCompletedRecvs += CompletedRecvs[Host];
    }
    printf("\n");
    for (i=0; i < num_subnets; i++) {
	Subnet = Subnets[i];
	Bytes = BytesSent[Subnet] + BytesRecvd[Subnet];
	Time = TimeSending[Subnet] + TimeReceiving[Subnet];
	Attempted = AttemptedSends[Subnet] + AttemptedRecvs[Subnet];
	Completed = CompletedSends[Subnet] + CompletedRecvs[Subnet];
	if (TimeSending[Subnet] > 0 && TimeReceiving[Subnet] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Subnet,
		   BytesSent[Subnet]*8/TimeSending[Subnet]/1024000,
		   CompletedSends[Subnet]/AttemptedSends[Subnet]*100,
		   BytesRecvd[Subnet]*8/TimeReceiving[Subnet]/1024000,
		   CompletedRecvs[Subnet]/AttemptedRecvs[Subnet]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    TotalBytes = TotalBytesSent + TotalBytesRecvd;
    TotalTime = TotalTimeSending + TotalTimeReceiving;
    TotalAttempted = TotalAttemptedSends + TotalAttemptedRecvs;
    TotalCompleted = TotalCompletedSends + TotalCompletedRecvs;
    if (TotalTimeSending > 0 && TotalTimeReceiving > 0) {
	printf("\n%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n\n",
	       "Overall",
	       TotalBytesSent*8/TotalTimeSending/1024000,
	       TotalCompletedSends/TotalAttemptedSends*100,
	       TotalBytesRecvd*8/TotalTimeReceiving/1024000,
	       TotalCompletedRecvs/TotalAttemptedRecvs*100,
	       TotalBytes*8/TotalTime/1024000,
	       TotalCompleted/TotalAttempted*100);
    }
    printf("\n%-16.16s %25.25s %25.25s\n\n", "User", "MB Sent", "MB Received");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	printf("%-16.16s %25.0f %25.0f\n",
	       User, BytesSent[User]/1024000, BytesRecvd[User]/1024000);
    }
    printf("\n%-16.16s %25.0f %25.0f\n",
	   "Total", TotalBytesSent/1024000, TotalBytesRecvd/1024000);
}
function qsort(v, left, right) {
  if (left >= right) return;
  swap(v, left, int((left + right)/2));
  last = left;
  for (i=left+1; i <= right; i++) {
    if (v[i] < v[left]) swap(v, ++last, i);
  }
  swap(v, left, last);
  qsort(v, left, last-1);
  qsort(v, last+1, right);
}
function swap(v, i, j) {
  temp = v[i];
  v[i] = v[j];
  v[j] = temp;
}

BEGIN {
    num_hosts = 0;
    num_subnets = 0;
    num_users = 0;
}
{
    if ($3 != "RECV" && $3 != "SENT") {
	if (num_hosts == 0) {
	    StartDate = $1;
	    StartTime = $2;
	} else {
	    EndDate = $1;
	    EndTime = $2;
	}
	if (AttemptedSends[$9] == 0 && AttemptedRecvs[$9] == 0) {
	    Hosts[num_hosts] = $9;
	    num_hosts++;
	}
	split($9, ip_tmp, ".");
	Subnet = sprintf("%s.%s.%s", ip_tmp[1], ip_tmp[2], ip_tmp[3]);
	if (AttemptedSends[Subnet] == 0 && AttemptedRecvs[Subnet] == 0) {
	    Subnets[num_subnets] = Subnet;
	    num_subnets++;
	}
	split($10, user_tmp, "\@");
	User = user_tmp[1];
	if (AttemptedSends[User] == 0 && AttemptedRecvs[User] == 0) {
	    Users[num_users] = User;
	    num_users++;
	}
	if ($3 == "S") {
	    AttemptedSends[$9]++;
	    AttemptedSends[Subnet]++;
	    AttemptedSends[User]++;
	    if ($4 != "F") {
		CompletedSends[$9]++;
		CompletedSends[Subnet]++;
		CompletedSends[User]++;
		BytesSent[$9] += $5;
		BytesSent[Subnet] += $5;
		BytesSent[User] += $5;
		TimeSending[$9] += $7;
		TimeSending[Subnet] += $7;
		TimeSending[User] += $7;
	    }
	} else {
	    AttemptedRecvs[$9]++;
	    AttemptedRecvs[Subnet]++;
	    AttemptedRecvs[User]++;
	    if ($4 != "F") {
		CompletedRecvs[$9]++;
		CompletedRecvs[Subnet]++;
		CompletedRecvs[User]++;
		BytesRecvd[$9] += $5;
		BytesRecvd[Subnet] += $5;
		BytesRecvd[User] += $5;
		TimeReceiving[$9] += $7;
		TimeReceiving[Subnet] += $7;
		TimeReceiving[User] += $7;
	    }
	}
    }
}
END {
    qsort(Hosts, 0, num_hosts-1);
    qsort(Subnets, 0, num_subnets-1);
    qsort(Users, 0, num_users-1);
    printf("\nCheckpoint Server Transfer Log Data from %s %s to %s %s\n\n",
	   StartDate, StartTime, EndDate, EndTime);
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Send", "Send", "Receive", "Receive", "Total", "Total");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n",
	   "", "Bandwidth", "Success", "Bandwidth", "Success", "Bandwidth",
	   "Success");
    printf("%-16.16s %9.9s %7.7s %9.9s %7.7s %9.9s %7.7s\n\n",
	   "User/Host", "(Mb/s)", "Rate", "(Mb/s)", "Rate", "(Mb/s)", "Rate");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	Bytes = BytesSent[User] + BytesRecvd[User];
	Time = TimeSending[User] + TimeReceiving[User];
	Attempted = AttemptedSends[User] + AttemptedRecvs[User];
	Completed = CompletedSends[User] + CompletedRecvs[User];
	if (TimeSending[User] > 0 && TimeReceiving[User] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   User,
		   BytesSent[User]*8/TimeSending[User]/1024000,
		   CompletedSends[User]/AttemptedSends[User]*100,
		   BytesRecvd[User]*8/TimeReceiving[User]/1024000,
		   CompletedRecvs[User]/AttemptedRecvs[User]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    printf("\n");
    for (i=0; i < num_hosts; i++) {
	Host = Hosts[i];
	Bytes = BytesSent[Host] + BytesRecvd[Host];
	Time = TimeSending[Host] + TimeReceiving[Host];
	Attempted = AttemptedSends[Host] + AttemptedRecvs[Host];
	Completed = CompletedSends[Host] + CompletedRecvs[Host];
	if (TimeSending[Host] > 0 && TimeReceiving[Host] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Host,
		   BytesSent[Host]*8/TimeSending[Host]/1024000,
		   CompletedSends[Host]/AttemptedSends[Host]*100,
		   BytesRecvd[Host]*8/TimeReceiving[Host]/1024000,
		   CompletedRecvs[Host]/AttemptedRecvs[Host]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
	TotalBytesSent += BytesSent[Host];
	TotalBytesRecvd += BytesRecvd[Host];
	TotalTimeSending += TimeSending[Host];
	TotalTimeReceiving += TimeReceiving[Host];
	TotalAttemptedSends += AttemptedSends[Host];
	TotalAttemptedRecvs += AttemptedRecvs[Host];
	TotalCompletedSends += CompletedSends[Host];
	TotalCompletedRecvs += CompletedRecvs[Host];
    }
    printf("\n");
    for (i=0; i < num_subnets; i++) {
	Subnet = Subnets[i];
	Bytes = BytesSent[Subnet] + BytesRecvd[Subnet];
	Time = TimeSending[Subnet] + TimeReceiving[Subnet];
	Attempted = AttemptedSends[Subnet] + AttemptedRecvs[Subnet];
	Completed = CompletedSends[Subnet] + CompletedRecvs[Subnet];
	if (TimeSending[Subnet] > 0 && TimeReceiving[Subnet] > 0) {
	    printf("%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n",
		   Subnet,
		   BytesSent[Subnet]*8/TimeSending[Subnet]/1024000,
		   CompletedSends[Subnet]/AttemptedSends[Subnet]*100,
		   BytesRecvd[Subnet]*8/TimeReceiving[Subnet]/1024000,
		   CompletedRecvs[Subnet]/AttemptedRecvs[Subnet]*100,
		   Bytes*8/Time/1024000,
		   Completed/Attempted*100);
	}
    }
    TotalBytes = TotalBytesSent + TotalBytesRecvd;
    TotalTime = TotalTimeSending + TotalTimeReceiving;
    TotalAttempted = TotalAttemptedSends + TotalAttemptedRecvs;
    TotalCompleted = TotalCompletedSends + TotalCompletedRecvs;
    if (TotalTimeSending > 0 && TotalTimeReceiving > 0) {
	printf("\n%-16.16s %9.2f %6.0f%% %9.2f %6.0f%% %9.2f %6.0f%%\n\n",
	       "Overall",
	       TotalBytesSent*8/TotalTimeSending/1024000,
	       TotalCompletedSends/TotalAttemptedSends*100,
	       TotalBytesRecvd*8/TotalTimeReceiving/1024000,
	       TotalCompletedRecvs/TotalAttemptedRecvs*100,
	       TotalBytes*8/TotalTime/1024000,
	       TotalCompleted/TotalAttempted*100);
    }
    printf("\n%-16.16s %25.25s %25.25s\n\n", "User", "MB Sent", "MB Received");
    for (i=0; i < num_users; i++) {
	User = Users[i];
	printf("%-16.16s %25.0f %25.0f\n",
	       User, BytesSent[User]/1024000, BytesRecvd[User]/1024000);
    }
    printf("\n%-16.16s %25.0f %25.0f\n",
	   "Total", TotalBytesSent/1024000, TotalBytesRecvd/1024000);
}
function qsort(v, left, right) {
  if (left >= right) return;
  swap(v, left, int((left + right)/2));
  last = left;
  for (i=left+1; i <= right; i++) {
    if (v[i] < v[left]) swap(v, ++last, i);
  }
  swap(v, left, last);
  qsort(v, left, last-1);
  qsort(v, last+1, right);
}
function swap(v, i, j) {
  temp = v[i];
  v[i] = v[j];
  v[j] = temp;
}


Translators/BingTranslator: use _sl instead of il

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 18 Mar 2018
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?&IG=" IG                                   \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl) {
    return "http://www.microsofttranslator.com/bv.aspx?"        \
        "from=" sl "&to=" tl "&a=" uri
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "translate")
        return HttpPathPrefix "/ttranslate?&IG=" IG
    else if (type == "translationlookup")
        return HttpPathPrefix "/ttranslationlookup?&IG=" IG
    else if (type == "transliterate")
        return HttpPathPrefix "/ttransliterate?&IG=" IG
    else # type == "detect"
        return HttpPathPrefix "/tdetect?&IG=" IG
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "translate")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "translationlookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "transliterate")
        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "detect"
        return "&text=" quote(text)
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetIG() # set IG

    # Language identification
    il = postResponse(text, _sl, _tl, _hl, "detect")
    if (!il) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    returnIl[0] = il
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = il
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    } else if (ast[0 SUBSEP "statusCode"] != "200") {
        e("[ERROR] statusCode: " ast[0 SUBSEP "statusCode"])
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "translationResponse"])

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        phonetics = unparameterize(content)
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(_sl, group, "-")
            content = postResponse(text, group[1], group[1], _hl, "transliterate")
            oPhonetics = unparameterize(content)
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, _sl))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), _sl))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(_sl))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(_sl))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, _sl, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}

####################################################################
# BingTranslator.awk                                               #
####################################################################
#
# Last Updated: 18 Mar 2018
BEGIN { provides("bing") }

function bingInit() {
    HttpProtocol = "http://"
    HttpHost = "www.bing.com"
    HttpPort = 80
}

# Retrieve the Cookie and set IG.
function bingSetIG(    content, cookie, group, header, isBody,
                       url, status, location) {
    url = HttpPathPrefix "/translator"

    header = "GET " url " HTTP/1.1\r\n"                                 \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"

    cookie = NULLSTR
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        match($0, /Set-Cookie: ([^;]*);/, group)
        if (group[1]) {
            cookie = cookie (cookie ?  "; " : NULLSTR) group[1]
        }
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        else { # interesting fields in header
            match($0, /^HTTP[^ ]* ([^ ]*)/, group)
            if (RSTART) status = group[1]
            match($0, /^Location: (.*)/, group)
            if (RSTART) location = squeeze(group[1]) # squeeze the URL!
        }
        l(sprintf("%4s bytes > %s", length($0), length($0) < 1024 ? $0 : "..."))
    }
    close(HttpService)

    if ((status == "301" || status == "302") && location)
        content = curl(location)
    # FIXME: cookie

    Cookie = cookie
    match(content, /IG:"([^"]+)"/, group)
    if (group[1]) {
        IG = group[1]
    } else {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        exit 1
    }
}

function bingTTSUrl(text, tl,
                    ####
                    country, gender, i, group,
                    header, content, isBody) {
    gender = "female"
    country = NULLSTR
    split(Option["narrator"], group, ",")
    for (i in group) {
        if (group[i] ~ /^(f(emale)?|w(oman)?)$/)
            gender = "female"
        else if (group[i] ~ /^m(ale|an)?$/)
            gender = "male"
        else
            country = group[i]
    }

    # Automatic ISO country code
    if (country) tl = tl "-" country
    else if (tl == "ar") tl = tl "-EG" # FIXME: sometimes doesn't work. Why?
    else if (tl == "da") tl = tl "-DK"
    else if (tl == "de") tl = tl "-DE"
    else if (tl == "en") tl = tl "-US"
    else if (tl == "es") tl = tl "-ES"
    else if (tl == "fi") tl = tl "-FI"
    else if (tl == "fr") tl = tl "-FR"
    else if (tl == "it") tl = tl "-IT"
    else if (tl == "ja") tl = tl "-JP"
    else if (tl == "ko") tl = tl "-KR"
    else if (tl == "nl") tl = tl "-NL"
    else if (tl == "nb") tl = tl "-NO" # Norwegian Bokmål
    else if (tl == "pl") tl = tl "-PL"
    else if (tl == "pt") tl = tl "-PT"
    else if (tl == "ru") tl = tl "-RU"
    else if (tl == "sv") tl = tl "-SE"
    else if (tl == "yue") ;
    else if (tl == "zh") tl = tl "-CN"

    header = "GET " "/tspeak?&IG=" IG                                   \
        "&language=" tl "&text=" preprocess(text)                       \
        "&options=" gender "&format=audio%2Fmp3" " HTTP/1.1\r\n"        \
        "Host: " HttpHost "\r\n"                                        \
        "Connection: close\r\n"
    if (Option["user-agent"])
        header = header "User-Agent: " Option["user-agent"] "\r\n"
    if (Cookie)
        header = header "Cookie: " Cookie "\r\n"

    content = NULLSTR; isBody = 0
    print header |& HttpService
    while ((HttpService |& getline) > 0) {
        if (isBody)
            content = content ? content "\n" $0 : $0
        else if (length($0) <= 1)
            isBody = 1
        #l(sprintf("%4s bytes > %s", length($0), $0))
    }
    close(HttpService)

    if (!TempFile)
        TempFile = getOutput("mktemp")
    printf("%s", content) > TempFile
    close(TempFile)
    return TempFile
}

function bingWebTranslateUrl(uri, sl, tl, hl) {
    return "http://www.microsofttranslator.com/bv.aspx?"        \
        "from=" sl "&to=" tl "&a=" uri
}

# Dictionary API (via HTTP GET).
function bingRequestUrl(text, sl, tl, hl) {
    return HttpPathPrefix "/translator/api/Dictionary/Lookup?"  \
        "from=" sl "&to=" tl "&text=" preprocess(text)
}

function bingPostRequestUrl(text, sl, tl, hl, type) {
    if (type == "translate")
        return HttpPathPrefix "/ttranslate?&IG=" IG
    else if (type == "translationlookup")
        return HttpPathPrefix "/ttranslationlookup?&IG=" IG
    else if (type == "transliterate")
        return HttpPathPrefix "/ttransliterate?&IG=" IG
    else # type == "detect"
        return HttpPathPrefix "/tdetect?&IG=" IG
}

function bingPostRequestContentType(text, sl, tl, hl, type) {
    return "application/x-www-form-urlencoded"
}

function bingPostRequestBody(text, sl, tl, hl, type) {
    if (type == "translate")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "translationlookup")
        return "&text=" quote(text) "&from=" sl "&to=" tl
    else if (type == "transliterate")
        return "&text=" quote(text) "&language=" sl "&toScript=" "latn"
    else # type == "detect"
        return "&text=" quote(text)
}

# Get the translation of a string.
function bingTranslate(text, sl, tl, hl,
                       isVerbose, toSpeech, returnPlaylist, returnIl,
                       ####
                       r,
                       content, tokens, ast,
                       _sl, _tl, _hl, il, isPhonetic,
                       translation,
                       wShowOriginal, wShowTranslation, wShowLanguages,
                       group, temp) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (isRTL(tl)) {
        # Check if target language is R-to-L
        if (!FriBidi)
            w("[WARNING] " getName(tl) " is a right-to-left language, but FriBidi is not found.")
    }
    _sl = getCode(sl); if (!_sl) _sl = sl
    _tl = getCode(tl); if (!_tl) _tl = tl
    _hl = getCode(hl); if (!_hl) _hl = hl

    bingSetIG() # set IG

    # Language identification
    il = postResponse(text, _sl, _tl, _hl, "detect")
    if (!il) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    }
    returnIl[0] = il
    if (Option["verbose"] < -1)
        return il
    if (Option["verbose"] < 0)
        return getList(il)

    # Hot-patches for Bing's own translator language codes
    # See: <https://msdn.microsoft.com/en-us/library/hh456380.aspx>
    if (_sl == "auto")  _sl = il
    if (_sl == "bs")    _sl = "bs-Latn" # 'bs' is not recognized as valid code
    if (_sl == "zh-CN") _sl = "zh-CHS"
    if (_sl == "zh-TW") _sl = "zh-CHT"
    if (_tl == "bs")    _tl = "bs-Latn"
    if (_tl == "zh-CN") _tl = "zh-CHS"
    if (_tl == "zh-TW") _tl = "zh-CHT"

    # Translation
    content = postResponse(text, _sl, _tl, _hl, "translate")
    if (Option["dump"])
        return content
    tokenize(tokens, content)
    parseJson(ast, tokens)

    l(content, "content", 1, 1)
    l(tokens, "tokens", 1, 0, 1)
    l(ast, "ast")
    if (!isarray(ast) || !anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
        return
    } else if (ast[0 SUBSEP "statusCode"] != "200") {
        e("[ERROR] statusCode: " ast[0 SUBSEP "statusCode"])
        ExitCode = 1
        return
    }

    translation = unparameterize(ast[0 SUBSEP "translationResponse"])

    # Transliteration
    wShowTranslationPhonetics = Option["show-translation-phonetics"]
    if (wShowTranslationPhonetics) {
        split(_tl, group, "-")
        content = postResponse(translation, group[1], group[1], _hl, "transliterate")
        phonetics = unparameterize(content)
        if (phonetics == translation) phonetics = ""
    }

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && phonetics ?
            prettify("brief-translation-phonetics", join(phonetics, " ")) :
            prettify("brief-translation", s(translation, tl))

    } else {
        # Verbose mode

        wShowOriginal = Option["show-original"]
        wShowTranslation = Option["show-translation"]
        wShowLanguages = Option["show-languages"]
        wShowDictionary = Option["show-dictionary"]

        # Transliteration (original)
        wShowOriginalPhonetics = Option["show-original-phonetics"]
        if (wShowOriginalPhonetics) {
            split(il, group, "-")
            content = postResponse(text, group[1], group[1], _hl, "transliterate")
            oPhonetics = unparameterize(content)
            if (oPhonetics == text) oPhonetics = ""
        }

        if (!oPhonetics) wShowOriginalPhonetics = 0
        if (!phonetics) wShowTranslationPhonetics = 0

        if (wShowOriginal) {
            # Display: original text & phonetics
            if (r) r = r RS RS
            r = r m("-- display original text")
            r = r prettify("original", s(text, il))
            if (wShowOriginalPhonetics)
                r = r RS prettify("original-phonetics", showPhonetics(join(oPhonetics, " "), il))
        }

        if (wShowTranslation) {
            # Display: major translation & phonetics
            if (r) r = r RS RS
            r = r m("-- display major translation")
            r = r prettify("translation", s(translation, tl))
            if (wShowTranslationPhonetics)
                r = r RS prettify("translation-phonetics", showPhonetics(join(phonetics, " "), tl))
        }

        if (wShowLanguages) {
            # Display: source language -> target language
            if (r) r = r RS RS
            r = r m("-- display source language -> target language")
            temp = Option["fmt-languages"]
            if (!temp) temp = "[ %s -> %t ]"
            split(temp, group, /(%s|%S|%t|%T)/)
            r = r prettify("languages", group[1])
            if (temp ~ /%s/)
                r = r prettify("languages-sl", getDisplay(il))
            if (temp ~ /%S/)
                r = r prettify("languages-sl", getName(il))
            r = r prettify("languages", group[2])
            if (temp ~ /%t/)
                r = r prettify("languages-tl", getDisplay(tl))
            if (temp ~ /%T/)
                r = r prettify("languages-tl", getName(tl))
            r = r prettify("languages", group[3])
        }

        if (wShowDictionary && false) { # FIXME!
            # Dictionary API
            # Note: source language must be identified
            dicContent = getResponse(text, il, _tl, _hl)
            tokenize(dicTokens, dicContent)
            parseJson(dicAst, dicTokens) # FIXME: inefficient parser

            if (anything(dicAst)) {
                # Display: dictionary entries
                if (r) r = r RS
                r = r m("-- display dictionary entries")

                # TODO
            }
        }
    }

    if (toSpeech) {
        returnPlaylist[0]["text"] = translation
        returnPlaylist[0]["tl"] = tl
    }

    return r
}


Script now directly invokes sox, and writes batch files instead of
creating a script to do so.

git-svn-id: a8b04003a33e1d3e001b9d20391fa392a9f62d91@1410 94700074-3cef-4d97-a70e-9c8c206c02f5

# An awk script that takes the hub4/eval99 data and
# generates a set of S4 batch files and raw audio
# to allow a supervised mode of testing
#
# Requires 'sox' the sound exchange program to extract the data
# Change 'directory' as appropriate for your system.
#
# Usage:
#       gawk -f unbundleHub4 *.stm
#
#

BEGIN {
    directory = "/lab/speech/sphinx4/data/hub4/eval99/supervised/";
    system("rm -rf " directory );
    system("mkdir -p " directory);
    audioCount = 0;
}

#
# Display info about stm file (for debugging purposes)
#
function displaySummary() {
    print "========================";
    print "Sphere   : "  sphereName;
    print "Audio    : " audioName;
    print "Speaker  : "  speaker;
    print "start    : "  startTime;
    print "end      : "  endTime;
    print "duration : " duration;
    print "class    : " speechClass;
    print "sox cmd  : " soxCommand;
    print "xcrpt    : " transcript;
}

#
# Output the batch file, generate the audio file
#
function outputCommands() {
    printf("%s %s\n", audioName, transcript) >> batchName;
    printf("%s %s\n", audioName, transcript) >> allBatch;
    system(soxCommand);
}

#
# Determines if the current utterance is worth keeping
#
function isGoodUtterance() {
    return speechClass != "";
}

#
# The main processing loop, extract the pertinent info
# and excute the commands
#
$1 != ";;"  {
    audioCount++;
    source = $1;
    sphereName = $1 ".sph";
    speaker = $3;
    startTime = $4;
    endTime = $5
    duration = endTime - startTime;
    info = $6;

    split(info, infoArray, ",");
    speechClass = infoArray[2];
    transcript = "";
    for (i = 7; i <= NF; i++) {
        # markup is in parans, so drop that.
        if (index($i, "(")  == 0) {
            transcript = transcript " " $i;
        }
    }

    audioName = directory source "." audioCount "." speechClass ".raw";
    batchName = directory speechClass "_hub4.batch"
    allBatch = directory "all_hub4.batch"

    soxCommand =  "sox " sphereName " -r 16000 -s -w " audioName  \
        " trim " startTime " " duration

    if (isGoodUtterance()) {
        displaySummary();
        outputCommands();
    }
}

# This is an awk script that will take the hub4 eval99 distribution
# and generate a script that will unpack the distribution into a set
# of audio and batch files suitable for s4.
#

BEGIN {
    audioCount = 0;
    directory = "/lab/speech/sphinx4/data/hub4/eval99/supervised/";

    print "#!/bin/sh"
    print "#  This Script unbundles the hub4/eval99 (broadcast news)"
    print "#  into a set of raw audio and batch files suitable for "
    print "#  processing by sphinx."
    print "#  "
    print "#  This script is automatically generated by unbundleHub99.awk"
    print "#  "
    print "#  "
    print "#  Create the destination directory"
    print "mkdir -p " directory;
    print "rm -rf " directory "/*";

}

function displaySummary() {
    print "========================";
    print "Sphere   : "  sphereName;
    print "Audio    : " audioName;
    print "Speaker  : "  speaker;
    print "start    : "  startTime;
    print "end      : "  endTime;
    print "duration : " duration;
    print "class    : " speechClass;
    print "sox cmd  : " soxCommand;
    print "xcrpt    : " transcript;
}

function outputCommands() {
    print "";
    print "";
    print "echo Processing" audioName
    print soxCommand;
    print "echo " audioName " " transcript " >> " batchName;
    print "echo " audioName " " transcript " >> " allBatch;
}

function isGoodUtterance() {
    return speechClass != "";
}

$1 != ";;"  {
    audioCount++;
    source = $1;
    sphereName = $1 ".sph";
    speaker = $3;
    startTime = $4;
    endTime = $5
    duration = endTime - startTime;
    info = $6;

    split(info, infoArray, ",");
    speechClass = infoArray[2];
    transcript = "";
    for (i = 7; i <= NF; i++) {
        # markup is in parans, so drop that.
        if (index($i, "(")  == 0) {
            transcript = transcript " " $i;
        }
    }

    audioName = directory source "." audioCount "." speechClass ".raw";
    batchName = directory speechClass "_hub4.batch"
    allBatch = directory "all_hub4.batch"

    soxCommand =  "sox " sphereName " -r 16000 -s -w " audioName  \
        " trim " startTime " " duration

    if (isGoodUtterance()) {
        outputCommands();
    }
    # displaySummary();
}


ahrf.awk: remove superfluous curly braces


slight formatting changes in comments

slight formatting changes in comments

# Process DBpedia data to extract subgraph corresponding to specified
# ontology types. This script expects the following three files to be
# passed in, in the following order:
#
#   * A list of DBpedia ontology types to select. Any DBpedia resource
#     belonging to at least one of these types according to the
#     DBpedia "mapping-based types" relations will be selected as a
#     node in the created graph. Example content:
#
#       <http://dbpedia.org/ontology/AdministrativeRegion>
#       <http://dbpedia.org/ontology/Country>
#       <http://dbpedia.org/ontology/City>
#       <http://dbpedia.org/ontology/Town>
#       <http://dbpedia.org/ontology/Village>
#
#   * A list of "mapping-based types" relations from DBpedia in Turtle
#     ("TTL") format. Example:
#
#       http://downloads.dbpedia.org/3.9/en/instance_types_en.ttl.bz2
#
#   * A list of "Wikipedia pagelinks" relations from DBpedia in Turtle
#     ("TTL") format. Example:
#
#       http://downloads.dbpedia.org/3.9/en/page_links_en.ttl.bz2
#
# This script will create four files:
#
#   * A map from created integer label ids to the provided ontology
#     types. Written to the file specified by the variable
#     label_index_file.
#
#     Format: [label id] [ontology type name]
#
#   * A map from created node ids to the corresponding DBpedia
#     resource names. Written to the file specified by the variable
#     node_id_index_file.
#
#     Format: [node id] [DBpedia resource name]
#
#   * A list of integer labels associated with the extracted nodes.
#     The ith line of this file is the label associated with node id
#     i. Written to the file specified by the variable labels_file.
#
#   * A list of edges corresponding to Wikipedia page links between
#     the extracted nodes. Written to stdout.
#
#     Format: [from node id] [to node id]
#
# Copyright (c) 2013, Roman Garnett (romangarnett@gmail.com)

# maintain variable containing ordinal number of current file in list
(FNR == 1) { file_number++ }

# read ontology types to keep and create index
(file_number == 1) {
	labels[$0] = ++label
	print label, $0 > label_index_file

	next
}

# read list of instance types and create (node id -> name) and
# (node id -> labels) maps
(file_number == 2) {
	# does this resource match one of the specified ontology types?
	if (labels[$3] && !node_ids[$1]) {
		node_ids[$1] = ++node_id

		print node_id, $1 > node_id_index_file
		print labels[$3] > labels_file
	}

	next
}

# read list of wikipedia page ids and create edge list corresponding
# to selected subgraph
((file_number == 3) && node_ids[$1] && node_ids[$3]) {
	print node_ids[$1], node_ids[$3]
}

# Process DBpedia data to extract subgraph corresponding to specified
# ontology types. This script expects the following three files to be
# passed in, in the following order:
#
#   - A list of DBpedia ontology types to select. Any DBpedia resource
#     belonging to at least one of these types according to the
#     DBpedia "mapping-based types" relations will be selected as a
#     node in the created graph. Example content:
#
#       <http://dbpedia.org/ontology/AdministrativeRegion>
#       <http://dbpedia.org/ontology/Country>
#       <http://dbpedia.org/ontology/City>
#       <http://dbpedia.org/ontology/Town>
#       <http://dbpedia.org/ontology/Village>
#
#   - A list of "mapping-based types" relations from DBpedia in Turtle
#     ("TTL") format. Example:
#
#       http://downloads.dbpedia.org/3.9/en/instance_types_en.ttl.bz2
#
#   - A list of "Wikipedia pagelinks" relations from DBpedia in Turtle
#     ("TTL") format. Example:
#
#       http://downloads.dbpedia.org/3.9/en/page_links_en.ttl.bz2
#
# This script will create four files:
#
#   - A map from created integer label ids to the provided ontology
#     types. Written to the file specified by the variable
#     label_index_file.
#
#       Format: [label id] [ontology type name]
#
#   - A map from created node ids to the corresponding DBpedia
#     resource names. Written to the file specified by the variable
#     node_id_index_file.
#
#       Format: [node id] [DBpedia resource name]
#
#   - A list of integer labels associated with the extracted nodes.
#     The ith line of this file is the label associated with node id
#     i. Written to the file specified by the variable labels_file.
#
#   - A list of edges corresponding to Wikipedia page links between
#     the extracted nodes. Written to stdout.
#
#       Format: [from node id] [to node id]
#
# Copyright (c) 2013, Roman Garnett (romangarnett@gmail.com)

# maintain variable containing ordinal number of current file in list
(FNR == 1) { file_number++ }

# read ontology types to keep and create index
(file_number == 1) {
	labels[$0] = ++label
	print label, $0 > label_index_file

	next
}

# read list of instance types and create (node id -> name) and
# (node id -> labels) maps
(file_number == 2) {
	# does this resource match one of the specified ontology types?
	if (labels[$3] && !node_ids[$1]) {
		node_ids[$1] = ++node_id

		print node_id, $1 > node_id_index_file
		print labels[$3] > labels_file
	}

	next
}

# read list of wikipedia page ids and create edge list corresponding
# to selected subgraph
((file_number == 3) && node_ids[$1] && node_ids[$3]) {
	print node_ids[$1], node_ids[$3]
}


gen-map: Always use internal version dependency on GLIBC

# Generate a version map file from a .map.in file.
#
# Written by Zack Weinberg <zackw at panix.com> in 2017.
# To the extent possible under law, Zack Weinberg has waived all
# copyright and related or neighboring rights to this work.
#
# See https://creativecommons.org/publicdomain/zero/1.0/ for further
# details.

# The .map.in file is the first input file, and we expect the Makefile
# to have set the variables SYMVER_MIN and SYMVER_FLOOR.  All symbol
# versions lower than SYMVER_MIN are discarded from the output.
# All symbol versions lower than SYMVER_FLOOR are replaced with
# SYMVER_FLOOR.  SYMVER_FLOOR must be greater than or equal to SYMVER_MIN.
#
# The ordering of symbol versions is entirely controlled by the %chain
# directive, which must therefore list both all of the versions
# actually used for symbols, and all of the versions that might be
# used as SYMVER_MIN or SYMVER_FLOOR.
#
# Note: if you change the format of .map.in files you probably need to
# update gen-vers.awk too.

BEGIN {
    split("", SYMBOLS) # ensure SYMBOLS is an array
    split("", VCHAIN)  # ditto VCHAIN
    NVCHAIN = 0

    # This arranges for sorted output if gawk is in use, and is
    # harmless otherwise.
    PROCINFO["sorted_in"] = "@ind_str_asc"
}

NF == 0   { next } # blank line, discard
$1 == "#" { next } # comment, discard
$1 == "%chain" {
    for (i = 2; i <= NF; i++) {
        VCHAIN[++NVCHAIN] = $i
    }
    next
}

{
    for (i = 2; i <= NF; i++) {
        if ($i != "-") {
            if ($i in SYMBOLS) {
                SYMBOLS[$i] = SYMBOLS[$i] SUBSEP $1
            } else {
                SYMBOLS[$i] = $1
            }
        }
    }
}

END {
    if (NVCHAIN == 0) {
        print ARGV[1] ": error: missing %chain directive" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }
    symver_min_idx = 0
    symver_floor_idx = 0
    for (i = 1; i <= NVCHAIN; i++) {
        if (VCHAIN[i] == SYMVER_MIN) {
            symver_min_idx = i
        }
        if (VCHAIN[i] == SYMVER_FLOOR) {
            symver_floor_idx = i
        }
    }
    if (symver_min_idx == 0) {
        print ARGV[1] ": error: SYMVER_MIN (" SYMVER_MIN ") " \
            "not found in %chain directives" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }
    if (symver_floor_idx == 0) {
        print ARGV[1] ": error: SYMVER_FLOOR (" SYMVER_FLOOR ") " \
            "not found in %chain directives" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }
    if (symver_floor_idx < symver_min_idx) {
        print ARGV[1] ": error: SYMVER_FLOOR (" SYMVER_FLOOR ") " \
            "is lower than SYMVER_MIN (" SYMVER_MIN ")" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }

    # Construct a pruned set of symbols and versions, including only
    # versions with symbols, discarding all symbols associated with
    # versions below SYMVER_MIN, raising symbols below SYMVER_FLOOR to
    # SYMVER_FLOOR, and removing duplicates.
    for (i = symver_min_idx; i <= NVCHAIN; i++) {
        v = VCHAIN[i]
        if (v in SYMBOLS) {
            nsyms = split(SYMBOLS[v], syms, SUBSEP)
            j = i;
            if (j < symver_floor_idx)
                j = symver_floor_idx;
            vr = VCHAIN[j]
            for (s = 1; s <= nsyms; s++) {
                if (syms[s]) {
                    symset[vr, syms[s]] = 1
                    allsyms[syms[s]] = 1
                }
            }
        }
    }

    vp = ""
    for (i = symver_floor_idx; i <= NVCHAIN; i++) {
        v = VCHAIN[i]
        split("", osyms)
        j = 0
        for (sym in allsyms) {
            if ((v, sym) in symset) {
                osyms[++j] = sym
            }
        }
        if (j > 0) {
            printf("%s {\n  global:\n", v);
            for (s = 1; s <= j; s++) {
                printf("    %s;\n", osyms[s]);
            }
            if (vp == "") {
                vp = v
                printf("  local:\n    *;\n};\n");
            } else {
                printf("} %s;\n", vp);
            }
        }
    }
}

# Generate a version map file from a .map.in file.
#
# Written by Zack Weinberg <zackw at panix.com> in 2017.
# To the extent possible under law, Zack Weinberg has waived all
# copyright and related or neighboring rights to this work.
#
# See https://creativecommons.org/publicdomain/zero/1.0/ for further
# details.

# The .map.in file is the first input file, and we expect the Makefile
# to have set the variables SYMVER_MIN and SYMVER_FLOOR.  All symbol
# versions lower than SYMVER_MIN are discarded from the output.
# All symbol versions lower than SYMVER_FLOOR are replaced with
# SYMVER_FLOOR.  SYMVER_FLOOR must be greater than or equal to SYMVER_MIN.
#
# The ordering of symbol versions is entirely controlled by the %chain
# directive, which must therefore list both all of the versions
# actually used for symbols, and all of the versions that might be
# used as SYMVER_MIN or SYMVER_FLOOR.
#
# Note: if you change the format of .map.in files you probably need to
# update gen-vers.awk too.

BEGIN {
    split("", SYMBOLS) # ensure SYMBOLS is an array
    split("", VCHAIN)  # ditto VCHAIN
    NVCHAIN = 0

    # This arranges for sorted output if gawk is in use, and is
    # harmless otherwise.
    PROCINFO["sorted_in"] = "@ind_str_asc"
}

NF == 0   { next } # blank line, discard
$1 == "#" { next } # comment, discard
$1 == "%chain" {
    for (i = 2; i <= NF; i++) {
        VCHAIN[++NVCHAIN] = $i
    }
    next
}

{
    for (i = 2; i <= NF; i++) {
        if ($i != "-") {
            if ($i in SYMBOLS) {
                SYMBOLS[$i] = SYMBOLS[$i] SUBSEP $1
            } else {
                SYMBOLS[$i] = $1
            }
        }
    }
}

END {
    if (NVCHAIN == 0) {
        print ARGV[1] ": error: missing %chain directive" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }
    symver_min_idx = 0
    symver_floor_idx = 0
    for (i = 1; i <= NVCHAIN; i++) {
        if (VCHAIN[i] == SYMVER_MIN) {
            symver_min_idx = i
        }
        if (VCHAIN[i] == SYMVER_FLOOR) {
            symver_floor_idx = i
        }
    }
    if (symver_min_idx == 0) {
        print ARGV[1] ": error: SYMVER_MIN (" SYMVER_MIN ") " \
            "not found in %chain directives" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }
    if (symver_floor_idx == 0) {
        print ARGV[1] ": error: SYMVER_FLOOR (" SYMVER_FLOOR ") " \
            "not found in %chain directives" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }
    if (symver_floor_idx < symver_min_idx) {
        print ARGV[1] ": error: SYMVER_FLOOR (" SYMVER_FLOOR ") " \
            "is lower than SYMVER_MIN (" SYMVER_MIN ")" > "/dev/stderr"
        close("/dev/stderr")
        exit 1
    }

    # Construct a pruned set of symbols and versions, including only
    # versions with symbols, discarding all symbols associated with
    # versions below SYMVER_MIN, raising symbols below SYMVER_FLOOR to
    # SYMVER_FLOOR, and removing duplicates.
    for (i = symver_min_idx; i <= NVCHAIN; i++) {
        v = VCHAIN[i]
        if (v in SYMBOLS) {
            nsyms = split(SYMBOLS[v], syms, SUBSEP)
            j = i;
            if (j < symver_floor_idx)
                j = symver_floor_idx;
            vr = VCHAIN[j]
            for (s = 1; s <= nsyms; s++) {
                if (syms[s]) {
                    symset[vr, syms[s]] = 1
                    allsyms[syms[s]] = 1
                }
            }
        }
    }

    vp = ""
    for (i = symver_floor_idx; i <= NVCHAIN; i++) {
        v = VCHAIN[i]
        split("", osyms)
        j = 0
        for (sym in allsyms) {
            if ((v, sym) in symset) {
                osyms[++j] = sym
            }
        }
        if (j > 0) {
            printf("%s {\n  global:\n", v);
            for (s = 1; s <= j; s++) {
                printf("    %s;\n", osyms[s]);
            }
            if (vp == "") {
                printf("  local:\n    *;\n};\n");
            } else {
                printf("} %s;\n", vp);
            }
            vp = v
        }
    }
}


opcode table parsing fix

opcode table parsing fix

BEGIN {
	FS = "`"
	PROCINFO["sorted_in"] = "@ind_str_asc"
}

function totitle(s,    tokens, i) {
	split(s, tokens, "_")
	s = ""
	for (i in tokens)
		s = s toupper(substr(tokens[i], 1, 1)) substr(tokens[i], 2)
	return s
}

/^\| +`.*` +\| +`0x[0-9a-f]{2}` +\|/ {
	names[$4] = $2
	sub("\\.", "_", $2)
	sub("/", "_", $2)
	symbols[$4] = totitle($2)
	immediates[$4] = $6
}

END {
	out = "gofmt > opcodes.go"

	print "package wag" | out
	print "import (" | out
	print "\"github.com/tsavola/wag/internal/opers\"" | out
	print "\"github.com/tsavola/wag/types\"" | out
	print ")" | out
	print "const (" | out
	for (key in symbols) {
		print "opcode" symbols[key] " = opcode(" key ")" | out
	}
	print ")" | out
	print "" | out
	print "var opcodeStrings = [256]string{" | out
	for (key in symbols) {
		print "opcode" symbols[key] ": \"" names[key] "\"," | out
	}
	print "}" | out
	print "" | out
	print "var opcodeImpls = [256]opImpl{" | out
	for (i = 1; i <= 256; i++) {
		key = sprintf("0x%02x", i-1)
		name = names[key]
		comment = ""

		if (!name) {
			gen = "badGen"
			info = "0"
		} else if (name == "else") {
			gen = "badGen"
			info = "0"
			key = "opcode" symbols[key]
		} else if (name == "end") {
			gen = "nil"
			info = "0"
			key = "opcode" symbols[key]
		} else {
			type = ""
			type2 = ""
			category = ""
			oper = ""

			if (match(name, "^(.)(..)\\.([^/]+)", groups)) {
				type = toupper(groups[1] groups[2])
				if (groups[1] == "i")
					category = "Int"
				else if (groups[1] == "f")
					category = "Float"
				oper = totitle(groups[3])
			}

			if (match(name, "/")) {
				gen = "genConversionOp"
				match(name, "/(...)$", groups)
				type2 = toupper(groups[1])
				category = ""
			} else if (match(name, "\\.const$")) {
				gen = "genConst" type
				oper = ""
			} else if (match(name, "\\.(abs|ceil|clz|copysign|ctz|floor|nearest|neg|popcnt|sqrt|trunc)$")) {
				gen = "genUnaryOp"
			} else if (match(name, "\\.eqz$")) {
				gen = "genUnaryConditionOp"
			} else if (match(name, "\\.(add|and|max|min|mul|or|xor)$", groups)) {
				gen = "genBinaryCommuteOp"
			} else if (match(name, "\\.(div(|_s|_u)|rem_(s|u)|rotl|rotr|shl|shr_(s|u)|sub)$")) {
				gen = "genBinaryOp"
			} else if (match(name, "\\.(eq|ge(|_s|_u)|gt(|_s|_u)|le(|_s|_u)|lt(|_s|_u)|ne)$", groups)) {
				if (match(name, "\\.(eq|ne)$"))
					gen = "genBinaryConditionCommuteOp"
				else
					gen = "genBinaryConditionOp"
				oper = totitle(groups[1])
			} else if (match(name, "^(.*)\\.load(.*)$", groups)) {
				gen = "genLoadOp"
				type2 = ""
				if (groups[2] == "")
					category = totitle(groups[1])
			} else if (match(name, "^(.*)\\.store(.*)$", groups)) {
				gen = "genStoreOp"
				type2 = ""
				if (groups[2] == "")
					category = totitle(groups[1])
			} else {
				if (match(name, "^(block|loop|if)$")) {
					gen = "nil"
					comment = " // initialized by init()"
				} else {
					gen = "gen" symbols[key]
				}
				oper = ""
			}

			info = "0"

			if (type)
				info = info " | opInfo(types." type ")"
			if (type2)
				info = info " | (opInfo(types." type2 ") << 8)"
			if (oper)
				info = info " | (opInfo(opers." category oper ") << 16)"

			sub("^0 \\| ", "", info)

			key = "opcode" symbols[key]
		}

		print key ": {" gen ", " info "}," comment | out
	}
	print "}" | out
	print "" | out
	print "var opcodeSkips = [256]func(reader, opcode){" | out
	for (i = 1; i <= 256; i++) {
		key = sprintf("0x%02x", i-1)
		sym = "opcode" symbols[key]
		comment = ""

		if (match(names[key], "^(block|loop|if)$")) {
			skip = "nil"
			comment = " // initialized by init()"
		} else if (match(names[key], "^(br_table)$")) {
			skip = "skip" symbols[key]
		} else if (names[key] == "end") {
			skip = "nil"
		} else if (names[key] == "else") {
			skip = "badSkip"
		} else if (immediates[key]) {
			skip = "skip" totitle(immediates[key])
		} else if (symbols[key]) {
			skip = "skipNothing"
		} else {
			sym = key
			skip = "badSkip"
		}

		print sym ": " skip "," comment | out
	}
	print "}" | out
}

BEGIN {
	FS = "`"
	PROCINFO["sorted_in"] = "@ind_str_asc"
}

function totitle(s,    tokens, i) {
	split(s, tokens, "_")
	s = ""
	for (i in tokens)
		s = s toupper(substr(tokens[i], 1, 1)) substr(tokens[i], 2)
	return s
}

/^\| +`.*` +\| +`0x[0-9a-f]{2}` +\|/ {
	names[$4] = $2
	sub("\\.", "_", $2)
	sub("/", "_", $2)
	symbols[$4] = totitle($2)
	immediates[$4] = $6
}

END {
	out = "gofmt > opcodes.go"

	print "package wag" | out
	print "import (" | out
	print "\"github.com/tsavola/wag/internal/opers\"" | out
	print "\"github.com/tsavola/wag/types\"" | out
	print ")" | out
	print "const (" | out
	for (key in symbols) {
		print "opcode" symbols[key] " = opcode(" key ")" | out
	}
	print ")" | out
	print | out
	print "var opcodeStrings = [256]string{" | out
	for (key in symbols) {
		print "opcode" symbols[key] ": \"" names[key] "\"," | out
	}
	print "}" | out
	print | out
	print "var opcodeImpls = [256]opImpl{" | out
	for (i = 1; i <= 256; i++) {
		key = sprintf("0x%02x", i-1)
		name = names[key]
		comment = ""

		if (!name) {
			gen = "badGen"
			info = "0"
		} else if (name == "else") {
			gen = "badGen"
			info = "0"
			key = "opcode" symbols[key]
		} else if (name == "end") {
			gen = "nil"
			info = "0"
			key = "opcode" symbols[key]
		} else {
			type = ""
			type2 = ""
			category = ""
			oper = ""

			if (match(name, "^(.)(..)\\.([^/]+)", groups)) {
				type = toupper(groups[1] groups[2])
				if (groups[1] == "i")
					category = "Int"
				else if (groups[1] == "f")
					category = "Float"
				oper = totitle(groups[3])
			}

			if (match(name, "/")) {
				gen = "genConversionOp"
				match(name, "/(...)$", groups)
				type2 = toupper(groups[1])
				category = ""
			} else if (match(name, "\\.const$")) {
				gen = "genConst" type
				oper = ""
			} else if (match(name, "\\.(abs|ceil|clz|copysign|ctz|floor|nearest|neg|popcnt|sqrt|trunc)$")) {
				gen = "genUnaryOp"
			} else if (match(name, "\\.eqz$")) {
				gen = "genUnaryConditionOp"
			} else if (match(name, "\\.(add|and|max|min|mul|or|xor)$", groups)) {
				gen = "genBinaryCommuteOp"
			} else if (match(name, "\\.(div(|_s|_u)|rem_(s|u)|rotl|rotr|shl|shr_(s|u)|sub)$")) {
				gen = "genBinaryOp"
			} else if (match(name, "\\.(eq|ge(|_s|_u)|gt(|_s|_u)|le(|_s|_u)|lt(|_s|_u)|ne)$", groups)) {
				if (match(name, "\\.(eq|ne)$"))
					gen = "genBinaryConditionCommuteOp"
				else
					gen = "genBinaryConditionOp"
				oper = totitle(groups[1])
			} else if (match(name, "^(.*)\\.load(.*)$", groups)) {
				gen = "genLoadOp"
				type2 = ""
				if (groups[2] == "")
					category = totitle(groups[1])
			} else if (match(name, "^(.*)\\.store(.*)$", groups)) {
				gen = "genStoreOp"
				type2 = ""
				if (groups[2] == "")
					category = totitle(groups[1])
			} else {
				if (match(name, "^(block|loop|if)$")) {
					gen = "nil"
					comment = " // initialized by init()"
				} else {
					gen = "gen" symbols[key]
				}
				oper = ""
			}

			info = "0"

			if (type)
				info = info " | opInfo(types." type ")"
			if (type2)
				info = info " | (opInfo(types." type2 ") << 8)"
			if (oper)
				info = info " | (opInfo(opers." category oper ") << 16)"

			sub("^0 \\| ", "", info)

			key = "opcode" symbols[key]
		}

		print key ": {" gen ", " info "}," comment | out
	}
	print "}" | out
	print | out
	print "var opcodeSkips = [256]func(reader, opcode){" | out
	for (i = 1; i <= 256; i++) {
		key = sprintf("0x%02x", i-1)
		sym = "opcode" symbols[key]
		comment = ""

		if (match(names[key], "^(block|loop|if)$")) {
			skip = "nil"
			comment = " // initialized by init()"
		} else if (match(names[key], "^(br_table)$")) {
			skip = "skip" symbols[key]
		} else if (names[key] == "end") {
			skip = "nil"
		} else if (names[key] == "else") {
			skip = "badSkip"
		} else if (immediates[key]) {
			skip = "skip" totitle(immediates[key])
		} else if (symbols[key]) {
			skip = "skipNothing"
		} else {
			sym = key
			skip = "badSkip"
		}

		print sym ": " skip "," comment | out
	}
	print "}" | out
}


Fix 0.00 should not pass as int

Fix 0.00 should not pass as int

Fix #4

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_integer(x) { return x ~ /^-?[0-9]+$/ }"
    print "function is_number(x) { return x ~ /^-?[0-9]+(\.[0-9]+)?$/ }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print  NR FS CSVFILENAME FS \"header\" FS \"invalid-header\" FS \"error\" FS  \"headers are invalid \"; exit 0;} }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "violation"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type == "integer") {
            cat       = "error"
            test      = field " != \"\" && !is_integer(" field ")"
            mini_msg  = field " should be an integer"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be an integer but was \" " field " \" "
        } else if (type == "number") {
            cat       = "error"
            test      = field " != \"\" && !is_number(" field ")"
            mini_msg  = field " should be a decimal number"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a decimal number but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = field " value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = field " value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = field " value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = field " value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = field " value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = field " max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        } else if (options["mode"] == "violation") {
            key=file_options["pkey"]
            gsub(/\|/, " \"-\" ", key)
            err_handler = handler_prefix "print  NR FS CSVFILENAME FS " key " FS \"" rule_type "\" FS \"" cat "\" FS  \"" mini_msg "\" FS \"" field "\"}"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    print \"COPY \" schema \"" file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    print "    if (action == \"insert\") print \"\\\\.\" RS"
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}

#!/usr/bin/awk -F, -f

BEGIN {
    FS=","
    print "#!/usr/bin/awk -F, -f" RS

    print "# generated by: awk-csvalid https://github.com/tesera/awk-csvalid" RS
    print "# awk-csvalid csv toolset generator: https://github.com/tesera/awk-csvalid"
    print "# usage:"
    print "#    validate:     awk -f action=validate validator.awk > violations.txt"
    print "#    create table: awk -v action=table -f validator.awk | psql afgo_dev"
    print "#    sanitize csv: awk -v action=sanitize -f validator.awk > sanitized.csv"
    print "#    insert sql:   awk -v action=insert -f validator.awk | psql afgo_dev" RS
    print "# awk is a simple unix text file parser: http://www.gnu.org/software/gawk/manual/gawk.html"
    print "# awk primer:"
    print "#    NR = number/index current record"
    print "#    RS = record seperator new line i.e. \\n"
    print "#    FS = field seperator i.e. ,"
    print "#    /pattern/ { expression } = if pattern is truthy run expression" RS

    print "BEGIN {"
    print "    FS=\",\"; OFS=\",\"; err_count=0; cats[\"na\"]=0;"
    print "    if(!action) action = \"validate\""
    print "    summary_header=\"file_name,field_name,rule,message,violation_count\""
    print "    CSVFILENAME = CSVFILENAME ? CSVFILENAME : FILENAME"
    print "    FPAT = \"([^,]*)|(\\\"[^\\\"]+\\\")\""
    print "}" RS

    print "# builtin helper functions"
    print "function eql(x,y) {v=1; for (i in x) v=(v&&x[i]==y[i]); return v;}"
    print "function are_headers_valid(valid_headers) { v=0; split($0, h, \",\"); split(valid_headers, vh, \"|\"); return eql(h, vh); }"
    print "function is_unique(i, val) { if (vals[i,val]) { return 0; } else { vals[i,val] = 1; return 1; } }"
    print "function is_integer(x) { return x ~ /^-?[0-9]+$/ }"
    print "function is_number(x) { return x ~ /^-?[0-9]+(\.[0-9]+)?$/ }"
    print "function print_cats(categories) { for (category in categories) { if (categories[category]) print \"      \" category \": \" categories[category]; } }"
    print "function log_err(cat) { cats[cat]++; err_count++; }"
    print RS
    print "#get rid of the evil windows cr"
    print "{ sub(\"\\r$\", \"\") }"
    print RS
    print "{"
    print "     for (i = 1; i <= NF; i++) {"
    print "         if (substr($i, 1, 1) == \"\\\"\") {"
    print "             len = length($i)"
    print "             $i = substr($i, 2, len - 2)"
    print "         }"
    print "     }"
    print "}"
    print RS
}

# BUILD OPTIONS MAP
$1 == "option" {
    options[$2]=$3;
}

# RENDER PFKEY CHECK
$1 == "file" {
    file_options[$2]=$3;
    if ($2 == "pkey") {
        pkeys=$3
        gsub(/\|/, " \"-\" ", pkeys)
        pkeycheck="action ~ /validate/ && NR > 1 { pkey=" pkeys "; if(keys[pkey]) { if (dupkeys[pkey]) dupkeys[pkey]++; else dupkeys[pkey] = 1 } else { keys[pkey] = NR } }"
    }
}

# RENDER HEADERS CHECK
$1 == "headers" && $2 == "names" {
    split($3, headers, "|")
    print "# make header index/map"
    print "NR > 1 {"
    print "field_width = " length(headers)
    for (i in headers) {
        header_name = headers[i]
        header_index[header_name] = i
        print "    " headers[i]  "=$" i
    }

    print "}" RS
    print "# awk rules based on user csv ruleset"
    print "NR == 1 && action == \"validate\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { gsub(/\\|/, FS, headers); print  NR FS CSVFILENAME FS \"header\" FS \"invalid-header\" FS \"error\" FS  \"headers are invalid \"; exit 0;} }"
    print "NR == 1 && action == \"validate:summary\" { headers=\"" $3 "\"; if (!are_headers_valid(headers)) { violations[CSVFILENAME FS \"headers\" FS  \"names\" FS \"csv headers are invalid\" FS \"error\"]=1; exit 0; } }"
    print pkeycheck
    cat = "error"
    test = "NF != field_width"
    msg = "row \" NR \" in \" CSVFILENAME \" has \" NF \" columns and \" field_width \" was expected"
    mini_msg = "row \" NR \" has an invalid column count"
    rule_type = "invalid-column-count"
    # print "action ~ /^validate/ && NR > 1 { if(NF != field_width) { print \"row \" NR \" in \" CSVFILENAME \" has \" NF \" records and \" field_width \" was expected\"; next;} }"
}

{
    # SET DEFAULTS IF NOT PASSED IN VIA RULES
    defaults["mode"]    = "violation"
    defaults["summary"] = "true"
    defaults["dcat"]    = "warning"
    for (option in defaults) {
        if (!options[option]) options[option] = defaults[option]
    }
}

# RENDER VALUE CHECKS
$1 == "field" {

    # PARSE RULE FIELDS
    rule_type     = $2
    split($3,params," ")
    field         = params[1]
    cat           = params[3]
    field_index   = header_index[field]
    test          = ""
    msg           = ""

    # BUILD VIOLATION TEST
    if (rule_type == "type") {
        type = params[2]

        if (type == "integer") {
            cat       = "error"
            test      = field " && !is_integer(" field ")"
            mini_msg  = field " should be an integer"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be an integer but was \" " field " \" "
        } else if (type == "number") {
            cat       = "error"
            test      = field " && !is_number(" field ")"
            mini_msg  = field " should be a decimal number"
            msg       = "Field " field " in \" CSVFILENAME \" line \" NR \" should be a decimal number but was \" " field " \" "
        }
    } else if (rule_type == "required") {
        req = params[2]

        if (req == "true") {
            test        = field " == \"\""
            mini_msg    = field " value is required but was empty"
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required"
        } else if (req != "false") {
            test        = req " && " field " == \"\""
            mini_msg    = field " value is required if " req
            msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is required if " req
        }

    } else if (rule_type == "unique") {
        cat         = "error"
        test        = "!is_unique(" field ")"
        mini_msg    = field " value should be unique but had duplicates"
        msg         = "Field " field " in \" CSVFILENAME \" line \" NR \" is a duplicate and should be unique"

    } else if (rule_type ~ /^minimum|maximum$/) {
        comparator  = $2 == "maximum" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maximum" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && " field " " comparator " " limit
        mini_msg    = field " value should be " term " or equal to: " limit
        msg         = field " in \" CSVFILENAME \" line \" NR \" should be " term " or equal to" limit " and was \" " field " \" "

    } else if (rule_type == "pattern") {
        pattern     = params[2]

        cat         = "error"
        test        = field " != \"\" && " field " !~ " pattern
        mini_msg    = field " value should match: " pattern
        msg         = field " in \" CSVFILENAME \" line \" NR \" should match the following pattern " pattern " and was \" " field " \" "

    } else if (rule_type ~ /^minLength|maxLength$/) {
        comparator  = $2 == "maxLength" ? ">" : "<"
        limit       = params[2]
        term        = $2 == "maxLength" ? "less" : "greater"

        cat         = "error"
        test        = field " != \"\" && length(" field ") " comparator " " limit
        mini_msg    = field " max length is: " limit
        msg         = field " length in \" CSVFILENAME \" line \" NR \" should be " term " or equal to " limit " and was \" length(" field ") \" "
    }
}

{
    # RENDER VIOLATION HANDLER
    if (test) {
        if (!cat) cat = options["dcat"]
        handler_prefix = "{ log_err(\"" cat "\"); "

        if (options["mode"] == "text") {
            err_handler = handler_prefix "print \"" msg "\" RS $0 RS; }"

        } else if (options["mode"] == "wrap") {
            acc = "$" field_index
            err_handler = handler_prefix "FS=\",\";" acc "=\">>\" " acc " \"<<\"; print $0 }"

        } else if (options["mode"] == "append") {
            err_handler = handler_prefix "print $0 FS \"" msg "\" }"
        } else if (options["mode"] == "violation") {
            key=file_options["pkey"]
            gsub(/\|/, " \"-\" ", key)
            err_handler = handler_prefix "print  NR FS CSVFILENAME FS " key " FS \"" rule_type "\" FS \"" cat "\" FS  \"" mini_msg "\" FS \"" field "\"}"
        }

        if (rule_type != "none") {
            print "action == \"validate\" && NR > 1 && " test " " err_handler " "
            summary_handler = "key=CSVFILENAME FS \"" field "\" FS  \"" rule_type "\" FS \"" mini_msg "\" FS \"" cat "\"; if(!violations[key]) { violations[key]=0; } violations[key]++;"
            print "action == \"validate:summary\" && NR > 1 && " test " { " summary_handler " } "
        }
    }
}

# BUILD MAP OF SQL TYPES FOR SQL CREATE AND COPY
$1 == "field" {
    sql_types_map["string"] = "text"
    sql_types_map["integer"] = "integer"
    sql_types_map["number"] = "numeric"

    rule_type = $2 ? sql_types_map[$2] : sql_types_map["string"]

    split($3, params, " ")
    field = params[1]

    sql_types[field] = rule_type
}

END {
    # SANITIZE EMPTY VALUES TO PASSED IN DEFAULTS OR NULL
    print RS "# sanitize rules"
    print "action ~ /^(sanitize|insert)$/ && NR > 1 {"
    for (field in sql_types) {
        sql_type    = sql_types[field]
        field_index = header_index[field]
        dval        = sql_type == "numeric" ? options["dvalnum"] : options["dvalstr"]
        dval        = dval ? dval : "\\\\N"

        print "    if (" field " == \"\") $" field_index " = \"" dval "\""
    }
    print "}"

    # SQL COPY
    print RS "# action handlers"
    print "action == \"insert\" && NR == 1 {"
    print "    print \"SET client_encoding = 'UTF8';\""
    print "    print \"COPY \" schema \"" file_options["table"] " (\" addfields FS \"source_row_index\" FS $0 \") FROM stdin;\""
    print "}"

    print   "action == \"insert\" && NR > 1 {"
    print   "   record = addvals \"\\t\" NR"
    print   "   for (i = 1; i <= NF; i++) {"
    print   "       record = record \"\\t\" $i"
    print   "   }"
    print   "   print record"
    print   "}"

    # SQL CREATE TABLE
    print "action == \"table\" && NR == 1 {"
    fields=""
    for (i = 1; i <= length(headers); i++) {
        field     = headers[i]
        sql_type  = sql_types[field]
        fields    = fields field " " sql_type
        if (i < length(headers)) fields = fields ","
    }

    if(file_options["pkey"]) {
        cols    = file_options["pkey"]
        gsub(/\|/, ",", cols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_pkey PRIMARY KEY (" cols ") "
    }
    if(file_options["fkey"]) {
        split(file_options["fkey"], ref, " ")
        ftable  = ref[1]
        fcols   = ref[2]
        gsub(/\|/, ",", fcols)
        fields  = fields ", CONSTRAINT " file_options["table"] "_" ftable "_fkey FOREIGN KEY (" fcols ") REFERENCES " ftable " (" fcols ") MATCH FULL "
        fields  = fields "ON UPDATE CASCADE ON DELETE NO ACTION"
    }
    print "     print \"CREATE TABLE IF NOT EXISTS " file_options["table"] " (" fields ");\""
    print "}"

    print "action == \"sanitize\" { print }"

    # END
    print "# la fin"
    print "END {"
    print "    if (action == \"validate:summary\" && length(dupkeys) > 0) for (dup in dupkeys) { violation=CSVFILENAME FS \"" file_options["pkey"] "\" FS  \"duplicate\" FS dup \" violates pkey\" FS \"error\"; violations[violation] = dupkeys[dup]}"
    print "    if (action == \"validate:summary\") { if (length(violations) > 0) for (violation in violations) { print violation FS violations[violation]; } }"
    print "    if (action == \"insert\") print \"\\\\.\" RS"
    print "    if (action == \"validate\" && options[\"summary\"] == \"true\") { print RS \"violation summary: \" RS \"   counts:   \" RS \"      total: \" err_count; print_cats(cats); }"
    print "}"
}


[unix][utils] Print language for translation

#!/usr/bin/env gawk -f
# ----------------------------------------------------------------------------
# "THE BEER-WARE LICENSE" (Revision 42):
# <mort.yao@gmail.com> wrote this file. As long as you retain this notice you
# can do whatever you want with this stuff. If we meet some day, and you think
# this stuff is worth it, you can buy me a beer in return Mort Yao
# ----------------------------------------------------------------------------
BEGIN {
    if (ARGC < 2) {
        help()
        exit
    }

    RS = ORS = "\r\n"
    FS = OFS = "\n"
    httpHost = "translate.google.com"
    httpPort = 80
    httpService = "/inet/tcp/0/" httpHost "/" httpPort
    percentEncoding["\n"]      = "%0A"
    percentEncoding[" "]       = "%20"
    percentEncoding["\""]      = "%22"
    percentEncoding["&"]       = "%26"

    match(ARGV[1], /{.*=.*}/)
    if (RSTART) {
        match(ARGV[1], /{.*=/)
        sl = substr(ARGV[1], RSTART + 1, RLENGTH - 2)
        if (sl == "") sl = "auto"

        match(ARGV[1], /=.*}/)
        split(substr(ARGV[1], RSTART + 1, RLENGTH - 2), tls, "+")
        if (length(tls) == 0) tls[1] = ""

        text_p = 1
    } else {
        sl = tls[1] = "auto"
        text_p = 0
    }

    while (ARGV[++text_p]) {
        $0 = ""
        getline < ARGV[text_p]
        if (NF)
            split($0, texts)
        else
            split(ARGV[text_p], texts)
        n = length(texts)

        for (tls_i = 1; tls_i <= length(tls); tls_i++) {
            tl = ((tls[tls_i] == "") ? "auto" : tls[tls_i])
            tl = substr(tl, 1 + (showPhonetically = index(tl, "@")))

            for (texts_i = 1; texts_i <= n; texts_i++) {
                text = texts[texts_i]
                for (key in percentEncoding)
                    gsub(key, percentEncoding[key], text)

                url = "http://translate.google.com/translate_a/t?client=t&ie=UTF-8&oe=UTF-8" \
                    "&sl=" sl "&tl=" tl "&text=" text

                print "GET " url |& httpService
                while ((httpService |& getline) > 0)
                    content = $0
                close(httpService)

                translation = phonetics = ""
                match(content, /\]\]/)
                s0 = substr(content, 3, RSTART - 2)
                while (s0 != "") {
                    match(s0, /\[[^\]]*\]/)
                    t0 = substr(s0, RSTART + 1, RLENGTH - 2)
                    s0 = substr(s0, RLENGTH + 2)

                    for (i = 0; t0 != ""; i++) {
                        match(t0, /[^\\]","/)
                        f[i] = substr(t0, 2, RSTART - 1)
                        t0 = substr(t0, RSTART + 3)
                    }

                    translation = translation f[0]
                    phonetics = phonetics ((f[2] != "") ? (f[2] " ") : f[0])
                }

                output = showPhonetically ? phonetics : translation
                output = gensub(/\\( |\\|\")/, "\\1", "g", output)
                output = gensub(/\\n/, "\n", "g", output)
                output = gensub(/(\") /, "\\1", "g", output)
                output = gensub(/ ('|;|:|\.|\,|\!|\?)/, "\\1", "g", output)
                printf tl ":" output "\n"
            }
        }
    }
}

function help() {
    print "Usage: translate {[SL]=[TL]} TEXT|TEXT_FILENAME"
    print "       translate {[SL]=[TL1]+[TL2]+...} TEXT|TEXT_FILENAME"
    print "       translate TEXT1 TEXT2 ..."
    print
    print "TEXT: Source text (The text to be translated)"
    print "      Can also be the filename of a plain text file."
    print "  SL: Source language (The language of the source text)"
    print "  TL: Target language (The language to translate the source text into)"
    print "      Language codes as listed here:"
    print "    * http://developers.google.com/translate/v2/using_rest#language-params"
    print "      Ignore the code where you want the system to identify it for you."
    print "      Prefix the code with an ampersat @ to show the result phonetically."
}

#!/usr/bin/env gawk -f
# ----------------------------------------------------------------------------
# "THE BEER-WARE LICENSE" (Revision 42):
# <mort.yao@gmail.com> wrote this file. As long as you retain this notice you
# can do whatever you want with this stuff. If we meet some day, and you think
# this stuff is worth it, you can buy me a beer in return Mort Yao
# ----------------------------------------------------------------------------
BEGIN {
    if (ARGC < 2) {
        help()
        exit
    }

    RS = ORS = "\r\n"
    FS = OFS = "\n"
    httpHost = "translate.google.com"
    httpPort = 80
    httpService = "/inet/tcp/0/" httpHost "/" httpPort
    percentEncoding["\n"]      = "%0A"
    percentEncoding[" "]       = "%20"
    percentEncoding["\""]      = "%22"
    percentEncoding["&"]       = "%26"

    match(ARGV[1], /{.*=.*}/)
    if (RSTART) {
        match(ARGV[1], /{.*=/)
        sl = substr(ARGV[1], RSTART + 1, RLENGTH - 2)
        if (sl == "") sl = "auto"

        match(ARGV[1], /=.*}/)
        split(substr(ARGV[1], RSTART + 1, RLENGTH - 2), tls, "+")
        if (length(tls) == 0) tls[1] = ""

        text_p = 1
    } else {
        sl = tls[1] = "auto"
        text_p = 0
    }

    while (ARGV[++text_p]) {
        $0 = ""
        getline < ARGV[text_p]
        if (NF)
            split($0, texts)
        else
            split(ARGV[text_p], texts)
        n = length(texts)

        for (tls_i = 1; tls_i <= length(tls); tls_i++) {
            tl = ((tls[tls_i] == "") ? "auto" : tls[tls_i])
            tl = substr(tl, 1 + (showPhonetically = index(tl, "@")))

            for (texts_i = 1; texts_i <= n; texts_i++) {
                text = texts[texts_i]
                for (key in percentEncoding)
                    gsub(key, percentEncoding[key], text)

                url = "http://translate.google.com/translate_a/t?client=t&ie=UTF-8&oe=UTF-8" \
                    "&sl=" sl "&tl=" tl "&text=" text

                print "GET " url |& httpService
                while ((httpService |& getline) > 0)
                    content = $0
                close(httpService)

                translation = phonetics = ""
                match(content, /\]\]/)
                s0 = substr(content, 3, RSTART - 2)
                while (s0 != "") {
                    match(s0, /\[[^\]]*\]/)
                    t0 = substr(s0, RSTART + 1, RLENGTH - 2)
                    s0 = substr(s0, RLENGTH + 2)

                    for (i = 0; t0 != ""; i++) {
                        match(t0, /[^\\]","/)
                        f[i] = substr(t0, 2, RSTART - 1)
                        t0 = substr(t0, RSTART + 3)
                    }

                    translation = translation f[0]
                    phonetics = phonetics ((f[2] != "") ? (f[2] " ") : f[0])
                }

                output = showPhonetically ? phonetics : translation
                output = gensub(/\\( |\\|\")/, "\\1", "g", output)
                output = gensub(/\\n/, "\n", "g", output)
                output = gensub(/(\") /, "\\1", "g", output)
                output = gensub(/ ('|;|:|\.|\,|\!|\?)/, "\\1", "g", output)
                printf output "\n"
            }
        }
    }
}

function help() {
    print "Usage: translate {[SL]=[TL]} TEXT|TEXT_FILENAME"
    print "       translate {[SL]=[TL1]+[TL2]+...} TEXT|TEXT_FILENAME"
    print "       translate TEXT1 TEXT2 ..."
    print
    print "TEXT: Source text (The text to be translated)"
    print "      Can also be the filename of a plain text file."
    print "  SL: Source language (The language of the source text)"
    print "  TL: Target language (The language to translate the source text into)"
    print "      Language codes as listed here:"
    print "    * http://developers.google.com/translate/v2/using_rest#language-params"
    print "      Ignore the code where you want the system to identify it for you."
    print "      Prefix the code with an ampersat @ to show the result phonetically."
}


A better Wiktionary processor

A better Wiktionary processor


AWK, this was really bad

AWK, this was really bad

#!/usr/bin/awk -f
# echo -e 'a\tb\na\tb\na\tc' | ./count.awk
BEGIN {
    FS = OFS = "\t";
}
NR == 1 {
    word1 = $1;
    word2 = $2;
}
("" word1 != $1) || ("" word2 != $2) {
    print word1, word2, count;
    word1 = $1;
    word2 = $2;
    count = 0;
}
{
    count++;
}
END {
    print word1, word2, count;
}

#!/usr/bin/awk -f
# echo -e 'a\tb\na\tb\na\tc' | ./count.awk
BEGIN {
    FS = OFS = "\t";
}
(!word1 || !word2) {
    word1 = $1;
    word2 = $2;
}
(word1 != $1 || word2 != $2) {
    print word1, word2, count;
    word1 = $1;
    word2 = $2;
    count = 0;
}
{
    count++;
}
END {
    print word1, word2, count;
}


ctx: recognize simple directory name in xterm's wm_name

### gather context data

# note: attr aren't available yet! upside is, `req -a ctx.foo=bar` can override this files code:

BEGIN {
    # interactive terminal
    if(attr["term"])
	setCtx("term", ENVIRON["TERM"])

    # x11 display
    if(ENVIRON["DISPLAY"]) {
	setCtx("display", ENVIRON["DISPLAY"])
	# x11 selection
	setCtx("sel", trim(btick("xclip -o")))
	# current x11 win id
	setCtx("xwin", word(btick("xprop -root _NET_ACTIVE_WINDOW"), 5))
	split_xprop(btick("xprop -id " get("ctx.xwin")), ctx, "ctx.xprop")
    }

    # x11 applications
    if(get("ctx.xprop.wm_class_name") == "Emacs") {
	setCtx("app.exe", "emacs")
	f = get("ctx.xprop.wm_name")
	split_file(f, ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "XTerm" &&
    (match(get("ctx.xprop.wm_name"), /^([^@]*)@([^:]*):(.*)$/, m) ||
     match(get("ctx.xprop.wm_name"), /^()()(.*)$/, m))) {
	setCtx("app.exe", "xterm")
	setCtx("app.user", m[1])
	setCtx("app.host", m[2])
	setCtx("app.dir", m[3])
	split_file(m[3], ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Rox" &&
    match(get("ctx.xprop.wm_name"), /^(.*) \+[ST]$/, m)) {
	setCtx("app.exe", "rox")
	f = gensub(/^~\//, HOME "/", 1, m[1])
	split_file(f, ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Sonata" &&
    match(get("ctx.xprop.wm_name"), /^(.*)\/([^/]*)$/, m)) {
	setCtx("app.exe", "sonata")
	setCtx("audio_time", m[3])
	setCtx("audio_total", m[4])
	f = "/home/rfigura/mnt/wirsing/home/rfigura/pub/audio/" m[1] "/" m[2]
	split_file(f, ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Hgtk" &&
    match(get("ctx.xprop.wm_name"), /^(.*) - (.*)$/, m)) {
	setCtx("app.exe", "hgtk")
	split_file(m[1], ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Surf")
	setCtx("app.url", get("ctx.xprop.url"))

    # working directory
    setCtx("wdir", ENVIRON["PWD"])
    if(get("ctx.file.dirname"))
	setCtx("wdir", get("ctx.file.dirname"))

    # current wifi
    split(btick("wpa_cli status"), m, "\n")
    for(i = 1; i <= length(m); ++i)
	if(match(m[i], /^([^=]*)=(.*)$/, a))
	    setCtx("wifi_" a[1], a[2])
}

### gather context data

# note: attr aren't available yet! upside is, `req -a ctx.foo=bar` can override this files code:

BEGIN {
    # interactive terminal
    if(attr["term"])
	setCtx("term", ENVIRON["TERM"])

    # x11 display
    if(ENVIRON["DISPLAY"]) {
	setCtx("display", ENVIRON["DISPLAY"])
	# x11 selection
	setCtx("sel", trim(btick("xclip -o")))
	# current x11 win id
	setCtx("xwin", word(btick("xprop -root _NET_ACTIVE_WINDOW"), 5))
	split_xprop(btick("xprop -id " get("ctx.xwin")), ctx, "ctx.xprop")
    }

    # x11 applications
    if(get("ctx.xprop.wm_class_name") == "Emacs") {
	setCtx("app.exe", "emacs")
	f = get("ctx.xprop.wm_name")
	split_file(f, ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "XTerm" &&
    match(get("ctx.xprop.wm_name"), /^([^@]*)@([^:]*):(.*)$/, m)) {
	setCtx("app.exe", "xterm")
	setCtx("app.user", m[1])
	setCtx("app.host", m[2])
	setCtx("app.dir", m[3])
	split_file(m[3], ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Rox" &&
    match(get("ctx.xprop.wm_name"), /^(.*) \+[ST]$/, m)) {
	setCtx("app.exe", "rox")
	f = gensub(/^~\//, HOME "/", 1, m[1])
	split_file(f, ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Sonata" &&
    match(get("ctx.xprop.wm_name"), /^(.*)\/([^/]*)$/, m)) {
	setCtx("app.exe", "sonata")
	setCtx("audio_time", m[3])
	setCtx("audio_total", m[4])
	f = "/home/rfigura/mnt/wirsing/home/rfigura/pub/audio/" m[1] "/" m[2]
	split_file(f, ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Hgtk" &&
    match(get("ctx.xprop.wm_name"), /^(.*) - (.*)$/, m)) {
	setCtx("app.exe", "hgtk")
	split_file(m[1], ctx, "ctx.file")
    }
    if(get("ctx.xprop.wm_class_name") == "Surf")
	setCtx("app.url", get("ctx.xprop.url"))

    # working directory
    setCtx("wdir", ENVIRON["PWD"])
    if(get("ctx.file.dirname"))
	setCtx("wdir", get("ctx.file.dirname"))

    # current wifi
    split(btick("wpa_cli status"), m, "\n")
    for(i = 1; i <= length(m); ++i)
	if(match(m[i], /^([^=]*)=(.*)$/, a))
	    setCtx("wifi_" a[1], a[2])
}