<commit_before>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package flow

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"

	"github.com/redhat-cip/skydive/logging"
)

func LayerFlow(l gopacket.Layer) gopacket.Flow {
	switch l.(type) {
	case gopacket.LinkLayer:
		return l.(gopacket.LinkLayer).LinkFlow()
	case gopacket.NetworkLayer:
		return l.(gopacket.NetworkLayer).NetworkFlow()
	case gopacket.TransportLayer:
		return l.(gopacket.TransportLayer).TransportFlow()
	}
	logging.GetLogger().Critical(fmt.Sprint("Unknown gopacket.Layer %T", l))
	return gopacket.Flow{}
}

type FlowKey struct {
	net, transport uint64
}

func (key FlowKey) fillFromGoPacket(p *gopacket.Packet) FlowKey {
	key.net = LayerFlow((*p).NetworkLayer()).FastHash()
	key.transport = LayerFlow((*p).TransportLayer()).FastHash()
	return key
}

func (key FlowKey) String() string {
	return fmt.Sprintf("%x-%x", key.net, key.transport)
}

func (flow *Flow) fillFromGoPacket(packet *gopacket.Packet) error {
	/* Continue if no ethernet layer */
	ethernetLayer := (*packet).Layer(layers.LayerTypeEthernet)
	_, ok := ethernetLayer.(*layers.Ethernet)
	if !ok {
		return errors.New("Unable to decode the ethernet layer")
	}

	newFlow := false
	fs := flow.GetStatistics()
	now := time.Now().Unix() //(*packet).Metadata().Timestamp.Unix()
	if fs == nil {
		newFlow = true
		fs = NewFlowStatistics()
		fs.Start = now
		fs.newEthernetEndpointStatistics(packet)
		fs.newIPV4EndpointStatistics(packet)
		fs.newTransportEndpointStatistics(packet)
		flow.Statistics = fs
	}
	fs.Last = now
	fs.updateEthernetFromGoPacket(packet)
	fs.updateIPV4FromGoPacket(packet)
	fs.updateTransportFromGoPacket(packet)

	if newFlow {
		hasher := sha1.New()
		path := ""
		for i, layer := range (*packet).Layers() {
			if i > 0 {
				path += "/"
			}
			path += layer.LayerType().String()
		}
		flow.LayersPath = path
		hasher.Write([]byte(flow.LayersPath))

		/* Generate an flow UUID */
		for _, ep := range fs.GetEndpoints() {
			hasher.Write([]byte(ep.AB.Value))
			hasher.Write([]byte(ep.BA.Value))
		}
		flow.UUID = hex.EncodeToString(hasher.Sum(nil))
	}
	return nil
}

func FromData(data []byte) (*Flow, error) {
	flow := new(Flow)

	err := proto.Unmarshal(data, flow)
	if err != nil {
		return nil, err
	}

	return flow, nil
}

func (flow *Flow) GetData() ([]byte, error) {
	data, err := proto.Marshal(flow)
	if err != nil {
		return []byte{}, err
	}

	return data, nil
}

func FLowsFromSFlowSample(ft *FlowTable, sample *layers.SFlowFlowSample, probePath *string) []*Flow {
	flows := []*Flow{}

	for _, rec := range sample.Records {

		/* FIX(safchain): just keeping the raw packet for now */
		record, ok := rec.(layers.SFlowRawPacketFlowRecord)
		if !ok {
			continue
		}

		packet := &record.Header
		key := (FlowKey{}).fillFromGoPacket(packet)
		flow, new := ft.GetFlow(key.String(), packet)
		if new {
			flow.ProbeGraphPath = *probePath

			flow.DebugTimestamp = time.Now().Unix()
			flow.DebugKeyNet = key.net
			flow.DebugKeyTransport = key.transport
		}
		flow.fillFromGoPacket(packet)
		flows = append(flows, flow)
	}

	return flows
}
<commit_msg>[debug] check SFlowRawPacketFlowRecord type<commit_after>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package flow

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"

	"github.com/redhat-cip/skydive/logging"
)

func LayerFlow(l gopacket.Layer) gopacket.Flow {
	switch l.(type) {
	case gopacket.LinkLayer:
		return l.(gopacket.LinkLayer).LinkFlow()
	case gopacket.NetworkLayer:
		return l.(gopacket.NetworkLayer).NetworkFlow()
	case gopacket.TransportLayer:
		return l.(gopacket.TransportLayer).TransportFlow()
	}
	logging.GetLogger().Critical(fmt.Sprint("Unknown gopacket.Layer %T", l))
	return gopacket.Flow{}
}

type FlowKey struct {
	net, transport uint64
}

func (key FlowKey) fillFromGoPacket(p *gopacket.Packet) FlowKey {
	key.net = LayerFlow((*p).NetworkLayer()).FastHash()
	key.transport = LayerFlow((*p).TransportLayer()).FastHash()
	return key
}

func (key FlowKey) String() string {
	return fmt.Sprintf("%x-%x", key.net, key.transport)
}

func (flow *Flow) fillFromGoPacket(packet *gopacket.Packet) error {
	/* Continue if no ethernet layer */
	ethernetLayer := (*packet).Layer(layers.LayerTypeEthernet)
	_, ok := ethernetLayer.(*layers.Ethernet)
	if !ok {
		return errors.New("Unable to decode the ethernet layer")
	}

	newFlow := false
	fs := flow.GetStatistics()
	now := time.Now().Unix() //(*packet).Metadata().Timestamp.Unix()
	if fs == nil {
		newFlow = true
		fs = NewFlowStatistics()
		fs.Start = now
		fs.newEthernetEndpointStatistics(packet)
		fs.newIPV4EndpointStatistics(packet)
		fs.newTransportEndpointStatistics(packet)
		flow.Statistics = fs
	}
	fs.Last = now
	fs.updateEthernetFromGoPacket(packet)
	fs.updateIPV4FromGoPacket(packet)
	fs.updateTransportFromGoPacket(packet)

	if newFlow {
		hasher := sha1.New()
		path := ""
		for i, layer := range (*packet).Layers() {
			if i > 0 {
				path += "/"
			}
			path += layer.LayerType().String()
		}
		flow.LayersPath = path
		hasher.Write([]byte(flow.LayersPath))

		/* Generate an flow UUID */
		for _, ep := range fs.GetEndpoints() {
			hasher.Write([]byte(ep.AB.Value))
			hasher.Write([]byte(ep.BA.Value))
		}
		flow.UUID = hex.EncodeToString(hasher.Sum(nil))
	}
	return nil
}

func FromData(data []byte) (*Flow, error) {
	flow := new(Flow)

	err := proto.Unmarshal(data, flow)
	if err != nil {
		return nil, err
	}

	return flow, nil
}

func (flow *Flow) GetData() ([]byte, error) {
	data, err := proto.Marshal(flow)
	if err != nil {
		return []byte{}, err
	}

	return data, nil
}

func FLowsFromSFlowSample(ft *FlowTable, sample *layers.SFlowFlowSample, probePath *string) []*Flow {
	flows := []*Flow{}

	for _, rec := range sample.Records {

		/* FIX(safchain): just keeping the raw packet for now */
		record, ok := rec.(layers.SFlowRawPacketFlowRecord)
		if !ok {
			logging.GetLogger().Critical("1st layer is not SFlowRawPacketFlowRecord type")
			continue
		}

		packet := &record.Header
		key := (FlowKey{}).fillFromGoPacket(packet)
		flow, new := ft.GetFlow(key.String(), packet)
		if new {
			flow.ProbeGraphPath = *probePath

			flow.DebugTimestamp = time.Now().Unix()
			flow.DebugKeyNet = key.net
			flow.DebugKeyTransport = key.transport
		}
		flow.fillFromGoPacket(packet)
		flows = append(flows, flow)
	}

	return flows
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package flow

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"

	"github.com/redhat-cip/skydive/logging"
)

func LayerFlow(l gopacket.Layer) gopacket.Flow {
	switch l.(type) {
	case gopacket.LinkLayer:
		return l.(gopacket.LinkLayer).LinkFlow()
	case gopacket.NetworkLayer:
		return l.(gopacket.NetworkLayer).NetworkFlow()
	case gopacket.TransportLayer:
		return l.(gopacket.TransportLayer).TransportFlow()
	}
	return gopacket.Flow{}
}

type FlowKey struct {
	net, transport uint64
}

func (key FlowKey) fillFromGoPacket(p *gopacket.Packet) FlowKey {
	key.net = LayerFlow((*p).NetworkLayer()).FastHash()
	key.transport = LayerFlow((*p).TransportLayer()).FastHash()
	return key
}

func (key FlowKey) String() string {
	return fmt.Sprintf("%x-%x", key.net, key.transport)
}

func (flow *Flow) fillFromGoPacket(packet *gopacket.Packet) error {
	/* Continue if no ethernet layer */
	ethernetLayer := (*packet).Layer(layers.LayerTypeEthernet)
	_, ok := ethernetLayer.(*layers.Ethernet)
	if !ok {
		return errors.New("Unable to decode the ethernet layer")
	}

	newFlow := false
	fs := flow.GetStatistics()
	now := time.Now().Unix() //(*packet).Metadata().Timestamp.Unix()
	if fs == nil {
		newFlow = true
		fs = NewFlowStatistics()
		fs.Start = now
		fs.newEthernetEndpointStatistics(packet)
		fs.newIPV4EndpointStatistics(packet)
		fs.newTransportEndpointStatistics(packet)
		flow.Statistics = fs
	}
	fs.Last = now
	fs.updateEthernetFromGoPacket(packet)
	fs.updateIPV4FromGoPacket(packet)
	fs.updateTransportFromGoPacket(packet)

	if newFlow {
		hasher := sha1.New()
		path := ""
		for i, layer := range (*packet).Layers() {
			if i > 0 {
				path += "/"
			}
			path += layer.LayerType().String()
		}
		flow.LayersPath = path
		hasher.Write([]byte(flow.LayersPath))

		/* Generate an flow UUID */
		for _, ep := range fs.GetEndpoints() {
			hasher.Write([]byte(ep.AB.Value))
			hasher.Write([]byte(ep.BA.Value))
		}
		flow.UUID = hex.EncodeToString(hasher.Sum(nil))
	}
	return nil
}

func FromData(data []byte) (*Flow, error) {
	flow := new(Flow)

	err := proto.Unmarshal(data, flow)
	if err != nil {
		return nil, err
	}

	return flow, nil
}

func (flow *Flow) GetData() ([]byte, error) {
	data, err := proto.Marshal(flow)
	if err != nil {
		return []byte{}, err
	}

	return data, nil
}

func FLowsFromSFlowSample(ft *FlowTable, sample *layers.SFlowFlowSample, probePath *string) []*Flow {
	flows := []*Flow{}

	for _, rec := range sample.Records {

		/* FIX(safchain): just keeping the raw packet for now */
		record, ok := rec.(layers.SFlowRawPacketFlowRecord)
		if !ok {
			logging.GetLogger().Critical("1st layer is not SFlowRawPacketFlowRecord type")
			continue
		}

		packet := &record.Header
		key := (FlowKey{}).fillFromGoPacket(packet)
		flow, new := ft.GetFlow(key.String())
		if new {
			flow.ProbeGraphPath = ""
			if probePath != nil {
				flow.ProbeGraphPath = *probePath
			}
		}
		flow.fillFromGoPacket(packet)
		flows = append(flows, flow)
	}

	return flows
}
<commit_msg>Adapt log of SFlow packet parser, log as critical only non SFlow packets<commit_after>/*
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package flow

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"

	"github.com/redhat-cip/skydive/logging"
)

func LayerFlow(l gopacket.Layer) gopacket.Flow {
	switch l.(type) {
	case gopacket.LinkLayer:
		return l.(gopacket.LinkLayer).LinkFlow()
	case gopacket.NetworkLayer:
		return l.(gopacket.NetworkLayer).NetworkFlow()
	case gopacket.TransportLayer:
		return l.(gopacket.TransportLayer).TransportFlow()
	}
	return gopacket.Flow{}
}

type FlowKey struct {
	net, transport uint64
}

func (key FlowKey) fillFromGoPacket(p *gopacket.Packet) FlowKey {
	key.net = LayerFlow((*p).NetworkLayer()).FastHash()
	key.transport = LayerFlow((*p).TransportLayer()).FastHash()
	return key
}

func (key FlowKey) String() string {
	return fmt.Sprintf("%x-%x", key.net, key.transport)
}

func (flow *Flow) fillFromGoPacket(packet *gopacket.Packet) error {
	/* Continue if no ethernet layer */
	ethernetLayer := (*packet).Layer(layers.LayerTypeEthernet)
	_, ok := ethernetLayer.(*layers.Ethernet)
	if !ok {
		return errors.New("Unable to decode the ethernet layer")
	}

	newFlow := false
	fs := flow.GetStatistics()
	now := time.Now().Unix() //(*packet).Metadata().Timestamp.Unix()
	if fs == nil {
		newFlow = true
		fs = NewFlowStatistics()
		fs.Start = now
		fs.newEthernetEndpointStatistics(packet)
		fs.newIPV4EndpointStatistics(packet)
		fs.newTransportEndpointStatistics(packet)
		flow.Statistics = fs
	}
	fs.Last = now
	fs.updateEthernetFromGoPacket(packet)
	fs.updateIPV4FromGoPacket(packet)
	fs.updateTransportFromGoPacket(packet)

	if newFlow {
		hasher := sha1.New()
		path := ""
		for i, layer := range (*packet).Layers() {
			if i > 0 {
				path += "/"
			}
			path += layer.LayerType().String()
		}
		flow.LayersPath = path
		hasher.Write([]byte(flow.LayersPath))

		/* Generate an flow UUID */
		for _, ep := range fs.GetEndpoints() {
			hasher.Write([]byte(ep.AB.Value))
			hasher.Write([]byte(ep.BA.Value))
		}
		flow.UUID = hex.EncodeToString(hasher.Sum(nil))
	}
	return nil
}

func FromData(data []byte) (*Flow, error) {
	flow := new(Flow)

	err := proto.Unmarshal(data, flow)
	if err != nil {
		return nil, err
	}

	return flow, nil
}

func (flow *Flow) GetData() ([]byte, error) {
	data, err := proto.Marshal(flow)
	if err != nil {
		return []byte{}, err
	}

	return data, nil
}

func FLowsFromSFlowSample(ft *FlowTable, sample *layers.SFlowFlowSample, probePath *string) []*Flow {
	flows := []*Flow{}

	for _, rec := range sample.Records {

		/* FIX(safchain): just keeping the raw packet for now */
		switch rec.(type) {
		case layers.SFlowRawPacketFlowRecord:
			/* We only support RawPacket from SFlow probe */
		case layers.SFlowExtendedSwitchFlowRecord:
			logging.GetLogger().Debug("1st layer is not SFlowRawPacketFlowRecord type")
			continue
		default:
			logging.GetLogger().Critical("1st layer is not a SFlow supported type")
			continue
		}

		record := rec.(layers.SFlowRawPacketFlowRecord)

		packet := &record.Header
		key := (FlowKey{}).fillFromGoPacket(packet)
		flow, new := ft.GetFlow(key.String())
		if new {
			flow.ProbeGraphPath = ""
			if probePath != nil {
				flow.ProbeGraphPath = *probePath
			}
		}
		flow.fillFromGoPacket(packet)
		flows = append(flows, flow)
	}

	return flows
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.259"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.226", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.260 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.260"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.227", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Michael Smith <mikejsmitty@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"os"

	"github.com/mikesmitty/curse/jinx/jinxlib"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "jinx",
	Short: "SSH certificate client",
	Long: `JINX is a client to the CURSE SSH certificate authority.
It is used to provide short-lived SSH certificates in place of semi-permanent SSH pubkeys
in authorized_keys files, which are difficult to manage at scale and over long periods
of time.`,
	Run: func(cmd *cobra.Command, args []string) {
		jinxlib.Jinx(args)
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	//RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.jinx.yaml)")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	//if cfgFile != "" { // enable ability to specify config file via flag
	//	viper.SetConfigFile(cfgFile)
	//}

	viper.SetConfigName("jinx") // name of config file (without extension)
	viper.AddConfigPath("/etc/jinx")
	viper.AddConfigPath("$HOME/.jinx/")

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		//fmt.Println("Using config file:", viper.ConfigFileUsed())
	}

	viper.SetDefault("autogenkeys", true)
	viper.SetDefault("bastionip", "")
	viper.SetDefault("insecure", false)
	viper.SetDefault("keygenbitsize", 2048)
	viper.SetDefault("keygenpubkey", "$HOME/.ssh/id_jinx.pub")
	viper.SetDefault("keygentype", "ed25519")
	viper.SetDefault("pubkey", "$HOME/.ssh/id_ed25519.pub")
	viper.SetDefault("sshuser", "root") // FIXME Need to revisit this?
	viper.SetDefault("sslcafile", "/etc/jinx/ca.crt")
	viper.SetDefault("sslcertfile", "$HOME/.jinx/client.crt")
	viper.SetDefault("sslkeyfile", "$HOME/.jinx/client.key")
	viper.SetDefault("timeout", 30)
	viper.SetDefault("url", "https://localhost/")
}
<commit_msg>Add missing explicit default mutualauth setting<commit_after>// Copyright © 2017 Michael Smith <mikejsmitty@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"os"

	"github.com/mikesmitty/curse/jinx/jinxlib"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "jinx",
	Short: "SSH certificate client",
	Long: `JINX is a client to the CURSE SSH certificate authority.
It is used to provide short-lived SSH certificates in place of semi-permanent SSH pubkeys
in authorized_keys files, which are difficult to manage at scale and over long periods
of time.`,
	Run: func(cmd *cobra.Command, args []string) {
		jinxlib.Jinx(args)
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	//RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.jinx.yaml)")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	//if cfgFile != "" { // enable ability to specify config file via flag
	//	viper.SetConfigFile(cfgFile)
	//}

	viper.SetConfigName("jinx") // name of config file (without extension)
	viper.AddConfigPath("/etc/jinx")
	viper.AddConfigPath("$HOME/.jinx/")

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		//fmt.Println("Using config file:", viper.ConfigFileUsed())
	}

	viper.SetDefault("autogenkeys", true)
	viper.SetDefault("bastionip", "")
	viper.SetDefault("insecure", false)
	viper.SetDefault("keygenbitsize", 2048)
	viper.SetDefault("keygenpubkey", "$HOME/.ssh/id_jinx.pub")
	viper.SetDefault("keygentype", "ed25519")
	viper.SetDefault("mutualauth", false)
	viper.SetDefault("pubkey", "$HOME/.ssh/id_ed25519.pub")
	viper.SetDefault("sshuser", "root") // FIXME Need to revisit this?
	viper.SetDefault("sslcafile", "/etc/jinx/ca.crt")
	viper.SetDefault("sslcertfile", "$HOME/.jinx/client.crt")
	viper.SetDefault("sslkeyfile", "$HOME/.jinx/client.key")
	viper.SetDefault("timeout", 30)
	viper.SetDefault("url", "https://localhost/")
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.229"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.196", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.230 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.230"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.197", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package jobq

import (
	"errors"
	"testing"
	"time"
)

var (
	errMock = errors.New("mock error")
)

// TODO; add some tests and benchmark.

type T struct {
	Size     int
	Len      int
	Expected error
}

var tests = []T{
	T{1, 2, nil},
	T{7, 6, nil},
	T{1, -1, errInvalidQueueSize},
	T{-1, 1, errInvalidWorkerSize},
}

func TestNew(t *testing.T) {
	for _, m := range tests {
		errc := make(chan error, 1)
		_, err := New(m.Size, m.Len, errc)
		if err != m.Expected {
			t.Fail()
		}
	}
}

func TestWork(t *testing.T) {
	errc := make(chan error, 1)
	go func() {
		for err := range errc {
			if err != errMock {
				t.Fail()
			}
		}
	}()
	jq, err := New(10, 20, errc)
	if err != nil {
		t.Fail()
	}
	jq.Add(func() error {
		return errMock
	})
	time.Sleep(time.Second)
	jq.Stop()
}
<commit_msg>test fmt.<commit_after>package jobq

import (
	"errors"
	"testing"
	"time"
)

var (
	errMock = errors.New("mock error")
)

// T struct.
type T struct {
	Size     int
	Len      int
	Expected error
}

var tests = []T{
	T{1, 2, nil},
	T{7, 6, nil},
	T{1, -1, errInvalidQueueSize},
	T{-1, 1, errInvalidWorkerSize},
}

// TestNew tests invalid inputs.
func TestNew(t *testing.T) {
	for _, m := range tests {
		errc := make(chan error, 1)
		_, err := New(m.Size, m.Len, errc)
		if err != m.Expected {
			t.Fail()
		}
	}
}

// TestWork it needs to be proved running.
func TestWork(t *testing.T) {
	errc := make(chan error, 1)
	go func() {
		for err := range errc {
			if err != errMock {
				t.Fail()
			}
		}
	}()
	jq, err := New(10, 20, errc)
	if err != nil {
		t.Fail()
	}
	jq.Add(func() error {
		return errMock
	})
	time.Sleep(time.Second)
	jq.Stop()
}
<|endoftext|>


<commit_before>package ask

import (
	"fmt"
	"math"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/byuoitav/common/db"
	"github.com/byuoitav/common/log"
	"github.com/byuoitav/common/nerr"
	"github.com/byuoitav/common/v2/events"
	"github.com/byuoitav/device-monitoring/localsystem"
	ping "github.com/sparrc/go-ping"
)

// PingJob is a job that pings each of the devices in the room and reports whether or not they are connected.
type PingJob struct {
	Count    int
	Interval time.Duration
	Timeout  time.Duration
}

// PingResult is returned after running the ping job for a room
type PingResult struct {
	Successful   []devicePingResult `json:"successful,omitempty"`
	Unsuccessful []devicePingResult `json:"unsuccessful,omitempty"`
}

type devicePingResult struct {
	DeviceID string `json:"deviceID"`
	Error    string `json:"error"`

	PacketsReceived  int     `json:"packets-received"`
	PacketsSent      int     `json:"packets-sent"`
	PacketLoss       float64 `json:"packet-loss"`
	IPAddr           net.IP  `json:"ip"`
	Addr             string  `json:"address"`
	AverageRoundTrip string  `json:"average-round-trip"`
}

// Run runs the job.
func (p *PingJob) Run(ctx interface{}, eventWrite chan events.Event) interface{} {
	log.L.Infof("Getting pinggable status of devices in room %s", localsystem.MustRoomID())

	devices, err := db.GetDB().GetDevicesByRoom(localsystem.MustRoomID())
	if err != nil {
		return nerr.Translate(err).Addf("unable to get devices in room %v: %v", localsystem.MustRoomID(), err)
	}

	ret := PingResult{}
	resultChan := make(chan devicePingResult, len(devices))

	wg := &sync.WaitGroup{}
	for i := range devices {
		if len(devices[i].Address) == 0 || strings.EqualFold(devices[i].Address, "0.0.0.0") {
			continue
		}
		wg.Add(1)

		go func(index int) {
			defer wg.Done()

			result := devicePingResult{
				DeviceID: devices[index].ID,
			}

			// build pinger
			pinger, err := ping.NewPinger(devices[index].Address)
			if err != nil {
				result.Error = fmt.Sprintf("unable to create pinger for device %v (address: %v): %v", devices[index].ID, devices[index].Address, err)
				resultChan <- result
				return
			}

			pinger.Count = p.Count
			pinger.Interval = p.Interval
			pinger.Timeout = p.Timeout
			pinger.SetPrivileged(true)

			// run ping test
			pinger.Run()

			// parse results
			result.PacketsReceived = pinger.Statistics().PacketsRecv
			result.PacketsSent = pinger.Statistics().PacketsSent
			result.PacketLoss = pinger.Statistics().PacketLoss
			result.IPAddr = pinger.Statistics().IPAddr.IP
			result.Addr = pinger.Statistics().Addr
			result.AverageRoundTrip = fmt.Sprintf("%dms", pinger.Statistics().AvgRtt/time.Millisecond)

			// add in errors
			if math.IsNaN(result.PacketLoss) {
				result.PacketLoss = -1
				result.Error = fmt.Sprintf("unknown error, but packet loss was NaN")
			} else if result.PacketLoss == 100 {
				result.Error = fmt.Sprintf("no packets were successful")
			}

			resultChan <- result
		}(i)
	}

	// timeout while waiting for waitgroup to finish
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(20 * time.Second):
	}

	close(resultChan)

	// build a generic event to send for every device
	event := events.Event{
		GeneratingSystem: localsystem.MustSystemID(),
		Timestamp:        time.Now(),
		EventTags: []string{
			events.Heartbeat,
		},
		AffectedRoom: events.GenerateBasicRoomInfo(localsystem.MustRoomID()),
	}

	resultCount := len(resultChan)

	for result := range resultChan {
		if len(result.Error) > 0 {
			ret.Unsuccessful = append(ret.Unsuccessful, result)
			log.L.Infof("error pinging %v: %v", result.DeviceID, result.Error)
		} else {
			ret.Successful = append(ret.Successful, result)

			event.TargetDevice = events.GenerateBasicDeviceInfo(result.DeviceID)
			event.Key = "last-heartbeat"
			event.Value = time.Now().Format(time.RFC3339)
			eventWrite <- event
		}
	}

	// send an event for my own heartbeat
	event.TargetDevice = events.GenerateBasicDeviceInfo(localsystem.MustSystemID())
	event.Key = "last-heartbeat"
	event.Value = time.Now().Format(time.RFC3339)
	eventWrite <- event

	if len(ret.Unsuccessful) == resultCount && resultCount > 0 {
		return nerr.Create(fmt.Sprintf("every single ping was unsuccessful. error: %s", ret.Unsuccessful[0].Error), "error")
	}

	return ret
}
<commit_msg>try this?<commit_after>package ask

import (
	"fmt"
	"math"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/byuoitav/common/db"
	"github.com/byuoitav/common/log"
	"github.com/byuoitav/common/nerr"
	"github.com/byuoitav/common/v2/events"
	"github.com/byuoitav/device-monitoring/localsystem"
	ping "github.com/sparrc/go-ping"
)

// PingJob is a job that pings each of the devices in the room and reports whether or not they are connected.
type PingJob struct {
	Count    int
	Interval time.Duration
	Timeout  time.Duration
}

// PingResult is returned after running the ping job for a room
type PingResult struct {
	Successful   []devicePingResult `json:"successful,omitempty"`
	Unsuccessful []devicePingResult `json:"unsuccessful,omitempty"`
}

type devicePingResult struct {
	DeviceID string `json:"deviceID"`
	Error    string `json:"error"`

	PacketsReceived  int     `json:"packets-received"`
	PacketsSent      int     `json:"packets-sent"`
	PacketLoss       float64 `json:"packet-loss"`
	IPAddr           net.IP  `json:"ip"`
	Addr             string  `json:"address"`
	AverageRoundTrip string  `json:"average-round-trip"`
}

// Run runs the job.
func (p *PingJob) Run(ctx interface{}, eventWrite chan events.Event) interface{} {
	log.L.Infof("Getting pinggable status of devices in room %s", localsystem.MustRoomID())

	devices, err := db.GetDB().GetDevicesByRoom(localsystem.MustRoomID())
	if err != nil {
		return nerr.Translate(err).Addf("unable to get devices in room %v: %v", localsystem.MustRoomID(), err)
	}

	ret := PingResult{}
	resultChan := make(chan devicePingResult, len(devices))

	wg := &sync.WaitGroup{}
	for i := range devices {
		if len(devices[i].Address) == 0 || strings.EqualFold(devices[i].Address, "0.0.0.0") {
			continue
		}
		wg.Add(1)

		go func(index int) {
			defer wg.Done()

			result := devicePingResult{
				DeviceID: devices[index].ID,
			}

			// build pinger
			pinger, err := ping.NewPinger(devices[index].Address)
			if err != nil {
				result.Error = fmt.Sprintf("unable to create pinger for device %v (address: %v): %v", devices[index].ID, devices[index].Address, err)
				resultChan <- result
				return
			}

			pinger.Count = p.Count
			pinger.Interval = p.Interval
			pinger.Timeout = p.Timeout
			pinger.SetPrivileged(true)

			// run ping test with timeout
			skip := false
			done := make(chan struct{})
			go func() {
				pinger.Run()
				close(done)
			}()

			select {
			case <-done:
			case <-time.After(p.Timeout):
				pinger.Stop()
				skip = true
			}

			if skip {
				result.PacketLoss = -1
				result.Error = fmt.Sprintf("ping timed out")
				resultChan <- result
				return
			}

			// parse results
			result.PacketsReceived = pinger.Statistics().PacketsRecv
			result.PacketsSent = pinger.Statistics().PacketsSent
			result.PacketLoss = pinger.Statistics().PacketLoss
			result.IPAddr = pinger.Statistics().IPAddr.IP
			result.Addr = pinger.Statistics().Addr
			result.AverageRoundTrip = fmt.Sprintf("%dms", pinger.Statistics().AvgRtt/time.Millisecond)

			// add in errors
			if math.IsNaN(result.PacketLoss) {
				result.PacketLoss = -1
				result.Error = fmt.Sprintf("unknown error, but packet loss was NaN")
			} else if result.PacketLoss == 100 {
				result.Error = fmt.Sprintf("no packets were successful")
			}

			resultChan <- result
		}(i)
	}

	wg.Wait()
	close(resultChan)

	// build a generic event to send for every device
	event := events.Event{
		GeneratingSystem: localsystem.MustSystemID(),
		Timestamp:        time.Now(),
		EventTags: []string{
			events.Heartbeat,
		},
		AffectedRoom: events.GenerateBasicRoomInfo(localsystem.MustRoomID()),
	}

	resultCount := len(resultChan)

	for result := range resultChan {
		if len(result.Error) > 0 {
			ret.Unsuccessful = append(ret.Unsuccessful, result)
			log.L.Infof("error pinging %v: %v", result.DeviceID, result.Error)
		} else {
			ret.Successful = append(ret.Successful, result)

			event.TargetDevice = events.GenerateBasicDeviceInfo(result.DeviceID)
			event.Key = "last-heartbeat"
			event.Value = time.Now().Format(time.RFC3339)
			eventWrite <- event
		}
	}

	// send an event for my own heartbeat
	event.TargetDevice = events.GenerateBasicDeviceInfo(localsystem.MustSystemID())
	event.Key = "last-heartbeat"
	event.Value = time.Now().Format(time.RFC3339)
	eventWrite <- event

	if len(ret.Unsuccessful) == resultCount && resultCount > 0 {
		return nerr.Create(fmt.Sprintf("every single ping was unsuccessful. error: %s", ret.Unsuccessful[0].Error), "error")
	}

	return ret
}
<|endoftext|>


<commit_before>package prefixed

import (
	"bytes"
	"fmt"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/mgutz/ansi"
)

const reset = ansi.Reset

var (
	baseTimestamp time.Time
	isTerminal    bool
)

func init() {
	baseTimestamp = time.Now()
	isTerminal = logrus.IsTerminal()
}

func miniTS() int {
	return int(time.Since(baseTimestamp) / time.Second)
}

type TextFormatter struct {
	// Set to true to bypass checking for a TTY before outputting colors.
	ForceColors bool

	// Force disabling colors.
	DisableColors bool

	// Disable timestamp logging. useful when output is redirected to logging
	// system that already adds timestamps.
	DisableTimestamp bool

	// Enable logging of just the time passed since beginning of execution.
	ShortTimestamp bool

	// Timestamp format to use for display when a full timestamp is printed.
	TimestampFormat string

	// The fields are sorted by default for a consistent output. For applications
	// that log extremely frequently and don't use the JSON formatter this may not
	// be desired.
	DisableSorting bool

	// Pad msg field with spaces on the right for display.
	// The value for this parameter will be the size of padding.
	// Its default value is zero, which means no padding will be applied for msg.
	SpacePadding int
}

func (f *TextFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	var keys []string = make([]string, 0, len(entry.Data))
	for k := range entry.Data {
		if k != "prefix" {
			keys = append(keys, k)
		}
	}

	if !f.DisableSorting {
		sort.Strings(keys)
	}

	b := &bytes.Buffer{}

	prefixFieldClashes(entry.Data)

	isColorTerminal := isTerminal && (runtime.GOOS != "windows")
	isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors

	timestampFormat := f.TimestampFormat
	if timestampFormat == "" {
		timestampFormat = time.Stamp
	}
	if isColored {
		f.printColored(b, entry, keys, timestampFormat)
	} else {
		if !f.DisableTimestamp {
			f.appendKeyValue(b, "time", entry.Time.Format(timestampFormat))
		}
		f.appendKeyValue(b, "level", entry.Level.String())
		if entry.Message != "" {
			f.appendKeyValue(b, "msg", entry.Message)
		}
		for _, key := range keys {
			f.appendKeyValue(b, key, entry.Data[key])
		}
	}

	b.WriteByte('\n')
	return b.Bytes(), nil
}

func (f *TextFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry, keys []string, timestampFormat string) {
	var levelColor string
	var levelText string
	switch entry.Level {
	case logrus.InfoLevel:
		levelColor = ansi.Green
	case logrus.WarnLevel:
		levelColor = ansi.Yellow
	case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
		levelColor = ansi.Red
	default:
		levelColor = ansi.Blue
	}

	if entry.Level != logrus.WarnLevel {
		levelText = strings.ToUpper(entry.Level.String())
	} else {
		levelText = "WARN"
	}

	prefix := ""
	message := entry.Message

	if prefixValue, ok := entry.Data["prefix"]; ok {
		prefix = fmt.Sprint(" ", ansi.Cyan, prefixValue, ":", reset)
	} else {
		prefixValue, trimmedMsg := extractPrefix(entry.Message)
		if len(prefixValue) > 0 {
			prefix = fmt.Sprint(" ", ansi.Cyan, prefixValue, ":", reset)
			message = trimmedMsg
		}
	}

	messageFormat := "%s"
	if f.SpacePadding != 0 {
		messageFormat = fmt.Sprintf("%%-%ds", f.SpacePadding)
	}

	if f.ShortTimestamp {
		fmt.Fprintf(b, "%s[%04d]%s %s%+5s%s%s "+messageFormat, ansi.LightBlack, miniTS(), reset, levelColor, levelText, reset, prefix, message)
	} else {
		fmt.Fprintf(b, "%s[%s]%s %s%+5s%s%s "+messageFormat, ansi.LightBlack, entry.Time.Format(timestampFormat), reset, levelColor, levelText, reset, prefix, message)
	}
	for _, k := range keys {
		v := entry.Data[k]
		fmt.Fprintf(b, " %s%s%s=%+v", levelColor, k, reset, v)
	}
}

func needsQuoting(text string) bool {
	for _, ch := range text {
		if !((ch >= 'a' && ch <= 'z') ||
			(ch >= 'A' && ch <= 'Z') ||
			(ch >= '0' && ch <= '9') ||
			ch == '-' || ch == '.') {
			return false
		}
	}
	return true
}

func extractPrefix(msg string) (string, string) {
	prefix := ""
	regex := regexp.MustCompile("^\\[(.*?)\\]")
	if regex.MatchString(msg) {
		match := regex.FindString(msg)
		prefix, msg = match[1:len(match)-1], strings.TrimSpace(msg[len(match):])
	}
	return prefix, msg
}

func (f *TextFormatter) appendKeyValue(b *bytes.Buffer, key string, value interface{}) {
	b.WriteString(key)
	b.WriteByte('=')

	switch value := value.(type) {
	case string:
		if needsQuoting(value) {
			b.WriteString(value)
		} else {
			fmt.Fprintf(b, "%q", value)
		}
	case error:
		errmsg := value.Error()
		if needsQuoting(errmsg) {
			b.WriteString(errmsg)
		} else {
			fmt.Fprintf(b, "%q", value)
		}
	default:
		fmt.Fprint(b, value)
	}

	b.WriteByte(' ')
}

func prefixFieldClashes(data logrus.Fields) {
	_, ok := data["time"]
	if ok {
		data["fields.time"] = data["time"]
	}
	_, ok = data["msg"]
	if ok {
		data["fields.msg"] = data["msg"]
	}
	_, ok = data["level"]
	if ok {
		data["fields.level"] = data["level"]
	}
}
<commit_msg>using DisableTimestamp with colored output ignores the DisableTimestamp setting<commit_after>package prefixed

import (
	"bytes"
	"fmt"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/mgutz/ansi"
)

const reset = ansi.Reset

var (
	baseTimestamp time.Time
	isTerminal    bool
)

func init() {
	baseTimestamp = time.Now()
	isTerminal = logrus.IsTerminal()
}

func miniTS() int {
	return int(time.Since(baseTimestamp) / time.Second)
}

type TextFormatter struct {
	// Set to true to bypass checking for a TTY before outputting colors.
	ForceColors bool

	// Force disabling colors.
	DisableColors bool

	// Disable timestamp logging. useful when output is redirected to logging
	// system that already adds timestamps.
	DisableTimestamp bool

	// Enable logging of just the time passed since beginning of execution.
	ShortTimestamp bool

	// Timestamp format to use for display when a full timestamp is printed.
	TimestampFormat string

	// The fields are sorted by default for a consistent output. For applications
	// that log extremely frequently and don't use the JSON formatter this may not
	// be desired.
	DisableSorting bool

	// Pad msg field with spaces on the right for display.
	// The value for this parameter will be the size of padding.
	// Its default value is zero, which means no padding will be applied for msg.
	SpacePadding int
}

func (f *TextFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	var keys []string = make([]string, 0, len(entry.Data))
	for k := range entry.Data {
		if k != "prefix" {
			keys = append(keys, k)
		}
	}

	if !f.DisableSorting {
		sort.Strings(keys)
	}

	b := &bytes.Buffer{}

	prefixFieldClashes(entry.Data)

	isColorTerminal := isTerminal && (runtime.GOOS != "windows")
	isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors

	timestampFormat := f.TimestampFormat
	if timestampFormat == "" {
		timestampFormat = time.Stamp
	}
	if isColored {
		f.printColored(b, entry, keys, timestampFormat)
	} else {
		if !f.DisableTimestamp {
			f.appendKeyValue(b, "time", entry.Time.Format(timestampFormat))
		}
		f.appendKeyValue(b, "level", entry.Level.String())
		if entry.Message != "" {
			f.appendKeyValue(b, "msg", entry.Message)
		}
		for _, key := range keys {
			f.appendKeyValue(b, key, entry.Data[key])
		}
	}

	b.WriteByte('\n')
	return b.Bytes(), nil
}

func (f *TextFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry, keys []string, timestampFormat string) {
	var levelColor string
	var levelText string
	switch entry.Level {
	case logrus.InfoLevel:
		levelColor = ansi.Green
	case logrus.WarnLevel:
		levelColor = ansi.Yellow
	case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
		levelColor = ansi.Red
	default:
		levelColor = ansi.Blue
	}

	if entry.Level != logrus.WarnLevel {
		levelText = strings.ToUpper(entry.Level.String())
	} else {
		levelText = "WARN"
	}

	prefix := ""
	message := entry.Message

	if prefixValue, ok := entry.Data["prefix"]; ok {
		prefix = fmt.Sprint(" ", ansi.Cyan, prefixValue, ":", reset)
	} else {
		prefixValue, trimmedMsg := extractPrefix(entry.Message)
		if len(prefixValue) > 0 {
			prefix = fmt.Sprint(" ", ansi.Cyan, prefixValue, ":", reset)
			message = trimmedMsg
		}
	}

	messageFormat := "%s"
	if f.SpacePadding != 0 {
		messageFormat = fmt.Sprintf("%%-%ds", f.SpacePadding)
	}

	if f.DisableTimestamp {
		fmt.Fprintf(b, "%s%s %s%+5s%s%s "+messageFormat, ansi.LightBlack, reset, levelColor, levelText, reset, prefix, message)
	} else {
		if f.ShortTimestamp {
			fmt.Fprintf(b, "%s[%04d]%s %s%+5s%s%s "+messageFormat, ansi.LightBlack, miniTS(), reset, levelColor, levelText, reset, prefix, message)
		} else {
			fmt.Fprintf(b, "%s[%s]%s %s%+5s%s%s "+messageFormat, ansi.LightBlack, entry.Time.Format(timestampFormat), reset, levelColor, levelText, reset, prefix, message)
		}
	}
	for _, k := range keys {
		v := entry.Data[k]
		fmt.Fprintf(b, " %s%s%s=%+v", levelColor, k, reset, v)
	}
}

func needsQuoting(text string) bool {
	for _, ch := range text {
		if !((ch >= 'a' && ch <= 'z') ||
			(ch >= 'A' && ch <= 'Z') ||
			(ch >= '0' && ch <= '9') ||
			ch == '-' || ch == '.') {
			return false
		}
	}
	return true
}

func extractPrefix(msg string) (string, string) {
	prefix := ""
	regex := regexp.MustCompile("^\\[(.*?)\\]")
	if regex.MatchString(msg) {
		match := regex.FindString(msg)
		prefix, msg = match[1:len(match)-1], strings.TrimSpace(msg[len(match):])
	}
	return prefix, msg
}

func (f *TextFormatter) appendKeyValue(b *bytes.Buffer, key string, value interface{}) {
	b.WriteString(key)
	b.WriteByte('=')

	switch value := value.(type) {
	case string:
		if needsQuoting(value) {
			b.WriteString(value)
		} else {
			fmt.Fprintf(b, "%q", value)
		}
	case error:
		errmsg := value.Error()
		if needsQuoting(errmsg) {
			b.WriteString(errmsg)
		} else {
			fmt.Fprintf(b, "%q", value)
		}
	default:
		fmt.Fprint(b, value)
	}

	b.WriteByte(' ')
}

func prefixFieldClashes(data logrus.Fields) {
	_, ok := data["time"]
	if ok {
		data["fields.time"] = data["time"]
	}
	_, ok = data["msg"]
	if ok {
		data["fields.msg"] = data["msg"]
	}
	_, ok = data["level"]
	if ok {
		data["fields.level"] = data["level"]
	}
}
<|endoftext|>


<commit_before>package prelude

const Prelude = prelude + types + numeric + goroutines + jsmapping

const prelude = `Error.stackTraceLimit = Infinity;

var $global = (function() { return this; })();
var $module;
if (typeof module !== "undefined") {
  $module = module;
}

var $packages = {}, $reflect, $idCounter = 0;
var $keys = function(m) { return m ? Object.keys(m) : []; };
var $min = Math.min;
var $mod = function(x, y) { return x % y; };
var $parseInt = parseInt;
var $parseFloat = function(f) {
  if (f !== undefined && f !== null && f.constructor === Number) {
    return f;
  }
  return parseFloat(f);
};
var $flushConsole = function() {};

var $mapArray = function(array, f) {
  var newArray = new array.constructor(array.length), i;
  for (i = 0; i < array.length; i++) {
    newArray[i] = f(array[i]);
  }
  return newArray;
};

var $methodVal = function(recv, name) {
  var vals = recv.$methodVals || {};
  recv.$methodVals = vals; /* noop for primitives */
  var f = vals[name];
  if (f !== undefined) {
    return f;
  }
  var method = recv[name];
  f = function() {
    $stackDepthOffset--;
    try {
      return method.apply(recv, arguments);
    } finally {
      $stackDepthOffset++;
    }
  };
  vals[name] = f;
  return f;
};

var $methodExpr = function(method) {
  if (method.$expr === undefined) {
    method.$expr = function() {
      $stackDepthOffset--;
      try {
        return Function.call.apply(method, arguments);
      } finally {
        $stackDepthOffset++;
      }
    };
  }
  return method.$expr;
};

var $subslice = function(slice, low, high, max) {
  if (low < 0 || high < low || max < high || high > slice.$capacity || max > slice.$capacity) {
    $throwRuntimeError("slice bounds out of range");
  }
  var s = new slice.constructor(slice.$array);
  s.$offset = slice.$offset + low;
  s.$length = slice.$length - low;
  s.$capacity = slice.$capacity - low;
  if (high !== undefined) {
    s.$length = high - low;
  }
  if (max !== undefined) {
    s.$capacity = max - low;
  }
  return s;
};

var $sliceToArray = function(slice) {
  if (slice.$length === 0) {
    return [];
  }
  if (slice.$array.constructor !== Array) {
    return slice.$array.subarray(slice.$offset, slice.$offset + slice.$length);
  }
  return slice.$array.slice(slice.$offset, slice.$offset + slice.$length);
};

var $decodeRune = function(str, pos) {
  var c0 = str.charCodeAt(pos);

  if (c0 < 0x80) {
    return [c0, 1];
  }

  if (c0 !== c0 || c0 < 0xC0) {
    return [0xFFFD, 1];
  }

  var c1 = str.charCodeAt(pos + 1);
  if (c1 !== c1 || c1 < 0x80 || 0xC0 <= c1) {
    return [0xFFFD, 1];
  }

  if (c0 < 0xE0) {
    var r = (c0 & 0x1F) << 6 | (c1 & 0x3F);
    if (r <= 0x7F) {
      return [0xFFFD, 1];
    }
    return [r, 2];
  }

  var c2 = str.charCodeAt(pos + 2);
  if (c2 !== c2 || c2 < 0x80 || 0xC0 <= c2) {
    return [0xFFFD, 1];
  }

  if (c0 < 0xF0) {
    var r = (c0 & 0x0F) << 12 | (c1 & 0x3F) << 6 | (c2 & 0x3F);
    if (r <= 0x7FF) {
      return [0xFFFD, 1];
    }
    if (0xD800 <= r && r <= 0xDFFF) {
      return [0xFFFD, 1];
    }
    return [r, 3];
  }

  var c3 = str.charCodeAt(pos + 3);
  if (c3 !== c3 || c3 < 0x80 || 0xC0 <= c3) {
    return [0xFFFD, 1];
  }

  if (c0 < 0xF8) {
    var r = (c0 & 0x07) << 18 | (c1 & 0x3F) << 12 | (c2 & 0x3F) << 6 | (c3 & 0x3F);
    if (r <= 0xFFFF || 0x10FFFF < r) {
      return [0xFFFD, 1];
    }
    return [r, 4];
  }

  return [0xFFFD, 1];
};

var $encodeRune = function(r) {
  if (r < 0 || r > 0x10FFFF || (0xD800 <= r && r <= 0xDFFF)) {
    r = 0xFFFD;
  }
  if (r <= 0x7F) {
    return String.fromCharCode(r);
  }
  if (r <= 0x7FF) {
    return String.fromCharCode(0xC0 | r >> 6, 0x80 | (r & 0x3F));
  }
  if (r <= 0xFFFF) {
    return String.fromCharCode(0xE0 | r >> 12, 0x80 | (r >> 6 & 0x3F), 0x80 | (r & 0x3F));
  }
  return String.fromCharCode(0xF0 | r >> 18, 0x80 | (r >> 12 & 0x3F), 0x80 | (r >> 6 & 0x3F), 0x80 | (r & 0x3F));
};

var $stringToBytes = function(str) {
  var array = new Uint8Array(str.length), i;
  for (i = 0; i < str.length; i++) {
    array[i] = str.charCodeAt(i);
  }
  return array;
};

var $bytesToString = function(slice) {
  if (slice.$length === 0) {
    return "";
  }
  var str = "", i;
  for (i = 0; i < slice.$length; i += 10000) {
    str += String.fromCharCode.apply(null, slice.$array.subarray(slice.$offset + i, slice.$offset + Math.min(slice.$length, i + 10000)));
  }
  return str;
};

var $stringToRunes = function(str) {
  var array = new Int32Array(str.length);
  var rune, i, j = 0;
  for (i = 0; i < str.length; i += rune[1], j++) {
    rune = $decodeRune(str, i);
    array[j] = rune[0];
  }
  return array.subarray(0, j);
};

var $runesToString = function(slice) {
  if (slice.$length === 0) {
    return "";
  }
  var str = "", i;
  for (i = 0; i < slice.$length; i++) {
    str += $encodeRune(slice.$array[slice.$offset + i]);
  }
  return str;
};

var $copyString = function(dst, src) {
  var n = Math.min(src.length, dst.$length), i;
  for (i = 0; i < n; i++) {
    dst.$array[dst.$offset + i] = src.charCodeAt(i);
  }
  return n;
};

var $copySlice = function(dst, src) {
  var n = Math.min(src.$length, dst.$length), i;
  $internalCopy(dst.$array, src.$array, dst.$offset, src.$offset, n, dst.constructor.elem);
  return n;
};

var $copy = function(dst, src, type) {
  switch (type.kind) {
  case $kindArray:
    $internalCopy(dst, src, 0, 0, src.length, type.elem);
    break;
  case $kindStruct:
    for (var i = 0; i < type.fields.length; i++) {
      var field = type.fields[i];
      var fieldName = field[0];
      var fieldType = field[3];
      switch (fieldType.kind) {
      case $kindArray:
      case $kindStruct:
        $copy(dst[fieldName], src[fieldName], fieldType);
        continue;
      default:
        dst[fieldName] = src[fieldName];
        continue;
      }
    }
    break;
  }
};

var $internalCopy = function(dst, src, dstOffset, srcOffset, n, elem) {
  if (n === 0 || (dst === src && dstOffset === srcOffset)) {
    return;
  }

  if (src.subarray) {
    dst.set(src.subarray(srcOffset, srcOffset + n), dstOffset);
    return;
  }

  switch (elem.kind) {
  case $kindArray:
  case $kindStruct:
    if (dst === src && dstOffset > srcOffset) {
      for (var i = n - 1; i >= 0; i--) {
        $copy(dst[dstOffset + i], src[srcOffset + i], elem);
      }
      return;
    }
    for (var i = 0; i < n; i++) {
      $copy(dst[dstOffset + i], src[srcOffset + i], elem);
    }
    return;
  }

  if (dst === src && dstOffset > srcOffset) {
    for (var i = n - 1; i >= 0; i--) {
      dst[dstOffset + i] = src[srcOffset + i];
    }
    return;
  }
  for (var i = 0; i < n; i++) {
    dst[dstOffset + i] = src[srcOffset + i];
  }
};

var $clone = function(src, type) {
  var clone = type.zero();
  $copy(clone, src, type);
  return clone;
};

var $pointerOfStructConversion = function(obj, type) {
  if(obj.$proxies === undefined) {
    obj.$proxies = {};
    obj.$proxies[obj.constructor.string] = obj;
  }
  var proxy = obj.$proxies[type.string];
  if (proxy === undefined) {
    var properties = {};
    for (var i = 0; i < type.Struct.fields.length; i++) {
      (function(fieldName) {
        properties[fieldName] = {
          get: function() { return obj[fieldName]; },
          set: function(value) { obj[fieldName] = value; },
        };
      })(type.Struct.fields[i][0]);
    }
    proxy = Object.create(type.prototype, properties);
    proxy.$val = proxy;
    obj.$proxies[type.string] = proxy;
    proxy.$proxies = obj.$proxies;
  }
  return proxy;
};

var $append = function(slice) {
  return $internalAppend(slice, arguments, 1, arguments.length - 1);
};

var $appendSlice = function(slice, toAppend) {
  return $internalAppend(slice, toAppend.$array, toAppend.$offset, toAppend.$length);
};

var $internalAppend = function(slice, array, offset, length) {
  if (length === 0) {
    return slice;
  }

  var newArray = slice.$array;
  var newOffset = slice.$offset;
  var newLength = slice.$length + length;
  var newCapacity = slice.$capacity;

  if (newLength > newCapacity) {
    newOffset = 0;
    newCapacity = Math.max(newLength, slice.$capacity < 1024 ? slice.$capacity * 2 : Math.floor(slice.$capacity * 5 / 4));

    if (slice.$array.constructor === Array) {
      newArray = slice.$array.slice(slice.$offset, slice.$offset + slice.$length);
      newArray.length = newCapacity;
      var zero = slice.constructor.elem.zero, i;
      for (i = slice.$length; i < newCapacity; i++) {
        newArray[i] = zero();
      }
    } else {
      newArray = new slice.$array.constructor(newCapacity);
      newArray.set(slice.$array.subarray(slice.$offset, slice.$offset + slice.$length));
    }
  }

  $internalCopy(newArray, array, newOffset + slice.$length, offset, length, slice.constructor.elem);

  var newSlice = new slice.constructor(newArray);
  newSlice.$offset = newOffset;
  newSlice.$length = newLength;
  newSlice.$capacity = newCapacity;
  return newSlice;
};

var $equal = function(a, b, type) {
  if (a === b) {
    return true;
  }
  var i;
  switch (type.kind) {
  case $kindFloat32:
    return $float32IsEqual(a, b);
  case $kindComplex64:
    return $float32IsEqual(a.$real, b.$real) && $float32IsEqual(a.$imag, b.$imag);
  case $kindComplex128:
    return a.$real === b.$real && a.$imag === b.$imag;
  case $kindInt64:
  case $kindUint64:
    return a.$high === b.$high && a.$low === b.$low;
  case $kindPtr:
    if (a.constructor.Struct) {
      return false;
    }
    return $pointerIsEqual(a, b);
  case $kindArray:
    if (a.length != b.length) {
      return false;
    }
    var i;
    for (i = 0; i < a.length; i++) {
      if (!$equal(a[i], b[i], type.elem)) {
        return false;
      }
    }
    return true;
  case $kindStruct:
    for (i = 0; i < type.fields.length; i++) {
      var field = type.fields[i];
      var name = field[0];
      if (!$equal(a[name], b[name], field[3])) {
        return false;
      }
    }
    return true;
  default:
    return false;
  }
};

var $interfaceIsEqual = function(a, b) {
  if (a === null || b === null || a === undefined || b === undefined || a.constructor !== b.constructor) {
    return a === b;
  }
  switch (a.constructor.kind) {
  case $kindFunc:
  case $kindMap:
  case $kindSlice:
  case $kindStruct:
    $throwRuntimeError("comparing uncomparable type " + a.constructor.string);
  case undefined: /* js.Object */
    return a === b;
  default:
    return $equal(a.$val, b.$val, a.constructor);
  }
};

var $float32IsEqual = function(a, b) {
  if (a === b) {
    return true;
  }
  if (a === 1/0 || b === 1/0 || a === -1/0 || b === -1/0 || a !== a || b !== b) {
    return false;
  }
  var math = $packages["math"];
  return math !== undefined && math.Float32bits(a) === math.Float32bits(b);
};

var $pointerIsEqual = function(a, b) {
  if (a === b) {
    return true;
  }
  if (a.$get === $throwNilPointerError || b.$get === $throwNilPointerError) {
    return a.$get === $throwNilPointerError && b.$get === $throwNilPointerError;
  }
  var va = a.$get();
  var vb = b.$get();
  if (va !== vb) {
    return false;
  }
  var dummy = va + 1;
  a.$set(dummy);
  var equal = b.$get() === dummy;
  a.$set(va);
  return equal;
};
`
<commit_msg>Revert "fixed $module"<commit_after>package prelude

const Prelude = prelude + types + numeric + goroutines + jsmapping

const prelude = `Error.stackTraceLimit = Infinity;

var $global = (function() { return this; })();
if (typeof module !== "undefined") {
  $module = module;
}

var $packages = {}, $reflect, $idCounter = 0;
var $keys = function(m) { return m ? Object.keys(m) : []; };
var $min = Math.min;
var $mod = function(x, y) { return x % y; };
var $parseInt = parseInt;
var $parseFloat = function(f) {
  if (f !== undefined && f !== null && f.constructor === Number) {
    return f;
  }
  return parseFloat(f);
};
var $flushConsole = function() {};

var $mapArray = function(array, f) {
  var newArray = new array.constructor(array.length), i;
  for (i = 0; i < array.length; i++) {
    newArray[i] = f(array[i]);
  }
  return newArray;
};

var $methodVal = function(recv, name) {
  var vals = recv.$methodVals || {};
  recv.$methodVals = vals; /* noop for primitives */
  var f = vals[name];
  if (f !== undefined) {
    return f;
  }
  var method = recv[name];
  f = function() {
    $stackDepthOffset--;
    try {
      return method.apply(recv, arguments);
    } finally {
      $stackDepthOffset++;
    }
  };
  vals[name] = f;
  return f;
};

var $methodExpr = function(method) {
  if (method.$expr === undefined) {
    method.$expr = function() {
      $stackDepthOffset--;
      try {
        return Function.call.apply(method, arguments);
      } finally {
        $stackDepthOffset++;
      }
    };
  }
  return method.$expr;
};

var $subslice = function(slice, low, high, max) {
  if (low < 0 || high < low || max < high || high > slice.$capacity || max > slice.$capacity) {
    $throwRuntimeError("slice bounds out of range");
  }
  var s = new slice.constructor(slice.$array);
  s.$offset = slice.$offset + low;
  s.$length = slice.$length - low;
  s.$capacity = slice.$capacity - low;
  if (high !== undefined) {
    s.$length = high - low;
  }
  if (max !== undefined) {
    s.$capacity = max - low;
  }
  return s;
};

var $sliceToArray = function(slice) {
  if (slice.$length === 0) {
    return [];
  }
  if (slice.$array.constructor !== Array) {
    return slice.$array.subarray(slice.$offset, slice.$offset + slice.$length);
  }
  return slice.$array.slice(slice.$offset, slice.$offset + slice.$length);
};

var $decodeRune = function(str, pos) {
  var c0 = str.charCodeAt(pos);

  if (c0 < 0x80) {
    return [c0, 1];
  }

  if (c0 !== c0 || c0 < 0xC0) {
    return [0xFFFD, 1];
  }

  var c1 = str.charCodeAt(pos + 1);
  if (c1 !== c1 || c1 < 0x80 || 0xC0 <= c1) {
    return [0xFFFD, 1];
  }

  if (c0 < 0xE0) {
    var r = (c0 & 0x1F) << 6 | (c1 & 0x3F);
    if (r <= 0x7F) {
      return [0xFFFD, 1];
    }
    return [r, 2];
  }

  var c2 = str.charCodeAt(pos + 2);
  if (c2 !== c2 || c2 < 0x80 || 0xC0 <= c2) {
    return [0xFFFD, 1];
  }

  if (c0 < 0xF0) {
    var r = (c0 & 0x0F) << 12 | (c1 & 0x3F) << 6 | (c2 & 0x3F);
    if (r <= 0x7FF) {
      return [0xFFFD, 1];
    }
    if (0xD800 <= r && r <= 0xDFFF) {
      return [0xFFFD, 1];
    }
    return [r, 3];
  }

  var c3 = str.charCodeAt(pos + 3);
  if (c3 !== c3 || c3 < 0x80 || 0xC0 <= c3) {
    return [0xFFFD, 1];
  }

  if (c0 < 0xF8) {
    var r = (c0 & 0x07) << 18 | (c1 & 0x3F) << 12 | (c2 & 0x3F) << 6 | (c3 & 0x3F);
    if (r <= 0xFFFF || 0x10FFFF < r) {
      return [0xFFFD, 1];
    }
    return [r, 4];
  }

  return [0xFFFD, 1];
};

var $encodeRune = function(r) {
  if (r < 0 || r > 0x10FFFF || (0xD800 <= r && r <= 0xDFFF)) {
    r = 0xFFFD;
  }
  if (r <= 0x7F) {
    return String.fromCharCode(r);
  }
  if (r <= 0x7FF) {
    return String.fromCharCode(0xC0 | r >> 6, 0x80 | (r & 0x3F));
  }
  if (r <= 0xFFFF) {
    return String.fromCharCode(0xE0 | r >> 12, 0x80 | (r >> 6 & 0x3F), 0x80 | (r & 0x3F));
  }
  return String.fromCharCode(0xF0 | r >> 18, 0x80 | (r >> 12 & 0x3F), 0x80 | (r >> 6 & 0x3F), 0x80 | (r & 0x3F));
};

var $stringToBytes = function(str) {
  var array = new Uint8Array(str.length), i;
  for (i = 0; i < str.length; i++) {
    array[i] = str.charCodeAt(i);
  }
  return array;
};

var $bytesToString = function(slice) {
  if (slice.$length === 0) {
    return "";
  }
  var str = "", i;
  for (i = 0; i < slice.$length; i += 10000) {
    str += String.fromCharCode.apply(null, slice.$array.subarray(slice.$offset + i, slice.$offset + Math.min(slice.$length, i + 10000)));
  }
  return str;
};

var $stringToRunes = function(str) {
  var array = new Int32Array(str.length);
  var rune, i, j = 0;
  for (i = 0; i < str.length; i += rune[1], j++) {
    rune = $decodeRune(str, i);
    array[j] = rune[0];
  }
  return array.subarray(0, j);
};

var $runesToString = function(slice) {
  if (slice.$length === 0) {
    return "";
  }
  var str = "", i;
  for (i = 0; i < slice.$length; i++) {
    str += $encodeRune(slice.$array[slice.$offset + i]);
  }
  return str;
};

var $copyString = function(dst, src) {
  var n = Math.min(src.length, dst.$length), i;
  for (i = 0; i < n; i++) {
    dst.$array[dst.$offset + i] = src.charCodeAt(i);
  }
  return n;
};

var $copySlice = function(dst, src) {
  var n = Math.min(src.$length, dst.$length), i;
  $internalCopy(dst.$array, src.$array, dst.$offset, src.$offset, n, dst.constructor.elem);
  return n;
};

var $copy = function(dst, src, type) {
  switch (type.kind) {
  case $kindArray:
    $internalCopy(dst, src, 0, 0, src.length, type.elem);
    break;
  case $kindStruct:
    for (var i = 0; i < type.fields.length; i++) {
      var field = type.fields[i];
      var fieldName = field[0];
      var fieldType = field[3];
      switch (fieldType.kind) {
      case $kindArray:
      case $kindStruct:
        $copy(dst[fieldName], src[fieldName], fieldType);
        continue;
      default:
        dst[fieldName] = src[fieldName];
        continue;
      }
    }
    break;
  }
};

var $internalCopy = function(dst, src, dstOffset, srcOffset, n, elem) {
  if (n === 0 || (dst === src && dstOffset === srcOffset)) {
    return;
  }

  if (src.subarray) {
    dst.set(src.subarray(srcOffset, srcOffset + n), dstOffset);
    return;
  }

  switch (elem.kind) {
  case $kindArray:
  case $kindStruct:
    if (dst === src && dstOffset > srcOffset) {
      for (var i = n - 1; i >= 0; i--) {
        $copy(dst[dstOffset + i], src[srcOffset + i], elem);
      }
      return;
    }
    for (var i = 0; i < n; i++) {
      $copy(dst[dstOffset + i], src[srcOffset + i], elem);
    }
    return;
  }

  if (dst === src && dstOffset > srcOffset) {
    for (var i = n - 1; i >= 0; i--) {
      dst[dstOffset + i] = src[srcOffset + i];
    }
    return;
  }
  for (var i = 0; i < n; i++) {
    dst[dstOffset + i] = src[srcOffset + i];
  }
};

var $clone = function(src, type) {
  var clone = type.zero();
  $copy(clone, src, type);
  return clone;
};

var $pointerOfStructConversion = function(obj, type) {
  if(obj.$proxies === undefined) {
    obj.$proxies = {};
    obj.$proxies[obj.constructor.string] = obj;
  }
  var proxy = obj.$proxies[type.string];
  if (proxy === undefined) {
    var properties = {};
    for (var i = 0; i < type.Struct.fields.length; i++) {
      (function(fieldName) {
        properties[fieldName] = {
          get: function() { return obj[fieldName]; },
          set: function(value) { obj[fieldName] = value; },
        };
      })(type.Struct.fields[i][0]);
    }
    proxy = Object.create(type.prototype, properties);
    proxy.$val = proxy;
    obj.$proxies[type.string] = proxy;
    proxy.$proxies = obj.$proxies;
  }
  return proxy;
};

var $append = function(slice) {
  return $internalAppend(slice, arguments, 1, arguments.length - 1);
};

var $appendSlice = function(slice, toAppend) {
  return $internalAppend(slice, toAppend.$array, toAppend.$offset, toAppend.$length);
};

var $internalAppend = function(slice, array, offset, length) {
  if (length === 0) {
    return slice;
  }

  var newArray = slice.$array;
  var newOffset = slice.$offset;
  var newLength = slice.$length + length;
  var newCapacity = slice.$capacity;

  if (newLength > newCapacity) {
    newOffset = 0;
    newCapacity = Math.max(newLength, slice.$capacity < 1024 ? slice.$capacity * 2 : Math.floor(slice.$capacity * 5 / 4));

    if (slice.$array.constructor === Array) {
      newArray = slice.$array.slice(slice.$offset, slice.$offset + slice.$length);
      newArray.length = newCapacity;
      var zero = slice.constructor.elem.zero, i;
      for (i = slice.$length; i < newCapacity; i++) {
        newArray[i] = zero();
      }
    } else {
      newArray = new slice.$array.constructor(newCapacity);
      newArray.set(slice.$array.subarray(slice.$offset, slice.$offset + slice.$length));
    }
  }

  $internalCopy(newArray, array, newOffset + slice.$length, offset, length, slice.constructor.elem);

  var newSlice = new slice.constructor(newArray);
  newSlice.$offset = newOffset;
  newSlice.$length = newLength;
  newSlice.$capacity = newCapacity;
  return newSlice;
};

var $equal = function(a, b, type) {
  if (a === b) {
    return true;
  }
  var i;
  switch (type.kind) {
  case $kindFloat32:
    return $float32IsEqual(a, b);
  case $kindComplex64:
    return $float32IsEqual(a.$real, b.$real) && $float32IsEqual(a.$imag, b.$imag);
  case $kindComplex128:
    return a.$real === b.$real && a.$imag === b.$imag;
  case $kindInt64:
  case $kindUint64:
    return a.$high === b.$high && a.$low === b.$low;
  case $kindPtr:
    if (a.constructor.Struct) {
      return false;
    }
    return $pointerIsEqual(a, b);
  case $kindArray:
    if (a.length != b.length) {
      return false;
    }
    var i;
    for (i = 0; i < a.length; i++) {
      if (!$equal(a[i], b[i], type.elem)) {
        return false;
      }
    }
    return true;
  case $kindStruct:
    for (i = 0; i < type.fields.length; i++) {
      var field = type.fields[i];
      var name = field[0];
      if (!$equal(a[name], b[name], field[3])) {
        return false;
      }
    }
    return true;
  default:
    return false;
  }
};

var $interfaceIsEqual = function(a, b) {
  if (a === null || b === null || a === undefined || b === undefined || a.constructor !== b.constructor) {
    return a === b;
  }
  switch (a.constructor.kind) {
  case $kindFunc:
  case $kindMap:
  case $kindSlice:
  case $kindStruct:
    $throwRuntimeError("comparing uncomparable type " + a.constructor.string);
  case undefined: /* js.Object */
    return a === b;
  default:
    return $equal(a.$val, b.$val, a.constructor);
  }
};

var $float32IsEqual = function(a, b) {
  if (a === b) {
    return true;
  }
  if (a === 1/0 || b === 1/0 || a === -1/0 || b === -1/0 || a !== a || b !== b) {
    return false;
  }
  var math = $packages["math"];
  return math !== undefined && math.Float32bits(a) === math.Float32bits(b);
};

var $pointerIsEqual = function(a, b) {
  if (a === b) {
    return true;
  }
  if (a.$get === $throwNilPointerError || b.$get === $throwNilPointerError) {
    return a.$get === $throwNilPointerError && b.$get === $throwNilPointerError;
  }
  var va = a.$get();
  var vb = b.$get();
  if (va !== vb) {
    return false;
  }
  var dummy = va + 1;
  a.$set(dummy);
  var equal = b.$get() === dummy;
  a.$set(va);
  return equal;
};
`
<|endoftext|>


<commit_before>package main

// merges multiple frequency count files into one
// this is could use a merge sort and be smart

import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"
)

// freqCount is mapping of string->count
type freqCount map[string]int

// make a new counter with some minor preallocation
//  each month has about 2.2M uniques
func newFreqCount() freqCount {
	return make(freqCount, 3000000)
}

// LoadCSV loads in a CSV in form of WORD,COUNT
func loadCSV(counts freqCount, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fi.Close()

	fizip, err := gzip.NewReader(fi)
	if err != nil {
		return err
	}
	defer fizip.Close()

	scanner := bufio.NewScanner(fizip)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.SplitN(line, ",", 2)
		if len(parts) != 2 {
			return fmt.Errorf("Got extra junk in line: %q", line)
		}
		c, err := strconv.Atoi(parts[1])
		if err != nil {
			return fmt.Errorf("Number conversion failed: %q", line)
		}

		counts[parts[0]] += c
	}
	return scanner.Err()
}

// returns true if any character is repeated more than N times
func repeated(s string, n int) bool {
	slen := len(s)
	if slen < n {
		return false
	}
	ch := s[0]
	count := 1
	for i := 1; i < slen; i++ {
		cnext := s[i]
		if cnext != ch {
			ch = cnext
			count = 1
			continue
		}
		count++
		if count == n {
			return true
		}
	}
	return false
}

// returns true if a haha words.  hahhahahhhahaha, etc.
func haha(s string) bool {
	return strings.Contains(s, "haha")
}

func main() {
	outfile := flag.String("o", "RC-total.csv.gz", "output file name")
	mincount := flag.Int("mincount", 2, "only output if freqcount >=, 0 = all")
	minlen := flag.Int("minlen", 7, "only output if word is >=, 0 = all")
	flag.Parse()
	if *outfile == "" {
		log.Fatalf("Must specificy outfile")
	}
	args := flag.Args()
	counts := newFreqCount()
	for _, arg := range args {
		log.Printf("Loading %s", arg)
		err := loadCSV(counts, arg)
		if err != nil {
			log.Fatalf("%s: %s", arg, err)
		}
	}
	fo, err := os.Create(*outfile)
	if err != nil {
		log.Fatalf("OH NO, unable to write: %s", err)
	}
	fout := gzip.NewWriter(fo)

	keys := make([]string, 0, len(counts))
	total := 0
	for k, v := range counts {
		total += v
		if v >= *mincount && len(k) >= *minlen && !repeated(k, 4) && !haha(k) {
			keys = append(keys, k)
		}
	}
	sort.Strings(keys)
	for _, k := range keys {
		fout.Write([]byte(fmt.Sprintf("%s,%d\n", k, counts[k])))
	}

	fout.Close()
	fo.Close()
	log.Printf("DONE: wrote %s got %d unique words from %d", *outfile, len(keys), total)
}
<commit_msg>add lol to haha<commit_after>package main

// merges multiple frequency count files into one
// this is could use a merge sort and be smart

import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"
)

// freqCount is mapping of string->count
type freqCount map[string]int

// make a new counter with some minor preallocation
//  each month has about 2.2M uniques
func newFreqCount() freqCount {
	return make(freqCount, 3000000)
}

// LoadCSV loads in a CSV in form of WORD,COUNT
func loadCSV(counts freqCount, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fi.Close()

	fizip, err := gzip.NewReader(fi)
	if err != nil {
		return err
	}
	defer fizip.Close()

	scanner := bufio.NewScanner(fizip)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.SplitN(line, ",", 2)
		if len(parts) != 2 {
			return fmt.Errorf("Got extra junk in line: %q", line)
		}
		c, err := strconv.Atoi(parts[1])
		if err != nil {
			return fmt.Errorf("Number conversion failed: %q", line)
		}

		counts[parts[0]] += c
	}
	return scanner.Err()
}

// returns true if any character is repeated more than N times
func repeated(s string, n int) bool {
	slen := len(s)
	if slen < n {
		return false
	}
	ch := s[0]
	count := 1
	for i := 1; i < slen; i++ {
		cnext := s[i]
		if cnext != ch {
			ch = cnext
			count = 1
			continue
		}
		count++
		if count == n {
			return true
		}
	}
	return false
}

// returns true if a haha words.  hahhahahhhahaha, lololo
func haha(s string) bool {
	return strings.Contains(s, "haha") || strings.Contains(s, "lolo")
}

func main() {
	outfile := flag.String("o", "RC-total.csv.gz", "output file name")
	mincount := flag.Int("mincount", 2, "only output if freqcount >=, 0 = all")
	minlen := flag.Int("minlen", 7, "only output if word is >=, 0 = all")
	flag.Parse()
	if *outfile == "" {
		log.Fatalf("Must specificy outfile")
	}
	args := flag.Args()
	counts := newFreqCount()
	for _, arg := range args {
		log.Printf("Loading %s", arg)
		err := loadCSV(counts, arg)
		if err != nil {
			log.Fatalf("%s: %s", arg, err)
		}
	}
	fo, err := os.Create(*outfile)
	if err != nil {
		log.Fatalf("OH NO, unable to write: %s", err)
	}
	fout := gzip.NewWriter(fo)

	keys := make([]string, 0, len(counts))
	total := 0
	for k, v := range counts {
		total += v
		if v >= *mincount && len(k) >= *minlen && !repeated(k, 4) && !haha(k) {
			keys = append(keys, k)
		}
	}
	sort.Strings(keys)
	for _, k := range keys {
		fout.Write([]byte(fmt.Sprintf("%s,%d\n", k, counts[k])))
	}

	fout.Close()
	fo.Close()
	log.Printf("DONE: wrote %s got %d unique words from %d", *outfile, len(keys), total)
}
<|endoftext|>


<commit_before>package apachelog

import (
	"net/url"
	"strings"
	"time"
)

// An AccessLogEntry represents a line of an access log.
type AccessLogEntry struct {
	RemoteIPAddr        string            // Remote IP address
	LocalIPAddr         string            // Local IP address
	ResponseSize        int64             // Size of response in bytes, excluding HTTP headers
	Cookies             map[string]string // Cookies value
	ElapsedTime         int64             // Time taken to serve the request, in microseconds
	EnvVars             map[string]string // Content of the environment variables
	Headers             map[string]string // Content of the request headers
	Filename            string            // Filename
	RemoteHost          string            // Remote host
	RequestProto        string            // Request protocol
	KeepAliveRequests   int64             // Number of keepalive requests handled on this connection
	RemoteLogname       string            // Remote logname. A "-" is returned, when not supplied
	RequestMethod       string            // Request method
	Port                string            // Canonical port of the server serving the request
	ProcessID           int64             // Process ID of the child that serviced the request
	QueryString         string            // Query string (prepended with a ? if exists)
	RequestFirstLine    RequestFirstLine  // First line of the request
	Status              string            // Status
	Time                time.Time         // Time the request was received (standard english format)
	ElapsedTimeSec      int64             // Time taken to serve the request, in seconds
	RemoteUser          string            // Remote user (from auth)
	URLPath             string            // URL path requested, not including any query string
	CanonicalServerName string            // Canonical ServerName of the server serving the request
	ServerName          string            // Server name according to the UseCanonicalName setting
	BytesReceived       int64             // Bytes received, including request and headers
	BytesSent           int64             // Bytes sent, including headers
}

// RequestFirstLine is a handy structure to hold the first line of the HTTP
// request. It provides accessors to access the HTTP method, the path and the
// protocol information contained in the raw first line.
type RequestFirstLine struct {
	raw string

	// Extracted fields
	method     string
	path       string
	pathParsed string
	proto      string

	parsedPath bool
	parsed     bool
}

// NewRequestFirstLine creates a new RequestFirstLine with the supplied raw
// first line.
//
// This function does not parse or do anything else than initializing the
// structure.
func NewRequestFirstLine(raw string) RequestFirstLine {
	return RequestFirstLine{
		raw: raw,
	}
}

// Method returns the method held in the HTTP request first line.
func (rfl *RequestFirstLine) Method() string {
	rfl.parse()
	return rfl.method
}

// RawPath returns the raw path held in the HTTP request first line.
func (rfl *RequestFirstLine) RawPath() string {
	rfl.parse()
	return rfl.path
}

// Path returns the parsed path (without url "percent encoding") held in the HTTP request first line.
func (rfl *RequestFirstLine) Path() string {
	rfl.parse()
	if rfl.parsedPath {
		return rfl.pathParsed
	}

	parsed, err := url.PathUnescape(rfl.path)
	if err != nil {
		rfl.pathParsed = rfl.path
	} else {
		rfl.pathParsed = parsed
	}
	rfl.parsedPath = true
	return rfl.pathParsed
}

// Protocol returns the protocol held in the HTTP request first line.
func (rfl *RequestFirstLine) Protocol() string {
	rfl.parse()
	return rfl.proto
}

func (rfl *RequestFirstLine) parse() {
	if rfl.parsed || rfl.raw == "" {
		return
	}

	s := strings.Split(rfl.raw, " ")
	if len(s) != 3 {
		return
	}
	rfl.method = s[0]
	rfl.path = s[1]
	rfl.proto = s[2]

	rfl.parsed = true
}

func (rfl RequestFirstLine) String() string {
	return rfl.raw
}
<commit_msg>apachelog: add support for url and query parameters in RequestFirstLine<commit_after>package apachelog

import (
	"net/url"
	"strings"
	"time"
)

// An AccessLogEntry represents a line of an access log.
type AccessLogEntry struct {
	RemoteIPAddr        string            // Remote IP address
	LocalIPAddr         string            // Local IP address
	ResponseSize        int64             // Size of response in bytes, excluding HTTP headers
	Cookies             map[string]string // Cookies value
	ElapsedTime         int64             // Time taken to serve the request, in microseconds
	EnvVars             map[string]string // Content of the environment variables
	Headers             map[string]string // Content of the request headers
	Filename            string            // Filename
	RemoteHost          string            // Remote host
	RequestProto        string            // Request protocol
	KeepAliveRequests   int64             // Number of keepalive requests handled on this connection
	RemoteLogname       string            // Remote logname. A "-" is returned, when not supplied
	RequestMethod       string            // Request method
	Port                string            // Canonical port of the server serving the request
	ProcessID           int64             // Process ID of the child that serviced the request
	QueryString         string            // Query string (prepended with a ? if exists)
	RequestFirstLine    RequestFirstLine  // First line of the request
	Status              string            // Status
	Time                time.Time         // Time the request was received (standard english format)
	ElapsedTimeSec      int64             // Time taken to serve the request, in seconds
	RemoteUser          string            // Remote user (from auth)
	URLPath             string            // URL path requested, not including any query string
	CanonicalServerName string            // Canonical ServerName of the server serving the request
	ServerName          string            // Server name according to the UseCanonicalName setting
	BytesReceived       int64             // Bytes received, including request and headers
	BytesSent           int64             // Bytes sent, including headers
}

// RequestFirstLine is a handy structure to hold the first line of the HTTP
// request. It provides accessors to access the HTTP method, the path and the
// protocol information contained in the raw first line.
type RequestFirstLine struct {
	raw string

	// Extracted fields
	method      string
	path        string
	pathParsed  string
	proto       string
	url         url.URL
	queryValues url.Values

	parsedQueryValues bool
	parsedURL         bool
	parsedPath        bool
	parsed            bool
}

// NewRequestFirstLine creates a new RequestFirstLine with the supplied raw
// first line.
//
// This function does not parse or do anything else than initializing the
// structure.
func NewRequestFirstLine(raw string) RequestFirstLine {
	return RequestFirstLine{
		raw: raw,
	}
}

// Method returns the method held in the HTTP request first line.
func (rfl *RequestFirstLine) Method() string {
	rfl.parse()
	return rfl.method
}

// RawPath returns the raw path held in the HTTP request first line.
func (rfl *RequestFirstLine) RawPath() string {
	rfl.parse()
	return rfl.path
}

// Path returns the parsed path (without url "percent encoding") held in the HTTP request first line.
func (rfl *RequestFirstLine) Path() string {
	rfl.parse()
	if rfl.parsedPath {
		return rfl.pathParsed
	}

	parsed, err := url.PathUnescape(rfl.path)
	if err != nil {
		rfl.pathParsed = rfl.path
	} else {
		rfl.pathParsed = parsed
	}
	rfl.parsedPath = true
	return rfl.pathParsed
}

// URL returns url.URL structure derived from the raw path
func (rfl *RequestFirstLine) URL() url.URL {

	if rfl.parsedURL {
		return rfl.url
	}

	urlParsed, err := url.ParseRequestURI(rfl.RawPath())
	if err != nil {
		rfl.url = url.URL{RawPath: rfl.RawPath()}
	} else {
		rfl.url = *urlParsed
	}
	rfl.parsedURL = true

	return rfl.url
}

// QueryValues returns query values (url.Values) derived from URL.RawQuery
func (rfl *RequestFirstLine) QueryValues() url.Values {

	if rfl.parsedQueryValues {
		return rfl.queryValues
	}

	query := rfl.URL().RawQuery
	val, _ := url.ParseQuery(query)
	rfl.queryValues = val
	rfl.parsedQueryValues = true
	return rfl.queryValues
}

// Protocol returns the protocol held in the HTTP request first line.
func (rfl *RequestFirstLine) Protocol() string {
	rfl.parse()
	return rfl.proto
}

func (rfl *RequestFirstLine) parse() {
	if rfl.parsed || rfl.raw == "" {
		return
	}

	s := strings.Split(rfl.raw, " ")
	if len(s) != 3 {
		return
	}
	rfl.method = s[0]
	rfl.path = s[1]
	rfl.proto = s[2]

	rfl.parsed = true
}

func (rfl RequestFirstLine) String() string {
	return rfl.raw
}
<|endoftext|>