<commit_before>package eapi

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
    "io/ioutil"
)

type Parameters struct {
	Version int      `json:"version"`
	Cmds    []string `json:"cmds"`
	Format  string   `json:"format"`
}

type Request struct {
	Jsonrpc string     `json:"jsonrpc"`
	Method  string     `json:"method"`
	Params  Parameters `json:"params"`
	Id      string     `json:"id"`
}

type JsonRpcResponse struct {
	Jsonrpc string                   `json:"jsonrpc"`
	Result  []map[string]interface{} `json:"result"`
	Error   map[string]interface{}   `json:"error"`
	Id      string                   `json:"id"`
}

type ShowVersion struct {
	ModelName        string  `json:"modelName"`
	InternalVersion  string  `json:"internalVersion"`
	SystemMacAddress string  `json:"systemMacAddress"`
	SerialNumber     string  `json:"serialNumber"`
	MemTotal         float64 `json:"memTotal"`
	BootupTimestap   float64 `json:"bootupTimestamp"`
	MemFree          float64 `json:"memFree"`
	Version          string  `json:"version"`
	Architecture     string  `json:"architecture"`
	InternalBuildId  string  `json:"internalBuildId"`
	HardwareRevision string  `json:"hardwareRevision"`
}

type ShowLldpNeighbors struct {
    TablesDeletes   int `json:"tablesDeletes"`
    TablesAgeOuts   int `json:"tablesAgeOuts"`
    TablesDrops     int `json:"tablesDrops"`
    TablesInserts   int `json:"tablesInserts"`
    TablesLastChangeTime   int `json:"tablesLastChangeTime"`
    LldpNeighbors   []LldpNeighbor `json:"lldpNeighbors"`
}

type LldpNeighbor struct{
    NeighborDevice  string
    NeighborPort    string
    Port            string
    Ttl             int
}

//TODO: May need to replace Interface with a generic interface
type ShowInterfaces struct {
	Interfaces map[string]Interface `json:"interfaces"`
}

// TODO: Account for different interface types, this is specific to Eth
type Interface struct {
	Bandwidth                 int
	BurnedInAddress           string
	Description               string //`json:"description"`
	ForwardingModel           string
	Hardware                  string
	InterfaceAddress          []InterfaceAddress
	InterfaceCounters         EthInterfaceCounters
	InterfaceMembership       string
	InterfaceStatistics       InterfaceStatistics
	InterfaceStatus           string
	L2Mtu                     int
	LastStatusChangeTimestamp float64
	LineProtocolStatus        string
	Mtu                       int
	Name                      string
	PhysicalAddress           string
}

type InterfaceAddress struct {
	BroadcastAddress       string
	PrimaryIp              IpAddress
	SecondaryIps           interface{}
	SecondaryIpOrderedList []IpAddress
	VirtualIp              IpAddress
}

type IpAddress struct {
	Address string
	MaskLen int
}

type EthInterfaceCounters struct {
	CounterRefreshTime float64
	InBroadcastPkts    int
	InDiscards         int
	InMulticastPkts    int
	InOctets           int
	InUcastPkts        int
	InputErrorsDetail  PhysicalInputErrors
	LastClear          float64
	LinkStatusChanges  int
	OutBroadcastPkts   int
	OutDiscards        int
	OutMulticastPkts   int
	OutOctets          int
	OutUcastPkts       int
	OutErrorsDetail    PhysicalOutputErrors
	TotalInErrors      int
	TotalOutErrors     int
}

type PhysicalInputErrors struct {
	AlignmentErrots int
	FcsErrors       int
	GiantFrames     int
	RuntFrames      int
	RxPause         int
	SymbolErrors    int
}

type PhysicalOutputErrors struct {
	Collisions            int
	DeferredTransmissions int
	LateCollisions        int
	TxPause               int
}

type InterfaceStatistics struct {
	InBitsRate     float64
	OutBitsRate    float64
	InPktsRate     float64
	UpdateInterval int
	OutPktsRate    float64
}

func call(url string, cmds []string, format string) *http.Response {
   p := Parameters{1, cmds, format}
    req := Request{"2.0", "runCmds", p, "1"}
    buf, err := json.Marshal(req)
    resp := new(http.Response)
    if err != nil {
        panic(err)
    }
    client := &http.Client{}
    if strings.HasPrefix(url, "https") {
        tr := &http.Transport{
            TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
        }
        client = &http.Client{Transport: tr}
    }
    resp, err = client.Post(url, "application/json", bytes.NewReader(buf))

    if err != nil {
        fmt.Println(err)
        panic(err)
    }
    return resp
}

func RawCall(url string, cmds []string, format string) []byte {
    resp := call(url, cmds, format)
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        fmt.Println("Error calling")
        fmt.Println(err)
    }
    return body
}

func Call(url string, cmds []string, format string) JsonRpcResponse {
	p := Parameters{1, cmds, format}
	req := Request{"2.0", "runCmds", p, "1"}
	buf, err := json.Marshal(req)
	resp := new(http.Response)
	if err != nil {
		panic(err)
	}
	client := &http.Client{}
	if strings.HasPrefix(url, "https") {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client = &http.Client{Transport: tr}
	}
	resp, err = client.Post(url, "application/json", bytes.NewReader(buf))
	defer resp.Body.Close()

	if err != nil {
		fmt.Println(err)
		panic(err)
	}
	return decodeEapiResponse(resp)
}

func decodeEapiResponse(resp *http.Response) JsonRpcResponse {
	dec := json.NewDecoder(resp.Body)
	var v JsonRpcResponse
	if err := dec.Decode(&v); err != nil {
		log.Println(err)
	}
	return v
}
<commit_msg>Added flows for directflow<commit_after>package eapi

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
    "io/ioutil"
)

type Parameters struct {
	Version int      `json:"version"`
	Cmds    []string `json:"cmds"`
	Format  string   `json:"format"`
}

type Request struct {
	Jsonrpc string     `json:"jsonrpc"`
	Method  string     `json:"method"`
	Params  Parameters `json:"params"`
	Id      string     `json:"id"`
}

type JsonRpcResponse struct {
	Jsonrpc string                   `json:"jsonrpc"`
	Result  []map[string]interface{} `json:"result"`
	Error   map[string]interface{}   `json:"error"`
	Id      string                   `json:"id"`
}

type ShowVersion struct {
	ModelName        string  `json:"modelName"`
	InternalVersion  string  `json:"internalVersion"`
	SystemMacAddress string  `json:"systemMacAddress"`
	SerialNumber     string  `json:"serialNumber"`
	MemTotal         float64 `json:"memTotal"`
	BootupTimestap   float64 `json:"bootupTimestamp"`
	MemFree          float64 `json:"memFree"`
	Version          string  `json:"version"`
	Architecture     string  `json:"architecture"`
	InternalBuildId  string  `json:"internalBuildId"`
	HardwareRevision string  `json:"hardwareRevision"`
}

type ShowLldpNeighbors struct {
    TablesDeletes   int `json:"tablesDeletes"`
    TablesAgeOuts   int `json:"tablesAgeOuts"`
    TablesDrops     int `json:"tablesDrops"`
    TablesInserts   int `json:"tablesInserts"`
    TablesLastChangeTime   int `json:"tablesLastChangeTime"`
    LldpNeighbors   []LldpNeighbor `json:"lldpNeighbors"`
}

type LldpNeighbor struct{
    NeighborDevice  string
    NeighborPort    string
    Port            string
    Ttl             int
}

type ShowDirectFlowFlows struct {
    Flows []Flow
}

type Flow struct {
    Priority    int
    MatchBytes  int
    Name    string
}

//TODO: May need to replace Interface with a generic interface
type ShowInterfaces struct {
	Interfaces map[string]Interface `json:"interfaces"`
}

// TODO: Account for different interface types, this is specific to Eth
type Interface struct {
	Bandwidth                 int
	BurnedInAddress           string
	Description               string //`json:"description"`
	ForwardingModel           string
	Hardware                  string
	InterfaceAddress          []InterfaceAddress
	InterfaceCounters         EthInterfaceCounters
	InterfaceMembership       string
	InterfaceStatistics       InterfaceStatistics
	InterfaceStatus           string
	L2Mtu                     int
	LastStatusChangeTimestamp float64
	LineProtocolStatus        string
	Mtu                       int
	Name                      string
	PhysicalAddress           string
}

type InterfaceAddress struct {
	BroadcastAddress       string
	PrimaryIp              IpAddress
	SecondaryIps           interface{}
	SecondaryIpOrderedList []IpAddress
	VirtualIp              IpAddress
}

type IpAddress struct {
	Address string
	MaskLen int
}

type EthInterfaceCounters struct {
	CounterRefreshTime float64
	InBroadcastPkts    int
	InDiscards         int
	InMulticastPkts    int
	InOctets           int
	InUcastPkts        int
	InputErrorsDetail  PhysicalInputErrors
	LastClear          float64
	LinkStatusChanges  int
	OutBroadcastPkts   int
	OutDiscards        int
	OutMulticastPkts   int
	OutOctets          int
	OutUcastPkts       int
	OutErrorsDetail    PhysicalOutputErrors
	TotalInErrors      int
	TotalOutErrors     int
}

type PhysicalInputErrors struct {
	AlignmentErrots int
	FcsErrors       int
	GiantFrames     int
	RuntFrames      int
	RxPause         int
	SymbolErrors    int
}

type PhysicalOutputErrors struct {
	Collisions            int
	DeferredTransmissions int
	LateCollisions        int
	TxPause               int
}

type InterfaceStatistics struct {
	InBitsRate     float64
	OutBitsRate    float64
	InPktsRate     float64
	UpdateInterval int
	OutPktsRate    float64
}

func call(url string, cmds []string, format string) *http.Response {
   p := Parameters{1, cmds, format}
    req := Request{"2.0", "runCmds", p, "1"}
    buf, err := json.Marshal(req)
    resp := new(http.Response)
    if err != nil {
        panic(err)
    }
    client := &http.Client{}
    if strings.HasPrefix(url, "https") {
        tr := &http.Transport{
            TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
        }
        client = &http.Client{Transport: tr}
    }
    resp, err = client.Post(url, "application/json", bytes.NewReader(buf))

    if err != nil {
        fmt.Println(err)
        panic(err)
    }
    return resp
}

func RawCall(url string, cmds []string, format string) []byte {
    resp := call(url, cmds, format)
    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        fmt.Println("Error calling")
        fmt.Println(err)
    }
    return body
}

func Call(url string, cmds []string, format string) JsonRpcResponse {
	p := Parameters{1, cmds, format}
	req := Request{"2.0", "runCmds", p, "1"}
	buf, err := json.Marshal(req)
	resp := new(http.Response)
	if err != nil {
		panic(err)
	}
	client := &http.Client{}
	if strings.HasPrefix(url, "https") {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client = &http.Client{Transport: tr}
	}
	resp, err = client.Post(url, "application/json", bytes.NewReader(buf))
	defer resp.Body.Close()

	if err != nil {
		fmt.Println(err)
		panic(err)
	}
	return decodeEapiResponse(resp)
}

func decodeEapiResponse(resp *http.Response) JsonRpcResponse {
	dec := json.NewDecoder(resp.Body)
	var v JsonRpcResponse
	if err := dec.Decode(&v); err != nil {
		log.Println(err)
	}
	return v
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	"github.com/codegangsta/cli"
	"github.com/mathpl/go-tsdmetrics"
	"github.com/rcrowley/go-metrics"
)

var version = "0.4"

func generateConsumerLag(r tsdmetrics.TaggedRegistry) {
	valsSent := make(map[string]int64, 0)
	valsCommitted := make(map[string]int64, 0)
	valsHWM := make(map[string]int64, 0)

	fn := func(n string, tm tsdmetrics.TaggedMetric) {
		switch n {
		case "kafka_httpcat.consumer.sent":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsSent[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.committed":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsCommitted[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.high_water_mark":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsHWM[tm.GetTags()["partition"]] = m.Value()
			}
		}
	}

	r.Each(fn)

	for partition, hwmOffset := range valsHWM {
		if sentOffset, ok := valsSent[partition]; ok {
			i := r.GetOrRegister("consumer.sent.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - sentOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}

		if committedOffset, ok := valsCommitted[partition]; ok {
			i := r.GetOrRegister("consumer.committed.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - committedOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}
}

func commaDelimitedToStringList(s string) []string {
	list := strings.Split(s, ",")
	cleanList := make([]string, 0)
	for _, v := range list {
		c := strings.TrimSpace(v)
		if c != "" {
			cleanList = append(cleanList, c)
		}
	}
	return cleanList
}

func stringListToHeaderMap(l []string) (map[string][]string, error) {
	headers := make(map[string][]string, len(l))
	for _, h := range l {
		idx := strings.Index(h, ":")
		if idx == -1 {
			return nil, fmt.Errorf("Unable to parse header %s", h)
		}
		headers[h[0:idx]] = append(headers[h[0:idx]], strings.TrimSpace(h[idx+1:]))
	}

	return headers, nil
}

func commaDelimitedToIntList(s string) ([]int, error) {
	list := strings.Split(s, ",")
	intList := make([]int, len(list))
	for i, v := range list {
		var err error
		intList[i], err = strconv.Atoi(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
	}
	return intList, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "kafka_httpcat"
	app.Usage = "Forward kafka data to http endpoint"
	app.Version = version

	host, err := os.Hostname()
	if err != nil {
		log.Printf("Unable to get hostname: %s", err)
		os.Exit(1)
	}

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:   "verbosity",
			Value:  2,
			Usage:  "verbosity (0-5)",
			EnvVar: "VERBOSITY",
		},
		cli.StringFlag{
			Name:   "target-host-list, t",
			Usage:  "Comma delimited target hosts",
			EnvVar: "TARGET_HOST_LIST",
		},
		cli.StringFlag{
			Name:   "target-path, p",
			Usage:  "HTTP path",
			EnvVar: "TARGET_PATH",
		},
		cli.StringFlag{
			Name:   "headers, H",
			Usage:  "Comma delimited headers",
			EnvVar: "HTTP_HEADERS",
		},
		cli.StringFlag{
			Name:   "method, m",
			Value:  "POST",
			Usage:  "HTTP method",
			EnvVar: "HTTP_METHOD",
		},
		cli.StringFlag{
			Name:   "expected-statuses, e",
			Value:  "200",
			Usage:  "Comma delimited list of expected HTTP status",
			EnvVar: "HTTP_EXPECTED_STATUSES",
		},
		cli.StringFlag{
			Name:   "kafka-broker-list, b",
			Usage:  "Comma delimited kafka broker list.",
			EnvVar: "KAFKA_BROKER_LIST",
		},
		cli.StringFlag{
			Name:   "kafka-topic, T",
			Usage:  "Kafka topic.",
			EnvVar: "KAFKA_TOPIC",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-id, i",
			Value:  fmt.Sprintf("%s-%d", host, os.Getpid()),
			Usage:  "Kafka consumer id.",
			EnvVar: "KAFKA_CONSUMER_ID",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-group, g",
			Usage:  "Kafka consumer group.",
			EnvVar: "KAFKA_CONSUMER_GROUP",
		},
		cli.StringFlag{
			Name:   "kafka-start-offset, o",
			Value:  "newest",
			Usage:  "Kafka offset to start with (newest or oldest)",
			EnvVar: "KAFKA_START_OFFSET",
		},
		cli.IntFlag{
			Name:   "kafka-commit-batch, c",
			Value:  1000,
			Usage:  "Commit consumed messages every X messages.",
			EnvVar: "KAFKA_COMMIT_BATCH",
		},
		cli.StringFlag{
			Name:   "metrics-report-url, r",
			Usage:  "Where to send OpenTSDB metrics.",
			EnvVar: "METRICS_REPORT_URL",
		},
		cli.StringFlag{
			Name:   "metrics-tags",
			Usage:  "Comma delimited list of default tags",
			EnvVar: "METRICS_TAGS",
		},
	}

	app.Action = func(c *cli.Context) error {
		expectedStatuses, err := commaDelimitedToIntList(c.String("expected-statuses"))
		if err != nil {
			return fmt.Errorf("Expected http status must be an integer: %s", err)
		}

		httpHeaderList := commaDelimitedToStringList(c.String("headers"))
		httpHeaders, err := stringListToHeaderMap(httpHeaderList)
		if err != nil {
			return fmt.Errorf("Unable to parse headers: %s", err)
		}

		targetHosts := commaDelimitedToStringList(c.String("target-host-list"))
		defaultTags, err := tsdmetrics.TagsFromString(c.String("metrics-tags"))
		if err != nil {
			return err
		}
		defaultTags = defaultTags.AddTags(tsdmetrics.Tags{"topic": c.String("kafka-topic"), "consumergroup": c.String("kafka-consumer-group")})

		rootRegistry := tsdmetrics.NewSegmentedTaggedRegistry("", defaultTags, nil)

		goMetrics := tsdmetrics.NewSegmentedTaggedRegistry("", tsdmetrics.Tags{"j_app": "http_kafkacat"}, rootRegistry)
		tsdmetrics.RegisterTaggedRuntimeMemStats(goMetrics)

		metricsRegistry := tsdmetrics.NewSegmentedTaggedRegistry("kafka_httpcat", nil, rootRegistry)

		metricsTsdb := tsdmetrics.TaggedOpenTSDB{Addr: c.String("metrics-report-url"), Registry: metricsRegistry, FlushInterval: 15 * time.Second, DurationUnit: time.Millisecond, Format: tsdmetrics.Json}

		log.Printf("Connecting to: %s", c.String("kafka-broker-list"))

		// Init config
		config := cluster.NewConfig()
		config.Consumer.Return.Errors = true
		config.Group.Return.Notifications = true

		switch c.String("kafka-start-offset") {
		case "oldest":
			config.Consumer.Offsets.Initial = sarama.OffsetOldest
		case "newest":
			config.Consumer.Offsets.Initial = sarama.OffsetNewest
		default:
			return fmt.Errorf("offset should be `oldest` or `newest`")
		}

		// Init consumer, consume errors & messages
		consumer, err := cluster.NewConsumer(strings.Split(c.String("kafka-broker-list"), ","), c.String("kafka-consumer-group"), strings.Split(c.String("kafka-topic"), ","), config)
		if err != nil {
			log.Fatal(err)
		}

		go func() {
			for err := range consumer.Errors() {
				fmt.Printf("Error: %s\n", err.Error())
			}
		}()

		go func() {
			for note := range consumer.Notifications() {
				fmt.Printf("Rebalanced: %+v\n", note)
			}
		}()

		batchSize := int64(c.Int("kafka-commit-batch"))

		go func() {
			sender := NewHTTPSender(targetHosts, c.String("target-path"), c.String("method"), httpHeaders, expectedStatuses)
			for msg := range consumer.Messages() {
				if err := sender.RRSend(msg.Value); err != nil {
					log.Printf("Error send data: %s", err)
				}

				tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", msg.Partition)}
				s := metricsRegistry.GetOrRegister("consumer.sent", tags, metrics.NewGauge())
				s.(metrics.Gauge).Update(msg.Offset)

				if msg.Offset%batchSize == 0 {
					consumer.MarkOffset(msg, "")
					c := metricsRegistry.GetOrRegister("consumer.committed", tags, metrics.NewGauge())
					c.(metrics.Gauge).Update(msg.Offset)
				}
			}
		}()

		collectFn := tsdmetrics.RuntimeCaptureFn
		collectFn = append(collectFn, generateConsumerLag)

		go metricsTsdb.RunWithPreprocessing(context.Background(), collectFn)

		wait := make(chan os.Signal, 1)
		signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM)

		<-wait
		if err := consumer.Close(); err != nil {
			fmt.Println("Failed to close consumer: ", err)
		}

		return nil
	}

	err = app.Run(os.Args)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>fix high water marks<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	"github.com/codegangsta/cli"
	"github.com/mathpl/go-tsdmetrics"
	"github.com/rcrowley/go-metrics"
)

var version = "0.4"

func generateConsumerLag(r tsdmetrics.TaggedRegistry) {
	valsSent := make(map[string]int64, 0)
	valsCommitted := make(map[string]int64, 0)
	valsHWM := make(map[string]int64, 0)

	fn := func(n string, tm tsdmetrics.TaggedMetric) {
		switch n {
		case "kafka_httpcat.consumer.sent":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsSent[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.committed":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsCommitted[tm.GetTags()["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.high_water_mark":
			if m, ok := tm.GetMetric().(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsHWM[tm.GetTags()["partition"]] = m.Value()
			}
		}
	}

	r.Each(fn)

	for partition, hwmOffset := range valsHWM {
		if sentOffset, ok := valsSent[partition]; ok {
			i := r.GetOrRegister("kafka_httpcat.consumer.sent.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - sentOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}

		if committedOffset, ok := valsCommitted[partition]; ok {
			i := r.GetOrRegister("kafka_httpcat.consumer.committed.offset_lag", tsdmetrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := hwmOffset - committedOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}
}

func commaDelimitedToStringList(s string) []string {
	list := strings.Split(s, ",")
	cleanList := make([]string, 0)
	for _, v := range list {
		c := strings.TrimSpace(v)
		if c != "" {
			cleanList = append(cleanList, c)
		}
	}
	return cleanList
}

func stringListToHeaderMap(l []string) (map[string][]string, error) {
	headers := make(map[string][]string, len(l))
	for _, h := range l {
		idx := strings.Index(h, ":")
		if idx == -1 {
			return nil, fmt.Errorf("Unable to parse header %s", h)
		}
		headers[h[0:idx]] = append(headers[h[0:idx]], strings.TrimSpace(h[idx+1:]))
	}

	return headers, nil
}

func commaDelimitedToIntList(s string) ([]int, error) {
	list := strings.Split(s, ",")
	intList := make([]int, len(list))
	for i, v := range list {
		var err error
		intList[i], err = strconv.Atoi(strings.TrimSpace(v))
		if err != nil {
			return nil, err
		}
	}
	return intList, nil
}

func main() {
	app := cli.NewApp()
	app.Name = "kafka_httpcat"
	app.Usage = "Forward kafka data to http endpoint"
	app.Version = version

	host, err := os.Hostname()
	if err != nil {
		log.Printf("Unable to get hostname: %s", err)
		os.Exit(1)
	}

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:   "verbosity",
			Value:  2,
			Usage:  "verbosity (0-5)",
			EnvVar: "VERBOSITY",
		},
		cli.StringFlag{
			Name:   "target-host-list, t",
			Usage:  "Comma delimited target hosts",
			EnvVar: "TARGET_HOST_LIST",
		},
		cli.StringFlag{
			Name:   "target-path, p",
			Usage:  "HTTP path",
			EnvVar: "TARGET_PATH",
		},
		cli.StringFlag{
			Name:   "headers, H",
			Usage:  "Comma delimited headers",
			EnvVar: "HTTP_HEADERS",
		},
		cli.StringFlag{
			Name:   "method, m",
			Value:  "POST",
			Usage:  "HTTP method",
			EnvVar: "HTTP_METHOD",
		},
		cli.StringFlag{
			Name:   "expected-statuses, e",
			Value:  "200",
			Usage:  "Comma delimited list of expected HTTP status",
			EnvVar: "HTTP_EXPECTED_STATUSES",
		},
		cli.StringFlag{
			Name:   "kafka-broker-list, b",
			Usage:  "Comma delimited kafka broker list.",
			EnvVar: "KAFKA_BROKER_LIST",
		},
		cli.StringFlag{
			Name:   "kafka-topic, T",
			Usage:  "Kafka topic.",
			EnvVar: "KAFKA_TOPIC",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-id, i",
			Value:  fmt.Sprintf("%s-%d", host, os.Getpid()),
			Usage:  "Kafka consumer id.",
			EnvVar: "KAFKA_CONSUMER_ID",
		},
		cli.StringFlag{
			Name:   "kafka-consumer-group, g",
			Usage:  "Kafka consumer group.",
			EnvVar: "KAFKA_CONSUMER_GROUP",
		},
		cli.StringFlag{
			Name:   "kafka-start-offset, o",
			Value:  "newest",
			Usage:  "Kafka offset to start with (newest or oldest)",
			EnvVar: "KAFKA_START_OFFSET",
		},
		cli.IntFlag{
			Name:   "kafka-commit-batch, c",
			Value:  1000,
			Usage:  "Commit consumed messages every X messages.",
			EnvVar: "KAFKA_COMMIT_BATCH",
		},
		cli.StringFlag{
			Name:   "metrics-report-url, r",
			Usage:  "Where to send OpenTSDB metrics.",
			EnvVar: "METRICS_REPORT_URL",
		},
		cli.StringFlag{
			Name:   "metrics-tags",
			Usage:  "Comma delimited list of default tags",
			EnvVar: "METRICS_TAGS",
		},
	}

	app.Action = func(c *cli.Context) error {
		expectedStatuses, err := commaDelimitedToIntList(c.String("expected-statuses"))
		if err != nil {
			return fmt.Errorf("Expected http status must be an integer: %s", err)
		}

		httpHeaderList := commaDelimitedToStringList(c.String("headers"))
		httpHeaders, err := stringListToHeaderMap(httpHeaderList)
		if err != nil {
			return fmt.Errorf("Unable to parse headers: %s", err)
		}

		targetHosts := commaDelimitedToStringList(c.String("target-host-list"))
		defaultTags, err := tsdmetrics.TagsFromString(c.String("metrics-tags"))
		if err != nil {
			return err
		}
		defaultTags = defaultTags.AddTags(tsdmetrics.Tags{"topic": c.String("kafka-topic"), "consumergroup": c.String("kafka-consumer-group")})

		rootRegistry := tsdmetrics.NewSegmentedTaggedRegistry("", defaultTags, nil)
		tsdmetrics.RegisterTaggedRuntimeMemStats(rootRegistry)
		metricsRegistry := tsdmetrics.NewSegmentedTaggedRegistry("kafka_httpcat", nil, rootRegistry)
		metricsTsdb := tsdmetrics.TaggedOpenTSDB{Addr: c.String("metrics-report-url"), Registry: rootRegistry, FlushInterval: 15 * time.Second, DurationUnit: time.Millisecond, Format: tsdmetrics.Json}

		log.Printf("Connecting to: %s", c.String("kafka-broker-list"))

		// Init config
		config := cluster.NewConfig()
		config.Consumer.Return.Errors = true
		config.Group.Return.Notifications = true

		switch c.String("kafka-start-offset") {
		case "oldest":
			config.Consumer.Offsets.Initial = sarama.OffsetOldest
		case "newest":
			config.Consumer.Offsets.Initial = sarama.OffsetNewest
		default:
			return fmt.Errorf("offset should be `oldest` or `newest`")
		}

		brokerList := strings.Split(c.String("kafka-broker-list"), ",")
		topicList := strings.Split(c.String("kafka-topic"), ",")

		// Init consumer, consume errors & messages
		consumer, err := cluster.NewConsumer(brokerList, c.String("kafka-consumer-group"), topicList, config)
		if err != nil {
			log.Fatal(err)
		}

		go func() {
			for err := range consumer.Errors() {
				fmt.Printf("Error: %s\n", err.Error())
			}
		}()

		go func() {
			for note := range consumer.Notifications() {
				fmt.Printf("Rebalanced: %+v\n", note)
			}
		}()

		batchSize := int64(c.Int("kafka-commit-batch"))

		go func() {
			for {
				hwms := consumer.HighWaterMarks()
				for _, hwm := range hwms {
					for partition, offset := range hwm {
						tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", partition)}
						m := metricsRegistry.GetOrRegister("consumer.high_water_mark", tags, metrics.NewGauge())
						m.(metrics.Gauge).Update(offset)
					}
				}
				time.Sleep(5 * time.Second)
			}
		}()

		go func() {
			sender := NewHTTPSender(targetHosts, c.String("target-path"), c.String("method"), httpHeaders, expectedStatuses)
			for msg := range consumer.Messages() {
				if err := sender.RRSend(msg.Value); err != nil {
					log.Printf("Error send data: %s\n", err)
				}

				tags := tsdmetrics.Tags{"partition": fmt.Sprintf("%d", msg.Partition)}
				s := metricsRegistry.GetOrRegister("consumer.sent", tags, metrics.NewGauge())
				s.(metrics.Gauge).Update(msg.Offset)

				if msg.Offset%batchSize == 0 {
					consumer.MarkOffset(msg, "")
					c := metricsRegistry.GetOrRegister("consumer.committed", tags, metrics.NewGauge())
					c.(metrics.Gauge).Update(msg.Offset)
					log.Printf("Commited offset partition:%d offset:%d\n", msg.Partition, msg.Offset)
				}
			}
		}()

		collectFn := tsdmetrics.RuntimeCaptureFn
		collectFn = append(collectFn, generateConsumerLag)

		go metricsTsdb.RunWithPreprocessing(context.Background(), collectFn)

		wait := make(chan os.Signal, 1)
		signal.Notify(wait, syscall.SIGINT, syscall.SIGTERM)

		<-wait
		consumer.CommitOffsets()

		if err := consumer.Close(); err != nil {
			fmt.Printf("Failed to close consumer: %s\n", err)
		}

		return nil
	}

	err = app.Run(os.Args)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

// This software is an expense tracker i made to read the transactions exported from
// my bank account.
// Currently is just crap code... :D

import (
	"database/sql"
	"errors"
	"expensetracker/categories"
	"expensetracker/datastore"
	"expensetracker/interactor"
	"fmt"
	"log"
	"os"

	_ "github.com/mattn/go-sqlite3"
	flag "github.com/ogier/pflag"
	"github.com/shopspring/decimal"
	"github.com/tealeg/xlsx"
)

var DATABASE_NAME string = "./expensetracker.db"
var DATABASE_ENGINE = "sqlite3"

func toExcel(value decimal.Decimal, description string) {
	var file *xlsx.File
	var sheet *xlsx.Sheet
	var row *xlsx.Row
	var cell *xlsx.Cell
	var err error

	file = xlsx.NewFile()
	sheet, err = file.AddSheet("Sheet1")
	if err != nil {
		fmt.Printf(err.Error())
	}
	row = sheet.AddRow()
	cell = row.AddCell()
	cell.Value = description
	cell = row.AddCell()
	// cell.Value = strconv.FormatFloat(value, 'f', 2, 64)
	cell.Value = value.String()

	err = file.Save("MyXLSXFile.xlsx")
	if err != nil {
		fmt.Printf(err.Error())
	}
}

// Database is the holder for database operations
type Database struct {
	db *sql.DB
}

// NewDBConnection provides a new connection to the database
func (d *Database) NewDBConnection() error {
	// log.Println("Creating new database connection")

	if _, err := os.Stat(DATABASE_NAME); os.IsNotExist(err) && d.db == nil {
		// os.Remove(DATABASE_NAME)
		db, err := sql.Open(DATABASE_ENGINE, DATABASE_NAME)
		if err != nil {
			return err
		}

		d.db = db

		return nil
	}

	if _, err := os.Stat(DATABASE_NAME); os.IsNotExist(err) && d.db != nil {
		log.Fatal("Connection exists, but database file does not... wtf??")
	}

	if d.db == nil {
		os.Remove(DATABASE_NAME)
		db, err := sql.Open(DATABASE_ENGINE, DATABASE_NAME)
		if err != nil {
			return err
		}

		d.db = db

		return nil
	}

	return errors.New("Database connection already exists")
}

// CreateExpenseDatabase creates the expense database
func (d *Database) CreateExpenseDatabase() error {
	// log.Println("Creating new database")

	sqlStmt := `
	create table expenses (id integer not null primary key, description text, value float);
	create table credits (id integer not null primary key, description text, value float);
	delete from expenses;
	delete from expenses;
	`
	d.db.Ping()
	_, err := d.db.Exec(sqlStmt)
	if err != nil {
		// log.Printf("%q: %s\n", err, sqlStmt)
		return err
	}

	return nil
}

func (d *Database) Close() {
	d.db.Close()
}

func (d *Database) SaveExpense(value decimal.Decimal, description string) {
	_, err := d.db.Exec("insert into expenses(value, description) values(?, ?)", value, description)
	if err != nil {
		log.Fatal("Error while saving expense: ", err)
	}
}

// func (d *Database) SaveCredit(value float64, description string) {
// 	tx, err := d.db.Begin()
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	stmt, err := tx.Prepare("insert into credits(id, name) values(?, ?)")
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	defer stmt.Close()
// 	_, err = stmt.Exec()
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	tx.Commit()
// }

// database := Database{}
// if err := database.NewDBConnection(); err != nil {
// 	log.Fatal(err)
// }
// defer database.Close()
// if err := database.CreateExpenseDatabase(); err != nil {
// 	log.Fatal(err)
// }
// database.SaveExpense(decimal.NewFromFloat(1), "descricao")

func main() {

	var inputFilePath string
	flag.StringVarP(&inputFilePath, "load", "l", "", "Specify the path to the input file")

	var showReport bool
	flag.BoolVarP(&showReport, "report", "r", false, "Show report")

	var showBalance bool
	flag.BoolVarP(&showBalance, "balance", "b", false, "Show current balance")

	var createCategory string
	flag.StringVarP(&createCategory, "category", "c", "", "Create category")

	flag.Parse()

	if createCategory != "" {
		i := categories.Interactor{
			Repository: nil,
		}
		_, err := i.NewCategory(createCategory)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		file, err := datastore.OpenFile(inputFilePath)
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()

		records, err := interactor.ImportData(file)
		if err != nil {
			log.Fatal("Failed to read input data ", err)
		}

		if showReport {
			interactor.MonthlyReport(records)
		}

		if showBalance {
			fmt.Println(interactor.CurrentBalance().String())
		}

	}
}
<commit_msg>Comments legacy  sql stuff<commit_after>package main

// This software is an expense tracker i made to read the transactions exported from
// my bank account.
// Currently is just crap code... :D

import (
	"expensetracker/categories"
	"expensetracker/datastore"
	"expensetracker/interactor"
	"fmt"
	"log"

	// _ "github.com/mattn/go-sqlite3"
	flag "github.com/ogier/pflag"
	"github.com/shopspring/decimal"
	"github.com/tealeg/xlsx"
)

var DATABASE_NAME string = "./expensetracker.db"
var DATABASE_ENGINE = "sqlite3"

func toExcel(value decimal.Decimal, description string) {
	var file *xlsx.File
	var sheet *xlsx.Sheet
	var row *xlsx.Row
	var cell *xlsx.Cell
	var err error

	file = xlsx.NewFile()
	sheet, err = file.AddSheet("Sheet1")
	if err != nil {
		fmt.Printf(err.Error())
	}
	row = sheet.AddRow()
	cell = row.AddCell()
	cell.Value = description
	cell = row.AddCell()
	// cell.Value = strconv.FormatFloat(value, 'f', 2, 64)
	cell.Value = value.String()

	err = file.Save("MyXLSXFile.xlsx")
	if err != nil {
		fmt.Printf(err.Error())
	}
}

// // Database is the holder for database operations
// type Database struct {
// 	db *sql.DB
// }

// // NewDBConnection provides a new connection to the database
// func (d *Database) NewDBConnection() error {
// 	// log.Println("Creating new database connection")

// 	if _, err := os.Stat(DATABASE_NAME); os.IsNotExist(err) && d.db == nil {
// 		// os.Remove(DATABASE_NAME)
// 		db, err := sql.Open(DATABASE_ENGINE, DATABASE_NAME)
// 		if err != nil {
// 			return err
// 		}

// 		d.db = db

// 		return nil
// 	}

// 	if _, err := os.Stat(DATABASE_NAME); os.IsNotExist(err) && d.db != nil {
// 		log.Fatal("Connection exists, but database file does not... wtf??")
// 	}

// 	if d.db == nil {
// 		os.Remove(DATABASE_NAME)
// 		db, err := sql.Open(DATABASE_ENGINE, DATABASE_NAME)
// 		if err != nil {
// 			return err
// 		}

// 		d.db = db

// 		return nil
// 	}

// 	return errors.New("Database connection already exists")
// }

// // CreateExpenseDatabase creates the expense database
// func (d *Database) CreateExpenseDatabase() error {
// 	// log.Println("Creating new database")

// 	sqlStmt := `
// 	create table expenses (id integer not null primary key, description text, value float);
// 	create table credits (id integer not null primary key, description text, value float);
// 	delete from expenses;
// 	delete from expenses;
// 	`
// 	d.db.Ping()
// 	_, err := d.db.Exec(sqlStmt)
// 	if err != nil {
// 		// log.Printf("%q: %s\n", err, sqlStmt)
// 		return err
// 	}

// 	return nil
// }

// func (d *Database) Close() {
// 	d.db.Close()
// }

// func (d *Database) SaveExpense(value decimal.Decimal, description string) {
// 	_, err := d.db.Exec("insert into expenses(value, description) values(?, ?)", value, description)
// 	if err != nil {
// 		log.Fatal("Error while saving expense: ", err)
// 	}
// }

// func (d *Database) SaveCredit(value float64, description string) {
// 	tx, err := d.db.Begin()
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	stmt, err := tx.Prepare("insert into credits(id, name) values(?, ?)")
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	defer stmt.Close()
// 	_, err = stmt.Exec()
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	tx.Commit()
// }

// database := Database{}
// if err := database.NewDBConnection(); err != nil {
// 	log.Fatal(err)
// }
// defer database.Close()
// if err := database.CreateExpenseDatabase(); err != nil {
// 	log.Fatal(err)
// }
// database.SaveExpense(decimal.NewFromFloat(1), "descricao")

func main() {

	var inputFilePath string
	flag.StringVarP(&inputFilePath, "load", "l", "", "Specify the path to the input file")

	var showReport bool
	flag.BoolVarP(&showReport, "report", "r", false, "Show report")

	var showBalance bool
	flag.BoolVarP(&showBalance, "balance", "b", false, "Show current balance")

	var createCategory string
	flag.StringVarP(&createCategory, "category", "c", "", "Create category")

	flag.Parse()

	if createCategory != "" {
		i := categories.Interactor{
			Repository: nil,
		}
		_, err := i.NewCategory(createCategory)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		file, err := datastore.OpenFile(inputFilePath)
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()

		records, err := interactor.ImportData(file)
		if err != nil {
			log.Fatal("Failed to read input data ", err)
		}

		if showReport {
			interactor.MonthlyReport(records)
		}

		if showBalance {
			fmt.Println(interactor.CurrentBalance().String())
		}

	}
}
<|endoftext|>


<commit_before>// Go equivalent of the "DNS & BIND" book check-soa program.
// Created by Stephane Bortzmeyer.
package main

import (
	"errors"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"

	"github.com/miekg/dns"
)

const (
	// DefaultTimeout is default timeout many operation in this program will
	// use.
	DefaultTimeout time.Duration = 5 * time.Second
)

var (
	localm          *dns.Msg
	localc          *dns.Client
	conf            *dns.ClientConfig
	domain          string
	assembledDomain string
	nextNs          string
)

func localQuery(qname string, qtype uint16, server string) (*dns.Msg, error) {
	localm.SetQuestion(qname, qtype)
	r, _, err := localc.Exchange(localm, server+":53")
	if err != nil {
		return nil, err
	}
	if r == nil || r.Rcode == dns.RcodeNameError || r.Rcode == dns.RcodeSuccess {
		return r, err
	}

	return nil, errors.New("No name server to answer the question")
}

func getNsRecords(zone string, server string) ([]string, string, error) {
	zone = dns.Fqdn(zone)
	r, err := localQuery(zone, dns.TypeNS, server)
	if err != nil || r == nil {
		return nil, "", err
	}

	var nameservers []string
	var random_ns string

	for _, ans := range r.Answer {
		switch t := ans.(type) {
		case *dns.NS:
			nameserver := t.Ns
			nameservers = append(nameservers, nameserver)
		}
	}

	if len(nameservers) == 0 {
		// No "Answer" given by the server, check the Authority section if
		// additional nameservers were provided.
		for _, ans := range r.Ns {
			switch t := ans.(type) {
			case *dns.NS:
				nameserver := t.Ns
				nameservers = append(nameservers, nameserver)
			}
		}
	}

	if len(nameservers) == 0 {
		return nil, "", errors.New("No nameservers found for " + zone)
	}

	// Pick a random NS record for the next queries
	random_ns = nameservers[rand.Intn(len(nameservers))]

	sort.Strings(nameservers)

	return nameservers, random_ns, nil

}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("%s ZONE\n", os.Args[0])
	}
	domain = os.Args[1]

	rand.Seed(time.Now().Unix())
	var err error
	conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil || conf == nil {
		log.Fatal("Cannot initialize the local resolver: %s\n", err)
	}
	localm = &dns.Msg{
		MsgHdr: dns.MsgHdr{
			RecursionDesired: true,
		},
		Question: make([]dns.Question, 1),
	}
	localc = &dns.Client{
		ReadTimeout: DefaultTimeout,
	}

	// Walk the root until you find the authoritative nameservers
	fmt.Printf("Retrieving list of root nameservers:\n")
	root_nameservers, nextNs, err := getNsRecords(".", conf.Servers[0])
	if err != nil {
		log.Fatal("Query failed: ", err)
	}
	for _, nameserver := range root_nameservers {
		if nameserver == nextNs {
			// We'll use this one for queries
			fmt.Println(" ➡ " + nameserver)
		} else {
			fmt.Println(" - " + nameserver)
		}
	}

	// We have list of root nameservers: split domain, query each part for NS records
	domain_pieces := dns.SplitDomainName(domain)
	assembledDomain = "."
	var ns []string

	for i := len(domain_pieces) - 1; i >= 0; i-- {
		fmt.Println("\n")
		element := domain_pieces[i]
		if assembledDomain == "." {
			assembledDomain = element + "."
		} else {
			assembledDomain = element + "." + assembledDomain
		}

		fmt.Println("Finding nameservers for zone '" + assembledDomain + "' using parent nameserver '" + nextNs + "'")
		ns, nextNs, err = getNsRecords(assembledDomain, nextNs)
		if err != nil {
			fmt.Println("Query failed: ", err)
		}

		// Print the nameservers for this zone, highlight the one we used to query
		for _, nameserver := range ns {
			if nameserver == nextNs && dns.Fqdn(domain) != assembledDomain {
				// We'll use this one for queries
				fmt.Println(" ➡ " + nameserver)
			} else {
				fmt.Println(" - " + nameserver)
			}
		}
	}

}
<commit_msg>go fmt + naming var fixes<commit_after>// Go equivalent of the "DNS & BIND" book check-soa program.
// Created by Stephane Bortzmeyer.
package main

import (
	"errors"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"

	"github.com/miekg/dns"
)

const (
	// DefaultTimeout is default timeout many operation in this program will
	// use.
	DefaultTimeout time.Duration = 5 * time.Second
)

var (
	localm          *dns.Msg
	localc          *dns.Client
	conf            *dns.ClientConfig
	domain          string
	assembledDomain string
	nextNs          string
)

func localQuery(qname string, qtype uint16, server string) (*dns.Msg, error) {
	localm.SetQuestion(qname, qtype)
	r, _, err := localc.Exchange(localm, server+":53")
	if err != nil {
		return nil, err
	}
	if r == nil || r.Rcode == dns.RcodeNameError || r.Rcode == dns.RcodeSuccess {
		return r, err
	}

	return nil, errors.New("No name server to answer the question")
}

func getNsRecords(zone string, server string) ([]string, string, error) {
	zone = dns.Fqdn(zone)
	r, err := localQuery(zone, dns.TypeNS, server)
	if err != nil || r == nil {
		return nil, "", err
	}

	var nameservers []string
	var randomNs string

	for _, ans := range r.Answer {
		switch t := ans.(type) {
		case *dns.NS:
			nameserver := t.Ns
			nameservers = append(nameservers, nameserver)
		}
	}

	if len(nameservers) == 0 {
		// No "Answer" given by the server, check the Authority section if
		// additional nameservers were provided.
		for _, ans := range r.Ns {
			switch t := ans.(type) {
			case *dns.NS:
				nameserver := t.Ns
				nameservers = append(nameservers, nameserver)
			}
		}
	}

	if len(nameservers) == 0 {
		return nil, "", errors.New("No nameservers found for " + zone)
	}

	// Pick a random NS record for the next queries
	randomNs = nameservers[rand.Intn(len(nameservers))]

	sort.Strings(nameservers)

	return nameservers, randomNs, nil

}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("%s ZONE\n", os.Args[0])
	}
	domain = os.Args[1]

	rand.Seed(time.Now().Unix())
	var err error
	conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil || conf == nil {
		log.Fatal("Cannot initialize the local resolver: %s\n", err)
	}
	localm = &dns.Msg{
		MsgHdr: dns.MsgHdr{
			RecursionDesired: true,
		},
		Question: make([]dns.Question, 1),
	}
	localc = &dns.Client{
		ReadTimeout: DefaultTimeout,
	}

	// Walk the root until you find the authoritative nameservers
	fmt.Printf("Retrieving list of root nameservers:\n")
	rootNameservers, nextNs, err := getNsRecords(".", conf.Servers[0])
	if err != nil {
		log.Fatal("Query failed: ", err)
	}
	for _, nameserver := range rootNameservers {
		if nameserver == nextNs {
			// We'll use this one for queries
			fmt.Println(" ➡ " + nameserver)
		} else {
			fmt.Println(" - " + nameserver)
		}
	}

	// We have list of root nameservers: split domain, query each part for NS records
	domainPieces := dns.SplitDomainName(domain)
	assembledDomain = "."
	var ns []string

	for i := len(domainPieces) - 1; i >= 0; i-- {
		fmt.Print("\n")
		element := domainPieces[i]
		if assembledDomain == "." {
			assembledDomain = element + "."
		} else {
			assembledDomain = element + "." + assembledDomain
		}

		fmt.Println("Finding nameservers for zone '" + assembledDomain + "' using parent nameserver '" + nextNs + "'")
		ns, nextNs, err = getNsRecords(assembledDomain, nextNs)
		if err != nil {
			fmt.Println("Query failed: ", err)
		}

		// Print the nameservers for this zone, highlight the one we used to query
		for _, nameserver := range ns {
			if nameserver == nextNs && dns.Fqdn(domain) != assembledDomain {
				// We'll use this one for queries
				fmt.Println(" ➡ " + nameserver)
			} else {
				fmt.Println(" - " + nameserver)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/codegangsta/cli"
	"github.com/gosuri/uitable"

	"iSystem/formats"
	"iSystem/providers/cpu"
	"iSystem/providers/host"
	"iSystem/providers/load"
	"iSystem/providers/mem"
	"iSystem/providers/net"
)

var VERSION string

func main() {
	app := cli.NewApp()
	app.Name = "iSystem"
	app.Usage = "Show Human-Readable system info of *nix."
	app.Authors = []cli.Author{
		{
			Name:  "Frank Yang",
			Email: "codebear4@gmail.com",
		},
	}
	app.Copyright = "Codebear (c) 2016"
	app.Version = VERSION

	var noCPU bool = false

	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "no-cpu", Usage: "Hide CPU information.", Destination: &noCPU},
	}

	app.Action = func(c *cli.Context) {
		infoTable := uitable.New()

		fmt.Println(formats.Title("System information as of ", time.Now().String()))

		if !noCPU {
			infoTable.AddRow("CPU Model:", cpu.ModelName, "Cores:", cpu.Cores)
		}
		infoTable.AddRow("Platform:", host.Platform, "PlatformVersion:", host.PlatformVersion)
		if host.VirtualizationSystem != "" {
			infoTable.AddRow("Virtualization:", host.VirtualizationSystem, "Role:", host.VirtualizationRole)
		}
		infoTable.AddRow("Hostname:", host.HostName, "Users:", host.Users)
		infoTable.AddRow("Boot time:", host.BootTime, "Uptime:", host.Uptime)
		infoTable.AddRow("Processes:", host.Procs, "System load:", load.Load)
		infoTable.AddRow("Memory total:", mem.Total, "Free:", mem.Free)
		infoTable.AddRow("Memory Used:", mem.Used, "Usage:", mem.Usage)
		infoTable.AddRow("Swap total:", mem.SwapTotal, "Free:", mem.SwapFree)
		infoTable.AddRow("Swap Used:", mem.SwapUsed, "Usage:", mem.SwapUsage)
		for i := 0; i <= net.NetInterfaceCount; i += 2 {
			if i >= net.NetInterfaceCount {
				break
			}
			cInter := net.NetInterfaces[i]
			if i < net.NetInterfaceCount-1 {
				nInter := net.NetInterfaces[i+1]
				infoTable.AddRow(cInter.Name+":", cInter.Addr, nInter.Name+":", nInter.Addr)
			} else {
				infoTable.AddRow(cInter.Name+":", cInter.Addr)
			}
		}
		fmt.Println(infoTable)
	}

	app.Run(os.Args)
}
<commit_msg>Rename package namespace.<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/codegangsta/cli"
	"github.com/gosuri/uitable"

	"github.com/iFrankYang/iSystem/formats"
	"github.com/iFrankYang/iSystem/providers/cpu"
	"github.com/iFrankYang/iSystem/providers/host"
	"github.com/iFrankYang/iSystem/providers/load"
	"github.com/iFrankYang/iSystem/providers/mem"
	"github.com/iFrankYang/iSystem/providers/net"
)

var VERSION string

func main() {
	app := cli.NewApp()
	app.Name = "iSystem"
	app.Usage = "Show Human-Readable system info of *nix."
	app.Authors = []cli.Author{
		{
			Name:  "Frank Yang",
			Email: "codebear4@gmail.com",
		},
	}
	app.Copyright = "Codebear (c) 2016"
	app.Version = VERSION

	var noCPU bool = false

	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "no-cpu", Usage: "Hide CPU information.", Destination: &noCPU},
	}

	app.Action = func(c *cli.Context) {
		infoTable := uitable.New()

		fmt.Println(formats.Title("System information as of ", time.Now().String()))

		if !noCPU {
			infoTable.AddRow("CPU Model:", cpu.ModelName, "Cores:", cpu.Cores)
		}
		infoTable.AddRow("Platform:", host.Platform, "PlatformVersion:", host.PlatformVersion)
		if host.VirtualizationSystem != "" {
			infoTable.AddRow("Virtualization:", host.VirtualizationSystem, "Role:", host.VirtualizationRole)
		}
		infoTable.AddRow("Hostname:", host.HostName, "Users:", host.Users)
		infoTable.AddRow("Boot time:", host.BootTime, "Uptime:", host.Uptime)
		infoTable.AddRow("Processes:", host.Procs, "System load:", load.Load)
		infoTable.AddRow("Memory total:", mem.Total, "Free:", mem.Free)
		infoTable.AddRow("Memory Used:", mem.Used, "Usage:", mem.Usage)
		infoTable.AddRow("Swap total:", mem.SwapTotal, "Free:", mem.SwapFree)
		infoTable.AddRow("Swap Used:", mem.SwapUsed, "Usage:", mem.SwapUsage)
		for i := 0; i <= net.NetInterfaceCount; i += 2 {
			if i >= net.NetInterfaceCount {
				break
			}
			cInter := net.NetInterfaces[i]
			if i < net.NetInterfaceCount-1 {
				nInter := net.NetInterfaces[i+1]
				infoTable.AddRow(cInter.Name+":", cInter.Addr, nInter.Name+":", nInter.Addr)
			} else {
				infoTable.AddRow(cInter.Name+":", cInter.Addr)
			}
		}
		fmt.Println(infoTable)
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zap

import (
	"bytes"
	"fmt"
	"sync"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/index/scorch/segment"
	"github.com/couchbase/vellum"
)

// Dictionary is the zap representation of the term dictionary
type Dictionary struct {
	sb        *SegmentBase
	field     string
	fieldID   uint16
	fst       *vellum.FST
	fstReader *vellum.Reader
}

var dictionaryPool = sync.Pool{New: func() interface{} { return &Dictionary{} }}

// PostingsList returns the postings list for the specified term
func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap,
	prealloc segment.PostingsList) (segment.PostingsList, error) {
	var preallocPL *PostingsList
	pl, ok := prealloc.(*PostingsList)
	if ok && pl != nil {
		preallocPL = pl
	}
	return d.postingsList(term, except, preallocPL)
}

func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	if d.fstReader == nil {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	postingsOffset, exists, err := d.fstReader.Get(term)
	if err != nil {
		return nil, fmt.Errorf("vellum err: %v", err)
	}
	if !exists {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	return d.postingsListFromOffset(postingsOffset, except, rv)
}

func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	rv = d.postingsListInit(rv, except)

	err := rv.read(postingsOffset, d)
	if err != nil {
		return nil, err
	}

	return rv, nil
}

func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList {
	if rv == nil || rv == emptyPostingsList {
		rv = postingsListPool.Get().(*PostingsList)
	} else {
		postings := rv.postings
		if postings != nil {
			postings.Clear()
		}

		*rv = PostingsList{} // clear the struct

		rv.postings = postings
	}
	rv.sb = d.sb
	rv.except = except
	return rv
}

// Iterator returns an iterator for this dictionary
func (d *Dictionary) Iterator() segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator(nil, nil)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// PrefixIterator returns an iterator which only visits terms having the
// the specified prefix
func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	kBeg := []byte(prefix)
	kEnd := segment.IncrementBytes(kBeg)

	if d.fst != nil {
		itr, err := d.fst.Iterator(kBeg, kEnd)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// RangeIterator returns an iterator which only visits terms between the
// start and end terms.  NOTE: bleve.index API specifies the end is inclusive.
func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	// need to increment the end position to be inclusive
	endBytes := []byte(end)
	if endBytes[len(endBytes)-1] < 0xff {
		endBytes[len(endBytes)-1]++
	} else {
		endBytes = append(endBytes, 0xff)
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator([]byte(start), endBytes)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// AutomatonIterator returns an iterator which only visits terms
// having the the vellum automaton and start/end key range
func (d *Dictionary) AutomatonIterator(a vellum.Automaton,
	startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

func (d *Dictionary) OnlyIterator(onlyTerms [][]byte,
	includeCount bool) segment.DictionaryIterator {

	rv := &DictionaryIterator{
		d:         d,
		omitCount: !includeCount,
	}

	var buf bytes.Buffer
	builder, err := vellum.New(&buf, nil)
	if err != nil {
		rv.err = err
		return rv
	}
	for _, term := range onlyTerms {
		err = builder.Insert(term, 0)
		if err != nil {
			rv.err = err
			return rv
		}
	}
	err = builder.Close()
	if err != nil {
		rv.err = err
		return rv
	}

	onlyFST, err := vellum.Load(buf.Bytes())
	if err != nil {
		rv.err = err
		return rv
	}

	itr, err := d.fst.Search(onlyFST, nil, nil)
	if err == nil {
		rv.itr = itr
	} else if err != nil && err != vellum.ErrIteratorDone {
		rv.err = err
	}

	return rv
}

func (d *Dictionary) Recycle() {
	if d.fst != nil {
		_ = d.fst.Close()
	}
	if d.fstReader != nil {
		d.fstReader.Close()
	}

	*d = Dictionary{} // clear fields
	dictionaryPool.Put(d)
}

// DictionaryIterator is an iterator for term dictionary
type DictionaryIterator struct {
	d         *Dictionary
	itr       vellum.Iterator
	err       error
	tmp       PostingsList
	entry     index.DictEntry
	omitCount bool
}

// Next returns the next entry in the dictionary
func (i *DictionaryIterator) Next() (*index.DictEntry, error) {
	if i.err != nil && i.err != vellum.ErrIteratorDone {
		return nil, i.err
	} else if i.itr == nil || i.err == vellum.ErrIteratorDone {
		return nil, nil
	}
	term, postingsOffset := i.itr.Current()
	i.entry.Term = string(term)
	if !i.omitCount {
		i.err = i.tmp.read(postingsOffset, i.d)
		if i.err != nil {
			return nil, i.err
		}
		i.entry.Count = i.tmp.Count()
	}
	i.err = i.itr.Next()
	return &i.entry, nil
}
<commit_msg>Revert "Invvoke fst/reader's Close() to reuse via sync pool on recycle"<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zap

import (
	"bytes"
	"fmt"
	"sync"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/index/scorch/segment"
	"github.com/couchbase/vellum"
)

// Dictionary is the zap representation of the term dictionary
type Dictionary struct {
	sb        *SegmentBase
	field     string
	fieldID   uint16
	fst       *vellum.FST
	fstReader *vellum.Reader
}

var dictionaryPool = sync.Pool{New: func() interface{} { return &Dictionary{} }}

// PostingsList returns the postings list for the specified term
func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap,
	prealloc segment.PostingsList) (segment.PostingsList, error) {
	var preallocPL *PostingsList
	pl, ok := prealloc.(*PostingsList)
	if ok && pl != nil {
		preallocPL = pl
	}
	return d.postingsList(term, except, preallocPL)
}

func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	if d.fstReader == nil {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	postingsOffset, exists, err := d.fstReader.Get(term)
	if err != nil {
		return nil, fmt.Errorf("vellum err: %v", err)
	}
	if !exists {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	return d.postingsListFromOffset(postingsOffset, except, rv)
}

func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	rv = d.postingsListInit(rv, except)

	err := rv.read(postingsOffset, d)
	if err != nil {
		return nil, err
	}

	return rv, nil
}

func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList {
	if rv == nil || rv == emptyPostingsList {
		rv = postingsListPool.Get().(*PostingsList)
	} else {
		postings := rv.postings
		if postings != nil {
			postings.Clear()
		}

		*rv = PostingsList{} // clear the struct

		rv.postings = postings
	}
	rv.sb = d.sb
	rv.except = except
	return rv
}

// Iterator returns an iterator for this dictionary
func (d *Dictionary) Iterator() segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator(nil, nil)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// PrefixIterator returns an iterator which only visits terms having the
// the specified prefix
func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	kBeg := []byte(prefix)
	kEnd := segment.IncrementBytes(kBeg)

	if d.fst != nil {
		itr, err := d.fst.Iterator(kBeg, kEnd)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// RangeIterator returns an iterator which only visits terms between the
// start and end terms.  NOTE: bleve.index API specifies the end is inclusive.
func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	// need to increment the end position to be inclusive
	endBytes := []byte(end)
	if endBytes[len(endBytes)-1] < 0xff {
		endBytes[len(endBytes)-1]++
	} else {
		endBytes = append(endBytes, 0xff)
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator([]byte(start), endBytes)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// AutomatonIterator returns an iterator which only visits terms
// having the the vellum automaton and start/end key range
func (d *Dictionary) AutomatonIterator(a vellum.Automaton,
	startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		itr, err := d.fst.Search(a, startKeyInclusive, endKeyExclusive)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

func (d *Dictionary) OnlyIterator(onlyTerms [][]byte,
	includeCount bool) segment.DictionaryIterator {

	rv := &DictionaryIterator{
		d:         d,
		omitCount: !includeCount,
	}

	var buf bytes.Buffer
	builder, err := vellum.New(&buf, nil)
	if err != nil {
		rv.err = err
		return rv
	}
	for _, term := range onlyTerms {
		err = builder.Insert(term, 0)
		if err != nil {
			rv.err = err
			return rv
		}
	}
	err = builder.Close()
	if err != nil {
		rv.err = err
		return rv
	}

	onlyFST, err := vellum.Load(buf.Bytes())
	if err != nil {
		rv.err = err
		return rv
	}

	itr, err := d.fst.Search(onlyFST, nil, nil)
	if err == nil {
		rv.itr = itr
	} else if err != nil && err != vellum.ErrIteratorDone {
		rv.err = err
	}

	return rv
}

func (d *Dictionary) Recycle() {
	*d = Dictionary{} // clear fields

	// TODO: need vellum API's to allow for recycled or prealloc'ed FST's

	dictionaryPool.Put(d)
}

// DictionaryIterator is an iterator for term dictionary
type DictionaryIterator struct {
	d         *Dictionary
	itr       vellum.Iterator
	err       error
	tmp       PostingsList
	entry     index.DictEntry
	omitCount bool
}

// Next returns the next entry in the dictionary
func (i *DictionaryIterator) Next() (*index.DictEntry, error) {
	if i.err != nil && i.err != vellum.ErrIteratorDone {
		return nil, i.err
	} else if i.itr == nil || i.err == vellum.ErrIteratorDone {
		return nil, nil
	}
	term, postingsOffset := i.itr.Current()
	i.entry.Term = string(term)
	if !i.omitCount {
		i.err = i.tmp.read(postingsOffset, i.d)
		if i.err != nil {
			return nil, i.err
		}
		i.entry.Count = i.tmp.Count()
	}
	i.err = i.itr.Next()
	return &i.entry, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/flosch/pongo2"
	"github.com/russross/blackfriday"
	"github.com/termie/go-shutil"
	"gopkg.in/yaml.v2"
)

// Config is a struct that will be used to store Blag's config.
type Config struct {
	Input            *string
	Theme            *string
	Output           *string
	PostsPerPage     *int
	StoryShortLength *int
	Title            *string
}

// BlagPostMeta is a struct that will hold a blogpost metadata
type BlagPostMeta struct {
	Title     string
	Timestamp int
	Author    string
	Slug      string
}

// BlagPost is a struct that holds post's content (in html) and its metadata
type BlagPost struct {
	BlagPostMeta
	Content string
}

// Theme holds templates that will be used to render HTML
type Theme struct {
	Page *pongo2.Template
	Post *pongo2.Template
}

// LoadTheme loads pongo2 templates for both pages and posts.
// It will try to load templates from themeDir/page.html and
// themeDir/post.html, and it will panic if that will not succeed.
func LoadTheme(themeDir string) Theme {
	t := Theme{}
	t.Page = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "templates", "page.html")))
	t.Post = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "templates", "post.html")))
	return t
}

// LoadPost loads post file specified by path argument, and returns BlagPost
// object with data loaded from that file.
func LoadPost(fpath string) BlagPost {
	file, err := os.Open(fpath)
	if err != nil {
		panic(err)
	}
	buf := bufio.NewReader(file)
	yamlMeta := ""
	for !strings.HasSuffix(yamlMeta, "\n\n") {
		var s string
		s, err = buf.ReadString('\n')
		yamlMeta += s
	}

	var meta BlagPostMeta
	yaml.Unmarshal([]byte(yamlMeta), &meta)

	if len(meta.Slug) == 0 {
		basename := filepath.Base(file.Name())
		meta.Slug = strings.TrimSuffix(basename, filepath.Ext(basename))
	}

	markdown, _ := ioutil.ReadAll(buf)
	html := string(blackfriday.MarkdownCommon(markdown))
	return BlagPost{
		meta,
		html,
	}
}

// LoadPosts loads all markdown files in inputDir (not recursive), and returns
// a slice []BlagPost, containing extracted metadata and HTML rendered from
// Markdown.
func LoadPosts(inputDir string) []BlagPost {
	var p []BlagPost
	filelist, err := ioutil.ReadDir(inputDir)
	if err != nil {
		panic(err)
	}
	for _, file := range filelist {
		p = append(p, LoadPost(path.Join(inputDir, file.Name())))
	}
	return p
}

// GenerateHTML generates page's static html and stores it in directory
// specified in config.
func GenerateHTML(config Config, theme Theme, posts []BlagPost) {
	os.RemoveAll(*config.Output)
	shutil.CopyTree(path.Join(*config.Theme, "static"), *config.Output, &shutil.CopyTreeOptions{
		Symlinks:               true,
		IgnoreDanglingSymlinks: true,
		CopyFunction:           shutil.Copy,
		Ignore:                 nil,
	})
	os.MkdirAll(*config.Output, 0755)
	os.MkdirAll(path.Join(*config.Output, "post"), 0755)
	for _, post := range posts {
		postFile, err := os.OpenFile(
			path.Join(*config.Output, "post", fmt.Sprintf("%s.html", post.Slug)),
			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer postFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Post.ExecuteWriter(pongo2.Context{
			"title": config.Title,
			"post":  post,
		}, postFile)
	}

	postCount := len(posts)
	pageCount := int(math.Floor(float64(postCount) / float64(*config.PostsPerPage)))
	pagePosts := make(map[int][]BlagPost)
	for i := postCount - 1; i >= 0; i-- {
		pageNo := int(math.Floor(float64(postCount-i-1)/float64(*config.PostsPerPage))) + 1
		pagePosts[pageNo] = append(pagePosts[pageNo], posts[i])
	}

	os.MkdirAll(path.Join(*config.Output, "page"), 0755)
	for k, v := range pagePosts {
		pageFile, err := os.OpenFile(
			path.Join(*config.Output, "page", fmt.Sprintf("%d.html", k)),
			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer pageFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Page.ExecuteWriter(pongo2.Context{
			"title":        config.Title,
			"posts":        v,
			"current_page": k,
			"page_count":   pageCount,
		}, pageFile)
		if k == 1 {
			indexFile, err := os.OpenFile(
				path.Join(*config.Output, "index.html"),
				os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
			defer pageFile.Close()
			if err != nil {
				panic(err)
			}
			theme.Page.ExecuteWriter(pongo2.Context{
				"title":        config.Title,
				"posts":        v,
				"current_page": k,
				"page_count":   pageCount,
			}, indexFile)
		}
	}
}

func main() {
	var config Config
	config.Input = flag.String("input", "input", "Directory where blog posts are stored (in markdown format)")
	config.Output = flag.String("output", "output", "Directory where generated html should be stored (IT WILL REMOVE ALL FILES INSIDE THAT DIR)")
	config.Theme = flag.String("theme", "theme", "Directory containing theme files (templates)")
	config.Title = flag.String("title", "Blag.", "Blag title")
	config.PostsPerPage = flag.Int("pps", 10, "Post count per page")
	config.StoryShortLength = flag.Int("short", 250, "Length of shortened versions of stories (-1 disables shortening)")
	flag.Parse()

	var theme Theme
	theme = LoadTheme(*config.Theme)

	var posts []BlagPost
	posts = LoadPosts(*config.Input)

	GenerateHTML(config, theme, posts)
}
<commit_msg>Fixed generation with no posts written.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/flosch/pongo2"
	"github.com/russross/blackfriday"
	"github.com/termie/go-shutil"
	"gopkg.in/yaml.v2"
)

// Config is a struct that will be used to store Blag's config.
type Config struct {
	Input            *string
	Theme            *string
	Output           *string
	PostsPerPage     *int
	StoryShortLength *int
	Title            *string
}

// BlagPostMeta is a struct that will hold a blogpost metadata
type BlagPostMeta struct {
	Title     string
	Timestamp int
	Author    string
	Slug      string
}

// BlagPost is a struct that holds post's content (in html) and its metadata
type BlagPost struct {
	BlagPostMeta
	Content string
}

// Theme holds templates that will be used to render HTML
type Theme struct {
	Page *pongo2.Template
	Post *pongo2.Template
}

// LoadTheme loads pongo2 templates for both pages and posts.
// It will try to load templates from themeDir/page.html and
// themeDir/post.html, and it will panic if that will not succeed.
func LoadTheme(themeDir string) Theme {
	t := Theme{}
	t.Page = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "templates", "page.html")))
	t.Post = pongo2.Must(pongo2.FromFile(path.Join(themeDir, "templates", "post.html")))
	return t
}

// LoadPost loads post file specified by path argument, and returns BlagPost
// object with data loaded from that file.
func LoadPost(fpath string) BlagPost {
	file, err := os.Open(fpath)
	if err != nil {
		panic(err)
	}
	buf := bufio.NewReader(file)
	yamlMeta := ""
	for !strings.HasSuffix(yamlMeta, "\n\n") {
		var s string
		s, err = buf.ReadString('\n')
		yamlMeta += s
	}

	var meta BlagPostMeta
	yaml.Unmarshal([]byte(yamlMeta), &meta)

	if len(meta.Slug) == 0 {
		basename := filepath.Base(file.Name())
		meta.Slug = strings.TrimSuffix(basename, filepath.Ext(basename))
	}

	markdown, _ := ioutil.ReadAll(buf)
	html := string(blackfriday.MarkdownCommon(markdown))
	return BlagPost{
		meta,
		html,
	}
}

// LoadPosts loads all markdown files in inputDir (not recursive), and returns
// a slice []BlagPost, containing extracted metadata and HTML rendered from
// Markdown.
func LoadPosts(inputDir string) []BlagPost {
	var p []BlagPost
	filelist, err := ioutil.ReadDir(inputDir)
	if err != nil {
		panic(err)
	}
	for _, file := range filelist {
		p = append(p, LoadPost(path.Join(inputDir, file.Name())))
	}
	return p
}

// GenerateHTML generates page's static html and stores it in directory
// specified in config.
func GenerateHTML(config Config, theme Theme, posts []BlagPost) {
	os.RemoveAll(*config.Output)
	shutil.CopyTree(path.Join(*config.Theme, "static"), *config.Output, &shutil.CopyTreeOptions{
		Symlinks:               true,
		IgnoreDanglingSymlinks: true,
		CopyFunction:           shutil.Copy,
		Ignore:                 nil,
	})
	os.MkdirAll(*config.Output, 0755)
	os.MkdirAll(path.Join(*config.Output, "post"), 0755)
	for _, post := range posts {
		postFile, err := os.OpenFile(
			path.Join(*config.Output, "post", fmt.Sprintf("%s.html", post.Slug)),
			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer postFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Post.ExecuteWriter(pongo2.Context{
			"title": config.Title,
			"post":  post,
		}, postFile)
	}

	postCount := len(posts)
	pageCount := int(math.Floor(float64(postCount) / float64(*config.PostsPerPage)))

	os.MkdirAll(path.Join(*config.Output, "page"), 0755)
	if pageCount > 0 {
		pagePosts := make(map[int][]BlagPost)
		for i := postCount - 1; i >= 0; i-- {
			pageNo := int(math.Floor(float64(postCount-i-1)/float64(*config.PostsPerPage))) + 1
			pagePosts[pageNo] = append(pagePosts[pageNo], posts[i])
		}

		for k, v := range pagePosts {
			pageFile, err := os.OpenFile(
				path.Join(*config.Output, "page", fmt.Sprintf("%d.html", k)),
				os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
			defer pageFile.Close()
			if err != nil {
				panic(err)
			}
			theme.Page.ExecuteWriter(pongo2.Context{
				"title":        config.Title,
				"posts":        v,
				"current_page": k,
				"page_count":   pageCount,
			}, pageFile)
			if k == 1 {
				indexFile, err := os.OpenFile(
					path.Join(*config.Output, "index.html"),
					os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
				defer pageFile.Close()
				if err != nil {
					panic(err)
				}
				theme.Page.ExecuteWriter(pongo2.Context{
					"title":        config.Title,
					"posts":        v,
					"current_page": k,
					"page_count":   pageCount,
				}, indexFile)
			}
		}
	} else {
		pageFile, err := os.OpenFile(
			path.Join(*config.Output, "page", "1.html"),
			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer pageFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Page.ExecuteWriter(pongo2.Context{
			"title":        config.Title,
			"posts":        make([]BlagPost, 0),
			"current_page": 1,
			"page_count":   1,
		}, pageFile)
		indexFile, err := os.OpenFile(
			path.Join(*config.Output, "index.html"),
			os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		defer pageFile.Close()
		if err != nil {
			panic(err)
		}
		theme.Page.ExecuteWriter(pongo2.Context{
			"title":        config.Title,
			"posts":        make([]BlagPost, 0),
			"current_page": 1,
			"page_count":   1,
		}, indexFile)
	}
}

func main() {
	var config Config
	config.Input = flag.String("input", "input", "Directory where blog posts are stored (in markdown format)")
	config.Output = flag.String("output", "output", "Directory where generated html should be stored (IT WILL REMOVE ALL FILES INSIDE THAT DIR)")
	config.Theme = flag.String("theme", "theme", "Directory containing theme files (templates)")
	config.Title = flag.String("title", "Blag.", "Blag title")
	config.PostsPerPage = flag.Int("pps", 10, "Post count per page")
	config.StoryShortLength = flag.Int("short", 250, "Length of shortened versions of stories (-1 disables shortening)")
	flag.Parse()

	var theme Theme
	theme = LoadTheme(*config.Theme)

	var posts []BlagPost
	posts = LoadPosts(*config.Input)

	GenerateHTML(config, theme, posts)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	// bugsnag "github.com/bugsnag/bugsnag-go"
	homehandler "github.com/themccallister/go-https-demo/handlers"
	usershandler "github.com/themccallister/go-https-demo/handlers/users"
	"github.com/themccallister/go-https-demo/helpers/env"
)

func main() {
	// grab the environment vars required
	port := env.GetOr("APP_PORT", "443")
	certFile := env.GetOr("APP_CERT_FILE", "certs/local.demodomain.com.crt")
	keyFile := env.GetOr("APP_KEY_FILE", "certs/local.demodomain.com.key")
	// bugsnagKey := helpers.EnvStringOr("BUGSNAG_KEY", "")
	// if bugsnagKey != "" {
	// 	bugsnag.Configure(bugsnag.Configuration{
	// 		APIKey: bugsnagKey,
	// 	})
	// }

	fmt.Println("Running on port", port)

	r := http.NewServeMux()

	// setup the routing
	r.HandleFunc("/", homehandler.Index)
	r.HandleFunc("/v2/users", usershandler.Index)

	if err := http.ListenAndServeTLS(":"+port, certFile, keyFile, r); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Updates a route to use the request logging middleware<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	// bugsnag "github.com/bugsnag/bugsnag-go"
	homehandler "github.com/themccallister/go-https-demo/handlers"
	usershandler "github.com/themccallister/go-https-demo/handlers/users"
	"github.com/themccallister/go-https-demo/helpers/env"
	"github.com/themccallister/go-https-demo/middleware"
)

func routes(r *http.ServeMux) {

}

func main() {
	// grab the environment vars required
	port := env.GetOr("APP_PORT", "443")
	certFile := env.GetOr("APP_CERT_FILE", "certs/local.demodomain.com.crt")
	keyFile := env.GetOr("APP_KEY_FILE", "certs/local.demodomain.com.key")
	// bugsnagKey := helpers.EnvStringOr("BUGSNAG_KEY", "")
	// if bugsnagKey != "" {
	// 	bugsnag.Configure(bugsnag.Configuration{
	// 		APIKey: bugsnagKey,
	// 	})
	// }

	fmt.Println("Running on port", port)

	r := http.NewServeMux()

	// setup the routing
	r.HandleFunc("/", homehandler.Index)
	r.HandleFunc("/v2/users", middleware.RequestLogging(usershandler.Index))

	if err := http.ListenAndServeTLS(":"+port, certFile, keyFile, r); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "log"
    "net/http"
    "os"
    "regexp"
    "strconv"
    "strings"
    "time"

    "github.com/PuerkitoBio/goquery"
    "github.com/sfreiberg/gotwilio"
)

var rgx *regexp.Regexp

func Scrape(proj string, idx uint) int64 {
    doc, err := goquery.NewDocument("https://www.kickstarter.com/projects/" + proj)
    if err != nil {
        log.Print("Error getting doc")
        log.Print(err)
        return -1
    }

    var remaining int64 = -1
    parseErr := false
    doc.Find(".backers-wrap .h6 .bold").EachWithBreak(func(i int, s *goquery.Selection) bool {
        if uint(i) != idx*2 {
            return true
        }

        span := s.Find(".limited-number").Text()                           // Limited (5093 left of 26000)
        raw := rgx.FindString(span)                                        // (5093  (trailing space)
        remaining, err = strconv.ParseInt(strings.Trim(raw, "( "), 10, 32) // 5093
        if err != nil {
            log.Print("Error parsing int")
            log.Print(err)
            parseErr = true
        }
        return false
    })

    if parseErr {
        return -1
    }

    return remaining
}

func doEvery(d time.Duration, f func()) {
    for {
        f()
        time.Sleep(d)
    }
}

func handler(w http.ResponseWriter, req *http.Request) {
    w.Header().Set("Content-Type", "text/html")
    w.Write([]byte("
Pebble availability
"))
}

// Given the last sent amount and the current remaining amount, determines if a new text should be sent
func sendMessage(lastSentAmount int64, currentAmount int64) bool {
    lastSentAmount-- // Adjust values since we want to send a new text at the diff multiple
    currentAmount--
    var factor int64 = 10 // Represents the multiple of 10 that is higher than the current amount
    for factor < currentAmount {
        factor *= 10
    }
    diff := factor / 10 // Threshold for sending a new text
    lastTier := (lastSentAmount - lastSentAmount%diff) / diff
    curTier := (currentAmount - currentAmount%diff) / diff
    isInitial := lastSentAmount < 0
    changed := curTier < lastTier
    return isInitial || changed
}

func main() {
    accountSid := os.Getenv("twilioSid")
    authToken := os.Getenv("twilioToken")
    twilio := gotwilio.NewTwilioClient(accountSid, authToken)
    from := os.Getenv("fromNum")
    to := os.Getenv("toNum")

    rgx = regexp.MustCompile("\\((.*?) ")
    var last int64 = -1

    go doEvery(60*60*time.Second, func() {
        var timeSteelIdx uint = 3
        remaining := Scrape("597507018/pebble-time-awesome-smartwatch-no-compromises", timeSteelIdx)
        if remaining < 0 {
            return
        }
        fmt.Println(remaining, "remaining")
        send := sendMessage(last, remaining)
        if !send {
            return
        }
        last = remaining
        message := fmt.Sprintf("%d Pebble Time Steels of %d are remaining.", remaining, 20000)
        fmt.Printf("Sending message: %s\n", message)
        twilio.SendSMS(from, to, message, "", "")
    })

    http.HandleFunc("/", handler)
    err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
    if err != nil {
        panic(err)
    }
}
<commit_msg>Use fmt instead of log<commit_after>package main

import (
    "fmt"
    "net/http"
    "os"
    "regexp"
    "strconv"
    "strings"
    "time"

    "github.com/PuerkitoBio/goquery"
    "github.com/sfreiberg/gotwilio"
)

var rgx *regexp.Regexp

func Scrape(proj string, idx uint) int64 {
    doc, err := goquery.NewDocument("https://www.kickstarter.com/projects/" + proj)
    if err != nil {
        fmt.Println("Error getting doc")
        fmt.Println(err)
        return -1
    }

    var remaining int64 = -1
    parseErr := false
    doc.Find(".backers-wrap .h6 .bold").EachWithBreak(func(i int, s *goquery.Selection) bool {
        if uint(i) != idx*2 {
            return true
        }

        span := s.Find(".limited-number").Text()                           // Limited (5093 left of 26000)
        raw := rgx.FindString(span)                                        // (5093  (trailing space)
        remaining, err = strconv.ParseInt(strings.Trim(raw, "( "), 10, 32) // 5093
        if err != nil {
            fmt.Println("Error parsing int")
            fmt.Println(err)
            parseErr = true
        }
        return false
    })

    if parseErr {
        return -1
    }

    return remaining
}

func doEvery(d time.Duration, f func()) {
    for {
        f()
        time.Sleep(d)
    }
}

func handler(w http.ResponseWriter, req *http.Request) {
    w.Header().Set("Content-Type", "text/html")
    w.Write([]byte("
Pebble availability
"))
}

// Given the last sent amount and the current remaining amount, determines if a new text should be sent
func sendMessage(lastSentAmount int64, currentAmount int64) bool {
    lastSentAmount-- // Adjust values since we want to send a new text at the diff multiple
    currentAmount--
    var factor int64 = 10 // Represents the multiple of 10 that is higher than the current amount
    for factor < currentAmount {
        factor *= 10
    }
    diff := factor / 10 // Threshold for sending a new text
    lastTier := (lastSentAmount - lastSentAmount%diff) / diff
    curTier := (currentAmount - currentAmount%diff) / diff
    isInitial := lastSentAmount < 0
    changed := curTier < lastTier
    return isInitial || changed
}

func main() {
    accountSid := os.Getenv("twilioSid")
    authToken := os.Getenv("twilioToken")
    twilio := gotwilio.NewTwilioClient(accountSid, authToken)
    from := os.Getenv("fromNum")
    to := os.Getenv("toNum")

    rgx = regexp.MustCompile("\\((.*?) ")
    var last int64 = -1

    go doEvery(60*60*time.Second, func() {
        var timeSteelIdx uint = 3
        remaining := Scrape("597507018/pebble-time-awesome-smartwatch-no-compromises", timeSteelIdx)
        if remaining < 0 {
            return
        }
        fmt.Println(remaining, "remaining")
        if !sendMessage(last, remaining) {
            return
        }

        last = remaining
        message := fmt.Sprintf("%d Pebble Time Steels of %d are remaining.", remaining, 20000)
        fmt.Printf("Sending message: %s\n", message)
        twilio.SendSMS(from, to, message, "", "")
    })

    http.HandleFunc("/", handler)
    err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
    if err != nil {
        panic(err)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchevents"
	"github.com/aws/aws-sdk-go/service/lambda"
)

var Version = "0.2.0"

func main() {
	var (
		apply, dryrun                           bool
		file, awsRegion                         string
		err                                     error
		sess                                    *session.Session
		cweRulesBeforeApply, cweRulesAfterApply *cloudwatchevents.ListRulesOutput
	)

	flag.BoolVar(&apply, "apply", false, "apply to CloudWatch Events")
	flag.BoolVar(&dryrun, "dry-run", false, "dry-run")
	flag.StringVar(&file, "file", "config.yml", "file path to setting yaml")
	flag.StringVar(&file, "f", "config.yml", "file path to setting yaml (shorthand)")
	flag.StringVar(&awsRegion, "region", os.Getenv("AWS_REGION"), "aws region")
	flag.Parse()

	sess, err = session.NewSession(
		&aws.Config{
			Region: aws.String(awsRegion),
		},
	)
	if err != nil {
		fmt.Printf("Session error %v\n", err)
		os.Exit(1)
	}

	cweClient := cloudwatchevents.New(sess)
	lambdaClient := lambda.New(sess)

	cweRulesBeforeApply, err = cweClient.ListRules(nil)
	if err != nil {
		fmt.Printf("API error %v\n", err)
		os.Exit(1)
	}

	describedRules := Rules{}
	err = loadYaml(file, &describedRules)
	if err != nil {
		fmt.Printf("File error %v\n", err)
		os.Exit(1)
	}

	describedRules.Rules = AssociateRules(cweRulesBeforeApply.Rules, describedRules.Rules)
	for i, rule := range describedRules.Rules {
		t, _ := fetchActualTargetsByRule(cweClient, rule)
		describedRules.Rules[i].Targets = AssociateTargets(t, describedRules.Rules[i].Targets)
	}
	CheckIsNeedUpdateOrDelete(describedRules.Rules)
	displayWhatWillChange(describedRules.Rules)

	if apply && !dryrun {
		err = updateCloudWatchEvents(cweClient, describedRules.Rules)
		if err != nil {
			fmt.Printf("API error %v\n", err)
			os.Exit(1)
		}
	}
}

func loadYaml(file string, r *Rules) error {

	buf, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(buf, &r)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Add permission to lambda to invoke from CloudWatch Events<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchevents"
	"github.com/aws/aws-sdk-go/service/lambda"
)

var Version = "0.2.0"

func main() {
	var (
		apply, dryrun                           bool
		file, awsRegion                         string
		err                                     error
		sess                                    *session.Session
		cweRulesBeforeApply, cweRulesAfterApply *cloudwatchevents.ListRulesOutput
	)

	flag.BoolVar(&apply, "apply", false, "apply to CloudWatch Events")
	flag.BoolVar(&dryrun, "dry-run", false, "dry-run")
	flag.StringVar(&file, "file", "config.yml", "file path to setting yaml")
	flag.StringVar(&file, "f", "config.yml", "file path to setting yaml (shorthand)")
	flag.StringVar(&awsRegion, "region", os.Getenv("AWS_REGION"), "aws region")
	flag.Parse()

	sess, err = session.NewSession(
		&aws.Config{
			Region: aws.String(awsRegion),
		},
	)
	if err != nil {
		fmt.Printf("Session error %v\n", err)
		os.Exit(1)
	}

	cweClient := cloudwatchevents.New(sess)
	lambdaClient := lambda.New(sess)

	cweRulesBeforeApply, err = cweClient.ListRules(nil)
	if err != nil {
		fmt.Printf("API error %v\n", err)
		os.Exit(1)
	}

	describedRules := Rules{}
	err = loadYaml(file, &describedRules)
	if err != nil {
		fmt.Printf("File error %v\n", err)
		os.Exit(1)
	}

	describedRules.Rules = AssociateRules(cweRulesBeforeApply.Rules, describedRules.Rules)
	for i, rule := range describedRules.Rules {
		t, _ := fetchActualTargetsByRule(cweClient, rule)
		describedRules.Rules[i].Targets = AssociateTargets(t, describedRules.Rules[i].Targets)
	}
	CheckIsNeedUpdateOrDelete(describedRules.Rules)
	displayWhatWillChange(describedRules.Rules)

	if apply && !dryrun {
		err = updateCloudWatchEvents(cweClient, describedRules.Rules)
		if err != nil {
			fmt.Printf("API error %v\n", err)
			os.Exit(1)
		}
		// Grant permission to invoke lambda function from CloudWatch Events
		cweRulesAfterApply, err = cweClient.ListRules(nil)
		describedRules.Rules = AssociateRules(cweRulesAfterApply.Rules, describedRules.Rules)
		for i, rule := range describedRules.Rules {
			t, _ := fetchActualTargetsByRule(cweClient, rule)
			describedRules.Rules[i].Targets = AssociateTargets(t, describedRules.Rules[i].Targets)
		}

		err = addPermissionToLambdaFromCloudWatchEvents(lambdaClient, describedRules.Rules)
		if err != nil {
			fmt.Print("Grant permission error %v\n", err)
		}
	}
}

func loadYaml(file string, r *Rules) error {

	buf, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(buf, &r)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"time"
)

const (
	// DefaultBaseURL is the API root URL
	defaultBaseURL = "https://pikacloud.com/api/"
	apiVersion     = "v1"
)

// Agent describes the agent
type Agent struct {
	Client   *Client
	ID       string `json:"aid"`
	PingURL  string `json:"ping_url"`
	TTL      int    `json:"ttl"`
	Hostname string `json:"hostname"`
}

// CreateAgentOptions represents the agent Create() options
type CreateAgentOptions struct {
	Hostname string `json:"hostname"`
}

// Task descibres a task
type Task struct {
	ID      string `json:"tid"`
	Payload string `json:"payload"`
	NeedACK bool   `json:"need_ack"`
}

// Create an agent
func (agent *Agent) Create(opt *CreateAgentOptions) error {
	status, err := agent.Client.Post("run/agents/", opt, &agent)
	if err != nil {
		return fmt.Errorf(err.Error())
	}
	if status != 200 {
		return fmt.Errorf("Failed to create agent http code: %d", status)
	}
	log.Printf("Agent %s registered with hostname %s\n", agent.ID, agent.Hostname)
	return nil
}

func (agent *Agent) infinitePing() {
	for {
		err := agent.Ping()
		if err != nil {
			log.Println(err)
		}
		log.Println("Ping OK")
		time.Sleep(3 * time.Second)
	}
}

// Ping agent
func (agent *Agent) Ping() error {
	pingURI := fmt.Sprintf("run/agents/%s/ping/", agent.ID)
	// for {
	status, err := agent.Client.Post(pingURI, nil, nil)
	if err != nil {
		log.Println("Trying to register lost agent")
		newAgentOpts := CreateAgentOptions{
			Hostname: agent.Hostname,
		}
		return agent.Create(&newAgentOpts)
	}
	if status != 200 {
		return fmt.Errorf("Ping to %s returns %d\n", pingURI, status)
	}
	return nil
}

func (agent *Agent) infinitePullTasks() {
	for {
		tasks, err := agent.pullTasks()
		if err != nil {
			log.Println(err)
		}
		if len(tasks) > 0 {
			log.Printf("Got %d new tasks", len(tasks))
		}
		time.Sleep(1 * time.Second)
	}
}

func (agent *Agent) pullTasks() ([]Task, error) {
	tasksURI := fmt.Sprintf("run/agents/%s/tasks/?requeue=false&size=50", agent.ID)
	tasks := []Task{}
	err := agent.Client.Get(tasksURI, &tasks)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	return tasks, nil
}

func main() {
	apiToken := os.Getenv("PIKACLOUD_API_TOKEN")
	baseURL := os.Getenv("PIKACLOUD_AGENT_URL")
	if apiToken == "" {
		log.Fatalln("PIKACLOUD_API_TOKEN is empty")
	}
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("Unable to retrieve agent hostname: %v", err)
	}
	if baseURL == "" {
		baseURL = defaultBaseURL
	}
	c := NewClient(apiToken)
	c.BaseURL = baseURL

	agent := Agent{
		Client:   c,
		Hostname: hostname,
	}
	newAgentOpts := CreateAgentOptions{
		Hostname: hostname,
	}
	err = agent.Create(&newAgentOpts)
	if err != nil {
		log.Fatal(err)
	}
	wg := sync.WaitGroup{}
	defer wg.Wait()
	wg.Add(1)
	go agent.infinitePing()
	wg.Add(1)
	go agent.syncDockerInfo()
	wg.Add(1)
	go agent.syncDockerContainers()
	wg.Add(1)
	go agent.infinitePullTasks()

}
<commit_msg>ack message if needed<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
	"time"
)

const (
	// DefaultBaseURL is the API root URL
	defaultBaseURL = "https://pikacloud.com/api/"
	apiVersion     = "v1"
)

// Agent describes the agent
type Agent struct {
	Client   *Client
	ID       string `json:"aid"`
	PingURL  string `json:"ping_url"`
	TTL      int    `json:"ttl"`
	Hostname string `json:"hostname"`
}

// CreateAgentOptions represents the agent Create() options
type CreateAgentOptions struct {
	Hostname string `json:"hostname"`
}

// Task descibres a task
type Task struct {
	ID      string `json:"tid"`
	Payload string `json:"payload"`
	NeedACK bool   `json:"need_ack"`
}

// Create an agent
func (agent *Agent) Create(opt *CreateAgentOptions) error {
	status, err := agent.Client.Post("run/agents/", opt, &agent)
	if err != nil {
		return fmt.Errorf(err.Error())
	}
	if status != 200 {
		return fmt.Errorf("Failed to create agent http code: %d", status)
	}
	log.Printf("Agent %s registered with hostname %s\n", agent.ID, agent.Hostname)
	return nil
}

func (agent *Agent) infinitePing() {
	for {
		err := agent.Ping()
		if err != nil {
			log.Println(err)
		}
		log.Println("Ping OK")
		time.Sleep(3 * time.Second)
	}
}

// Ping agent
func (agent *Agent) Ping() error {
	pingURI := fmt.Sprintf("run/agents/%s/ping/", agent.ID)
	// for {
	status, err := agent.Client.Post(pingURI, nil, nil)
	if err != nil {
		log.Println("Trying to register lost agent")
		newAgentOpts := CreateAgentOptions{
			Hostname: agent.Hostname,
		}
		return agent.Create(&newAgentOpts)
	}
	if status != 200 {
		return fmt.Errorf("Ping to %s returns %d\n", pingURI, status)
	}
	return nil
}

func (agent *Agent) infinitePullTasks() {
	for {
		tasks, err := agent.pullTasks()
		if err != nil {
			log.Println(err)
		}
		if len(tasks) > 0 {
			log.Printf("Got %d new tasks", len(tasks))
		}
		for _, task := range tasks {
			if task.NeedACK {
				err := agent.ackTask(task)
				if err != nil {
					log.Printf("Unable to ack task %s: %s", task.ID, err)
				}
				log.Printf("task %s ACKed", task.ID)

			}
		}
		time.Sleep(1 * time.Second)
	}
}

func (agent *Agent) pullTasks() ([]Task, error) {
	tasksURI := fmt.Sprintf("run/agents/%s/tasks/ready/?requeue=false&size=50", agent.ID)
	tasks := []Task{}
	err := agent.Client.Get(tasksURI, &tasks)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	return tasks, nil
}

func (agent *Agent) ackTask(t Task) error {
	ackURI := fmt.Sprintf("run/agents/%s/tasks/unack/%s/", agent.ID, t.ID)
	err := agent.Client.Delete(ackURI, nil)
	if err != nil {
		log.Println(err)
		return err
	}
	return nil
}

func main() {
	apiToken := os.Getenv("PIKACLOUD_API_TOKEN")
	baseURL := os.Getenv("PIKACLOUD_AGENT_URL")
	if apiToken == "" {
		log.Fatalln("PIKACLOUD_API_TOKEN is empty")
	}
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("Unable to retrieve agent hostname: %v", err)
	}
	if baseURL == "" {
		baseURL = defaultBaseURL
	}
	c := NewClient(apiToken)
	c.BaseURL = baseURL

	agent := Agent{
		Client:   c,
		Hostname: hostname,
	}
	newAgentOpts := CreateAgentOptions{
		Hostname: hostname,
	}
	err = agent.Create(&newAgentOpts)
	if err != nil {
		log.Fatal(err)
	}
	wg := sync.WaitGroup{}
	defer wg.Wait()
	wg.Add(1)
	go agent.infinitePing()
	wg.Add(1)
	go agent.syncDockerInfo()
	wg.Add(1)
	go agent.syncDockerContainers()
	wg.Add(1)
	go agent.infinitePullTasks()

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/parser"
	"go/token"
	"os"
	"strings"
)

const VERSION = "0.0.1"

var fset *token.FileSet

func main() {
	if len(os.Args) != 2 {
		printUsage()
		os.Exit(1)
	}

	filename := os.Args[1]

	fset = token.NewFileSet()

	f, err := parser.ParseFile(fset, filename, nil, 0)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// header
	fmt.Println("!_TAG_FILE_FORMAT\t2\t")
	fmt.Println("!_TAG_FILE_SORTED\t0\t")

	// package
	if f.Name != nil {
		printTag(f.Name.Name, "p", f.Name.Pos())
	}

	// imports
	for _, im := range f.Imports {
		if im.Path != nil {
			printTag(strings.Trim(im.Path.Value, "\""), "i", im.Path.Pos())
		}
	}
}

func printUsage() {
	fmt.Printf("gotags version %s\n\n", VERSION)
	fmt.Printf("Usage: %s file\n", os.Args[0])
}

func printTag(tag, kind string, pos token.Pos) {
	line := fset.Position(pos).Line
	file := fset.File(pos).Name()
	fmt.Printf("%s\t%s\t%d;\"\t%s\n", tag, file, line, kind)
}
<commit_msg>Print functions<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"strings"
)

const VERSION = "0.0.1"

var fset *token.FileSet

func main() {
	if len(os.Args) != 2 {
		printUsage()
		os.Exit(1)
	}

	filename := os.Args[1]

	fset = token.NewFileSet()

	f, err := parser.ParseFile(fset, filename, nil, 0)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// header
	fmt.Println("!_TAG_FILE_FORMAT\t2\t")
	fmt.Println("!_TAG_FILE_SORTED\t0\t")

	// package
	if f.Name != nil {
		printTag(f.Name.Name, "p", f.Name.Pos())
	}

	// imports
	for _, im := range f.Imports {
		if im.Path != nil {
			printTag(strings.Trim(im.Path.Value, "\""), "i", im.Path.Pos())
		}
	}

	// declarations
	for _, d := range f.Decls {
		if fd, ok := d.(*ast.FuncDecl); ok {
			if fd.Name != nil {
				// TODO: add params, receiver, etc
				printTag(fd.Name.Name, "f", fd.Pos())
			}
		}
	}
}

func printUsage() {
	fmt.Printf("gotags version %s\n\n", VERSION)
	fmt.Printf("Usage: %s file\n", os.Args[0])
}

func printTag(tag, kind string, pos token.Pos) {
	line := fset.Position(pos).Line
	file := fset.File(pos).Name()
	fmt.Printf("%s\t%s\t%d;\"\t%s\n", tag, file, line, kind)
}
<|endoftext|>