<commit_before>package main

import (
	"bufio"
	"flag"
	"github.com/saymedia/packer2terraform/packer2terraform"
	"io"
	"io/ioutil"
	"log"
	"os"
)

func help() {
	log.Println(`Usage packer2terraform [options...]
packer2terraform turns Packer's machine-readable output into a Terraform-readable format.

Options:
    -f Filename of the input CSV. Alternatively use STDIN.
    -h This help information.
    -template Filename of the template to use in the output.

Example:
    packer -machine-readable build app.json | \
        packer2terraform -template templates/amazon-ebs.hcl > app.tfvars
`)
}

func main() {

	tmpl := flag.String("template", "", "a template file")
	csv := flag.String("f", "", "a csv file")
	helpMe := flag.Bool("h", false, "help")

	flag.Parse()

	if *helpMe {
		help()
		os.Exit(0)
	}

	// Read a file or use STDIN
	var csvFile io.Reader
	if len(*csv) > 0 {
		f, err := os.Open(*csv)
		if err != nil {
			log.Fatalf("CSV file read failed %s", err)
		}
		csvFile = bufio.NewReader(f)
	} else if stat, _ := os.Stdin.Stat(); stat.Mode()&os.ModeCharDevice == 0 {
		// has STDIN data
		csvFile = bufio.NewReader(os.Stdin)
	} else {
		// No input data
		help()
		os.Exit(0)
	}

	// Get the CSV as a string array
	parsed, err := packer2terraform.ReadCSV(csvFile)
	if err != nil {
		log.Fatalf("CSV read failed %s", err)
	}

	// Extract the artifacts
	artifacts, err := packer2terraform.ExtractArtifacts(parsed)
	if err != nil {
		// fmt.Errorf("Packer build failed: %s", err)
		log.Fatalf("Packer build failed: %s", err)
	}

	// Print artifacts using a template
	var templateString string
	if len(*tmpl) == 0 {
		templateString = packer2terraform.TemplateAmazonEBS
	} else {
		buf, err := ioutil.ReadFile(*tmpl)
		if err != nil {
			log.Fatalf("Template file read failed: %s", err)
		}
		templateString = string(buf)
	}
	doc, err := packer2terraform.ToTemplate(artifacts, templateString)
	if err != nil {
		log.Fatalf("Template render failed: %s", err)
	}
	log.Println(doc)

	// Done
	os.Exit(0)

}
<commit_msg>print using fmt instead of log<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/saymedia/packer2terraform/packer2terraform"
	"io"
	"io/ioutil"
	"log"
	"os"
)

func help() {
	log.Println(`Usage packer2terraform [options...]
packer2terraform turns Packer's machine-readable output into a Terraform-readable format.

Options:
    -f Filename of the input CSV. Alternatively use STDIN.
    -h This help information.
    -template Filename of the template to use in the output.

Example:
    packer -machine-readable build app.json | \
        packer2terraform -template templates/amazon-ebs.hcl > app.tfvars
`)
}

func main() {

	tmpl := flag.String("template", "", "a template file")
	csv := flag.String("f", "", "a csv file")
	helpMe := flag.Bool("h", false, "help")

	flag.Parse()

	if *helpMe {
		help()
		os.Exit(0)
	}

	// Read a file or use STDIN
	var csvFile io.Reader
	if len(*csv) > 0 {
		f, err := os.Open(*csv)
		if err != nil {
			log.Fatalf("CSV file read failed %s", err)
		}
		csvFile = bufio.NewReader(f)
	} else if stat, _ := os.Stdin.Stat(); stat.Mode()&os.ModeCharDevice == 0 {
		// has STDIN data
		csvFile = bufio.NewReader(os.Stdin)
	} else {
		// No input data
		help()
		os.Exit(0)
	}

	// Get the CSV as a string array
	parsed, err := packer2terraform.ReadCSV(csvFile)
	if err != nil {
		log.Fatalf("CSV read failed %s", err)
	}

	// Extract the artifacts
	artifacts, err := packer2terraform.ExtractArtifacts(parsed)
	if err != nil {
		// fmt.Errorf("Packer build failed: %s", err)
		log.Fatalf("Packer build failed: %s", err)
	}

	// Print artifacts using a template
	var templateString string
	if len(*tmpl) == 0 {
		templateString = packer2terraform.TemplateAmazonEBS
	} else {
		buf, err := ioutil.ReadFile(*tmpl)
		if err != nil {
			log.Fatalf("Template file read failed: %s", err)
		}
		templateString = string(buf)
	}
	doc, err := packer2terraform.ToTemplate(artifacts, templateString)
	if err != nil {
		log.Fatalf("Template render failed: %s", err)
	}
	fmt.Println(doc)

	// Done
	os.Exit(0)

}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "strconv"
    "strings"
    "time"

    "github.com/BurntSushi/toml"
    "github.com/akrennmair/gopcap"
    "github.com/nranchev/go-libGeoIP"
)

type Packet struct {
    ts      time.Time
    tuple   IpPortTuple
    payload []byte
}

type protocolType uint16

const (
    UnknownProtocol protocolType = iota
    HttpProtocol
    MysqlProtocol
    RedisProtocol
)

var protocolNames = []string{"unknown", "http", "mysql", "redis"}

type tomlConfig struct {
    Interfaces    tomlInterfaces
    RunOptions    tomlRunOptions
    Protocols     map[string]tomlProtocol
    Procs         tomlProcs
    Elasticsearch tomlMothership
    Agent         tomlAgent
    Logging       tomlLogging
}

type tomlInterfaces struct {
    Device string
}

type tomlRunOptions struct {
    Uid int
    Gid int
}

type tomlLogging struct {
    Selectors []string
}

var _Config tomlConfig
var _ConfigMeta toml.MetaData
var _GeoLite *libgeo.GeoIP

func Bytes_Ipv4_Ntoa(bytes []byte) string {
    var strarr []string = make([]string, 4)
    for i, b := range bytes {
        strarr[i] = strconv.Itoa(int(b))
    }
    return strings.Join(strarr, ".")
}

func decodePktEth(datalink int, pkt *pcap.Packet) {
    defer RECOVER("decodePktEth exception")

    packet := new(Packet)
    var l2hlen int
    var eth_type uint16

    if datalink != pcap.LINKTYPE_LINUX_SLL {
        // we are just assuming ETH
        l2hlen = 14

        // bytes 12 and 13 are the pkt type
        eth_type = Bytes_Ntohs(pkt.Data[12:14])

    } else {
        l2hlen = 16

        // bytes 14 and 15 are the pkt type
        eth_type = Bytes_Ntohs(pkt.Data[14:16])
    }

    if eth_type != 0x800 {
        DEBUG("ip", "Not ipv4 packet. Ignoring")
        return
    }

    if len(pkt.Data) < l2hlen+20 {
        DEBUG("ip", "Packet too short to be ethernet")
        return
    }

    // IP header
    iphl := int((uint16(pkt.Data[l2hlen]) & 0x0f) * 4)
    if len(pkt.Data) < l2hlen+iphl {
        DEBUG("ip", "Packet too short to be IP")
        return
    }
    iphdr := pkt.Data[l2hlen : l2hlen+iphl]

    //DEBUG("Packet timestamp: %s", pkt.Time)
    packet.ts = pkt.Time

    packet.tuple.Src_ip = Bytes_Ntohl(iphdr[12:16])
    packet.tuple.Dst_ip = Bytes_Ntohl(iphdr[16:20])

    mf := (uint8(iphdr[6]&0x20) != 0)
    frag_offset := (uint16(iphdr[7]) << 3) | (uint16(iphdr[6]) & 0x1F)

    if mf || frag_offset > 0 {
        DEBUG("ip", "Fragmented packets not yet supported")
        return
    }

    ip_length := int(Bytes_Ntohs(iphdr[2:4]))

    protocol := uint8(iphdr[9])
    if protocol != 6 {
        DEBUG("ip", "Not TCP packet. Ignoring")
        return
    }

    if len(pkt.Data) < l2hlen+iphl+20 {
        DEBUG("ip", "Packet too short to be TCP")
        return
    }

    tcphl := int((uint16(pkt.Data[l2hlen+iphl+12]) >> 4) * 4)
    if tcphl > 20 && len(pkt.Data) < l2hlen+iphl+tcphl {
        DEBUG("ip", "Packet too short to contain TCP header")
        return
    }

    tcphdr := pkt.Data[l2hlen+iphl : l2hlen+iphl+tcphl]
    packet.tuple.Src_port = Bytes_Ntohs(tcphdr[0:2])
    packet.tuple.Dst_port = Bytes_Ntohs(tcphdr[2:4])

    //DEBUG(" %s:%d -> %s:%d", packet.tuple.src_ip, packet.tuple.src_port,
    //      packet.tuple.dst_ip, packet.tuple.dst_port)

    packet.payload = pkt.Data[l2hlen+iphl+tcphl : l2hlen+ip_length]

    FollowTcp(tcphdr, packet)
}

func main() {

    configfile := flag.String("c", "packetbeat.conf", "Configuration file")
    file := flag.String("I", "", "file")
    loop := flag.Int("l", 1, "Loop file. 0 - loop forever")
    debugSelectorsStr := flag.String("d", "", "Enable certain debug selectors")
    oneAtAtime := flag.Bool("O", false, "Read packets one at a time (press Enter)")
    toStdout := flag.Bool("e", false, "Output to stdout instead of syslog")
    topSpeed := flag.Bool("t", false, "Read packets as fast as possible, without sleeping")
    publishDisabled := flag.Bool("N", false, "Disable actual publishing for testing")

    flag.Parse()

    debugSelectors := []string{}
    logLevel := LOG_DEBUG
    if len(*debugSelectorsStr) > 0 {
        debugSelectors = strings.Split(*debugSelectorsStr, ",")
    }

    var h *pcap.Pcap
    var err error

    if _ConfigMeta, err = toml.DecodeFile(*configfile, &_Config); err != nil {
        fmt.Printf("TOML config parsing failed on %s: %s. Exiting.\n", *configfile, err)
        return
    }

    if len(debugSelectors) == 0 {
        debugSelectors = _Config.Logging.Selectors
    }
    LogInit(logLevel, "", !*toStdout, debugSelectors)

    if *file != "" {
        h, err = pcap.Openoffline(*file)
        if h == nil {
            ERR("Openoffline(%s) failed: %s", *file, err)
            return
        }
    } else {
        device := _Config.Interfaces.Device

        h, err = pcap.Openlive(device, 65535, true, 0)
        if h == nil {
            ERR("pcap.Openlive failed(%s): %s", device, err)
            return
        }
    }
    defer h.Close()

    if err = DropPrivileges(); err != nil {
        CRIT(err.Error())
        return
    }

    filter := configToFilter(&_Config)
    if filter != "" {
        DEBUG("pcapfilter", "Installing filter '%s'", filter)
        err := h.Setfilter(filter)
        if err != nil {
            ERR("pcap.Setfilter failed: %s", err)
            return
        }
    }

    if err = Publisher.Init(*publishDisabled); err != nil {
        CRIT(err.Error())
        return
    }

    if err = procWatcher.Init(&_Config.Procs); err != nil {
        CRIT(err.Error())
        return
    }

    if err = TcpInit(); err != nil {
        CRIT(err.Error())
        return
    }

    datalink := h.Datalink()
    if datalink != pcap.LINKTYPE_ETHERNET && datalink != pcap.LINKTYPE_LINUX_SLL {
        WARN("Unsupported link type: %d", datalink)
    }

    _GeoLite, err = libgeo.Load("/usr/share/GeoIP/GeoIP.dat")
    if err != nil {
        WARN("Could not load GeoIP data: %s", err.Error())
    }

    counter := 0
    live := true
    loopCount := 1
    var lastPktTime *time.Time = nil
    for live {
        var pkt *pcap.Packet
        var res int32

        if *oneAtAtime {
            fmt.Println("Press enter to read packet")
            fmt.Scanln()
        }

        pkt, res = h.NextEx()
        switch res {
        case -1:
            ERR("pcap.NextEx() error: %s", h.Geterror())
            live = false
            continue
        case -2:
            DEBUG("pcapread", "End of file")
            loopCount += 1
            if *loop > 0 && loopCount > *loop {
                // give a bit of time to the publish goroutine
                // to flush
                time.Sleep(300 * time.Millisecond)
                live = false
            }
            // reopen the file
            h, err = pcap.Openoffline(*file)
            if h == nil {
                ERR("Openoffline(%s) failed: %s", *file, err)
                return
            }
            lastPktTime = nil
            continue
        case 0:
            // timeout
            continue
        }
        if res != 1 {
            panic(fmt.Sprintf("Unexpected return code from pcap.NextEx: %d", res))
        }

        if pkt == nil {
            panic("Nil packet despite res=1")
        }

        if *file != "" {
            if lastPktTime != nil && !*topSpeed {
                sleep := pkt.Time.Sub(*lastPktTime)
                if sleep > 0 {
                    time.Sleep(sleep)
                } else {
                    WARN("Time in pcap went backwards: %d", sleep)
                }
            }
            _lastPktTime := pkt.Time
            lastPktTime = &_lastPktTime
            pkt.Time = time.Now() // overwrite what we get from the pcap
        }
        counter++

        decodePktEth(datalink, pkt)
    }
    INFO("Input finish. Processed %d packets. Have a nice day!", counter)
}
<commit_msg>Added a -version flag to help  during support<commit_after>package main

import (
    "flag"
    "fmt"
    "runtime"
    "strconv"
    "strings"
    "time"

    "github.com/BurntSushi/toml"
    "github.com/akrennmair/gopcap"
    "github.com/nranchev/go-libGeoIP"
)

const Version = "0.1.1"

type Packet struct {
    ts      time.Time
    tuple   IpPortTuple
    payload []byte
}

type protocolType uint16

const (
    UnknownProtocol protocolType = iota
    HttpProtocol
    MysqlProtocol
    RedisProtocol
)

var protocolNames = []string{"unknown", "http", "mysql", "redis"}

type tomlConfig struct {
    Interfaces    tomlInterfaces
    RunOptions    tomlRunOptions
    Protocols     map[string]tomlProtocol
    Procs         tomlProcs
    Elasticsearch tomlMothership
    Agent         tomlAgent
    Logging       tomlLogging
}

type tomlInterfaces struct {
    Device string
}

type tomlRunOptions struct {
    Uid int
    Gid int
}

type tomlLogging struct {
    Selectors []string
}

var _Config tomlConfig
var _ConfigMeta toml.MetaData
var _GeoLite *libgeo.GeoIP

func Bytes_Ipv4_Ntoa(bytes []byte) string {
    var strarr []string = make([]string, 4)
    for i, b := range bytes {
        strarr[i] = strconv.Itoa(int(b))
    }
    return strings.Join(strarr, ".")
}

func decodePktEth(datalink int, pkt *pcap.Packet) {
    defer RECOVER("decodePktEth exception")

    packet := new(Packet)
    var l2hlen int
    var eth_type uint16

    if datalink != pcap.LINKTYPE_LINUX_SLL {
        // we are just assuming ETH
        l2hlen = 14

        // bytes 12 and 13 are the pkt type
        eth_type = Bytes_Ntohs(pkt.Data[12:14])

    } else {
        l2hlen = 16

        // bytes 14 and 15 are the pkt type
        eth_type = Bytes_Ntohs(pkt.Data[14:16])
    }

    if eth_type != 0x800 {
        DEBUG("ip", "Not ipv4 packet. Ignoring")
        return
    }

    if len(pkt.Data) < l2hlen+20 {
        DEBUG("ip", "Packet too short to be ethernet")
        return
    }

    // IP header
    iphl := int((uint16(pkt.Data[l2hlen]) & 0x0f) * 4)
    if len(pkt.Data) < l2hlen+iphl {
        DEBUG("ip", "Packet too short to be IP")
        return
    }
    iphdr := pkt.Data[l2hlen : l2hlen+iphl]

    //DEBUG("Packet timestamp: %s", pkt.Time)
    packet.ts = pkt.Time

    packet.tuple.Src_ip = Bytes_Ntohl(iphdr[12:16])
    packet.tuple.Dst_ip = Bytes_Ntohl(iphdr[16:20])

    mf := (uint8(iphdr[6]&0x20) != 0)
    frag_offset := (uint16(iphdr[7]) << 3) | (uint16(iphdr[6]) & 0x1F)

    if mf || frag_offset > 0 {
        DEBUG("ip", "Fragmented packets not yet supported")
        return
    }

    ip_length := int(Bytes_Ntohs(iphdr[2:4]))

    protocol := uint8(iphdr[9])
    if protocol != 6 {
        DEBUG("ip", "Not TCP packet. Ignoring")
        return
    }

    if len(pkt.Data) < l2hlen+iphl+20 {
        DEBUG("ip", "Packet too short to be TCP")
        return
    }

    tcphl := int((uint16(pkt.Data[l2hlen+iphl+12]) >> 4) * 4)
    if tcphl > 20 && len(pkt.Data) < l2hlen+iphl+tcphl {
        DEBUG("ip", "Packet too short to contain TCP header")
        return
    }

    tcphdr := pkt.Data[l2hlen+iphl : l2hlen+iphl+tcphl]
    packet.tuple.Src_port = Bytes_Ntohs(tcphdr[0:2])
    packet.tuple.Dst_port = Bytes_Ntohs(tcphdr[2:4])

    //DEBUG(" %s:%d -> %s:%d", packet.tuple.src_ip, packet.tuple.src_port,
    //      packet.tuple.dst_ip, packet.tuple.dst_port)

    packet.payload = pkt.Data[l2hlen+iphl+tcphl : l2hlen+ip_length]

    FollowTcp(tcphdr, packet)
}

func main() {

    configfile := flag.String("c", "packetbeat.conf", "Configuration file")
    file := flag.String("I", "", "file")
    loop := flag.Int("l", 1, "Loop file. 0 - loop forever")
    debugSelectorsStr := flag.String("d", "", "Enable certain debug selectors")
    oneAtAtime := flag.Bool("O", false, "Read packets one at a time (press Enter)")
    toStdout := flag.Bool("e", false, "Output to stdout instead of syslog")
    topSpeed := flag.Bool("t", false, "Read packets as fast as possible, without sleeping")
    publishDisabled := flag.Bool("N", false, "Disable actual publishing for testing")
    printVersion := flag.Bool("version", false, "Print version and exit")

    flag.Parse()

    if *printVersion {
        fmt.Printf("Packetbat verision %s (%s)\n", Version, runtime.GOARCH)
    }

    debugSelectors := []string{}
    logLevel := LOG_DEBUG
    if len(*debugSelectorsStr) > 0 {
        debugSelectors = strings.Split(*debugSelectorsStr, ",")
    }

    var h *pcap.Pcap
    var err error

    if _ConfigMeta, err = toml.DecodeFile(*configfile, &_Config); err != nil {
        fmt.Printf("TOML config parsing failed on %s: %s. Exiting.\n", *configfile, err)
        return
    }

    if len(debugSelectors) == 0 {
        debugSelectors = _Config.Logging.Selectors
    }
    LogInit(logLevel, "", !*toStdout, debugSelectors)

    if *file != "" {
        h, err = pcap.Openoffline(*file)
        if h == nil {
            ERR("Openoffline(%s) failed: %s", *file, err)
            return
        }
    } else {
        device := _Config.Interfaces.Device

        h, err = pcap.Openlive(device, 65535, true, 0)
        if h == nil {
            ERR("pcap.Openlive failed(%s): %s", device, err)
            return
        }
    }
    defer h.Close()

    if err = DropPrivileges(); err != nil {
        CRIT(err.Error())
        return
    }

    filter := configToFilter(&_Config)
    if filter != "" {
        DEBUG("pcapfilter", "Installing filter '%s'", filter)
        err := h.Setfilter(filter)
        if err != nil {
            ERR("pcap.Setfilter failed: %s", err)
            return
        }
    }

    if err = Publisher.Init(*publishDisabled); err != nil {
        CRIT(err.Error())
        return
    }

    if err = procWatcher.Init(&_Config.Procs); err != nil {
        CRIT(err.Error())
        return
    }

    if err = TcpInit(); err != nil {
        CRIT(err.Error())
        return
    }

    datalink := h.Datalink()
    if datalink != pcap.LINKTYPE_ETHERNET && datalink != pcap.LINKTYPE_LINUX_SLL {
        WARN("Unsupported link type: %d", datalink)
    }

    _GeoLite, err = libgeo.Load("/usr/share/GeoIP/GeoIP.dat")
    if err != nil {
        WARN("Could not load GeoIP data: %s", err.Error())
    }

    counter := 0
    live := true
    loopCount := 1
    var lastPktTime *time.Time = nil
    for live {
        var pkt *pcap.Packet
        var res int32

        if *oneAtAtime {
            fmt.Println("Press enter to read packet")
            fmt.Scanln()
        }

        pkt, res = h.NextEx()
        switch res {
        case -1:
            ERR("pcap.NextEx() error: %s", h.Geterror())
            live = false
            continue
        case -2:
            DEBUG("pcapread", "End of file")
            loopCount += 1
            if *loop > 0 && loopCount > *loop {
                // give a bit of time to the publish goroutine
                // to flush
                time.Sleep(300 * time.Millisecond)
                live = false
            }
            // reopen the file
            h, err = pcap.Openoffline(*file)
            if h == nil {
                ERR("Openoffline(%s) failed: %s", *file, err)
                return
            }
            lastPktTime = nil
            continue
        case 0:
            // timeout
            continue
        }
        if res != 1 {
            panic(fmt.Sprintf("Unexpected return code from pcap.NextEx: %d", res))
        }

        if pkt == nil {
            panic("Nil packet despite res=1")
        }

        if *file != "" {
            if lastPktTime != nil && !*topSpeed {
                sleep := pkt.Time.Sub(*lastPktTime)
                if sleep > 0 {
                    time.Sleep(sleep)
                } else {
                    WARN("Time in pcap went backwards: %d", sleep)
                }
            }
            _lastPktTime := pkt.Time
            lastPktTime = &_lastPktTime
            pkt.Time = time.Now() // overwrite what we get from the pcap
        }
        counter++

        decodePktEth(datalink, pkt)
    }
    INFO("Input finish. Processed %d packets. Have a nice day!", counter)
}
<|endoftext|>


<commit_before>// Copyright 2013 The glamor authors.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"net/mail"
	"net/smtp"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const Version = "0.1"

var flagHelp bool
var flagHost string
var flagInterval int64
var flagMaxErrors int64
var flagSMTP string
var flagSMTPFrom string
var flagSMTPSkipCertificateVerify bool
var flagSMTPTLS bool
var flagSMTPTo string
var flagVerbose bool

func sendMail() {
	if flagSMTP == "" {
		return
	}

	c, err := smtp.Dial(flagSMTP)

	if err != nil {
		v("Cannot open SMTP connection: %v\n", err)

		return
	}

	if flagSMTPTLS {
		err := c.StartTLS(&tls.Config{InsecureSkipVerify: flagSMTPSkipCertificateVerify})

		if err != nil {
			v("Cannot start SMTP TLS: %v\n", err)

			return
		}
	}

	c.Mail(flagSMTPFrom)
	c.Rcpt(flagSMTPTo)

	wc, err := c.Data()

	if err != nil {
		v("Cannot open Data writer: %v\n", err)

		return
	}

	defer wc.Close()

	buf := bytes.NewBufferString(`Return-path: <` + flagSMTPFrom + `>
From: ` + flagSMTPFrom + `
To: ` + flagSMTPTo + `
Subject: ` + flagHost + ` is down
Content-Transfer-Encoding: 7Bit
Content-Type: text/plain; charset="us-ascii"

` + flagHost + ` is not reachable via ping
`)

	if _, err = buf.WriteTo(wc); err != nil {
		v("Cannot write mail body: %v\n", err)
	}

	v("Mail sent\n")
}

func v(format string, a ...interface{}) {
	if flagVerbose {
		fmt.Fprintf(os.Stderr, format, a...)
	}
}

func main() {
	flag.BoolVar(&flagHelp, "help", false, "Show this help")
	flag.StringVar(&flagHost, "host", "", "The host to ping")
	flag.Int64Var(&flagInterval, "interval", 60, "Ping interval in seconds")
	flag.Int64Var(&flagMaxErrors, "max-errors", 5, "How many pings can fail before a report is sent")
	flag.StringVar(&flagSMTP, "smtp", "", "The SMTP server + port for sending report mails")
	flag.StringVar(&flagSMTPFrom, "smtp-from", "", "From-mail address")
	flag.BoolVar(&flagSMTPSkipCertificateVerify, "smtp-skip-certificate-verify", false, "Do not verify the SMTP certificate")
	flag.BoolVar(&flagSMTPTLS, "smtp-tls", false, "Use TLS for the SMTP connection")
	flag.StringVar(&flagSMTPTo, "smtp-to", "", "To-mail address")
	flag.BoolVar(&flagVerbose, "verbose", false, "Do verbose output")

	flag.Parse()

	if flagHost == "" || flagInterval <= 0 || flagMaxErrors <= 0 || flagHelp {
		fmt.Printf("glamor v%s\n", Version)
		fmt.Printf("usage:\n")
		fmt.Printf("\t%s -host <host> -interval <interval>\n", os.Args[0])
		fmt.Printf("options\n")
		flag.PrintDefaults()
		fmt.Printf("\n")

		if !flagHelp {
			panic("Wrong arguments")
		}
	}

	if _, err := mail.ParseAddress(flagSMTPFrom); flagSMTPFrom != "" && err != nil {
		panic("smtp-from is not a valid mail address")
	} else if _, err := mail.ParseAddress(flagSMTPTo); flagSMTPFrom != "" && err != nil {
		panic("smtp-to is not a valid mail address")
	}

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)

	go func() {
		s := <-sig
		v("Caught signal \"%v\", will exit now\n", s)

		os.Exit(0)
	}()

	var sentMail = false
	var errors int64 = 0

	for {
		var cmd = exec.Command("ping", []string{"-w", "1", "-c", "1", flagHost}...)

		out, err := cmd.CombinedOutput()

		if err != nil {
			v("Ping failed to %s: %v\n", flagHost, err)
		}

		if strings.Contains(string(out), "1 received") {
			errors--

			if sentMail && errors < 100 {
				errors = 0
				sentMail = false

				v("Reset mail sent flag\n")
			}
		} else {
			errors++

			if errors >= flagMaxErrors {
				errors = 0

				v("Reached error count\n")

				if !sentMail {
					sendMail()

					sentMail = true
				}
			}
		}

		time.Sleep(time.Duration(flagInterval) * time.Second)
	}

	return
}
<commit_msg>add reset host down status argument<commit_after>// Copyright 2013 The glamor authors.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"net/mail"
	"net/smtp"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const Version = "0.1"

var flagHelp bool
var flagHost string
var flagInterval int64
var flagMaxErrors int64
var flagResetHostDown int64
var flagSMTP string
var flagSMTPFrom string
var flagSMTPSkipCertificateVerify bool
var flagSMTPTLS bool
var flagSMTPTo string
var flagVerbose bool

func sendMail() {
	if flagSMTP == "" {
		return
	}

	c, err := smtp.Dial(flagSMTP)

	if err != nil {
		v("Cannot open SMTP connection: %v\n", err)

		return
	}

	if flagSMTPTLS {
		err := c.StartTLS(&tls.Config{InsecureSkipVerify: flagSMTPSkipCertificateVerify})

		if err != nil {
			v("Cannot start SMTP TLS: %v\n", err)

			return
		}
	}

	c.Mail(flagSMTPFrom)
	c.Rcpt(flagSMTPTo)

	wc, err := c.Data()

	if err != nil {
		v("Cannot open Data writer: %v\n", err)

		return
	}

	defer wc.Close()

	buf := bytes.NewBufferString(`Return-path: <` + flagSMTPFrom + `>
From: ` + flagSMTPFrom + `
To: ` + flagSMTPTo + `
Subject: ` + flagHost + ` is down
Content-Transfer-Encoding: 7Bit
Content-Type: text/plain; charset="us-ascii"

` + flagHost + ` is not reachable via ping
`)

	if _, err = buf.WriteTo(wc); err != nil {
		v("Cannot write mail body: %v\n", err)
	}

	v("Mail sent\n")
}

func v(format string, a ...interface{}) {
	if flagVerbose {
		fmt.Fprintf(os.Stderr, format, a...)
	}
}

func main() {
	flag.BoolVar(&flagHelp, "help", false, "Show this help")
	flag.StringVar(&flagHost, "host", "", "The host to ping")
	flag.Int64Var(&flagInterval, "interval", 60, "Ping interval in seconds")
	flag.Int64Var(&flagMaxErrors, "max-errors", 5, "How many pings can fail before a report is sent")
	flag.Int64Var(&flagResetHostDown, "reset-host-down", 50, "How many pings have to be successful to reset the host down status")
	flag.StringVar(&flagSMTP, "smtp", "", "The SMTP server + port for sending report mails")
	flag.StringVar(&flagSMTPFrom, "smtp-from", "", "From-mail address")
	flag.BoolVar(&flagSMTPSkipCertificateVerify, "smtp-skip-certificate-verify", false, "Do not verify the SMTP certificate")
	flag.BoolVar(&flagSMTPTLS, "smtp-tls", false, "Use TLS for the SMTP connection")
	flag.StringVar(&flagSMTPTo, "smtp-to", "", "To-mail address")
	flag.BoolVar(&flagVerbose, "verbose", false, "Do verbose output")

	flag.Parse()

	if flagHost == "" || flagInterval <= 0 || flagMaxErrors <= 0 || flagResetHostDown <= 0 || flagHelp {
		fmt.Printf("glamor v%s\n", Version)
		fmt.Printf("usage:\n")
		fmt.Printf("\t%s -host <host> -interval <interval>\n", os.Args[0])
		fmt.Printf("options\n")
		flag.PrintDefaults()
		fmt.Printf("\n")

		if !flagHelp {
			panic("Wrong arguments")
		}
	}

	if _, err := mail.ParseAddress(flagSMTPFrom); flagSMTPFrom != "" && err != nil {
		panic("smtp-from is not a valid mail address")
	} else if _, err := mail.ParseAddress(flagSMTPTo); flagSMTPFrom != "" && err != nil {
		panic("smtp-to is not a valid mail address")
	}

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)

	go func() {
		s := <-sig
		v("Caught signal \"%v\", will exit now\n", s)

		os.Exit(0)
	}()

	var sentMail = false
	var errors int64 = 0

	for {
		var cmd = exec.Command("ping", []string{"-w", "1", "-c", "1", flagHost}...)

		out, err := cmd.CombinedOutput()

		if err != nil {
			v("Ping failed to %s: %v\n", flagHost, err)
		}

		if strings.Contains(string(out), "1 received") {
			errors--

			if sentMail && errors <= flagResetHostDown {
				errors = 0
				sentMail = false

				v("Reset mail sent flag\n")
			}
		} else {
			errors++

			if errors >= flagMaxErrors {
				errors = 0

				v("Reached error count\n")

				if !sentMail {
					sendMail()

					sentMail = true
				}
			}
		}

		time.Sleep(time.Duration(flagInterval) * time.Second)
	}

	return
}
<|endoftext|>


<commit_before>
<commit_msg>implemented !quote and !print<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var host *string = flag.String("host", "irc.ps0ke.de:2342", "IRC server")
var channel *string = flag.String("channel", "#metttest", "IRC channel")
var nick *string = flag.String("nick", "rohmett", "IRC nick")
var longnick *string = flag.String("longnick", "Le MettBot", "IRC fullname")
var timeformat *string = flag.String("timeformat", "2006-01-02T15:04", "Time format string (standard date: 2006-01-02T15:04:05")
var quotes *string = flag.String("quotes", "mett_quotes.txt", "Quote database file")

func init() {
	flag.Parse()
}

type Mettbot struct {
	*irc.Conn
	Quitted    chan bool
	Prnt       chan string
	Input      chan string
	ReallyQuit bool
}

func NewMettbot(nick string, args ...string) *Mettbot {
	bot := &Mettbot{
		irc.SimpleClient(nick, args...),
		make(chan bool),
		make(chan string),
		make(chan string, 4),
		false,
	}
	bot.EnableStateTracking()
	return bot
}

func (bot *Mettbot) hConnected()    { bot.Join(*channel) }
func (bot *Mettbot) hDisconnected() { bot.Quitted <- true }

func (bot *Mettbot) hPrivmsg(line *irc.Line) {
	actChannel := line.Args[0]
	msg := line.Args[1]

	if msg[0] == '!' {
		cmd := msg
		args := ""
		idx := strings.Index(msg, " ")
		if idx != -1 {
			cmd = msg[:idx]
			args = msg[idx+1:]
		}

		switch {
		case cmd == "!help":
			bot.Help(actChannel)
		case args == "":
			bot.Syntax(actChannel)
		case cmd == "!quote":
			bot.cQuote(actChannel, args, line.Time)
		case cmd == "!print":
			bot.cPrint(actChannel, args)
		default:
			bot.Syntax(actChannel)
		}
	}
}

func (bot *Mettbot) Syntax(channel string) {
	bot.Notice(channel, "Wrong Syntax. Try !help")
}

func (bot *Mettbot) Help(channel string) {
	bot.Notice(channel, "Mett")
}

func (bot *Mettbot) cQuote(channel string, msg string, t time.Time) {
	s := fmt.Sprintln(t.Format(*timeformat), msg)
	fmt.Print(s)
	bot.Prnt <- s
	bot.Notice(channel, "Added Quote to Database")
}

func (bot *Mettbot) cPrint(channel string, msg string) {
	num, err := strconv.Atoi(msg)
	if err != nil {
		bot.Syntax(channel)
		return
	}

	fi, err := os.Open(*quotes)
	if err != nil {
		log.Println(err)
		bot.Notice(channel, "Failed to open quote database")
		return
	}
	defer fi.Close()

	reader := bufio.NewReader(fi)
	quote := ""
	for ; num >= 0; num-- {
		quote, err = reader.ReadString('\n')
		if err == io.EOF {
			bot.Notice(channel, "Quote not found")
			return
		}
		if err != nil {
			log.Println(err)
			bot.Notice(channel, "Failed to read from quote database")
			return
		}
	}
	bot.Notice(channel, quote)
}

func (bot *Mettbot) readStdin() {
	con := bufio.NewReader(os.Stdin)
	for {
		s, err := con.ReadString('\n')
		if err != nil {
			// wha?, maybe ctrl-D...
			close(bot.Input)
			break
		}
		// no point in sending empty lines down the channel
		if len(s) > 2 {
			bot.Input <- s[0 : len(s)-1]
		}
	}
}

func (bot *Mettbot) parseStdin() {
	for cmd := range bot.Input {
		if cmd[0] == ':' {
			switch idx := strings.Index(cmd, " "); {
			case cmd[1] == 'd':
				fmt.Printf(bot.String())
			case cmd[1] == 'f':
				if len(cmd) > 2 && cmd[2] == 'e' {
					// enable flooding
					bot.Flood = true
				} else if len(cmd) > 2 && cmd[2] == 'd' {
					// disable flooding
					bot.Flood = false
				}
				for i := 0; i < 20; i++ {
					bot.Privmsg(*channel, "salami!1!")
				}
			case idx == -1:
				continue
			case cmd[1] == 'q':
				bot.ReallyQuit = true
				bot.Quit(cmd[idx+1 : len(cmd)])
			case cmd[1] == 'j':
				bot.Join(cmd[idx+1 : len(cmd)])
			case cmd[1] == 'p':
				bot.Part(cmd[idx+1 : len(cmd)])
			case cmd[1] == 'm':
				bot.Privmsg(*channel, cmd[idx+1:len(cmd)])
			case cmd[1] == 'a':
				bot.Action(*channel, cmd[idx+1:len(cmd)])
			}
		} else {
			bot.Raw(cmd)
		}
	}
}

func (bot *Mettbot) writeQuote() {
	for messages := range bot.Prnt {
		fo, err := os.OpenFile(*quotes, syscall.O_WRONLY+syscall.O_CREAT+syscall.O_APPEND, 0644)
		if err != nil {
			log.Println(err)
			bot.Notice(*channel, "Couldn't open quote database")
			continue
		}
		defer fo.Close()

		_, err = fo.WriteString(messages)
		if err != nil {
			log.Println(err)
			bot.Notice(*channel, "Couldn't write to quote database")
		}
		fo.Close()
	}
}

func main() {
	// create new IRC connection
	mett := NewMettbot(*nick, *nick, *longnick)

	// Set up handlers
	mett.AddHandler("connected", func(conn *irc.Conn, line *irc.Line) { mett.hConnected() })
	mett.AddHandler("disconnected", func(conn *irc.Conn, line *irc.Line) { mett.hDisconnected() })
	mett.AddHandler("privmsg", func(conn *irc.Conn, line *irc.Line) { mett.hPrivmsg(line) })

	// set up a goroutine to read commands from stdin
	go mett.readStdin()

	// set up a goroutine to do parsey things with the stuff from stdin
	go mett.parseStdin()

	// Set up a go routine to write quotes to file
	go mett.writeQuote()

	for !mett.ReallyQuit {
		// connect to server
		if err := mett.Connect(*host); err != nil {
			fmt.Printf("Connection error: %s\n", err)
			return
		}

		// wait on quit channel
		<-mett.Quitted
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/peterbourgon/g2g"
)

type zipper string

var Zipper zipper

var Metrics = struct {
	Requests         *expvar.Int
	RequestCacheHits *expvar.Int

	FindRequests  *expvar.Int
	FindCacheHits *expvar.Int

	RenderRequests *expvar.Int
}{
	Requests:         expvar.NewInt("requests"),
	RequestCacheHits: expvar.NewInt("request_cache_hits"),

	FindRequests:  expvar.NewInt("find_requests"),
	FindCacheHits: expvar.NewInt("find_cache_hits"),

	RenderRequests: expvar.NewInt("render_requests"),
}

var queryCache bytesCache
var findCache bytesCache

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) string {

	if s == "" {
		// return the default if nothing was passed
		return strconv.Itoa(int(d))
	}

	// relative timestamp
	if s[0] == '-' {

		offset, err := intervalString(s[1:])
		if err != nil {
			return strconv.Itoa(int(d))
		}

		return strconv.Itoa(int(timeNow().Add(-time.Duration(offset) * time.Second).Unix()))
	}

	if s == "now" {
		return strconv.Itoa(int(timeNow().Unix()))
	}

	_, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return s // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return strconv.Itoa(int(t.Unix()))
		}
	}
	return strconv.Itoa(int(d))
}

func intervalString(s string) (int32, error) {

	var j int

	for j < len(s) && s[j] >= '0' && s[j] <= '9' {
		j++
	}
	offsetStr, unitStr := s[:j], s[j:]

	var units int
	switch unitStr {
	case "s", "sec", "secs", "second", "seconds":
		units = 1
	case "min", "minute", "minutes":
		units = 60
	case "h", "hour", "hours":
		units = 60 * 60
	case "d", "day", "days":
		units = 24 * 60 * 60
	case "w", "week", "weeks":
		units = 7 * 24 * 60 * 60
	case "mon", "month", "months":
		units = 30 * 24 * 60 * 60
	case "y", "year", "years":
		units = 365 * 24 * 60 * 60
	default:
		return 0, errors.New("unknown time units")
	}

	offset, err := strconv.Atoi(offsetStr)
	if err != nil {
		return 0, err
	}

	return int32(offset * units), nil
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric, from, until string) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{from},
		"until":  []string{until},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	Metrics.Requests.Add(1)

	r.ParseForm()
	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	useCache := r.FormValue("noCache") == ""

	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	cacheKey := r.Form.Encode()

	if response, ok := queryCache.get(cacheKey); useCache && ok {
		Metrics.RequestCacheHits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		w.Write(response)
		return
	}

	// normalize from and until values
	from = dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until = dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	metricMap := make(map[string][]*pb.FetchResponse)

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		for _, metric := range exp.metrics() {

			if _, ok := metricMap[metric]; ok {
				// already fetched this metric for this request
				continue
			}

			var glob pb.GlobResponse
			var haveCacheData bool

			if response, ok := findCache.get(metric); useCache && ok {
				Metrics.FindCacheHits.Add(1)
				err := proto.Unmarshal(response, &glob)
				haveCacheData = err == nil
			}

			if !haveCacheData {
				var err error
				Metrics.FindRequests.Add(1)
				glob, err = Zipper.Find(metric)
				if err != nil {
					continue
				}
				b, err := proto.Marshal(&glob)
				if err == nil {
					findCache.set(metric, b)
				}
			}

			// For each metric returned in the Find response, query Render
			// This is a conscious decision to *not* cache render data
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			leaves := 0
			for _, m := range glob.GetMatches() {
				if !m.GetIsLeaf() {
					continue
				}
				Metrics.RenderRequests.Add(1)
				leaves++
				Limiter.enter()
				go func(m *pb.GlobMatch) {
					var rptr *pb.FetchResponse
					r, err := Zipper.Render(m.GetPath(), from, until)
					if err == nil {
						rptr = &r
					}
					rch <- rptr
					Limiter.leave()
				}(m)
			}

			for i := 0; i < leaves; i++ {
				r := <-rch
				if r != nil {
					metricMap[metric] = append(metricMap[metric], r)
				}
			}
		}

		exprs := evalExpr(exp, metricMap)
		results = append(results, exprs...)
	}

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jout, err := json.Marshal(jresults)
	if err != nil {
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	queryCache.set(cacheKey, jout)

	w.Header().Set("Content-Type", "application/json")
	w.Write(jout)
}

func lbcheckHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Ok\n")
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")
	mc := flag.String("mc", "", "comma separated memcached server list")
	cpus := flag.Int("cpus", 0, "number of CPUs to use")

	flag.Parse()

	if *z == "" {
		log.Fatal("no zipper (-z) provided")
	}

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	if *mc != "" {
		servers := strings.Split(*mc, ",")
		log.Println("using memcache servers:", servers)
		queryCache = &memcachedCache{client: memcache.New(servers...)}
		findCache = &memcachedCache{client: memcache.New(servers...)}
	} else {
		queryCache = &expireCache{cache: make(map[string]cacheElement)}
		go queryCache.(*expireCache).cleaner()

		findCache = &expireCache{cache: make(map[string]cacheElement)}
		go findCache.(*expireCache).cleaner()
	}

	Zipper = zipper(*z)

	if *cpus != 0 {
		log.Println("using GOMAXPROCS", *cpus)
		runtime.GOMAXPROCS(*cpus)
	}

	if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {

		log.Println("Using graphite host", host)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(host, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatal("unable to connect to to graphite: ", host, ":", err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.api.%s.requests", hostname), Metrics.Requests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.request_cache_hits", hostname), Metrics.RequestCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.find_requests", hostname), Metrics.FindRequests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.find_cache_hits", hostname), Metrics.FindCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.render_requests", hostname), Metrics.RenderRequests)
	}

	http.HandleFunc("/render/", renderHandler)
	http.HandleFunc("/lbcheck", lbcheckHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
<commit_msg>bugs++<commit_after>package main

import (
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/peterbourgon/g2g"
)

type zipper string

var Zipper zipper

var Metrics = struct {
	Requests         *expvar.Int
	RequestCacheHits *expvar.Int

	FindRequests  *expvar.Int
	FindCacheHits *expvar.Int

	RenderRequests *expvar.Int
}{
	Requests:         expvar.NewInt("requests"),
	RequestCacheHits: expvar.NewInt("request_cache_hits"),

	FindRequests:  expvar.NewInt("find_requests"),
	FindCacheHits: expvar.NewInt("find_cache_hits"),

	RenderRequests: expvar.NewInt("render_requests"),
}

var queryCache bytesCache
var findCache bytesCache

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) string {

	if s == "" {
		// return the default if nothing was passed
		return strconv.Itoa(int(d))
	}

	// relative timestamp
	if s[0] == '-' {

		offset, err := intervalString(s[1:])
		if err != nil {
			return strconv.Itoa(int(d))
		}

		return strconv.Itoa(int(timeNow().Add(-time.Duration(offset) * time.Second).Unix()))
	}

	if s == "now" {
		return strconv.Itoa(int(timeNow().Unix()))
	}

	_, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return s // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return strconv.Itoa(int(t.Unix()))
		}
	}
	return strconv.Itoa(int(d))
}

func intervalString(s string) (int32, error) {

	var j int

	for j < len(s) && s[j] >= '0' && s[j] <= '9' {
		j++
	}
	offsetStr, unitStr := s[:j], s[j:]

	var units int
	switch unitStr {
	case "s", "sec", "secs", "second", "seconds":
		units = 1
	case "min", "minute", "minutes":
		units = 60
	case "h", "hour", "hours":
		units = 60 * 60
	case "d", "day", "days":
		units = 24 * 60 * 60
	case "w", "week", "weeks":
		units = 7 * 24 * 60 * 60
	case "mon", "month", "months":
		units = 30 * 24 * 60 * 60
	case "y", "year", "years":
		units = 365 * 24 * 60 * 60
	default:
		return 0, errors.New("unknown time units")
	}

	offset, err := strconv.Atoi(offsetStr)
	if err != nil {
		return 0, err
	}

	return int32(offset * units), nil
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric, from, until string) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{from},
		"until":  []string{until},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	Metrics.Requests.Add(1)

	r.ParseForm()
	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	useCache := r.FormValue("noCache") == ""

	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	cacheKey := r.Form.Encode()

	if response, ok := queryCache.get(cacheKey); useCache && ok {
		Metrics.RequestCacheHits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		w.Write(response)
		return
	}

	// normalize from and until values
	// BUG(dgryski): doesn't handle timezones the same as graphite-web
	from = dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until = dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	metricMap := make(map[string][]*pb.FetchResponse)

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		for _, metric := range exp.metrics() {

			if _, ok := metricMap[metric]; ok {
				// already fetched this metric for this request
				continue
			}

			var glob pb.GlobResponse
			var haveCacheData bool

			if response, ok := findCache.get(metric); useCache && ok {
				Metrics.FindCacheHits.Add(1)
				err := proto.Unmarshal(response, &glob)
				haveCacheData = err == nil
			}

			if !haveCacheData {
				var err error
				Metrics.FindRequests.Add(1)
				glob, err = Zipper.Find(metric)
				if err != nil {
					continue
				}
				b, err := proto.Marshal(&glob)
				if err == nil {
					findCache.set(metric, b)
				}
			}

			// For each metric returned in the Find response, query Render
			// This is a conscious decision to *not* cache render data
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			leaves := 0
			for _, m := range glob.GetMatches() {
				if !m.GetIsLeaf() {
					continue
				}
				Metrics.RenderRequests.Add(1)
				leaves++
				Limiter.enter()
				go func(m *pb.GlobMatch) {
					var rptr *pb.FetchResponse
					r, err := Zipper.Render(m.GetPath(), from, until)
					if err == nil {
						rptr = &r
					}
					rch <- rptr
					Limiter.leave()
				}(m)
			}

			for i := 0; i < leaves; i++ {
				r := <-rch
				if r != nil {
					metricMap[metric] = append(metricMap[metric], r)
				}
			}
		}

		exprs := evalExpr(exp, metricMap)
		results = append(results, exprs...)
	}

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jout, err := json.Marshal(jresults)
	if err != nil {
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	queryCache.set(cacheKey, jout)

	w.Header().Set("Content-Type", "application/json")
	w.Write(jout)
}

func lbcheckHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Ok\n")
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")
	mc := flag.String("mc", "", "comma separated memcached server list")
	cpus := flag.Int("cpus", 0, "number of CPUs to use")

	flag.Parse()

	if *z == "" {
		log.Fatal("no zipper (-z) provided")
	}

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	if *mc != "" {
		servers := strings.Split(*mc, ",")
		log.Println("using memcache servers:", servers)
		queryCache = &memcachedCache{client: memcache.New(servers...)}
		findCache = &memcachedCache{client: memcache.New(servers...)}
	} else {
		queryCache = &expireCache{cache: make(map[string]cacheElement)}
		go queryCache.(*expireCache).cleaner()

		findCache = &expireCache{cache: make(map[string]cacheElement)}
		go findCache.(*expireCache).cleaner()
	}

	Zipper = zipper(*z)

	if *cpus != 0 {
		log.Println("using GOMAXPROCS", *cpus)
		runtime.GOMAXPROCS(*cpus)
	}

	if host := os.Getenv("GRAPHITEHOST") + ":" + os.Getenv("GRAPHITEPORT"); host != ":" {

		log.Println("Using graphite host", host)

		// register our metrics with graphite
		graphite, err := g2g.NewGraphite(host, 60*time.Second, 10*time.Second)
		if err != nil {
			log.Fatal("unable to connect to to graphite: ", host, ":", err)
		}

		hostname, _ := os.Hostname()
		hostname = strings.Replace(hostname, ".", "_", -1)

		graphite.Register(fmt.Sprintf("carbon.api.%s.requests", hostname), Metrics.Requests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.request_cache_hits", hostname), Metrics.RequestCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.find_requests", hostname), Metrics.FindRequests)
		graphite.Register(fmt.Sprintf("carbon.api.%s.find_cache_hits", hostname), Metrics.FindCacheHits)

		graphite.Register(fmt.Sprintf("carbon.api.%s.render_requests", hostname), Metrics.RenderRequests)
	}

	http.HandleFunc("/render/", renderHandler)
	http.HandleFunc("/lbcheck", lbcheckHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
<|endoftext|>


<commit_before>package main

import (
//		"net"
		"fmt"
		"io/ioutil"
		"encoding/json"
//		"os"
//		"bufio"
		"log"
//		"flag"
	)

type AuthSource struct {
	Name string					// String name, used for cli
	URL string					// URL to query
	HTTPVerb string				// GET or POST
	Headers string				// Any other headers
	DetectionMethod string		// Valid options: ResponseCode, Regex (have to go beneath the javascript to the api js is querying)
	LoginSuccess string			// respose code to match if response code, regex to match if regex
	LoginFailed string			// same as success but for failure.  Need both to confirm.
}

func authentication_source_from_file(filename string) (AuthSource, error) {
	var auth_source AuthSource
	file_contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return auth_source, err
	}
	json_err := json.Unmarshal(file_contents, &auth_source)
	return auth_source, json_err
}

func build_auth_source_slice(directory string) []AuthSource {
	var auth_sources []AuthSource
	entries, dir_err := ioutil.ReadDir(directory)
	if dir_err != nil {
		log.Fatal(dir_err)
	}
	for _, filename := range entries {
		auth_source, err := authentication_source_from_file(directory+"/"+filename.Name())
		if err != nil {
			fmt.Println("Error loading ", filename.Name() ,": ", err)
		} else {
			auth_sources = append(auth_sources, auth_source)
		}
	}
	return auth_sources
}

func main() {
	fmt.Println("\n\tStarting AuthenticationScanner 1.0...\n")
	auth_source_array := build_auth_source_slice("sites")
	site_count := len(auth_source_array)
	if site_count == 0 {
		log.Fatal("No usable sites loaded, exiting.")
	}
	fmt.Println("Sites loaded:", site_count)
	// parse arguments for proxies and such here
	for _, site := range auth_source_array {
		fmt.Println(site.Name)
		// each site will "go attempt_to_login(site, results_channel)".  The attemp_to_login method with handle proxies.
	}
	// a recieving function will loop site_count times and read login results from the results_channel
	// the recieving method returns a list of results, with are parsed and printed out.
}
<commit_msg>getting some ideas in<commit_after>package main

import (
//		"net"
		"fmt"
		"io/ioutil"
		"encoding/json"
//		"os"
//		"bufio"
		"log"
//		"flag"
	)

type AuthSource struct {
	Name string					// String name, used for cli
	URL string					// URL to query
	HTTPVerb string				// GET or POST
	Headers string				// Any other headers
	DetectionMethod string		// Valid options: ResponseCode, Regex (have to go beneath the javascript to the api js is querying)
	LoginSuccess string			// respose code to match if response code, regex to match if regex
	LoginFailed string			// same as success but for failure.  Need both to confirm.
	// Maybe have a field for login type (username or email), then apply the sites to the crentials dynamically
}

func authentication_source_from_file(filename string) (AuthSource, error) {
	var auth_source AuthSource
	file_contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return auth_source, err
	}
	json_err := json.Unmarshal(file_contents, &auth_source)
	return auth_source, json_err
}

func build_auth_source_slice(directory string) []AuthSource {
	var auth_sources []AuthSource
	entries, dir_err := ioutil.ReadDir(directory)
	if dir_err != nil {
		log.Fatal(dir_err)
	}
	for _, filename := range entries {
		auth_source, err := authentication_source_from_file(directory+"/"+filename.Name())
		if err != nil {
			fmt.Println("Error loading ", filename.Name() ,": ", err)
		} else {
			auth_sources = append(auth_sources, auth_source)
		}
	}
	return auth_sources
}

func main() {
	fmt.Println("\n\tStarting AuthenticationScanner 1.0...\n")
	auth_source_array := build_auth_source_slice("sites")
	site_count := len(auth_source_array)
	if site_count == 0 {
		log.Fatal("No usable sites loaded, exiting.")
	}
	fmt.Println("Sites loaded:", site_count)
	// parse arguments for proxies and such here
	// single credentials to test?  a whole file of crenetials?
	for _, site := range auth_source_array {
		fmt.Println(site.Name)
		// each site will "go attempt_to_login(site, results_channel)".  The attemp_to_login method with handle proxies.
	}
	// a recieving function will loop site_count times and read login results from the results_channel
	// the recieving method returns a list of results, with are parsed and printed out.
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"log"
)

var (
	debug       = flag.Bool("debug", false, "enable debug output")
	templateDir = flag.String("template-dir", "_templates", "templates directory")
	sourceDir   = flag.String("source-dir", "_source", "source directory")
	staticDir   = flag.String("static-dir", "_static", "static directory")
	outputDir   = flag.String("output-dir", "_output", "output directory")
)

func Debugf(format string, args ...interface{}) {
	if *debug {
		log.Printf("DEBUG "+format, args...)
	}
}

func Problemf(format string, args ...interface{}) {
	log.Printf("PROBLEM "+format, args...)
}

func Fatalf(format string, args ...interface{}) {
	log.Fatalf("FATAL "+format, args...)
}

func init() {
	flag.Parse()
}

// A source file will specify
//  - the template it wants to use to render itself
//  - the data it wants to put into that template
//  - its output path
//
// A template file will specify
//  - subtemplates it can pull in
//  - somehow defaults for unspecified variables

func main() {
	if err := os.MkdirAll(*outputDir, 0755); err != nil {
		Fatalf("%s: %s", *outputDir, err)
	}

	// copy static
	copyStatic := exec.Command("cp", "-r", *staticDir+"/", *outputDir+"/")
	buf, err := copyStatic.CombinedOutput()
	if err != nil {
		Fatalf("copying %s: %s (%s)", *staticDir, err, strings.TrimSpace(string(buf)))
	}

	// render source files
	sourceFiles, err := filepath.Glob(*sourceDir + "/*")
	if err != nil {
		Fatalf("%s: %s", *sourceDir, err)
	}
	for _, sourceFile := range sourceFiles {
		// extract context
		templateFile, ctx, outputFile, err := ContextFrom(sourceFile)
		if err != nil {
			Problemf("Context: %s: %s", sourceFile, err)
			continue
		}

		// render using specified template
		buf, err := RenderTemplate(*templateDir, templateFile, ctx)
		if err != nil {
			Problemf("Render: %s: %s", sourceFile, err)
			continue
		}

		// write output
		totalOutputFile := *outputDir + "/" + outputFile
		if err := os.MkdirAll(filepath.Dir(totalOutputFile), 0755); err != nil {
			Problemf("MkdirAll: %s: %s: %s", sourceFile, outputFile, err)
			continue
		}
		f, err := os.Create(totalOutputFile)
		if err != nil {
			Problemf("Create: %s: %s: %s", sourceFile, outputFile, err)
			continue
		}
		defer f.Close()
		n, err := f.Write(buf)
		if err != nil {
			Problemf("Write: %s: %s: %s", sourceFile, outputFile, err)
			continue
		}
		if n != len(buf) {
			Problemf("%s: %s: %d < %d", sourceFile, outputFile, n, len(buf))
			continue
		}
	}
}
<commit_msg>Better logging<commit_after>package main

import (
	"flag"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"log"
)

var (
	debug       = flag.Bool("debug", false, "enable debug output")
	templateDir = flag.String("template-dir", "_templates", "templates directory")
	sourceDir   = flag.String("source-dir", "_source", "source directory")
	staticDir   = flag.String("static-dir", "_static", "static directory")
	outputDir   = flag.String("output-dir", "_output", "output directory")
)

func Debugf(format string, args ...interface{}) {
	if *debug {
		log.Printf("DEBUG "+format, args...)
	}
}

func Problemf(format string, args ...interface{}) {
	log.Printf("PROBLEM "+format, args...)
}

func Fatalf(format string, args ...interface{}) {
	log.Fatalf("FATAL "+format, args...)
}

func init() {
	flag.Parse()
	log.SetFlags(0)
}

// A source file will specify
//  - the template it wants to use to render itself
//  - the data it wants to put into that template
//  - its output path
//
// A template file will specify
//  - subtemplates it can pull in
//  - somehow defaults for unspecified variables

func main() {
	if err := os.MkdirAll(*outputDir, 0755); err != nil {
		Fatalf("%s: %s", *outputDir, err)
	}

	// copy static
	copyStatic := exec.Command("cp", "-r", *staticDir+"/", *outputDir+"/")
	buf, err := copyStatic.CombinedOutput()
	if err != nil {
		Fatalf("copying %s: %s (%s)", *staticDir, err, strings.TrimSpace(string(buf)))
	}

	// render source files
	sourceFiles, err := filepath.Glob(*sourceDir + "/*")
	if err != nil {
		Fatalf("%s: %s", *sourceDir, err)
	}
	for i, sourceFile := range sourceFiles {
		// extract context
		templateFile, ctx, outputFile, err := ContextFrom(sourceFile)
		if err != nil {
			Problemf("Context: %s: %s", sourceFile, err)
			continue
		}

		// render using specified template
		buf, err := RenderTemplate(*templateDir, templateFile, ctx)
		if err != nil {
			Problemf("Render: %s: %s", sourceFile, err)
			continue
		}

		// write output
		totalOutputFile := *outputDir + "/" + outputFile
		if err := os.MkdirAll(filepath.Dir(totalOutputFile), 0755); err != nil {
			Problemf("MkdirAll: %s: %s: %s", sourceFile, outputFile, err)
			continue
		}
		f, err := os.Create(totalOutputFile)
		if err != nil {
			Problemf("Create: %s: %s: %s", sourceFile, outputFile, err)
			continue
		}
		defer f.Close()
		n, err := f.Write(buf)
		if err != nil {
			Problemf("Write: %s: %s: %s", sourceFile, outputFile, err)
			continue
		}
		if n != len(buf) {
			Problemf("%s: %s: %d < %d", sourceFile, outputFile, n, len(buf))
			continue
		}

		log.Printf(
			" %d/%d) %s → %s OK",
			i+1,
			len(sourceFiles),
			sourceFile,
			totalOutputFile,
		)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 IBM Corp.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package k8s

import (
	"github.com/skydive-project/skydive/topology/graph"
)

func newMetadata(typ, name string, extra interface{}) graph.Metadata {
	return graph.Metadata{
		"Type":    typ,
		"Probe":   "k8s",
		"Manager": "k8s",
		"Name":    name,
		"K8s":     extra,
	}
}

func addMetadata(g *graph.Graph, n *graph.Node, extra interface{}) {
	g.AddMetadata(n, "Manager", "k8s")
	g.AddMetadata(n, "K8s", extra)
}
<commit_msg>k8s: use transacitons for addMetadata<commit_after>/*
 * Copyright 2017 IBM Corp.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package k8s

import (
	"github.com/skydive-project/skydive/topology/graph"
)

func newMetadata(typ, name string, extra interface{}) graph.Metadata {
	return graph.Metadata{
		"Type":    typ,
		"Probe":   "k8s",
		"Manager": "k8s",
		"Name":    name,
		"K8s":     extra,
	}
}

func addMetadata(g *graph.Graph, n *graph.Node, extra interface{}) {
	tr := g.StartMetadataTransaction(n)
	defer tr.Commit()
	tr.AddMetadata("Manager", "k8s")
	tr.AddMetadata("K8s", extra)
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package http

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/adapters/http/parser"
	. "github.com/TheThingsNetwork/ttn/core/errors"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
)

// Adapter type materializes an http adapter which implements the basic http protocol
type Adapter struct {
	parser.PacketParser                // The adapter's parser contract
	http.Client                        // Adapter is also an http client
	serveMux            *http.ServeMux // Holds a references to the adapter servemux in order to dynamically define endpoints
	packets             chan pktReq    // Channel used to "transforms" incoming request to something we can handle concurrently
	ctx                 log.Interface  // Just a logger, no one really cares about him.
}

// Message sent through the packets channel when an incoming request arrives
type pktReq struct {
	core.Packet             // The actual packet that has been parsed
	response    chan pktRes // A response channel waiting for an success or reject confirmation
}

// Message sent through the response channel of a pktReq
type pktRes struct {
	statusCode int    // The http status code to set as an answer
	content    []byte // The response content.
}

// NewAdapter constructs and allocates a new http adapter
func NewAdapter(port uint, parser parser.PacketParser, ctx log.Interface) (*Adapter, error) {
	a := Adapter{
		PacketParser: parser,
		serveMux:     http.NewServeMux(),
		packets:      make(chan pktReq),
		ctx:          ctx,
		Client:       http.Client{},
	}

	a.RegisterEndpoint("/packets", a.handlePostPacket)
	go a.listenRequests(port)

	return &a, nil
}

// Send implements the core.Adapter interface
func (a *Adapter) Send(p core.Packet, r ...core.Recipient) (core.Packet, error) {
	stats.MarkMeter("http_adapter.send")
	stats.UpdateHistogram("http_adapter.send.recipients", int64(len(r)))

	// Generate payload from core packet
	m, err := json.Marshal(p.Metadata)
	if err != nil {
		a.ctx.WithError(err).Warn("Invalid Packet")
		return core.Packet{}, errors.New(ErrInvalidStructure, err)
	}
	pl, err := p.Payload.MarshalBinary()
	if err != nil {
		a.ctx.WithError(err).Warn("Invalid Packet")
		return core.Packet{}, errors.New(ErrInvalidStructure, err)
	}
	payload := fmt.Sprintf(`{"payload":"%s","metadata":%s}`, base64.StdEncoding.EncodeToString(pl), m)

	devAddr, _ := p.DevAddr()
	ctx := a.ctx.WithField("devAddr", devAddr)
	ctx.Debug("Sending Packet")

	// Prepare ground for parrallel http request
	nb := len(r)
	cherr := make(chan error, nb)
	chresp := make(chan core.Packet, nb)
	wg := sync.WaitGroup{}
	wg.Add(nb)

	// Run each request
	for _, recipient := range r {
		go func(recipient core.Recipient) {
			defer wg.Done()

			ctx := ctx.WithField("recipient", recipient.Address)
			ctx.Debug("POST Request")

			buf := new(bytes.Buffer)
			buf.Write([]byte(payload))

			// Send request
			resp, err := a.Post(fmt.Sprintf("http://%s", recipient.Address.(string)), "application/json", buf)
			if err != nil {
				cherr <- err
				return
			}

			defer func() {
				// This is needed because the default HTTP client's Transport does not
				// attempt to reuse HTTP/1.0 or HTTP/1.1 TCP connections unless the Body
				// is read to completion and is closed.
				io.Copy(ioutil.Discard, resp.Body)
				resp.Body.Close()
			}()

			// Check response code
			if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusCreated {
				ctx.WithField("response", resp.StatusCode).Warn("Unexpected response")
				cherr <- fmt.Errorf("Unexpected response from server: %s (%d)", resp.Status, resp.StatusCode)
				return
			}

			// Process response body
			raw, err := ioutil.ReadAll(resp.Body)
			if err != nil && err != io.EOF {
				cherr <- err
				return
			}

			// Process packet
			var packet core.Packet
			if err := json.Unmarshal(raw, &packet); err != nil {
				cherr <- err
				return
			}

			chresp <- packet
		}(recipient)
	}

	// Wait for each request to be done, and return
	stats.IncCounter("http_adapter.send.waiting")
	wg.Wait()
	stats.DecCounter("http_adapter.send.waiting")

	// Collect errors
	var errs []error
	for i := 0; i < len(cherr); i += 1 {
		err := <-cherr
		ctx.WithError(err).Error("POST Failed")
		errs = append(errs, err)
	}

	// Check responses
	if len(chresp) > 1 {
		ctx.WithField("response_count", len(chresp)).Error("Received too many positive answers")
		return core.Packet{}, errors.New(ErrWrongBehavior, "Received too many positive answers")
	}

	// Get packet
	select {
	case packet := <-chresp:
		return packet, nil
	default:
		if errs != nil {
			return core.Packet{}, errors.New(ErrFailedOperation, fmt.Sprintf("%+v", errs))
		}
		ctx.Error("No response packet available")
		return core.Packet{}, errors.New(ErrWrongBehavior, "No response packet available")
	}
}

// RegisterEndpoint can be used by an external agent to register a handler to the adapter servemux
func (a *Adapter) RegisterEndpoint(url string, handler func(w http.ResponseWriter, req *http.Request)) {
	a.ctx.WithField("url", url).Info("Register new endpoint")
	a.serveMux.HandleFunc(url, handler)
}

// Next implements the core.Adapter interface
func (a *Adapter) Next() (core.Packet, core.AckNacker, error) {
	pktReq := <-a.packets
	return pktReq.Packet, packetAckNacker{response: pktReq.response}, nil
}

// NextRegistration implements the core.Adapter interface. Not implemented for this adapter.
//
// See broadcast and pubsub adapters for mechanisms to handle registrations.
func (a *Adapter) NextRegistration() (core.Packet, core.AckNacker, error) {
	return core.Packet{}, nil, errors.New(ErrNotSupported, "NextRegistration not supported for http adapter")
}

// listenRequests handles incoming registration request sent through http to the adapter
func (a *Adapter) listenRequests(port uint) {
	server := http.Server{
		Addr:    fmt.Sprintf("0.0.0.0:%d", port),
		Handler: a.serveMux,
	}
	a.ctx.WithField("port", port).Info("Starting Server")
	err := server.ListenAndServe()
	a.ctx.WithError(err).Warn("HTTP connection lost")
}
<commit_msg>[refactor] Add /healthz endpoint in HTTP adapter<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package http

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/adapters/http/parser"
	. "github.com/TheThingsNetwork/ttn/core/errors"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
)

// Adapter type materializes an http adapter which implements the basic http protocol
type Adapter struct {
	parser.PacketParser                // The adapter's parser contract
	http.Client                        // Adapter is also an http client
	serveMux            *http.ServeMux // Holds a references to the adapter servemux in order to dynamically define endpoints
	packets             chan pktReq    // Channel used to "transforms" incoming request to something we can handle concurrently
	ctx                 log.Interface  // Just a logger, no one really cares about him.
}

// Message sent through the packets channel when an incoming request arrives
type pktReq struct {
	core.Packet             // The actual packet that has been parsed
	response    chan pktRes // A response channel waiting for an success or reject confirmation
}

// Message sent through the response channel of a pktReq
type pktRes struct {
	statusCode int    // The http status code to set as an answer
	content    []byte // The response content.
}

// NewAdapter constructs and allocates a new http adapter
func NewAdapter(port uint, parser parser.PacketParser, ctx log.Interface) (*Adapter, error) {
	a := Adapter{
		PacketParser: parser,
		serveMux:     http.NewServeMux(),
		packets:      make(chan pktReq),
		ctx:          ctx,
		Client:       http.Client{},
	}

	a.RegisterEndpoint("/healthz", func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("ok"))
	})

	a.RegisterEndpoint("/packets", a.handlePostPacket)
	go a.listenRequests(port)

	return &a, nil
}

// Send implements the core.Adapter interface
func (a *Adapter) Send(p core.Packet, r ...core.Recipient) (core.Packet, error) {
	stats.MarkMeter("http_adapter.send")
	stats.UpdateHistogram("http_adapter.send.recipients", int64(len(r)))

	// Generate payload from core packet
	m, err := json.Marshal(p.Metadata)
	if err != nil {
		a.ctx.WithError(err).Warn("Invalid Packet")
		return core.Packet{}, errors.New(ErrInvalidStructure, err)
	}
	pl, err := p.Payload.MarshalBinary()
	if err != nil {
		a.ctx.WithError(err).Warn("Invalid Packet")
		return core.Packet{}, errors.New(ErrInvalidStructure, err)
	}
	payload := fmt.Sprintf(`{"payload":"%s","metadata":%s}`, base64.StdEncoding.EncodeToString(pl), m)

	devAddr, _ := p.DevAddr()
	ctx := a.ctx.WithField("devAddr", devAddr)
	ctx.Debug("Sending Packet")

	// Prepare ground for parrallel http request
	nb := len(r)
	cherr := make(chan error, nb)
	chresp := make(chan core.Packet, nb)
	wg := sync.WaitGroup{}
	wg.Add(nb)

	// Run each request
	for _, recipient := range r {
		go func(recipient core.Recipient) {
			defer wg.Done()

			ctx := ctx.WithField("recipient", recipient.Address)
			ctx.Debug("POST Request")

			buf := new(bytes.Buffer)
			buf.Write([]byte(payload))

			// Send request
			resp, err := a.Post(fmt.Sprintf("http://%s", recipient.Address.(string)), "application/json", buf)
			if err != nil {
				cherr <- err
				return
			}

			defer func() {
				// This is needed because the default HTTP client's Transport does not
				// attempt to reuse HTTP/1.0 or HTTP/1.1 TCP connections unless the Body
				// is read to completion and is closed.
				io.Copy(ioutil.Discard, resp.Body)
				resp.Body.Close()
			}()

			// Check response code
			if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusCreated {
				ctx.WithField("response", resp.StatusCode).Warn("Unexpected response")
				cherr <- fmt.Errorf("Unexpected response from server: %s (%d)", resp.Status, resp.StatusCode)
				return
			}

			// Process response body
			raw, err := ioutil.ReadAll(resp.Body)
			if err != nil && err != io.EOF {
				cherr <- err
				return
			}

			// Process packet
			var packet core.Packet
			if err := json.Unmarshal(raw, &packet); err != nil {
				cherr <- err
				return
			}

			chresp <- packet
		}(recipient)
	}

	// Wait for each request to be done, and return
	stats.IncCounter("http_adapter.send.waiting")
	wg.Wait()
	stats.DecCounter("http_adapter.send.waiting")

	// Collect errors
	var errs []error
	for i := 0; i < len(cherr); i += 1 {
		err := <-cherr
		ctx.WithError(err).Error("POST Failed")
		errs = append(errs, err)
	}

	// Check responses
	if len(chresp) > 1 {
		ctx.WithField("response_count", len(chresp)).Error("Received too many positive answers")
		return core.Packet{}, errors.New(ErrWrongBehavior, "Received too many positive answers")
	}

	// Get packet
	select {
	case packet := <-chresp:
		return packet, nil
	default:
		if errs != nil {
			return core.Packet{}, errors.New(ErrFailedOperation, fmt.Sprintf("%+v", errs))
		}
		ctx.Error("No response packet available")
		return core.Packet{}, errors.New(ErrWrongBehavior, "No response packet available")
	}
}

// RegisterEndpoint can be used by an external agent to register a handler to the adapter servemux
func (a *Adapter) RegisterEndpoint(url string, handler func(w http.ResponseWriter, req *http.Request)) {
	a.ctx.WithField("url", url).Info("Register new endpoint")
	a.serveMux.HandleFunc(url, handler)
}

// Next implements the core.Adapter interface
func (a *Adapter) Next() (core.Packet, core.AckNacker, error) {
	pktReq := <-a.packets
	return pktReq.Packet, packetAckNacker{response: pktReq.response}, nil
}

// NextRegistration implements the core.Adapter interface. Not implemented for this adapter.
//
// See broadcast and pubsub adapters for mechanisms to handle registrations.
func (a *Adapter) NextRegistration() (core.Packet, core.AckNacker, error) {
	return core.Packet{}, nil, errors.New(ErrNotSupported, "NextRegistration not supported for http adapter")
}

// listenRequests handles incoming registration request sent through http to the adapter
func (a *Adapter) listenRequests(port uint) {
	server := http.Server{
		Addr:    fmt.Sprintf("0.0.0.0:%d", port),
		Handler: a.serveMux,
	}
	a.ctx.WithField("port", port).Info("Starting Server")
	err := server.ListenAndServe()
	a.ctx.WithError(err).Warn("HTTP connection lost")
}
<|endoftext|>


<commit_before>package client

import (
	"github.com/gopherjs/jquery"
	//"honnef.co/go/js/console"
	"fmt"
)

type FormElement interface {
	Selector() string
	ContentAttribute() Attribute
	Val() string
	SetVal(string)
}

//InputTextId is a special case of HtmlId that is the exported text
//of a type in field.
type InputTextId interface {
	HtmlId
	FormElement
}

type inputTextIdImpl struct {
	htmlIdImpl
	attr *AttributeImpl
}

type RadioGroup interface {
	FormElement
}

type radioGroupImpl struct {
	dom      NarrowDom
	selector string
	attr     *AttributeImpl
}

//NewInputTextId returns a reference to a static part of the DOM that is the
//an input tag with the given id.
func NewInputTextId(id string) InputTextId {
	result := inputTextIdImpl{
		htmlIdImpl: NewHtmlId("input", id).(htmlIdImpl),
	}
	result.attr = NewAttribute(VALUE_ONLY, result.value, nil)
	result.htmlIdImpl.Dom().On(INPUT_EVENT, func(jquery.Event) {
		print("marking dirty")
		result.attr.markDirty()
		//		print("drain")
		//		DrainEagerQueue()
	})
	return result
}

//SetText puts the text provided into the value (such as with an input field).
func (self inputTextIdImpl) SetVal(s string) {
	self.htmlIdImpl.t.SetVal(s)
	self.htmlIdImpl.Dom().Trigger(INPUT_EVENT)
}

//Val returns the value of an input field.
func (self inputTextIdImpl) Val() string {
	v := self.htmlIdImpl.t.Val()
	if v == "undefined" {
		return ""
	}
	return v
}

func (self inputTextIdImpl) value() Equaler {
	return StringEqualer{S: self.Val()}
}

func (self inputTextIdImpl) ContentAttribute() Attribute {
	return self.attr
}

//NewRadioGroup selects a named set of radio button elements
//with a given name.
func NewRadioGroup(name string) RadioGroup {
	selector := "input:radio[name=\"" + name + "\"]"
	result := radioGroupImpl{selector: selector}
	if TestMode {
		result.dom = newTestOps()
	} else {
		result.dom = wrap(jquery.NewJQuery(selector))
	}
	result.attr = NewAttribute(VALUE_ONLY, result.value, nil)
	result.dom.On(CLICK, func(jquery.Event) {
		result.attr.markDirty()
	})
	return result
}

func (self radioGroupImpl) value() Equaler {
	return StringEqualer{S: self.dom.Val()}
}

func (self radioGroupImpl) Selector() string {
	return self.selector
}

func (self radioGroupImpl) ContentAttribute() Attribute {
	return self.attr
}

//Val returns the currently selected item or "" if no item is selected. It
//cannot deal with values that are "" or are exactly the string "undefined".
func (self radioGroupImpl) Val() string {
	var v string
	switch d := self.dom.(type) {
	case jqueryWrapper:
		v = d.jq.Filter(":checked").Val()
	case *testOpsImpl:
		v = d.Val()
	default:
		panic("unknown dom type")
	}
	if v == "undefined" {
		return ""
	}
	return v
}

//
func (self radioGroupImpl) SetVal(s string) {
	switch d := self.dom.(type) {
	case jqueryWrapper:
		child := d.jq.Filter(fmt.Sprintf("[value=\"%s\"]", s))
		child.SetProp("checked", true)
	case *testOpsImpl:
		d.SetVal(s)
		d.Trigger(CLICK)
	default:
		panic("unknown type of dom pointer!")
	}
}

//
// FORM VALIDATION CONSTRAINT
//
type formValidFunc func(map[string]Equaler) Equaler

type formValidConstraint struct {
	id []FormElement
	fn formValidFunc
}

func (self formValidConstraint) Inputs() []Attribute {
	result := make([]Attribute, len(self.id))
	for i, id := range self.id {
		result[i] = id.ContentAttribute()
	}
	return result
}

func (self formValidConstraint) Fn(v []Equaler) Equaler {
	args := make(map[string]Equaler)
	for i, id := range self.id {
		args[id.Selector()] = v[i]
	}
	return self.fn(args)
}

//NewFormValidConstraint creates a constraint that uses the function
//supplied to compute a value and has dependencies on all the remaining
//parameters.
func NewFormValidConstraint(fn formValidFunc, id ...FormElement) formValidConstraint {
	return formValidConstraint{fn: fn, id: id}
}
<commit_msg>remove leftover prints<commit_after>package client

import (
	"github.com/gopherjs/jquery"
	//"honnef.co/go/js/console"
	"fmt"
)

type FormElement interface {
	Selector() string
	ContentAttribute() Attribute
	Val() string
	SetVal(string)
}

//InputTextId is a special case of HtmlId that is the exported text
//of a type in field.
type InputTextId interface {
	HtmlId
	FormElement
}

type inputTextIdImpl struct {
	htmlIdImpl
	attr *AttributeImpl
}

type RadioGroup interface {
	FormElement
}

type radioGroupImpl struct {
	dom      NarrowDom
	selector string
	attr     *AttributeImpl
}

//NewInputTextId returns a reference to a static part of the DOM that is the
//an input tag with the given id.
func NewInputTextId(id string) InputTextId {
	result := inputTextIdImpl{
		htmlIdImpl: NewHtmlId("input", id).(htmlIdImpl),
	}
	result.attr = NewAttribute(VALUE_ONLY, result.value, nil)
	result.htmlIdImpl.Dom().On(INPUT_EVENT, func(jquery.Event) {
		result.attr.markDirty()
	})
	return result
}

//SetText puts the text provided into the value (such as with an input field).
func (self inputTextIdImpl) SetVal(s string) {
	self.htmlIdImpl.t.SetVal(s)
	self.htmlIdImpl.Dom().Trigger(INPUT_EVENT)
}

//Val returns the value of an input field.
func (self inputTextIdImpl) Val() string {
	v := self.htmlIdImpl.t.Val()
	if v == "undefined" {
		return ""
	}
	return v
}

func (self inputTextIdImpl) value() Equaler {
	return StringEqualer{S: self.Val()}
}

func (self inputTextIdImpl) ContentAttribute() Attribute {
	return self.attr
}

//NewRadioGroup selects a named set of radio button elements
//with a given name.
func NewRadioGroup(name string) RadioGroup {
	selector := "input:radio[name=\"" + name + "\"]"
	result := radioGroupImpl{selector: selector}
	if TestMode {
		result.dom = newTestOps()
	} else {
		result.dom = wrap(jquery.NewJQuery(selector))
	}
	result.attr = NewAttribute(VALUE_ONLY, result.value, nil)
	result.dom.On(CLICK, func(jquery.Event) {
		result.attr.markDirty()
	})
	return result
}

func (self radioGroupImpl) value() Equaler {
	return StringEqualer{S: self.dom.Val()}
}

func (self radioGroupImpl) Selector() string {
	return self.selector
}

func (self radioGroupImpl) ContentAttribute() Attribute {
	return self.attr
}

//Val returns the currently selected item or "" if no item is selected. It
//cannot deal with values that are "" or are exactly the string "undefined".
func (self radioGroupImpl) Val() string {
	var v string
	switch d := self.dom.(type) {
	case jqueryWrapper:
		v = d.jq.Filter(":checked").Val()
	case *testOpsImpl:
		v = d.Val()
	default:
		panic("unknown dom type")
	}
	if v == "undefined" {
		return ""
	}
	return v
}

//
func (self radioGroupImpl) SetVal(s string) {
	switch d := self.dom.(type) {
	case jqueryWrapper:
		child := d.jq.Filter(fmt.Sprintf("[value=\"%s\"]", s))
		child.SetProp("checked", true)
	case *testOpsImpl:
		d.SetVal(s)
		d.Trigger(CLICK)
	default:
		panic("unknown type of dom pointer!")
	}
}

//
// FORM VALIDATION CONSTRAINT
//
type formValidFunc func(map[string]Equaler) Equaler

type formValidConstraint struct {
	id []FormElement
	fn formValidFunc
}

func (self formValidConstraint) Inputs() []Attribute {
	result := make([]Attribute, len(self.id))
	for i, id := range self.id {
		result[i] = id.ContentAttribute()
	}
	return result
}

func (self formValidConstraint) Fn(v []Equaler) Equaler {
	args := make(map[string]Equaler)
	for i, id := range self.id {
		args[id.Selector()] = v[i]
	}
	return self.fn(args)
}

//NewFormValidConstraint creates a constraint that uses the function
//supplied to compute a value and has dependencies on all the remaining
//parameters.
func NewFormValidConstraint(fn formValidFunc, id ...FormElement) formValidConstraint {
	return formValidConstraint{fn: fn, id: id}
}
<|endoftext|>


<commit_before>// +build windows

package pb

import (
	"os"
	"syscall"
	"unsafe"
)

var tty = os.Stdin

var (
	kernel32 = syscall.NewLazyDLL("kernel32.dll")

	// GetConsoleScreenBufferInfo retrieves information about the
	// specified console screen buffer.
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx
	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
)

type (
	// Defines the coordinates of the upper left and lower right corners
	// of a rectangle.
	// See
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311(v=vs.85).aspx
	smallRect struct {
		Left, Top, Right, Bottom int16
	}

	// Defines the coordinates of a character cell in a console screen
	// buffer. The origin of the coordinate system (0,0) is at the top, left cell
	// of the buffer.
	// See
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx
	coordinates struct {
		X, Y int16
	}

	// Contains information about a console screen buffer.
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093(v=vs.85).aspx
	consoleScreenBufferInfo struct {
		dwSize              coordinates
		dwCursorPosition    coordinates
		wAttributes         word
		srWindow            smallRect
		dwMaximumWindowSize coordinates
	}
)

// terminalWidth returns width of the terminal.
func terminalWidth() (width int, err error) {
	var info consoleScreenBufferInfo
	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0)
	if e != 0 {
		return 0, error(e)
	}
	return int(buffer.dwSize.X), nil
}
<commit_msg>fix #67<commit_after>// +build windows

package pb

import (
	"os"
	"syscall"
	"unsafe"
)

var tty = os.Stdin

var (
	kernel32 = syscall.NewLazyDLL("kernel32.dll")

	// GetConsoleScreenBufferInfo retrieves information about the
	// specified console screen buffer.
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx
	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
)

type (
	// Defines the coordinates of the upper left and lower right corners
	// of a rectangle.
	// See
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311(v=vs.85).aspx
	smallRect struct {
		Left, Top, Right, Bottom int16
	}

	// Defines the coordinates of a character cell in a console screen
	// buffer. The origin of the coordinate system (0,0) is at the top, left cell
	// of the buffer.
	// See
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx
	coordinates struct {
		X, Y int16
	}

	word int16

	// Contains information about a console screen buffer.
	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093(v=vs.85).aspx
	consoleScreenBufferInfo struct {
		dwSize              coordinates
		dwCursorPosition    coordinates
		wAttributes         word
		srWindow            smallRect
		dwMaximumWindowSize coordinates
	}
)

// terminalWidth returns width of the terminal.
func terminalWidth() (width int, err error) {
	var info consoleScreenBufferInfo
	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0)
	if e != 0 {
		return 0, error(e)
	}
	return int(info.dwSize.X), nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/etix/mirrorbits/cli"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/core"
	"github.com/etix/mirrorbits/daemon"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/http"
	"github.com/etix/mirrorbits/logs"
	"github.com/etix/mirrorbits/mirrors"
	"github.com/etix/mirrorbits/process"
	"github.com/op/go-logging"
)

var (
	log = logging.MustGetLogger("main")
)

func init() {
	// Improves perf in 1.1.2 linux/amd64
	runtime.GOMAXPROCS(runtime.NumCPU())

	LoadConfig()
	logs.ReloadLogs()
}

func main() {

	if core.CpuProfile != "" {
		f, err := os.Create(core.CpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if core.Daemon {
		process.WritePidFile()

		// Show our nice welcome logo
		fmt.Printf(core.Banner+"\n\n", core.VERSION)

		/* Connect to the database */
		r := database.NewRedis()
		r.ConnectPubsub()
		c := mirrors.NewCache(r)
		h := http.HTTPServer(r, c)

		if r.CheckVersion() == database.ErrUpgradeRequired {
			log.Fatalf("Unsupported Redis version, please upgrade to Redis >= %s", database.RedisMinimumVersion)
			return
		}

		/* Start the background monitor */
		m := daemon.NewMonitor(r, c)
		if core.Monitor {
			go m.MonitorLoop()
		}

		/* Handle SIGNALS */
		k := make(chan os.Signal, 1)
		signal.Notify(k,
			syscall.SIGINT,  // Terminate
			syscall.SIGTERM, // Terminate
			syscall.SIGQUIT, // Stop gracefully
			syscall.SIGHUP,  // Reload config
			syscall.SIGUSR1, // Reopen log files
			syscall.SIGUSR2, // Seamless binary upgrade
		)
		go func() {
			for {
				sig := <-k
				switch sig {
				case syscall.SIGINT:
					fallthrough
				case syscall.SIGTERM:
					process.RemovePidFile()
					os.Exit(0)
				case syscall.SIGQUIT:
					m.Stop()
					if h.Listener != nil {
						log.Notice("Waiting for running tasks to finish...")
						h.Stop(5 * time.Second)
					} else {
						process.RemovePidFile()
						os.Exit(0)
					}
				case syscall.SIGHUP:
					listenAddress := GetConfig().ListenAddress
					if err := ReloadConfig(); err != nil {
						log.Warningf("SIGHUP Received: %s\n", err)
					} else {
						log.Notice("SIGHUP Received: Reloading configuration...")
					}
					if GetConfig().ListenAddress != listenAddress {
						h.Restarting = true
						h.Stop(1 * time.Second)
					}
					h.Reload()
				case syscall.SIGUSR1:
					log.Notice("SIGUSR1 Received: Re-opening logs...")
					logs.ReloadLogs()
				case syscall.SIGUSR2:
					log.Notice("SIGUSR2 Received: Seamless binary upgrade...")
					err := process.Relaunch(*h.Listener)
					if err != nil {
						log.Errorf("Relaunch failed: %s\n", err)
					} else {
						m.Stop()
						h.Stop(10 * time.Second)
					}
				}
			}
		}()

		// Recover an existing listener (see process.go)
		if l, ppid, err := process.Recover(); err == nil {
			h.SetListener(l)
			go func() {
				time.Sleep(500 * time.Millisecond)
				process.KillParent(ppid)
			}()
		}

		/* Finally start the HTTP server */
		var err error
		for {
			err = h.RunServer()
			if h.Restarting {
				h.Restarting = false
				continue
			}
			// This check is ugly but there's still no way to detect this error by type
			if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
				// This error is expected during a graceful shutdown
				err = nil
			}
			break
		}

		log.Debug("Waiting for monitor termination")
		m.Wait()

		log.Debug("Terminating server")
		h.Terminate()

		r.Close()

		process.RemovePidFile()

		if err != nil {
			log.Fatal(err)
		} else {
			log.Notice("Server stopped gracefully.")
		}
	} else {
		if err := cli.ParseCommands(core.Args()...); err != nil {
			log.Fatal(err)
		}
	}
	os.Exit(0)
}
<commit_msg>Remove forced GOMAXPROCS<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"fmt"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/etix/mirrorbits/cli"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/core"
	"github.com/etix/mirrorbits/daemon"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/http"
	"github.com/etix/mirrorbits/logs"
	"github.com/etix/mirrorbits/mirrors"
	"github.com/etix/mirrorbits/process"
	"github.com/op/go-logging"
)

var (
	log = logging.MustGetLogger("main")
)

func init() {
	LoadConfig()
	logs.ReloadLogs()
}

func main() {

	if core.CpuProfile != "" {
		f, err := os.Create(core.CpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if core.Daemon {
		process.WritePidFile()

		// Show our nice welcome logo
		fmt.Printf(core.Banner+"\n\n", core.VERSION)

		/* Connect to the database */
		r := database.NewRedis()
		r.ConnectPubsub()
		c := mirrors.NewCache(r)
		h := http.HTTPServer(r, c)

		if r.CheckVersion() == database.ErrUpgradeRequired {
			log.Fatalf("Unsupported Redis version, please upgrade to Redis >= %s", database.RedisMinimumVersion)
			return
		}

		/* Start the background monitor */
		m := daemon.NewMonitor(r, c)
		if core.Monitor {
			go m.MonitorLoop()
		}

		/* Handle SIGNALS */
		k := make(chan os.Signal, 1)
		signal.Notify(k,
			syscall.SIGINT,  // Terminate
			syscall.SIGTERM, // Terminate
			syscall.SIGQUIT, // Stop gracefully
			syscall.SIGHUP,  // Reload config
			syscall.SIGUSR1, // Reopen log files
			syscall.SIGUSR2, // Seamless binary upgrade
		)
		go func() {
			for {
				sig := <-k
				switch sig {
				case syscall.SIGINT:
					fallthrough
				case syscall.SIGTERM:
					process.RemovePidFile()
					os.Exit(0)
				case syscall.SIGQUIT:
					m.Stop()
					if h.Listener != nil {
						log.Notice("Waiting for running tasks to finish...")
						h.Stop(5 * time.Second)
					} else {
						process.RemovePidFile()
						os.Exit(0)
					}
				case syscall.SIGHUP:
					listenAddress := GetConfig().ListenAddress
					if err := ReloadConfig(); err != nil {
						log.Warningf("SIGHUP Received: %s\n", err)
					} else {
						log.Notice("SIGHUP Received: Reloading configuration...")
					}
					if GetConfig().ListenAddress != listenAddress {
						h.Restarting = true
						h.Stop(1 * time.Second)
					}
					h.Reload()
				case syscall.SIGUSR1:
					log.Notice("SIGUSR1 Received: Re-opening logs...")
					logs.ReloadLogs()
				case syscall.SIGUSR2:
					log.Notice("SIGUSR2 Received: Seamless binary upgrade...")
					err := process.Relaunch(*h.Listener)
					if err != nil {
						log.Errorf("Relaunch failed: %s\n", err)
					} else {
						m.Stop()
						h.Stop(10 * time.Second)
					}
				}
			}
		}()

		// Recover an existing listener (see process.go)
		if l, ppid, err := process.Recover(); err == nil {
			h.SetListener(l)
			go func() {
				time.Sleep(500 * time.Millisecond)
				process.KillParent(ppid)
			}()
		}

		/* Finally start the HTTP server */
		var err error
		for {
			err = h.RunServer()
			if h.Restarting {
				h.Restarting = false
				continue
			}
			// This check is ugly but there's still no way to detect this error by type
			if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
				// This error is expected during a graceful shutdown
				err = nil
			}
			break
		}

		log.Debug("Waiting for monitor termination")
		m.Wait()

		log.Debug("Terminating server")
		h.Terminate()

		r.Close()

		process.RemovePidFile()

		if err != nil {
			log.Fatal(err)
		} else {
			log.Notice("Server stopped gracefully.")
		}
	} else {
		if err := cli.ParseCommands(core.Args()...); err != nil {
			log.Fatal(err)
		}
	}
	os.Exit(0)
}
<|endoftext|>