<commit_before>// Package scan TCP ports
package scan

import (
	"context"
	"fmt"
	"math/rand"
	"net"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/olekukonko/tablewriter"
)

// Scan represents the scan parameters
type Scan struct {
	minPort int
	maxPort int
	target  string
	lport   int
	rport   int
	laddr   net.IP
	raddr   net.IP
	ifName  string
	network string
	forceV4 bool
	forceV6 bool
	synScan bool
	ip      gopacket.NetworkLayer
}

// NewScan creats scan object
func NewScan(args string, cfg cli.Config) (Scan, error) {
	var (
		scan Scan
		flag map[string]interface{}
		err  error
	)

	args, flag = cli.Flag(args)
	// help
	if _, ok := flag["help"]; ok || args == "" {
		help(cfg)
		return scan, fmt.Errorf("")
	}

	scan.forceV4 = cli.SetFlag(flag, "4", true).(bool)
	scan.forceV6 = cli.SetFlag(flag, "6", false).(bool)
	scan.synScan = cli.SetFlag(flag, "s", false).(bool)

	pRange := cli.SetFlag(flag, "p", cfg.Scan.Port).(string)

	re := regexp.MustCompile(`(\d+)(\-{0,1}(\d*))`)
	f := re.FindStringSubmatch(pRange)

	if len(f) != 4 {
		return scan, fmt.Errorf("error! please try scan help")
	}

	scan.target = args
	if len(f) == 4 && f[2] != "" {
		scan.minPort, err = strconv.Atoi(f[1])
		scan.maxPort, err = strconv.Atoi(f[3])
	} else {
		scan.minPort, err = strconv.Atoi(f[1])
		scan.maxPort, err = strconv.Atoi(f[1])
	}

	if err != nil {
		return scan, err
	}

	if scan.IsCIDR() {
		return scan, fmt.Errorf("it doesn't support CIDR")
	}

	scan.setIP()

	return scan, nil
}

// IsCIDR checks the target if it's CIDR
func (s *Scan) IsCIDR() bool {
	_, _, err := net.ParseCIDR(s.target)
	if err != nil {
		return false
	}
	return true
}

func isIPv4(ip net.IP) bool {
	return len(ip.To4()) == net.IPv4len
}

func isIPv6(ip net.IP) bool {
	if r := strings.Index(ip.String(), ":"); r != -1 {
		return true
	}
	return false
}

func (s *Scan) setIP() error {
	ips, err := net.LookupIP(s.target)
	if err != nil {
		return err
	}
	for _, ip := range ips {
		if isIPv4(ip) && !s.forceV6 {
			s.raddr = ip
			s.network = "ip4"
			break
		} else if isIPv6(ip) && !s.forceV4 {
			s.raddr = ip
			s.network = "ip6"
			break
		}
	}
	return nil
}

// Run tries to scan wide range ports (TCP)
func (s *Scan) Run() {
	var (
		openPorts []int
		err       error
	)

	fmt.Printf("Scan %s (%s)\n", s.target, s.raddr)

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Protocol", "Port", "Status"})

	tStart := time.Now()
	if !s.synScan {
		openPorts = s.tcpConnScan()
	} else {
		openPorts, err = s.tcpSYNScan()
	}

	if err != nil {
		println(err.Error())
		return
	}

	for _, p := range openPorts {
		table.Append([]string{"TCP", fmt.Sprintf("%d", p), "Open"})
	}
	table.Render()

	if len(openPorts) == 0 {
		println("there isn't any opened port")
	} else {
		elapsed := fmt.Sprintf("%.3f seconds", time.Since(tStart).Seconds())
		println("Scan done:", len(openPorts), "opened port(s) found in", elapsed)
	}

}

func (s *Scan) packetDataTCP(rport int) (error, []byte) {
	tcp := &layers.TCP{
		SrcPort: layers.TCPPort(s.lport),
		DstPort: layers.TCPPort(rport),
		Seq:     1,
		SYN:     true,
		Window:  15000,
	}

	tcp.SetNetworkLayerForChecksum(s.ip)
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{
		ComputeChecksums: true,
		FixLengths:       true,
	}

	if err := gopacket.SerializeLayers(buf, opts, tcp); err != nil {
		return err, []byte{}
	}

	return nil, buf.Bytes()
}

func (s *Scan) setProto(proto string) error {
	var netProto = fmt.Sprintf("%s:%s", s.network, proto)

	switch netProto {
	case "ip4:tcp":
		s.ip = &layers.IPv4{
			SrcIP:    s.laddr,
			DstIP:    s.raddr,
			Protocol: layers.IPProtocolTCP,
		}
	case "ip4:udp":
		s.ip = &layers.IPv4{
			SrcIP:    s.laddr,
			DstIP:    s.raddr,
			Protocol: layers.IPProtocolUDP,
		}
	case "ip6:tcp":
		s.ip = &layers.IPv6{
			SrcIP:      s.laddr,
			DstIP:      s.raddr,
			NextHeader: layers.IPProtocolTCP,
		}
	case "ip6:udp":
		s.ip = &layers.IPv6{
			SrcIP:      s.laddr,
			DstIP:      s.raddr,
			NextHeader: layers.IPProtocolUDP,
		}

	}

	return nil
}

func (s *Scan) setLocalNet() error {
	conn, err := net.Dial("udp", net.JoinHostPort(s.raddr.String(), "80"))
	if err != nil {
		return err
	}
	defer conn.Close()

	if lAddr, ok := conn.LocalAddr().(*net.UDPAddr); ok {
		s.laddr = lAddr.IP
		s.lport = lAddr.Port
	} else {
		return fmt.Errorf("can not find local address/port")
	}

	ifs, _ := net.Interfaces()
	for _, i := range ifs {
		addrs, _ := i.Addrs()
		for _, addr := range addrs {
			ip, _, _ := net.ParseCIDR(addr.String())
			if ip.String() == s.laddr.String() {
				s.ifName = i.Name
				break
			}
		}
	}

	return nil
}

func (s *Scan) tcpSYNScan() ([]int, error) {
	var err error

	ctx, cancel := context.WithCancel(context.Background())

	if err = s.setLocalNet(); err != nil {
		println(err.Error())
	}
	if err = s.setProto("tcp"); err != nil {
		println(err.Error())
	}
	go func() {
		if err := s.sendTCPSYN(); err != nil {
			println(err.Error())
		}
		cancel()
	}()

	openPorts, err := s.pCapture(ctx)
	return openPorts, err
}

func (s *Scan) pCapture(ctx context.Context) ([]int, error) {
	var (
		tcp       *layers.TCP
		timeout   = 100 * time.Nanosecond
		openPorts []int
		ok        bool
	)
	handle, err := pcap.OpenLive(s.ifName, 6*1024, false, timeout)
	if err != nil {
		return nil, err
	}
	defer handle.Close()

	filter := "(tcp or udp) and src host " + s.raddr.String()
	if err := handle.SetBPFFilter(filter); err != nil {
		return nil, err
	}

	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
LOOP:
	for {
		select {
		case <-ctx.Done():
			break LOOP
		case packet := <-packetSource.Packets():
			tcpLayer := packet.Layer(layers.LayerTypeTCP)
			if tcpLayer != nil {
				if tcp, ok = tcpLayer.(*layers.TCP); !ok {
					continue
				}
				if tcp.SYN && tcp.ACK {
					p, _ := strconv.Atoi(fmt.Sprintf("%d", tcp.SrcPort))
					openPorts = append(openPorts, p)
				}
				continue
			}
		}
	}
	//TODO: unique .....
	sort.Ints(openPorts)
	return openPorts, nil
}

func (s *Scan) sendTCPSYN() error {
	var (
		buf  []byte
		err  error
		conn net.PacketConn
	)
	if s.network != "ip6" {
		conn, err = net.ListenPacket(s.network+":tcp", "0.0.0.0")
	} else {
		conn, err = net.ListenPacket(s.network+":tcp", "[::]")
	}
	if err != nil {
		return err
	}
	for i := s.minPort; i <= s.maxPort; i++ {
		if err, buf = s.packetDataTCP(i); err != nil {
			return err
		}
		if _, err := conn.WriteTo(buf, &net.IPAddr{IP: s.raddr}); err != nil {
			println(err.Error())
		}
		time.Sleep(10 * time.Millisecond)
	}
	time.Sleep(2 * time.Second)
	return nil
}

// tcpConnScan tries to scan a single host
func (s *Scan) tcpConnScan() []int {
	var wg sync.WaitGroup

	var ports []int
	for i := s.minPort; i <= s.maxPort; i++ {
		wg.Add(1)
		go func(i int) {
			for {
				host := net.JoinHostPort(s.raddr.String(), fmt.Sprintf("%d", i))
				conn, err := net.DialTimeout("tcp", host, 2*time.Second)
				if err != nil {
					if strings.Contains(err.Error(), "too many open files") {
						// random back-off
						time.Sleep(time.Duration(10+rand.Int31n(30)) * time.Millisecond)
						continue
					}
					wg.Done()
					return
				}
				conn.Close()
				break
			}
			ports = append(ports, i)
			wg.Done()
		}(i)
	}

	wg.Wait()
	sort.Ints(ports)
	return ports
}

// help represents guide to user
func help(cfg cli.Config) {
	fmt.Printf(`
    usage:
          scan ip/host [option]
    options:
          -p port-range or port number      specified range or port number (default is %s)
    example:
          scan 8.8.8.8 -p 53
          scan www.google.com -p 1-500
	`,
		cfg.Scan.Port)
}
<commit_msg>add uniq slice func<commit_after>// Package scan TCP ports
package scan

import (
	"context"
	"fmt"
	"math/rand"
	"net"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/olekukonko/tablewriter"
)

// Scan represents the scan parameters
type Scan struct {
	minPort int
	maxPort int
	target  string
	lport   int
	rport   int
	laddr   net.IP
	raddr   net.IP
	ifName  string
	network string
	forceV4 bool
	forceV6 bool
	synScan bool
	ip      gopacket.NetworkLayer
}

// NewScan creats scan object
func NewScan(args string, cfg cli.Config) (Scan, error) {
	var (
		scan Scan
		flag map[string]interface{}
		err  error
	)

	args, flag = cli.Flag(args)
	// help
	if _, ok := flag["help"]; ok || args == "" {
		help(cfg)
		return scan, fmt.Errorf("")
	}

	scan.forceV4 = cli.SetFlag(flag, "4", true).(bool)
	scan.forceV6 = cli.SetFlag(flag, "6", false).(bool)
	scan.synScan = cli.SetFlag(flag, "s", false).(bool)

	pRange := cli.SetFlag(flag, "p", cfg.Scan.Port).(string)

	re := regexp.MustCompile(`(\d+)(\-{0,1}(\d*))`)
	f := re.FindStringSubmatch(pRange)

	if len(f) != 4 {
		return scan, fmt.Errorf("error! please try scan help")
	}

	scan.target = args
	if len(f) == 4 && f[2] != "" {
		scan.minPort, err = strconv.Atoi(f[1])
		scan.maxPort, err = strconv.Atoi(f[3])
	} else {
		scan.minPort, err = strconv.Atoi(f[1])
		scan.maxPort, err = strconv.Atoi(f[1])
	}

	if err != nil {
		return scan, err
	}

	if scan.IsCIDR() {
		return scan, fmt.Errorf("it doesn't support CIDR")
	}

	scan.setIP()

	return scan, nil
}

// IsCIDR checks the target if it's CIDR
func (s *Scan) IsCIDR() bool {
	_, _, err := net.ParseCIDR(s.target)
	if err != nil {
		return false
	}
	return true
}

func isIPv4(ip net.IP) bool {
	return len(ip.To4()) == net.IPv4len
}

func isIPv6(ip net.IP) bool {
	if r := strings.Index(ip.String(), ":"); r != -1 {
		return true
	}
	return false
}

func (s *Scan) setIP() error {
	ips, err := net.LookupIP(s.target)
	if err != nil {
		return err
	}
	for _, ip := range ips {
		if isIPv4(ip) && !s.forceV6 {
			s.raddr = ip
			s.network = "ip4"
			break
		} else if isIPv6(ip) && !s.forceV4 {
			s.raddr = ip
			s.network = "ip6"
			break
		}
	}
	return nil
}

// Run tries to scan wide range ports (TCP)
func (s *Scan) Run() {
	var (
		openPorts []int
		err       error
	)

	if s.minPort != s.maxPort {
		fmt.Printf("Scan %s (%s) TCP ports %d-%d\n", s.target, s.raddr, s.minPort, s.maxPort)
	} else {
		fmt.Printf("Scan %s (%s) TCP port %d\n", s.target, s.raddr, s.minPort)
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Protocol", "Port", "Status"})

	tStart := time.Now()
	if !s.synScan {
		openPorts = s.tcpConnScan()
	} else {
		openPorts, err = s.tcpSYNScan()
	}

	if err != nil {
		println(err.Error())
		return
	}

	for _, p := range openPorts {
		table.Append([]string{"TCP", fmt.Sprintf("%d", p), "Open"})
	}

	if len(openPorts) == 0 {
		println("there isn't any opened port")
	} else {
		table.Render()
		elapsed := fmt.Sprintf("%.3f seconds", time.Since(tStart).Seconds())
		println("Scan done:", len(openPorts), "opened port(s) found in", elapsed)
	}

}

func (s *Scan) packetDataTCP(rport int) (error, []byte) {
	tcp := &layers.TCP{
		SrcPort: layers.TCPPort(s.lport),
		DstPort: layers.TCPPort(rport),
		Seq:     1,
		SYN:     true,
		Window:  15000,
	}

	tcp.SetNetworkLayerForChecksum(s.ip)
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{
		ComputeChecksums: true,
		FixLengths:       true,
	}

	if err := gopacket.SerializeLayers(buf, opts, tcp); err != nil {
		return err, []byte{}
	}

	return nil, buf.Bytes()
}

func (s *Scan) setProto(proto string) error {
	var netProto = fmt.Sprintf("%s:%s", s.network, proto)

	switch netProto {
	case "ip4:tcp":
		s.ip = &layers.IPv4{
			SrcIP:    s.laddr,
			DstIP:    s.raddr,
			Protocol: layers.IPProtocolTCP,
		}
	case "ip4:udp":
		s.ip = &layers.IPv4{
			SrcIP:    s.laddr,
			DstIP:    s.raddr,
			Protocol: layers.IPProtocolUDP,
		}
	case "ip6:tcp":
		s.ip = &layers.IPv6{
			SrcIP:      s.laddr,
			DstIP:      s.raddr,
			NextHeader: layers.IPProtocolTCP,
		}
	case "ip6:udp":
		s.ip = &layers.IPv6{
			SrcIP:      s.laddr,
			DstIP:      s.raddr,
			NextHeader: layers.IPProtocolUDP,
		}

	}

	return nil
}

func (s *Scan) setLocalNet() error {
	conn, err := net.Dial("udp", net.JoinHostPort(s.raddr.String(), "80"))
	if err != nil {
		return err
	}
	defer conn.Close()

	if lAddr, ok := conn.LocalAddr().(*net.UDPAddr); ok {
		s.laddr = lAddr.IP
		s.lport = lAddr.Port
	} else {
		return fmt.Errorf("can not find local address/port")
	}

	ifs, _ := net.Interfaces()
	for _, i := range ifs {
		addrs, _ := i.Addrs()
		for _, addr := range addrs {
			ip, _, _ := net.ParseCIDR(addr.String())
			if ip.String() == s.laddr.String() {
				s.ifName = i.Name
				break
			}
		}
	}

	return nil
}

func (s *Scan) tcpSYNScan() ([]int, error) {
	var err error

	ctx, cancel := context.WithCancel(context.Background())

	if err = s.setLocalNet(); err != nil {
		println(err.Error())
	}
	if err = s.setProto("tcp"); err != nil {
		println(err.Error())
	}
	go func() {
		if err := s.sendTCPSYN(); err != nil {
			println(err.Error())
		}
		cancel()
	}()

	openPorts, err := s.pCapture(ctx)
	return openPorts, err
}

func (s *Scan) pCapture(ctx context.Context) ([]int, error) {
	var (
		tcp       *layers.TCP
		timeout   = 100 * time.Nanosecond
		openPorts []int
		ok        bool
	)
	handle, err := pcap.OpenLive(s.ifName, 6*1024, false, timeout)
	if err != nil {
		return nil, err
	}
	defer handle.Close()

	filter := "tcp and src host " + s.raddr.String()
	if err := handle.SetBPFFilter(filter); err != nil {
		return nil, err
	}

	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
LOOP:
	for {
		select {
		case <-ctx.Done():
			break LOOP
		case packet := <-packetSource.Packets():
			tcpLayer := packet.Layer(layers.LayerTypeTCP)
			if tcpLayer != nil {
				if tcp, ok = tcpLayer.(*layers.TCP); !ok {
					continue
				}
				if tcp.SYN && tcp.ACK {
					p, _ := strconv.Atoi(fmt.Sprintf("%d", tcp.SrcPort))
					openPorts = append(openPorts, p)
				}
				continue
			}
		}
	}
	openPorts = uniqSlice(openPorts)
	sort.Ints(openPorts)
	return openPorts, nil
}

func (s *Scan) sendTCPSYN() error {
	var (
		buf  []byte
		err  error
		conn net.PacketConn
	)
	if s.network != "ip6" {
		conn, err = net.ListenPacket(s.network+":tcp", "0.0.0.0")
	} else {
		conn, err = net.ListenPacket(s.network+":tcp", "[::]")
	}
	if err != nil {
		return err
	}
	for i := s.minPort; i <= s.maxPort; i++ {
		if err, buf = s.packetDataTCP(i); err != nil {
			return err
		}
		if _, err := conn.WriteTo(buf, &net.IPAddr{IP: s.raddr}); err != nil {
			println(err.Error())
		}
		time.Sleep(5 * time.Millisecond)
	}
	time.Sleep(1 * time.Second)
	return nil
}

// tcpConnScan tries to scan a single host
func (s *Scan) tcpConnScan() []int {
	var wg sync.WaitGroup

	var ports []int
	for i := s.minPort; i <= s.maxPort; i++ {
		wg.Add(1)
		go func(i int) {
			for {
				host := net.JoinHostPort(s.raddr.String(), fmt.Sprintf("%d", i))
				conn, err := net.DialTimeout("tcp", host, 2*time.Second)
				if err != nil {
					if strings.Contains(err.Error(), "too many open files") {
						// random back-off
						time.Sleep(time.Duration(10+rand.Int31n(30)) * time.Millisecond)
						continue
					}
					wg.Done()
					return
				}
				conn.Close()
				break
			}
			ports = append(ports, i)
			wg.Done()
		}(i)
	}

	wg.Wait()
	sort.Ints(ports)
	return ports
}

func uniqSlice(s []int) []int {
	m := map[int]bool{}
	r := []int{}
	for _, v := range s {
		if _, ok := m[v]; !ok {
			m[v] = true
			r = append(r, v)
		}
	}
	return r
}

// help represents guide to user
func help(cfg cli.Config) {
	fmt.Printf(`
    usage:
          scan ip/host [option]
    options:
          -p port-range or port number      specified range or port number (default is %s)
    example:
          scan 8.8.8.8 -p 53
          scan www.google.com -p 1-500
	`,
		cfg.Scan.Port)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis *database.Redis

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	// Also make the key expire automatically in case our process
	// gets killed.
	lockKey := fmt.Sprintf("SCANNING_%s", identifier)
	_, err := redis.String(conn.Do("SET", lockKey, 1, "NX", "EX", 10))
	if err == redis.ErrNil {
		return ScanInProgress
	} else if err != nil {
		return err
	}

	// Lock acquired, delete the lock once done
	defer conn.Do("DEL", lockKey)

	// Maintain the lock active while the scan is in progress
	done := make(chan struct{})
	defer close(done)
	go func() {
		conn := s.redis.Get()
		defer conn.Close()

		for {
			select {
			case <-done:
				return
			case <-stop:
				return
			case <-time.After(5 * time.Second):
				result, err := redis.Int(conn.Do("EXPIRE", lockKey, 10))
				if err != nil {
					log.Errorf("Renewing lock for %s failed: %s", identifier, err)
					return
				} else if result == 0 {
					log.Errorf("Renewing lock for %s failed: lock disappeared", identifier)
					return
				}
				log.Debugf("[%s] Lock renewed", identifier)
			}
		}
	}()

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(conn redis.Conn, path string, f os.FileInfo, err error) (*filedata, error) {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil, nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return nil, err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Infof("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Infof("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Infof("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	return d, nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	conn := s.redis.Get()
	defer conn.Close()

	if conn.Err() != nil {
		return conn.Err()
	}

	sourceFiles := make([]*filedata, 0, 1000)

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, func(path string, f os.FileInfo, err error) error {
		fd, err := s.walkSource(conn, path, f, err)
		if err != nil {
			return err
		}
		if fd != nil {
			sourceFiles = append(sourceFiles, fd)
		}
		return nil
	})

	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	conn.Send("MULTI")

	// Remove any left over
	conn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range sourceFiles {
		conn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(conn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	conn.Send("MULTI")
	for _, e := range sourceFiles {
		conn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(conn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			conn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(conn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	conn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}
<commit_msg>scan: only show the lock renewed debug messages when DEBUG is defined<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis *database.Redis

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	// Also make the key expire automatically in case our process
	// gets killed.
	lockKey := fmt.Sprintf("SCANNING_%s", identifier)
	_, err := redis.String(conn.Do("SET", lockKey, 1, "NX", "EX", 10))
	if err == redis.ErrNil {
		return ScanInProgress
	} else if err != nil {
		return err
	}

	// Lock acquired, delete the lock once done
	defer conn.Do("DEL", lockKey)

	// Maintain the lock active while the scan is in progress
	done := make(chan struct{})
	defer close(done)
	go func() {
		conn := s.redis.Get()
		defer conn.Close()

		for {
			select {
			case <-done:
				return
			case <-stop:
				return
			case <-time.After(5 * time.Second):
				result, err := redis.Int(conn.Do("EXPIRE", lockKey, 10))
				if err != nil {
					log.Errorf("Renewing lock for %s failed: %s", identifier, err)
					return
				} else if result == 0 {
					log.Errorf("Renewing lock for %s failed: lock disappeared", identifier)
					return
				}
				if os.Getenv("DEBUG") != "" {
					log.Debugf("[%s] Lock renewed", identifier)
				}
			}
		}
	}()

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(conn redis.Conn, path string, f os.FileInfo, err error) (*filedata, error) {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil, nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return nil, err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Infof("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Infof("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Infof("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	return d, nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	conn := s.redis.Get()
	defer conn.Close()

	if conn.Err() != nil {
		return conn.Err()
	}

	sourceFiles := make([]*filedata, 0, 1000)

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, func(path string, f os.FileInfo, err error) error {
		fd, err := s.walkSource(conn, path, f, err)
		if err != nil {
			return err
		}
		if fd != nil {
			sourceFiles = append(sourceFiles, fd)
		}
		return nil
	})

	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	conn.Send("MULTI")

	// Remove any left over
	conn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range sourceFiles {
		conn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(conn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	conn.Send("MULTI")
	for _, e := range sourceFiles {
		conn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(conn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			conn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(conn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	conn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}
<|endoftext|>


<commit_before>package geoip

import (
	"fmt"
	. "launchpad.net/gocheck"
	"testing"
)

// Hook up gocheck into the gotest runner.
func Test(t *testing.T) { TestingT(t) }

type GeoIPSuite struct {
}

var _ = Suite(&GeoIPSuite{})

func (s *GeoIPSuite) Testv4(c *C) {
	gi := Open()
	if gi == nil {
		fmt.Printf("Could not open GeoIP database\n")
	}

	c.Check(gi, NotNil)

	country := gi.GetCountry("207.171.7.51")
	c.Check(country, Equals, "US")
}
<commit_msg>Commit test updates<commit_after>package geoip

import (
	"fmt"
	. "launchpad.net/gocheck"
	"testing"
)

// Hook up gocheck into the gotest runner.
func Test(t *testing.T) { TestingT(t) }

type GeoIPSuite struct {
}

var _ = Suite(&GeoIPSuite{})

func (s *GeoIPSuite) Testv4(c *C) {
	gi, err := Open()
	if gi == nil || err != nil {
		fmt.Printf("Could not open GeoIP database: %s\n", err)
	}

	c.Check(gi, NotNil)

	country := gi.GetCountry("207.171.7.51")
	c.Check(country, Equals, "US")
}
<|endoftext|>


<commit_before>package geth

import (
	"errors"
	"fmt"
	"io"
	"math/big"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"syscall"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/eth"
	"github.com/ethereum/go-ethereum/les"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/node"
	"github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/rlp"
	whisper "github.com/ethereum/go-ethereum/whisper/whisperv2"
)

const (
	ClientIdentifier = "StatusIM" // Client identifier to advertise over the network
	VersionMajor     = 1          // Major version component of the current release
	VersionMinor     = 1          // Minor version component of the current release
	VersionPatch     = 0          // Patch version component of the current release
	VersionMeta      = "stable" // Version metadata to append to the version string

	RPCPort         = 8545 // RPC port (replaced in unit tests)
	NetworkPort     = 30303
	MaxPeers        = 25
	MaxLightPeers   = 20
	MaxPendingPeers = 0

	ProcessFileDescriptorLimit = uint64(2048)
	DatabaseCacheSize          = 128 // Megabytes of memory allocated to internal caching (min 16MB / database forced)

	EventNodeStarted = "node.started"
)

// Gas price settings
var (
	GasPrice                = new(big.Int).Mul(big.NewInt(20), common.Shannon)  // Minimal gas price to accept for mining a transactions
	GpoMinGasPrice          = new(big.Int).Mul(big.NewInt(20), common.Shannon)  // Minimum suggested gas price
	GpoMaxGasPrice          = new(big.Int).Mul(big.NewInt(500), common.Shannon) // Maximum suggested gas price
	GpoFullBlockRatio       = 80                                                // Full block threshold for gas price calculation (%)
	GpobaseStepDown         = 10                                                // Suggested gas price base step down ratio (1/1000)
	GpobaseStepUp           = 100                                               // Suggested gas price base step up ratio (1/1000)
	GpobaseCorrectionFactor = 110                                               // Suggested gas price base correction factor (%)
)

// default node configuration options
var (
	UseTestnetFlag = "true" // to be overridden via -ldflags '-X geth.UseTestnetFlag'
	UseTestnet     = false
)

func init() {
	if UseTestnetFlag == "true" { // set at compile time, here we make sure to set corresponding boolean flag
		UseTestnet = true
	}
}

// node-related errors
var (
	ErrRLimitRaiseFailure            = errors.New("failed to register the whisper service")
	ErrDatabaseAccessFailure         = errors.New("could not open database")
	ErrChainConfigurationFailure     = errors.New("could not make chain configuration")
	ErrEthServiceRegistrationFailure = errors.New("failed to register the Ethereum service")
	ErrSshServiceRegistrationFailure = errors.New("failed to register the Whisper service")
	ErrLightEthRegistrationFailure   = errors.New("failed to register the LES service")
)

type Node struct {
	geth    *node.Node    // reference to the running Geth node
	started chan struct{} // channel to wait for node to start
}

// Inited checks whether status node has been properly initialized
func (n *Node) Inited() bool {
	return n != nil && n.geth != nil
}

// MakeNode create a geth node entity
func MakeNode(dataDir string, rpcPort int) *Node {
	glog.CopyStandardLogTo("INFO")
	glog.SetToStderr(true)

	bootstrapNodes := params.MainnetBootnodes
	if UseTestnet {
		dataDir = filepath.Join(dataDir, "testnet")
		bootstrapNodes = params.TestnetBootnodes
	}

	// configure required node (should you need to update node's config, e.g. add bootstrap nodes, see node.Config)
	config := &node.Config{
		DataDir:           dataDir,
		UseLightweightKDF: true,
		Name:              ClientIdentifier,
		Version:           fmt.Sprintf("%d.%d.%d-%s", VersionMajor, VersionMinor, VersionPatch, VersionMeta),
		NoDiscovery:       true,
		DiscoveryV5:       true,
		DiscoveryV5Addr:   fmt.Sprintf(":%d", NetworkPort+1),
		BootstrapNodes:    bootstrapNodes,
		BootstrapNodesV5:  params.DiscoveryV5Bootnodes,
		ListenAddr:        fmt.Sprintf(":%d", NetworkPort),
		MaxPeers:          MaxPeers,
		MaxPendingPeers:   MaxPendingPeers,
		HTTPHost:          node.DefaultHTTPHost,
		HTTPPort:          rpcPort,
		HTTPCors:          "*",
		HTTPModules:       strings.Split("db,eth,net,web3,shh,personal,admin", ","), // TODO remove "admin" on main net
	}

	stack, err := node.New(config)
	if err != nil {
		Fatalf(ErrNodeMakeFailure)
	}

	// start Ethereum service
	if err := activateEthService(stack, makeDefaultExtra()); err != nil {
		Fatalf(fmt.Errorf("%v: %v", ErrEthServiceRegistrationFailure, err))
	}

	// start Whisper service
	if err := activateShhService(stack); err != nil {
		Fatalf(fmt.Errorf("%v: %v", ErrSshServiceRegistrationFailure, err))
	}

	return &Node{
		geth:    stack,
		started: make(chan struct{}),
	}
}

// activateEthService configures and registers the eth.Ethereum service with a given node.
func activateEthService(stack *node.Node, extra []byte) error {
	ethConf := &eth.Config{
		Etherbase:               common.Address{},
		ChainConfig:             makeChainConfig(stack),
		FastSync:                false,
		LightMode:               true,
		LightServ:               60,
		LightPeers:              MaxLightPeers,
		MaxPeers:                MaxPeers,
		DatabaseCache:           DatabaseCacheSize,
		DatabaseHandles:         makeDatabaseHandles(),
		NetworkId:               1, // Olympic
		MinerThreads:            runtime.NumCPU(),
		GasPrice:                GasPrice,
		GpoMinGasPrice:          GpoMinGasPrice,
		GpoMaxGasPrice:          GpoMaxGasPrice,
		GpoFullBlockRatio:       GpoFullBlockRatio,
		GpobaseStepDown:         GpobaseStepDown,
		GpobaseStepUp:           GpobaseStepUp,
		GpobaseCorrectionFactor: GpobaseCorrectionFactor,
		SolcPath:                "solc",
		AutoDAG:                 false,
	}

	if UseTestnet {
		ethConf.NetworkId = 3
		ethConf.Genesis = core.DefaultTestnetGenesisBlock()
	}

	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
		return les.New(ctx, ethConf)
	}); err != nil {
		return fmt.Errorf("%v: %v", ErrLightEthRegistrationFailure, err)
	}

	return nil
}

// activateShhService configures Whisper and adds it to the given node.
func activateShhService(stack *node.Node) error {
	serviceConstructor := func(*node.ServiceContext) (node.Service, error) {
		return whisper.New(), nil
	}
	if err := stack.Register(serviceConstructor); err != nil {
		return err
	}

	return nil
}

// makeChainConfig reads the chain configuration from the database in the datadir.
func makeChainConfig(stack *node.Node) *params.ChainConfig {
	config := new(params.ChainConfig)

	if UseTestnet {
		config = params.TestnetChainConfig
	} else {
		// Homestead fork
		config.HomesteadBlock = params.MainNetHomesteadBlock
		// DAO fork
		config.DAOForkBlock = params.MainNetDAOForkBlock
		config.DAOForkSupport = true

		// DoS reprice fork
		config.EIP150Block = params.MainNetHomesteadGasRepriceBlock
		config.EIP150Hash = params.MainNetHomesteadGasRepriceHash

		// DoS state cleanup fork
		config.EIP155Block = params.MainNetSpuriousDragon
		config.EIP158Block = params.MainNetSpuriousDragon
		config.ChainId = params.MainNetChainID
	}

	return config
}

// makeDatabaseHandles makes sure that enough file descriptors are available to the process
// (and returns half of them for node's database to use)
func makeDatabaseHandles() int {
	// current limit
	var limit syscall.Rlimit
	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
		Fatalf(err)
	}

	// increase limit
	limit.Cur = limit.Max
	if limit.Cur > ProcessFileDescriptorLimit {
		limit.Cur = ProcessFileDescriptorLimit
	}
	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
		Fatalf(err)
	}

	// re-query limit
	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
		Fatalf(err)
	}

	// cap limit
	if limit.Cur > ProcessFileDescriptorLimit {
		limit.Cur = ProcessFileDescriptorLimit
	}

	return int(limit.Cur) / 2
}

func makeDefaultExtra() []byte {
	var clientInfo = struct {
		Version   uint
		Name      string
		GoVersion string
		Os        string
	}{uint(VersionMajor<<16 | VersionMinor<<8 | VersionPatch), ClientIdentifier, runtime.Version(), runtime.GOOS}
	extra, err := rlp.EncodeToBytes(clientInfo)
	if err != nil {
		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
	}

	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
		glog.V(logger.Debug).Infof("extra: %x\n", extra)
		return nil
	}

	return extra
}

func Fatalf(reason interface{}, args ...interface{}) {
	// decide on output stream
	w := io.MultiWriter(os.Stdout, os.Stderr)
	outf, _ := os.Stdout.Stat()
	errf, _ := os.Stderr.Stat()
	if outf != nil && errf != nil && os.SameFile(outf, errf) {
		w = os.Stderr
	}

	// find out whether error or string has been passed as a reason
	r := reflect.ValueOf(reason)
	if r.Kind() == reflect.String {
		fmt.Fprintf(w, "Fatal Failure: "+reason.(string)+"\n", args)
	} else {
		fmt.Fprintf(w, "Fatal Failure: %v\n", reason.(error))
	}

	os.Exit(1)
}
<commit_msg>version bump<commit_after>package geth

import (
	"errors"
	"fmt"
	"io"
	"math/big"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"syscall"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/eth"
	"github.com/ethereum/go-ethereum/les"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/node"
	"github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/rlp"
	whisper "github.com/ethereum/go-ethereum/whisper/whisperv2"
)

const (
	ClientIdentifier = "StatusIM" // Client identifier to advertise over the network
	VersionMajor     = 1          // Major version component of the current release
	VersionMinor     = 2          // Minor version component of the current release
	VersionPatch     = 0          // Patch version component of the current release
	VersionMeta      = "unstable" // Version metadata to append to the version string

	RPCPort         = 8545 // RPC port (replaced in unit tests)
	NetworkPort     = 30303
	MaxPeers        = 25
	MaxLightPeers   = 20
	MaxPendingPeers = 0

	ProcessFileDescriptorLimit = uint64(2048)
	DatabaseCacheSize          = 128 // Megabytes of memory allocated to internal caching (min 16MB / database forced)

	EventNodeStarted = "node.started"
)

// Gas price settings
var (
	GasPrice                = new(big.Int).Mul(big.NewInt(20), common.Shannon)  // Minimal gas price to accept for mining a transactions
	GpoMinGasPrice          = new(big.Int).Mul(big.NewInt(20), common.Shannon)  // Minimum suggested gas price
	GpoMaxGasPrice          = new(big.Int).Mul(big.NewInt(500), common.Shannon) // Maximum suggested gas price
	GpoFullBlockRatio       = 80                                                // Full block threshold for gas price calculation (%)
	GpobaseStepDown         = 10                                                // Suggested gas price base step down ratio (1/1000)
	GpobaseStepUp           = 100                                               // Suggested gas price base step up ratio (1/1000)
	GpobaseCorrectionFactor = 110                                               // Suggested gas price base correction factor (%)
)

// default node configuration options
var (
	UseTestnetFlag = "true" // to be overridden via -ldflags '-X geth.UseTestnetFlag'
	UseTestnet     = false
)

func init() {
	if UseTestnetFlag == "true" { // set at compile time, here we make sure to set corresponding boolean flag
		UseTestnet = true
	}
}

// node-related errors
var (
	ErrRLimitRaiseFailure            = errors.New("failed to register the whisper service")
	ErrDatabaseAccessFailure         = errors.New("could not open database")
	ErrChainConfigurationFailure     = errors.New("could not make chain configuration")
	ErrEthServiceRegistrationFailure = errors.New("failed to register the Ethereum service")
	ErrSshServiceRegistrationFailure = errors.New("failed to register the Whisper service")
	ErrLightEthRegistrationFailure   = errors.New("failed to register the LES service")
)

type Node struct {
	geth    *node.Node    // reference to the running Geth node
	started chan struct{} // channel to wait for node to start
}

// Inited checks whether status node has been properly initialized
func (n *Node) Inited() bool {
	return n != nil && n.geth != nil
}

// MakeNode create a geth node entity
func MakeNode(dataDir string, rpcPort int) *Node {
	glog.CopyStandardLogTo("INFO")
	glog.SetToStderr(true)

	bootstrapNodes := params.MainnetBootnodes
	if UseTestnet {
		dataDir = filepath.Join(dataDir, "testnet")
		bootstrapNodes = params.TestnetBootnodes
	}

	// configure required node (should you need to update node's config, e.g. add bootstrap nodes, see node.Config)
	config := &node.Config{
		DataDir:           dataDir,
		UseLightweightKDF: true,
		Name:              ClientIdentifier,
		Version:           fmt.Sprintf("%d.%d.%d-%s", VersionMajor, VersionMinor, VersionPatch, VersionMeta),
		NoDiscovery:       true,
		DiscoveryV5:       true,
		DiscoveryV5Addr:   fmt.Sprintf(":%d", NetworkPort+1),
		BootstrapNodes:    bootstrapNodes,
		BootstrapNodesV5:  params.DiscoveryV5Bootnodes,
		ListenAddr:        fmt.Sprintf(":%d", NetworkPort),
		MaxPeers:          MaxPeers,
		MaxPendingPeers:   MaxPendingPeers,
		HTTPHost:          node.DefaultHTTPHost,
		HTTPPort:          rpcPort,
		HTTPCors:          "*",
		HTTPModules:       strings.Split("db,eth,net,web3,shh,personal,admin", ","), // TODO remove "admin" on main net
	}

	stack, err := node.New(config)
	if err != nil {
		Fatalf(ErrNodeMakeFailure)
	}

	// start Ethereum service
	if err := activateEthService(stack, makeDefaultExtra()); err != nil {
		Fatalf(fmt.Errorf("%v: %v", ErrEthServiceRegistrationFailure, err))
	}

	// start Whisper service
	if err := activateShhService(stack); err != nil {
		Fatalf(fmt.Errorf("%v: %v", ErrSshServiceRegistrationFailure, err))
	}

	return &Node{
		geth:    stack,
		started: make(chan struct{}),
	}
}

// activateEthService configures and registers the eth.Ethereum service with a given node.
func activateEthService(stack *node.Node, extra []byte) error {
	ethConf := &eth.Config{
		Etherbase:               common.Address{},
		ChainConfig:             makeChainConfig(stack),
		FastSync:                false,
		LightMode:               true,
		LightServ:               60,
		LightPeers:              MaxLightPeers,
		MaxPeers:                MaxPeers,
		DatabaseCache:           DatabaseCacheSize,
		DatabaseHandles:         makeDatabaseHandles(),
		NetworkId:               1, // Olympic
		MinerThreads:            runtime.NumCPU(),
		GasPrice:                GasPrice,
		GpoMinGasPrice:          GpoMinGasPrice,
		GpoMaxGasPrice:          GpoMaxGasPrice,
		GpoFullBlockRatio:       GpoFullBlockRatio,
		GpobaseStepDown:         GpobaseStepDown,
		GpobaseStepUp:           GpobaseStepUp,
		GpobaseCorrectionFactor: GpobaseCorrectionFactor,
		SolcPath:                "solc",
		AutoDAG:                 false,
	}

	if UseTestnet {
		ethConf.NetworkId = 3
		ethConf.Genesis = core.DefaultTestnetGenesisBlock()
	}

	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
		return les.New(ctx, ethConf)
	}); err != nil {
		return fmt.Errorf("%v: %v", ErrLightEthRegistrationFailure, err)
	}

	return nil
}

// activateShhService configures Whisper and adds it to the given node.
func activateShhService(stack *node.Node) error {
	serviceConstructor := func(*node.ServiceContext) (node.Service, error) {
		return whisper.New(), nil
	}
	if err := stack.Register(serviceConstructor); err != nil {
		return err
	}

	return nil
}

// makeChainConfig reads the chain configuration from the database in the datadir.
func makeChainConfig(stack *node.Node) *params.ChainConfig {
	config := new(params.ChainConfig)

	if UseTestnet {
		config = params.TestnetChainConfig
	} else {
		// Homestead fork
		config.HomesteadBlock = params.MainNetHomesteadBlock
		// DAO fork
		config.DAOForkBlock = params.MainNetDAOForkBlock
		config.DAOForkSupport = true

		// DoS reprice fork
		config.EIP150Block = params.MainNetHomesteadGasRepriceBlock
		config.EIP150Hash = params.MainNetHomesteadGasRepriceHash

		// DoS state cleanup fork
		config.EIP155Block = params.MainNetSpuriousDragon
		config.EIP158Block = params.MainNetSpuriousDragon
		config.ChainId = params.MainNetChainID
	}

	return config
}

// makeDatabaseHandles makes sure that enough file descriptors are available to the process
// (and returns half of them for node's database to use)
func makeDatabaseHandles() int {
	// current limit
	var limit syscall.Rlimit
	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
		Fatalf(err)
	}

	// increase limit
	limit.Cur = limit.Max
	if limit.Cur > ProcessFileDescriptorLimit {
		limit.Cur = ProcessFileDescriptorLimit
	}
	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
		Fatalf(err)
	}

	// re-query limit
	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
		Fatalf(err)
	}

	// cap limit
	if limit.Cur > ProcessFileDescriptorLimit {
		limit.Cur = ProcessFileDescriptorLimit
	}

	return int(limit.Cur) / 2
}

func makeDefaultExtra() []byte {
	var clientInfo = struct {
		Version   uint
		Name      string
		GoVersion string
		Os        string
	}{uint(VersionMajor<<16 | VersionMinor<<8 | VersionPatch), ClientIdentifier, runtime.Version(), runtime.GOOS}
	extra, err := rlp.EncodeToBytes(clientInfo)
	if err != nil {
		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
	}

	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
		glog.V(logger.Debug).Infof("extra: %x\n", extra)
		return nil
	}

	return extra
}

func Fatalf(reason interface{}, args ...interface{}) {
	// decide on output stream
	w := io.MultiWriter(os.Stdout, os.Stderr)
	outf, _ := os.Stdout.Stat()
	errf, _ := os.Stderr.Stat()
	if outf != nil && errf != nil && os.SameFile(outf, errf) {
		w = os.Stderr
	}

	// find out whether error or string has been passed as a reason
	r := reflect.ValueOf(reason)
	if r.Kind() == reflect.String {
		fmt.Fprintf(w, "Fatal Failure: "+reason.(string)+"\n", args)
	} else {
		fmt.Fprintf(w, "Fatal Failure: %v\n", reason.(error))
	}

	os.Exit(1)
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"strconv"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccASGNotification_basic(t *testing.T) {
	var asgn autoscaling.DescribeNotificationConfigurationsOutput

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckASGNDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccASGNotificationConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckASGNotificationExists("aws_autoscaling_notification.example", []string{"foobar1-terraform-test"}, &asgn),
					testAccCheckAWSASGNotificationAttributes("aws_autoscaling_notification.example", &asgn),
				),
			},
		},
	})
}

func TestAccASGNotification_update(t *testing.T) {
	var asgn autoscaling.DescribeNotificationConfigurationsOutput

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckASGNDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccASGNotificationConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckASGNotificationExists("aws_autoscaling_notification.example", []string{"foobar1-terraform-test"}, &asgn),
					testAccCheckAWSASGNotificationAttributes("aws_autoscaling_notification.example", &asgn),
				),
			},

			resource.TestStep{
				Config: testAccASGNotificationConfig_update,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckASGNotificationExists("aws_autoscaling_notification.example", []string{"foobar1-terraform-test", "barfoo-terraform-test"}, &asgn),
					testAccCheckAWSASGNotificationAttributes("aws_autoscaling_notification.example", &asgn),
				),
			},
		},
	})
}

func testAccCheckASGNotificationExists(n string, groups []string, asgn *autoscaling.DescribeNotificationConfigurationsOutput) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ASG Notification ID is set")
		}

		var gl []*string
		for _, g := range groups {
			gl = append(gl, aws.String(g))
		}

		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn
		opts := &autoscaling.DescribeNotificationConfigurationsInput{
			AutoScalingGroupNames: gl,
		}

		resp, err := conn.DescribeNotificationConfigurations(opts)
		if err != nil {
			return fmt.Errorf("Error describing notifications")
		}

		*asgn = *resp

		return nil
	}
}

func testAccCheckASGNDestroy(s *terraform.State) error {
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_autoscaling_notification" {
			continue
		}

		groups := []*string{aws.String("foobar1-terraform-test")}
		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn
		opts := &autoscaling.DescribeNotificationConfigurationsInput{
			AutoScalingGroupNames: groups,
		}

		resp, err := conn.DescribeNotificationConfigurations(opts)
		if err != nil {
			return fmt.Errorf("Error describing notifications")
		}

		if len(resp.NotificationConfigurations) != 0 {
			fmt.Errorf("Error finding notification descriptions")
		}

	}
	return nil
}

func testAccCheckAWSASGNotificationAttributes(n string, asgn *autoscaling.DescribeNotificationConfigurationsOutput) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ASG Notification ID is set")
		}

		if len(asgn.NotificationConfigurations) == 0 {
			return fmt.Errorf("Error: no ASG Notifications found")
		}

		// build a unique list of groups, notification types
		gRaw := make(map[string]bool)
		nRaw := make(map[string]bool)
		for _, n := range asgn.NotificationConfigurations {
			if *n.TopicARN == rs.Primary.Attributes["topic_arn"] {
				gRaw[*n.AutoScalingGroupName] = true
				nRaw[*n.NotificationType] = true
			}
		}

		// Grab the keys here as the list of Groups
		var gList []string
		for k, _ := range gRaw {
			gList = append(gList, k)
		}

		// Grab the keys here as the list of Types
		var nList []string
		for k, _ := range nRaw {
			nList = append(nList, k)
		}

		typeCount, _ := strconv.Atoi(rs.Primary.Attributes["notifications.#"])

		if len(nList) != typeCount {
			return fmt.Errorf("Error: Bad ASG Notification count, expected (%d), got (%d)", typeCount, len(nList))
		}

		groupCount, _ := strconv.Atoi(rs.Primary.Attributes["group_names.#"])

		if len(gList) != groupCount {
			return fmt.Errorf("Error: Bad ASG Group count, expected (%d), got (%d)", typeCount, len(gList))
		}

		return nil
	}
}

const testAccASGNotificationConfig_basic = `
resource "aws_sns_topic" "topic_example" {
  name = "user-updates-topic"
}

resource "aws_launch_configuration" "foobar" {
  name = "foobarautoscaling-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
}

resource "aws_autoscaling_group" "bar" {
  availability_zones = ["us-west-2a"]
  name = "foobar1-terraform-test"
  max_size = 1
  min_size = 1
  health_check_grace_period = 100
  health_check_type = "ELB"
  desired_capacity = 1
  force_delete = true
  termination_policies = ["OldestInstance"]
  launch_configuration = "${aws_launch_configuration.foobar.name}"
}

resource "aws_autoscaling_notification" "example" {
  group_names     = ["${aws_autoscaling_group.bar.name}"]
  notifications  = [
	"autoscaling:EC2_INSTANCE_LAUNCH",
	"autoscaling:EC2_INSTANCE_TERMINATE",
  ]
  topic_arn = "${aws_sns_topic.topic_example.arn}"
}
`

const testAccASGNotificationConfig_update = `
resource "aws_sns_topic" "user_updates" {
  name = "user-updates-topic"
}

resource "aws_launch_configuration" "foobar" {
  name = "foobarautoscaling-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
}

resource "aws_autoscaling_group" "bar" {
  availability_zones = ["us-west-2a"]
  name = "foobar1-terraform-test"
  max_size = 1
  min_size = 1
  health_check_grace_period = 100
  health_check_type = "ELB"
  desired_capacity = 1
  force_delete = true
  termination_policies = ["OldestInstance"]
  launch_configuration = "${aws_launch_configuration.foobar.name}"
}

resource "aws_autoscaling_group" "foo" {
  availability_zones = ["us-west-2b"]
  name = "barfoo-terraform-test"
  max_size = 1
  min_size = 1
  health_check_grace_period = 200
  health_check_type = "ELB"
  desired_capacity = 1
  force_delete = true
  termination_policies = ["OldestInstance"]
  launch_configuration = "${aws_launch_configuration.foobar.name}"
}

resource "aws_autoscaling_notification" "example" {
	group_names     = [
	"${aws_autoscaling_group.bar.name}",
	"${aws_autoscaling_group.foo.name}",
	]
	notifications  = [
		"autoscaling:EC2_INSTANCE_LAUNCH",
		"autoscaling:EC2_INSTANCE_TERMINATE",
		"autoscaling:EC2_INSTANCE_LAUNCH_ERROR"
	]
	topic_arn = "${aws_sns_topic.user_updates.arn}"
}`
<commit_msg>tests: test name consistency<commit_after>package aws

import (
	"fmt"
	"strconv"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSASGNotification_basic(t *testing.T) {
	var asgn autoscaling.DescribeNotificationConfigurationsOutput

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckASGNDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccASGNotificationConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckASGNotificationExists("aws_autoscaling_notification.example", []string{"foobar1-terraform-test"}, &asgn),
					testAccCheckAWSASGNotificationAttributes("aws_autoscaling_notification.example", &asgn),
				),
			},
		},
	})
}

func TestAccAWSASGNotification_update(t *testing.T) {
	var asgn autoscaling.DescribeNotificationConfigurationsOutput

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckASGNDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccASGNotificationConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckASGNotificationExists("aws_autoscaling_notification.example", []string{"foobar1-terraform-test"}, &asgn),
					testAccCheckAWSASGNotificationAttributes("aws_autoscaling_notification.example", &asgn),
				),
			},

			resource.TestStep{
				Config: testAccASGNotificationConfig_update,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckASGNotificationExists("aws_autoscaling_notification.example", []string{"foobar1-terraform-test", "barfoo-terraform-test"}, &asgn),
					testAccCheckAWSASGNotificationAttributes("aws_autoscaling_notification.example", &asgn),
				),
			},
		},
	})
}

func testAccCheckASGNotificationExists(n string, groups []string, asgn *autoscaling.DescribeNotificationConfigurationsOutput) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ASG Notification ID is set")
		}

		var gl []*string
		for _, g := range groups {
			gl = append(gl, aws.String(g))
		}

		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn
		opts := &autoscaling.DescribeNotificationConfigurationsInput{
			AutoScalingGroupNames: gl,
		}

		resp, err := conn.DescribeNotificationConfigurations(opts)
		if err != nil {
			return fmt.Errorf("Error describing notifications")
		}

		*asgn = *resp

		return nil
	}
}

func testAccCheckASGNDestroy(s *terraform.State) error {
	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_autoscaling_notification" {
			continue
		}

		groups := []*string{aws.String("foobar1-terraform-test")}
		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn
		opts := &autoscaling.DescribeNotificationConfigurationsInput{
			AutoScalingGroupNames: groups,
		}

		resp, err := conn.DescribeNotificationConfigurations(opts)
		if err != nil {
			return fmt.Errorf("Error describing notifications")
		}

		if len(resp.NotificationConfigurations) != 0 {
			fmt.Errorf("Error finding notification descriptions")
		}

	}
	return nil
}

func testAccCheckAWSASGNotificationAttributes(n string, asgn *autoscaling.DescribeNotificationConfigurationsOutput) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ASG Notification ID is set")
		}

		if len(asgn.NotificationConfigurations) == 0 {
			return fmt.Errorf("Error: no ASG Notifications found")
		}

		// build a unique list of groups, notification types
		gRaw := make(map[string]bool)
		nRaw := make(map[string]bool)
		for _, n := range asgn.NotificationConfigurations {
			if *n.TopicARN == rs.Primary.Attributes["topic_arn"] {
				gRaw[*n.AutoScalingGroupName] = true
				nRaw[*n.NotificationType] = true
			}
		}

		// Grab the keys here as the list of Groups
		var gList []string
		for k, _ := range gRaw {
			gList = append(gList, k)
		}

		// Grab the keys here as the list of Types
		var nList []string
		for k, _ := range nRaw {
			nList = append(nList, k)
		}

		typeCount, _ := strconv.Atoi(rs.Primary.Attributes["notifications.#"])

		if len(nList) != typeCount {
			return fmt.Errorf("Error: Bad ASG Notification count, expected (%d), got (%d)", typeCount, len(nList))
		}

		groupCount, _ := strconv.Atoi(rs.Primary.Attributes["group_names.#"])

		if len(gList) != groupCount {
			return fmt.Errorf("Error: Bad ASG Group count, expected (%d), got (%d)", typeCount, len(gList))
		}

		return nil
	}
}

const testAccASGNotificationConfig_basic = `
resource "aws_sns_topic" "topic_example" {
  name = "user-updates-topic"
}

resource "aws_launch_configuration" "foobar" {
  name = "foobarautoscaling-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
}

resource "aws_autoscaling_group" "bar" {
  availability_zones = ["us-west-2a"]
  name = "foobar1-terraform-test"
  max_size = 1
  min_size = 1
  health_check_grace_period = 100
  health_check_type = "ELB"
  desired_capacity = 1
  force_delete = true
  termination_policies = ["OldestInstance"]
  launch_configuration = "${aws_launch_configuration.foobar.name}"
}

resource "aws_autoscaling_notification" "example" {
  group_names     = ["${aws_autoscaling_group.bar.name}"]
  notifications  = [
	"autoscaling:EC2_INSTANCE_LAUNCH",
	"autoscaling:EC2_INSTANCE_TERMINATE",
  ]
  topic_arn = "${aws_sns_topic.topic_example.arn}"
}
`

const testAccASGNotificationConfig_update = `
resource "aws_sns_topic" "user_updates" {
  name = "user-updates-topic"
}

resource "aws_launch_configuration" "foobar" {
  name = "foobarautoscaling-terraform-test"
  image_id = "ami-21f78e11"
  instance_type = "t1.micro"
}

resource "aws_autoscaling_group" "bar" {
  availability_zones = ["us-west-2a"]
  name = "foobar1-terraform-test"
  max_size = 1
  min_size = 1
  health_check_grace_period = 100
  health_check_type = "ELB"
  desired_capacity = 1
  force_delete = true
  termination_policies = ["OldestInstance"]
  launch_configuration = "${aws_launch_configuration.foobar.name}"
}

resource "aws_autoscaling_group" "foo" {
  availability_zones = ["us-west-2b"]
  name = "barfoo-terraform-test"
  max_size = 1
  min_size = 1
  health_check_grace_period = 200
  health_check_type = "ELB"
  desired_capacity = 1
  force_delete = true
  termination_policies = ["OldestInstance"]
  launch_configuration = "${aws_launch_configuration.foobar.name}"
}

resource "aws_autoscaling_notification" "example" {
	group_names     = [
	"${aws_autoscaling_group.bar.name}",
	"${aws_autoscaling_group.foo.name}",
	]
	notifications  = [
		"autoscaling:EC2_INSTANCE_LAUNCH",
		"autoscaling:EC2_INSTANCE_TERMINATE",
		"autoscaling:EC2_INSTANCE_LAUNCH_ERROR"
	]
	topic_arn = "${aws_sns_topic.user_updates.arn}"
}`
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package unix_test

import (
	"testing"
	"time"

	"golang.org/x/sys/unix"
)

func TestPpoll(t *testing.T) {
	f, cleanup := mktmpfifo(t)
	defer cleanup()

	const timeout = 100 * time.Millisecond

	ok := make(chan bool, 1)
	go func() {
		select {
		case <-time.After(10 * timeout):
			t.Errorf("Ppoll: failed to timeout after %d", 10*timeout)
		case <-ok:
		}
	}()

	fds := []unix.PollFd{{Fd: int32(f.Fd()), Events: unix.POLLIN}}
	timeoutTs := unix.NsecToTimespec(int64(timeout))
	n, err := unix.Ppoll(fds, &timeoutTs, nil)
	ok <- true
	if err != nil {
		t.Errorf("Ppoll: unexpected error: %v", err)
		return
	}
	if n != 0 {
		t.Errorf("Ppoll: wrong number of events: got %v, expected %v", n, 0)
		return
	}
}

func TestSysctlUvmexp(t *testing.T) {
	uvm, err := unix.SysctlUvmexp("vm.uvmexp")
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("free = %v", uvm.Free)
}
<commit_msg>unix: add missing chtmpdir in TestPpoll on openbsd<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package unix_test

import (
	"testing"
	"time"

	"golang.org/x/sys/unix"
)

func TestPpoll(t *testing.T) {
	defer chtmpdir(t)()
	f, cleanup := mktmpfifo(t)
	defer cleanup()

	const timeout = 100 * time.Millisecond

	ok := make(chan bool, 1)
	go func() {
		select {
		case <-time.After(10 * timeout):
			t.Errorf("Ppoll: failed to timeout after %d", 10*timeout)
		case <-ok:
		}
	}()

	fds := []unix.PollFd{{Fd: int32(f.Fd()), Events: unix.POLLIN}}
	timeoutTs := unix.NsecToTimespec(int64(timeout))
	n, err := unix.Ppoll(fds, &timeoutTs, nil)
	ok <- true
	if err != nil {
		t.Errorf("Ppoll: unexpected error: %v", err)
		return
	}
	if n != 0 {
		t.Errorf("Ppoll: wrong number of events: got %v, expected %v", n, 0)
		return
	}
}

func TestSysctlUvmexp(t *testing.T) {
	uvm, err := unix.SysctlUvmexp("vm.uvmexp")
	if err != nil {
		t.Fatal(err)
	}
	t.Logf("free = %v", uvm.Free)
}
<|endoftext|>


<commit_before>package etcd

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/compose/transporter/pkg/adaptor"
	"github.com/compose/transporter/pkg/message"
	"github.com/compose/transporter/pkg/message/adaptor/etcd"
	"github.com/compose/transporter/pkg/pipe"
	"github.com/coreos/etcd/client"
)

// Etcd is an adaptor to read / write to Etcd.
// it works as a source by copying files, and then optionally watching all keys
type Etcd struct {
	// pull these in from the node
	uri  string
	tail bool // run the tail oplog

	// save time by setting these once
	tableMatch *regexp.Regexp
	database   string

	//
	pipe *pipe.Pipe
	path string

	// etcd connection and options
	session        client.Client
	sessionTimeout time.Duration
}

// Config provides configuration options for a postgres adaptor
// the notable difference between this and dbConfig is the presence of the Tail option
type Config struct {
	URI       string `json:"uri" doc:"the uri to connect to, in the form TODO"`
	Namespace string `json:"namespace" doc:"mongo namespace to read/write"`
	Timeout   string `json:"timeout" doc:"timeout for establishing connection, format must be parsable by time.ParseDuration and defaults to 10s"`
	Tail      bool   `json:"tail" doc:"if tail is true, then the etcd source will track changes after copying the namespace"`
}

func init() {
	adaptor.Add("etcd", adaptor.Creator(func(ppipe *pipe.Pipe, path string, extra adaptor.Config) (adaptor.Adaptor, error) {
		var (
			conf Config
			err  error
		)
		if err = extra.Construct(&conf); err != nil {
			return nil, err
		}

		if conf.URI == "" || conf.Namespace == "" {
			return nil, fmt.Errorf("both endpoints and namespace required, but missing ")
		}
		e := &Etcd{
			sessionTimeout: time.Second * 10,
			pipe:           ppipe,
			uri:            conf.URI,
			tail:           conf.Tail,
			path:           path,
		}
		e.database, e.tableMatch, err = extra.CompileNamespace()
		if err != nil {
			return e, err
		}
		if conf.Timeout != "" {
			t, err := time.ParseDuration(conf.Timeout)
			if err != nil {
				log.Printf("error parsing timeout, defaulting to 10s, %v", err)
			} else {
				e.sessionTimeout = t
			}
		}
		return e, nil
	}))
}

// Description for etcd adaptor
func (e *Etcd) Description() string {
	return "an etcd adaptor that functions as both a source and a sink"
}

const sampleConfig = `
- localetcd:
    type: postgres
    uri: 127.0.0.1:5432,127.0.0.1:2345
`

// SampleConfig for postgres adaptor
func (e *Etcd) SampleConfig() string {
	return sampleConfig
}

func (e *Etcd) Connect() error {
	cfg := client.Config{
		Endpoints: strings.Split(e.uri, ","),
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second * 5,
	}
	var err error
	e.session, err = client.New(cfg)
	if err != nil {
		return fmt.Errorf("unable to make etcd session (%v), %v\n", e.uri, err)
	}
	return nil
}

// Start the adaptor as a source
func (e *Etcd) Start() (err error) {
	defer func() {
		e.pipe.Stop()
	}()

	err = e.catData()
	if err != nil {
		e.pipe.Err <- err
		return fmt.Errorf("Error connecting to Postgres: %v", err)
	}
	if e.tail {
		err = e.tailData()
		if err != nil {
			e.pipe.Err <- err
			return err
		}
	}
	return
}

// Listen starts the pipe's listener
func (e *Etcd) Listen() (err error) {
	defer func() {
		e.pipe.Stop()
	}()

	return e.pipe.Listen(e.writeMessage, e.tableMatch)
}

// Stop the adaptor
func (e *Etcd) Stop() error {
	e.pipe.Stop()
	return nil
}

// writeMessage writes one message to the destination Postgres, or sends an error down the pipe
// TODO this can be cleaned up.  I'm not sure whether this should pipe the error, or whether the
//   caller should pipe the error
func (e *Etcd) writeMessage(msg message.Msg) (message.Msg, error) {
	m, err := message.Exec(message.MustUseAdaptor("etcd").(etcd.Adaptor).UseClient(e.session), msg)
	if err != nil {
		e.pipe.Err <- adaptor.NewError(adaptor.ERROR, e.path, fmt.Sprintf("etcd error (%v)", err), msg.Data())
	}

	return m, err
}

// catdata pulls down the original tables
func (e *Etcd) catData() error {
	return nil
}

// tail the logical data
func (e *Etcd) tailData() error {
	return nil
}
<commit_msg>fix sample config for etcd<commit_after>package etcd

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/compose/transporter/pkg/adaptor"
	"github.com/compose/transporter/pkg/message"
	"github.com/compose/transporter/pkg/message/adaptor/etcd"
	"github.com/compose/transporter/pkg/pipe"
	"github.com/coreos/etcd/client"
)

// Etcd is an adaptor to read / write to Etcd.
// it works as a source by copying files, and then optionally watching all keys
type Etcd struct {
	// pull these in from the node
	uri  string
	tail bool // run the tail oplog

	// save time by setting these once
	tableMatch *regexp.Regexp
	database   string

	//
	pipe *pipe.Pipe
	path string

	// etcd connection and options
	session        client.Client
	sessionTimeout time.Duration
}

// Config provides configuration options for a postgres adaptor
// the notable difference between this and dbConfig is the presence of the Tail option
type Config struct {
	URI       string `json:"uri" doc:"the uri to connect to, in the form TODO"`
	Namespace string `json:"namespace" doc:"mongo namespace to read/write"`
	Timeout   string `json:"timeout" doc:"timeout for establishing connection, format must be parsable by time.ParseDuration and defaults to 10s"`
	Tail      bool   `json:"tail" doc:"if tail is true, then the etcd source will track changes after copying the namespace"`
}

func init() {
	adaptor.Add("etcd", adaptor.Creator(func(ppipe *pipe.Pipe, path string, extra adaptor.Config) (adaptor.Adaptor, error) {
		var (
			conf Config
			err  error
		)
		if err = extra.Construct(&conf); err != nil {
			return nil, err
		}

		if conf.URI == "" || conf.Namespace == "" {
			return nil, fmt.Errorf("both endpoints and namespace required, but missing ")
		}
		e := &Etcd{
			sessionTimeout: time.Second * 10,
			pipe:           ppipe,
			uri:            conf.URI,
			tail:           conf.Tail,
			path:           path,
		}
		e.database, e.tableMatch, err = extra.CompileNamespace()
		if err != nil {
			return e, err
		}
		if conf.Timeout != "" {
			t, err := time.ParseDuration(conf.Timeout)
			if err != nil {
				log.Printf("error parsing timeout, defaulting to 10s, %v", err)
			} else {
				e.sessionTimeout = t
			}
		}
		return e, nil
	}))
}

// Description for etcd adaptor
func (e *Etcd) Description() string {
	return "an etcd adaptor that functions as both a source and a sink"
}

const sampleConfig = `
- localetcd:
    type: etcd
    uri: 127.0.0.1:5432,127.0.0.1:2345
`

// SampleConfig for postgres adaptor
func (e *Etcd) SampleConfig() string {
	return sampleConfig
}

func (e *Etcd) Connect() error {
	cfg := client.Config{
		Endpoints: strings.Split(e.uri, ","),
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second * 5,
	}
	var err error
	e.session, err = client.New(cfg)
	if err != nil {
		return fmt.Errorf("unable to make etcd session (%v), %v\n", e.uri, err)
	}
	return nil
}

// Start the adaptor as a source
func (e *Etcd) Start() (err error) {
	defer func() {
		e.pipe.Stop()
	}()

	err = e.catData()
	if err != nil {
		e.pipe.Err <- err
		return fmt.Errorf("Error connecting to Postgres: %v", err)
	}
	if e.tail {
		err = e.tailData()
		if err != nil {
			e.pipe.Err <- err
			return err
		}
	}
	return
}

// Listen starts the pipe's listener
func (e *Etcd) Listen() (err error) {
	defer func() {
		e.pipe.Stop()
	}()

	return e.pipe.Listen(e.writeMessage, e.tableMatch)
}

// Stop the adaptor
func (e *Etcd) Stop() error {
	e.pipe.Stop()
	return nil
}

// writeMessage writes one message to the destination Postgres, or sends an error down the pipe
// TODO this can be cleaned up.  I'm not sure whether this should pipe the error, or whether the
//   caller should pipe the error
func (e *Etcd) writeMessage(msg message.Msg) (message.Msg, error) {
	m, err := message.Exec(message.MustUseAdaptor("etcd").(etcd.Adaptor).UseClient(e.session), msg)
	if err != nil {
		e.pipe.Err <- adaptor.NewError(adaptor.ERROR, e.path, fmt.Sprintf("etcd error (%v)", err), msg.Data())
	}

	return m, err
}

// catdata pulls down the original tables
func (e *Etcd) catData() error {
	return nil
}

// tail the logical data
func (e *Etcd) tailData() error {
	return nil
}
<|endoftext|>