<commit_before>package wavefront

import (
	"bufio"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/datapoint/dpsink"
	"github.com/signalfx/golib/errors"
	"github.com/signalfx/golib/log"
	"github.com/signalfx/golib/pointer"
	"github.com/signalfx/golib/sfxclient"
	"github.com/signalfx/metricproxy/logkey"
	"github.com/signalfx/metricproxy/protocol"
	"github.com/signalfx/metricproxy/protocol/collectd"
	"golang.org/x/net/context"
	"io"
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// Listener once setup will listen for wavefront protocol points to forward on
type Listener struct {
	protocol.CloseableHealthCheck
	psocket                   net.Listener
	sink                      dpsink.Sink
	serverAcceptDeadline      time.Duration
	connectionTimeout         time.Duration
	listenfunc                func()
	logger                    log.Logger
	stats                     listenerStats
	wg                        sync.WaitGroup
	extractCollectdDimensions bool
}

var _ protocol.Listener = &Listener{}

type listenerStats struct {
	totalDatapoints     int64
	idleTimeouts        int64
	retriedListenErrors int64
	totalEOFCloses      int64
	invalidDatapoints   int64
	totalConnections    int64
	activeConnections   int64
}

// Datapoints reports information about the total points seen by wavefront
func (listener *Listener) Datapoints() []*datapoint.Datapoint {
	return []*datapoint.Datapoint{
		sfxclient.Cumulative("invalid_datapoints", nil, atomic.LoadInt64(&listener.stats.invalidDatapoints)),
		sfxclient.Cumulative("total_connections", nil, atomic.LoadInt64(&listener.stats.totalConnections)),
		sfxclient.Gauge("active_connections", nil, atomic.LoadInt64(&listener.stats.activeConnections)),
		sfxclient.Cumulative("idle_timeouts", nil, atomic.LoadInt64(&listener.stats.idleTimeouts)),
		sfxclient.Cumulative("retry_listen_errors", nil, atomic.LoadInt64(&listener.stats.retriedListenErrors)),
	}
}

// Close the exposed wavefront port
func (listener *Listener) Close() error {
	var err error
	if listener.psocket != nil {
		err = listener.psocket.Close()
	}

	listener.wg.Wait()
	return err
}

type wavefrontListenConn interface {
	io.Reader
	Close() error
	SetDeadline(t time.Time) error
	RemoteAddr() net.Addr
}

// TODO need to run this and prometheus through some benchmarks for allocations
func extractCollectdDimensions(doit bool, metricName string) (string, map[string]string) {
	dimensions := make(map[string]string)
	if !doit {
		return metricName, dimensions
	}
	index := strings.Index(metricName, "..")
	var toAddDims map[string]string
	for {
		metricName, toAddDims = collectd.GetDimensionsFromName(&metricName)
		if len(toAddDims) == 0 {
			// we may have put two dots next to each other extracting this stuff, see if there were two beforehand and if not, replace them if they occur with a single dot
			// TODO could do better here if wavefront users prefixed all metrics with collectd, we could know how they're constructed and could make them be almost as good as coming from collectd
			// could even be crazy and put the types.db file in here and provide accurate metric types... overkill?
			if index == -1 {
				metricName = strings.Replace(metricName, "..", ".", -1)
			}

			return metricName, dimensions
		}
		for k, v := range toAddDims {
			dimensions[k] = v
		}
	}
}

// https://docs.wavefront.com/wavefront_data_format.html
func (listener *Listener) fromWavefrontDatapoint(line string) *datapoint.Datapoint {
	// TODO rewrite without Split using Index and slices as they don't allocate
	pieces := strings.Split(line, " ")
	if len(pieces) >= 3 {
		metricName, dimensions := extractCollectdDimensions(listener.extractCollectdDimensions, pieces[0])

		valueString := pieces[1]

		var value datapoint.Value
		if i, err := strconv.ParseInt(valueString, 10, 64); err == nil {
			value = datapoint.NewIntValue(i)
		} else if f, err := strconv.ParseFloat(valueString, 64); err == nil {
			value = datapoint.NewFloatValue(f)
		} else {
			return nil
		}
		var timestamp time.Time
		epoch, err := strconv.ParseInt(pieces[2], 10, 64)
		if err != nil {
			// probably not the timestamp, probably a dimension
			pieces = pieces[2:]
			timestamp = time.Now()
		} else {
			timestamp = time.Unix(epoch, 0)
			pieces = pieces[3:]
		}
		for _, p := range pieces {
			dimPieces := strings.SplitN(p, "=", 2)
			if len(dimPieces) == 2 {
				dimensions[dimPieces[0]] = dimPieces[1]
			}
			// ignore malformed dimensions
		}
		return datapoint.New(metricName, dimensions, value, datapoint.Gauge, timestamp)
	}
	return nil
}

var errInvalidDatapoint = errors.New("invalid wavefront datapoint")

func (listener *Listener) handleTCPConnection(ctx context.Context, conn wavefrontListenConn) error {
	connLogger := log.NewContext(listener.logger).With(logkey.RemoteAddr, conn.RemoteAddr())
	defer func() {
		log.IfErr(connLogger, conn.Close())
	}()
	reader := bufio.NewReader(conn)
	atomic.AddInt64(&listener.stats.totalConnections, 1)
	atomic.AddInt64(&listener.stats.activeConnections, 1)
	defer atomic.AddInt64(&listener.stats.activeConnections, -1)
	for {
		log.IfErr(connLogger, conn.SetDeadline(time.Now().Add(listener.connectionTimeout)))
		bytes, err := reader.ReadBytes((byte)('\n'))
		if err != nil && err != io.EOF {
			atomic.AddInt64(&listener.stats.idleTimeouts, 1)
			connLogger.Log(log.Err, err, "Listening for wavefront data returned an error (Note: We timeout idle connections)")
			return err
		}
		line := strings.TrimSpace(string(bytes))
		if line != "" {
			dp := listener.fromWavefrontDatapoint(line)
			if dp == nil {
				atomic.AddInt64(&listener.stats.invalidDatapoints, 1)
				connLogger.Log(logkey.WavefrontLine, line, log.Err, err, "Received data on a wavefront port, but it doesn't look like wavefront data")
				return errInvalidDatapoint
			}
			log.IfErr(connLogger, listener.sink.AddDatapoints(ctx, []*datapoint.Datapoint{dp}))
			atomic.AddInt64(&listener.stats.totalDatapoints, 1)
		}

		if err == io.EOF {
			atomic.AddInt64(&listener.stats.totalEOFCloses, 1)
			return nil
		}
	}
}

func (listener *Listener) startListeningTCP() {
	defer listener.wg.Done()
	defer listener.logger.Log("Stop listening wavefront TCP")
	for {
		deadlineable, ok := listener.psocket.(*net.TCPListener)
		if ok {
			log.IfErr(listener.logger, deadlineable.SetDeadline(time.Now().Add(listener.serverAcceptDeadline)))
		}
		conn, err := listener.psocket.Accept()
		if err != nil {
			if netErr, ok := err.(net.Error); ok {
				if netErr.Timeout() || netErr.Temporary() {
					atomic.AddInt64(&listener.stats.retriedListenErrors, 1)
					continue
				}
			}
			listener.logger.Log(log.Err, err, "Unable to accept a socket connection")
			return
		}
		go func() {
			log.IfErr(listener.logger, listener.handleTCPConnection(context.Background(), conn))
		}()
	}
}

// ListenerConfig controls optional parameters for wavefront listeners
type ListenerConfig struct {
	ServerAcceptDeadline      *time.Duration
	ConnectionTimeout         *time.Duration
	ListenAddr                *string
	Logger                    log.Logger
	ExtractCollectdDimensions *bool
}

var defaultListenerConfig = &ListenerConfig{
	ServerAcceptDeadline:      pointer.Duration(time.Second),
	ConnectionTimeout:         pointer.Duration(time.Second * 30),
	ListenAddr:                pointer.String("127.0.0.1:2003"),
	ExtractCollectdDimensions: pointer.Bool(true),
}

// Addr returns the listening address of this wavefront listener
func (listener *Listener) Addr() net.Addr {
	return listener.psocket.Addr()
}

func (listener *Listener) getServer(conf *ListenerConfig) error {
	server, err := net.Listen("tcp", *conf.ListenAddr)
	if err != nil {
		return errors.Annotatef(err, "cannot listen to addr %s", *conf.ListenAddr)
	}
	listener.psocket = server
	listener.listenfunc = listener.startListeningTCP
	return nil
}

// NewListener creates a new listener for wavefront datapoints
func NewListener(sendTo dpsink.Sink, passedConf *ListenerConfig) (*Listener, error) {
	conf := pointer.FillDefaultFrom(passedConf, defaultListenerConfig).(*ListenerConfig)
	receiver := Listener{
		sink:                 sendTo,
		serverAcceptDeadline: *conf.ServerAcceptDeadline,
		connectionTimeout:    *conf.ConnectionTimeout,
		logger:               log.NewContext(conf.Logger).With(logkey.Protocol, "wavefront", logkey.Direction, "listener"),
		extractCollectdDimensions: *conf.ExtractCollectdDimensions,
	}
	err := receiver.getServer(conf)
	if err != nil {
		return nil, err
	}
	receiver.wg.Add(1)
	go receiver.listenfunc()
	return &receiver, nil
}
<commit_msg>changes max requested<commit_after>package wavefront

import (
	"bufio"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/datapoint/dpsink"
	"github.com/signalfx/golib/errors"
	"github.com/signalfx/golib/log"
	"github.com/signalfx/golib/pointer"
	"github.com/signalfx/golib/sfxclient"
	"github.com/signalfx/metricproxy/logkey"
	"github.com/signalfx/metricproxy/protocol"
	"github.com/signalfx/metricproxy/protocol/collectd"
	"golang.org/x/net/context"
	"io"
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// Listener once setup will listen for wavefront protocol points to forward on
type Listener struct {
	protocol.CloseableHealthCheck
	psocket                   net.Listener
	sink                      dpsink.Sink
	serverAcceptDeadline      time.Duration
	connectionTimeout         time.Duration
	listenfunc                func()
	logger                    log.Logger
	stats                     listenerStats
	wg                        sync.WaitGroup
	extractCollectdDimensions bool
}

var _ protocol.Listener = &Listener{}

type listenerStats struct {
	totalDatapoints     int64
	idleTimeouts        int64
	retriedListenErrors int64
	totalEOFCloses      int64
	invalidDatapoints   int64
	totalConnections    int64
	activeConnections   int64
}

// Datapoints reports information about the total points seen by wavefront
func (listener *Listener) Datapoints() []*datapoint.Datapoint {
	return []*datapoint.Datapoint{
		sfxclient.Cumulative("invalid_datapoints", nil, atomic.LoadInt64(&listener.stats.invalidDatapoints)),
		sfxclient.Cumulative("total_connections", nil, atomic.LoadInt64(&listener.stats.totalConnections)),
		sfxclient.Gauge("active_connections", nil, atomic.LoadInt64(&listener.stats.activeConnections)),
		sfxclient.Cumulative("idle_timeouts", nil, atomic.LoadInt64(&listener.stats.idleTimeouts)),
		sfxclient.Cumulative("retry_listen_errors", nil, atomic.LoadInt64(&listener.stats.retriedListenErrors)),
	}
}

// Close the exposed wavefront port
func (listener *Listener) Close() error {
	var err error
	if listener.psocket != nil {
		err = listener.psocket.Close()
	}

	listener.wg.Wait()
	return err
}

type wavefrontListenConn interface {
	io.Reader
	Close() error
	SetDeadline(t time.Time) error
	RemoteAddr() net.Addr
}

// TODO need to run this and prometheus through some benchmarks for allocations
func extractCollectdDimensions(doit bool, metricName string) (string, map[string]string) {
	dimensions := make(map[string]string)
	if !doit {
		return metricName, dimensions
	}
	index := strings.Index(metricName, "..")
	var toAddDims map[string]string
	for {
		metricName, toAddDims = collectd.GetDimensionsFromName(&metricName)
		if len(toAddDims) == 0 {
			// we may have put two dots next to each other extracting this stuff, see
			// if there were two beforehand and if not, replace them if they occur with
			// a single dot
			// TODO could do better here if wavefront users prefixed all metrics with
			// collectd, we could know how they're constructed and could make them be
			// almost as good as coming from collectd.
			// could even be crazy and put the types.db file in here and provide
			// accurate metric types... overkill?
			if index == -1 {
				metricName = strings.Replace(metricName, "..", ".", -1)
			}

			return metricName, dimensions
		}
		for k, v := range toAddDims {
			dimensions[k] = v
		}
	}
}

// https://docs.wavefront.com/wavefront_data_format.html
func (listener *Listener) fromWavefrontDatapoint(line string) *datapoint.Datapoint {
	// TODO rewrite without Split using Index and slices as they don't allocate
	pieces := strings.Split(line, " ")
	if len(pieces) < 3 {
		return nil
	}
	metricName, dimensions := extractCollectdDimensions(listener.extractCollectdDimensions, pieces[0])

	valueString := pieces[1]

	var value datapoint.Value
	if i, err := strconv.ParseInt(valueString, 10, 64); err == nil {
		value = datapoint.NewIntValue(i)
	} else if f, err := strconv.ParseFloat(valueString, 64); err == nil {
		value = datapoint.NewFloatValue(f)
	} else {
		return nil
	}
	var timestamp time.Time
	epoch, err := strconv.ParseInt(pieces[2], 10, 64)
	if err != nil {
		// probably not the timestamp, probably a dimension
		pieces = pieces[2:]
		timestamp = time.Now()
	} else {
		timestamp = time.Unix(epoch, 0)
		pieces = pieces[3:]
	}
	for _, p := range pieces {
		dimPieces := strings.SplitN(p, "=", 2)
		if len(dimPieces) == 2 {
			dimensions[dimPieces[0]] = dimPieces[1]
		}
		// ignore malformed dimensions
	}
	return datapoint.New(metricName, dimensions, value, datapoint.Gauge, timestamp)
}

var errInvalidDatapoint = errors.New("invalid wavefront datapoint")

func (listener *Listener) handleTCPConnection(ctx context.Context, conn wavefrontListenConn) error {
	connLogger := log.NewContext(listener.logger).With(logkey.RemoteAddr, conn.RemoteAddr())
	defer func() {
		log.IfErr(connLogger, conn.Close())
	}()
	reader := bufio.NewReader(conn)
	atomic.AddInt64(&listener.stats.totalConnections, 1)
	atomic.AddInt64(&listener.stats.activeConnections, 1)
	defer atomic.AddInt64(&listener.stats.activeConnections, -1)
	for {
		log.IfErr(connLogger, conn.SetDeadline(time.Now().Add(listener.connectionTimeout)))
		bytes, err := reader.ReadBytes((byte)('\n'))
		if err != nil && err != io.EOF {
			atomic.AddInt64(&listener.stats.idleTimeouts, 1)
			connLogger.Log(log.Err, err, "Listening for wavefront data returned an error (Note: We timeout idle connections)")
			return err
		}
		line := strings.TrimSpace(string(bytes))
		if line != "" {
			dp := listener.fromWavefrontDatapoint(line)
			if dp == nil {
				atomic.AddInt64(&listener.stats.invalidDatapoints, 1)
				connLogger.Log(logkey.WavefrontLine, line, log.Err, err, "Received data on a wavefront port, but it doesn't look like wavefront data")
				return errInvalidDatapoint
			}
			log.IfErr(connLogger, listener.sink.AddDatapoints(ctx, []*datapoint.Datapoint{dp}))
			atomic.AddInt64(&listener.stats.totalDatapoints, 1)
		}

		if err == io.EOF {
			atomic.AddInt64(&listener.stats.totalEOFCloses, 1)
			return nil
		}
	}
}

func (listener *Listener) startListeningTCP() {
	defer listener.wg.Done()
	defer listener.logger.Log("Stop listening wavefront TCP")
	for {
		deadlineable, ok := listener.psocket.(*net.TCPListener)
		if ok {
			log.IfErr(listener.logger, deadlineable.SetDeadline(time.Now().Add(listener.serverAcceptDeadline)))
		}
		conn, err := listener.psocket.Accept()
		if err != nil {
			if netErr, ok := err.(net.Error); ok {
				if netErr.Timeout() || netErr.Temporary() {
					atomic.AddInt64(&listener.stats.retriedListenErrors, 1)
					continue
				}
			}
			listener.logger.Log(log.Err, err, "Unable to accept a socket connection")
			return
		}
		go func() {
			log.IfErr(listener.logger, listener.handleTCPConnection(context.Background(), conn))
		}()
	}
}

// ListenerConfig controls optional parameters for wavefront listeners
type ListenerConfig struct {
	ServerAcceptDeadline      *time.Duration
	ConnectionTimeout         *time.Duration
	ListenAddr                *string
	Logger                    log.Logger
	ExtractCollectdDimensions *bool
}

var defaultListenerConfig = &ListenerConfig{
	ServerAcceptDeadline:      pointer.Duration(time.Second),
	ConnectionTimeout:         pointer.Duration(time.Second * 30),
	ListenAddr:                pointer.String("127.0.0.1:2003"),
	ExtractCollectdDimensions: pointer.Bool(true),
}

// Addr returns the listening address of this wavefront listener
func (listener *Listener) Addr() net.Addr {
	return listener.psocket.Addr()
}

func (listener *Listener) getServer(conf *ListenerConfig) error {
	server, err := net.Listen("tcp", *conf.ListenAddr)
	if err != nil {
		return errors.Annotatef(err, "cannot listen to addr %s", *conf.ListenAddr)
	}
	listener.psocket = server
	listener.listenfunc = listener.startListeningTCP
	return nil
}

// NewListener creates a new listener for wavefront datapoints
func NewListener(sendTo dpsink.Sink, passedConf *ListenerConfig) (*Listener, error) {
	conf := pointer.FillDefaultFrom(passedConf, defaultListenerConfig).(*ListenerConfig)
	receiver := Listener{
		sink:                 sendTo,
		serverAcceptDeadline: *conf.ServerAcceptDeadline,
		connectionTimeout:    *conf.ConnectionTimeout,
		logger:               log.NewContext(conf.Logger).With(logkey.Protocol, "wavefront", logkey.Direction, "listener"),
		extractCollectdDimensions: *conf.ExtractCollectdDimensions,
	}
	err := receiver.getServer(conf)
	if err != nil {
		return nil, err
	}
	receiver.wg.Add(1)
	go receiver.listenfunc()
	return &receiver, nil
}
<|endoftext|>


<commit_before>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package schema

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	nvd "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema"
	"github.com/facebookincubator/nvdtools/rpm"
	"github.com/facebookincubator/nvdtools/wfn"
)

const (
	timeLayout = "2006-01-02T15:04:05"
)

var (
	// cwe regex to match CWEs
	cweRegex = regexp.MustCompile("CWE-[0-9]+")
)

func convertTime(redhatTime string) (string, error) {
	t, err := time.Parse(time.RFC3339, redhatTime)
	if err != nil {
		t, err = time.Parse(timeLayout, redhatTime)
	}
	if err != nil { // should be parsable
		log.Printf("unable to parse time: %v", err)
		return redhatTime, err
	}
	return t.Format(nvd.TimeLayout), nil
}

func findCWEs(s string) []string {
	return cweRegex.FindAllString(s, -1)
}

func IsFixed(fixState string) bool {
	// $ jq 'to_entries | .[].value.package_state' redhat.json | grep fix_state | sort -u
	// "fix_state": "Affected",
	// "fix_state": "Fix deferred",
	// "fix_state": "New",
	// "fix_state": "Not affected",
	// "fix_state": "Out of support scope",
	// "fix_state": "Under investigation",
	// "fix_state": "Will not fix",

	switch strings.TrimSpace(strings.ToLower(fixState)) {
	case "affected", "fix deferred", "new", "out of support scope", "will not fix", "under investigation":
		return false
	case "not affected":
		return true
	default:
		log.Printf("unknown fix state: %q", fixState)
		return true
	}
}

func packageName2wfn(packageName string) (*wfn.Attributes, error) {
	product, err := wfn.WFNize(packageName)
	if err != nil {
		return nil, fmt.Errorf("can't wfnize package name %q: %v", packageName, err)
	}
	attrs := wfn.Attributes{
		Part:    "a",
		Product: product,
	}
	return &attrs, nil
}

func package2wfn(pkg string) (*wfn.Attributes, error) {
	attrs := wfn.NewAttributesWithAny()
	err := rpm.ToWFN(attrs, pkg+".src") // add release .src so it parses correctly
	return attrs, err
}
<commit_msg>[redhat] Err on the side of caution when facing unknown resolution<commit_after>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package schema

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	nvd "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema"
	"github.com/facebookincubator/nvdtools/rpm"
	"github.com/facebookincubator/nvdtools/wfn"
)

const (
	timeLayout = "2006-01-02T15:04:05"
)

var (
	// cwe regex to match CWEs
	cweRegex = regexp.MustCompile("CWE-[0-9]+")
)

func convertTime(redhatTime string) (string, error) {
	t, err := time.Parse(time.RFC3339, redhatTime)
	if err != nil {
		t, err = time.Parse(timeLayout, redhatTime)
	}
	if err != nil { // should be parsable
		log.Printf("unable to parse time: %v", err)
		return redhatTime, err
	}
	return t.Format(nvd.TimeLayout), nil
}

func findCWEs(s string) []string {
	return cweRegex.FindAllString(s, -1)
}

// IsFixed returns true if the state string describe a CVE resolution meaning a
// packaged isn't vulnerable.
func IsFixed(fixState string) bool {
	// $ jq 'to_entries | .[].value.package_state' redhat.json | grep fix_state | sort -u
	// "fix_state": "Affected",
	// "fix_state": "Fix deferred",
	// "fix_state": "New",
	// "fix_state": "Not affected",
	// "fix_state": "Out of support scope",
	// "fix_state": "Under investigation",
	// "fix_state": "Will not fix",

	switch strings.TrimSpace(strings.ToLower(fixState)) {
	case "affected", "fix deferred", "new", "out of support scope", "will not fix", "under investigation":
		return false
	case "not affected":
		return true
	default:
		log.Printf("unknown fix state: %q", fixState)
		return false
	}
}

func packageName2wfn(packageName string) (*wfn.Attributes, error) {
	product, err := wfn.WFNize(packageName)
	if err != nil {
		return nil, fmt.Errorf("can't wfnize package name %q: %v", packageName, err)
	}
	attrs := wfn.Attributes{
		Part:    "a",
		Product: product,
	}
	return &attrs, nil
}

func package2wfn(pkg string) (*wfn.Attributes, error) {
	attrs := wfn.NewAttributesWithAny()
	err := rpm.ToWFN(attrs, pkg+".src") // add release .src so it parses correctly
	return attrs, err
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/pointer"
	"io"
	"time"
)

type Forwarder struct {
	Id       [8]byte              // Gateway's Identifier
	alti     int                  // GPS altitude in RX meters
	upnb     uint                 // Number of upstream datagrams sent
	ackn     uint                 // Number of upstream datagrams that were acknowledged
	dwnb     uint                 // Number of downlink datagrams received
	lati     float64              // GPS latitude, North is +
	long     float64              // GPS longitude, East is +
	rxfw     uint                 // Number of radio packets forwarded
	rxnb     uint                 // Number of radio packets received
	adapters []io.ReadWriteCloser // List of downlink adapters
	packets  []semtech.Packet     // Downlink packets received
	done     chan chan error      // Done channel
	commands chan command         // Concurrent access on gateway stats
}

type commandName string
type command struct {
	name commandName
	data interface{}
}

const (
	cmd_ACK     commandName = "Acknowledged"
	cmd_EMIT    commandName = "Emitted"
	cmd_RECVUP  commandName = "Radio Packet Received"
	cmd_RECVDWN commandName = "Dowlink Datagram Received"
	cmd_FWD     commandName = "Forwarded"
	cmd_FLUSH   commandName = "Flush"
	cmd_STATS   commandName = "Stats"
)

// NewForwarder create a forwarder instance bound to a set of routers.
func NewForwarder(id [8]byte, adapters ...io.ReadWriteCloser) (*Forwarder, error) {
	if len(adapters) == 0 {
		return nil, fmt.Errorf("At least one adapter must be supplied")
	}

	fwd := &Forwarder{
		Id:       id,
		alti:     120,
		lati:     53.3702,
		long:     4.8952,
		adapters: adapters,
		done:     make(chan chan error, len(adapters)),
		commands: make(chan command),
	}

	go fwd.handleCommands()
	go fwd.listen()

	return fwd, nil
}

// listen get downlink packets from routers and store them until a flush is requested
func (fwd *Forwarder) listen() {
	dwnl := make(chan semtech.Packet, len(fwd.adapters))

	// Star listening to each adapter Read() method
	for _, adapter := range fwd.adapters {
		go asChannel(adapter, dwnl)
	}

	for {
		select {
		case fwd.commands <- command{cmd_RECVDWN, <-dwnl}:

		case errc := <-fwd.done:
			// Empty the buffer first to avoid leaking goroutines
			nb := len(dwnl)
			for i := 0; i < nb; i += 1 {
				fwd.commands <- command{cmd_RECVDWN, <-dwnl}
			}

			// Then stop
			errc <- nil
			return
		}
	}
}

// asChannel listen to incoming connection from an adapter and forward them into a dedicated
// channel. Non-valid packets are ignored.
func asChannel(adapter io.ReadWriteCloser, dwnl chan<- semtech.Packet) {
	for {
		buf := make([]byte, 1024)
		n, err := adapter.Read(buf)
		if err != nil {
			fmt.Println(err)
			return // Error on reading, we assume the connection is closed / lost
		}
		packet, err := semtech.Unmarshal(buf[:n])
		if err != nil {
			fmt.Println(err)
			continue
		}
		if packet.Identifier != semtech.PUSH_DATA {
			continue
		}
		dwnl <- *packet // Only valid PUSH_DATA packet are transmitted through the chan
	}
}

// handleCommands acts as a mediator between all goroutines that attempt to modify the forwarder
// attributes. All sensitive operations are done by commands send though an appropriate channel.
// This method consume commands from the channel until it's closed.
func (fwd *Forwarder) handleCommands() {
	for cmd := range fwd.commands {
		switch cmd.name {
		case cmd_ACK:
			fwd.ackn += 1
		case cmd_FWD:
			fwd.rxfw += 1
		case cmd_EMIT:
			fwd.upnb += 1
		case cmd_RECVUP:
			fwd.rxnb += 1
		case cmd_RECVDWN:
			fwd.dwnb += 1
			fwd.packets = append(fwd.packets, cmd.data.(semtech.Packet))
		case cmd_FLUSH:
			cmd.data.(chan []semtech.Packet) <- fwd.packets
			fwd.packets = make([]semtech.Packet, 0)
		case cmd_STATS:
			var ackr float64
			if fwd.upnb != 0 {
				ackr = float64(fwd.ackn) / float64(fwd.upnb)
			}

			cmd.data.(chan semtech.Stat) <- semtech.Stat{
				Ackr: &ackr,
				Alti: pointer.Int(fwd.alti),
				Dwnb: pointer.Uint(fwd.dwnb),
				Lati: pointer.Float64(fwd.lati),
				Long: pointer.Float64(fwd.long),
				Rxfw: pointer.Uint(fwd.rxfw),
				Rxnb: pointer.Uint(fwd.rxnb),
				Rxok: pointer.Uint(fwd.rxnb),
				Time: pointer.Time(time.Now()),
				Txnb: pointer.Uint(0),
			}
		}
	}
}

// Forward dispatch a packet to all connected routers.
func (fwd *Forwarder) Forward(packet semtech.Packet) error {
	fwd.commands <- command{cmd_RECVUP, nil}
	if packet.Identifier != semtech.PUSH_DATA {
		return fmt.Errorf("Unable to forward with identifier %x", packet.Identifier)
	}

	raw, err := semtech.Marshal(packet)
	if err != nil {
		return err
	}

	for _, adapter := range fwd.adapters {
		n, err := adapter.Write(raw)
		if err != nil {
			return err
		}
		if n < len(raw) {
			return fmt.Errorf("Packet was too long")
		}
		fwd.commands <- command{cmd_EMIT, nil}
	}

	fwd.commands <- command{cmd_FWD, nil}
	return nil
}

// Flush spits out all downlink packet received by the forwarder since the last flush.
func (fwd *Forwarder) Flush() []semtech.Packet {
	chpkt := make(chan []semtech.Packet)
	fwd.commands <- command{cmd_FLUSH, chpkt}
	return <-chpkt
}

// Stats computes and return the forwarder statistics since it was created
func (fwd Forwarder) Stats() semtech.Stat {
	chstats := make(chan semtech.Stat)
	fwd.commands <- command{cmd_STATS, chstats}
	return <-chstats
}

// Stop terminate the forwarder activity. Closing all routers connections
func (fwd *Forwarder) Stop() error {
	var errors []error

	// Close the uplink adapters
	for _, adapter := range fwd.adapters {
		err := adapter.Close()
		if err != nil {
			errors = append(errors, err)
		}
	}

	if len(errors) > 0 {
		return fmt.Errorf("Unable to stop the forwarder: %+v", errors)
	}

	// Stop listening to downlink packets
	errc := make(chan error)
	fwd.done <- errc
	if err := <-errc; err != nil {
		return err
	}

	// Close the commands channel
	close(fwd.commands)

	return nil
}
<commit_msg>[simulators.gateway] Add some comments in the forwarder<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"fmt"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/pointer"
	"io"
	"time"
)

type Forwarder struct {
	Id       [8]byte              // Gateway's Identifier
	alti     int                  // GPS altitude in RX meters
	upnb     uint                 // Number of upstream datagrams sent
	ackn     uint                 // Number of upstream datagrams that were acknowledged
	dwnb     uint                 // Number of downlink datagrams received
	lati     float64              // GPS latitude, North is +
	long     float64              // GPS longitude, East is +
	rxfw     uint                 // Number of radio packets forwarded
	rxnb     uint                 // Number of radio packets received
	adapters []io.ReadWriteCloser // List of downlink adapters
	packets  []semtech.Packet     // Downlink packets received
	done     chan chan error      // Done channel
	commands chan command         // Concurrent access on gateway stats
}

type commandName string
type command struct {
	name commandName
	data interface{}
}

const (
	cmd_ACK     commandName = "Acknowledged"
	cmd_EMIT    commandName = "Emitted"
	cmd_RECVUP  commandName = "Radio Packet Received"
	cmd_RECVDWN commandName = "Dowlink Datagram Received"
	cmd_FWD     commandName = "Forwarded"
	cmd_FLUSH   commandName = "Flush"
	cmd_STATS   commandName = "Stats"
)

// NewForwarder create a forwarder instance bound to a set of routers.
func NewForwarder(id [8]byte, adapters ...io.ReadWriteCloser) (*Forwarder, error) {
	if len(adapters) == 0 {
		return nil, fmt.Errorf("At least one adapter must be supplied")
	}

	fwd := &Forwarder{
		Id:       id,
		alti:     120,
		lati:     53.3702,
		long:     4.8952,
		adapters: adapters,
		done:     make(chan chan error, len(adapters)),
		commands: make(chan command),
	}

	go fwd.handleCommands()
	go fwd.listen()

	return fwd, nil
}

// listen get downlink packets from routers and store them until a flush is requested
func (fwd *Forwarder) listen() {
	dwnl := make(chan semtech.Packet, len(fwd.adapters))

	// Star listening to each adapter Read() method
	for _, adapter := range fwd.adapters {
		go asChannel(adapter, dwnl)
	}

	for {
		select {
		case fwd.commands <- command{cmd_RECVDWN, <-dwnl}:

		case errc := <-fwd.done:
			// Empty the buffer first to avoid leaking goroutines
			nb := len(dwnl)
			for i := 0; i < nb; i += 1 {
				fwd.commands <- command{cmd_RECVDWN, <-dwnl}
			}

			// Then stop
			errc <- nil
			return
		}
	}
}

// asChannel listen to incoming connection from an adapter and forward them into a dedicated
// channel. Non-valid packets are ignored.
func asChannel(adapter io.ReadWriteCloser, dwnl chan<- semtech.Packet) {
	for {
		buf := make([]byte, 1024)
		fmt.Printf("Forwarder listens to downlink datagrams\n")
		n, err := adapter.Read(buf)
		if err != nil {
			fmt.Println(err)
			return // Error on reading, we assume the connection is closed / lost
		}
		fmt.Printf("Forwarder unmarshals datagram %x\n", buf[:n])
		packet, err := semtech.Unmarshal(buf[:n])
		if err != nil {
			fmt.Println(err)
			continue
		}
		if packet.Identifier != semtech.PUSH_DATA {
			fmt.Printf("Forwarder ignores contingent packet %+v\n", packet)
			continue
		}
		dwnl <- *packet // Only valid PUSH_DATA packet are transmitted through the chan
	}
}

// handleCommands acts as a mediator between all goroutines that attempt to modify the forwarder
// attributes. All sensitive operations are done by commands send though an appropriate channel.
// This method consume commands from the channel until it's closed.
func (fwd *Forwarder) handleCommands() {
	for cmd := range fwd.commands {
		fmt.Printf("Fowarder executes command: %v\n", cmd.name)
		switch cmd.name {
		case cmd_ACK:
			fwd.ackn += 1
		case cmd_FWD:
			fwd.rxfw += 1
		case cmd_EMIT:
			fwd.upnb += 1
		case cmd_RECVUP:
			fwd.rxnb += 1
		case cmd_RECVDWN:
			fwd.dwnb += 1
			fwd.packets = append(fwd.packets, cmd.data.(semtech.Packet))
		case cmd_FLUSH:
			cmd.data.(chan []semtech.Packet) <- fwd.packets
			fwd.packets = make([]semtech.Packet, 0)
		case cmd_STATS:
			var ackr float64
			if fwd.upnb != 0 {
				ackr = float64(fwd.ackn) / float64(fwd.upnb)
			}

			cmd.data.(chan semtech.Stat) <- semtech.Stat{
				Ackr: &ackr,
				Alti: pointer.Int(fwd.alti),
				Dwnb: pointer.Uint(fwd.dwnb),
				Lati: pointer.Float64(fwd.lati),
				Long: pointer.Float64(fwd.long),
				Rxfw: pointer.Uint(fwd.rxfw),
				Rxnb: pointer.Uint(fwd.rxnb),
				Rxok: pointer.Uint(fwd.rxnb),
				Time: pointer.Time(time.Now()),
				Txnb: pointer.Uint(0),
			}
		}
	}
}

// Forward dispatch a packet to all connected routers.
func (fwd *Forwarder) Forward(packet semtech.Packet) error {
	fwd.commands <- command{cmd_RECVUP, nil}
	if packet.Identifier != semtech.PUSH_DATA {
		return fmt.Errorf("Unable to forward with identifier %x", packet.Identifier)
	}

	raw, err := semtech.Marshal(packet)
	if err != nil {
		return err
	}

	for _, adapter := range fwd.adapters {
		n, err := adapter.Write(raw)
		if err != nil {
			return err
		}
		if n < len(raw) {
			return fmt.Errorf("Packet was too long")
		}
		fwd.commands <- command{cmd_EMIT, nil}
	}

	fwd.commands <- command{cmd_FWD, nil}
	return nil
}

// Flush spits out all downlink packet received by the forwarder since the last flush.
func (fwd *Forwarder) Flush() []semtech.Packet {
	chpkt := make(chan []semtech.Packet)
	fwd.commands <- command{cmd_FLUSH, chpkt}
	return <-chpkt
}

// Stats computes and return the forwarder statistics since it was created
func (fwd Forwarder) Stats() semtech.Stat {
	chstats := make(chan semtech.Stat)
	fwd.commands <- command{cmd_STATS, chstats}
	return <-chstats
}

// Stop terminate the forwarder activity. Closing all routers connections
func (fwd *Forwarder) Stop() error {
	var errors []error

	// Close the uplink adapters
	for _, adapter := range fwd.adapters {
		err := adapter.Close()
		if err != nil {
			errors = append(errors, err)
		}
	}

	if len(errors) > 0 {
		return fmt.Errorf("Unable to stop the forwarder: %+v", errors)
	}

	// Stop listening to downlink packets
	errc := make(chan error)
	fwd.done <- errc
	if err := <-errc; err != nil {
		return err
	}

	// Close the commands channel
	close(fwd.commands)

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

var taskDefinitionRE = regexp.MustCompile("^([a-zA-Z0-9_-]+):([0-9]+)$")

func resourceAwsEcsService() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEcsServiceCreate,
		Read:   resourceAwsEcsServiceRead,
		Update: resourceAwsEcsServiceUpdate,
		Delete: resourceAwsEcsServiceDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"cluster": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"task_definition": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"desired_count": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
			},

			"iam_role": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"load_balancer": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"elb_name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"container_name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"container_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
				Set: resourceAwsEcsLoadBalancerHash,
			},
		},
	}
}

func resourceAwsEcsServiceCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	input := ecs.CreateServiceInput{
		ServiceName:    aws.String(d.Get("name").(string)),
		TaskDefinition: aws.String(d.Get("task_definition").(string)),
		DesiredCount:   aws.Long(int64(d.Get("desired_count").(int))),
	}

	if v, ok := d.GetOk("cluster"); ok {
		input.Cluster = aws.String(v.(string))
	}

	loadBalancers := expandEcsLoadBalancers(d.Get("load_balancer").(*schema.Set).List())
	if len(loadBalancers) > 0 {
		log.Printf("[DEBUG] Adding ECS load balancers: %#v", loadBalancers)
		input.LoadBalancers = loadBalancers
	}
	if v, ok := d.GetOk("iam_role"); ok {
		input.Role = aws.String(v.(string))
	}

	log.Printf("[DEBUG] Creating ECS service: %#v", input)
	out, err := conn.CreateService(&input)
	if err != nil {
		return err
	}

	service := *out.Service

	log.Printf("[DEBUG] ECS service created: %s", *service.ServiceARN)
	d.SetId(*service.ServiceARN)
	d.Set("cluster", *service.ClusterARN)

	return resourceAwsEcsServiceUpdate(d, meta)
}

func resourceAwsEcsServiceRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	log.Printf("[DEBUG] Reading ECS service %s", d.Id())
	input := ecs.DescribeServicesInput{
		Services: []*string{aws.String(d.Id())},
		Cluster:  aws.String(d.Get("cluster").(string)),
	}

	out, err := conn.DescribeServices(&input)
	if err != nil {
		return err
	}

	service := out.Services[0]
	log.Printf("[DEBUG] Received ECS service %#v", service)

	d.SetId(*service.ServiceARN)
	d.Set("name", *service.ServiceName)

	// Save task definition in the same format
	if strings.HasPrefix(d.Get("task_definition").(string), "arn:aws:ecs:") {
		d.Set("task_definition", *service.TaskDefinition)
	} else {
		taskDefinition := buildFamilyAndRevisionFromARN(*service.TaskDefinition)
		d.Set("task_definition", taskDefinition)
	}

	d.Set("desired_count", *service.DesiredCount)
	d.Set("cluster", *service.ClusterARN)

	if service.RoleARN != nil {
		d.Set("iam_role", *service.RoleARN)
	}

	if service.LoadBalancers != nil {
		d.Set("load_balancers", flattenEcsLoadBalancers(service.LoadBalancers))
	}

	return nil
}

func resourceAwsEcsServiceUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	log.Printf("[DEBUG] Updating ECS service %s", d.Id())
	input := ecs.UpdateServiceInput{
		Service: aws.String(d.Id()),
		Cluster: aws.String(d.Get("cluster").(string)),
	}

	if d.HasChange("desired_count") {
		_, n := d.GetChange("desired_count")
		input.DesiredCount = aws.Long(int64(n.(int)))
	}
	if d.HasChange("task_definition") {
		_, n := d.GetChange("task_definition")
		input.TaskDefinition = aws.String(n.(string))
	}

	out, err := conn.UpdateService(&input)
	if err != nil {
		return err
	}
	service := out.Service
	log.Printf("[DEBUG] Updated ECS service %#v", service)

	return resourceAwsEcsServiceRead(d, meta)
}

func resourceAwsEcsServiceDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	// Check if it's not already gone
	resp, err := conn.DescribeServices(&ecs.DescribeServicesInput{
		Services: []*string{aws.String(d.Id())},
		Cluster:  aws.String(d.Get("cluster").(string)),
	})
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] ECS service %s is currently %s", d.Id(), *resp.Services[0].Status)

	if *resp.Services[0].Status == "INACTIVE" {
		return nil
	}

	// Drain the ECS service
	if *resp.Services[0].Status != "DRAINING" {
		log.Printf("[DEBUG] Draining ECS service %s", d.Id())
		_, err = conn.UpdateService(&ecs.UpdateServiceInput{
			Service:      aws.String(d.Id()),
			Cluster:      aws.String(d.Get("cluster").(string)),
			DesiredCount: aws.Long(int64(0)),
		})
		if err != nil {
			return err
		}
	}

	input := ecs.DeleteServiceInput{
		Service: aws.String(d.Id()),
		Cluster: aws.String(d.Get("cluster").(string)),
	}

	log.Printf("[DEBUG] Deleting ECS service %#v", input)
	out, err := conn.DeleteService(&input)
	if err != nil {
		return err
	}

	// Wait until it's deleted
	wait := resource.StateChangeConf{
		Pending:    []string{"DRAINING"},
		Target:     "INACTIVE",
		Timeout:    5 * time.Minute,
		MinTimeout: 1 * time.Second,
		Refresh: func() (interface{}, string, error) {
			log.Printf("[DEBUG] Checking if ECS service %s is INACTIVE", d.Id())
			resp, err := conn.DescribeServices(&ecs.DescribeServicesInput{
				Services: []*string{aws.String(d.Id())},
				Cluster:  aws.String(d.Get("cluster").(string)),
			})
			if err != nil {
				return resp, "FAILED", err
			}

			return resp, *resp.Services[0].Status, nil
		},
	}

	_, err = wait.WaitForState()
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] ECS service %s deleted.", *out.Service.ServiceARN)
	return nil
}

func resourceAwsEcsLoadBalancerHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%s-", m["elb_name"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["container_name"].(string)))
	buf.WriteString(fmt.Sprintf("%d-", m["container_port"].(int)))

	return hashcode.String(buf.String())
}

func buildFamilyAndRevisionFromARN(arn string) string {
	return strings.Split(arn, "/")[1]
}

func buildTaskDefinitionARN(taskDefinition string, meta interface{}) (string, error) {
	// If it's already an ARN, just return it
	if strings.HasPrefix(taskDefinition, "arn:aws:ecs:") {
		return taskDefinition, nil
	}

	// Parse out family & revision
	family, revision, err := parseTaskDefinition(taskDefinition)
	if err != nil {
		return "", err
	}

	iamconn := meta.(*AWSClient).iamconn
	region := meta.(*AWSClient).region

	// An zero value GetUserInput{} defers to the currently logged in user
	resp, err := iamconn.GetUser(&iam.GetUserInput{})
	if err != nil {
		return "", fmt.Errorf("GetUser ERROR: %#v", err)
	}

	// arn:aws:iam::0123456789:user/username
	userARN := *resp.User.ARN
	accountID := strings.Split(userARN, ":")[4]

	// arn:aws:ecs:us-west-2:01234567890:task-definition/mongodb:3
	arn := fmt.Sprintf("arn:aws:ecs:%s:%s:task-definition/%s:%s",
		region, accountID, family, revision)
	log.Printf("[DEBUG] Built task definition ARN: %s", arn)
	return arn, nil
}

func parseTaskDefinition(taskDefinition string) (string, string, error) {
	matches := taskDefinitionRE.FindAllStringSubmatch(taskDefinition, 2)

	if len(matches) == 0 || len(matches[0]) != 3 {
		return "", "", fmt.Errorf(
			"Invalid task definition format, family:rev or ARN expected (%#v)",
			taskDefinition)
	}

	return matches[0][1], matches[0][2], nil
}
<commit_msg>aws: Allow migrating (recreating) ecs_service to another cluster<commit_after>package aws

import (
	"bytes"
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

var taskDefinitionRE = regexp.MustCompile("^([a-zA-Z0-9_-]+):([0-9]+)$")

func resourceAwsEcsService() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEcsServiceCreate,
		Read:   resourceAwsEcsServiceRead,
		Update: resourceAwsEcsServiceUpdate,
		Delete: resourceAwsEcsServiceDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"cluster": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"task_definition": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"desired_count": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
			},

			"iam_role": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},

			"load_balancer": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"elb_name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"container_name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},

						"container_port": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
				Set: resourceAwsEcsLoadBalancerHash,
			},
		},
	}
}

func resourceAwsEcsServiceCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	input := ecs.CreateServiceInput{
		ServiceName:    aws.String(d.Get("name").(string)),
		TaskDefinition: aws.String(d.Get("task_definition").(string)),
		DesiredCount:   aws.Long(int64(d.Get("desired_count").(int))),
	}

	if v, ok := d.GetOk("cluster"); ok {
		input.Cluster = aws.String(v.(string))
	}

	loadBalancers := expandEcsLoadBalancers(d.Get("load_balancer").(*schema.Set).List())
	if len(loadBalancers) > 0 {
		log.Printf("[DEBUG] Adding ECS load balancers: %#v", loadBalancers)
		input.LoadBalancers = loadBalancers
	}
	if v, ok := d.GetOk("iam_role"); ok {
		input.Role = aws.String(v.(string))
	}

	log.Printf("[DEBUG] Creating ECS service: %#v", input)
	out, err := conn.CreateService(&input)
	if err != nil {
		return err
	}

	service := *out.Service

	log.Printf("[DEBUG] ECS service created: %s", *service.ServiceARN)
	d.SetId(*service.ServiceARN)
	d.Set("cluster", *service.ClusterARN)

	return resourceAwsEcsServiceUpdate(d, meta)
}

func resourceAwsEcsServiceRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	log.Printf("[DEBUG] Reading ECS service %s", d.Id())
	input := ecs.DescribeServicesInput{
		Services: []*string{aws.String(d.Id())},
		Cluster:  aws.String(d.Get("cluster").(string)),
	}

	out, err := conn.DescribeServices(&input)
	if err != nil {
		return err
	}

	if len(out.Services) < 1 {
		return nil
	}

	service := out.Services[0]
	log.Printf("[DEBUG] Received ECS service %#v", service)

	d.SetId(*service.ServiceARN)
	d.Set("name", *service.ServiceName)

	// Save task definition in the same format
	if strings.HasPrefix(d.Get("task_definition").(string), "arn:aws:ecs:") {
		d.Set("task_definition", *service.TaskDefinition)
	} else {
		taskDefinition := buildFamilyAndRevisionFromARN(*service.TaskDefinition)
		d.Set("task_definition", taskDefinition)
	}

	d.Set("desired_count", *service.DesiredCount)
	d.Set("cluster", *service.ClusterARN)

	if service.RoleARN != nil {
		d.Set("iam_role", *service.RoleARN)
	}

	if service.LoadBalancers != nil {
		d.Set("load_balancers", flattenEcsLoadBalancers(service.LoadBalancers))
	}

	return nil
}

func resourceAwsEcsServiceUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	log.Printf("[DEBUG] Updating ECS service %s", d.Id())
	input := ecs.UpdateServiceInput{
		Service: aws.String(d.Id()),
		Cluster: aws.String(d.Get("cluster").(string)),
	}

	if d.HasChange("desired_count") {
		_, n := d.GetChange("desired_count")
		input.DesiredCount = aws.Long(int64(n.(int)))
	}
	if d.HasChange("task_definition") {
		_, n := d.GetChange("task_definition")
		input.TaskDefinition = aws.String(n.(string))
	}

	out, err := conn.UpdateService(&input)
	if err != nil {
		return err
	}
	service := out.Service
	log.Printf("[DEBUG] Updated ECS service %#v", service)

	return resourceAwsEcsServiceRead(d, meta)
}

func resourceAwsEcsServiceDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecsconn

	// Check if it's not already gone
	resp, err := conn.DescribeServices(&ecs.DescribeServicesInput{
		Services: []*string{aws.String(d.Id())},
		Cluster:  aws.String(d.Get("cluster").(string)),
	})
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] ECS service %s is currently %s", d.Id(), *resp.Services[0].Status)

	if *resp.Services[0].Status == "INACTIVE" {
		return nil
	}

	// Drain the ECS service
	if *resp.Services[0].Status != "DRAINING" {
		log.Printf("[DEBUG] Draining ECS service %s", d.Id())
		_, err = conn.UpdateService(&ecs.UpdateServiceInput{
			Service:      aws.String(d.Id()),
			Cluster:      aws.String(d.Get("cluster").(string)),
			DesiredCount: aws.Long(int64(0)),
		})
		if err != nil {
			return err
		}
	}

	input := ecs.DeleteServiceInput{
		Service: aws.String(d.Id()),
		Cluster: aws.String(d.Get("cluster").(string)),
	}

	log.Printf("[DEBUG] Deleting ECS service %#v", input)
	out, err := conn.DeleteService(&input)
	if err != nil {
		return err
	}

	// Wait until it's deleted
	wait := resource.StateChangeConf{
		Pending:    []string{"DRAINING"},
		Target:     "INACTIVE",
		Timeout:    5 * time.Minute,
		MinTimeout: 1 * time.Second,
		Refresh: func() (interface{}, string, error) {
			log.Printf("[DEBUG] Checking if ECS service %s is INACTIVE", d.Id())
			resp, err := conn.DescribeServices(&ecs.DescribeServicesInput{
				Services: []*string{aws.String(d.Id())},
				Cluster:  aws.String(d.Get("cluster").(string)),
			})
			if err != nil {
				return resp, "FAILED", err
			}

			return resp, *resp.Services[0].Status, nil
		},
	}

	_, err = wait.WaitForState()
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] ECS service %s deleted.", *out.Service.ServiceARN)
	return nil
}

func resourceAwsEcsLoadBalancerHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%s-", m["elb_name"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["container_name"].(string)))
	buf.WriteString(fmt.Sprintf("%d-", m["container_port"].(int)))

	return hashcode.String(buf.String())
}

func buildFamilyAndRevisionFromARN(arn string) string {
	return strings.Split(arn, "/")[1]
}

func buildTaskDefinitionARN(taskDefinition string, meta interface{}) (string, error) {
	// If it's already an ARN, just return it
	if strings.HasPrefix(taskDefinition, "arn:aws:ecs:") {
		return taskDefinition, nil
	}

	// Parse out family & revision
	family, revision, err := parseTaskDefinition(taskDefinition)
	if err != nil {
		return "", err
	}

	iamconn := meta.(*AWSClient).iamconn
	region := meta.(*AWSClient).region

	// An zero value GetUserInput{} defers to the currently logged in user
	resp, err := iamconn.GetUser(&iam.GetUserInput{})
	if err != nil {
		return "", fmt.Errorf("GetUser ERROR: %#v", err)
	}

	// arn:aws:iam::0123456789:user/username
	userARN := *resp.User.ARN
	accountID := strings.Split(userARN, ":")[4]

	// arn:aws:ecs:us-west-2:01234567890:task-definition/mongodb:3
	arn := fmt.Sprintf("arn:aws:ecs:%s:%s:task-definition/%s:%s",
		region, accountID, family, revision)
	log.Printf("[DEBUG] Built task definition ARN: %s", arn)
	return arn, nil
}

func parseTaskDefinition(taskDefinition string) (string, string, error) {
	matches := taskDefinitionRE.FindAllStringSubmatch(taskDefinition, 2)

	if len(matches) == 0 || len(matches[0]) != 3 {
		return "", "", fmt.Errorf(
			"Invalid task definition format, family:rev or ARN expected (%#v)",
			taskDefinition)
	}

	return matches[0][1], matches[0][2], nil
}
<|endoftext|>


<commit_before>package mobi

import (
	"crypto/sha1"
	"fmt"
	"image"
	"io"
	"os"
	"path/filepath"

	"github.com/geek1011/BookBrowser/booklist"
	"github.com/geek1011/BookBrowser/formats"
	"github.com/pkg/errors"

	mobirdr "github.com/sblinch/mobi"
		"encoding/binary"
)

type mobi struct {
	book *booklist.Book
	coverstart int64
	coverend int64
}

func (e *mobi) Book() *booklist.Book {
	return e.book
}

func (e *mobi) HasCover() bool {
	return e.coverstart > 0
}

func (e *mobi) GetCover() (i image.Image, err error) {
	if !e.HasCover() {
		return nil, errors.New("no cover")
	}

	f, err := os.Open(e.book.FilePath)
	if err != nil {
		return nil, errors.Wrap(err, "unable to open book file")
	}
	defer f.Close()

	if _, err := f.Seek(e.coverstart, 0); err != nil {
		return nil, errors.Wrap(err, "unable to see to cover offset")
	}

	if i, _, err = image.Decode(f); err != nil {
		return nil, errors.Wrap(err, "unable to decode book cover")
	}

	return i, nil
}

func load(filename string) (bi formats.BookInfo, ferr error) {
	defer func() {
		if r := recover(); r != nil {
			bi = nil
			ferr = fmt.Errorf("unknown error: %s", r)
		}
	}()

	m := &mobi{book: &booklist.Book{}}

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, errors.Wrapf(err, "could not stat book")
	}
	m.book.FilePath = filename
	m.book.FileSize = fi.Size()
	m.book.ModTime = fi.ModTime()

	s := sha1.New()
	i, err := io.Copy(s, f)
	if err == nil && i != fi.Size() {
		err = errors.New("could not read whole file")
	}
	if err != nil {
		f.Close()
		return nil, errors.Wrap(err, "could not hash book")
	}
	m.book.Hash = fmt.Sprintf("%x", s.Sum(nil))
	f.Close()

	r, err := mobirdr.NewReader(filename)
	if err != nil {
		return nil, err
	}

	for _, rec := range r.Exth.Records {
		switch(rec.RecordType) {
		case mobirdr.EXTH_COVEROFFSET:
			v := binary.BigEndian.Uint32([]byte(rec.Value))
			coverPDBOffset := r.Header.FirstImageIndex + v

			n := int(coverPDBOffset)
			if n <= int(r.Pdf.RecordsNum)-1 {
				if n+1 < int(r.Pdf.RecordsNum) {
					m.coverend = int64(r.Offsets[n+1].Offset)
				} else {
					m.coverend = -1
				}
				m.coverstart = int64(r.Offsets[n].Offset)
			}

		case mobirdr.EXTH_TITLE:
			if len(m.book.Title)==0 {
				m.book.Title = string(rec.Value)
			}
		case mobirdr.EXTH_UPDATEDTITLE:
			if len(m.book.Title)==0 {
				m.book.Title = string(rec.Value)
			}
		case mobirdr.EXTH_AUTHOR:
			if len(m.book.Author) == 0 {
				m.book.Author = string(rec.Value)
			}
		case mobirdr.EXTH_DESCRIPTION:
			if len(m.book.Description) == 0 {
				m.book.Description = string(rec.Value)
			}
		case mobirdr.EXTH_PUBLISHER:
			if len(m.book.Publisher) == 0 {
				m.book.Publisher = string(rec.Value)
			}
/* // uncomment after merging ISBN/publishing date pull request :)
		case mobirdr.EXTH_ISBN:
			if len(m.book.ISBN) == 0 {
				m.book.ISBN = string(rec.Value)
			}
		case mobirdr.EXTH_PUBLISHINGDATE:
			if m.book.PublishDate.IsZero() {
				m.book.PublishDate = parsePublishDate(string(rec.Value))
			}
*/
		}
	}

	if len(m.book.Title)==0 {
		m.book.Title = filepath.Base(filename)
	}

	return m, nil
}

func init() {
	formats.Register("mobi", load)
}

/* // uncomment after merging ISBN/publishing date pull request :)
func parsePublishDate(s string) time.Time {
	// handle the various dumb decisions people make when encoding dates
	format := ""
	switch len(s) {
	case 32:
		//2012-02-13T20:20:58.175203+00:00
		format = "2006-01-02T15:04:05.000000-07:00"
	case 25:
		//2000-10-31 00:00:00-06:00
		//2009-04-19T22:00:00+00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05-07:00"
	case 20:
		//2016-08-11T14:09:25Z
		format = "2006-01-02T15:04:05Z"
	case 19:
		//2008-01-28T07:00:00
		//2000-10-31 00:00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05"
	case 10:
		//1998-07-01
		format = "2006-01-02"
	default:
		return time.Time{}
	}

	t, err := time.Parse(format,s)
	if err != nil {
		t = time.Time{}
	}
	return t
}
*/<commit_msg>Validate ISBNs extracted from .mobi files<commit_after>package mobi

import (
	"crypto/sha1"
	"fmt"
	"image"
	"io"
	"os"
	"path/filepath"

	"github.com/geek1011/BookBrowser/booklist"
	"github.com/geek1011/BookBrowser/formats"
	"github.com/pkg/errors"

	mobirdr "github.com/sblinch/mobi"
	"encoding/binary"
	)

type mobi struct {
	book *booklist.Book
	coverstart int64
	coverend int64
}

func (e *mobi) Book() *booklist.Book {
	return e.book
}

func (e *mobi) HasCover() bool {
	return e.coverstart > 0
}

func (e *mobi) GetCover() (i image.Image, err error) {
	if !e.HasCover() {
		return nil, errors.New("no cover")
	}

	f, err := os.Open(e.book.FilePath)
	if err != nil {
		return nil, errors.Wrap(err, "unable to open book file")
	}
	defer f.Close()

	if _, err := f.Seek(e.coverstart, 0); err != nil {
		return nil, errors.Wrap(err, "unable to see to cover offset")
	}

	if i, _, err = image.Decode(f); err != nil {
		return nil, errors.Wrap(err, "unable to decode book cover")
	}

	return i, nil
}

func load(filename string) (bi formats.BookInfo, ferr error) {
	defer func() {
		if r := recover(); r != nil {
			bi = nil
			ferr = fmt.Errorf("unknown error: %s", r)
		}
	}()

	m := &mobi{book: &booklist.Book{}}

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, errors.Wrapf(err, "could not stat book")
	}
	m.book.FilePath = filename
	m.book.FileSize = fi.Size()
	m.book.ModTime = fi.ModTime()

	s := sha1.New()
	i, err := io.Copy(s, f)
	if err == nil && i != fi.Size() {
		err = errors.New("could not read whole file")
	}
	if err != nil {
		f.Close()
		return nil, errors.Wrap(err, "could not hash book")
	}
	m.book.Hash = fmt.Sprintf("%x", s.Sum(nil))
	f.Close()

	r, err := mobirdr.NewReader(filename)
	if err != nil {
		return nil, err
	}

	for _, rec := range r.Exth.Records {
		switch(rec.RecordType) {
		case mobirdr.EXTH_COVEROFFSET:
			v := binary.BigEndian.Uint32([]byte(rec.Value))
			coverPDBOffset := r.Header.FirstImageIndex + v

			n := int(coverPDBOffset)
			if n <= int(r.Pdf.RecordsNum)-1 {
				if n+1 < int(r.Pdf.RecordsNum) {
					m.coverend = int64(r.Offsets[n+1].Offset)
				} else {
					m.coverend = -1
				}
				m.coverstart = int64(r.Offsets[n].Offset)
			}

		case mobirdr.EXTH_TITLE:
			if len(m.book.Title)==0 {
				m.book.Title = string(rec.Value)
			}
		case mobirdr.EXTH_UPDATEDTITLE:
			if len(m.book.Title)==0 {
				m.book.Title = string(rec.Value)
			}
		case mobirdr.EXTH_AUTHOR:
			if len(m.book.Author) == 0 {
				m.book.Author = string(rec.Value)
			}
		case mobirdr.EXTH_DESCRIPTION:
			if len(m.book.Description) == 0 {
				m.book.Description = string(rec.Value)
			}
		case mobirdr.EXTH_PUBLISHER:
			if len(m.book.Publisher) == 0 {
				m.book.Publisher = string(rec.Value)
			}
 /* // uncomment after merging ISBN/publishing date pull request :)
		case mobirdr.EXTH_ISBN:
			if len(m.book.ISBN) == 0 {
				isbnStr := string(rec.Value)
				if isbn.Validate(isbnStr) {
					m.book.ISBN = isbnStr
				}
			}
		case mobirdr.EXTH_PUBLISHINGDATE:
			if m.book.PublishDate.IsZero() {
				m.book.PublishDate = parsePublishDate(string(rec.Value))
			}
*/
		}
	}

	if len(m.book.Title)==0 {
		m.book.Title = filepath.Base(filename)
	}

	return m, nil
}

func init() {
	formats.Register("mobi", load)
}

/* // uncomment after merging ISBN/publishing date pull request :)
func parsePublishDate(s string) time.Time {
	// handle the various dumb decisions people make when encoding dates
	format := ""
	switch len(s) {
	case 32:
		//2012-02-13T20:20:58.175203+00:00
		format = "2006-01-02T15:04:05.000000-07:00"
	case 25:
		//2000-10-31 00:00:00-06:00
		//2009-04-19T22:00:00+00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05-07:00"
	case 20:
		//2016-08-11T14:09:25Z
		format = "2006-01-02T15:04:05Z"
	case 19:
		//2008-01-28T07:00:00
		//2000-10-31 00:00:00
		format = "2006-01-02" + string(s[10]) + "15:04:05"
	case 10:
		//1998-07-01
		format = "2006-01-02"
	default:
		return time.Time{}
	}

	t, err := time.Parse(format,s)
	if err != nil {
		t = time.Time{}
	}
	return t
}
*/<|endoftext|>


<commit_before>package qingcloud

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	qc "github.com/yunify/qingcloud-sdk-go/service"
)

func resourceQingcloudVpc() *schema.Resource {
	return &schema.Resource{
		Create: resourceQingcloudVpcCreate,
		Read:   resourceQingcloudVpcRead,
		Update: resourceQingcloudVpcUpdate,
		Delete: resourceQingcloudVpcDelete,
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The name of Vpc",
			},
			"type": &schema.Schema{
				Type:         schema.TypeInt,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: withinArrayInt(0, 1, 2),
				Description: "Type of Vpc: 0 - medium,1 - small,2 - large,default 1	",
			},
			"vpc_network": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: withinArrayString("192.168.0.0/16", "172.16.0.0/16", "172.17.0.0/16",
					"172.18.0.0/16", "172.19.0.0/16", "172.20.0.0/16", "172.21.0.0/16", "172.22.0.0/16",
					"172.23.0.0/16", "172.24.0.0/16", "172.25.0.0/16"),
				Description: "Network address range of vpc.",
			},
			"tag_ids": &schema.Schema{
				Type:        schema.TypeSet,
				Optional:    true,
				Elem:        &schema.Schema{Type: schema.TypeString},
				Set:         schema.HashString,
				Description: "The tagids' id used by the vpc",
			},
			"tag_names": &schema.Schema{
				Type:        schema.TypeSet,
				Computed:    true,
				Elem:        &schema.Schema{Type: schema.TypeString},
				Set:         schema.HashString,
				Description: "compute by tag ids ",
			},
			"eip_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The eip's id used by the vpc",
			},
			"security_group_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The security group's id used by the vpc",
			},
			"description": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The description of vpc",
			},
			"private_ip": &schema.Schema{
				Type:        schema.TypeString,
				Computed:    true,
				Description: "The private ip of vpc",
			},
			"public_ip": &schema.Schema{
				Type:        schema.TypeString,
				Computed:    true,
				Description: "The public ip of vpc",
			},
		},
	}
}

// resourceQingcloudRouterCreate
func resourceQingcloudVpcCreate(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	input := new(qc.CreateRoutersInput)
	input.RouterName = qc.String(d.Get("name").(string))
	if d.Get("name").(string) != "" {
		input.RouterName = qc.String(d.Get("name").(string))
	}
	input.RouterType = qc.Int(d.Get("type").(int))
	if d.Get("vpc_network").(string) != "" {
		input.VpcNetwork = qc.String(d.Get("vpc_network").(string))
	}
	if d.Get("security_group_id").(string) != "" {
		input.SecurityGroup = qc.String(d.Get("security_group_id").(string))
	}
	input.Count = qc.Int(1)
	output, err := clt.CreateRouters(input)
	if err != nil {
		return fmt.Errorf("Error create router: %s", err.Error())
	}
	if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
		return fmt.Errorf("Error create router: %s", *output.Message)
	}
	d.SetId(qc.StringValue(output.Routers[0]))
	_, err = RouterTransitionStateRefresh(clt, d.Id())
	if err != nil {
		return fmt.Errorf("Error waiting for router (%s) to start: %s", d.Id(), err.Error())
	}
	return resourceQingcloudVpcUpdate(d, meta)
}

func resourceQingcloudVpcRead(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	input := new(qc.DescribeRoutersInput)
	input.Routers = []*string{qc.String(d.Id())}
	input.Verbose = qc.Int(1)
	output, err := clt.DescribeRouters(input)
	if err != nil {
		return fmt.Errorf("Error describe router: %s", err)
	}
	if *output.RetCode != 0 {
		return fmt.Errorf("Error describe router: %s", *output.Message)
	}
	if len(output.RouterSet) == 0 {
		d.SetId("")
		return nil
	}
	rtr := output.RouterSet[0]
	d.Set("name", qc.StringValue(rtr.RouterName))
	d.Set("type", qc.IntValue(rtr.RouterType))
	d.Set("security_group_id", qc.StringValue(rtr.SecurityGroupID))
	d.Set("description", qc.StringValue(rtr.Description))
	d.Set("private_ip", qc.StringValue(rtr.PrivateIP))
	d.Set("eip_id", qc.StringValue(rtr.EIP.EIPID))
	d.Set("public_ip", qc.StringValue(rtr.EIP.EIPAddr))
	return nil
}

func resourceQingcloudVpcUpdate(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	if _, err := RouterTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	waitRouterLease(d, meta)
	if err := modifyRouterAttributes(d, meta); err != nil {
		return err
	}
	if d.HasChange("eip_id") {
		input := new(qc.UpdateRoutersInput)
		input.Routers = []*string{qc.String(d.Id())}
		output, err := clt.UpdateRouters(input)
		if err != nil {
			return fmt.Errorf("Error update router: %s", err.Error())
		}
		if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
			return fmt.Errorf("Error update router: %s", *output.Message)
		}
		_, err = RouterTransitionStateRefresh(clt, d.Id())
		if err != nil {
			return fmt.Errorf("Error waiting for router (%s) to start: %s", d.Id(), err.Error())
		}
	}
	if d.HasChange("security_group_id") && !d.IsNewResource() {
		sgClt := meta.(*QingCloudClient).securitygroup
		input := new(qc.ApplySecurityGroupInput)
		input.SecurityGroup = qc.String(d.Get("security_group_id").(string))
		output, err := sgClt.ApplySecurityGroup(input)
		if err != nil {
			return fmt.Errorf("Error apply security group (%s) update %s", *input.SecurityGroup, *output.Message)
		}
	}
	if err := resourceUpdateTag(d, meta, qingcloudResourceTypeRouter); err != nil {
		return err
	}
	return resourceQingcloudRouterRead(d, meta)
}

func resourceQingcloudVpcDelete(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	if _, err := RouterTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	waitRouterLease(d, meta)
	input := new(qc.DeleteRoutersInput)
	input.Routers = []*string{qc.String(d.Id())}
	output, err := clt.DeleteRouters(input)
	if err != nil {
		return fmt.Errorf("Error delete router: %s", err)
	}
	if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
		return fmt.Errorf("Error delete router: %s", *output.Message)
	}
	if _, err := RouterTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	d.SetId("")
	return nil
}
<commit_msg>reformat code , Partial update vpc<commit_after>package qingcloud

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	qc "github.com/yunify/qingcloud-sdk-go/service"
)

func resourceQingcloudVpc() *schema.Resource {
	return &schema.Resource{
		Create: resourceQingcloudVpcCreate,
		Read:   resourceQingcloudVpcRead,
		Update: resourceQingcloudVpcUpdate,
		Delete: resourceQingcloudVpcDelete,
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The name of Vpc",
			},
			"type": &schema.Schema{
				Type:         schema.TypeInt,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: withinArrayInt(0, 1, 2),
				Description: "Type of Vpc: 0 - medium,1 - small,2 - large,default 1	",
			},
			"vpc_network": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: withinArrayString("192.168.0.0/16", "172.16.0.0/16", "172.17.0.0/16",
					"172.18.0.0/16", "172.19.0.0/16", "172.20.0.0/16", "172.21.0.0/16", "172.22.0.0/16",
					"172.23.0.0/16", "172.24.0.0/16", "172.25.0.0/16"),
				Description: "Network address range of vpc.",
			},
			"tag_ids": &schema.Schema{
				Type:        schema.TypeSet,
				Optional:    true,
				Elem:        &schema.Schema{Type: schema.TypeString},
				Set:         schema.HashString,
				Description: "The tag ids' id used by the vpc",
			},
			"tag_names": &schema.Schema{
				Type:        schema.TypeSet,
				Computed:    true,
				Elem:        &schema.Schema{Type: schema.TypeString},
				Set:         schema.HashString,
				Description: "compute by tag ids ",
			},
			"eip_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The eip's id used by the vpc",
			},
			"security_group_id": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				Description: "The security group's id used by the vpc",
			},
			"description": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The description of vpc",
			},
			"private_ip": &schema.Schema{
				Type:        schema.TypeString,
				Computed:    true,
				Description: "The private ip of vpc",
			},
			"public_ip": &schema.Schema{
				Type:        schema.TypeString,
				Computed:    true,
				Description: "The public ip of vpc",
			},
		},
	}
}

// resourceQingcloudRouterCreate
func resourceQingcloudVpcCreate(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	input := new(qc.CreateRoutersInput)
	input.RouterName = qc.String(d.Get("name").(string))
	if d.Get("name").(string) != "" {
		input.RouterName = qc.String(d.Get("name").(string))
	}
	input.RouterType = qc.Int(d.Get("type").(int))
	if d.Get("vpc_network").(string) != "" {
		input.VpcNetwork = qc.String(d.Get("vpc_network").(string))
	}
	if d.Get("security_group_id").(string) != "" {
		input.SecurityGroup = qc.String(d.Get("security_group_id").(string))
	}
	input.Count = qc.Int(1)
	output, err := clt.CreateRouters(input)
	if err != nil {
		return fmt.Errorf("Error create router: %s", err.Error())
	}
	if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
		return fmt.Errorf("Error create router: %s", *output.Message)
	}
	d.SetId(qc.StringValue(output.Routers[0]))
	_, err = RouterTransitionStateRefresh(clt, d.Id())
	if err != nil {
		return fmt.Errorf("Error waiting for router (%s) to start: %s", d.Id(), err.Error())
	}
	return resourceQingcloudVpcUpdate(d, meta)
}

func resourceQingcloudVpcRead(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	input := new(qc.DescribeRoutersInput)
	input.Routers = []*string{qc.String(d.Id())}
	input.Verbose = qc.Int(1)
	output, err := clt.DescribeRouters(input)
	if err != nil {
		return fmt.Errorf("Error describe router: %s", err)
	}
	if *output.RetCode != 0 {
		return fmt.Errorf("Error describe router: %s", *output.Message)
	}
	if len(output.RouterSet) == 0 {
		d.SetId("")
		return nil
	}
	rtr := output.RouterSet[0]
	d.Set("name", qc.StringValue(rtr.RouterName))
	d.Set("type", qc.IntValue(rtr.RouterType))
	d.Set("security_group_id", qc.StringValue(rtr.SecurityGroupID))
	d.Set("description", qc.StringValue(rtr.Description))
	d.Set("private_ip", qc.StringValue(rtr.PrivateIP))
	d.Set("eip_id", qc.StringValue(rtr.EIP.EIPID))
	d.Set("public_ip", qc.StringValue(rtr.EIP.EIPAddr))
	return nil
}

func resourceQingcloudVpcUpdate(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	if _, err := RouterTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	waitRouterLease(d, meta)
	d.Partial(true)
	if err := modifyRouterAttributes(d, meta); err != nil {
		return err
	}
	d.SetPartial("name")
	d.SetPartial("description")
	if d.HasChange("eip_id") {
		input := new(qc.UpdateRoutersInput)
		input.Routers = []*string{qc.String(d.Id())}
		output, err := clt.UpdateRouters(input)
		if err != nil {
			return fmt.Errorf("Error update router: %s", err.Error())
		}
		if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
			return fmt.Errorf("Error update router: %s", *output.Message)
		}
		_, err = RouterTransitionStateRefresh(clt, d.Id())
		if err != nil {
			return fmt.Errorf("Error waiting for router (%s) to start: %s", d.Id(), err.Error())
		}
	}
	d.SetPartial("eip_id")
	if d.HasChange("security_group_id") && !d.IsNewResource() {
		sgClt := meta.(*QingCloudClient).securitygroup
		input := new(qc.ApplySecurityGroupInput)
		input.SecurityGroup = qc.String(d.Get("security_group_id").(string))
		output, err := sgClt.ApplySecurityGroup(input)
		if err != nil {
			return fmt.Errorf("Error apply security group (%s) update %s", *input.SecurityGroup, *output.Message)
		}
	}
	d.SetPartial("security_group_id")
	if err := resourceUpdateTag(d, meta, qingcloudResourceTypeRouter); err != nil {
		return err
	}
	d.SetPartial("tag_ids")
	d.Partial(false)
	return resourceQingcloudRouterRead(d, meta)
}

func resourceQingcloudVpcDelete(d *schema.ResourceData, meta interface{}) error {
	clt := meta.(*QingCloudClient).router
	if _, err := RouterTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	waitRouterLease(d, meta)
	input := new(qc.DeleteRoutersInput)
	input.Routers = []*string{qc.String(d.Id())}
	output, err := clt.DeleteRouters(input)
	if err != nil {
		return fmt.Errorf("Error delete router: %s", err)
	}
	if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
		return fmt.Errorf("Error delete router: %s", *output.Message)
	}
	if _, err := RouterTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	d.SetId("")
	return nil
}
<|endoftext|>


<commit_before>package gcm

import (
	"bytes"
	"errors"
	"testing"

	"github.com/alistanis/goenc/generate"
	. "github.com/smartystreets/goconvey/convey"
)

var (
	text = "this is some text to encrypt"
)

func TestEncrypt(t *testing.T) {
	Convey("We can encrypt and decrypt messages using a key", t, func() {
		k, err := generate.Key()
		So(err, ShouldBeNil)
		key := k[:]
		data, err := Encrypt(key, []byte(text))
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeFalse)

		data, err = Decrypt(key, data)
		So(bytes.Equal([]byte(text), data), ShouldBeTrue)

		s, err := EncryptString(string(key), text)
		So(err, ShouldBeNil)
		So(s, ShouldNotEqual, text)

		s, err = DecryptString(string(key), s)
		So(err, ShouldBeNil)
		So(s, ShouldEqual, text)
	})
}

func TestEncryptWithID(t *testing.T) {
	Convey("We can encrypt and decrypt messages using an ID", t, func() {
		k, err := generate.Key()
		So(err, ShouldBeNil)
		key := k[:]

		id := uint32(1)

		keyFunc := func(uint32) ([]byte, error) {
			return key, nil
		}
		h := NewGCMHelper(keyFunc)
		data, err := EncryptWithID(key, []byte(text), id)
		So(err, ShouldBeNil)

		So(bytes.Equal([]byte(text), data), ShouldBeFalse)

		data, err = DecryptWithID(data, h)
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeTrue)
		s := "You can't read this"
		s1, err := EncryptStringWithID(string(key), s, 47)
		So(err, ShouldBeNil)
		So(s, ShouldNotEqual, s1)

		s1, err = DecryptStringWithID(s1, h)
		So(err, ShouldBeNil)
		So(s1, ShouldEqual, s)

	})
}

func TestCipher_Encrypt(t *testing.T) {
	Convey("We can encrypt and decrypt messages using a cipher struct", t, func() {
		c := New()
		So(c.KeySize(), ShouldEqual, KeySize)
		k, err := generate.Key()
		So(err, ShouldBeNil)
		key := k[:]
		data, err := c.Encrypt(key, []byte(text))
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeFalse)

		data, err = c.Decrypt(key, data)
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeTrue)
	})
}

var errTestError = errors.New("test error")

type R struct{}

func (r *R) KeyForID(u uint32) ([]byte, error) {
	return nil, errTestError
}

type T struct{}

func (t *T) KeyForID(u uint32) ([]byte, error) {
	return []byte{}, nil
}

func TestErrors(t *testing.T) {
	Convey("We can get appropriate errors", t, func() {
		k, err := generate.RandBytes(15)
		So(err, ShouldBeNil)
		key := k[:]
		errText := "crypto/aes: invalid key size 15"
		_, err = Encrypt(key, []byte{})
		So(err.Error(), ShouldEqual, errText)

		_, err = EncryptWithID(key, []byte{}, 1)
		So(err.Error(), ShouldEqual, errText)

		_, err = DecryptWithID([]byte{}, nil)

		data, err := generate.RandBytes(NonceSize + 5)
		So(err, ShouldBeNil)
		_, err = DecryptWithID(data, &R{})
		So(err, ShouldEqual, errTestError)

		_, err = DecryptWithID(data, &T{})
		So(err.Error(), ShouldEqual, "crypto/aes: invalid key size 0")

	})
}
<commit_msg>cover missed case in gcm<commit_after>package gcm

import (
	"bytes"
	"errors"
	"testing"

	"github.com/alistanis/goenc/generate"
	. "github.com/smartystreets/goconvey/convey"
)

var (
	text = "this is some text to encrypt"
)

func TestEncrypt(t *testing.T) {
	Convey("We can encrypt and decrypt messages using a key", t, func() {
		k, err := generate.Key()
		So(err, ShouldBeNil)
		key := k[:]
		data, err := Encrypt(key, []byte(text))
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeFalse)

		data, err = Decrypt(key, data)
		So(bytes.Equal([]byte(text), data), ShouldBeTrue)

		s, err := EncryptString(string(key), text)
		So(err, ShouldBeNil)
		So(s, ShouldNotEqual, text)

		s, err = DecryptString(string(key), s)
		So(err, ShouldBeNil)
		So(s, ShouldEqual, text)
	})
}

func TestEncryptWithID(t *testing.T) {
	Convey("We can encrypt and decrypt messages using an ID", t, func() {
		k, err := generate.Key()
		So(err, ShouldBeNil)
		key := k[:]

		id := uint32(1)

		keyFunc := func(uint32) ([]byte, error) {
			return key, nil
		}
		h := NewGCMHelper(keyFunc)
		data, err := EncryptWithID(key, []byte(text), id)
		So(err, ShouldBeNil)

		So(bytes.Equal([]byte(text), data), ShouldBeFalse)

		data, err = DecryptWithID(data, h)
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeTrue)
		s := "You can't read this"
		s1, err := EncryptStringWithID(string(key), s, 47)
		So(err, ShouldBeNil)
		So(s, ShouldNotEqual, s1)

		s1, err = DecryptStringWithID(s1, h)
		So(err, ShouldBeNil)
		So(s1, ShouldEqual, s)

	})
}

func TestCipher_Encrypt(t *testing.T) {
	Convey("We can encrypt and decrypt messages using a cipher struct", t, func() {
		c := New()
		So(c.KeySize(), ShouldEqual, KeySize)
		k, err := generate.Key()
		So(err, ShouldBeNil)
		key := k[:]
		data, err := c.Encrypt(key, []byte(text))
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeFalse)

		data, err = c.Decrypt(key, data)
		So(err, ShouldBeNil)
		So(bytes.Equal([]byte(text), data), ShouldBeTrue)
	})
}

var errTestError = errors.New("test error")

type R struct{}

func (r *R) KeyForID(u uint32) ([]byte, error) {
	return nil, errTestError
}

type T struct{}

func (t *T) KeyForID(u uint32) ([]byte, error) {
	return []byte{}, nil
}

func TestErrors(t *testing.T) {
	Convey("We can get appropriate errors", t, func() {
		k, err := generate.RandBytes(15)
		So(err, ShouldBeNil)
		key := k[:]
		errText := "crypto/aes: invalid key size 15"
		_, err = Encrypt(key, []byte{})
		So(err.Error(), ShouldEqual, errText)

		_, err = Decrypt(key, []byte{})
		So(err, ShouldNotBeNil)

		_, err = EncryptWithID(key, []byte{}, 1)
		So(err.Error(), ShouldEqual, errText)

		_, err = DecryptWithID([]byte{}, nil)
		So(err, ShouldNotBeNil)
		data, err := generate.RandBytes(NonceSize + 5)
		So(err, ShouldBeNil)
		_, err = DecryptWithID(data, &R{})
		So(err, ShouldEqual, errTestError)

		_, err = DecryptWithID(data, &T{})
		So(err.Error(), ShouldEqual, "crypto/aes: invalid key size 0")

	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Command for running the agent.

package main

import (
	"flag"
	"github.com/romana/core/agent"
)

// main function is entrypoint to everything.
func main() {
	var rootUrl = flag.String("rootUrl", "", "URL to root service URL")
	flag.Parse()
	if (rootUrl == nil)  {
		fmt.Println("Must specify rootUrl.")
		return
	}
	channel, err := agent.Run(rootUrl)
	if err != nil {
		panic(err)
	}
	for {
		msg := <-channel
		fmt.Println(msg)
	}
}
<commit_msg>Fix imports and pointers<commit_after>// Copyright (c) 2015 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Command for running the agent.

package main

import (
	"flag"
	"fmt"
	"github.com/romana/core/agent"
)

// main function is entrypoint to everything.
func main() {
	var rootUrl = flag.String("rootUrl", "", "URL to root service URL")
	flag.Parse()
	if rootUrl == nil {
		fmt.Println("Must specify rootUrl.")
		return
	}
	channel, err := agent.Run(*rootUrl)
	if err != nil {
		panic(err)
	}
	for {
		msg := <-channel
		fmt.Println(msg)
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

type templ struct {
	name    string
	file    string
	version string
	branch  string
	id      string
	hash    string
}

func templId(t templ, arch string, kurjun *http.Client) string {
	url := config.Cdn.Kurjun + "/template/info?name=" + t.name + "&type=text"
	if t.name == "management" && len(t.branch) != 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "-" + t.branch + "&type=text"
	} else if t.name == "management" {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug("Retrieving id, get: " + url)

	if err == nil && response.StatusCode == 204 && t.name == "management" {
		log.Warn("Cannot get management with specified version, trying without version")
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/info?name=" + t.name + "&type=text")
	}
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(t templ) bool {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		if t.file == f.Name() {
			if len(t.hash) == 0 {
				fmt.Print("Cannot check md5 of local archive. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return true
				}
				return false
			}
			if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return true
			}
		}
	}
	return false
}

func download(t templ, kurjun *http.Client) bool {
	if len(t.id) == 0 {
		return false
	}
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
	log.Check(log.FatalLevel, "Creating file "+t.file, err)
	defer out.Close()
	log.Info("Downloading " + t.name)

	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + t.id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+t.id, err)
	defer response.Body.Close()
	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	for c := 0; err != nil && c < 5; _, err = io.Copy(out, rd) {
		log.Info("Download interrupted, retrying")
		time.Sleep(3 * time.Second)
		c++

		//Repeating GET request to CDN, while need to continue interrupted download
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + t.id)
		log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+t.id, err)
		defer response.Body.Close()
		bar = pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
		bar.Start()
		rd = bar.NewProxyReader(response.Body)
	}

	log.Check(log.FatalLevel, "Writing response body to file", err)

	if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+t.file) {
		return true
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return false
}

func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(name, version, token string) {
	if container.IsContainer(name) && name == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	log.Info("Importing " + name)
	for !lockSubutai(name + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	if container.IsContainer(name) {
		log.Info(name + " instance exist")
		return
	}

	var t templ

	t.name = name
	t.version = config.Template.Version
	t.branch = config.Template.Branch
	if len(version) != 0 {
		if strings.Contains(version, "-") {
			verstr := strings.Split(version, "-")
			if len(verstr) == 2 {
				t.version = verstr[0]
				t.branch = verstr[1]
			} else {
				log.Error("Invalid version")
			}
		} else {
			if strings.Contains(version, ".") {
				t.version = version
				t.branch = config.Template.Branch
			} else {
				t.version = config.Template.Version
				t.branch = version
			}
		}
	}

	log.Info("Version: " + t.version + ", branch: " + t.branch)

	if t.branch == "" || t.name != "management" {
		t.file = t.name + "-subutai-template_" + t.version + "_" + config.Template.Arch + ".tar.gz"
	} else {
		t.file = t.name + "-subutai-template_" + t.version + "-" + t.branch + "_" + config.Template.Arch + ".tar.gz"
	}

	kurjun := config.CheckKurjun()
	t.id = templId(t, runtime.GOARCH, kurjun)
	if len(strings.Split(t.id, ".")) > 1 {
		t.hash = strings.Split(t.id, ".")[1]
	}

	if !checkLocal(t) && !download(t, kurjun) {
		log.Error(t.name + " template not found")
	}

	log.Info("Unpacking template " + t.name)
	tgz := extractor.NewTgz()
	templdir := config.Agent.LxcPrefix + "tmpdir/" + t.name
	tgz.Extract(config.Agent.LxcPrefix+"tmpdir/"+t.file, templdir)
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != t.name && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + t.name)
	template.Install(parent, t.name)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-home", config.Agent.LxcPrefix+t.name+"/home")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-var", config.Agent.LxcPrefix+t.name+"/var")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-opt", config.Agent.LxcPrefix+t.name+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if t.name == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(t.name, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + t.name + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + t.name + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/var var none bind,rw 0 0"},
	})
}
<commit_msg>Repeated import bug fix<commit_after>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

type templ struct {
	name    string
	file    string
	version string
	branch  string
	id      string
	hash    string
}

func templId(t templ, arch string, kurjun *http.Client) string {
	url := config.Cdn.Kurjun + "/template/info?name=" + t.name + "&type=text"
	if t.name == "management" && len(t.branch) != 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "-" + t.branch + "&type=text"
	} else if t.name == "management" {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug("Retrieving id, get: " + url)

	if err == nil && response.StatusCode == 204 && t.name == "management" {
		log.Warn("Cannot get management with specified version, trying without version")
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/info?name=" + t.name + "&type=text")
	}
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(t templ) bool {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		if t.file == f.Name() {
			if len(t.hash) == 0 {
				fmt.Print("Cannot check md5 of local archive. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return true
				}
				return false
			}
			if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return true
			}
		}
	}
	return false
}

func download(t templ, kurjun *http.Client) bool {
	if len(t.id) == 0 {
		return false
	}
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
	log.Check(log.FatalLevel, "Creating file "+t.file, err)
	defer out.Close()
	log.Info("Downloading " + t.name)

	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + t.id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+t.id, err)
	defer response.Body.Close()
	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	for c := 0; err != nil && c < 5; _, err = io.Copy(out, rd) {
		log.Info("Download interrupted, retrying")
		time.Sleep(3 * time.Second)
		c++

		//Repeating GET request to CDN, while need to continue interrupted download
		out, err = os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
		log.Check(log.FatalLevel, "Creating file "+t.file, err)
		defer out.Close()
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + t.id)
		log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+t.id, err)
		defer response.Body.Close()
		bar = pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
		bar.Start()
		rd = bar.NewProxyReader(response.Body)
	}

	log.Check(log.FatalLevel, "Writing response body to file", err)

	if t.hash == md5sum(config.Agent.LxcPrefix+"tmpdir/"+t.file) {
		return true
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return false
}

func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(name, version, token string) {
	if container.IsContainer(name) && name == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	log.Info("Importing " + name)
	for !lockSubutai(name + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	if container.IsContainer(name) {
		log.Info(name + " instance exist")
		return
	}

	var t templ

	t.name = name
	t.version = config.Template.Version
	t.branch = config.Template.Branch
	if len(version) != 0 {
		if strings.Contains(version, "-") {
			verstr := strings.Split(version, "-")
			if len(verstr) == 2 {
				t.version = verstr[0]
				t.branch = verstr[1]
			} else {
				log.Error("Invalid version")
			}
		} else {
			if strings.Contains(version, ".") {
				t.version = version
				t.branch = config.Template.Branch
			} else {
				t.version = config.Template.Version
				t.branch = version
			}
		}
	}

	log.Info("Version: " + t.version + ", branch: " + t.branch)

	if t.branch == "" || t.name != "management" {
		t.file = t.name + "-subutai-template_" + t.version + "_" + config.Template.Arch + ".tar.gz"
	} else {
		t.file = t.name + "-subutai-template_" + t.version + "-" + t.branch + "_" + config.Template.Arch + ".tar.gz"
	}

	kurjun := config.CheckKurjun()
	t.id = templId(t, runtime.GOARCH, kurjun)
	if len(strings.Split(t.id, ".")) > 1 {
		t.hash = strings.Split(t.id, ".")[1]
	}

	if !checkLocal(t) && !download(t, kurjun) {
		log.Error(t.name + " template not found")
	}

	log.Info("Unpacking template " + t.name)
	tgz := extractor.NewTgz()
	templdir := config.Agent.LxcPrefix + "tmpdir/" + t.name
	tgz.Extract(config.Agent.LxcPrefix+"tmpdir/"+t.file, templdir)
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != t.name && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + t.name)
	template.Install(parent, t.name)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-home", config.Agent.LxcPrefix+t.name+"/home")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-var", config.Agent.LxcPrefix+t.name+"/var")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-opt", config.Agent.LxcPrefix+t.name+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if t.name == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(t.name, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + t.name + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + t.name + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/var var none bind,rw 0 0"},
	})
}
<|endoftext|>