<commit_before>package kasper

import (
	"strconv"

	"github.com/Shopify/sarama"
)

type partitionProcessor struct {
	topicProcessor     *TopicProcessor
	coordinator        Coordinator
	consumer           sarama.Consumer
	partitionConsumers []sarama.PartitionConsumer
	offsetManagers     map[string]sarama.PartitionOffsetManager
	messageProcessor   MessageProcessor
	inputTopics        []string
	partition          int
}

func (pp *partitionProcessor) consumerMessageChannels() []<-chan *sarama.ConsumerMessage {
	chans := make([]<-chan *sarama.ConsumerMessage, len(pp.partitionConsumers))
	for i, consumer := range pp.partitionConsumers {
		chans[i] = consumer.Messages()
	}
	return chans
}

func getPartitionOffsetManager(tp *TopicProcessor, topic string, partition int) sarama.PartitionOffsetManager {
	pom, err := tp.offsetManager.ManagePartition(topic, int32(partition))
	if err != nil {
		logger.Panic(err)
	}
	return pom
}

func getPartitionConsumer(tp *TopicProcessor, consumer sarama.Consumer, pom sarama.PartitionOffsetManager, topic string, partition int) sarama.PartitionConsumer {
	newestOffset, err := tp.config.Client.GetOffset(topic, int32(partition), sarama.OffsetNewest)
	if err != nil {
		logger.Panic(err)
	}
	nextOffset, _ := pom.NextOffset()
	if nextOffset > newestOffset {
		nextOffset = sarama.OffsetNewest
	}
	c, err := consumer.ConsumePartition(topic, int32(partition), nextOffset)
	if err != nil {
		logger.Panic(err)
	}
	return c
}

func newPartitionProcessor(tp *TopicProcessor, bmp MessageProcessor, partition int) *partitionProcessor {
	consumer, err := sarama.NewConsumerFromClient(tp.config.Client)
	if err != nil {
		logger.Panic(err)
	}
	partitionConsumers := make([]sarama.PartitionConsumer, len(tp.inputTopics))
	partitionOffsetManagers := make(map[string]sarama.PartitionOffsetManager)
	for i, topic := range tp.inputTopics {
		partitionOffsetManager := getPartitionOffsetManager(tp, topic, partition)
		partitionConsumer := getPartitionConsumer(tp, consumer, partitionOffsetManager, topic, partition)
		partitionConsumers[i] = partitionConsumer
		partitionOffsetManagers[topic] = partitionOffsetManager
	}
	pp := &partitionProcessor{
		tp,
		nil,
		consumer,
		partitionConsumers,
		partitionOffsetManagers,
		bmp,
		tp.inputTopics,
		partition,
	}
	pp.coordinator = &partitionProcessorCoordinator{pp}
	return pp
}

func (pp *partitionProcessor) process(msgs []*sarama.ConsumerMessage) []*sarama.ProducerMessage {
	sender := newSender(pp)
	pp.messageProcessor.Process(msgs, sender, pp.coordinator)
	return sender.producerMessages
}

func (pp *partitionProcessor) countMessagesBehindHighWaterMark() {
	partition := strconv.Itoa(pp.partition)
	highWaterMarks := pp.consumer.HighWaterMarks()
	for _, topic := range pp.topicProcessor.inputTopics {
		offsetManager := pp.offsetManagers[topic]
		currentOffset, _ := offsetManager.NextOffset()
		highWaterMark := highWaterMarks[topic][int32(pp.partition)]
		if currentOffset == sarama.OffsetNewest {
			pp.topicProcessor.messagesBehindHighWaterMark.Set(0, topic, partition)
		} else if currentOffset != sarama.OffsetOldest {
			messagesBehindHighWaterMark := highWaterMark - currentOffset
			pp.topicProcessor.messagesBehindHighWaterMark.Set(float64(messagesBehindHighWaterMark), topic, partition)
		}
	}
}

func (pp *partitionProcessor) hasConsumedAllMessages() bool {
	highWaterMarks := pp.consumer.HighWaterMarks()
	for _, topic := range pp.topicProcessor.inputTopics {
		offsetManager := pp.offsetManagers[topic]
		currentOffset, _ := offsetManager.NextOffset()
		highWaterMark := highWaterMarks[topic][int32(pp.partition)]
		if highWaterMark != currentOffset {
			logger.Debugf("Topic %s partition %d has messages remaining to consume (offset = %d, hight water mark = %d)", topic, pp.partition, currentOffset, highWaterMark)
			return false
		}
	}
	logger.Debug("Partitions %d of all input topics have been consumed", pp.partition)
	return true
}

func (pp *partitionProcessor) onMetricsTick() {
	pp.countMessagesBehindHighWaterMark()
}

func (pp *partitionProcessor) markOffsets(messages []*sarama.ConsumerMessage) {
	latestOffset := make(map[string]int64)
	for _, message := range messages {
		latestOffset[message.Topic] = message.Offset
	}
	for topic, offset := range latestOffset {
		logger.Debugf("Marking offset %s:%d", topic, offset+1)
		pp.offsetManagers[topic].MarkOffset(offset+1, "")
	}
}

func (pp *partitionProcessor) onClose() {
	var err error
	for _, pom := range pp.offsetManagers {
		err = pom.Close()
		if err != nil {
			logger.Panicf("Cannot close offset manager: %s", err)
		}

	}
	for _, pc := range pp.partitionConsumers {
		err = pc.Close()
		if err != nil {
			logger.Panicf("Cannot close partition consumer: %s", err)
		}
	}
	err = pp.consumer.Close()
	if err != nil {
		logger.Panic(err)
	}
}
<commit_msg>Renaming<commit_after>package kasper

import (
	"strconv"

	"github.com/Shopify/sarama"
)

type partitionProcessor struct {
	topicProcessor     *TopicProcessor
	coordinator        Coordinator
	consumer           sarama.Consumer
	partitionConsumers []sarama.PartitionConsumer
	offsetManagers     map[string]sarama.PartitionOffsetManager
	messageProcessor   MessageProcessor
	inputTopics        []string
	partition          int
}

func (pp *partitionProcessor) consumerMessageChannels() []<-chan *sarama.ConsumerMessage {
	chans := make([]<-chan *sarama.ConsumerMessage, len(pp.partitionConsumers))
	for i, consumer := range pp.partitionConsumers {
		chans[i] = consumer.Messages()
	}
	return chans
}

func getPartitionOffsetManager(tp *TopicProcessor, topic string, partition int) sarama.PartitionOffsetManager {
	pom, err := tp.offsetManager.ManagePartition(topic, int32(partition))
	if err != nil {
		logger.Panic(err)
	}
	return pom
}

func getPartitionConsumer(tp *TopicProcessor, consumer sarama.Consumer, pom sarama.PartitionOffsetManager, topic string, partition int) sarama.PartitionConsumer {
	newestOffset, err := tp.config.Client.GetOffset(topic, int32(partition), sarama.OffsetNewest)
	if err != nil {
		logger.Panic(err)
	}
	nextOffset, _ := pom.NextOffset()
	if nextOffset > newestOffset {
		nextOffset = sarama.OffsetNewest
	}
	c, err := consumer.ConsumePartition(topic, int32(partition), nextOffset)
	if err != nil {
		logger.Panic(err)
	}
	return c
}

func newPartitionProcessor(tp *TopicProcessor, mp MessageProcessor, partition int) *partitionProcessor {
	consumer, err := sarama.NewConsumerFromClient(tp.config.Client)
	if err != nil {
		logger.Panic(err)
	}
	partitionConsumers := make([]sarama.PartitionConsumer, len(tp.inputTopics))
	partitionOffsetManagers := make(map[string]sarama.PartitionOffsetManager)
	for i, topic := range tp.inputTopics {
		partitionOffsetManager := getPartitionOffsetManager(tp, topic, partition)
		partitionConsumer := getPartitionConsumer(tp, consumer, partitionOffsetManager, topic, partition)
		partitionConsumers[i] = partitionConsumer
		partitionOffsetManagers[topic] = partitionOffsetManager
	}
	pp := &partitionProcessor{
		tp,
		nil,
		consumer,
		partitionConsumers,
		partitionOffsetManagers,
		mp,
		tp.inputTopics,
		partition,
	}
	pp.coordinator = &partitionProcessorCoordinator{pp}
	return pp
}

func (pp *partitionProcessor) process(msgs []*sarama.ConsumerMessage) []*sarama.ProducerMessage {
	sender := newSender(pp)
	pp.messageProcessor.Process(msgs, sender, pp.coordinator)
	return sender.producerMessages
}

func (pp *partitionProcessor) countMessagesBehindHighWaterMark() {
	partition := strconv.Itoa(pp.partition)
	highWaterMarks := pp.consumer.HighWaterMarks()
	for _, topic := range pp.topicProcessor.inputTopics {
		offsetManager := pp.offsetManagers[topic]
		currentOffset, _ := offsetManager.NextOffset()
		highWaterMark := highWaterMarks[topic][int32(pp.partition)]
		if currentOffset == sarama.OffsetNewest {
			pp.topicProcessor.messagesBehindHighWaterMark.Set(0, topic, partition)
		} else if currentOffset != sarama.OffsetOldest {
			messagesBehindHighWaterMark := highWaterMark - currentOffset
			pp.topicProcessor.messagesBehindHighWaterMark.Set(float64(messagesBehindHighWaterMark), topic, partition)
		}
	}
}

func (pp *partitionProcessor) hasConsumedAllMessages() bool {
	highWaterMarks := pp.consumer.HighWaterMarks()
	for _, topic := range pp.topicProcessor.inputTopics {
		offsetManager := pp.offsetManagers[topic]
		currentOffset, _ := offsetManager.NextOffset()
		highWaterMark := highWaterMarks[topic][int32(pp.partition)]
		if highWaterMark != currentOffset {
			logger.Debugf("Topic %s partition %d has messages remaining to consume (offset = %d, hight water mark = %d)", topic, pp.partition, currentOffset, highWaterMark)
			return false
		}
	}
	logger.Debug("Partitions %d of all input topics have been consumed", pp.partition)
	return true
}

func (pp *partitionProcessor) onMetricsTick() {
	pp.countMessagesBehindHighWaterMark()
}

func (pp *partitionProcessor) markOffsets(messages []*sarama.ConsumerMessage) {
	latestOffset := make(map[string]int64)
	for _, message := range messages {
		latestOffset[message.Topic] = message.Offset
	}
	for topic, offset := range latestOffset {
		logger.Debugf("Marking offset %s:%d", topic, offset+1)
		pp.offsetManagers[topic].MarkOffset(offset+1, "")
	}
}

func (pp *partitionProcessor) onClose() {
	var err error
	for _, pom := range pp.offsetManagers {
		err = pom.Close()
		if err != nil {
			logger.Panicf("Cannot close offset manager: %s", err)
		}

	}
	for _, pc := range pp.partitionConsumers {
		err = pc.Close()
		if err != nil {
			logger.Panicf("Cannot close partition consumer: %s", err)
		}
	}
	err = pp.consumer.Close()
	if err != nil {
		logger.Panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	kftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfapp/coordinator"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfupgrade"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/utils"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var applyCfg = viper.New()
var kfApp kftypes.KfApp
var err error

// KFDef example configs to be printed out from apply --help
const (
	arritkoConfig = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_existing_arrikto.0.7.0.yaml"
	awsConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_aws.0.7.0.yaml"
	gcpConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_gcp_iap.0.7.0.yaml"
	k8sConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_k8s_istio.0.7.0.yaml"
)

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply -f ${CONFIG}",
	Short: "deploys a kubeflow application.",
	Long: `'kfctl apply' builds and deploys a kubeflow application from a KFDef config.` + "\n" +
		`To install run -> ` + ColorPrint("kfctl apply -f ${CONFIG}") + "\n" +
		`For more information, run 'kfctl apply -h' or read the docs at www.kubeflow.org.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		log.SetLevel(log.InfoLevel)
		if applyCfg.GetBool(string(kftypes.VERBOSE)) != true {
			log.SetLevel(log.WarnLevel)
		}

		// Load config from exisiting app.yaml
		if configFilePath == "" {
			return fmt.Errorf("Must pass in -f configFile")
		}

		kind, err := utils.GetObjectKindFromUri(configFilePath)
		if err != nil {
			return fmt.Errorf("Cannot determine the object kind: %v", err)
		}
		switch kind {
		case string(kftypes.KFDEF):
			kfApp, err = coordinator.NewLoadKfAppFromURI(configFilePath)
			if err != nil {
				return fmt.Errorf("failed to build kfApp from URI %s: %v", configFilePath, err)
			}
			return kfApp.Apply(kftypes.ALL)
		case string(kftypes.KFUPGRADE):
			kfUpgrade, err := kfupgrade.NewKfUpgrade(configFilePath)
			if err != nil {
				return fmt.Errorf("couldn't load KfUpgrade: %v", err)
			}

			err = kfUpgrade.Apply()
			if err != nil {
				return fmt.Errorf("couldn't apply KfUpgrade: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("Unsupported object kind: %v", kind)
		}
	},
}

func init() {
	rootCmd.AddCommand(applyCmd)

	applyCfg.SetConfigName("app")
	applyCfg.SetConfigType("yaml")

	// Config file option
	applyCmd.PersistentFlags().StringVarP(&configFilePath, string(kftypes.FILE), "f", "",
		`Static config file to use. Can be either a local path:
		export CONFIG=./kfctl_gcp_iap.yaml
	or a URL:
		export CONFIG=`+gcpConfig+`
		export CONFIG=`+arritkoConfig+`
		export CONFIG=`+awsConfig+`
		export CONFIG=`+k8sConfig+`
	kfctl apply -V --file=${CONFIG}`)

	// verbose output
	applyCmd.Flags().BoolP(string(kftypes.VERBOSE), "V", false,
		string(kftypes.VERBOSE)+" output default is false")
	bindErr := applyCfg.BindPFlag(string(kftypes.VERBOSE), applyCmd.Flags().Lookup(string(kftypes.VERBOSE)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.VERBOSE), bindErr)
		return
	}
}
<commit_msg>Add a message after kfctl apply succeeds (#4443)<commit_after>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	kftypes "github.com/kubeflow/kubeflow/bootstrap/v3/pkg/apis/apps"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfapp/coordinator"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/kfupgrade"
	"github.com/kubeflow/kubeflow/bootstrap/v3/pkg/utils"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var applyCfg = viper.New()
var kfApp kftypes.KfApp
var err error

// KFDef example configs to be printed out from apply --help
const (
	arritkoConfig = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_existing_arrikto.0.7.0.yaml"
	awsConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_aws.0.7.0.yaml"
	gcpConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_gcp_iap.0.7.0.yaml"
	k8sConfig     = "https://raw.githubusercontent.com/kubeflow/manifests/v0.7-branch/kfdef/kfctl_k8s_istio.0.7.0.yaml"
)

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply -f ${CONFIG}",
	Short: "deploys a kubeflow application.",
	Long: `'kfctl apply' builds and deploys a kubeflow application from a KFDef config.` + "\n" +
		`To install run -> ` + ColorPrint("kfctl apply -f ${CONFIG}") + "\n" +
		`For more information, run 'kfctl apply -h' or read the docs at www.kubeflow.org.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		log.SetLevel(log.InfoLevel)
		if applyCfg.GetBool(string(kftypes.VERBOSE)) != true {
			log.SetLevel(log.WarnLevel)
		}

		// Load config from exisiting app.yaml
		if configFilePath == "" {
			return fmt.Errorf("Must pass in -f configFile")
		}

		kind, err := utils.GetObjectKindFromUri(configFilePath)
		if err != nil {
			return fmt.Errorf("Cannot determine the object kind: %v", err)
		}
		switch kind {
		case string(kftypes.KFDEF):
			kfApp, err = coordinator.NewLoadKfAppFromURI(configFilePath)
			if err != nil {
				return fmt.Errorf("failed to build kfApp from URI %s: %v", configFilePath, err)
			}
			if err := kfApp.Apply(kftypes.ALL); err != nil {
				return fmt.Errorf("failed to apply: %s", err)
			}
			log.Info("Applied the configuration Successfully!")
			return nil
		case string(kftypes.KFUPGRADE):
			kfUpgrade, err := kfupgrade.NewKfUpgrade(configFilePath)
			if err != nil {
				return fmt.Errorf("couldn't load KfUpgrade: %v", err)
			}

			err = kfUpgrade.Apply()
			if err != nil {
				return fmt.Errorf("couldn't apply KfUpgrade: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("Unsupported object kind: %v", kind)
		}
	},
}

func init() {
	rootCmd.AddCommand(applyCmd)

	applyCfg.SetConfigName("app")
	applyCfg.SetConfigType("yaml")

	// Config file option
	applyCmd.PersistentFlags().StringVarP(&configFilePath, string(kftypes.FILE), "f", "",
		`Static config file to use. Can be either a local path:
		export CONFIG=./kfctl_gcp_iap.yaml
	or a URL:
		export CONFIG=`+gcpConfig+`
		export CONFIG=`+arritkoConfig+`
		export CONFIG=`+awsConfig+`
		export CONFIG=`+k8sConfig+`
	kfctl apply -V --file=${CONFIG}`)

	// verbose output
	applyCmd.Flags().BoolP(string(kftypes.VERBOSE), "V", false,
		string(kftypes.VERBOSE)+" output default is false")
	bindErr := applyCfg.BindPFlag(string(kftypes.VERBOSE), applyCmd.Flags().Lookup(string(kftypes.VERBOSE)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.VERBOSE), bindErr)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package upstream

import (
	"encoding/binary"
	"fmt"
	"sync"
	"time"

	zmq "github.com/pebbe/zmq4"

	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/blockdigest"
	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
)

const (
	cycleInterval = 30 * time.Second
	queueSize     = 50 // 0 => synchronous queue
)

type Upstream struct {
	sync.RWMutex
	log         *logger.L
	client      *zmqutil.Client
	registered  bool
	blockHeight uint64
	shutdown    chan<- struct{}
}

// atomically incremented counter for log names
var upstreamCounter counter.Counter

func New(privateKey []byte, publicKey []byte, timeout time.Duration) (*Upstream, error) {
	client, err := zmqutil.NewClient(zmq.REQ, privateKey, publicKey, timeout)
	if nil != err {
		return nil, err
	}

	n := upstreamCounter.Increment()

	shutdown := make(chan struct{})
	u := &Upstream{
		log:         logger.New(fmt.Sprintf("upstream@%d", n)),
		client:      client,
		registered:  false,
		blockHeight: 0,
		shutdown:    shutdown,
	}
	go upstreamRunner(u, shutdown)
	return u, nil
}

func (u *Upstream) Destroy() {
	if nil != u {
		close(u.shutdown)
	}
}

// check the current destination
//
// does not mean actually connected, as could be in a timeout and
// reconnect state
func (u *Upstream) IsConnectedTo(serverPublicKey []byte) bool {
	return u.client.IsConnectedTo(serverPublicKey)
}

// if registered the have avalid connection
func (u *Upstream) IsOK() bool {
	return u.registered
}

// if registered the have avalid connection
func (u *Upstream) ConnectedTo() *zmqutil.Connected {
	return u.client.ConnectedTo()
}

// connect (or reconnect) to a specific server
func (u *Upstream) Connect(address *util.Connection, serverPublicKey []byte) error {
	u.log.Infof("connecting to address: %s", address)
	u.log.Infof("connecting to server: %x", serverPublicKey)
	u.Lock()
	err := u.client.Connect(address, serverPublicKey, mode.ChainName())
	if nil == err {
		err = register(u.client, u.log)
	}
	u.Unlock()
	return err
}

// fetch height from last polled value
func (u *Upstream) GetHeight() uint64 {
	return u.blockHeight
}

// fetch block digest
func (u *Upstream) GetBlockDigest(blockNumber uint64) (blockdigest.Digest, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("H", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return blockdigest.Digest{}, err
	}

	if 2 != len(data) {
		return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return blockdigest.Digest{}, fault.InvalidError(string(data[1]))
	case "H":
		d := blockdigest.Digest{}
		if blockdigest.Length == len(data[1]) {
			err := blockdigest.DigestFromBytes(&d, data[1])
			return d, err
		}
	default:
	}
	return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
}

// fetch block data
func (u *Upstream) GetBlockData(blockNumber uint64) ([]byte, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("B", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return nil, err
	}

	if 2 != len(data) {
		return nil, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return nil, fault.InvalidError(string(data[1]))
	case "B":
		return data[1], nil
	default:
	}
	return nil, fault.ErrInvalidPeerResponse
}

func upstreamRunner(u *Upstream, shutdown <-chan struct{}) {
	log := u.log

	log.Debug("starting…")

	queue := messagebus.Bus.Broadcast.Chan(queueSize)

loop:
	for {
		log.Debug("waiting…")

		select {
		case <-shutdown:
			break loop

		case item := <-queue:
			log.Debugf("from queue: %q  %x", item.Command, item.Parameters)
			if u.registered {
				u.Lock()
				err := push(u.client, u.log, &item)
				if nil != err {
					log.Errorf("push: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("push: reconnect error: %s", err)
					}
				}
				u.Unlock()
			}

		case <-time.After(cycleInterval):
			u.Lock()
			if !u.registered {
				err := register(u.client, u.log)
				if fault.ErrNotConnected == err {
					log.Infof("register: %s", err)
					u.Unlock()
					continue loop // try again later
				} else if nil != err {
					log.Errorf("register: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("register: reconnect error: %s", err)
					}
					u.Unlock()
					continue loop // try again later
				}
				u.registered = true
			}

			h, err := getHeight(u.client, u.log)
			if nil == err {
				u.blockHeight = h
			} else {
				u.registered = false
				log.Errorf("getHeight: error: %s", err)
				err := u.client.Reconnect()
				if nil != err {
					log.Errorf("highestBlock: reconnect error: %s", err)
				}
			}
			u.Unlock()
		}
	}
	log.Info("shutting down…")
	u.client.Close()
	log.Info("stopped")
}

func register(client *zmqutil.Client, log *logger.L) error {

	log.Debugf("register: client: %s", client)

	err := announce.SendRegistration(client, "R")
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}

	if len(data) < 2 {
		return fmt.Errorf("register received: %d  expected at least: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("register error: %q", data[1])
	case "R":
		if len(data) < 5 {
			return fmt.Errorf("register response incorrect: %x", data)
		}
		chain := mode.ChainName()
		received := string(data[1])
		if received != chain {
			log.Criticalf("expected chain: %q but received: %q", chain, received)
			logger.Panicf("expected chain: %q but received: %q", chain, received)
		}

		timestamp := binary.BigEndian.Uint64(data[4])
		log.Infof("register replied: public key: %x:  listeners: %x  timestamp: %d", data[2], data[3], timestamp)
		announce.AddPeer(data[2], data[3], timestamp) // publicKey, broadcasts, listeners
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func getHeight(client *zmqutil.Client, log *logger.L) (uint64, error) {

	log.Debugf("getHeight: client: %s", client)

	err := client.Send("N")
	if nil != err {
		return 0, err
	}

	data, err := client.Receive(0)
	if nil != err {
		return 0, err
	}
	if 2 != len(data) {
		return 0, fmt.Errorf("getHeight received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return 0, fmt.Errorf("rpc error response: %q", data[1])
	case "N":
		if 8 != len(data[1]) {
			return 0, fmt.Errorf("highestBlock: rpc invalid response: %q", data[1])
		}
		height := binary.BigEndian.Uint64(data[1])
		log.Infof("height: %d", height)
		return height, nil
	default:
		return 0, fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func push(client *zmqutil.Client, log *logger.L, item *messagebus.Message) error {

	log.Debugf("push: client: %s  %q %x", client, item.Command, item.Parameters)

	err := client.Send(item.Command, item.Parameters)
	if nil != err {
		log.Errorf("push: %s send error: %s", client, err)
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		log.Errorf("push: %s receive error: %s", client, err)
		return err
	}
	if 2 != len(data) {
		return fmt.Errorf("push received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("rpc error response: %q", data[1])
	case item.Command:
		log.Debugf("push: client: %s complete: %q", client, data[1])
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}
<commit_msg>[peer] change to warning<commit_after>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package upstream

import (
	"encoding/binary"
	"fmt"
	"sync"
	"time"

	zmq "github.com/pebbe/zmq4"

	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/blockdigest"
	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
)

const (
	cycleInterval = 30 * time.Second
	queueSize     = 50 // 0 => synchronous queue
)

type Upstream struct {
	sync.RWMutex
	log         *logger.L
	client      *zmqutil.Client
	registered  bool
	blockHeight uint64
	shutdown    chan<- struct{}
}

// atomically incremented counter for log names
var upstreamCounter counter.Counter

func New(privateKey []byte, publicKey []byte, timeout time.Duration) (*Upstream, error) {
	client, err := zmqutil.NewClient(zmq.REQ, privateKey, publicKey, timeout)
	if nil != err {
		return nil, err
	}

	n := upstreamCounter.Increment()

	shutdown := make(chan struct{})
	u := &Upstream{
		log:         logger.New(fmt.Sprintf("upstream@%d", n)),
		client:      client,
		registered:  false,
		blockHeight: 0,
		shutdown:    shutdown,
	}
	go upstreamRunner(u, shutdown)
	return u, nil
}

func (u *Upstream) Destroy() {
	if nil != u {
		close(u.shutdown)
	}
}

// check the current destination
//
// does not mean actually connected, as could be in a timeout and
// reconnect state
func (u *Upstream) IsConnectedTo(serverPublicKey []byte) bool {
	return u.client.IsConnectedTo(serverPublicKey)
}

// if registered the have avalid connection
func (u *Upstream) IsOK() bool {
	return u.registered
}

// if registered the have avalid connection
func (u *Upstream) ConnectedTo() *zmqutil.Connected {
	return u.client.ConnectedTo()
}

// connect (or reconnect) to a specific server
func (u *Upstream) Connect(address *util.Connection, serverPublicKey []byte) error {
	u.log.Infof("connecting to address: %s", address)
	u.log.Infof("connecting to server: %x", serverPublicKey)
	u.Lock()
	err := u.client.Connect(address, serverPublicKey, mode.ChainName())
	if nil == err {
		err = register(u.client, u.log)
	}
	u.Unlock()
	return err
}

// fetch height from last polled value
func (u *Upstream) GetHeight() uint64 {
	return u.blockHeight
}

// fetch block digest
func (u *Upstream) GetBlockDigest(blockNumber uint64) (blockdigest.Digest, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("H", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return blockdigest.Digest{}, err
	}

	if 2 != len(data) {
		return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return blockdigest.Digest{}, fault.InvalidError(string(data[1]))
	case "H":
		d := blockdigest.Digest{}
		if blockdigest.Length == len(data[1]) {
			err := blockdigest.DigestFromBytes(&d, data[1])
			return d, err
		}
	default:
	}
	return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
}

// fetch block data
func (u *Upstream) GetBlockData(blockNumber uint64) ([]byte, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("B", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return nil, err
	}

	if 2 != len(data) {
		return nil, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return nil, fault.InvalidError(string(data[1]))
	case "B":
		return data[1], nil
	default:
	}
	return nil, fault.ErrInvalidPeerResponse
}

func upstreamRunner(u *Upstream, shutdown <-chan struct{}) {
	log := u.log

	log.Debug("starting…")

	queue := messagebus.Bus.Broadcast.Chan(queueSize)

loop:
	for {
		log.Debug("waiting…")

		select {
		case <-shutdown:
			break loop

		case item := <-queue:
			log.Debugf("from queue: %q  %x", item.Command, item.Parameters)
			if u.registered {
				u.Lock()
				err := push(u.client, u.log, &item)
				if nil != err {
					log.Errorf("push: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("push: reconnect error: %s", err)
					}
				}
				u.Unlock()
			}

		case <-time.After(cycleInterval):
			u.Lock()
			if !u.registered {
				err := register(u.client, u.log)
				if fault.ErrNotConnected == err {
					log.Infof("register: %s", err)
					u.Unlock()
					continue loop // try again later
				} else if nil != err {
					log.Warnf("register: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("register: reconnect error: %s", err)
					}
					u.Unlock()
					continue loop // try again later
				}
				u.registered = true
			}

			h, err := getHeight(u.client, u.log)
			if nil == err {
				u.blockHeight = h
			} else {
				u.registered = false
				log.Errorf("getHeight: error: %s", err)
				err := u.client.Reconnect()
				if nil != err {
					log.Errorf("highestBlock: reconnect error: %s", err)
				}
			}
			u.Unlock()
		}
	}
	log.Info("shutting down…")
	u.client.Close()
	log.Info("stopped")
}

func register(client *zmqutil.Client, log *logger.L) error {

	log.Debugf("register: client: %s", client)

	err := announce.SendRegistration(client, "R")
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}

	if len(data) < 2 {
		return fmt.Errorf("register received: %d  expected at least: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("register error: %q", data[1])
	case "R":
		if len(data) < 5 {
			return fmt.Errorf("register response incorrect: %x", data)
		}
		chain := mode.ChainName()
		received := string(data[1])
		if received != chain {
			log.Criticalf("expected chain: %q but received: %q", chain, received)
			logger.Panicf("expected chain: %q but received: %q", chain, received)
		}

		timestamp := binary.BigEndian.Uint64(data[4])
		log.Infof("register replied: public key: %x:  listeners: %x  timestamp: %d", data[2], data[3], timestamp)
		announce.AddPeer(data[2], data[3], timestamp) // publicKey, broadcasts, listeners
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func getHeight(client *zmqutil.Client, log *logger.L) (uint64, error) {

	log.Debugf("getHeight: client: %s", client)

	err := client.Send("N")
	if nil != err {
		return 0, err
	}

	data, err := client.Receive(0)
	if nil != err {
		return 0, err
	}
	if 2 != len(data) {
		return 0, fmt.Errorf("getHeight received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return 0, fmt.Errorf("rpc error response: %q", data[1])
	case "N":
		if 8 != len(data[1]) {
			return 0, fmt.Errorf("highestBlock: rpc invalid response: %q", data[1])
		}
		height := binary.BigEndian.Uint64(data[1])
		log.Infof("height: %d", height)
		return height, nil
	default:
		return 0, fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func push(client *zmqutil.Client, log *logger.L, item *messagebus.Message) error {

	log.Debugf("push: client: %s  %q %x", client, item.Command, item.Parameters)

	err := client.Send(item.Command, item.Parameters)
	if nil != err {
		log.Errorf("push: %s send error: %s", client, err)
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		log.Errorf("push: %s receive error: %s", client, err)
		return err
	}
	if 2 != len(data) {
		return fmt.Errorf("push received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("rpc error response: %q", data[1])
	case item.Command:
		log.Debugf("push: client: %s complete: %q", client, data[1])
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}
<|endoftext|>


<commit_before>package language

import (
	"encoding/xml"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/helpers/address"
	"github.com/jmoiron/sqlx"
)

var regexCurrency = regexp.MustCompile(`\d+\.?\d*`)
var regexNum = regexp.MustCompile(`\d+`)
var regexNonWords = regexp.MustCompile(`[^\w\s]`)

// ErrNotFound is thrown when the requested type cannot be found in the string
var ErrNotFound error = errors.New("couldn't extract requested type from string")

// ExtractCurrency returns an int64 if a currency is found, and throws an
// error if one isn't.
func ExtractCurrency(s string) (int64, error) {
	log.Debug("extracting currency")
	s = regexCurrency.FindString(s)
	if len(s) == 0 {
		return 0, ErrNotFound
	}
	val, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0, err
	}
	log.Debug("found value", val)
	// convert parsed float into an int64 with precision of 2 decimal places
	return int64(val * 100), nil
}

// ExtractYesNo determines whether a string contains a Yes or No response.
// This is useful for plugins to determine a user's answer to a Yes/No question.
func ExtractYesNo(s string) (bool, error) {
	ss := strings.Fields(strings.ToLower(s))
	for _, w := range ss {
		w = strings.TrimRight(w, " .,;:!?'\"")
		if yes[w] {
			return true, nil
		}
		if no[w] {
			return false, nil
		}
	}
	return false, ErrNotFound
}

// ExtractAddress will return an address from a user's message, whether it's a
// labeled address (e.g. "home", "office"), or a full U.S. address (e.g. 100
// Penn St., CA 90000)
func ExtractAddress(db *sqlx.DB, u *dt.User, s string) (*dt.Address, bool, error) {
	addr, err := address.Parse(s)
	if err != nil {
		// check if user's address is in DB already
		log.Debug("Checking if user address already in DB...")
		if addr, err = u.GetAddress(db, s); err == nil {
			return addr, true, nil
		}
		return nil, false, err
	}

	type addr2S struct {
		XMLName  xml.Name `xml:"Address"`
		ID       string   `xml:"ID,attr"`
		FirmName string
		Address1 string
		Address2 string
		City     string
		State    string
		Zip5     string
		Zip4     string
	}
	addr2Stmp := addr2S{
		ID:       "0",
		Address1: addr.Line2,
		Address2: addr.Line1,
		City:     addr.City,
		State:    addr.State,
		Zip5:     addr.Zip5,
		Zip4:     addr.Zip4,
	}
	if len(addr.Zip) > 0 {
		addr2Stmp.Zip5 = addr.Zip[:5]
	}
	if len(addr.Zip) > 5 {
		addr2Stmp.Zip4 = addr.Zip[5:]
	}
	addrS := struct {
		XMLName    xml.Name `xml:"AddressValidateRequest"`
		USPSUserID string   `xml:"USERID,attr"`
		Address    addr2S
	}{
		USPSUserID: os.Getenv("USPS_USER_ID"),
		Address:    addr2Stmp,
	}
	xmlAddr, err := xml.Marshal(addrS)
	if err != nil {
		return nil, false, err
	}
	log.Debug(string(xmlAddr))
	ul := "https://secure.shippingapis.com/ShippingAPI.dll?API=Verify&XML="
	ul += url.QueryEscape(string(xmlAddr))
	response, err := http.Get(ul)
	if err != nil {
		return nil, false, err
	}
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, false, err
	}
	if err = response.Body.Close(); err != nil {
		return nil, false, err
	}
	resp := struct {
		XMLName    xml.Name `xml:"AddressValidateResponse"`
		USPSUserID string   `xml:"USERID,attr"`
		Address    addr2S
	}{
		USPSUserID: os.Getenv("USPS_USER_ID"),
		Address:    addr2Stmp,
	}
	if err = xml.Unmarshal(contents, &resp); err != nil {
		log.Debug("USPS response", string(contents))
		return nil, false, err
	}
	a := dt.Address{
		Name:  resp.Address.FirmName,
		Line1: resp.Address.Address2,
		Line2: resp.Address.Address1,
		City:  resp.Address.City,
		State: resp.Address.State,
		Zip5:  resp.Address.Zip5,
		Zip4:  resp.Address.Zip4,
	}
	if len(resp.Address.Zip4) > 0 {
		a.Zip = resp.Address.Zip5 + "-" + resp.Address.Zip4
	} else {
		a.Zip = resp.Address.Zip5
	}
	return &a, false, nil
}

// ExtractCount returns a number from a user's message, useful in situations
// like:
//	Ava>  How many would you like to buy?
//	User> Order 5
func ExtractCount(s string) (int64, error) {
	s = regexNum.FindString(s)
	if len(s) == 0 {
		return 0, ErrNotFound
	}
	val, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0, err
	}
	return val, nil
}

// ExtractCities efficiently from a user's message.
func ExtractCities(db *sqlx.DB, in *dt.Msg) ([]dt.City, error) {
	// Interface type is used to expand the args in db.Select below.
	// Although we're only storing strings, []string{} doesn't work.
	var args []interface{}

	// Look for "at", "in", "on" prepositions to signal that locations
	// follow, skipping everything before
	var start int
	for i := range in.Stems {
		switch in.Stems[i] {
		case "at", "in", "on":
			start = i
			break
		}
	}

	// Prepare sentence for iteration
	tmp := regexNonWords.ReplaceAllString(in.Sentence, "")
	words := strings.Fields(strings.Title(tmp))

	// Iterate through words and bigrams to assemble a DB query
	for i := start; i < len(words); i++ {
		args = append(args, words[i])
	}
	bgs := bigrams(words, start)
	for i := 0; i < len(bgs); i++ {
		args = append(args, bgs[i])
	}

	cities := []dt.City{}
	q := `SELECT name, countrycode FROM cities
	      WHERE countrycode='US' AND name IN (?)
	      ORDER BY LENGTH(name) DESC`
	query, arguments, err := sqlx.In(q, args)
	query = db.Rebind(query)
	rows, err := db.Query(query, arguments...)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err = rows.Close(); err != nil {
			log.Info("failed to close db rows.", err)
		}
	}()
	for rows.Next() {
		city := dt.City{}
		if err = rows.Scan(&city.Name, &city.CountryCode); err != nil {
			return nil, err
		}
		cities = append(cities, city)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}
	if len(cities) == 0 {
		return nil, ErrNotFound
	}
	return cities, nil
}

func bigrams(words []string, startIndex int) (bigrams []string) {
	for i := startIndex; i < len(words)-1; i++ {
		bigrams = append(bigrams, words[i]+" "+words[i+1])
	}
	return bigrams
}
<commit_msg>Remove unnecessary log, fix comment formatting<commit_after>package language

import (
	"encoding/xml"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/helpers/address"
	"github.com/jmoiron/sqlx"
)

var regexCurrency = regexp.MustCompile(`\d+\.?\d*`)
var regexNum = regexp.MustCompile(`\d+`)
var regexNonWords = regexp.MustCompile(`[^\w\s]`)

// ErrNotFound is thrown when the requested type cannot be found in the string
var ErrNotFound error = errors.New("couldn't extract requested type from string")

// ExtractCurrency returns an int64 if a currency is found, and throws an
// error if one isn't.
func ExtractCurrency(s string) (int64, error) {
	s = regexCurrency.FindString(s)
	if len(s) == 0 {
		return 0, ErrNotFound
	}
	val, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0, err
	}
	log.Debug("found value", val)
	// Convert parsed float into an int64 with precision of 2 decimal places
	return int64(val * 100), nil
}

// ExtractYesNo determines whether a string contains a Yes or No response.
// This is useful for plugins to determine a user's answer to a Yes/No question.
func ExtractYesNo(s string) (bool, error) {
	ss := strings.Fields(strings.ToLower(s))
	for _, w := range ss {
		w = strings.TrimRight(w, " .,;:!?'\"")
		if yes[w] {
			return true, nil
		}
		if no[w] {
			return false, nil
		}
	}
	return false, ErrNotFound
}

// ExtractAddress will return an address from a user's message, whether it's a
// labeled address (e.g. "home", "office"), or a full U.S. address (e.g. 100
// Penn St., CA 90000)
func ExtractAddress(db *sqlx.DB, u *dt.User, s string) (*dt.Address, bool, error) {
	addr, err := address.Parse(s)
	if err != nil {
		// check if user's address is in DB already
		log.Debug("Checking if user address already in DB...")
		if addr, err = u.GetAddress(db, s); err == nil {
			return addr, true, nil
		}
		return nil, false, err
	}

	type addr2S struct {
		XMLName  xml.Name `xml:"Address"`
		ID       string   `xml:"ID,attr"`
		FirmName string
		Address1 string
		Address2 string
		City     string
		State    string
		Zip5     string
		Zip4     string
	}
	addr2Stmp := addr2S{
		ID:       "0",
		Address1: addr.Line2,
		Address2: addr.Line1,
		City:     addr.City,
		State:    addr.State,
		Zip5:     addr.Zip5,
		Zip4:     addr.Zip4,
	}
	if len(addr.Zip) > 0 {
		addr2Stmp.Zip5 = addr.Zip[:5]
	}
	if len(addr.Zip) > 5 {
		addr2Stmp.Zip4 = addr.Zip[5:]
	}
	addrS := struct {
		XMLName    xml.Name `xml:"AddressValidateRequest"`
		USPSUserID string   `xml:"USERID,attr"`
		Address    addr2S
	}{
		USPSUserID: os.Getenv("USPS_USER_ID"),
		Address:    addr2Stmp,
	}
	xmlAddr, err := xml.Marshal(addrS)
	if err != nil {
		return nil, false, err
	}
	log.Debug(string(xmlAddr))
	ul := "https://secure.shippingapis.com/ShippingAPI.dll?API=Verify&XML="
	ul += url.QueryEscape(string(xmlAddr))
	response, err := http.Get(ul)
	if err != nil {
		return nil, false, err
	}
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, false, err
	}
	if err = response.Body.Close(); err != nil {
		return nil, false, err
	}
	resp := struct {
		XMLName    xml.Name `xml:"AddressValidateResponse"`
		USPSUserID string   `xml:"USERID,attr"`
		Address    addr2S
	}{
		USPSUserID: os.Getenv("USPS_USER_ID"),
		Address:    addr2Stmp,
	}
	if err = xml.Unmarshal(contents, &resp); err != nil {
		log.Debug("USPS response", string(contents))
		return nil, false, err
	}
	a := dt.Address{
		Name:  resp.Address.FirmName,
		Line1: resp.Address.Address2,
		Line2: resp.Address.Address1,
		City:  resp.Address.City,
		State: resp.Address.State,
		Zip5:  resp.Address.Zip5,
		Zip4:  resp.Address.Zip4,
	}
	if len(resp.Address.Zip4) > 0 {
		a.Zip = resp.Address.Zip5 + "-" + resp.Address.Zip4
	} else {
		a.Zip = resp.Address.Zip5
	}
	return &a, false, nil
}

// ExtractCount returns a number from a user's message, useful in situations
// like:
//	Ava>  How many would you like to buy?
//	User> Order 5
func ExtractCount(s string) (int64, error) {
	s = regexNum.FindString(s)
	if len(s) == 0 {
		return 0, ErrNotFound
	}
	val, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		return 0, err
	}
	return val, nil
}

// ExtractCities efficiently from a user's message.
func ExtractCities(db *sqlx.DB, in *dt.Msg) ([]dt.City, error) {
	// Interface type is used to expand the args in db.Select below.
	// Although we're only storing strings, []string{} doesn't work.
	var args []interface{}

	// Look for "at", "in", "on" prepositions to signal that locations
	// follow, skipping everything before
	var start int
	for i := range in.Stems {
		switch in.Stems[i] {
		case "at", "in", "on":
			start = i
			break
		}
	}

	// Prepare sentence for iteration
	tmp := regexNonWords.ReplaceAllString(in.Sentence, "")
	words := strings.Fields(strings.Title(tmp))

	// Iterate through words and bigrams to assemble a DB query
	for i := start; i < len(words); i++ {
		args = append(args, words[i])
	}
	bgs := bigrams(words, start)
	for i := 0; i < len(bgs); i++ {
		args = append(args, bgs[i])
	}

	cities := []dt.City{}
	q := `SELECT name, countrycode FROM cities
	      WHERE countrycode='US' AND name IN (?)
	      ORDER BY LENGTH(name) DESC`
	query, arguments, err := sqlx.In(q, args)
	query = db.Rebind(query)
	rows, err := db.Query(query, arguments...)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err = rows.Close(); err != nil {
			log.Info("failed to close db rows.", err)
		}
	}()
	for rows.Next() {
		city := dt.City{}
		if err = rows.Scan(&city.Name, &city.CountryCode); err != nil {
			return nil, err
		}
		cities = append(cities, city)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}
	if len(cities) == 0 {
		return nil, ErrNotFound
	}
	return cities, nil
}

func bigrams(words []string, startIndex int) (bigrams []string) {
	for i := startIndex; i < len(words)-1; i++ {
		bigrams = append(bigrams, words[i]+" "+words[i+1])
	}
	return bigrams
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"

	"github.com/go-check/check"
	"golang.org/x/net/websocket"
)

func (s *DockerSuite) TestGetContainersAttachWebsocket(c *check.C) {
	testRequires(c, DaemonIsLinux)
	out, _ := dockerCmd(c, "run", "-dit", "busybox", "cat")

	rwc, err := sockConn(time.Duration(10 * time.Second))
	if err != nil {
		c.Fatal(err)
	}

	cleanedContainerID := strings.TrimSpace(out)
	config, err := websocket.NewConfig(
		"/containers/"+cleanedContainerID+"/attach/ws?stream=1&stdin=1&stdout=1&stderr=1",
		"http://localhost",
	)
	if err != nil {
		c.Fatal(err)
	}

	ws, err := websocket.NewClient(config, rwc)
	if err != nil {
		c.Fatal(err)
	}
	defer ws.Close()

	expected := []byte("hello")
	actual := make([]byte, len(expected))

	outChan := make(chan error)
	go func() {
		_, err := ws.Read(actual)
		outChan <- err
		close(outChan)
	}()

	inChan := make(chan error)
	go func() {
		_, err := ws.Write(expected)
		inChan <- err
		close(inChan)
	}()

	select {
	case err := <-inChan:
		if err != nil {
			c.Fatal(err)
		}
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout writing to ws")
	}

	select {
	case err := <-outChan:
		if err != nil {
			c.Fatal(err)
		}
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout reading from ws")
	}

	if !bytes.Equal(expected, actual) {
		c.Fatal("Expected output on websocket to match input")
	}
}

// regression gh14320
func (s *DockerSuite) TestPostContainersAttachContainerNotFound(c *check.C) {
	status, body, err := sockRequest("POST", "/containers/doesnotexist/attach", nil)
	c.Assert(status, check.Equals, http.StatusNotFound)
	c.Assert(err, check.IsNil)
	expected := "no such id: doesnotexist\n"
	if !strings.Contains(string(body), expected) {
		c.Fatalf("Expected response body to contain %q", expected)
	}
}

func (s *DockerSuite) TestGetContainersWsAttachContainerNotFound(c *check.C) {
	status, body, err := sockRequest("GET", "/containers/doesnotexist/attach/ws", nil)
	c.Assert(status, check.Equals, http.StatusNotFound)
	c.Assert(err, check.IsNil)
	expected := "no such id: doesnotexist\n"
	if !strings.Contains(string(body), expected) {
		c.Fatalf("Expected response body to contain %q", expected)
	}
}

func (s *DockerSuite) TestPostContainersAttach(c *check.C) {
	testRequires(c, DaemonIsLinux)
	out, _ := dockerCmd(c, "run", "-dit", "busybox", "cat")

	r, w := io.Pipe()
	defer r.Close()
	defer w.Close()

	conn, err := sockConn(time.Duration(10 * time.Second))
	c.Assert(err, check.IsNil)

	containerID := strings.TrimSpace(out)

	req, err := http.NewRequest("POST", "/containers/"+containerID+"/attach?stream=1&stdin=1&stdout=1&stderr=1", bytes.NewReader([]byte{}))
	c.Assert(err, check.IsNil)

	client := httputil.NewClientConn(conn, nil)
	defer client.Close()

	// Do POST attach request
	resp, err := client.Do(req)
	c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
	// If we check the err, we get a ErrPersistEOF = &http.ProtocolError{ErrorString: "persistent connection closed"}
	// This means that the remote requested this be the last request serviced, is this okay?

	// Test read and write to the attached container
	expected := []byte("hello")
	actual := make([]byte, len(expected))

	outChan := make(chan error)
	go func() {
		_, err := r.Read(actual)
		outChan <- err
		close(outChan)
	}()

	inChan := make(chan error)
	go func() {
		_, err := w.Write(expected)
		inChan <- err
		close(inChan)
	}()

	select {
	case err := <-inChan:
		c.Assert(err, check.IsNil)
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout writing to stdout")
	}

	select {
	case err := <-outChan:
		c.Assert(err, check.IsNil)
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout reading from stdin")
	}

	if !bytes.Equal(expected, actual) {
		c.Fatal("Expected output to match input")
	}

	resp.Body.Close()
}

func (s *DockerSuite) TestPostContainersAttachStderr(c *check.C) {
	testRequires(c, DaemonIsLinux)
	out, _ := dockerCmd(c, "run", "-dit", "busybox", "/bin/sh", "-c", "cat >&2")

	r, w := io.Pipe()
	defer r.Close()
	defer w.Close()

	conn, err := sockConn(time.Duration(10 * time.Second))
	c.Assert(err, check.IsNil)

	containerID := strings.TrimSpace(out)

	req, err := http.NewRequest("POST", "/containers/"+containerID+"/attach?stream=1&stdin=1&stdout=1&stderr=1", bytes.NewReader([]byte{}))
	c.Assert(err, check.IsNil)

	client := httputil.NewClientConn(conn, nil)
	defer client.Close()

	// Do POST attach request
	resp, err := client.Do(req)
	c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
	// If we check the err, we get a ErrPersistEOF = &http.ProtocolError{ErrorString: "persistent connection closed"}
	// This means that the remote requested this be the last request serviced, is this okay?

	// Test read and write to the attached container
	expected := []byte("hello")
	actual := make([]byte, len(expected))

	outChan := make(chan error)
	go func() {
		_, err := r.Read(actual)
		outChan <- err
		close(outChan)
	}()

	inChan := make(chan error)
	go func() {
		_, err := w.Write(expected)
		inChan <- err
		close(inChan)
	}()

	select {
	case err := <-inChan:
		c.Assert(err, check.IsNil)
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout writing to stdout")
	}

	select {
	case err := <-outChan:
		c.Assert(err, check.IsNil)
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout reading from stdin")
	}

	if !bytes.Equal(expected, actual) {
		c.Fatal("Expected output to match input")
	}

	resp.Body.Close()
}
<commit_msg>fix the flaws in the test of the attach API<commit_after>package main

import (
	"bufio"
	"bytes"
	"io"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/go-check/check"
	"golang.org/x/net/websocket"
)

func (s *DockerSuite) TestGetContainersAttachWebsocket(c *check.C) {
	testRequires(c, DaemonIsLinux)
	out, _ := dockerCmd(c, "run", "-dit", "busybox", "cat")

	rwc, err := sockConn(time.Duration(10 * time.Second))
	if err != nil {
		c.Fatal(err)
	}

	cleanedContainerID := strings.TrimSpace(out)
	config, err := websocket.NewConfig(
		"/containers/"+cleanedContainerID+"/attach/ws?stream=1&stdin=1&stdout=1&stderr=1",
		"http://localhost",
	)
	if err != nil {
		c.Fatal(err)
	}

	ws, err := websocket.NewClient(config, rwc)
	if err != nil {
		c.Fatal(err)
	}
	defer ws.Close()

	expected := []byte("hello")
	actual := make([]byte, len(expected))

	outChan := make(chan error)
	go func() {
		_, err := ws.Read(actual)
		outChan <- err
		close(outChan)
	}()

	inChan := make(chan error)
	go func() {
		_, err := ws.Write(expected)
		inChan <- err
		close(inChan)
	}()

	select {
	case err := <-inChan:
		if err != nil {
			c.Fatal(err)
		}
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout writing to ws")
	}

	select {
	case err := <-outChan:
		if err != nil {
			c.Fatal(err)
		}
	case <-time.After(5 * time.Second):
		c.Fatal("Timeout reading from ws")
	}

	if !bytes.Equal(expected, actual) {
		c.Fatal("Expected output on websocket to match input")
	}
}

// regression gh14320
func (s *DockerSuite) TestPostContainersAttachContainerNotFound(c *check.C) {
	status, body, err := sockRequest("POST", "/containers/doesnotexist/attach", nil)
	c.Assert(status, check.Equals, http.StatusNotFound)
	c.Assert(err, check.IsNil)
	expected := "no such id: doesnotexist\n"
	if !strings.Contains(string(body), expected) {
		c.Fatalf("Expected response body to contain %q", expected)
	}
}

func (s *DockerSuite) TestGetContainersWsAttachContainerNotFound(c *check.C) {
	status, body, err := sockRequest("GET", "/containers/doesnotexist/attach/ws", nil)
	c.Assert(status, check.Equals, http.StatusNotFound)
	c.Assert(err, check.IsNil)
	expected := "no such id: doesnotexist\n"
	if !strings.Contains(string(body), expected) {
		c.Fatalf("Expected response body to contain %q", expected)
	}
}

func (s *DockerSuite) TestPostContainersAttach(c *check.C) {
	testRequires(c, DaemonIsLinux)

	expectSuccess := func(conn net.Conn, br *bufio.Reader, stream string, tty bool) {
		defer conn.Close()
		expected := []byte("success")
		_, err := conn.Write(expected)
		c.Assert(err, check.IsNil)

		conn.SetReadDeadline(time.Now().Add(time.Second))
		lenHeader := 0
		if !tty {
			lenHeader = 8
		}
		actual := make([]byte, len(expected)+lenHeader)
		_, err = io.ReadFull(br, actual)
		c.Assert(err, check.IsNil)
		if !tty {
			fdMap := map[string]byte{
				"stdin":  0,
				"stdout": 1,
				"stderr": 2,
			}
			c.Assert(actual[0], check.Equals, fdMap[stream])
		}
		c.Assert(actual[lenHeader:], check.DeepEquals, expected, check.Commentf("Attach didn't return the expected data from %s", stream))
	}

	expectTimeout := func(conn net.Conn, br *bufio.Reader, stream string) {
		defer conn.Close()
		_, err := conn.Write([]byte{'t'})
		c.Assert(err, check.IsNil)

		conn.SetReadDeadline(time.Now().Add(time.Second))
		actual := make([]byte, 1)
		_, err = io.ReadFull(br, actual)
		opErr, ok := err.(*net.OpError)
		c.Assert(ok, check.Equals, true, check.Commentf("Error is expected to be *net.OpError, got %v", err))
		c.Assert(opErr.Timeout(), check.Equals, true, check.Commentf("Read from %s is expected to timeout", stream))
	}

	// Create a container that only emits stdout.
	cid, _ := dockerCmd(c, "run", "-di", "busybox", "cat")
	cid = strings.TrimSpace(cid)
	// Attach to the container's stdout stream.
	conn, br, err := sockRequestHijack("POST", "/containers/"+cid+"/attach?stream=1&stdin=1&stdout=1", nil, "text/plain")
	c.Assert(err, check.IsNil)
	// Check if the data from stdout can be received.
	expectSuccess(conn, br, "stdout", false)
	// Attach to the container's stderr stream.
	conn, br, err = sockRequestHijack("POST", "/containers/"+cid+"/attach?stream=1&stdin=1&stderr=1", nil, "text/plain")
	c.Assert(err, check.IsNil)
	// Since the container only emits stdout, attaching to stderr should return nothing.
	expectTimeout(conn, br, "stdout")

	// Test the simlar functions of the stderr stream.
	cid, _ = dockerCmd(c, "run", "-di", "busybox", "/bin/sh", "-c", "cat >&2")
	cid = strings.TrimSpace(cid)
	conn, br, err = sockRequestHijack("POST", "/containers/"+cid+"/attach?stream=1&stdin=1&stderr=1", nil, "text/plain")
	c.Assert(err, check.IsNil)
	expectSuccess(conn, br, "stderr", false)
	conn, br, err = sockRequestHijack("POST", "/containers/"+cid+"/attach?stream=1&stdin=1&stdout=1", nil, "text/plain")
	c.Assert(err, check.IsNil)
	expectTimeout(conn, br, "stderr")

	// Test with tty.
	cid, _ = dockerCmd(c, "run", "-dit", "busybox", "/bin/sh", "-c", "cat >&2")
	cid = strings.TrimSpace(cid)
	// Attach to stdout only.
	conn, br, err = sockRequestHijack("POST", "/containers/"+cid+"/attach?stream=1&stdin=1&stdout=1", nil, "text/plain")
	c.Assert(err, check.IsNil)
	expectSuccess(conn, br, "stdout", true)

	// Attach without stdout stream.
	conn, br, err = sockRequestHijack("POST", "/containers/"+cid+"/attach?stream=1&stdin=1&stderr=1", nil, "text/plain")
	c.Assert(err, check.IsNil)
	// Nothing should be received because both the stdout and stderr of the container will be
	// sent to the client as stdout when tty is enabled.
	expectTimeout(conn, br, "stdout")
}
<|endoftext|>


<commit_before>package pigae

import (
	"fmt"
	"net/http"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"appengine"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]
	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "c++", "cpp", -1)
	terms := SplitForSearching(q, true)

	if len(terms) == 0 {
		// Search query is empty or illegible...
		redirURL := hostPrefix() + "/about#about-block-all-idioms"
		http.Redirect(w, r, redirURL, http.StatusFound)
		return nil
	}

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
	})

	if len(words) == 0 {
		words, typedLangs = typedLangs, nil
	}

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByWordsWithFavorites(c, words, typedLangs, userProfile.FavoriteLanguages, userProfile.SeeNonFavorite, numberMaxResults)
	if err != nil {
		return err
	}

	typedLangsSet := make(map[string]bool, len(typedLangs))
	for _, lang := range typedLangs {
		typedLangsSet[lang] = true
	}

	// Highlight matching impls :)
	matchingImplIDs, err := dao.searchImplIDs(c, words, typedLangs)
	if err != nil {
		return err
	}
	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
		for i := range idiom.Implementations {
			impl := &idiom.Implementations[i]
			implIDStr := fmt.Sprintf("%d", impl.Id)
			if matchingImplIDs[implIDStr] {
				impl.Deco.Matching = true
			}
			if typedLangsSet[impl.LanguageName] {
				impl.Deco.SearchedLang = true
			}
		}
	}

	normalizedQ := strings.Join(terms, " ")
	return listResults(w, r, normalizedQ, hits)
}

func listResults(w http.ResponseWriter, r *http.Request, q string, idioms []*Idiom) error {
	data := &SearchResultsFacade{
		PageMeta: PageMeta{
			PageTitle:   "Idioms for \"" + q + "\"",
			Toggles:     toggles,
			SearchQuery: q,
		},
		UserProfile: readUserProfile(r),
		Q:           q,
		Results:     idioms,
	}

	return templates.ExecuteTemplate(w, "page-list-results", data)
}

func searchRedirect(w http.ResponseWriter, r *http.Request) error {
	q := r.FormValue("q")
	if q == "" {
		// Small hack for own convenience: empty search -> homepage
		http.Redirect(w, r, "/", 301)
		return nil
	}
	q = strings.Replace(q, " ", "+", -1)

	http.Redirect(w, r, hostPrefix()+"/search/"+q, http.StatusMovedPermanently)
	return nil
}

func listByLanguage(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	langsStr := vars["langs"]
	langs := strings.Split(langsStr, "_")
	langs = MapStrings(langs, normLang)
	langs = RemoveEmptyStrings(langs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByLangs(c, langs, numberMaxResults)
	if err != nil {
		return err
	}

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
	}

	niceLangs := MapStrings(langs, printNiceLang)
	return listResults(w, r, fmt.Sprintf("Language=%v", niceLangs), hits)
}
<commit_msg>Search: fix for uppercase C++.<commit_after>package pigae

import (
	"fmt"
	"net/http"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"appengine"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]

	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "c++", "cpp", -1)
	q = strings.Replace(q, "C++", "cpp", -1)
	// TODO find a graceful way to handle "c#".

	terms := SplitForSearching(q, true)

	if len(terms) == 0 {
		// Search query is empty or illegible...
		redirURL := hostPrefix() + "/about#about-block-all-idioms"
		http.Redirect(w, r, redirURL, http.StatusFound)
		return nil
	}

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
	})

	if len(words) == 0 {
		words, typedLangs = typedLangs, nil
	}

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByWordsWithFavorites(c, words, typedLangs, userProfile.FavoriteLanguages, userProfile.SeeNonFavorite, numberMaxResults)
	if err != nil {
		return err
	}

	typedLangsSet := make(map[string]bool, len(typedLangs))
	for _, lang := range typedLangs {
		typedLangsSet[lang] = true
	}

	// Highlight matching impls :)
	matchingImplIDs, err := dao.searchImplIDs(c, words, typedLangs)
	if err != nil {
		return err
	}
	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
		for i := range idiom.Implementations {
			impl := &idiom.Implementations[i]
			implIDStr := fmt.Sprintf("%d", impl.Id)
			if matchingImplIDs[implIDStr] {
				impl.Deco.Matching = true
			}
			if typedLangsSet[impl.LanguageName] {
				impl.Deco.SearchedLang = true
			}
		}
	}

	normalizedQ := strings.Join(terms, " ")
	return listResults(w, r, normalizedQ, hits)
}

func listResults(w http.ResponseWriter, r *http.Request, q string, idioms []*Idiom) error {
	data := &SearchResultsFacade{
		PageMeta: PageMeta{
			PageTitle:   "Idioms for \"" + q + "\"",
			Toggles:     toggles,
			SearchQuery: q,
		},
		UserProfile: readUserProfile(r),
		Q:           q,
		Results:     idioms,
	}

	return templates.ExecuteTemplate(w, "page-list-results", data)
}

func searchRedirect(w http.ResponseWriter, r *http.Request) error {
	q := r.FormValue("q")
	if q == "" {
		// Small hack for own convenience: empty search -> homepage
		http.Redirect(w, r, "/", 301)
		return nil
	}
	q = strings.Replace(q, " ", "+", -1)

	http.Redirect(w, r, hostPrefix()+"/search/"+q, http.StatusMovedPermanently)
	return nil
}

func listByLanguage(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	langsStr := vars["langs"]
	langs := strings.Split(langsStr, "_")
	langs = MapStrings(langs, normLang)
	langs = RemoveEmptyStrings(langs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByLangs(c, langs, numberMaxResults)
	if err != nil {
		return err
	}

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
	}

	niceLangs := MapStrings(langs, printNiceLang)
	return listResults(w, r, fmt.Sprintf("Language=%v", niceLangs), hits)
}
<|endoftext|>


<commit_before>package collector

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"fullerite/config"
	"fullerite/metric"
	"fullerite/util"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	l "github.com/Sirupsen/logrus"
)

var (
	getNerveHTTPDMetrics = (*NerveHTTPD).getMetrics
	knownApacheMetrics   = map[string]bool{
		"ReqPerSec":        true,
		"BytesPerSec":      true,
		"BytesPerReq":      true,
		"BusyWorkers":      true,
		"Total Accesses":   true,
		"IdleWorkers":      true,
		"StartingWorkers":  true,
		"ReadingWorkers":   true,
		"WritingWorkers":   true,
		"KeepaliveWorkers": true,
		"DnsWorkers":       true,
		"ClosingWorkers":   true,
		"LoggingWorkers":   true,
		"FinishingWorkers": true,
		"CleanupWorkers":   true,
		"StandbyWorkers":   true,
		"CPULoad":          true,
	}
	metricRegexp = regexp.MustCompile(`^([A-Za-z ]+):\s+(.+)$`)
)

// NerveHTTPD discovers Apache servers via Nerve config
// and reports metric for them
type NerveHTTPD struct {
	baseCollector

	configFilePath    string
	queryPath         string
	host              string
	timeout           int
	statusTTL         time.Duration
	servicesWhitelist []string
}

type nerveHTTPDResponse struct {
	data   []byte
	err    error
	status int
}

func init() {
	RegisterCollector("NerveHTTPD", newNerveHTTPD)
}

func newNerveHTTPD(channel chan metric.Metric, initialInterval int, log *l.Entry) Collector {
	c := new(NerveHTTPD)
	c.channel = channel
	c.interval = initialInterval
	c.log = log

	c.name = "NerveHTTPD"
	c.configFilePath = "/etc/nerve/nerve.conf.json"
	c.queryPath = "server-status?auto"
	c.host = "localhost"
	c.timeout = 2
	c.statusTTL = time.Duration(60) * time.Minute
	return c
}

// Configure the collector
func (c *NerveHTTPD) Configure(configMap map[string]interface{}) {
	if val, exists := configMap["queryPath"]; exists {
		c.queryPath = val.(string)
	}

	if val, exists := configMap["configFilePath"]; exists {
		c.configFilePath = val.(string)
	}

	if val, exists := configMap["host"]; exists {
		c.host = val.(string)
	}

	if val, exists := configMap["status_ttl"]; exists {
		tmpStatusTTL := config.GetAsInt(val, 3600)
		c.statusTTL = time.Duration(tmpStatusTTL) * time.Second
	}

	if val, exists := configMap["servicesWhitelist"]; exists {
		c.servicesWhitelist = config.GetAsSlice(val)
	}

	c.configureCommonParams(configMap)
}

// Collect the metrics
func (c *NerveHTTPD) Collect() {
	rawFileContents, err := ioutil.ReadFile(c.configFilePath)
	if err != nil {
		c.log.Warn("Failed to read the contents of file ", c.configFilePath, " because ", err)
		return
	}
	services, err := util.ParseNerveConfig(&rawFileContents, true)
	if err != nil {
		c.log.Warn("Failed to parse the nerve config at ", c.configFilePath, ": ", err)
		return
	}
	c.log.Debug("Finished parsing Nerve config into ", services)

	for _, service := range services {
		if c.serviceInWhitelist(service) {
			go c.emitHTTPDMetric(service, service.Port)
		}
	}
}

func (c *NerveHTTPD) serviceInWhitelist(service util.NerveService) bool {
	for _, s := range c.servicesWhitelist {
		if s == service.Name+"."+service.Namespace {
			return true
		}
	}
	return false
}

func (c *NerveHTTPD) emitHTTPDMetric(service util.NerveService, port int) {
	metrics := getNerveHTTPDMetrics(c, service, port)
	for _, metric := range metrics {
		c.Channel() <- metric
	}
	c.Channel() <- metric.Sentinel()
}

func (c *NerveHTTPD) getMetrics(service util.NerveService, port int) []metric.Metric {
	results := []metric.Metric{}
	serviceLog := c.log.WithField("service", service.Name)

	endpoint := fmt.Sprintf("http://%s:%d/%s", c.host, port, c.queryPath)
	serviceLog.Debug("making GET request to ", endpoint)

	httpResponse := fetchApacheMetrics(endpoint, port)

	if httpResponse.status != 200 {
		serviceLog.Warn("Failed to query endpoint ", endpoint, ": ", httpResponse.err)
		return results
	}
	apacheMetrics := extractApacheMetrics(httpResponse.data)
	metric.AddToAll(&apacheMetrics, map[string]string{
		"service_name":      service.Name,
		"service_namespace": service.Namespace,
		"port":              strconv.Itoa(port),
	})
	return apacheMetrics
}

func extractApacheMetrics(data []byte) []metric.Metric {
	results := []metric.Metric{}
	reader := bytes.NewReader(data)
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		metricLine := scanner.Text()
		resultMatch := metricRegexp.FindStringSubmatch(metricLine)
		if len(resultMatch) > 0 {
			k := resultMatch[1]
			v := resultMatch[2]
			if k == "IdleWorkers" {
				continue
			}

			if k == "Scoreboard" {
				scoreBoardMetrics := extractScoreBoardMetrics(k, v)
				results = append(results, scoreBoardMetrics...)
			}

			metric, err := buildApacheMetric(k, v)
			if err == nil {
				results = append(results, metric)
			}
		}

	}
	return results
}

func buildApacheMetric(key, value string) (metric.Metric, error) {
	var tmpMetric metric.Metric
	if _, ok := knownApacheMetrics[key]; ok {
		whiteRegexp := regexp.MustCompile(`\s+`)
		metricName := whiteRegexp.ReplaceAllString(key, "")
		metricValue, err := strconv.ParseFloat(value, 64)

		if err != nil {
			return tmpMetric, err
		}
		return metric.WithValue(metricName, metricValue), nil
	}
	return tmpMetric, errors.New("invalid metric")
}

func extractScoreBoardMetrics(key, value string) []metric.Metric {
	results := []metric.Metric{}
	charCounter := func(str string, pattern string) float64 {
		return float64(strings.Count(str, pattern))
	}
	results = append(results, metric.WithValue("IdleWorkers", charCounter(value, "_")))
	results = append(results, metric.WithValue("StartingWorkers", charCounter(value, "S")))
	results = append(results, metric.WithValue("ReadingWorkers", charCounter(value, "R")))
	results = append(results, metric.WithValue("WritingWorkers", charCounter(value, "W")))
	results = append(results, metric.WithValue("KeepaliveWorkers", charCounter(value, "K")))
	results = append(results, metric.WithValue("DnsWorkers", charCounter(value, "D")))
	results = append(results, metric.WithValue("ClosingWorkers", charCounter(value, "C")))
	results = append(results, metric.WithValue("LoggingWorkers", charCounter(value, "L")))
	results = append(results, metric.WithValue("FinishingWorkers", charCounter(value, "G")))
	results = append(results, metric.WithValue("CleanupWorkers", charCounter(value, "I")))
	results = append(results, metric.WithValue("StandbyWorkers", charCounter(value, "_")))
	return results
}

func fetchApacheMetrics(endpoint string, timeout int) *nerveHTTPDResponse {
	response := new(nerveHTTPDResponse)
	client := http.Client{
		Timeout: time.Duration(timeout) * time.Second,
	}

	rsp, err := client.Get(endpoint)
	response.err = err
	if rsp != nil {
		response.status = rsp.StatusCode
	}

	if err != nil {
		return response
	}

	txt, err := ioutil.ReadAll(rsp.Body)
	defer rsp.Body.Close()
	if err != nil {
		response.err = err
		return response
	}
	response.data = txt
	return response
}
<commit_msg>Emit TotalAccesses as CumulativeCounter<commit_after>package collector

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"fullerite/config"
	"fullerite/metric"
	"fullerite/util"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	l "github.com/Sirupsen/logrus"
)

var (
	getNerveHTTPDMetrics = (*NerveHTTPD).getMetrics
	knownApacheMetrics   = map[string]string{
		"ReqPerSec":        metric.Gauge,
		"BytesPerSec":      metric.Gauge,
		"BytesPerReq":      metric.Gauge,
		"BusyWorkers":      metric.Gauge,
		"Total Accesses":   metric.CumulativeCounter,
		"IdleWorkers":      metric.Gauge,
		"StartingWorkers":  metric.Gauge,
		"ReadingWorkers":   metric.Gauge,
		"WritingWorkers":   metric.Gauge,
		"KeepaliveWorkers": metric.Gauge,
		"DnsWorkers":       metric.Gauge,
		"ClosingWorkers":   metric.Gauge,
		"LoggingWorkers":   metric.Gauge,
		"FinishingWorkers": metric.Gauge,
		"CleanupWorkers":   metric.Gauge,
		"StandbyWorkers":   metric.Gauge,
		"CPULoad":          metric.Gauge,
	}
	metricRegexp = regexp.MustCompile(`^([A-Za-z ]+):\s+(.+)$`)
)

// NerveHTTPD discovers Apache servers via Nerve config
// and reports metric for them
type NerveHTTPD struct {
	baseCollector

	configFilePath    string
	queryPath         string
	host              string
	timeout           int
	statusTTL         time.Duration
	servicesWhitelist []string
}

type nerveHTTPDResponse struct {
	data   []byte
	err    error
	status int
}

func init() {
	RegisterCollector("NerveHTTPD", newNerveHTTPD)
}

func newNerveHTTPD(channel chan metric.Metric, initialInterval int, log *l.Entry) Collector {
	c := new(NerveHTTPD)
	c.channel = channel
	c.interval = initialInterval
	c.log = log

	c.name = "NerveHTTPD"
	c.configFilePath = "/etc/nerve/nerve.conf.json"
	c.queryPath = "server-status?auto"
	c.host = "localhost"
	c.timeout = 2
	c.statusTTL = time.Duration(60) * time.Minute
	return c
}

// Configure the collector
func (c *NerveHTTPD) Configure(configMap map[string]interface{}) {
	if val, exists := configMap["queryPath"]; exists {
		c.queryPath = val.(string)
	}

	if val, exists := configMap["configFilePath"]; exists {
		c.configFilePath = val.(string)
	}

	if val, exists := configMap["host"]; exists {
		c.host = val.(string)
	}

	if val, exists := configMap["status_ttl"]; exists {
		tmpStatusTTL := config.GetAsInt(val, 3600)
		c.statusTTL = time.Duration(tmpStatusTTL) * time.Second
	}

	if val, exists := configMap["servicesWhitelist"]; exists {
		c.servicesWhitelist = config.GetAsSlice(val)
	}

	c.configureCommonParams(configMap)
}

// Collect the metrics
func (c *NerveHTTPD) Collect() {
	rawFileContents, err := ioutil.ReadFile(c.configFilePath)
	if err != nil {
		c.log.Warn("Failed to read the contents of file ", c.configFilePath, " because ", err)
		return
	}
	services, err := util.ParseNerveConfig(&rawFileContents, true)
	if err != nil {
		c.log.Warn("Failed to parse the nerve config at ", c.configFilePath, ": ", err)
		return
	}
	c.log.Debug("Finished parsing Nerve config into ", services)

	for _, service := range services {
		if c.serviceInWhitelist(service) {
			go c.emitHTTPDMetric(service, service.Port)
		}
	}
}

func (c *NerveHTTPD) serviceInWhitelist(service util.NerveService) bool {
	for _, s := range c.servicesWhitelist {
		if s == service.Name+"."+service.Namespace {
			return true
		}
	}
	return false
}

func (c *NerveHTTPD) emitHTTPDMetric(service util.NerveService, port int) {
	metrics := getNerveHTTPDMetrics(c, service, port)
	for _, metric := range metrics {
		c.Channel() <- metric
	}
	c.Channel() <- metric.Sentinel()
}

func (c *NerveHTTPD) getMetrics(service util.NerveService, port int) []metric.Metric {
	results := []metric.Metric{}
	serviceLog := c.log.WithField("service", service.Name)

	endpoint := fmt.Sprintf("http://%s:%d/%s", c.host, port, c.queryPath)
	serviceLog.Debug("making GET request to ", endpoint)

	httpResponse := fetchApacheMetrics(endpoint, port)

	if httpResponse.status != 200 {
		serviceLog.Warn("Failed to query endpoint ", endpoint, ": ", httpResponse.err)
		return results
	}
	apacheMetrics := extractApacheMetrics(httpResponse.data)
	metric.AddToAll(&apacheMetrics, map[string]string{
		"service_name":      service.Name,
		"service_namespace": service.Namespace,
		"port":              strconv.Itoa(port),
	})
	return apacheMetrics
}

func extractApacheMetrics(data []byte) []metric.Metric {
	results := []metric.Metric{}
	reader := bytes.NewReader(data)
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		metricLine := scanner.Text()
		resultMatch := metricRegexp.FindStringSubmatch(metricLine)
		if len(resultMatch) > 0 {
			k := resultMatch[1]
			v := resultMatch[2]
			if k == "IdleWorkers" {
				continue
			}

			if k == "Scoreboard" {
				scoreBoardMetrics := extractScoreBoardMetrics(k, v)
				results = append(results, scoreBoardMetrics...)
			}

			metric, err := buildApacheMetric(k, v)
			if err == nil {
				results = append(results, metric)
			}
		}

	}
	return results
}

func buildApacheMetric(key, value string) (metric.Metric, error) {
	var tmpMetric metric.Metric
	if metricType, ok := knownApacheMetrics[key]; ok {
		whiteRegexp := regexp.MustCompile(`\s+`)
		metricName := whiteRegexp.ReplaceAllString(key, "")
		metricValue, err := strconv.ParseFloat(value, 64)

		if err != nil {
			return tmpMetric, err
		}
		m := metric.WithValue(metricName, metricValue)
		m.MetricType = metricType
		return m, nil
	}
	return tmpMetric, errors.New("invalid metric")
}

func extractScoreBoardMetrics(key, value string) []metric.Metric {
	results := []metric.Metric{}
	charCounter := func(str string, pattern string) float64 {
		return float64(strings.Count(str, pattern))
	}
	metricWithValueAndType := func(str string, value float64) metric.Metric {
		m := metric.WithValue(str, value)
		m.MetricType = knownApacheMetrics[str]
		return m
	}
	results = append(results, metricWithValueAndType("IdleWorkers", charCounter(value, "_")))
	results = append(results, metricWithValueAndType("StartingWorkers", charCounter(value, "S")))
	results = append(results, metricWithValueAndType("ReadingWorkers", charCounter(value, "R")))
	results = append(results, metricWithValueAndType("WritingWorkers", charCounter(value, "W")))
	results = append(results, metricWithValueAndType("KeepaliveWorkers", charCounter(value, "K")))
	results = append(results, metricWithValueAndType("DnsWorkers", charCounter(value, "D")))
	results = append(results, metricWithValueAndType("ClosingWorkers", charCounter(value, "C")))
	results = append(results, metricWithValueAndType("LoggingWorkers", charCounter(value, "L")))
	results = append(results, metricWithValueAndType("FinishingWorkers", charCounter(value, "G")))
	results = append(results, metricWithValueAndType("CleanupWorkers", charCounter(value, "I")))
	results = append(results, metricWithValueAndType("StandbyWorkers", charCounter(value, "_")))
	return results
}

func fetchApacheMetrics(endpoint string, timeout int) *nerveHTTPDResponse {
	response := new(nerveHTTPDResponse)
	client := http.Client{
		Timeout: time.Duration(timeout) * time.Second,
	}

	rsp, err := client.Get(endpoint)
	response.err = err
	if rsp != nil {
		response.status = rsp.StatusCode
	}

	if err != nil {
		return response
	}

	txt, err := ioutil.ReadAll(rsp.Body)
	defer rsp.Body.Close()
	if err != nil {
		response.err = err
		return response
	}
	response.data = txt
	return response
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"
	"strconv"
	"strings"

	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/pkg/api/v1"

	"github.com/zalando-incubator/postgres-operator/pkg/spec"
	"github.com/zalando-incubator/postgres-operator/pkg/util"
	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
	"github.com/zalando-incubator/postgres-operator/pkg/util/filesystems"
	"github.com/zalando-incubator/postgres-operator/pkg/util/volumes"
)

func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) {
	ns := c.Namespace
	listOptions := metav1.ListOptions{
		LabelSelector: c.labelsSet(false).String(),
	}

	pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(listOptions)
	if err != nil {
		return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err)
	}
	return pvcs.Items, nil
}

func (c *Cluster) deletePersistenVolumeClaims() error {
	c.logger.Debugln("deleting PVCs")
	pvcs, err := c.listPersistentVolumeClaims()
	if err != nil {
		return err
	}
	for _, pvc := range pvcs {
		c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta))
		if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, c.deleteOptions); err != nil {
			c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err)
		}
	}
	if len(pvcs) > 0 {
		c.logger.Debugln("PVCs have been deleted")
	} else {
		c.logger.Debugln("no PVCs to delete")
	}

	return nil
}

func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
	result := make([]*v1.PersistentVolume, 0)

	pvcs, err := c.listPersistentVolumeClaims()
	if err != nil {
		return nil, fmt.Errorf("could not list cluster's PersistentVolumeClaims: %v", err)
	}
	lastPodIndex := *c.Statefulset.Spec.Replicas - 1
	for _, pvc := range pvcs {
		lastDash := strings.LastIndex(pvc.Name, "-")
		if lastDash > 0 && lastDash < len(pvc.Name)-1 {
			pvcNumber, err := strconv.Atoi(pvc.Name[lastDash+1:])
			if err != nil {
				return nil, fmt.Errorf("could not convert last part of the persistent volume claim name %q to a number", pvc.Name)
			}
			if int32(pvcNumber) > lastPodIndex {
				c.logger.Debugf("skipping persistent volume %q corresponding to a non-running pods", pvc.Name)
				continue
			}
		}
		pv, err := c.KubeClient.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
		if err != nil {
			return nil, fmt.Errorf("could not get PersistentVolume: %v", err)
		}
		result = append(result, pv)
	}

	return result, nil
}

// resizeVolumes resize persistent volumes compatible with the given resizer interface
func (c *Cluster) resizeVolumes(newVolume spec.Volume, resizers []volumes.VolumeResizer) error {
	c.setProcessName("resizing volumes")

	totalCompatible := 0
	newQuantity, err := resource.ParseQuantity(newVolume.Size)
	if err != nil {
		return fmt.Errorf("could not parse volume size: %v", err)
	}
	pvs, newSize, err := c.listVolumesWithManifestSize(newVolume)
	if err != nil {
		return fmt.Errorf("could not list persistent volumes: %v", err)
	}
	for _, pv := range pvs {
		volumeSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
		if volumeSize > newSize {
			return fmt.Errorf("cannot shrink persistent volume")
		}
		if volumeSize == newSize {
			continue
		}
		for _, resizer := range resizers {
			if !resizer.VolumeBelongsToProvider(pv) {
				continue
			}
			totalCompatible++
			if !resizer.IsConnectedToProvider() {
				err := resizer.ConnectToProvider()
				if err != nil {
					return fmt.Errorf("could not connect to the volume provider: %v", err)
				}
				defer func() {
					if err := resizer.DisconnectFromProvider(); err != nil {
						c.logger.Errorf("%v", err)
					}
				}()
			}
			awsVolumeID, err := resizer.GetProviderVolumeID(pv)
			if err != nil {
				return err
			}
			c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
			if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
				return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
			}
			c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
			podName := getPodNameFromPersistentVolume(pv)
			if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
				return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
			}
			c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
			pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
			c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
			if _, err := c.KubeClient.PersistentVolumes().Update(pv); err != nil {
				return fmt.Errorf("could not update persistent volume: %q", err)
			}
			c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
		}
	}
	if len(pvs) > 0 && totalCompatible == 0 {
		return fmt.Errorf("could not resize EBS volumes: persistent volumes are not compatible with existing resizing providers")
	}
	return nil
}

func (c *Cluster) volumesNeedResizing(newVolume spec.Volume) (bool, error) {
	vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
	if err != nil {
		return false, err
	}
	for _, pv := range vols {
		currentSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
		if currentSize != manifestSize {
			return true, nil
		}
	}
	return false, nil
}

func (c *Cluster) listVolumesWithManifestSize(newVolume spec.Volume) ([]*v1.PersistentVolume, int64, error) {
	newSize, err := resource.ParseQuantity(newVolume.Size)
	if err != nil {
		return nil, 0, fmt.Errorf("could not parse volume size from the manifest: %v", err)
	}
	manifestSize := quantityToGigabyte(newSize)
	vols, err := c.listPersistentVolumes()
	if err != nil {
		return nil, 0, fmt.Errorf("could not list persistent volumes: %v", err)
	}
	return vols, manifestSize, nil
}

// getPodNameFromPersistentVolume returns a pod name that it extracts from the volume claim ref.
func getPodNameFromPersistentVolume(pv *v1.PersistentVolume) *spec.NamespacedName {
	namespace := pv.Spec.ClaimRef.Namespace
	name := pv.Spec.ClaimRef.Name[len(constants.DataVolumeName)+1:]
	return &spec.NamespacedName{Namespace: namespace, Name: name}
}

func quantityToGigabyte(q resource.Quantity) int64 {
	return q.ScaledValue(0) / (1 * constants.Gigabyte)
}
<commit_msg>Fix crash during sync.<commit_after>package cluster

import (
	"fmt"
	"strconv"
	"strings"

	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/pkg/api/v1"

	"github.com/zalando-incubator/postgres-operator/pkg/spec"
	"github.com/zalando-incubator/postgres-operator/pkg/util"
	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
	"github.com/zalando-incubator/postgres-operator/pkg/util/filesystems"
	"github.com/zalando-incubator/postgres-operator/pkg/util/volumes"
)

func (c *Cluster) listPersistentVolumeClaims() ([]v1.PersistentVolumeClaim, error) {
	ns := c.Namespace
	listOptions := metav1.ListOptions{
		LabelSelector: c.labelsSet(false).String(),
	}

	pvcs, err := c.KubeClient.PersistentVolumeClaims(ns).List(listOptions)
	if err != nil {
		return nil, fmt.Errorf("could not list of PersistentVolumeClaims: %v", err)
	}
	return pvcs.Items, nil
}

func (c *Cluster) deletePersistenVolumeClaims() error {
	c.logger.Debugln("deleting PVCs")
	pvcs, err := c.listPersistentVolumeClaims()
	if err != nil {
		return err
	}
	for _, pvc := range pvcs {
		c.logger.Debugf("deleting PVC %q", util.NameFromMeta(pvc.ObjectMeta))
		if err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, c.deleteOptions); err != nil {
			c.logger.Warningf("could not delete PersistentVolumeClaim: %v", err)
		}
	}
	if len(pvcs) > 0 {
		c.logger.Debugln("PVCs have been deleted")
	} else {
		c.logger.Debugln("no PVCs to delete")
	}

	return nil
}

func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) {
	result := make([]*v1.PersistentVolume, 0)

	pvcs, err := c.listPersistentVolumeClaims()
	if err != nil {
		return nil, fmt.Errorf("could not list cluster's PersistentVolumeClaims: %v", err)
	}

	pods, err := c.listPods()
	if err != nil {
		return nil, fmt.Errorf("could not get list of running pods for resizing persistent volumes: %v", err)
	}

	lastPodIndex := len(pods) - 1

	for _, pvc := range pvcs {
		lastDash := strings.LastIndex(pvc.Name, "-")
		if lastDash > 0 && lastDash < len(pvc.Name)-1 {
			pvcNumber, err := strconv.Atoi(pvc.Name[lastDash+1:])
			if err != nil {
				return nil, fmt.Errorf("could not convert last part of the persistent volume claim name %q to a number", pvc.Name)
			}
			if pvcNumber > lastPodIndex {
				c.logger.Debugf("skipping persistent volume %q corresponding to a non-running pods", pvc.Name)
				continue
			}
		}
		pv, err := c.KubeClient.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{})
		if err != nil {
			return nil, fmt.Errorf("could not get PersistentVolume: %v", err)
		}
		result = append(result, pv)
	}

	return result, nil
}

// resizeVolumes resize persistent volumes compatible with the given resizer interface
func (c *Cluster) resizeVolumes(newVolume spec.Volume, resizers []volumes.VolumeResizer) error {
	c.setProcessName("resizing volumes")

	totalCompatible := 0
	newQuantity, err := resource.ParseQuantity(newVolume.Size)
	if err != nil {
		return fmt.Errorf("could not parse volume size: %v", err)
	}
	pvs, newSize, err := c.listVolumesWithManifestSize(newVolume)
	if err != nil {
		return fmt.Errorf("could not list persistent volumes: %v", err)
	}

	for _, pv := range pvs {
		volumeSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
		if volumeSize > newSize {
			return fmt.Errorf("cannot shrink persistent volume")
		}
		if volumeSize == newSize {
			continue
		}
		for _, resizer := range resizers {
			if !resizer.VolumeBelongsToProvider(pv) {
				continue
			}
			totalCompatible++
			if !resizer.IsConnectedToProvider() {
				err := resizer.ConnectToProvider()
				if err != nil {
					return fmt.Errorf("could not connect to the volume provider: %v", err)
				}
				defer func() {
					if err := resizer.DisconnectFromProvider(); err != nil {
						c.logger.Errorf("%v", err)
					}
				}()
			}
			awsVolumeID, err := resizer.GetProviderVolumeID(pv)
			if err != nil {
				return err
			}
			c.logger.Debugf("updating persistent volume %q to %d", pv.Name, newSize)
			if err := resizer.ResizeVolume(awsVolumeID, newSize); err != nil {
				return fmt.Errorf("could not resize EBS volume %q: %v", awsVolumeID, err)
			}
			c.logger.Debugf("resizing the filesystem on the volume %q", pv.Name)
			podName := getPodNameFromPersistentVolume(pv)
			if err := c.resizePostgresFilesystem(podName, []filesystems.FilesystemResizer{&filesystems.Ext234Resize{}}); err != nil {
				return fmt.Errorf("could not resize the filesystem on pod %q: %v", podName, err)
			}
			c.logger.Debugf("filesystem resize successful on volume %q", pv.Name)
			pv.Spec.Capacity[v1.ResourceStorage] = newQuantity
			c.logger.Debugf("updating persistent volume definition for volume %q", pv.Name)
			if _, err := c.KubeClient.PersistentVolumes().Update(pv); err != nil {
				return fmt.Errorf("could not update persistent volume: %q", err)
			}
			c.logger.Debugf("successfully updated persistent volume %q", pv.Name)
		}
	}
	if len(pvs) > 0 && totalCompatible == 0 {
		return fmt.Errorf("could not resize EBS volumes: persistent volumes are not compatible with existing resizing providers")
	}
	return nil
}

func (c *Cluster) volumesNeedResizing(newVolume spec.Volume) (bool, error) {
	vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume)
	if err != nil {
		return false, err
	}
	for _, pv := range vols {
		currentSize := quantityToGigabyte(pv.Spec.Capacity[v1.ResourceStorage])
		if currentSize != manifestSize {
			return true, nil
		}
	}
	return false, nil
}

func (c *Cluster) listVolumesWithManifestSize(newVolume spec.Volume) ([]*v1.PersistentVolume, int64, error) {
	newSize, err := resource.ParseQuantity(newVolume.Size)
	if err != nil {
		return nil, 0, fmt.Errorf("could not parse volume size from the manifest: %v", err)
	}
	manifestSize := quantityToGigabyte(newSize)
	vols, err := c.listPersistentVolumes()
	if err != nil {
		return nil, 0, fmt.Errorf("could not list persistent volumes: %v", err)
	}
	return vols, manifestSize, nil
}

// getPodNameFromPersistentVolume returns a pod name that it extracts from the volume claim ref.
func getPodNameFromPersistentVolume(pv *v1.PersistentVolume) *spec.NamespacedName {
	namespace := pv.Spec.ClaimRef.Namespace
	name := pv.Spec.ClaimRef.Name[len(constants.DataVolumeName)+1:]
	return &spec.NamespacedName{Namespace: namespace, Name: name}
}

func quantityToGigabyte(q resource.Quantity) int64 {
	return q.ScaledValue(0) / (1 * constants.Gigabyte)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	osclientcmd "github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/radanalyticsio/oshinko-cli/pkg/cmd/cli/auth"
	"github.com/spf13/cobra"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)

type CmdOptions struct {
	Name            string `json:"name,omitempty"`
	Image           string `json:"image"`
	WorkerCount     int    `json:"workerCount"`
	MasterCount     int    `json:"masterCount,omitempty"`
	MasterConfig    string `json:"sparkMasterConfig,omitempty"`
	WorkerConfig    string `json:"workerConfig,omitempty"`
	StoredConfig    string `json:"storedConfig,omitempty"`
	Verbose         bool
	Output          string

	auth.AuthOptions
}

func (o *CmdOptions) Complete(f *osclientcmd.Factory, cmd *cobra.Command, args []string) error {
	o.Image = defaultImage

	// Pod counts will be assigned by the default config in oshinko-core,
	// here values should be defaulted to 0
	o.WorkerCount = 0
	o.MasterCount = 0

	currentCluster, err := NameFromCommandArgs(cmd, args)
	if err != nil {
		return err
	}
	o.Name = currentCluster
	if err := o.AuthOptions.Complete(f, cmd, args); err != nil {
		kcmdutil.CheckErr(kcmdutil.UsageError(cmd, err.Error()))
	}
	if err := o.GatherInfo(); err != nil {
		return err
	}
	if cmd.Flags().Lookup("verbose") != nil {
		o.Verbose = kcmdutil.GetFlagBool(cmd, "verbose")
	}
	if cmd.Flags().Lookup("output") != nil {
		o.Output = kcmdutil.GetFlagString(cmd, "output")
		if o.Output != "yaml" || o.Output != "json" {
			cmdutil.UsageError(cmd, "INVALID output format only yaml|json allowed")
		}
	}
	if cmd.Flags().Lookup("workers") != nil {
		o.WorkerCount = kcmdutil.GetFlagInt(cmd, "workers")
	}
	if cmd.Flags().Lookup("masters") != nil {
		o.MasterCount = kcmdutil.GetFlagInt(cmd, "masters")
	}
	if cmd.Flags().Lookup("image") != nil {
		o.Image = kcmdutil.GetFlagString(cmd, "image")
	}
        if cmd.Flags().Lookup("storedconfig") != nil {
		o.StoredConfig = kcmdutil.GetFlagString(cmd, "storedconfig")
	}
	return nil
}

func (o *CmdOptions) GatherInfo() error {
	var msg string
	var err error
	if msg, err = o.AuthOptions.GatherAuthInfo(); err != nil {
		return err
	}
	if o.Verbose {
		fmt.Printf(msg)
	}
	if msg, err = o.AuthOptions.GatherProjectInfo(); err != nil {
		return err
	}
	if o.Verbose {
		fmt.Printf(msg)
	}
	return nil
}
<commit_msg>Master and worker config flags not being set from commandline<commit_after>package cmd

import (
	"fmt"
	osclientcmd "github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/radanalyticsio/oshinko-cli/pkg/cmd/cli/auth"
	"github.com/spf13/cobra"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)

type CmdOptions struct {
	Name            string `json:"name,omitempty"`
	Image           string `json:"image"`
	WorkerCount     int    `json:"workerCount"`
	MasterCount     int    `json:"masterCount,omitempty"`
	MasterConfig    string `json:"sparkMasterConfig,omitempty"`
	WorkerConfig    string `json:"workerConfig,omitempty"`
	StoredConfig    string `json:"storedConfig,omitempty"`
	Verbose         bool
	Output          string

	auth.AuthOptions
}

func (o *CmdOptions) Complete(f *osclientcmd.Factory, cmd *cobra.Command, args []string) error {
	o.Image = defaultImage

	// Pod counts will be assigned by the default config in oshinko-core,
	// here values should be defaulted to 0
	o.WorkerCount = 0
	o.MasterCount = 0

	currentCluster, err := NameFromCommandArgs(cmd, args)
	if err != nil {
		return err
	}
	o.Name = currentCluster
	if err := o.AuthOptions.Complete(f, cmd, args); err != nil {
		kcmdutil.CheckErr(kcmdutil.UsageError(cmd, err.Error()))
	}
	if err := o.GatherInfo(); err != nil {
		return err
	}
	if cmd.Flags().Lookup("verbose") != nil {
		o.Verbose = kcmdutil.GetFlagBool(cmd, "verbose")
	}
	if cmd.Flags().Lookup("output") != nil {
		o.Output = kcmdutil.GetFlagString(cmd, "output")
		if o.Output != "yaml" || o.Output != "json" {
			cmdutil.UsageError(cmd, "INVALID output format only yaml|json allowed")
		}
	}
	if cmd.Flags().Lookup("workers") != nil {
		o.WorkerCount = kcmdutil.GetFlagInt(cmd, "workers")
	}
	if cmd.Flags().Lookup("masters") != nil {
		o.MasterCount = kcmdutil.GetFlagInt(cmd, "masters")
	}
	if cmd.Flags().Lookup("image") != nil {
		o.Image = kcmdutil.GetFlagString(cmd, "image")
	}
	if cmd.Flags().Lookup("masterconfig") != nil {
		o.MasterConfig = kcmdutil.GetFlagString(cmd, "masterconfig")
	}
	if cmd.Flags().Lookup("workerconfig") != nil {
		o.WorkerConfig = kcmdutil.GetFlagString(cmd, "workerconfig")
	}
        if cmd.Flags().Lookup("storedconfig") != nil {
		o.StoredConfig = kcmdutil.GetFlagString(cmd, "storedconfig")
	}
	return nil
}

func (o *CmdOptions) GatherInfo() error {
	var msg string
	var err error
	if msg, err = o.AuthOptions.GatherAuthInfo(); err != nil {
		return err
	}
	if o.Verbose {
		fmt.Printf(msg)
	}
	if msg, err = o.AuthOptions.GatherProjectInfo(); err != nil {
		return err
	}
	if o.Verbose {
		fmt.Printf(msg)
	}
	return nil
}
<|endoftext|>