<commit_before>/* Licensed to Elodina Inc. under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package gonzo

import (
	"github.com/elodina/siesta"
	"github.com/rcrowley/go-metrics"
	"sync/atomic"
	"time"
)

// PartitionConsumer is an interface responsible for consuming exactly one topic/partition
// from Kafka. Used to switch between PartitionConsumer in live mode and MockPartitionConsumer in tests.
type PartitionConsumer interface {
	// Start starts consuming given topic/partition.
	Start()

	// Stop stops consuming given topic/partition.
	Stop()

	// Offset returns the last fetched offset for this partition consumer.
	Offset() int64

	// Commit commits the given offset to Kafka. Returns an error on unsuccessful commit.
	Commit(offset int64) error

	// SetOffset overrides the current fetch offset value for given topic/partition.
	// This does not commit offset but allows you to move back and forth throughout the partition.
	SetOffset(offset int64)

	// Lag returns the difference between the latest available offset in the partition and the
	// latest fetched offset by this consumer. This allows you to see how much behind the consumer is.
	Lag() int64

	// Metrics returns a metrics registry for this partition consumer. An error is returned if metrics are disabled.
	Metrics() (metrics.Registry, error)
}

// KafkaPartitionConsumer serves to consume exactly one topic/partition from Kafka.
// This is very similar to JVM SimpleConsumer except the PartitionConsumer is able to handle
// leader changes and supports committing offsets to Kafka via Siesta client.
type KafkaPartitionConsumer struct {
	client              Client
	config              *ConsumerConfig
	topic               string
	partition           int32
	offset              int64
	highwaterMarkOffset int64
	strategy            Strategy
	metrics             PartitionConsumerMetrics
	stop                chan struct{}
}

// NewPartitionConsumer creates a new PartitionConsumer for given client and config that will
// consume given topic and partition.
// The message processing logic is passed via strategy.
func NewPartitionConsumer(client Client, config *ConsumerConfig, topic string, partition int32, strategy Strategy) PartitionConsumer {
	var metrics PartitionConsumerMetrics = noOpPartitionConsumerMetrics
	if config.EnableMetrics {
		metrics = NewKafkaPartitionConsumerMetrics(topic, partition)
	}

	return &KafkaPartitionConsumer{
		client:    client,
		config:    config,
		topic:     topic,
		partition: partition,
		strategy:  strategy,
		metrics:   metrics,
		stop:      make(chan struct{}, 1),
	}
}

// Start starts consuming a single partition from Kafka.
// This call blocks until Stop() is called.
func (pc *KafkaPartitionConsumer) Start() {
	Logger.Info("Starting partition consumer for topic %s, partition %d", pc.topic, pc.partition)
	proceed := pc.initOffset()
	if !proceed {
		return
	}

	for {
		select {
		case <-pc.stop:
			{
				Logger.Info("Stopping fetcher loop for topic %s, partition %d", pc.topic, pc.partition)
				return
			}
		default:
			{
				response, err := pc.client.Fetch(pc.topic, pc.partition, atomic.LoadInt64(&pc.offset))
				pc.metrics.NumFetches(func(numFetches metrics.Counter) {
					numFetches.Inc(1)
				})
				if err != nil {
					Logger.Warn("Fetch error: %s", err)
					pc.metrics.NumFailedFetches(func(numFailedFetches metrics.Counter) {
						numFailedFetches.Inc(1)
					})
					pc.strategy(&FetchData{
						Messages: nil,
						Error:    err,
					}, pc)
					continue
				}

				data := response.Data[pc.topic][pc.partition]
				atomic.StoreInt64(&pc.highwaterMarkOffset, data.HighwaterMarkOffset)
				pc.metrics.Lag(func(lag metrics.Gauge) {
					lag.Update(pc.Lag())
				})

				if len(data.Messages) == 0 {
					pc.metrics.NumEmptyFetches(func(numEmptyFetches metrics.Counter) {
						numEmptyFetches.Inc(1)
					})
					continue
				}

				// store the offset before we actually hand off messages to user
				if len(data.Messages) > 0 {
					offsetIndex := len(data.Messages) - 1
					atomic.StoreInt64(&pc.offset, data.Messages[offsetIndex].Offset+1)
				}

				//TODO siesta could probably support size hints? feel like quick traversal of messages should be quicker
				// than appending to a slice if it resizes internally, should benchmark this
				var messages []*MessageAndMetadata
				collector := pc.collectorFunc(&messages)
				err = response.CollectMessages(collector)
				if err != nil {
					pc.metrics.NumFailedFetches(func(numFetches metrics.Counter) {
						numFetches.Inc(1)
					})
				}

				pc.metrics.NumFetchedMessages(func(numFetchedMessages metrics.Counter) {
					numFetchedMessages.Inc(int64(len(data.Messages)))
				})

				pc.strategy(&FetchData{
					Messages: messages,
					Error:    err,
				}, pc)

				if pc.config.AutoCommitEnable && len(messages) > 0 {
					offset := messages[len(messages)-1].Offset
					err = pc.Commit(offset)
					if err != nil {
						Logger.Warn("Could not commit offset %d for topic %s, partition %d", offset, pc.topic, pc.partition)
					}
				}
			}
		}
	}
}

// Stop stops consuming partition from Kafka.
// This means the PartitionConsumer will stop accepting new batches but will have a chance to finish its current work.
func (pc *KafkaPartitionConsumer) Stop() {
	Logger.Info("Stopping partition consumer for topic %s, partition %d", pc.topic, pc.partition)
	pc.stop <- struct{}{}
	pc.metrics.Stop()
}

// Commit commits the given offset to Kafka. Returns an error on unsuccessful commit.
func (pc *KafkaPartitionConsumer) Commit(offset int64) error {
	pc.metrics.NumOffsetCommits(func(numOffsetCommits metrics.Counter) {
		numOffsetCommits.Inc(1)
	})
	err := pc.client.CommitOffset(pc.config.Group, pc.topic, pc.partition, offset)
	if err != nil {
		pc.metrics.NumFailedOffsetCommits(func(numFetchedMessages metrics.Counter) {
			numFetchedMessages.Inc(1)
		})
	}

	return err
}

// SetOffset overrides the current fetch offset value for given topic/partition.
// This does not commit offset but allows you to move back and forth throughout the partition.
func (pc *KafkaPartitionConsumer) SetOffset(offset int64) {
	atomic.StoreInt64(&pc.offset, offset)
}

// Offset returns the last fetched offset for this partition consumer.
func (pc *KafkaPartitionConsumer) Offset() int64 {
	return atomic.LoadInt64(&pc.offset)
}

// Lag returns the difference between the latest available offset in the partition and the
// latest fetched offset by this consumer. This allows you to see how much behind the consumer is.
func (pc *KafkaPartitionConsumer) Lag() int64 {
	return atomic.LoadInt64(&pc.highwaterMarkOffset) - atomic.LoadInt64(&pc.offset)
}

// Metrics returns a metrics registry for this partition consumer. An error is returned if metrics are disabled.
func (pc *KafkaPartitionConsumer) Metrics() (metrics.Registry, error) {
	if !pc.config.EnableMetrics {
		return nil, ErrMetricsDisabled
	}

	return pc.metrics.Registry(), nil
}

func (pc *KafkaPartitionConsumer) initOffset() bool {
	for {
		offset, err := pc.client.GetOffset(pc.config.Group, pc.topic, pc.partition)
		if err != nil {
			if err == siesta.ErrUnknownTopicOrPartition {
				return pc.resetOffset()
			}
			Logger.Warn("Cannot get offset for group %s, topic %s, partition %d: %s\n", pc.config.Group, pc.topic, pc.partition, err)
			select {
			case <-pc.stop:
				{
					Logger.Warn("PartitionConsumer told to stop trying to get offset, returning")
					return false
				}
			default:
			}
		} else {
			atomic.StoreInt64(&pc.offset, offset)
			atomic.StoreInt64(&pc.highwaterMarkOffset, offset)
			return true
		}
		time.Sleep(pc.config.InitOffsetBackoff)
	}
}

func (pc *KafkaPartitionConsumer) resetOffset() bool {
	for {
		offset, err := pc.client.GetAvailableOffset(pc.topic, pc.partition, pc.config.AutoOffsetReset)
		if err != nil {
			Logger.Warn("Cannot get available offset for topic %s, partition %d: %s", pc.topic, pc.partition, err)
			select {
			case <-pc.stop:
				{
					Logger.Warn("PartitionConsumer told to stop trying to get offset, returning")
					return false
				}
			default:
			}
		} else {
			atomic.StoreInt64(&pc.offset, offset)
			atomic.StoreInt64(&pc.highwaterMarkOffset, offset)
			return true
		}
		time.Sleep(pc.config.InitOffsetBackoff)
	}
}

func (pc *KafkaPartitionConsumer) collectorFunc(messages *[]*MessageAndMetadata) func(topic string, partition int32, offset int64, key []byte, value []byte) {
	return func(topic string, partition int32, offset int64, key []byte, value []byte) {
		decodedKey, err := pc.config.KeyDecoder.Decode(key)
		if err != nil {
			//TODO siesta should support collector function to return an error
			Logger.Warn(err.Error())
		}
		decodedValue, err := pc.config.ValueDecoder.Decode(value)
		if err != nil {
			//TODO siesta should support collector function to return an error
			Logger.Warn(err.Error())
		}

		*messages = append(*messages, &MessageAndMetadata{
			Key:          key,
			Value:        value,
			Topic:        topic,
			Partition:    partition,
			Offset:       offset,
			DecodedKey:   decodedKey,
			DecodedValue: decodedValue,
		})
	}
}

// Strategy is a function that actually processes Kafka messages.
// FetchData contains actual messages, highwater mark offset and fetch error.
// PartitionConsumer which is passed to this function allows to commit/rewind offset if necessary,
// track offset/lag, stop the consumer. Please note that you should NOT stop the consumer if using
// Consumer but rather use consumer.Remove(topic, partition) call.
// The processing happens on per-partition level - the amount of strategies running simultaneously is defined by the
// number of partitions being consumed. The next batch for topic/partition won't start until the previous one
// finishes.
type Strategy func(data *FetchData, consumer *KafkaPartitionConsumer)
<commit_msg>Fix collectorFunc to return error.<commit_after>/* Licensed to Elodina Inc. under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package gonzo

import (
	"github.com/elodina/siesta"
	"github.com/rcrowley/go-metrics"
	"sync/atomic"
	"time"
)

// PartitionConsumer is an interface responsible for consuming exactly one topic/partition
// from Kafka. Used to switch between PartitionConsumer in live mode and MockPartitionConsumer in tests.
type PartitionConsumer interface {
	// Start starts consuming given topic/partition.
	Start()

	// Stop stops consuming given topic/partition.
	Stop()

	// Offset returns the last fetched offset for this partition consumer.
	Offset() int64

	// Commit commits the given offset to Kafka. Returns an error on unsuccessful commit.
	Commit(offset int64) error

	// SetOffset overrides the current fetch offset value for given topic/partition.
	// This does not commit offset but allows you to move back and forth throughout the partition.
	SetOffset(offset int64)

	// Lag returns the difference between the latest available offset in the partition and the
	// latest fetched offset by this consumer. This allows you to see how much behind the consumer is.
	Lag() int64

	// Metrics returns a metrics registry for this partition consumer. An error is returned if metrics are disabled.
	Metrics() (metrics.Registry, error)
}

// KafkaPartitionConsumer serves to consume exactly one topic/partition from Kafka.
// This is very similar to JVM SimpleConsumer except the PartitionConsumer is able to handle
// leader changes and supports committing offsets to Kafka via Siesta client.
type KafkaPartitionConsumer struct {
	client              Client
	config              *ConsumerConfig
	topic               string
	partition           int32
	offset              int64
	highwaterMarkOffset int64
	strategy            Strategy
	metrics             PartitionConsumerMetrics
	stop                chan struct{}
}

// NewPartitionConsumer creates a new PartitionConsumer for given client and config that will
// consume given topic and partition.
// The message processing logic is passed via strategy.
func NewPartitionConsumer(client Client, config *ConsumerConfig, topic string, partition int32, strategy Strategy) PartitionConsumer {
	var metrics PartitionConsumerMetrics = noOpPartitionConsumerMetrics
	if config.EnableMetrics {
		metrics = NewKafkaPartitionConsumerMetrics(topic, partition)
	}

	return &KafkaPartitionConsumer{
		client:    client,
		config:    config,
		topic:     topic,
		partition: partition,
		strategy:  strategy,
		metrics:   metrics,
		stop:      make(chan struct{}, 1),
	}
}

// Start starts consuming a single partition from Kafka.
// This call blocks until Stop() is called.
func (pc *KafkaPartitionConsumer) Start() {
	Logger.Info("Starting partition consumer for topic %s, partition %d", pc.topic, pc.partition)
	proceed := pc.initOffset()
	if !proceed {
		return
	}

	for {
		select {
		case <-pc.stop:
			{
				Logger.Info("Stopping fetcher loop for topic %s, partition %d", pc.topic, pc.partition)
				return
			}
		default:
			{
				response, err := pc.client.Fetch(pc.topic, pc.partition, atomic.LoadInt64(&pc.offset))
				pc.metrics.NumFetches(func(numFetches metrics.Counter) {
					numFetches.Inc(1)
				})
				if err != nil {
					Logger.Warn("Fetch error: %s", err)
					pc.metrics.NumFailedFetches(func(numFailedFetches metrics.Counter) {
						numFailedFetches.Inc(1)
					})
					pc.strategy(&FetchData{
						Messages: nil,
						Error:    err,
					}, pc)
					continue
				}

				data := response.Data[pc.topic][pc.partition]
				atomic.StoreInt64(&pc.highwaterMarkOffset, data.HighwaterMarkOffset)
				pc.metrics.Lag(func(lag metrics.Gauge) {
					lag.Update(pc.Lag())
				})

				if len(data.Messages) == 0 {
					pc.metrics.NumEmptyFetches(func(numEmptyFetches metrics.Counter) {
						numEmptyFetches.Inc(1)
					})
					continue
				}

				// store the offset before we actually hand off messages to user
				if len(data.Messages) > 0 {
					offsetIndex := len(data.Messages) - 1
					atomic.StoreInt64(&pc.offset, data.Messages[offsetIndex].Offset+1)
				}

				//TODO siesta could probably support size hints? feel like quick traversal of messages should be quicker
				// than appending to a slice if it resizes internally, should benchmark this
				var messages []*MessageAndMetadata
				collector := pc.collectorFunc(&messages)
				err = response.CollectMessages(collector)
				if err != nil {
					pc.metrics.NumFailedFetches(func(numFetches metrics.Counter) {
						numFetches.Inc(1)
					})
				}

				pc.metrics.NumFetchedMessages(func(numFetchedMessages metrics.Counter) {
					numFetchedMessages.Inc(int64(len(data.Messages)))
				})

				pc.strategy(&FetchData{
					Messages: messages,
					Error:    err,
				}, pc)

				if pc.config.AutoCommitEnable && len(messages) > 0 {
					offset := messages[len(messages)-1].Offset
					err = pc.Commit(offset)
					if err != nil {
						Logger.Warn("Could not commit offset %d for topic %s, partition %d", offset, pc.topic, pc.partition)
					}
				}
			}
		}
	}
}

// Stop stops consuming partition from Kafka.
// This means the PartitionConsumer will stop accepting new batches but will have a chance to finish its current work.
func (pc *KafkaPartitionConsumer) Stop() {
	Logger.Info("Stopping partition consumer for topic %s, partition %d", pc.topic, pc.partition)
	pc.stop <- struct{}{}
	pc.metrics.Stop()
}

// Commit commits the given offset to Kafka. Returns an error on unsuccessful commit.
func (pc *KafkaPartitionConsumer) Commit(offset int64) error {
	pc.metrics.NumOffsetCommits(func(numOffsetCommits metrics.Counter) {
		numOffsetCommits.Inc(1)
	})
	err := pc.client.CommitOffset(pc.config.Group, pc.topic, pc.partition, offset)
	if err != nil {
		pc.metrics.NumFailedOffsetCommits(func(numFetchedMessages metrics.Counter) {
			numFetchedMessages.Inc(1)
		})
	}

	return err
}

// SetOffset overrides the current fetch offset value for given topic/partition.
// This does not commit offset but allows you to move back and forth throughout the partition.
func (pc *KafkaPartitionConsumer) SetOffset(offset int64) {
	atomic.StoreInt64(&pc.offset, offset)
}

// Offset returns the last fetched offset for this partition consumer.
func (pc *KafkaPartitionConsumer) Offset() int64 {
	return atomic.LoadInt64(&pc.offset)
}

// Lag returns the difference between the latest available offset in the partition and the
// latest fetched offset by this consumer. This allows you to see how much behind the consumer is.
func (pc *KafkaPartitionConsumer) Lag() int64 {
	return atomic.LoadInt64(&pc.highwaterMarkOffset) - atomic.LoadInt64(&pc.offset)
}

// Metrics returns a metrics registry for this partition consumer. An error is returned if metrics are disabled.
func (pc *KafkaPartitionConsumer) Metrics() (metrics.Registry, error) {
	if !pc.config.EnableMetrics {
		return nil, ErrMetricsDisabled
	}

	return pc.metrics.Registry(), nil
}

func (pc *KafkaPartitionConsumer) initOffset() bool {
	for {
		offset, err := pc.client.GetOffset(pc.config.Group, pc.topic, pc.partition)
		if err != nil {
			if err == siesta.ErrUnknownTopicOrPartition {
				return pc.resetOffset()
			}
			Logger.Warn("Cannot get offset for group %s, topic %s, partition %d: %s\n", pc.config.Group, pc.topic, pc.partition, err)
			select {
			case <-pc.stop:
				{
					Logger.Warn("PartitionConsumer told to stop trying to get offset, returning")
					return false
				}
			default:
			}
		} else {
			atomic.StoreInt64(&pc.offset, offset)
			atomic.StoreInt64(&pc.highwaterMarkOffset, offset)
			return true
		}
		time.Sleep(pc.config.InitOffsetBackoff)
	}
}

func (pc *KafkaPartitionConsumer) resetOffset() bool {
	for {
		offset, err := pc.client.GetAvailableOffset(pc.topic, pc.partition, pc.config.AutoOffsetReset)
		if err != nil {
			Logger.Warn("Cannot get available offset for topic %s, partition %d: %s", pc.topic, pc.partition, err)
			select {
			case <-pc.stop:
				{
					Logger.Warn("PartitionConsumer told to stop trying to get offset, returning")
					return false
				}
			default:
			}
		} else {
			atomic.StoreInt64(&pc.offset, offset)
			atomic.StoreInt64(&pc.highwaterMarkOffset, offset)
			return true
		}
		time.Sleep(pc.config.InitOffsetBackoff)
	}
}

func (pc *KafkaPartitionConsumer) collectorFunc(messages *[]*MessageAndMetadata) func(topic string, partition int32, offset int64, key []byte, value []byte) error {
	return func(topic string, partition int32, offset int64, key []byte, value []byte) error {
		decodedKey, err := pc.config.KeyDecoder.Decode(key)
		if err != nil {
			//TODO siesta should support collector function to return an error
			Logger.Warn(err.Error())
			return err
		}
		decodedValue, err := pc.config.ValueDecoder.Decode(value)
		if err != nil {
			//TODO siesta should support collector function to return an error
			Logger.Warn(err.Error())
			return err
		}

		*messages = append(*messages, &MessageAndMetadata{
			Key:          key,
			Value:        value,
			Topic:        topic,
			Partition:    partition,
			Offset:       offset,
			DecodedKey:   decodedKey,
			DecodedValue: decodedValue,
		})
		return nil
	}
}

// Strategy is a function that actually processes Kafka messages.
// FetchData contains actual messages, highwater mark offset and fetch error.
// PartitionConsumer which is passed to this function allows to commit/rewind offset if necessary,
// track offset/lag, stop the consumer. Please note that you should NOT stop the consumer if using
// Consumer but rather use consumer.Remove(topic, partition) call.
// The processing happens on per-partition level - the amount of strategies running simultaneously is defined by the
// number of partitions being consumed. The next batch for topic/partition won't start until the previous one
// finishes.
type Strategy func(data *FetchData, consumer *KafkaPartitionConsumer)
<|endoftext|>


<commit_before><commit_msg>Prometheus config map parse error bug fix<commit_after><|endoftext|>


<commit_before>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main
type I int
type S struct { f map[I]int }
var v1 = S{ make(map[int]int) }		// OK--names are ignored.
var v2 map[I]int = map[int]int{}	// OK.
var v3 = S{ make(map[uint]int) }	// ERROR "cannot|illegal|incompatible|wrong"
<commit_msg>We should not silently permit a named type to match an unnamed type.  This is OK in a conversion but not in an assignment.<commit_after>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main
type I int
type S struct { f map[I]int }
var v1 = S{ make(map[int]int) }		// ERROR "cannot|illegal|incompatible|wrong"
var v2 map[I]int = map[int]int{}	// ERROR "cannot|illegal|incompatible|wrong"
var v3 = S{ make(map[uint]int) }	// ERROR "cannot|illegal|incompatible|wrong"
<|endoftext|>


<commit_before>package main

import (
	"os"
	"archive/zip"
	"path/filepath"
	"io"
	"fmt"
	"errors"
)

const (
	target   string = "./"
	archive  string = "./testfixture/phlow-test-pkg.zip"
	testPath string = ""
)

var (
	GoPathNotSet error = errors.New("GOPATH is empty")
	goPath       string
)

//init
//Runs before functions to setup variable gopath
func init() {
	goPath = os.Getenv("GOPATH")
	if len(goPath) == 0 {
		fmt.Fprintln(os.Stdout, GoPathNotSet)
		os.Exit(1)
	}
}

//unzip
//unzips archive to target directory
func unzip(archive, target string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}

	for _, file := range reader.File {
		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, file.Mode())
			continue
		}

		fileReader, err := file.Open()
		if err != nil {
			return err
		}
		defer fileReader.Close()

		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer targetFile.Close()

		if _, err := io.Copy(targetFile, fileReader); err != nil {
			return err
		}
	}

	return nil
}

//SetupTestRepo
func SetupTestRepo() {
	err := unzip(archive, target)

	if err != nil {
		fmt.Fprintln(os.Stdout, err.Error())
		os.Exit(1)
	}
	fmt.Fprintln(os.Stdout, "Local test repository created from 'zip'")
}

//TearDownTestRepo
func TearDownTestRepo() {

}

func main() {
	SetupTestRepo()
}
<commit_msg>added comments to testfixture<commit_after>package main

import (
	"os"
	"archive/zip"
	"path/filepath"
	"io"
	"fmt"
	"errors"
)

const (
	target   string = "./"
	archive  string = "./testfixture/phlow-test-pkg.zip"
	testPath string = "./phlow-test-pkg"
)

var (
	GoPathNotSet error = errors.New("GOPATH is empty")
	goPath       string
)

//init
//Runs before functions to setup variable gopath
func init() {
	goPath = os.Getenv("GOPATH")
	if len(goPath) == 0 {
		fmt.Fprintln(os.Stdout, GoPathNotSet)
		os.Exit(1)
	}
}

//unzip
//unzips archive to target directory
func unzip(archive, target string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}

	for _, file := range reader.File {
		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, file.Mode())
			continue
		}

		fileReader, err := file.Open()
		if err != nil {
			return err
		}
		defer fileReader.Close()

		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer targetFile.Close()

		if _, err := io.Copy(targetFile, fileReader); err != nil {
			return err
		}
	}

	return nil
}

//SetupTestRepo
//Creates git test repository from a zip file in /testfixture
func SetupTestRepo() {
	err := unzip(archive, target)

	if err != nil {
		fmt.Fprintln(os.Stdout, err.Error())
		os.Exit(1)
	}
	fmt.Fprintln(os.Stdout, "Local test repository created from 'zip'")
}

//TearDownTestRepo
//removes the unzipped test repository is it exists
func TearDownTestRepo() {

	err := os.RemoveAll(testPath)
	if err != nil {
		fmt.Fprintln(os.Stdout, err.Error())
		os.Exit(1)
	}
	fmt.Fprintln(os.Stdout, "Deleted local test repository")
}

func main() {
	SetupTestRepo()
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package semtech

import (
	"fmt"
	"net"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/semtech"
	"github.com/apex/log"
)

// Adapter represents a semtech adapter which sends and receives packet via UDP in respect of the
// semtech forwarder protocol
type Adapter struct {
	ctx  log.Interface // Just a logger
	conn chan udpMsg   // Channel used to manage response transmissions made by multiple goroutines
	next chan rxpkMsg  // Incoming valid RXPK packets are pushed to this channel
}

// udpMsg type materializes response messages transmitted towards existing recipients (commonly,
// gateways).
type udpMsg struct {
	conn *net.UDPConn // Provide if you intent to change the current adapter connection
	addr *net.UDPAddr // The target recipient address
	raw  []byte       // The raw byte sequence that has to be sent
}

// rxpkMsg type materializes valid uplink messages coming from a given recipient
type rxpkMsg struct {
	rxpk      semtech.RXPK   // The actual RXPK message
	recipient core.Recipient // The address and id of the source emitter
}

var ErrInvalidPort error = fmt.Errorf("Invalid port supplied. The connection might be already taken")
var ErrNotInitialized error = fmt.Errorf("Illegal call on non-initialized adapter")
var ErrNotSupported error = fmt.Errorf("Unsupported operation")
var ErrInvalidPacket error = fmt.Errorf("Invalid packet supplied")

// NewAdapter constructs and allocates a new semtech adapter
func NewAdapter(port uint, ctx log.Interface) (*Adapter, error) {
	a := Adapter{
		ctx:  ctx,
		conn: make(chan udpMsg),
		next: make(chan rxpkMsg),
	}

	// Create the udp connection and start listening with a goroutine
	var udpConn *net.UDPConn
	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("0.0.0.0:%d", port))
	a.ctx.WithField("port", port).Info("Starting Server")
	if udpConn, err = net.ListenUDP("udp", addr); err != nil {
		a.ctx.WithError(err).Error("Unable to start server")
		return nil, ErrInvalidPort
	}

	go a.monitorConnection()
	a.conn <- udpMsg{conn: udpConn}
	go a.listen(udpConn) // Terminates when the connection is closed

	return &a, nil
}

// ok controls whether or not the adapter has been initialized via NewAdapter()
func (a *Adapter) ok() bool {
	return a != nil && a.conn != nil && a.next != nil
}

// Send implements the core.Adapter interface. Not implemented for the semtech adapter.
func (a *Adapter) Send(p core.Packet, r ...core.Recipient) (core.Packet, error) {
	return core.Packet{}, ErrNotSupported
}

// Next implements the core.Adapter interface
func (a *Adapter) Next() (core.Packet, core.AckNacker, error) {
	if !a.ok() {
		return core.Packet{}, nil, ErrNotInitialized
	}
	msg := <-a.next
	packet, err := core.ConvertRXPK(msg.rxpk)
	if err != nil {
		a.ctx.Debug("Received invalid packet")
		return core.Packet{}, nil, ErrInvalidPacket
	}
	return packet, semtechAckNacker{recipient: msg.recipient, conn: a.conn}, nil
}

// NextRegistration implements the core.Adapter interface
func (a *Adapter) NextRegistration() (core.Registration, core.AckNacker, error) {
	return core.Registration{}, nil, ErrNotSupported
}

// listen Handle incoming packets and forward them
func (a *Adapter) listen(conn *net.UDPConn) {
	defer conn.Close()
	a.ctx.WithField("address", conn.LocalAddr()).Debug("Starting accept loop")
	for {
		buf := make([]byte, 512)
		n, addr, err := conn.ReadFromUDP(buf)
		if err != nil { // Problem with the connection
			a.ctx.WithError(err).Error("Connection error")
			continue
		}
		a.ctx.WithField("datagram", buf[:n]).Debug("Incoming datagram")

		pkt := new(semtech.Packet)
		err = pkt.UnmarshalBinary(buf[:n])
		if err != nil {
			a.ctx.WithError(err).Warn("Invalid packet")
			continue
		}

		switch pkt.Identifier {
		case semtech.PULL_DATA: // PULL_DATA -> Respond to the recipient with an ACK
			pullAck, err := semtech.Packet{
				Version:    semtech.VERSION,
				Token:      pkt.Token,
				Identifier: semtech.PULL_ACK,
			}.MarshalBinary()
			if err != nil {
				a.ctx.WithError(err).Error("Unexpected error while marshaling PULL_ACK")
				continue
			}
			a.ctx.WithField("recipient", addr).Debug("Sending PULL_ACK")
			a.conn <- udpMsg{addr: addr, raw: pullAck}
		case semtech.PUSH_DATA: // PUSH_DATA -> Transfer all RXPK to the component
			pushAck, err := semtech.Packet{
				Version:    semtech.VERSION,
				Token:      pkt.Token,
				Identifier: semtech.PUSH_ACK,
			}.MarshalBinary()
			if err != nil {
				a.ctx.WithError(err).Error("Unexpected error while marshaling PUSH_ACK")
				continue
			}
			a.ctx.WithField("Recipient", addr).Debug("Sending PUSH_ACK")
			a.conn <- udpMsg{addr: addr, raw: pushAck}

			if pkt.Payload == nil {
				a.ctx.WithField("packet", pkt).Warn("Invalid PUSH_DATA packet")
				continue
			}
			for _, rxpk := range pkt.Payload.RXPK {
				a.next <- rxpkMsg{
					rxpk:      rxpk,
					recipient: core.Recipient{Address: addr, Id: pkt.GatewayId},
				}
			}
		default:
			a.ctx.WithField("packet", pkt).Debug("Ignoring unexpected packet")
			continue
		}
	}
}

// monitorConnection manages udpConnection of the adapter and send message through that connection
//
// That function executes into a single goroutine and is the only one allowed to write UDP messages.
// Doing this makes sure that only 1 goroutine is interacting with the connection. It thereby allows
// the connection to be replaced at any moment (in case of failure for instance) without disturbing
// the ongoing process.
func (a *Adapter) monitorConnection() {
	var udpConn *net.UDPConn
	for msg := range a.conn {
		if msg.conn != nil { // Change the connection
			if udpConn != nil {
				a.ctx.Debug("Define new UDP connection")
				udpConn.Close()
			}
			udpConn = msg.conn
		}

		if udpConn != nil && msg.raw != nil { // Send the given udp message
			if _, err := udpConn.WriteToUDP(msg.raw, msg.addr); err != nil {
				a.ctx.WithError(err).Error("Error while sending UDP message")
			}
		}
	}
	if udpConn != nil {
		udpConn.Close() // Make sure we close the connection before leaving if we dare ever leave.
	}
}
<commit_msg>Don't log the datagram payload<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package semtech

import (
	"fmt"
	"net"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/semtech"
	"github.com/apex/log"
)

// Adapter represents a semtech adapter which sends and receives packet via UDP in respect of the
// semtech forwarder protocol
type Adapter struct {
	ctx  log.Interface // Just a logger
	conn chan udpMsg   // Channel used to manage response transmissions made by multiple goroutines
	next chan rxpkMsg  // Incoming valid RXPK packets are pushed to this channel
}

// udpMsg type materializes response messages transmitted towards existing recipients (commonly,
// gateways).
type udpMsg struct {
	conn *net.UDPConn // Provide if you intent to change the current adapter connection
	addr *net.UDPAddr // The target recipient address
	raw  []byte       // The raw byte sequence that has to be sent
}

// rxpkMsg type materializes valid uplink messages coming from a given recipient
type rxpkMsg struct {
	rxpk      semtech.RXPK   // The actual RXPK message
	recipient core.Recipient // The address and id of the source emitter
}

var ErrInvalidPort error = fmt.Errorf("Invalid port supplied. The connection might be already taken")
var ErrNotInitialized error = fmt.Errorf("Illegal call on non-initialized adapter")
var ErrNotSupported error = fmt.Errorf("Unsupported operation")
var ErrInvalidPacket error = fmt.Errorf("Invalid packet supplied")

// NewAdapter constructs and allocates a new semtech adapter
func NewAdapter(port uint, ctx log.Interface) (*Adapter, error) {
	a := Adapter{
		ctx:  ctx,
		conn: make(chan udpMsg),
		next: make(chan rxpkMsg),
	}

	// Create the udp connection and start listening with a goroutine
	var udpConn *net.UDPConn
	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("0.0.0.0:%d", port))
	a.ctx.WithField("port", port).Info("Starting Server")
	if udpConn, err = net.ListenUDP("udp", addr); err != nil {
		a.ctx.WithError(err).Error("Unable to start server")
		return nil, ErrInvalidPort
	}

	go a.monitorConnection()
	a.conn <- udpMsg{conn: udpConn}
	go a.listen(udpConn) // Terminates when the connection is closed

	return &a, nil
}

// ok controls whether or not the adapter has been initialized via NewAdapter()
func (a *Adapter) ok() bool {
	return a != nil && a.conn != nil && a.next != nil
}

// Send implements the core.Adapter interface. Not implemented for the semtech adapter.
func (a *Adapter) Send(p core.Packet, r ...core.Recipient) (core.Packet, error) {
	return core.Packet{}, ErrNotSupported
}

// Next implements the core.Adapter interface
func (a *Adapter) Next() (core.Packet, core.AckNacker, error) {
	if !a.ok() {
		return core.Packet{}, nil, ErrNotInitialized
	}
	msg := <-a.next
	packet, err := core.ConvertRXPK(msg.rxpk)
	if err != nil {
		a.ctx.Debug("Received invalid packet")
		return core.Packet{}, nil, ErrInvalidPacket
	}
	return packet, semtechAckNacker{recipient: msg.recipient, conn: a.conn}, nil
}

// NextRegistration implements the core.Adapter interface
func (a *Adapter) NextRegistration() (core.Registration, core.AckNacker, error) {
	return core.Registration{}, nil, ErrNotSupported
}

// listen Handle incoming packets and forward them
func (a *Adapter) listen(conn *net.UDPConn) {
	defer conn.Close()
	a.ctx.WithField("address", conn.LocalAddr()).Debug("Starting accept loop")
	for {
		buf := make([]byte, 512)
		n, addr, err := conn.ReadFromUDP(buf)
		if err != nil { // Problem with the connection
			a.ctx.WithError(err).Error("Connection error")
			continue
		}
		a.ctx.Debug("Incoming datagram")

		pkt := new(semtech.Packet)
		err = pkt.UnmarshalBinary(buf[:n])
		if err != nil {
			a.ctx.WithError(err).Warn("Invalid packet")
			continue
		}

		switch pkt.Identifier {
		case semtech.PULL_DATA: // PULL_DATA -> Respond to the recipient with an ACK
			pullAck, err := semtech.Packet{
				Version:    semtech.VERSION,
				Token:      pkt.Token,
				Identifier: semtech.PULL_ACK,
			}.MarshalBinary()
			if err != nil {
				a.ctx.WithError(err).Error("Unexpected error while marshaling PULL_ACK")
				continue
			}
			a.ctx.WithField("recipient", addr).Debug("Sending PULL_ACK")
			a.conn <- udpMsg{addr: addr, raw: pullAck}
		case semtech.PUSH_DATA: // PUSH_DATA -> Transfer all RXPK to the component
			pushAck, err := semtech.Packet{
				Version:    semtech.VERSION,
				Token:      pkt.Token,
				Identifier: semtech.PUSH_ACK,
			}.MarshalBinary()
			if err != nil {
				a.ctx.WithError(err).Error("Unexpected error while marshaling PUSH_ACK")
				continue
			}
			a.ctx.WithField("Recipient", addr).Debug("Sending PUSH_ACK")
			a.conn <- udpMsg{addr: addr, raw: pushAck}

			if pkt.Payload == nil {
				a.ctx.WithField("packet", pkt).Warn("Invalid PUSH_DATA packet")
				continue
			}
			for _, rxpk := range pkt.Payload.RXPK {
				a.next <- rxpkMsg{
					rxpk:      rxpk,
					recipient: core.Recipient{Address: addr, Id: pkt.GatewayId},
				}
			}
		default:
			a.ctx.WithField("packet", pkt).Debug("Ignoring unexpected packet")
			continue
		}
	}
}

// monitorConnection manages udpConnection of the adapter and send message through that connection
//
// That function executes into a single goroutine and is the only one allowed to write UDP messages.
// Doing this makes sure that only 1 goroutine is interacting with the connection. It thereby allows
// the connection to be replaced at any moment (in case of failure for instance) without disturbing
// the ongoing process.
func (a *Adapter) monitorConnection() {
	var udpConn *net.UDPConn
	for msg := range a.conn {
		if msg.conn != nil { // Change the connection
			if udpConn != nil {
				a.ctx.Debug("Define new UDP connection")
				udpConn.Close()
			}
			udpConn = msg.conn
		}

		if udpConn != nil && msg.raw != nil { // Send the given udp message
			if _, err := udpConn.WriteToUDP(msg.raw, msg.addr); err != nil {
				a.ctx.WithError(err).Error("Error while sending UDP message")
			}
		}
	}
	if udpConn != nil {
		udpConn.Close() // Make sure we close the connection before leaving if we dare ever leave.
	}
}
<|endoftext|>


<commit_before>//Transactions can represent a charge, payment, refund, or adjustment.
// We record charge and refund transactions for you that happen through Invoiced. The payment transaction type is designated for recording offline payments like checks. Finally, an adjustment transaction represents any additional credit or debits to a customer’s balance.
// Most transactions will be associated with an invoice, however, not all. For example, if you wanted to credit your customer for $20 you would create an adjustment transaction for -$20 using the customer ID only instead of the invoice ID.
// We currently support the following payment methods on transactions:
// credit_card
// ach
// bitcoin
// paypal
// wire_transfer
// check
// cash
// other
package invdapi

import (
	"github.com/Invoiced/invoiced-go/invdendpoint"
	"strconv"
)

type Transaction struct {
	*Connection
	*invdendpoint.Transaction
}

type Transactions []*Transaction

func (c *Connection) NewTransaction() *Transaction {
	transaction := new(invdendpoint.Transaction)
	return &Transaction{c, transaction}

}

func (c *Transaction) Count() (int64, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)

	count, apiErr := c.count(endPoint)

	if apiErr != nil {
		return -1, apiErr
	}

	return count, nil

}

func (c *Transaction) Create(transaction *Transaction) (*Transaction, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)
	txnResp := new(Transaction)

	apiErr := c.create(endPoint, transaction, txnResp)

	if apiErr != nil {
		return nil, apiErr
	}

	txnResp.Connection = c.Connection

	return txnResp, nil

}

func (c *Transaction) Delete() error {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), c.Id)

	apiErr := c.delete(endPoint)

	if apiErr != nil {
		return apiErr
	}

	return nil

}

func (c *Transaction) Save() error {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), c.Id)
	txnResp := new(Transaction)
	apiErr := c.update(endPoint, c, txnResp)

	if apiErr != nil {
		return apiErr
	}

	c.Transaction = txnResp.Transaction

	return nil

}

func (c *Transaction) Retrieve(id int64) (*Transaction, error) {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), id)

	custEndPoint := new(invdendpoint.Transaction)

	transaction := &Transaction{c.Connection, custEndPoint}

	_, apiErr := c.retrieveDataFromAPI(endPoint, transaction)

	if apiErr != nil {
		return nil, apiErr
	}

	return transaction, nil

}

func (c *Transaction) ListAll(filter *invdendpoint.Filter, sort *invdendpoint.Sort) (Transactions, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)
	endPoint = addFilterSortToEndPoint(endPoint, filter, sort)

	transactions := make(Transactions, 0)

NEXT:
	tmpTransactions := make(Transactions, 0)

	endPoint, apiErr := c.retrieveDataFromAPI(endPoint, &tmpTransactions)

	if apiErr != nil {
		return nil, apiErr
	}

	transactions = append(transactions, tmpTransactions...)

	if endPoint != "" {
		goto NEXT
	}

	for _, transaction := range transactions {
		transaction.Connection = c.Connection

	}

	return transactions, nil

}

func (c *Transaction) List(filter *invdendpoint.Filter, sort *invdendpoint.Sort) (Transactions, string, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)
	endPoint = addFilterSortToEndPoint(endPoint, filter, sort)

	transactions := make(Transactions, 0)

	nextEndPoint, apiErr := c.retrieveDataFromAPI(endPoint, &transactions)

	if apiErr != nil {
		return nil, "", apiErr
	}

	for _, transaction := range transactions {
		transaction.Connection = c.Connection

	}

	return transactions, nextEndPoint, nil

}

func (c *Transaction) ListByNumber(transactionNumber string) (*Transaction, error) {

	filter := invdendpoint.NewFilter()
	filter.Set("number", transactionNumber)

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions[0], nil

}

func (c *Transaction) ListSuccessfulByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulChargesByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")
	filter.Set("type", "charge")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulRefundsByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")
	filter.Set("type", "refund")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulPaymentsByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")
	filter.Set("type", "payment")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulChargesAndPaymentsByInvoiceID(invoiceID int64) (Transactions, error) {

	charges, err := c.ListSuccessfulChargesByInvoiceID(invoiceID)

	if err != nil {
		return nil, err
	}

	payments, err := c.ListSuccessfulPaymentsByInvoiceID(invoiceID)

	if err != nil {
		return nil, err
	}

	chargesPayments := append(charges, payments...)

	return chargesPayments, nil

}

func (c *Transaction) SendReceipt(emailReq *invdendpoint.EmailRequest) (invdendpoint.EmailResponses, error) {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.InvoicesEndPoint), c.Id) + "/emails"

	emailResp := new(invdendpoint.EmailResponses)

	err := c.create(endPoint, emailReq, emailResp)

	if err != nil {
		return nil, err
	}

	return *emailResp, nil

}

func (c *Transaction) Refund(refund *invdendpoint.Refund) error {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), c.Id) + "/refunds"
	transaction := new(invdendpoint.Transaction)
	err := c.create(endPoint, nil, transaction)

	if err != nil {
		return nil
	}

	c.Transaction = transaction

	return nil

}
<commit_msg>transaction number property does not exist<commit_after>//Transactions can represent a charge, payment, refund, or adjustment.
// We record charge and refund transactions for you that happen through Invoiced. The payment transaction type is designated for recording offline payments like checks. Finally, an adjustment transaction represents any additional credit or debits to a customer’s balance.
// Most transactions will be associated with an invoice, however, not all. For example, if you wanted to credit your customer for $20 you would create an adjustment transaction for -$20 using the customer ID only instead of the invoice ID.
// We currently support the following payment methods on transactions:
// credit_card
// ach
// bitcoin
// paypal
// wire_transfer
// check
// cash
// other
package invdapi

import (
	"github.com/Invoiced/invoiced-go/invdendpoint"
	"strconv"
)

type Transaction struct {
	*Connection
	*invdendpoint.Transaction
}

type Transactions []*Transaction

func (c *Connection) NewTransaction() *Transaction {
	transaction := new(invdendpoint.Transaction)
	return &Transaction{c, transaction}

}

func (c *Transaction) Count() (int64, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)

	count, apiErr := c.count(endPoint)

	if apiErr != nil {
		return -1, apiErr
	}

	return count, nil

}

func (c *Transaction) Create(transaction *Transaction) (*Transaction, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)
	txnResp := new(Transaction)

	apiErr := c.create(endPoint, transaction, txnResp)

	if apiErr != nil {
		return nil, apiErr
	}

	txnResp.Connection = c.Connection

	return txnResp, nil

}

func (c *Transaction) Delete() error {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), c.Id)

	apiErr := c.delete(endPoint)

	if apiErr != nil {
		return apiErr
	}

	return nil

}

func (c *Transaction) Save() error {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), c.Id)
	txnResp := new(Transaction)
	apiErr := c.update(endPoint, c, txnResp)

	if apiErr != nil {
		return apiErr
	}

	c.Transaction = txnResp.Transaction

	return nil

}

func (c *Transaction) Retrieve(id int64) (*Transaction, error) {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), id)

	custEndPoint := new(invdendpoint.Transaction)

	transaction := &Transaction{c.Connection, custEndPoint}

	_, apiErr := c.retrieveDataFromAPI(endPoint, transaction)

	if apiErr != nil {
		return nil, apiErr
	}

	return transaction, nil

}

func (c *Transaction) ListAll(filter *invdendpoint.Filter, sort *invdendpoint.Sort) (Transactions, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)
	endPoint = addFilterSortToEndPoint(endPoint, filter, sort)

	transactions := make(Transactions, 0)

NEXT:
	tmpTransactions := make(Transactions, 0)

	endPoint, apiErr := c.retrieveDataFromAPI(endPoint, &tmpTransactions)

	if apiErr != nil {
		return nil, apiErr
	}

	transactions = append(transactions, tmpTransactions...)

	if endPoint != "" {
		goto NEXT
	}

	for _, transaction := range transactions {
		transaction.Connection = c.Connection

	}

	return transactions, nil

}

func (c *Transaction) List(filter *invdendpoint.Filter, sort *invdendpoint.Sort) (Transactions, string, error) {
	endPoint := c.makeEndPointURL(invdendpoint.TransactionsEndPoint)
	endPoint = addFilterSortToEndPoint(endPoint, filter, sort)

	transactions := make(Transactions, 0)

	nextEndPoint, apiErr := c.retrieveDataFromAPI(endPoint, &transactions)

	if apiErr != nil {
		return nil, "", apiErr
	}

	for _, transaction := range transactions {
		transaction.Connection = c.Connection

	}

	return transactions, nextEndPoint, nil

}

func (c *Transaction) ListSuccessfulByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulChargesByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")
	filter.Set("type", "charge")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulRefundsByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")
	filter.Set("type", "refund")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulPaymentsByInvoiceID(invoiceID int64) (Transactions, error) {

	invoiceIDString := strconv.FormatInt(invoiceID, 10)

	filter := invdendpoint.NewFilter()
	filter.Set("invoice", invoiceIDString)
	filter.Set("status", "succeeded")
	filter.Set("type", "payment")

	transactions, apiError := c.ListAll(filter, nil)

	if apiError != nil {
		return nil, apiError
	}

	if len(transactions) == 0 {
		return nil, nil
	}

	return transactions, nil

}

func (c *Transaction) ListSuccessfulChargesAndPaymentsByInvoiceID(invoiceID int64) (Transactions, error) {

	charges, err := c.ListSuccessfulChargesByInvoiceID(invoiceID)

	if err != nil {
		return nil, err
	}

	payments, err := c.ListSuccessfulPaymentsByInvoiceID(invoiceID)

	if err != nil {
		return nil, err
	}

	chargesPayments := append(charges, payments...)

	return chargesPayments, nil

}

func (c *Transaction) SendReceipt(emailReq *invdendpoint.EmailRequest) (invdendpoint.EmailResponses, error) {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.InvoicesEndPoint), c.Id) + "/emails"

	emailResp := new(invdendpoint.EmailResponses)

	err := c.create(endPoint, emailReq, emailResp)

	if err != nil {
		return nil, err
	}

	return *emailResp, nil

}

func (c *Transaction) Refund(refund *invdendpoint.Refund) error {
	endPoint := makeEndPointSingular(c.makeEndPointURL(invdendpoint.TransactionsEndPoint), c.Id) + "/refunds"
	transaction := new(invdendpoint.Transaction)
	err := c.create(endPoint, nil, transaction)

	if err != nil {
		return nil
	}

	c.Transaction = transaction

	return nil

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"log/syslog"
	"os"
	"os/exec"
)

type Syslogger struct {
	logger *log.Logger
	stream string
	buffer *bytes.Buffer
}

func (s *Syslogger) Write(p []byte) (n int, err error) {
	for b := range p {
		s.buffer.WriteByte(p[b])
		if p[b] == 10 { // newline
			msg := string(s.buffer.Bytes())
			s.logger.Print(msg)
			s.buffer = bytes.NewBuffer([]byte{})
		}
	}
	return len(p), nil
}

func (s *Syslogger) Close() error {
	return nil
}

func NewSysLogger(stream, hostPort, prefix string) (*Syslogger, error) {
	var priority syslog.Priority
	if stream == "stderr" {
		priority = syslog.LOG_ERR | syslog.LOG_LOCAL0
	} else if stream == "stdout" {
		priority = syslog.LOG_INFO | syslog.LOG_LOCAL0
	} else {
		return nil, errors.New("cannot create syslogger for stream " + stream)
	}
	logFlags := 0

	s, err := syslog.Dial("tcp", hostPort, priority, prefix)
	if err != nil {
		return nil, err
	}

	logger := log.New(s, "", logFlags)
	return &Syslogger{logger, stream, bytes.NewBuffer([]byte{})}, nil
}

func usage() {
	fmt.Errorf("usage: %s -h syslog_host:port -n name -- executable [arg ...]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flHostPort := flag.String("h", "", "Host port of where to connect to the syslog daemon")
	flLogName := flag.String("n", "", "Name to log as")
	flag.Parse()

	if *flHostPort == "" {
		fmt.Println("Must set the syslog host:port argument")
		usage()
	}

	if *flLogName == "" {
		fmt.Println("Must set the syslog log name argument")
		usage()
	}

	//Example ./syslog-redirector -h 10.0.3.1:6514 -n test-ls-thingy -- \
	//            /bin/bash -c 'while true; do date; echo $SHELL; sleep 1; done'
	if len(os.Args) < 4 {
		fmt.Printf("at least 3 arguments required\n")
		usage()
	}
	hostPort := *flHostPort
	name := *flLogName

	if len(flag.Args()) == 0 {
		fmt.Printf("must supply a command")
		usage()
	}

	cmdArgs := flag.Args()[1:]
	cmd := exec.Command(flag.Args()[0], cmdArgs...)

	var err error

	// TODO (dano): tolerate syslog downtime by reconnecting

	cmd.Stdout, err = NewSysLogger("stdout", hostPort, name)
	if err != nil {
		fmt.Errorf("error creating syslog writer for stdout: " + err.Error())
	}

	cmd.Stderr, err = NewSysLogger("stderr", hostPort, name)
	if err != nil {
		fmt.Errorf("error creating syslog writer for stderr: " + err.Error())
	}

	err = cmd.Run()
	if err != nil {
		fmt.Errorf("error running command: " + err.Error())
	}
}
<commit_msg>propagate exit status code<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"log/syslog"
	"os"
	"os/exec"
)

type Syslogger struct {
	logger *log.Logger
	stream string
	buffer *bytes.Buffer
}

func (s *Syslogger) Write(p []byte) (n int, err error) {
	for b := range p {
		s.buffer.WriteByte(p[b])
		if p[b] == 10 { // newline
			msg := string(s.buffer.Bytes())
			s.logger.Print(msg)
			s.buffer = bytes.NewBuffer([]byte{})
		}
	}
	return len(p), nil
}

func (s *Syslogger) Close() error {
	return nil
}

func NewSysLogger(stream, hostPort, prefix string) (*Syslogger, error) {
	var priority syslog.Priority
	if stream == "stderr" {
		priority = syslog.LOG_ERR | syslog.LOG_LOCAL0
	} else if stream == "stdout" {
		priority = syslog.LOG_INFO | syslog.LOG_LOCAL0
	} else {
		return nil, errors.New("cannot create syslogger for stream " + stream)
	}
	logFlags := 0

	s, err := syslog.Dial("tcp", hostPort, priority, prefix)
	if err != nil {
		return nil, err
	}

	logger := log.New(s, "", logFlags)
	return &Syslogger{logger, stream, bytes.NewBuffer([]byte{})}, nil
}

func usage() {
	fmt.Errorf("usage: %s -h syslog_host:port -n name -- executable [arg ...]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flHostPort := flag.String("h", "", "Host port of where to connect to the syslog daemon")
	flLogName := flag.String("n", "", "Name to log as")
	flag.Parse()

	if *flHostPort == "" {
		fmt.Println("Must set the syslog host:port argument")
		usage()
	}

	if *flLogName == "" {
		fmt.Println("Must set the syslog log name argument")
		usage()
	}

	//Example ./syslog-redirector -h 10.0.3.1:6514 -n test-ls-thingy -- \
	//            /bin/bash -c 'while true; do date; echo $SHELL; sleep 1; done'
	if len(os.Args) < 4 {
		fmt.Printf("at least 3 arguments required\n")
		usage()
	}
	hostPort := *flHostPort
	name := *flLogName

	if len(flag.Args()) == 0 {
		fmt.Printf("must supply a command")
		usage()
	}

	cmdArgs := flag.Args()[1:]
	cmd := exec.Command(flag.Args()[0], cmdArgs...)

	var err error

	// TODO (dano): tolerate syslog downtime by reconnecting

	cmd.Stdout, err = NewSysLogger("stdout", hostPort, name)
	if err != nil {
		fmt.Errorf("error creating syslog writer for stdout: " + err.Error())
	}

	cmd.Stderr, err = NewSysLogger("stderr", hostPort, name)
	if err != nil {
		fmt.Errorf("error creating syslog writer for stderr: " + err.Error())
	}

	err = cmd.Run()
	if err != nil {
		if msg, ok := err.(*exec.ExitError); ok {
			os.Exit(msg.Sys().(syscall.WaitStatus).ExitStatus())
		} else {
			fmt.Errorf("error running command: " + err.Error())
			os.Exit(1)
		}
	}

	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/workers/common/mux"
	"socialapi/workers/integration/webhook/api"

	"github.com/koding/runner"
)

var (
	Name = "IntegrationWebhook"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}
	defer r.Close()

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	redisConn := r.Bongo.MustGetRedisConn()

	iConfig := appConfig.Integration

	mc := mux.NewConfig(Name, iConfig.Host, iConfig.Port)
	m := mux.New(mc, r.Log, r.Metrics)

	h, err := api.NewHandler(appConfig, redisConn, r.Log)
	if err != nil {
		r.Log.Fatal("Could not initialize webhook worker: %s", err)
	}

	h.AddHandlers(m)

	go r.Listen()

	m.Listen()
	defer m.Close()

	r.Wait()
}
<commit_msg>integration: updating integration is added to cmd<commit_after>package main

import (
	"fmt"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/workers/common/mux"
	"socialapi/workers/integration/webhook/api"
	"socialapi/workers/migrator/controller"
	"socialapi/workers/realtime/models"

	"github.com/koding/runner"
)

var (
	Name = "IntegrationWebhook"
)

func main() {
	r := runner.New(Name)
	if err := r.Init(); err != nil {
		fmt.Println(err)
		return
	}
	defer r.Close()

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	redisConn := r.Bongo.MustGetRedisConn()

	iConfig := appConfig.Integration

	mc := mux.NewConfig(Name, iConfig.Host, iConfig.Port)
	m := mux.New(mc, r.Log, r.Metrics)

	h, err := api.NewHandler(appConfig, redisConn, r.Log)
	if err != nil {
		r.Log.Fatal("Could not initialize webhook worker: %s", err)
	}

	pubnub := models.NewPubNub(appConfig.GateKeeper.Pubnub, r.Log)
	defer pubnub.Close()

	mwc, err := controller.New(r.Log, pubnub)
	if err != nil {
		panic(err)
	}

	go mwc.UpdateIntegrations()

	h.AddHandlers(m)

	go r.Listen()

	m.Listen()
	defer m.Close()

	r.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	ggio "github.com/gogo/protobuf/io"
	mux "github.com/gorilla/mux"
	p2p_peer "github.com/ipfs/go-libp2p-peer"
	p2p_pstore "github.com/ipfs/go-libp2p-peerstore"
	multiaddr "github.com/jbenet/go-multiaddr"
	p2p_host "github.com/libp2p/go-libp2p/p2p/host"
	p2p_net "github.com/libp2p/go-libp2p/p2p/net"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"
	"time"
)

type Node struct {
	mc.Identity
	host    p2p_host.Host
	dir     p2p_pstore.PeerInfo
	home    string
	mx      sync.Mutex
	stmt    map[string]*pb.Statement
	counter int
}

func (node *Node) pingHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	log.Printf("node/ping: new stream from %s", pid.Pretty())

	var ping pb.Ping
	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&ping)
		if err != nil {
			return
		}

		log.Printf("node/ping: ping from %s; ponging", pid.Pretty())

		err = w.WriteMsg(&pong)
		if err != nil {
			return
		}
	}
}

func (node *Node) registerPeer(addrs ...multiaddr.Multiaddr) {
	// directory failure is a fatality for now
	ctx := context.Background()

	err := node.host.Connect(ctx, node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory")
		log.Fatal(err)
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream")
		log.Fatal(err)
	}
	defer s.Close()

	pinfo := p2p_pstore.PeerInfo{node.ID, addrs}
	var pbpi pb.PeerInfo
	mc.PBFromPeerInfo(&pbpi, pinfo)
	msg := pb.RegisterPeer{&pbpi}

	w := ggio.NewDelimitedWriter(s)
	for {
		log.Printf("Registering with directory")
		err = w.WriteMsg(&msg)
		if err != nil {
			log.Printf("Failed to register with directory")
			log.Fatal(err)
		}

		time.Sleep(5 * time.Minute)
	}
}

func (node *Node) httpId(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, node.Identity.Pretty())
}

func (node *Node) httpPing(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	peerId := vars["peerId"]
	pid, err := p2p_peer.IDB58Decode(peerId)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error: Bad id: %s\n", err.Error())
		return
	}

	err = node.doPing(r.Context(), pid)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	fmt.Fprintf(w, "OK\n")
}

func (node *Node) doPing(ctx context.Context, pid p2p_peer.ID) error {
	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	err = node.host.Connect(ctx, pinfo)
	if err != nil {
		return err
	}

	s, err := node.host.NewStream(ctx, pinfo.ID, "/mediachain/node/ping")
	if err != nil {
		return err
	}
	defer s.Close()

	var ping pb.Ping
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&ping)
	if err != nil {
		return err
	}

	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&pong)

	return err
}

var UnknownPeer = errors.New("Unknown peer")

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{string(pid)}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}

func (node *Node) httpPublish(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	ns := vars["namespace"]

	rbody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Printf("http/publish: Error reading request body: %s", err.Error())
		return
	}

	// just simple statements for now
	sbody := new(pb.SimpleStatement)
	err = json.Unmarshal(rbody, sbody)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	sid, err := node.doPublish(ns, sbody)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	fmt.Println(w, sid)
}

var BadStatementBody = errors.New("Unrecognized statement body")

func (node *Node) doPublish(ns string, body interface{}) (string, error) {
	stmt := new(pb.Statement)
	pid := node.ID.Pretty()
	ts := time.Now().Unix()
	counter := node.stmtCounter()
	stmt.Id = fmt.Sprintf("%s:%d:%d", pid, ts, counter)
	stmt.Publisher = pid // this should be the pubkey when we have ECC keys
	stmt.Namespace = ns
	switch body := body.(type) {
	case *pb.SimpleStatement:
		stmt.Body = &pb.Statement_Simple{body}

	default:
		return "", BadStatementBody
	}
	// only sign it with shiny ECC keys, don't bother with RSA
	node.mx.Lock()
	node.stmt[stmt.Id] = stmt
	node.mx.Unlock()

	return stmt.Id, nil
}

func (node *Node) stmtCounter() int {
	node.mx.Lock()
	counter := node.counter
	node.counter++
	node.mx.Unlock()
	return counter
}

func main() {
	pport := flag.Int("l", 9001, "Peer listen port")
	cport := flag.Int("c", 9002, "Peer control interface port [http]")
	home := flag.String("d", "/tmp/mcnode", "Node home")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Fprintf(os.Stderr, "Usage: %s [options ...] directory\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	addr, err := mc.ParseAddress(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", *pport))
	if err != nil {
		log.Fatal(err)
	}

	dir, err := mc.ParseHandle(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	err = os.MkdirAll(*home, 0755)
	if err != nil {
		log.Fatal(err)
	}

	id, err := mc.NodeIdentity(*home)
	if err != nil {
		log.Fatal(err)
	}

	host, err := mc.NewHost(id, addr)
	if err != nil {
		log.Fatal(err)
	}

	node := &Node{Identity: id, host: host, dir: dir, home: *home, stmt: make(map[string]*pb.Statement)}
	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	go node.registerPeer(addr)

	log.Printf("I am %s/%s", addr, id.Pretty())

	haddr := fmt.Sprintf("127.0.0.1:%d", *cport)
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/id", node.httpId)
	router.HandleFunc("/ping/{peerId}", node.httpPing)
	router.HandleFunc("/publish/{namespace}", node.httpPublish)

	log.Printf("Serving client interface at %s", haddr)
	err = http.ListenAndServe(haddr, router)
	if err != nil {
		log.Fatal(err)
	}

	select {}
}
<commit_msg>mcnode: statement retrieval api<commit_after>package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	ggio "github.com/gogo/protobuf/io"
	mux "github.com/gorilla/mux"
	p2p_peer "github.com/ipfs/go-libp2p-peer"
	p2p_pstore "github.com/ipfs/go-libp2p-peerstore"
	multiaddr "github.com/jbenet/go-multiaddr"
	p2p_host "github.com/libp2p/go-libp2p/p2p/host"
	p2p_net "github.com/libp2p/go-libp2p/p2p/net"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"
	"time"
)

type Node struct {
	mc.Identity
	host    p2p_host.Host
	dir     p2p_pstore.PeerInfo
	home    string
	mx      sync.Mutex
	stmt    map[string]*pb.Statement
	counter int
}

func (node *Node) pingHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	log.Printf("node/ping: new stream from %s", pid.Pretty())

	var ping pb.Ping
	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&ping)
		if err != nil {
			return
		}

		log.Printf("node/ping: ping from %s; ponging", pid.Pretty())

		err = w.WriteMsg(&pong)
		if err != nil {
			return
		}
	}
}

func (node *Node) registerPeer(addrs ...multiaddr.Multiaddr) {
	// directory failure is a fatality for now
	ctx := context.Background()

	err := node.host.Connect(ctx, node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory")
		log.Fatal(err)
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream")
		log.Fatal(err)
	}
	defer s.Close()

	pinfo := p2p_pstore.PeerInfo{node.ID, addrs}
	var pbpi pb.PeerInfo
	mc.PBFromPeerInfo(&pbpi, pinfo)
	msg := pb.RegisterPeer{&pbpi}

	w := ggio.NewDelimitedWriter(s)
	for {
		log.Printf("Registering with directory")
		err = w.WriteMsg(&msg)
		if err != nil {
			log.Printf("Failed to register with directory")
			log.Fatal(err)
		}

		time.Sleep(5 * time.Minute)
	}
}

func (node *Node) httpId(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, node.Identity.Pretty())
}

func (node *Node) httpPing(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	peerId := vars["peerId"]
	pid, err := p2p_peer.IDB58Decode(peerId)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error: Bad id: %s\n", err.Error())
		return
	}

	err = node.doPing(r.Context(), pid)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	fmt.Fprintf(w, "OK\n")
}

func (node *Node) doPing(ctx context.Context, pid p2p_peer.ID) error {
	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	err = node.host.Connect(ctx, pinfo)
	if err != nil {
		return err
	}

	s, err := node.host.NewStream(ctx, pinfo.ID, "/mediachain/node/ping")
	if err != nil {
		return err
	}
	defer s.Close()

	var ping pb.Ping
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&ping)
	if err != nil {
		return err
	}

	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&pong)

	return err
}

var UnknownPeer = errors.New("Unknown peer")

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{string(pid)}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}

func (node *Node) httpPublish(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	ns := vars["namespace"]

	rbody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Printf("http/publish: Error reading request body: %s", err.Error())
		return
	}

	// just simple statements for now
	sbody := new(pb.SimpleStatement)
	err = json.Unmarshal(rbody, sbody)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	sid, err := node.doPublish(ns, sbody)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	fmt.Fprintln(w, sid)
}

var BadStatementBody = errors.New("Unrecognized statement body")

func (node *Node) doPublish(ns string, body interface{}) (string, error) {
	stmt := new(pb.Statement)
	pid := node.ID.Pretty()
	ts := time.Now().Unix()
	counter := node.stmtCounter()
	stmt.Id = fmt.Sprintf("%s:%d:%d", pid, ts, counter)
	stmt.Publisher = pid // this should be the pubkey when we have ECC keys
	stmt.Namespace = ns
	switch body := body.(type) {
	case *pb.SimpleStatement:
		stmt.Body = &pb.Statement_Simple{body}

	default:
		return "", BadStatementBody
	}
	// only sign it with shiny ECC keys, don't bother with RSA
	node.mx.Lock()
	node.stmt[stmt.Id] = stmt
	node.mx.Unlock()

	log.Printf("Published statement %s", stmt.Id)

	return stmt.Id, nil
}

func (node *Node) stmtCounter() int {
	node.mx.Lock()
	counter := node.counter
	node.counter++
	node.mx.Unlock()
	return counter
}

func (node *Node) httpStatement(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["statementId"]

	var stmt *pb.Statement
	node.mx.Lock()
	stmt = node.stmt[id]
	node.mx.Unlock()

	if stmt == nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "No such statement\n")
		return
	}

	err := json.NewEncoder(w).Encode(stmt)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}
}

func main() {
	pport := flag.Int("l", 9001, "Peer listen port")
	cport := flag.Int("c", 9002, "Peer control interface port [http]")
	home := flag.String("d", "/tmp/mcnode", "Node home")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Fprintf(os.Stderr, "Usage: %s [options ...] directory\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	addr, err := mc.ParseAddress(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", *pport))
	if err != nil {
		log.Fatal(err)
	}

	dir, err := mc.ParseHandle(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	err = os.MkdirAll(*home, 0755)
	if err != nil {
		log.Fatal(err)
	}

	id, err := mc.NodeIdentity(*home)
	if err != nil {
		log.Fatal(err)
	}

	host, err := mc.NewHost(id, addr)
	if err != nil {
		log.Fatal(err)
	}

	node := &Node{Identity: id, host: host, dir: dir, home: *home, stmt: make(map[string]*pb.Statement)}
	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	go node.registerPeer(addr)

	log.Printf("I am %s/%s", addr, id.Pretty())

	haddr := fmt.Sprintf("127.0.0.1:%d", *cport)
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/id", node.httpId)
	router.HandleFunc("/ping/{peerId}", node.httpPing)
	router.HandleFunc("/publish/{namespace}", node.httpPublish)
	router.HandleFunc("/stmt/{statementId}", node.httpStatement)

	log.Printf("Serving client interface at %s", haddr)
	err = http.ListenAndServe(haddr, router)
	if err != nil {
		log.Fatal(err)
	}

	select {}
}
<|endoftext|>


<commit_before><commit_msg>Add support for chain deletion in iptsave package<commit_after><|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"errors"
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"koding/helpers"
	"reflect"
	"socialapi/models"
	"time"

	"github.com/koding/bongo"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	ErrMigrated     = errors.New("already migrated")
	kodingChannelId int64
)

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) (*Controller, error) {
	wc := &Controller{
		log: log,
	}

	return wc, nil
}

func (mwc *Controller) Start() error {
	if err := mwc.migrateAllAccounts(); err != nil {
		return err
	}

	if err := mwc.migrateAllGroups(); err != nil {
		return err
	}

	if err := mwc.migrateAllTags(); err != nil {
		return err
	}

	if err := mwc.migrateAllPosts(); err != nil {
		return err
	}

	return nil
}

func (mwc *Controller) migrateAllPosts() error {
	s := modelhelper.Selector{
		"socialMessageId": modelhelper.Selector{"$exists": false},
	}
	kodingChannel, err := mwc.createGroupChannel("koding")
	if err != nil {
		return fmt.Errorf("Koding channel cannot be created: %s", err)
	}
	kodingChannelId = kodingChannel.Id

	errCount := 0
	successCount := 0

	handleError := func(su *mongomodels.StatusUpdate, err error) {
		mwc.log.Error("an error occured for %s: %s", su.Id.Hex(), err)
		errCount++
	}

	migratePost := func(post interface{}) error {
		su := post.(*mongomodels.StatusUpdate)
		channelId, err := mwc.fetchGroupChannelId(su.Group)
		if err != nil {
			if err == bongo.RecordNotFound {
				handleError(su, err)
				return nil
			}

			return err
		}

		// create channel message
		cm, err := mapStatusUpdateToChannelMessage(su)
		if err != nil {
			handleError(su, err)
			return nil
		}

		cm.InitialChannelId = channelId
		if err := insertChannelMessage(cm, su.OriginId.Hex()); err != nil {
			handleError(su, err)
			return nil
		}

		if err := addChannelMessageToMessageList(cm); err != nil {
			handleError(su, err)
			return nil
		}

		// create reply messages
		if err := mwc.migrateComments(cm, su); err != nil {
			handleError(su, err)
			return nil
		}

		if err := mwc.migrateLikes(cm, su.Id); err != nil {
			handleError(su, err)
			return nil
		}

		if err := mwc.migrateTags(cm, su.Id); err != nil {
			handleError(su, err)
			return nil
		}

		// update mongo status update channelMessageId field
		if err := completePostMigration(su, cm); err != nil {
			handleError(su, err)
			return nil
		}
		successCount++

		return nil
	}

	iterOptions := helpers.NewIterOptions()
	iterOptions.CollectionName = "jNewStatusUpdates"
	iterOptions.F = migratePost
	iterOptions.Filter = s
	iterOptions.Result = &mongomodels.StatusUpdate{}
	iterOptions.Limit = 10000000
	iterOptions.Skip = 0

	if err := helpers.Iter(modelhelper.Mongo, iterOptions); err != nil {
		mwc.log.Fatal("Post migration is interrupted with %d errors: channel id cannot be fetched :%s", errCount, err)
	}

	mwc.log.Notice("Post migration completed for %d status updates with %d errors", successCount, errCount)

	return nil
}

func insertChannelMessage(cm *models.ChannelMessage, accountOldId string) error {

	if err := prepareMessageAccount(cm, accountOldId); err != nil {
		return err
	}

	if err := cm.CreateRaw(); err != nil {
		return err
	}

	return nil
}

func addChannelMessageToMessageList(cm *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	cml.ChannelId = cm.InitialChannelId
	cml.MessageId = cm.Id
	cml.AddedAt = cm.CreatedAt

	return cml.CreateRaw()
}

func (mwc *Controller) migrateComments(parentMessage *models.ChannelMessage, su *mongomodels.StatusUpdate) error {

	s := modelhelper.Selector{
		"sourceId":   su.Id,
		"targetName": "JComment",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		if err == modelhelper.ErrNotFound {
			return nil
		}
		return fmt.Errorf("comment relationships cannot be fetched: %s", err)
	}

	for _, r := range rels {
		comment, err := modelhelper.GetCommentById(r.TargetId.Hex())
		if err != nil {
			return fmt.Errorf("comment cannot be fetched %s", err)
		}
		// comment is already migrated
		if comment.SocialMessageId != 0 {
			continue
		}

		reply := mapCommentToChannelMessage(comment)
		reply.InitialChannelId = parentMessage.InitialChannelId
		// insert as channel message
		if err := insertChannelMessage(reply, comment.OriginId.Hex()); err != nil {
			return fmt.Errorf("comment cannot be inserted %s", err)
		}

		// insert as message reply
		mr := models.NewMessageReply()
		mr.MessageId = parentMessage.Id
		mr.ReplyId = reply.Id
		mr.CreatedAt = reply.CreatedAt
		if err := mr.CreateRaw(); err != nil {
			return fmt.Errorf("comment cannot be inserted to message reply %s", err)
		}

		if err := mwc.migrateLikes(reply, comment.Id); err != nil {
			return fmt.Errorf("likes cannot be migrated %s", err)
		}

		if err := completeCommentMigration(comment, reply); err != nil {
			return fmt.Errorf("old comment cannot be flagged with new message id %s", err)
		}
	}

	return nil
}

func (mwc *Controller) migrateLikes(cm *models.ChannelMessage, oldId bson.ObjectId) error {
	s := modelhelper.Selector{
		"sourceId": oldId,
		"as":       "like",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		return fmt.Errorf("likes cannot be fetched %s", err)
	}
	for _, r := range rels {
		a := models.NewAccount()
		a.OldId = r.TargetId.Hex()
		if err := a.FetchOrCreate(); err != nil {
			mwc.log.Error("interactor account could not found: %s", err)
			continue
		}
		i := models.NewInteraction()
		i.MessageId = cm.Id
		i.AccountId = a.Id
		i.TypeConstant = models.Interaction_TYPE_LIKE
		i.CreatedAt = r.TimeStamp
		if err := i.CreateRaw(); err != nil {
			mwc.log.Error("interaction could not created: %s", err)
		}
	}

	return nil
}

func prepareMessageAccount(cm *models.ChannelMessage, accountOldId string) error {
	id, err := models.AccountIdByOldId(accountOldId, "")
	if err != nil {
		return fmt.Errorf("account could not found: %s", err)
	}

	cm.AccountId = id

	return nil
}

func (mwc *Controller) fetchGroupChannelId(groupName string) (int64, error) {
	// koding group channel id is prefetched
	if groupName == "koding" {
		return kodingChannelId, nil
	}

	c := models.NewChannel()
	channelId, err := c.FetchChannelIdByNameAndGroupName(groupName, groupName)
	if err != nil {
		return 0, err
	}

	return channelId, nil
}

func mapStatusUpdateToChannelMessage(su *mongomodels.StatusUpdate) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	cm.Slug = su.Slug
	cm.Body = su.Body // for now do not modify tags
	cm.TypeConstant = models.ChannelMessage_TYPE_POST
	cm.CreatedAt = su.Meta.CreatedAt
	payload, err := mapEmbeddedLink(su.Link)
	if err != nil {
		return nil, err
	}
	cm.Payload = payload

	prepareMessageMetaDates(cm, &su.Meta)

	return cm, nil
}

func mapEmbeddedLink(link map[string]interface{}) (map[string]*string, error) {
	resultMap := make(map[string]*string)
	for key, value := range link {
		// when value is a map, then marshal it
		if reflect.ValueOf(value).Kind() == reflect.Map {
			res, err := json.Marshal(value)
			if err != nil {
				return nil, err
			}

			s := string(res)
			resultMap[key] = &s
			continue
		}

		// for the other types convert value to string
		str := fmt.Sprintf("%v", value)
		resultMap[key] = &str
	}

	return resultMap, nil
}

func mapCommentToChannelMessage(c *mongomodels.Comment) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Body = c.Body
	cm.TypeConstant = models.ChannelMessage_TYPE_REPLY
	cm.CreatedAt = c.Meta.CreatedAt
	cm.DeletedAt = c.DeletedAt
	prepareMessageMetaDates(cm, &c.Meta)

	return cm
}

func prepareMessageMetaDates(cm *models.ChannelMessage, meta *mongomodels.Meta) {
	lowerLimit := cm.CreatedAt.Add(-time.Second)
	upperLimit := cm.CreatedAt.Add(time.Second)
	if meta.ModifiedAt.After(lowerLimit) && meta.ModifiedAt.Before(upperLimit) {
		cm.UpdatedAt = cm.CreatedAt
	} else {
		cm.UpdatedAt = meta.ModifiedAt
	}
}

func completePostMigration(su *mongomodels.StatusUpdate, cm *models.ChannelMessage) error {
	su.SocialMessageId = cm.Id

	return modelhelper.UpdateStatusUpdate(su)
}

func completeCommentMigration(reply *mongomodels.Comment, cm *models.ChannelMessage) error {
	reply.SocialMessageId = cm.Id

	return modelhelper.UpdateComment(reply)
}
<commit_msg>Migrator: Add AccountIdByOldId method for fetching id from cache with fallback to mongo account if not found<commit_after>package controller

import (
	"encoding/json"
	"errors"
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"koding/helpers"
	"reflect"
	"socialapi/models"
	"time"

	"github.com/koding/bongo"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	ErrMigrated     = errors.New("already migrated")
	kodingChannelId int64
)

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) (*Controller, error) {
	wc := &Controller{
		log: log,
	}

	return wc, nil
}

func (mwc *Controller) Start() error {
	if err := mwc.migrateAllAccounts(); err != nil {
		return err
	}

	if err := mwc.migrateAllGroups(); err != nil {
		return err
	}

	if err := mwc.migrateAllTags(); err != nil {
		return err
	}

	if err := mwc.migrateAllPosts(); err != nil {
		return err
	}

	return nil
}

func (mwc *Controller) migrateAllPosts() error {
	s := modelhelper.Selector{
		"socialMessageId": modelhelper.Selector{"$exists": false},
	}
	kodingChannel, err := mwc.createGroupChannel("koding")
	if err != nil {
		return fmt.Errorf("Koding channel cannot be created: %s", err)
	}
	kodingChannelId = kodingChannel.Id

	errCount := 0
	successCount := 0

	handleError := func(su *mongomodels.StatusUpdate, err error) {
		mwc.log.Error("an error occured for %s: %s", su.Id.Hex(), err)
		errCount++
	}

	migratePost := func(post interface{}) error {
		su := post.(*mongomodels.StatusUpdate)
		channelId, err := mwc.fetchGroupChannelId(su.Group)
		if err != nil {
			if err == bongo.RecordNotFound {
				handleError(su, err)
				return nil
			}

			return err
		}

		// create channel message
		cm, err := mapStatusUpdateToChannelMessage(su)
		if err != nil {
			handleError(su, err)
			return nil
		}

		cm.InitialChannelId = channelId
		if err := insertChannelMessage(cm, su.OriginId.Hex()); err != nil {
			handleError(su, err)
			return nil
		}

		if err := addChannelMessageToMessageList(cm); err != nil {
			handleError(su, err)
			return nil
		}

		// create reply messages
		if err := mwc.migrateComments(cm, su); err != nil {
			handleError(su, err)
			return nil
		}

		if err := mwc.migrateLikes(cm, su.Id); err != nil {
			handleError(su, err)
			return nil
		}

		if err := mwc.migrateTags(cm, su.Id); err != nil {
			handleError(su, err)
			return nil
		}

		// update mongo status update channelMessageId field
		if err := completePostMigration(su, cm); err != nil {
			handleError(su, err)
			return nil
		}
		successCount++

		return nil
	}

	iterOptions := helpers.NewIterOptions()
	iterOptions.CollectionName = "jNewStatusUpdates"
	iterOptions.F = migratePost
	iterOptions.Filter = s
	iterOptions.Result = &mongomodels.StatusUpdate{}
	iterOptions.Limit = 10000000
	iterOptions.Skip = 0

	if err := helpers.Iter(modelhelper.Mongo, iterOptions); err != nil {
		mwc.log.Fatal("Post migration is interrupted with %d errors: channel id cannot be fetched :%s", errCount, err)
	}

	mwc.log.Notice("Post migration completed for %d status updates with %d errors", successCount, errCount)

	return nil
}

func insertChannelMessage(cm *models.ChannelMessage, accountOldId string) error {

	if err := prepareMessageAccount(cm, accountOldId); err != nil {
		return err
	}

	if err := cm.CreateRaw(); err != nil {
		return err
	}

	return nil
}

func addChannelMessageToMessageList(cm *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	cml.ChannelId = cm.InitialChannelId
	cml.MessageId = cm.Id
	cml.AddedAt = cm.CreatedAt

	return cml.CreateRaw()
}

func (mwc *Controller) migrateComments(parentMessage *models.ChannelMessage, su *mongomodels.StatusUpdate) error {

	s := modelhelper.Selector{
		"sourceId":   su.Id,
		"targetName": "JComment",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		if err == modelhelper.ErrNotFound {
			return nil
		}
		return fmt.Errorf("comment relationships cannot be fetched: %s", err)
	}

	for _, r := range rels {
		comment, err := modelhelper.GetCommentById(r.TargetId.Hex())
		if err != nil {
			return fmt.Errorf("comment cannot be fetched %s", err)
		}
		// comment is already migrated
		if comment.SocialMessageId != 0 {
			continue
		}

		reply := mapCommentToChannelMessage(comment)
		reply.InitialChannelId = parentMessage.InitialChannelId
		// insert as channel message
		if err := insertChannelMessage(reply, comment.OriginId.Hex()); err != nil {
			return fmt.Errorf("comment cannot be inserted %s", err)
		}

		// insert as message reply
		mr := models.NewMessageReply()
		mr.MessageId = parentMessage.Id
		mr.ReplyId = reply.Id
		mr.CreatedAt = reply.CreatedAt
		if err := mr.CreateRaw(); err != nil {
			return fmt.Errorf("comment cannot be inserted to message reply %s", err)
		}

		if err := mwc.migrateLikes(reply, comment.Id); err != nil {
			return fmt.Errorf("likes cannot be migrated %s", err)
		}

		if err := completeCommentMigration(comment, reply); err != nil {
			return fmt.Errorf("old comment cannot be flagged with new message id %s", err)
		}
	}

	return nil
}

func (mwc *Controller) migrateLikes(cm *models.ChannelMessage, oldId bson.ObjectId) error {
	s := modelhelper.Selector{
		"sourceId": oldId,
		"as":       "like",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		return fmt.Errorf("likes cannot be fetched %s", err)
	}
	for _, r := range rels {
		a := models.NewAccount()
		a.OldId = r.TargetId.Hex()
		if err := a.FetchOrCreate(); err != nil {
			mwc.log.Error("interactor account could not found: %s", err)
			continue
		}
		i := models.NewInteraction()
		i.MessageId = cm.Id
		i.AccountId = a.Id
		i.TypeConstant = models.Interaction_TYPE_LIKE
		i.CreatedAt = r.TimeStamp
		if err := i.CreateRaw(); err != nil {
			mwc.log.Error("interaction could not created: %s", err)
		}
	}

	return nil
}

func prepareMessageAccount(cm *models.ChannelMessage, accountOldId string) error {
	id, err := models.AccountIdByOldId(accountOldId, "")
	if err != nil {
		return fmt.Errorf("account could not found: %s", err)
	}

	cm.AccountId = id

	return nil
}

func (mwc *Controller) fetchGroupChannelId(groupName string) (int64, error) {
	// koding group channel id is prefetched
	if groupName == "koding" {
		return kodingChannelId, nil
	}

	c := models.NewChannel()
	channelId, err := c.FetchChannelIdByNameAndGroupName(groupName, groupName)
	if err != nil {
		return 0, err
	}

	return channelId, nil
}

func (mwc *Controller) AccountIdByOldId(oldId string) (int64, error) {
	id := models.FetchAccountIdByOldId(oldId)
	if id != 0 {
		return id, nil
	}

	acc, err := modelhelper.GetAccountById(oldId)
	if err != nil {
		return 0, fmt.Errorf("Participant account %s cannot be fetched: %s", oldId, err)
	}

	id, err = models.AccountIdByOldId(oldId, acc.Profile.Nickname)
	if err != nil {
		mwc.log.Warning("Could not update cache for %s: %s", oldId, err)
	}

	return id, nil
}

func mapStatusUpdateToChannelMessage(su *mongomodels.StatusUpdate) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	cm.Slug = su.Slug
	cm.Body = su.Body // for now do not modify tags
	cm.TypeConstant = models.ChannelMessage_TYPE_POST
	cm.CreatedAt = su.Meta.CreatedAt
	payload, err := mapEmbeddedLink(su.Link)
	if err != nil {
		return nil, err
	}
	cm.Payload = payload

	prepareMessageMetaDates(cm, &su.Meta)

	return cm, nil
}

func mapEmbeddedLink(link map[string]interface{}) (map[string]*string, error) {
	resultMap := make(map[string]*string)
	for key, value := range link {
		// when value is a map, then marshal it
		if reflect.ValueOf(value).Kind() == reflect.Map {
			res, err := json.Marshal(value)
			if err != nil {
				return nil, err
			}

			s := string(res)
			resultMap[key] = &s
			continue
		}

		// for the other types convert value to string
		str := fmt.Sprintf("%v", value)
		resultMap[key] = &str
	}

	return resultMap, nil
}

func mapCommentToChannelMessage(c *mongomodels.Comment) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Body = c.Body
	cm.TypeConstant = models.ChannelMessage_TYPE_REPLY
	cm.CreatedAt = c.Meta.CreatedAt
	cm.DeletedAt = c.DeletedAt
	prepareMessageMetaDates(cm, &c.Meta)

	return cm
}

func prepareMessageMetaDates(cm *models.ChannelMessage, meta *mongomodels.Meta) {
	lowerLimit := cm.CreatedAt.Add(-time.Second)
	upperLimit := cm.CreatedAt.Add(time.Second)
	if meta.ModifiedAt.After(lowerLimit) && meta.ModifiedAt.Before(upperLimit) {
		cm.UpdatedAt = cm.CreatedAt
	} else {
		cm.UpdatedAt = meta.ModifiedAt
	}
}

func completePostMigration(su *mongomodels.StatusUpdate, cm *models.ChannelMessage) error {
	su.SocialMessageId = cm.Id

	return modelhelper.UpdateStatusUpdate(su)
}

func completeCommentMigration(reply *mongomodels.Comment, cm *models.ChannelMessage) error {
	reply.SocialMessageId = cm.Id

	return modelhelper.UpdateComment(reply)
}
<|endoftext|>


<commit_before>package devastator

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"sync"
)

var (
	ping   = []byte("ping")
	closed = []byte("close")
)

// Listener accepts connections from devices.
type Listener struct {
	Conns    []*Conn
	debug    bool
	listener net.Listener
	connWG   sync.WaitGroup
}

// Listen creates a TCP listener with the given PEM encoded X.509 certificate and the private key on the local network address laddr.
// Debug mode logs all server activity.
func Listen(cert, privKey []byte, laddr string, debug bool) (*Listener, error) {
	tlsCert, err := tls.X509KeyPair(cert, privKey)
	pool := x509.NewCertPool()
	ok := pool.AppendCertsFromPEM(cert)
	if err != nil || !ok {
		return nil, fmt.Errorf("failed to parse the certificate or the private key: %v", err)
	}

	conf := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    pool,
		ClientAuth:   tls.VerifyClientCertIfGiven,
	}

	l, err := tls.Listen("tcp", laddr, &conf)
	if err != nil {
		return nil, fmt.Errorf("failed to create TLS listener no network address %v: %v", laddr, err)
	}
	if debug {
		log.Printf("Listener created: %v\n", laddr)
	}

	return &Listener{
		Conns:    make([]*Conn, 0),
		debug:    debug,
		listener: l,
	}, nil
}

// Accept waits for incoming connections and forwards the client connect/message/disconnect events to provided handlers in a new goroutine.
// This function blocks and never returns, unless there is an error while accepting a new connection.
func (l *Listener) Accept(handleMsg func(conn *Conn, session *Session, msg []byte), handleDisconn func(conn *Conn, session *Session)) error {
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			if operr, ok := err.(*net.OpError); ok && operr.Op == "accept" && operr.Err.Error() == "use of closed network connection" {
				return nil
			}
			return fmt.Errorf("error while accepting a new connection from a client: %v", err)
			// todo: it might not be appropriate to break the loop on recoverable errors (like client disconnect during handshake)
			// the underlying fd.accept() does some basic recovery though we might need more: http://golang.org/src/net/fd_unix.go
		}

		tlsconn, ok := conn.(*tls.Conn)
		if !ok {
			conn.Close()
			return errors.New("cannot cast net.Conn interface to tls.Conn type")
		}
		if l.debug {
			l.connWG.Add(1)
			log.Println("Client connected:", conn.RemoteAddr())
		}

		c := NewConn(tlsconn, 0, 0, 0)
		l.Conns = append(l.Conns, c)
		go handleClient(l, c, handleMsg, handleDisconn)
	}
}

// handleClient waits for messages from the connected client and forwards the client message/disconnect
// events to provided handlers in a new goroutine.
// This function never returns, unless there is an error while reading from the channel or the client disconnects.
func handleClient(l *Listener, conn *Conn, handleMsg func(conn *Conn, session *Session, msg []byte), handleDisconn func(conn *Conn, session *Session)) error {
	session := &Session{}

	if l.debug {
		defer func() {
			l.connWG.Done()
			if session.Disconnected {
				log.Println("Client disconnected:", conn.RemoteAddr())
			} else {
				log.Println("Closed client connection:", conn.RemoteAddr())
			}
		}()
	}
	defer func() {
		session.Error = conn.Close() // todo: handle close error, store the error in conn object and return it to handleMsg/handleErr/handleDisconn or one level up (to server)
	}()

	for {
		if session.Error != nil {
			// todo: send error message to user, log the error, and close the conn and return
			return session.Error
		}

		n, msg, err := conn.Read()
		if err != nil {
			if err == io.EOF {
				session.Disconnected = true
				break
			}
			if operr, ok := err.(*net.OpError); ok && operr.Op == "read" && operr.Err.Error() == "use of closed network connection" {
				session.Disconnected = true
				break
			}
			log.Fatalln("Errored while reading:", err)
		}

		// shortcut 'ping' and 'close' messages, saves some processing time
		if n == 4 && bytes.Equal(msg, ping) {
			continue // send back pong?
		}
		if n == 5 && bytes.Equal(msg, closed) {
			go func() {
				handleDisconn(conn, session)
			}()
			return session.Error
		}

		go func() {
			handleMsg(conn, session, msg)
		}()
	}

	return session.Error
}

// Close closes the listener.
func (l *Listener) Close() error {
	if l.debug {
		defer log.Println("Listener closed:", l.listener.Addr())
	}
	return l.listener.Close()
}
<commit_msg>expose conn waiter<commit_after>package devastator

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"sync"
)

var (
	ping   = []byte("ping")
	closed = []byte("close")
)

// Listener accepts connections from devices.
type Listener struct {
	Conns    []*Conn
	debug    bool
	listener net.Listener
	connWG   sync.WaitGroup
}

// Listen creates a TCP listener with the given PEM encoded X.509 certificate and the private key on the local network address laddr.
// Debug mode logs all server activity.
func Listen(cert, privKey []byte, laddr string, debug bool) (*Listener, error) {
	tlsCert, err := tls.X509KeyPair(cert, privKey)
	pool := x509.NewCertPool()
	ok := pool.AppendCertsFromPEM(cert)
	if err != nil || !ok {
		return nil, fmt.Errorf("failed to parse the certificate or the private key: %v", err)
	}

	conf := tls.Config{
		Certificates: []tls.Certificate{tlsCert},
		ClientCAs:    pool,
		ClientAuth:   tls.VerifyClientCertIfGiven,
	}

	l, err := tls.Listen("tcp", laddr, &conf)
	if err != nil {
		return nil, fmt.Errorf("failed to create TLS listener no network address %v: %v", laddr, err)
	}
	if debug {
		log.Printf("Listener created: %v\n", laddr)
	}

	return &Listener{
		Conns:    make([]*Conn, 0),
		debug:    debug,
		listener: l,
	}, nil
}

// Accept waits for incoming connections and forwards the client connect/message/disconnect events to provided handlers in a new goroutine.
// This function blocks and never returns, unless there is an error while accepting a new connection.
func (l *Listener) Accept(handleMsg func(conn *Conn, session *Session, msg []byte), handleDisconn func(conn *Conn, session *Session)) error {
	for {
		conn, err := l.listener.Accept()
		if err != nil {
			if operr, ok := err.(*net.OpError); ok && operr.Op == "accept" && operr.Err.Error() == "use of closed network connection" {
				return nil
			}
			return fmt.Errorf("error while accepting a new connection from a client: %v", err)
			// todo: it might not be appropriate to break the loop on recoverable errors (like client disconnect during handshake)
			// the underlying fd.accept() does some basic recovery though we might need more: http://golang.org/src/net/fd_unix.go
		}

		tlsconn, ok := conn.(*tls.Conn)
		if !ok {
			conn.Close()
			return errors.New("cannot cast net.Conn interface to tls.Conn type")
		}
		if l.debug {
			l.connWG.Add(1)
			log.Println("Client connected:", conn.RemoteAddr())
		}

		c := NewConn(tlsconn, 0, 0, 0)
		l.Conns = append(l.Conns, c)
		go handleClient(l, c, handleMsg, handleDisconn)
	}
}

// handleClient waits for messages from the connected client and forwards the client message/disconnect
// events to provided handlers in a new goroutine.
// This function never returns, unless there is an error while reading from the channel or the client disconnects.
func handleClient(l *Listener, conn *Conn, handleMsg func(conn *Conn, session *Session, msg []byte), handleDisconn func(conn *Conn, session *Session)) error {
	session := &Session{}

	if l.debug {
		defer func() {
			l.connWG.Done()
			if session.Disconnected {
				log.Println("Client disconnected:", conn.RemoteAddr())
			} else {
				log.Println("Closed client connection:", conn.RemoteAddr())
			}
		}()
	}
	defer func() {
		session.Error = conn.Close() // todo: handle close error, store the error in conn object and return it to handleMsg/handleErr/handleDisconn or one level up (to server)
	}()

	for {
		if session.Error != nil {
			// todo: send error message to user, log the error, and close the conn and return
			return session.Error
		}

		n, msg, err := conn.Read()
		if err != nil {
			if err == io.EOF {
				session.Disconnected = true
				break
			}
			if operr, ok := err.(*net.OpError); ok && operr.Op == "read" && operr.Err.Error() == "use of closed network connection" {
				session.Disconnected = true
				break
			}
			log.Fatalln("Errored while reading:", err)
		}

		// shortcut 'ping' and 'close' messages, saves some processing time
		if n == 4 && bytes.Equal(msg, ping) {
			continue // send back pong?
		}
		if n == 5 && bytes.Equal(msg, closed) {
			go func() {
				handleDisconn(conn, session)
			}()
			return session.Error
		}

		go func() {
			handleMsg(conn, session, msg)
		}()
	}

	return session.Error
}

// Close closes the listener.
func (l *Listener) Close() error {
	if l.debug {
		defer log.Println("Listener closed:", l.listener.Addr())
	}
	return l.listener.Close()
}

// WaitConnClose blocks until all the client connections are closed.
func (l *Listener) WaitConnClose() {
	l.connWG.Wait()
}
<|endoftext|>