<commit_before>/*******************************************************************************
 * This is very experimental code and probably a long way from perfect or
 * ideal.  Please provide feed back on areas that would improve performance
 *
 */

// Package dgvoice provides opus encoding and audio file playback for the
// Discordgo package.
package dgvoice

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"sync"

	"github.com/bwmarrin/discordgo"
	"layeh.com/gopus"
)

// NOTE: This API is not final and these are likely to change.

// Technically the below settings can be adjusted however that poses
// a lot of other problems that are not handled well at this time.
// These below values seem to provide the best overall performance
const (
	channels  int = 2                   // 1 for mono, 2 for stereo
	frameRate int = 48000               // audio sampling rate
	frameSize int = 960                 // uint16 size of each audio frame
	maxBytes  int = (frameSize * 2) * 2 // max size of opus data
)

var (
	speakers    map[uint32]*gopus.Decoder
	opusEncoder *gopus.Encoder
	mu          sync.Mutex
)

// OnError gets called by dgvoice when an error is encountered.
// By default logs to STDERR
var OnError = func(str string, err error) {
	prefix := "dgVoice: " + str

	if err != nil {
		os.Stderr.WriteString(prefix + ", " + err.Error())
	} else {
		os.Stderr.WriteString(prefix)
	}
}

// SendPCM will receive on the provied channel encode
// received PCM data into Opus then send that to Discordgo
func SendPCM(v *discordgo.VoiceConnection, pcm <-chan []int16) {
	if pcm == nil {
		return
	}

	var err error

	opusEncoder, err = gopus.NewEncoder(frameRate, channels, gopus.Audio)

	if err != nil {
		OnError("NewEncoder Error", err)
		return
	}

	for {

		// read pcm from chan, exit if channel is closed.
		recv, ok := <-pcm
		if !ok {
			OnError("PCM Channel closed.", nil)
			return
		}

		// try encoding pcm frame with Opus
		opus, err := opusEncoder.Encode(recv, frameSize, maxBytes)
		if err != nil {
			OnError("Encoding Error", err)
			return
		}

		if v.Ready == false || v.OpusSend == nil {
			// OnError(fmt.Sprintf("Discordgo not ready for opus packets. %+v : %+v", v.Ready, v.OpusSend), nil)
			// Sending errors here might not be suited
			return
		}
		// send encoded opus data to the sendOpus channel
		v.OpusSend <- opus
	}
}

// ReceivePCM will receive on the the Discordgo OpusRecv channel and decode
// the opus audio into PCM then send it on the provided channel.
func ReceivePCM(v *discordgo.VoiceConnection, c chan *discordgo.Packet) {
	if c == nil {
		return
	}

	var err error

	for {
		if v.Ready == false || v.OpusRecv == nil {
			OnError(fmt.Sprintf("Discordgo not to receive opus packets. %+v : %+v", v.Ready, v.OpusSend), nil)
			return
		}

		p, ok := <-v.OpusRecv
		if !ok {
			return
		}

		if speakers == nil {
			speakers = make(map[uint32]*gopus.Decoder)
		}

		_, ok = speakers[p.SSRC]
		if !ok {
			speakers[p.SSRC], err = gopus.NewDecoder(48000, 2)
			if err != nil {
				OnError("error creating opus decoder", err)
				continue
			}
		}

		p.PCM, err = speakers[p.SSRC].Decode(p.Opus, 960, false)
		if err != nil {
			OnError("Error decoding opus data", err)
			continue
		}

		c <- p
	}
}

// PlayAudioFile will play the given filename to the already connected
// Discord voice server/channel.  voice websocket and udp socket
// must already be setup before this will work.
func PlayAudioFile(v *discordgo.VoiceConnection, filename string, stop <-chan bool) {

	// Create a shell command "object" to run.
	run := exec.Command("ffmpeg", "-i", filename, "-f", "s16le", "-ar", strconv.Itoa(frameRate), "-ac", strconv.Itoa(channels), "pipe:1")
	ffmpegout, err := run.StdoutPipe()
	if err != nil {
		OnError("StdoutPipe Error", err)
		return
	}

	ffmpegbuf := bufio.NewReaderSize(ffmpegout, 16384)

	// Starts the ffmpeg command
	err = run.Start()
	if err != nil {
		OnError("RunStart Error", err)
		return
	}

	go func() {
		<-stop
		err = run.Process.Kill()
	}()

	// Send "speaking" packet over the voice websocket
	err = v.Speaking(true)
	if err != nil {
		OnError("Couldn't set speaking", err)
	}

	// Send not "speaking" packet over the websocket when we finish
	defer func() {
		err := v.Speaking(false)
		if err != nil {
			OnError("Couldn't stop speaking", err)
		}
	}()

	send := make(chan []int16, 2)
	defer close(send)

	close := make(chan bool)
	go func() {
		SendPCM(v, send)
		close <- true
	}()

	for {
		// read data from ffmpeg stdout
		audiobuf := make([]int16, frameSize*channels)
		err = binary.Read(ffmpegbuf, binary.LittleEndian, &audiobuf)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return
		}
		if err != nil {
			OnError("error reading from ffmpeg stdout", err)
			return
		}

		// Send received PCM to the sendPCM channel
		select {
		case send <- audiobuf:
		case <-close:
			return
		}
	}
}
<commit_msg>Fixed strange punctation<commit_after>/*******************************************************************************
 * This is very experimental code and probably a long way from perfect or
 * ideal.  Please provide feed back on areas that would improve performance
 *
 */

// Package dgvoice provides opus encoding and audio file playback for the
// Discordgo package.
package dgvoice

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"sync"

	"github.com/bwmarrin/discordgo"
	"layeh.com/gopus"
)

// NOTE: This API is not final and these are likely to change.

// Technically the below settings can be adjusted however that poses
// a lot of other problems that are not handled well at this time.
// These below values seem to provide the best overall performance
const (
	channels  int = 2                   // 1 for mono, 2 for stereo
	frameRate int = 48000               // audio sampling rate
	frameSize int = 960                 // uint16 size of each audio frame
	maxBytes  int = (frameSize * 2) * 2 // max size of opus data
)

var (
	speakers    map[uint32]*gopus.Decoder
	opusEncoder *gopus.Encoder
	mu          sync.Mutex
)

// OnError gets called by dgvoice when an error is encountered.
// By default logs to STDERR
var OnError = func(str string, err error) {
	prefix := "dgVoice: " + str

	if err != nil {
		os.Stderr.WriteString(prefix + ": " + err.Error())
	} else {
		os.Stderr.WriteString(prefix)
	}
}

// SendPCM will receive on the provied channel encode
// received PCM data into Opus then send that to Discordgo
func SendPCM(v *discordgo.VoiceConnection, pcm <-chan []int16) {
	if pcm == nil {
		return
	}

	var err error

	opusEncoder, err = gopus.NewEncoder(frameRate, channels, gopus.Audio)

	if err != nil {
		OnError("NewEncoder Error", err)
		return
	}

	for {

		// read pcm from chan, exit if channel is closed.
		recv, ok := <-pcm
		if !ok {
			OnError("PCM Channel closed", nil)
			return
		}

		// try encoding pcm frame with Opus
		opus, err := opusEncoder.Encode(recv, frameSize, maxBytes)
		if err != nil {
			OnError("Encoding Error", err)
			return
		}

		if v.Ready == false || v.OpusSend == nil {
			// OnError(fmt.Sprintf("Discordgo not ready for opus packets. %+v : %+v", v.Ready, v.OpusSend), nil)
			// Sending errors here might not be suited
			return
		}
		// send encoded opus data to the sendOpus channel
		v.OpusSend <- opus
	}
}

// ReceivePCM will receive on the the Discordgo OpusRecv channel and decode
// the opus audio into PCM then send it on the provided channel.
func ReceivePCM(v *discordgo.VoiceConnection, c chan *discordgo.Packet) {
	if c == nil {
		return
	}

	var err error

	for {
		if v.Ready == false || v.OpusRecv == nil {
			OnError(fmt.Sprintf("Discordgo not to receive opus packets. %+v : %+v", v.Ready, v.OpusSend), nil)
			return
		}

		p, ok := <-v.OpusRecv
		if !ok {
			return
		}

		if speakers == nil {
			speakers = make(map[uint32]*gopus.Decoder)
		}

		_, ok = speakers[p.SSRC]
		if !ok {
			speakers[p.SSRC], err = gopus.NewDecoder(48000, 2)
			if err != nil {
				OnError("error creating opus decoder", err)
				continue
			}
		}

		p.PCM, err = speakers[p.SSRC].Decode(p.Opus, 960, false)
		if err != nil {
			OnError("Error decoding opus data", err)
			continue
		}

		c <- p
	}
}

// PlayAudioFile will play the given filename to the already connected
// Discord voice server/channel.  voice websocket and udp socket
// must already be setup before this will work.
func PlayAudioFile(v *discordgo.VoiceConnection, filename string, stop <-chan bool) {

	// Create a shell command "object" to run.
	run := exec.Command("ffmpeg", "-i", filename, "-f", "s16le", "-ar", strconv.Itoa(frameRate), "-ac", strconv.Itoa(channels), "pipe:1")
	ffmpegout, err := run.StdoutPipe()
	if err != nil {
		OnError("StdoutPipe Error", err)
		return
	}

	ffmpegbuf := bufio.NewReaderSize(ffmpegout, 16384)

	// Starts the ffmpeg command
	err = run.Start()
	if err != nil {
		OnError("RunStart Error", err)
		return
	}

	go func() {
		<-stop
		err = run.Process.Kill()
	}()

	// Send "speaking" packet over the voice websocket
	err = v.Speaking(true)
	if err != nil {
		OnError("Couldn't set speaking", err)
	}

	// Send not "speaking" packet over the websocket when we finish
	defer func() {
		err := v.Speaking(false)
		if err != nil {
			OnError("Couldn't stop speaking", err)
		}
	}()

	send := make(chan []int16, 2)
	defer close(send)

	close := make(chan bool)
	go func() {
		SendPCM(v, send)
		close <- true
	}()

	for {
		// read data from ffmpeg stdout
		audiobuf := make([]int16, frameSize*channels)
		err = binary.Read(ffmpegbuf, binary.LittleEndian, &audiobuf)
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return
		}
		if err != nil {
			OnError("error reading from ffmpeg stdout", err)
			return
		}

		// Send received PCM to the sendPCM channel
		select {
		case send <- audiobuf:
		case <-close:
			return
		}
	}
}
<|endoftext|>


<commit_before>package q

import "github.com/oov/q/qutil"

var (
	// DefaultDialect is default setting of builder's dialect. When not set a Dialect in builder, this value is used.
	DefaultDialect qutil.Dialect

	// MySQL implements a dialect in MySQL.
	MySQL = qutil.MySQL
	// PostgreSQL implements a dialect in PostgreSQL.
	PostgreSQL = qutil.PostgreSQL
	// SQLite implements a dialect in SQLite.
	SQLite = qutil.SQLite
)
<commit_msg>cosmetic change<commit_after>package q

import "github.com/oov/q/qutil"

var (
	// DefaultDialect is default setting of builder's dialect.
	// When not set a Dialect in builder, this value is used.
	DefaultDialect qutil.Dialect

	// MySQL implements a dialect in MySQL.
	MySQL = qutil.MySQL
	// PostgreSQL implements a dialect in PostgreSQL.
	PostgreSQL = qutil.PostgreSQL
	// SQLite implements a dialect in SQLite.
	SQLite = qutil.SQLite
)
<|endoftext|>


<commit_before>package quiztool

import (
	"fmt"
	"log"
	"strconv"
	"strings"
)

func (wrp *Wrapper) NextLF() bool {
	if wrp.Debug {
		log.Printf("VERBOSE Doing NextLF: bufPtr = %d bufLen = %d\n", wrp.bufPtr, wrp.bufLen)
	}

	// find next LF
	for {
		if wrp.bufPtr == wrp.bufLen-1 {
			return false
		}
		if wrp.buf[wrp.bufPtr] == '\r' && wrp.buf[wrp.bufPtr+1] == '\n' {
			wrp.bufPtr++
			if wrp.bufPtr < wrp.bufLen {
				wrp.bufPtr++
			}
			if wrp.Debug {
				log.Printf("VERBOSE NextLF returned bufPtr=%d\n", wrp.bufPtr)
			}
			return true
		}
		wrp.bufPtr++
	}
}

func (wrp *Wrapper) GetBulkString() (s string, err error) {
	if wrp.Debug {
		log.Printf("VERBOSE GetBulkString processing: %q\n", wrp.buf[wrp.bufPtr:wrp.bufLen])
	}

	// Locate size
	i := wrp.bufPtr
	if !wrp.NextLF() {
		err = fmt.Errorf("Bulk string size EOLN not found: %s", wrp.buf[wrp.bufPtr:wrp.bufLen-2])
		return
	}
	l, err := strconv.Atoi(string(wrp.buf[i+1 : wrp.bufPtr-2]))
	if l == -1 {
		err = fmt.Errorf("%s Bulk string not found", wrp.msg.Command)
		s = "null"
		return
	}

	if wrp.Debug {
		log.Printf("VERBOSE GetBulkString expected length %d\n", l)
	}

	// Locate value
	i = wrp.bufPtr
	if !wrp.NextLF() {
		err = fmt.Errorf("Bulk string value EOLN not found: %s", wrp.buf[wrp.bufPtr:wrp.bufLen-2])
		return
	}

	// Make sure Bulk String length matches RESP size value
	if l != wrp.bufPtr-i-2 {
		err = fmt.Errorf("Expected length %d != actual length %d for Bulk string %s",
			l, wrp.bufPtr-i, wrp.buf[i:wrp.bufPtr-2])
		return
	}

	// Convert to JSON (string will be in "")
	s = fmt.Sprintf("%q", wrp.buf[i:wrp.bufPtr-2])
	if wrp.Debug {
		log.Printf("VERBOSE GetBulkString returned: %s\n", s)
	}
	return
}

func (wrp *Wrapper) ParseBuf() (s string, err error) {
	if wrp.Debug {
		log.Printf("VERBOSE Parsebuf processing: %q\n", wrp.buf[wrp.bufPtr:wrp.bufLen])
	}

	resp_type := wrp.buf[wrp.bufPtr]

	if resp_type == '+' {
		// Simple string, convert to JSON (string will be in "")
		s = fmt.Sprintf("%q", wrp.buf[wrp.bufPtr+1:wrp.bufLen-2])
	} else if resp_type == ':' {
		// Integer, convert to JSON (string)
		s = fmt.Sprintf("%s", wrp.buf[wrp.bufPtr+1:wrp.bufLen-2])
	} else if resp_type == '$' {
		// Bulk string
		s, err = wrp.GetBulkString()
		if err != nil {
			return
		}
	} else {
		err = fmt.Errorf("Unrecognized response data: %s", wrp.buf[wrp.bufPtr+1:wrp.bufLen-2])
		return
	}

	if wrp.Debug {
		log.Printf("VERBOSE Parsebuf returned: %s\n", s)
	}
	return
}

func (wrp *Wrapper) GetArray() (s string, err error) {
	if wrp.Debug {
		log.Printf("VERBOSE GetArray processing: %q\n", wrp.buf[wrp.bufPtr:wrp.bufLen])
	}

	// Locate size
	i := wrp.bufPtr
	if !wrp.NextLF() {
		err = fmt.Errorf("Array EOLN not found: %s", wrp.buf[wrp.bufPtr:wrp.bufLen])
		return
	}
	l, err := strconv.Atoi(string(wrp.buf[i+1 : wrp.bufPtr-2]))
	if l == -1 {
		err = fmt.Errorf("%s Array not found", wrp.msg.Command)
		s = "null"
		return
	}

	if wrp.Debug {
		log.Printf("VERBOSE GetArray expected length %d\n", l)
	}

	// Iterate through values
	arr := make([]string, l)
	var elem string

	for a := 0; a < l; a++ {
		// Make sure there are more entries to parse
		if wrp.bufPtr == wrp.bufLen {
			err = fmt.Errorf("Array expected length %d != actual length %d", l, a+1)
			return
		}
		elem, err = wrp.ParseBuf()

		if wrp.keyPair {
			// Form { elem1 : elem2 }, ...
			if a%2 == 0 {
				arr[a] = elem + ":"
			} else {
				arr[a] = elem + ","
			}
		} else {
			arr[a] = elem + ","
		}

	}

	// Join array of key-pairs, or values
	var data string
	if wrp.keyPair {
		data = "{"
	} else {
		data = "["
	}
	data = data + strings.Join(arr, "")
	// Drop trailing ','
	if wrp.keyPair {
		s = data[:len(data)-1] + "}"
	} else {
		s = data[:len(data)-1] + "]"
	}
	if wrp.Debug {
		log.Printf("VERBOSE GetArray returned: %s\n", s)
	}
	return
}

func (wrp *Wrapper) ParseSocket() (data string, err error) {

	wrp.bufPtr = 0
	resp_type := wrp.buf[0]

	// Array data is list if HMGET
	wrp.keyPair = wrp.msg.Command != "HMGET"

	// Check for Redis error
	if resp_type == '-' {
		err = fmt.Errorf("%s", wrp.buf[1:wrp.bufLen-2])
		return
	}

	// Check for Array
	if resp_type == '*' {
		data, err = wrp.GetArray()
		return
	}

	// Check for Simple string, Integer or Bulk string
	data, err = wrp.ParseBuf()
	return
}
<commit_msg>Improve error messaging<commit_after>package quiztool

import (
	"fmt"
	"log"
	"strconv"
	"strings"
)

func (wrp *Wrapper) NextLF() bool {
	if wrp.Debug {
		log.Printf("VERBOSE Doing NextLF: bufPtr = %d bufLen = %d\n", wrp.bufPtr, wrp.bufLen)
	}

	// find next LF
	for {
		if wrp.bufPtr == wrp.bufLen-1 {
			return false
		}
		if wrp.buf[wrp.bufPtr] == '\r' && wrp.buf[wrp.bufPtr+1] == '\n' {
			wrp.bufPtr++
			if wrp.bufPtr < wrp.bufLen {
				wrp.bufPtr++
			}
			if wrp.Debug {
				log.Printf("VERBOSE NextLF returned bufPtr=%d\n", wrp.bufPtr)
			}
			return true
		}
		wrp.bufPtr++
	}
}

func (wrp *Wrapper) GetBulkString() (s string, err error) {
	if wrp.Debug {
		log.Printf("VERBOSE GetBulkString processing: %q\n", wrp.buf[wrp.bufPtr:wrp.bufLen])
	}

	// Locate size
	i := wrp.bufPtr
	if !wrp.NextLF() {
		err = fmt.Errorf("Bulk string size EOLN not found: %s", wrp.buf[wrp.bufPtr:wrp.bufLen-2])
		return
	}
	l, err := strconv.Atoi(string(wrp.buf[i+1 : wrp.bufPtr-2]))
	if l == -1 {
		err = fmt.Errorf("%s Bulk string not found", wrp.msg.Command)
		s = "null"
		return
	}

	if wrp.Debug {
		log.Printf("VERBOSE GetBulkString expected length %d\n", l)
	}

	// Locate value
	i = wrp.bufPtr
	if !wrp.NextLF() {
		err = fmt.Errorf("Bulk string value EOLN not found: %s", wrp.buf[wrp.bufPtr:wrp.bufLen-2])
		return
	}

	// Make sure Bulk String length matches RESP size value
	if l != wrp.bufPtr-i-2 {
		err = fmt.Errorf("Expected length %d != actual length %d for Bulk string %s",
			l, wrp.bufPtr-i, wrp.buf[i:wrp.bufPtr-2])
		return
	}

	// Convert to JSON (string will be in "")
	s = fmt.Sprintf("%q", wrp.buf[i:wrp.bufPtr-2])
	if wrp.Debug {
		log.Printf("VERBOSE GetBulkString returned: %s\n", s)
	}
	return
}

func (wrp *Wrapper) ParseBuf() (s string, err error) {
	if wrp.Debug {
		log.Printf("VERBOSE Parsebuf processing: %q\n", wrp.buf[wrp.bufPtr:wrp.bufLen])
	}

	resp_type := wrp.buf[wrp.bufPtr]

	if resp_type == '+' {
		// Simple string, convert to JSON (string will be in "")
		s = fmt.Sprintf("%q", wrp.buf[wrp.bufPtr+1:wrp.bufLen-2])
	} else if resp_type == ':' {
		// Integer, convert to JSON (string)
		s = fmt.Sprintf("%s", wrp.buf[wrp.bufPtr+1:wrp.bufLen-2])
	} else if resp_type == '$' {
		// Bulk string
		s, err = wrp.GetBulkString()
		if err != nil {
			return
		}
	} else {
		err = fmt.Errorf("Unrecognized response data: %s", wrp.buf[wrp.bufPtr+1:wrp.bufLen-2])
		return
	}

	if wrp.Debug {
		log.Printf("VERBOSE Parsebuf returned: %s\n", s)
	}
	return
}

func (wrp *Wrapper) GetKeyPairOrArray() (s string, err error) {
	var method = "Array"
	if wrp.keyPair {
		method = "KeyPair"
	}
	if wrp.Debug {
		log.Printf("VERBOSE Get%s processing: %q\n", method, wrp.buf[wrp.bufPtr:wrp.bufLen])
	}

	// Locate size
	i := wrp.bufPtr
	if !wrp.NextLF() {
		err = fmt.Errorf("Get%s EOLN not found: %s", method, wrp.buf[wrp.bufPtr:wrp.bufLen])
		return
	}
	l, err := strconv.Atoi(string(wrp.buf[i+1 : wrp.bufPtr-2]))
	if l == -1 || l == 0 {
		err = fmt.Errorf("%s %s not found", wrp.msg.Command, method)
		s = "null"
		return
	}

	if wrp.Debug {
		log.Printf("VERBOSE Get%s expected length %d\n", method, l)
	}

	// Iterate through values
	arr := make([]string, l)
	var elem string

	for a := 0; a < l; a++ {
		// Make sure there are more entries to parse
		if wrp.bufPtr == wrp.bufLen {
			err = fmt.Errorf("Get%s expected length %d != actual length %d", method, l, a+1)
			return
		}
		elem, err = wrp.ParseBuf()

		if wrp.keyPair {
			// Form { elem1 : elem2 }, ...
			if a%2 == 0 {
				arr[a] = elem + ":"
			} else {
				arr[a] = elem + ","
			}
		} else {
			arr[a] = elem + ","
		}

	}

	// Join array of key-pairs, or values
	var data string
	if wrp.keyPair {
		data = "{"
	} else {
		data = "["
	}
	data = data + strings.Join(arr, "")
	// Drop trailing ','
	if wrp.keyPair {
		s = data[:len(data)-1] + "}"
	} else {
		s = data[:len(data)-1] + "]"
	}
	if wrp.Debug {
		log.Printf("VERBOSE Get%s returned: %s\n", method, s)
	}
	return
}

func (wrp *Wrapper) ParseSocket() (data string, err error) {

	wrp.bufPtr = 0
	resp_type := wrp.buf[0]

	// Array data is list if HMGET
	wrp.keyPair = wrp.msg.Command != "HMGET"

	// Check for Redis error
	if resp_type == '-' {
		err = fmt.Errorf("%s", wrp.buf[1:wrp.bufLen-2])
		return
	}

	// Check for Array
	if resp_type == '*' {
		data, err = wrp.GetKeyPairOrArray()
		return
	}

	// Check for Simple string, Integer or Bulk string
	data, err = wrp.ParseBuf()
	return
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/docker-deploy/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 67108864
const maxMemory = 536870912
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const linkSeparator = `:`

var networkConfig = network.NetworkingConfig{
	EndpointsConfig: map[string]*network.EndpointSettings{
		`traefik`: &network.EndpointSettings{},
	},
}

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)
var commandSplit = regexp.MustCompile(`["']([^"']+)["']|(\S+)`)

type dockerComposeService struct {
	Image       string
	Command     string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, loggedUser *user, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = loggedUser.username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if service.Command != `` {
		config.Cmd = strslice.StrSlice{}
		for _, args := range commandSplit.FindAllStringSubmatch(service.Command, -1) {
			config.Cmd = append(config.Cmd, args[1]+args[2])
		}
	}

	return &config
}

func getHostConfig(service *dockerComposeService, deployedServices map[string]deployedService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit < maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		id := linkParts[0]
		if linkedService, ok := deployedServices[linkParts[0]]; ok {
			id := linkedService.ID
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		hostConfig.Links = append(hostConfig.Links, id+linkSeparator+alias)
	}

	return &hostConfig
}

func pullImage(image string, loggedUser *user) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(loggedUser.username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(loggedUser.username + ` ends pulling for ` + image)
	return nil
}

func cleanContainers(containers *[]types.Container, loggedUser *user) {
	for _, container := range *containers {
		log.Print(loggedUser.username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(loggedUser.username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.id, strings.TrimSuffix(service.Name, deploySuffix)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(appName string, serviceName string) string {
	return appName + `_` + serviceName + deploySuffix
}

func createAppHandler(w http.ResponseWriter, loggedUser *user, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(loggedUser.username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(loggedUser, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, loggedUser); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Print(loggedUser.username + ` starts ` + serviceFullName)

		id, err := docker.ContainerCreate(context.Background(), getConfig(&service, loggedUser, appNameStr), getHostConfig(&service, deployedServices), &networkConfig, serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			return
		}

		startContainer(id.ID)
		deployedServices[serviceName] = deployedService{ID: id.ID, Name: serviceFullName}
	}

	log.Print(`Waiting 10 seconds for containers to start...`)
	time.Sleep(10 * time.Second)

	cleanContainers(&ownerContainers, loggedUser)
	if err := renameDeployedContainers(&deployedServices); err != nil {
		errorHandler(w, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<commit_msg>Update compose.go<commit_after>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/docker-deploy/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 67108864
const maxMemory = 536870912
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const linkSeparator = `:`

var networkConfig = network.NetworkingConfig{
	EndpointsConfig: map[string]*network.EndpointSettings{
		`traefik`: &network.EndpointSettings{},
	},
}

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)
var commandSplit = regexp.MustCompile(`["']([^"']+)["']|(\S+)`)

type dockerComposeService struct {
	Image       string
	Command     string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, loggedUser *user, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = loggedUser.username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if service.Command != `` {
		config.Cmd = strslice.StrSlice{}
		for _, args := range commandSplit.FindAllStringSubmatch(service.Command, -1) {
			config.Cmd = append(config.Cmd, args[1]+args[2])
		}
	}

	return &config
}

func getHostConfig(service *dockerComposeService, deployedServices map[string]deployedService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit < maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		id := linkParts[0]
		if linkedService, ok := deployedServices[linkParts[0]]; ok {
			id := linkedService.ID
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		hostConfig.Links = append(hostConfig.Links, id+linkSeparator+alias)
	}

	return &hostConfig
}

func pullImage(image string, loggedUser *user) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(loggedUser.username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(loggedUser.username + ` ends pulling for ` + image)
	return nil
}

func cleanContainers(containers *[]types.Container, loggedUser *user) {
	for _, container := range *containers {
		log.Print(loggedUser.username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(loggedUser.username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, strings.TrimSuffix(service.Name, deploySuffix)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(appName string, serviceName string) string {
	return appName + `_` + serviceName + deploySuffix
}

func createAppHandler(w http.ResponseWriter, loggedUser *user, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(loggedUser.username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(loggedUser, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, loggedUser); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Print(loggedUser.username + ` starts ` + serviceFullName)

		id, err := docker.ContainerCreate(context.Background(), getConfig(&service, loggedUser, appNameStr), getHostConfig(&service, deployedServices), &networkConfig, serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			return
		}

		startContainer(id.ID)
		deployedServices[serviceName] = deployedService{ID: id.ID, Name: serviceFullName}
	}

	log.Print(`Waiting 10 seconds for containers to start...`)
	time.Sleep(10 * time.Second)

	cleanContainers(&ownerContainers, loggedUser)
	if err := renameDeployedContainers(&deployedServices); err != nil {
		errorHandler(w, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"github.com/ViBiOh/docker-deploy/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 67108864
const maxMemory = 536870912
const defaultTag = `:latest`
const deploySuffix = `_deploy`

var networkConfig = network.NetworkingConfig{
	EndpointsConfig: map[string]*network.EndpointSettings{
		`traefik`: &network.EndpointSettings{},
	},
}

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)
var commandSplit = regexp.MustCompile(`["']([^"']+)["']|(\S+)`)

type dockerComposeService struct {
	Image       string
	Command     string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

func getConfig(service *dockerComposeService, loggedUser *user, appName string) (*container.Config, error) {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = loggedUser.username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if service.Command != `` {
		config.Cmd = strslice.StrSlice{}
		for _, args := range commandSplit.FindAllStringSubmatch(service.Command, -1) {
			config.Cmd = append(config.Cmd, args[1]+args[2])
		}
	}

	return &config, nil
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit < maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func pullImage(image string, loggedUser *user) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(loggedUser.username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return err
	}

	readBody(pull)
	log.Print(loggedUser.username + ` ends pulling for ` + image)
	return nil
}

func cleanContainers(containers *[]types.Container, loggedUser *user) {
	for _, container := range *containers {
		log.Print(loggedUser.username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(loggedUser.username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]string) error {
	for id, name := range *containers {
		if err := docker.ContainerRename(context.Background(), id, strings.TrimSuffix(name, deploySuffix)); err != nil {
			return err
		}
	}

	return nil
}

func createAppHandler(w http.ResponseWriter, loggedUser *user, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, err)
		return
	}

	appNameStr := string(appName)
	log.Print(loggedUser.username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(loggedUser, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]string)
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, loggedUser); err != nil {
			errorHandler(w, err)
			return
		}

		config, err := getConfig(&service, loggedUser, appNameStr)
		if err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := appNameStr + `_` + serviceName + deploySuffix
		log.Print(loggedUser.username + ` starts ` + serviceFullName)

		id, err := docker.ContainerCreate(context.Background(), config, getHostConfig(&service), &networkConfig, serviceFullName)
		if err != nil {
			errorHandler(w, err)
			return
		}

		startContainer(id.ID)
		deployedServices[id.ID] = serviceFullName
	}

	log.Print(`Waiting 5 seconds for containers to start...`)
	time.Sleep(5 * time.Second)

	cleanContainers(&ownerContainers, loggedUser)
	if err := renameDeployedContainers(&deployedServices); err != nil {
		errorHandler(w, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<commit_msg>Adding link for compose<commit_after>package docker

import (
	"context"
	"github.com/ViBiOh/docker-deploy/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 67108864
const maxMemory = 536870912
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const linkSeparator = `:`

var networkConfig = network.NetworkingConfig{
	EndpointsConfig: map[string]*network.EndpointSettings{
		`traefik`: &network.EndpointSettings{},
	},
}

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)
var commandSplit = regexp.MustCompile(`["']([^"']+)["']|(\S+)`)

type dockerComposeService struct {
	Image       string
	Command     string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

func getConfig(service *dockerComposeService, loggedUser *user, appName string) (*container.Config, error) {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = loggedUser.username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if service.Command != `` {
		config.Cmd = strslice.StrSlice{}
		for _, args := range commandSplit.FindAllStringSubmatch(service.Command, -1) {
			config.Cmd = append(config.Cmd, args[1]+args[2])
		}
	}

	return &config, nil
}

func getHostConfig(service *dockerComposeService, appName string) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit < maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		hostConfig.Links = append(hostConfig.Links, getServiceFullName(appName, linkParts[0])+linkSeparator+alias)
	}

	return &hostConfig
}

func pullImage(image string, loggedUser *user) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(loggedUser.username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return err
	}

	readBody(pull)
	log.Print(loggedUser.username + ` ends pulling for ` + image)
	return nil
}

func cleanContainers(containers *[]types.Container, loggedUser *user) {
	for _, container := range *containers {
		log.Print(loggedUser.username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(loggedUser.username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]string) error {
	for id, name := range *containers {
		if err := docker.ContainerRename(context.Background(), id, strings.TrimSuffix(name, deploySuffix)); err != nil {
			return err
		}
	}

	return nil
}

func getServiceFullName(appName string, serviceName string) string {
	return appName + `_` + serviceName + deploySuffix
}

func createAppHandler(w http.ResponseWriter, loggedUser *user, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, err)
		return
	}

	appNameStr := string(appName)
	log.Print(loggedUser.username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(loggedUser, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]string)
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, loggedUser); err != nil {
			errorHandler(w, err)
			return
		}

		config, err := getConfig(&service, loggedUser, appNameStr)
		if err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Print(loggedUser.username + ` starts ` + serviceFullName)

		id, err := docker.ContainerCreate(context.Background(), config, getHostConfig(&service, appNameStr), &networkConfig, serviceFullName)
		if err != nil {
			errorHandler(w, err)
			return
		}

		startContainer(id.ID)
		deployedServices[id.ID] = serviceFullName
	}

	log.Print(`Waiting 5 seconds for containers to start...`)
	time.Sleep(5 * time.Second)

	cleanContainers(&ownerContainers, loggedUser)
	if err := renameDeployedContainers(&deployedServices); err != nil {
		errorHandler(w, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<|endoftext|>


<commit_before>// +build linux darwin dragonfly freebsd netbsd openbsd rumprun

// Package reuseport provides TCP net.Listener with SO_REUSEPORT support.
//
// SO_REUSEPORT allows linear scaling server performance on multi-CPU servers.
// See https://www.nginx.com/blog/socket-sharding-nginx-release-1-9-1/ for more details :)
//
// The package is based on https://github.com/kavu/go_reuseport .
package reuseport

import (
	"errors"
	"fmt"
	"net"
	"os"
	"syscall"
)

func getSockaddr(network, addr string) (sa syscall.Sockaddr, soType int, err error) {
	// TODO: add support for tcp and tcp6 networks.

	if network != "tcp4" {
		return nil, -1, errors.New("only tcp4 network is supported")
	}

	tcpAddr, err := net.ResolveTCPAddr(network, addr)
	if err != nil {
		return nil, -1, err
	}

	var sa4 syscall.SockaddrInet4
	sa4.Port = tcpAddr.Port
	copy(sa4.Addr[:], tcpAddr.IP.To4())
	return &sa4, syscall.AF_INET, nil
}

// ErrNoReusePort is returned if the OS doesn't support SO_REUSEPORT.
type ErrNoReusePort struct {
	err error
}

// Error implements error interface.
func (e *ErrNoReusePort) Error() string {
	return fmt.Sprintf("The OS doesn't support SO_REUSEPORT: %s", e.err)
}

// Listen returns TCP listener with SO_REUSEPORT option set.
//
// Only tcp4 network is supported.
//
// ErrNoReusePort error is returned if the system doesn't support SO_REUSEPORT.
func Listen(network, addr string) (l net.Listener, err error) {
	var (
		soType, fd int
		file       *os.File
		sockaddr   syscall.Sockaddr
	)

	if sockaddr, soType, err = getSockaddr(network, addr); err != nil {
		return nil, err
	}

	if fd, err = syscall.Socket(soType, syscall.SOCK_STREAM, syscall.IPPROTO_TCP); err != nil {
		return nil, err
	}

	if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, soReusePort, 1); err != nil {
		syscall.Close(fd)
		return nil, &ErrNoReusePort{err}
	}

	if err = syscall.Bind(fd, sockaddr); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	name := fmt.Sprintf("reuseport.%d.%s.%s", os.Getpid(), network, addr)
	file = os.NewFile(uintptr(fd), name)
	if l, err = net.FileListener(file); err != nil {
		file.Close()
		return nil, err
	}

	if err = file.Close(); err != nil {
		l.Close()
		return nil, err
	}

	return l, err
}
<commit_msg>reuseport: set FD_CLOSEEXEC for the sockets (#117)<commit_after>// +build linux darwin dragonfly freebsd netbsd openbsd rumprun

// Package reuseport provides TCP net.Listener with SO_REUSEPORT support.
//
// SO_REUSEPORT allows linear scaling server performance on multi-CPU servers.
// See https://www.nginx.com/blog/socket-sharding-nginx-release-1-9-1/ for more details :)
//
// The package is based on https://github.com/kavu/go_reuseport .
package reuseport

import (
	"errors"
	"fmt"
	"net"
	"os"
	"syscall"
)

func getSockaddr(network, addr string) (sa syscall.Sockaddr, soType int, err error) {
	// TODO: add support for tcp and tcp6 networks.

	if network != "tcp4" {
		return nil, -1, errors.New("only tcp4 network is supported")
	}

	tcpAddr, err := net.ResolveTCPAddr(network, addr)
	if err != nil {
		return nil, -1, err
	}

	var sa4 syscall.SockaddrInet4
	sa4.Port = tcpAddr.Port
	copy(sa4.Addr[:], tcpAddr.IP.To4())
	return &sa4, syscall.AF_INET, nil
}

// ErrNoReusePort is returned if the OS doesn't support SO_REUSEPORT.
type ErrNoReusePort struct {
	err error
}

// Error implements error interface.
func (e *ErrNoReusePort) Error() string {
	return fmt.Sprintf("The OS doesn't support SO_REUSEPORT: %s", e.err)
}

// Listen returns TCP listener with SO_REUSEPORT option set.
//
// Only tcp4 network is supported.
//
// ErrNoReusePort error is returned if the system doesn't support SO_REUSEPORT.
func Listen(network, addr string) (l net.Listener, err error) {
	var (
		soType, fd int
		file       *os.File
		sockaddr   syscall.Sockaddr
	)

	if sockaddr, soType, err = getSockaddr(network, addr); err != nil {
		return nil, err
	}

	syscall.ForkLock.RLock()
	fd, err = syscall.Socket(soType, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
	if err == nil {
		syscall.CloseOnExec(fd)
	}
	syscall.ForkLock.RUnlock()
	if err != nil {
		return nil, err
	}

	if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, soReusePort, 1); err != nil {
		syscall.Close(fd)
		return nil, &ErrNoReusePort{err}
	}

	if err = syscall.Bind(fd, sockaddr); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil {
		syscall.Close(fd)
		return nil, err
	}

	name := fmt.Sprintf("reuseport.%d.%s.%s", os.Getpid(), network, addr)
	file = os.NewFile(uintptr(fd), name)
	if l, err = net.FileListener(file); err != nil {
		file.Close()
		return nil, err
	}

	if err = file.Close(); err != nil {
		l.Close()
		return nil, err
	}

	return l, err
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd

package commands

import (
	"fmt"
	"strings"
	"time"

	cmds "github.com/jbenet/go-ipfs/commands"
	config "github.com/jbenet/go-ipfs/config"
	core "github.com/jbenet/go-ipfs/core"
	ipns "github.com/jbenet/go-ipfs/fuse/ipns"
	mount "github.com/jbenet/go-ipfs/fuse/mount"
	rofs "github.com/jbenet/go-ipfs/fuse/readonly"
)

// amount of time to wait for mount errors
// TODO is this non-deterministic?
const mountTimeout = time.Second

// fuseNoDirectory used to check the returning fuse error
const fuseNoDirectory = "fusermount: failed to access mountpoint"

var mountCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Mounts IPFS to the filesystem (read-only)",
		Synopsis: `
ipfs [-f <ipfs mount path>] [-n <ipns mount path>]
`,
		ShortDescription: `
Mount ipfs at a read-only mountpoint on the OS (default: /ipfs and /ipns).
All ipfs objects will be accessible under that directory. Note that the
root will not be listable, as it is virtual. Access known paths directly.

You may have to create /ipfs and /ipfs before using 'ipfs mount':

> sudo mkdir /ipfs /ipns
> sudo chown ` + "`" + `whoami` + "`" + ` /ipfs /ipns
> ipfs daemon &
> ipfs mount
`,
		LongDescription: `
Mount ipfs at a read-only mountpoint on the OS (default: /ipfs and /ipns).
All ipfs objects will be accessible under that directory. Note that the
root will not be listable, as it is virtual. Access known paths directly.

You may have to create /ipfs and /ipfs before using 'ipfs mount':

> sudo mkdir /ipfs /ipns
> sudo chown ` + "`" + `whoami` + "`" + ` /ipfs /ipns
> ipfs daemon &
> ipfs mount

EXAMPLE:

# setup
> mkdir foo
> echo "baz" > foo/bar
> ipfs add -r foo
added QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR foo/bar
added QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC foo
> ipfs ls QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC
QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR 12 bar
> ipfs cat QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR
baz

# mount
> ipfs daemon &
> ipfs mount
IPFS mounted at: /ipfs
IPNS mounted at: /ipns
> cd /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC
> ls
bar
> cat bar
baz
> cat /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC/bar
baz
> cat /ipfs/QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR
baz
`,
	},

	Options: []cmds.Option{
		// TODO longform
		cmds.StringOption("f", "The path where IPFS should be mounted"),

		// TODO longform
		cmds.StringOption("n", "The path where IPNS should be mounted"),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		cfg, err := req.Context().GetConfig()
		if err != nil {
			return nil, err
		}

		node, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		// check if we already have live mounts.
		// if the user said "Mount", then there must be something wrong.
		// so, close them and try again.
		if node.Mounts.Ipfs != nil {
			node.Mounts.Ipfs.Unmount()
		}
		if node.Mounts.Ipns != nil {
			node.Mounts.Ipns.Unmount()
		}

		// error if we aren't running node in online mode
		if node.Network == nil {
			return nil, errNotOnline
		}

		if err := platformFuseChecks(); err != nil {
			return nil, err
		}

		fsdir, found, err := req.Option("f").String()
		if err != nil {
			return nil, err
		}
		if !found {
			fsdir = cfg.Mounts.IPFS // use default value
		}

		// get default mount points
		nsdir, found, err := req.Option("n").String()
		if err != nil {
			return nil, err
		}
		if !found {
			nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare!
		}

		if err := doMount(node, fsdir, nsdir); err != nil {
			return nil, err
		}

		var output config.Mounts
		output.IPFS = fsdir
		output.IPNS = nsdir
		return &output, nil
	},
	Type: &config.Mounts{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) ([]byte, error) {
			v := res.Output().(*config.Mounts)
			s := fmt.Sprintf("IPFS mounted at: %s\n", v.IPFS)
			s += fmt.Sprintf("IPNS mounted at: %s\n", v.IPNS)
			return []byte(s), nil
		},
	},
}

func doMount(node *core.IpfsNode, fsdir, nsdir string) error {
	fmtFuseErr := func(err error) error {
		s := err.Error()
		if strings.Contains(s, fuseNoDirectory) {
			s = strings.Replace(s, `fusermount: "fusermount:`, "", -1)
			s = strings.Replace(s, `\n", exit status 1`, "", -1)
			return cmds.ClientError(s)
		}
		return err
	}

	// this sync stuff is so that both can be mounted simultaneously.
	var fsmount mount.Mount
	var nsmount mount.Mount
	var err1 error
	var err2 error

	done := make(chan struct{})

	go func() {
		fsmount, err1 = rofs.Mount(node, fsdir)
		done <- struct{}{}
	}()

	go func() {
		nsmount, err2 = ipns.Mount(node, nsdir, fsdir)
		done <- struct{}{}
	}()

	<-done
	<-done

	if err1 != nil || err2 != nil {
		fsmount.Close()
		nsmount.Close()
		if err1 != nil {
			return fmtFuseErr(err1)
		} else {
			return fmtFuseErr(err2)
		}
	}

	// setup node state, so that it can be cancelled
	node.Mounts.Ipfs = fsmount
	node.Mounts.Ipns = nsmount
	return nil
}

var platformFuseChecks = func() error {
	return nil
}
<commit_msg>Add missing 'mount' in mount synopsis<commit_after>// +build linux darwin freebsd

package commands

import (
	"fmt"
	"strings"
	"time"

	cmds "github.com/jbenet/go-ipfs/commands"
	config "github.com/jbenet/go-ipfs/config"
	core "github.com/jbenet/go-ipfs/core"
	ipns "github.com/jbenet/go-ipfs/fuse/ipns"
	mount "github.com/jbenet/go-ipfs/fuse/mount"
	rofs "github.com/jbenet/go-ipfs/fuse/readonly"
)

// amount of time to wait for mount errors
// TODO is this non-deterministic?
const mountTimeout = time.Second

// fuseNoDirectory used to check the returning fuse error
const fuseNoDirectory = "fusermount: failed to access mountpoint"

var mountCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Mounts IPFS to the filesystem (read-only)",
		Synopsis: `
ipfs mount [-f <ipfs mount path>] [-n <ipns mount path>]
`,
		ShortDescription: `
Mount ipfs at a read-only mountpoint on the OS (default: /ipfs and /ipns).
All ipfs objects will be accessible under that directory. Note that the
root will not be listable, as it is virtual. Access known paths directly.

You may have to create /ipfs and /ipfs before using 'ipfs mount':

> sudo mkdir /ipfs /ipns
> sudo chown ` + "`" + `whoami` + "`" + ` /ipfs /ipns
> ipfs daemon &
> ipfs mount
`,
		LongDescription: `
Mount ipfs at a read-only mountpoint on the OS (default: /ipfs and /ipns).
All ipfs objects will be accessible under that directory. Note that the
root will not be listable, as it is virtual. Access known paths directly.

You may have to create /ipfs and /ipfs before using 'ipfs mount':

> sudo mkdir /ipfs /ipns
> sudo chown ` + "`" + `whoami` + "`" + ` /ipfs /ipns
> ipfs daemon &
> ipfs mount

EXAMPLE:

# setup
> mkdir foo
> echo "baz" > foo/bar
> ipfs add -r foo
added QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR foo/bar
added QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC foo
> ipfs ls QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC
QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR 12 bar
> ipfs cat QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR
baz

# mount
> ipfs daemon &
> ipfs mount
IPFS mounted at: /ipfs
IPNS mounted at: /ipns
> cd /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC
> ls
bar
> cat bar
baz
> cat /ipfs/QmSh5e7S6fdcu75LAbXNZAFY2nGyZUJXyLCJDvn2zRkWyC/bar
baz
> cat /ipfs/QmWLdkp93sNxGRjnFHPaYg8tCQ35NBY3XPn6KiETd3Z4WR
baz
`,
	},

	Options: []cmds.Option{
		// TODO longform
		cmds.StringOption("f", "The path where IPFS should be mounted"),

		// TODO longform
		cmds.StringOption("n", "The path where IPNS should be mounted"),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		cfg, err := req.Context().GetConfig()
		if err != nil {
			return nil, err
		}

		node, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		// check if we already have live mounts.
		// if the user said "Mount", then there must be something wrong.
		// so, close them and try again.
		if node.Mounts.Ipfs != nil {
			node.Mounts.Ipfs.Unmount()
		}
		if node.Mounts.Ipns != nil {
			node.Mounts.Ipns.Unmount()
		}

		// error if we aren't running node in online mode
		if node.Network == nil {
			return nil, errNotOnline
		}

		if err := platformFuseChecks(); err != nil {
			return nil, err
		}

		fsdir, found, err := req.Option("f").String()
		if err != nil {
			return nil, err
		}
		if !found {
			fsdir = cfg.Mounts.IPFS // use default value
		}

		// get default mount points
		nsdir, found, err := req.Option("n").String()
		if err != nil {
			return nil, err
		}
		if !found {
			nsdir = cfg.Mounts.IPNS // NB: be sure to not redeclare!
		}

		if err := doMount(node, fsdir, nsdir); err != nil {
			return nil, err
		}

		var output config.Mounts
		output.IPFS = fsdir
		output.IPNS = nsdir
		return &output, nil
	},
	Type: &config.Mounts{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) ([]byte, error) {
			v := res.Output().(*config.Mounts)
			s := fmt.Sprintf("IPFS mounted at: %s\n", v.IPFS)
			s += fmt.Sprintf("IPNS mounted at: %s\n", v.IPNS)
			return []byte(s), nil
		},
	},
}

func doMount(node *core.IpfsNode, fsdir, nsdir string) error {
	fmtFuseErr := func(err error) error {
		s := err.Error()
		if strings.Contains(s, fuseNoDirectory) {
			s = strings.Replace(s, `fusermount: "fusermount:`, "", -1)
			s = strings.Replace(s, `\n", exit status 1`, "", -1)
			return cmds.ClientError(s)
		}
		return err
	}

	// this sync stuff is so that both can be mounted simultaneously.
	var fsmount mount.Mount
	var nsmount mount.Mount
	var err1 error
	var err2 error

	done := make(chan struct{})

	go func() {
		fsmount, err1 = rofs.Mount(node, fsdir)
		done <- struct{}{}
	}()

	go func() {
		nsmount, err2 = ipns.Mount(node, nsdir, fsdir)
		done <- struct{}{}
	}()

	<-done
	<-done

	if err1 != nil || err2 != nil {
		fsmount.Close()
		nsmount.Close()
		if err1 != nil {
			return fmtFuseErr(err1)
		} else {
			return fmtFuseErr(err2)
		}
	}

	// setup node state, so that it can be cancelled
	node.Mounts.Ipfs = fsmount
	node.Mounts.Ipns = nsmount
	return nil
}

var platformFuseChecks = func() error {
	return nil
}
<|endoftext|>


<commit_before>package bigflake

import (
	"encoding/hex"
	"errors"
	"fmt"
	"math/big"
	"regexp"

	"github.com/mattheath/base62"
)

// BigflakeId represents a globally unique ID
type BigflakeId struct {
	id *big.Int
}

// String returns the raw id as a string
func (bf *BigflakeId) String() string {
	return bf.id.String()
}

// Base62 returns a base62 encoded version
func (bf *BigflakeId) Base62() string {
	return base62.EncodeBigInt(bf.id)
}

// Uuid returns the id encoded in UUID format
func (bf *BigflakeId) Uuid() string {
	b := bf.id.Bytes()

	// Pad numbers less than the full 128bit (16 byte) width
	if len(b) < 16 {
		pad := 16 - len(b)
		padslice := make([]byte, pad)
		// prepend to bytes
		b = append(padslice, b...)
	}

	// Return hex formatted with delimiters
	h := hex.EncodeToString(b)
	return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:])
}

// Raw returns a raw 128bit integer
func (bf *BigflakeId) Raw() *big.Int {
	return bf.id
}
<commit_msg>Add BinaryString method to IDs<commit_after>package bigflake

import (
	"encoding/hex"
	"errors"
	"fmt"
	"math/big"
	"regexp"

	"github.com/mattheath/base62"
)

// BigflakeId represents a globally unique ID
type BigflakeId struct {
	id *big.Int
}

// String returns the raw id as a string
func (bf *BigflakeId) String() string {
	return bf.id.String()
}

// BinaryString returns a padded 128bit binary number formatted as a string
func (bf *BigflakeId) BinaryString() string {
	return fmt.Sprintf("%0128b", bf.id)
}

// Base62 returns a base62 encoded version
func (bf *BigflakeId) Base62() string {
	return base62.EncodeBigInt(bf.id)
}

// Uuid returns the id encoded in UUID format
func (bf *BigflakeId) Uuid() string {
	b := bf.id.Bytes()

	// Pad numbers less than the full 128bit (16 byte) width
	if len(b) < 16 {
		pad := 16 - len(b)
		padslice := make([]byte, pad)
		// prepend to bytes
		b = append(padslice, b...)
	}

	// Return hex formatted with delimiters
	h := hex.EncodeToString(b)
	return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:])
}

// Raw returns a raw 128bit integer
func (bf *BigflakeId) Raw() *big.Int {
	return bf.id
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	acceptance "github.com/cloudfoundry/bosh-bootloader/acceptance-tests"
	"github.com/cloudfoundry/bosh-bootloader/acceptance-tests/actors"
	"github.com/cloudfoundry/bosh-bootloader/storage"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("plan", func() {
	var (
		bbl      actors.BBL
		stateDir string
		iaas     string
	)

	BeforeEach(func() {
		acceptance.SkipUnless("plan")

		configuration, err := acceptance.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		iaas = configuration.IAAS

		stateDir = configuration.StateFileDir

		bbl = actors.NewBBL(stateDir, pathToBBL, configuration, "plan-env")
	})

	It("sets up the bbl state directory", func() {
		session := bbl.Plan("--name", bbl.PredefinedEnvID())
		Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

		expectedArtifacts := []string{
			filepath.Join(stateDir, "create-jumpbox.sh"),
			filepath.Join(stateDir, "create-director.sh"),
			filepath.Join(stateDir, "delete-jumpbox.sh"),
			filepath.Join(stateDir, "delete-director.sh"),
			filepath.Join(stateDir, ".bbl", "cloudconfig", "cloud-config.yml"),
			filepath.Join(stateDir, ".bbl", "cloudconfig", "ops.yml"),
			filepath.Join(stateDir, "bosh-deployment", "bosh.yml"),
			filepath.Join(stateDir, "bosh-deployment", iaas, "cpi.yml"),
			filepath.Join(stateDir, "bosh-deployment", "credhub.yml"),
			filepath.Join(stateDir, "bosh-deployment", "jumpbox-user.yml"),
			filepath.Join(stateDir, "bosh-deployment", "uaa.yml"),
			filepath.Join(stateDir, "jumpbox-deployment", "jumpbox.yml"),
			filepath.Join(stateDir, "jumpbox-deployment", iaas, "cpi.yml"),
			filepath.Join(stateDir, "terraform", "template.tf"),
			filepath.Join(stateDir, "vars", "user-ops-file.yml"),
		}

		By("verifying that artifacts are created in state dir", func() {
			for _, f := range expectedArtifacts {
				fileinfo, err := os.Stat(f)
				Expect(err).NotTo(HaveOccurred())
				if strings.HasSuffix(f, ".sh") {
					Expect(fileinfo.Mode().String()).To(Equal("-rwxr-x---"))
				} else {
					Expect(fileinfo.Mode().String()).To(Equal("-rwxr-----"))
				}
			}
		})

		By("modifying artifacts", func() {
			for _, f := range expectedArtifacts {
				err := ioutil.WriteFile(f, []byte("modified after plan"), storage.StateMode)
				Expect(err).NotTo(HaveOccurred())
			}
		})

		By("rerunning bbl plan", func() {
			session := bbl.Plan("--name", bbl.PredefinedEnvID())
			Eventually(session, 5*time.Minute).Should(gexec.Exit(0))
		})

		By("verifying that modified artifacts were overwritten", func() {
			for _, f := range expectedArtifacts {
				contents, err := ioutil.ReadFile(f)
				Expect(err).NotTo(HaveOccurred())
				Expect(string(contents)).NotTo(ContainSubstring("modified after plan"))
			}
		})
	})
})
<commit_msg>Fix plan test now that cloud-config has moved.<commit_after>package acceptance_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	acceptance "github.com/cloudfoundry/bosh-bootloader/acceptance-tests"
	"github.com/cloudfoundry/bosh-bootloader/acceptance-tests/actors"
	"github.com/cloudfoundry/bosh-bootloader/storage"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("plan", func() {
	var (
		bbl      actors.BBL
		stateDir string
		iaas     string
	)

	BeforeEach(func() {
		acceptance.SkipUnless("plan")

		configuration, err := acceptance.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		iaas = configuration.IAAS

		stateDir = configuration.StateFileDir

		bbl = actors.NewBBL(stateDir, pathToBBL, configuration, "plan-env")
	})

	It("sets up the bbl state directory", func() {
		session := bbl.Plan("--name", bbl.PredefinedEnvID())
		Eventually(session, 5*time.Minute).Should(gexec.Exit(0))

		expectedArtifacts := []string{
			filepath.Join(stateDir, "create-jumpbox.sh"),
			filepath.Join(stateDir, "create-director.sh"),
			filepath.Join(stateDir, "delete-jumpbox.sh"),
			filepath.Join(stateDir, "delete-director.sh"),
			filepath.Join(stateDir, "cloudconfig", "cloud-config.yml"),
			filepath.Join(stateDir, "cloudconfig", "ops.yml"),
			filepath.Join(stateDir, "bosh-deployment", "bosh.yml"),
			filepath.Join(stateDir, "bosh-deployment", iaas, "cpi.yml"),
			filepath.Join(stateDir, "bosh-deployment", "credhub.yml"),
			filepath.Join(stateDir, "bosh-deployment", "jumpbox-user.yml"),
			filepath.Join(stateDir, "bosh-deployment", "uaa.yml"),
			filepath.Join(stateDir, "jumpbox-deployment", "jumpbox.yml"),
			filepath.Join(stateDir, "jumpbox-deployment", iaas, "cpi.yml"),
			filepath.Join(stateDir, "terraform", "template.tf"),
			filepath.Join(stateDir, "vars", "user-ops-file.yml"),
		}

		By("verifying that artifacts are created in state dir", func() {
			for _, f := range expectedArtifacts {
				fileinfo, err := os.Stat(f)
				Expect(err).NotTo(HaveOccurred())
				if strings.HasSuffix(f, ".sh") {
					Expect(fileinfo.Mode().String()).To(Equal("-rwxr-x---"))
				} else {
					Expect(fileinfo.Mode().String()).To(Equal("-rwxr-----"))
				}
			}
		})

		By("modifying artifacts", func() {
			for _, f := range expectedArtifacts {
				err := ioutil.WriteFile(f, []byte("modified after plan"), storage.StateMode)
				Expect(err).NotTo(HaveOccurred())
			}
		})

		By("rerunning bbl plan", func() {
			session := bbl.Plan("--name", bbl.PredefinedEnvID())
			Eventually(session, 5*time.Minute).Should(gexec.Exit(0))
		})

		By("verifying that modified artifacts were overwritten", func() {
			for _, f := range expectedArtifacts {
				contents, err := ioutil.ReadFile(f)
				Expect(err).NotTo(HaveOccurred())
				Expect(string(contents)).NotTo(ContainSubstring("modified after plan"))
			}
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2015 LinkedIn Corp. Licensed under the Apache License,
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License.
 You may obtain a copy of the License
// at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache
// License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License.
 You may obtain a copy of
// the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.

package goavro

import (
	"bufio"
	"bytes"
	"code.google.com/p/snappy-go/snappy"
	"compress/flate"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
)

// ErrReaderInit is returned when an error is created during Reader
// initialization.
type ErrReaderInit struct {
	Message string
	Err     error
}

func (e *ErrReaderInit) Error() string {
	if e.Err == nil {
		return "cannot create Reader: " + e.Message
	} else if e.Message == "" {
		return "cannot create Reader: " + e.Err.Error()
	} else {
		return "cannot create Reader: " + e.Message + "; " + e.Err.Error()
	}
}

// ErrReaderBlockCount is returned when a reader detects an error
// while attempting to read the block count and block size.
type ErrReaderBlockCount struct {
	Err error
}

func (e *ErrReaderBlockCount) Error() string {
	return "cannot read block count and size: " + e.Err.Error()
}

// ReaderSetter functions are those those which are used to instantiate
// a new Reader.
type ReaderSetter func(*Reader) error

// BufferFromReader wraps the specified `io.Reader` using a
// `bufio.Reader` to read from a file.
func BufferFromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = bufio.NewReader(r)
		return nil
	}
}

// FromReader specifies the `io.Reader` to use when reading a file.
func FromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = r
		return nil
	}
}

// Reader structure contains data necessary to read Avro files.
type Reader struct {
	CompressionCodec string
	DataSchema       string
	Sync             []byte
	dataCodec        Codec
	datum            Datum
	deblocked        chan Datum
	done             bool
	err              error
	r                io.Reader
}

// NewReader returns a object to read data from an io.Reader using the
// Avro Object Container Files format.
//
//     func main() {
//         conn, err := net.Dial("tcp", "127.0.0.1:8080")
//         if err != nil {
//             log.Fatal(err)
//         }
//         fr, err := goavro.NewReader(goavro.FromReader(conn))
//         if err != nil {
//             log.Fatal("cannot create Reader: ", err)
//         }
//         defer func() {
//             if err := fr.Close(); err != nil {
//                 log.Fatal(err)
//             }
//         }()
//
//         for fr.Scan() {
//             datum, err := fr.Read()
//             if err != nil {
//                 log.Println("cannot read datum: ", err)
//                 continue
//             }
//             fmt.Println("RECORD: ", datum)
//         }
//     }
func NewReader(setters ...ReaderSetter) (*Reader, error) {
	var err error
	fr := &Reader{}
	for _, setter := range setters {
		err = setter(fr)
		if err != nil {
			return nil, &ErrReaderInit{Err: err}
		}
	}
	if fr.r == nil {
		return nil, &ErrReaderInit{Message: "must specify io.Reader"}
	}
	// read in header information and use it to initialize Reader
	magic := make([]byte, 4)
	_, err = fr.r.Read(magic)
	if err != nil {
		return nil, &ErrReaderInit{Message: "cannot read magic number", Err: err}
	}
	if bytes.Compare(magic, []byte(magicBytes)) != 0 {
		return nil, &ErrReaderInit{Message: "invalid magic number: " + string(magic)}
	}
	meta, err := decodeHeaderMetadata(fr.r)
	if err != nil {
		return nil, &ErrReaderInit{Message: "cannot read header metadata", Err: err}
	}
	fr.CompressionCodec, err = getHeaderString("avro.codec", meta)
	if err != nil {
		return nil, &ErrReaderInit{Message: "cannot read header metadata", Err: err}
	}
	if !IsCompressionCodecSupported(fr.CompressionCodec) {
		return nil, &ErrWriterInit{Message: fmt.Sprintf("unsupported codec: %s", fr.CompressionCodec)}
	}
	fr.DataSchema, err = getHeaderString("avro.schema", meta)
	if err != nil {
		return nil, &ErrReaderInit{Message: "cannot read header metadata", Err: err}
	}
	if fr.dataCodec, err = NewCodec(fr.DataSchema); err != nil {
		return nil, &ErrWriterInit{Message: "cannot compile schema", Err: err}
	}
	fr.Sync = make([]byte, syncLength)
	if _, err = fr.r.Read(fr.Sync); err != nil {
		return nil, &ErrReaderInit{Message: "cannot read sync marker", Err: err}
	}
	// setup reading pipeline
	toDecompress := make(chan *readerBlock)
	toDecode := make(chan *readerBlock)
	fr.deblocked = make(chan Datum)
	go read(fr, toDecompress)
	go decompress(fr, toDecompress, toDecode)
	go decode(fr, toDecode)
	return fr, nil
}

// Close releases resources and returns any Reader errors.
func (fr *Reader) Close() error {
	return fr.err
}

// Scan returns true if more data is ready to be read.
func (fr *Reader) Scan() bool {
	fr.datum = <-fr.deblocked
	return !fr.done
}

// Read returns the next element from the Reader.
func (fr *Reader) Read() (interface{}, error) {
	return fr.datum.Value, fr.datum.Err
}

func decodeHeaderMetadata(r io.Reader) (map[string]interface{}, error) {
	md, err := metadataCodec.Decode(r)
	if err != nil {
		return nil, err
	}
	return md.(map[string]interface{}), nil
}

func getHeaderString(someKey string, header map[string]interface{}) (string, error) {
	v, ok := header[someKey]
	if !ok {
		return "", fmt.Errorf("header ought to have %v key", someKey)
	}
	return string(v.([]byte)), nil
}

type readerBlock struct {
	data       []Datum
	datumCount int
	err        error
	r          io.Reader
}

func read(fr *Reader, toDecompress chan<- *readerBlock) {
	// NOTE: these variables created outside loop to eliminate churn
	var lr io.Reader
	var bits []byte
	sync := make([]byte, syncLength)

	blockCount, blockSize, err := readBlockCountAndSize(fr.r)
	if err != nil {
		blockCount = 0
	}
	for blockCount != 0 {
		lr = io.LimitReader(fr.r, int64(blockSize))
		if bits, err = ioutil.ReadAll(lr); err != nil {
			err = fmt.Errorf("cannot read block: %v", err)
			break
		}
		toDecompress <- &readerBlock{datumCount: blockCount, r: bytes.NewReader(bits)}
		if _, err = fr.r.Read(sync); err != nil {
			err = fmt.Errorf("cannot read sync marker: %v", err)
			break
		}
		if bytes.Compare(fr.Sync, sync) != 0 {
			err = fmt.Errorf("sync marker mismatch: %#v != %#v", sync, fr.Sync)
			break
		}
		if blockCount, blockSize, err = readBlockCountAndSize(fr.r); err != nil {
			break
		}
	}
	if err != nil {
		fr.err = fmt.Errorf("error reading: %v", err)
	}
	close(toDecompress)
}

func readBlockCountAndSize(r io.Reader) (blockCount, blockSize int, err error) {

	// io.EOF
	// io.ErrUnexpectedEOF

	bc, err := longCodec.Decode(r)
	switch err {
	case io.EOF:
		// we're done
		// log.Printf("io.EOF: error type %T\n", err)
		return 0, 0, nil
	case nil:
		// no error: ignore
	default:
		// TODO: this could be optimized
		if strings.Contains(err.Error(), "EOF") {
			// log.Printf("EOF: error type: %T", err)
			// if ed, ok := err.(*ErrDecode); ok {
			// 	log.Printf("EOF: error value: %#v", ed.Err)
			// 	log.Printf("EOF: error string: %v", ed.Err)
			// }
			// we're done
			return 0, 0, nil
		}
		return 0, 0, &ErrReaderBlockCount{err}
	}
	bs, err := longCodec.Decode(r)
	if err != nil {
		return 0, 0, &ErrReaderBlockCount{err}
	}
	return int(bc.(int64)), int(bs.(int64)), nil
}

func decompress(fr *Reader, toDecompress <-chan *readerBlock, toDecode chan<- *readerBlock) {
	switch fr.CompressionCodec {
	case CompressionDeflate:
		var rc io.ReadCloser
		var bits []byte
		for block := range toDecompress {
			rc = flate.NewReader(block.r)
			bits, block.err = ioutil.ReadAll(rc)
			if block.err != nil {
				block.err = fmt.Errorf("cannot read from deflate: %v", block.err)
				toDecode <- block
				rc.Close() // ignore any close error
				continue
			}
			block.err = rc.Close()
			if block.err != nil {
				block.err = fmt.Errorf("cannot close deflate: %v", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(bits)
			toDecode <- block
		}
	case CompressionNull:
		for block := range toDecompress {
			toDecode <- block
		}
	case CompressionSnappy:
		var src, dst []byte
		for block := range toDecompress {
			src, block.err = ioutil.ReadAll(block.r)
			if block.err != nil {
				block.err = fmt.Errorf("cannot read: %v", block.err)
				toDecode <- block
				continue
			}
			dst, block.err = snappy.Decode(dst, src)
			if block.err != nil {
				block.err = fmt.Errorf("cannot decompress: %v", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(dst)
			toDecode <- block
		}
	}
	close(toDecode)
}

func decode(fr *Reader, toDecode <-chan *readerBlock) {
decodeLoop:
	for block := range toDecode {
		for i := 0; i < block.datumCount; i++ {
			var datum Datum
			datum.Value, datum.Err = fr.dataCodec.Decode(block.r)
			if datum.Value == nil && datum.Err == nil {
				break decodeLoop
			}
			fr.deblocked <- datum
		}
	}
	fr.done = true
	close(fr.deblocked)
}
<commit_msg>Reader uses goavro specific error type structures<commit_after>// Copyright 2015 LinkedIn Corp. Licensed under the Apache License,
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License.
 You may obtain a copy of the License
// at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache
// License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License.
 You may obtain a copy of
// the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.

package goavro

import (
	"bufio"
	"bytes"
	"code.google.com/p/snappy-go/snappy"
	"compress/flate"
	"fmt"
	"io"
	"io/ioutil"
)

// ErrReaderInit is returned when the encoder encounters an error.
type ErrReaderInit struct {
	Message string
	Err     error
}

func (e ErrReaderInit) Error() string {
	if e.Err == nil {
		return "cannot build " + e.Message
	} else {
		return "cannot build " + e.Message + ": " + e.Err.Error()
	}
}

func newReaderInitError(a ...interface{}) *ErrReaderInit {
	var err error
	var format, message string
	var ok bool
	if len(a) == 0 {
		return &ErrReaderInit{"cannot create reader: no reason given", nil}
	}
	// if last item is error: save it
	if err, ok = a[len(a)-1].(error); ok {
		a = a[:len(a)-1] // pop it
	}
	// if items left, first ought to be format string
	if len(a) > 0 {
		if format, ok = a[0].(string); ok {
			a = a[1:] // unshift
			message = fmt.Sprintf(format, a...)
		}
	}
	return &ErrReaderInit{message, err}
}

// ErrReaderBlockCount is returned when a reader detects an error
// while attempting to read the block count and block size.
type ErrReaderBlockCount struct {
	Err error
}

func (e *ErrReaderBlockCount) Error() string {
	return "cannot read block count and size: " + e.Err.Error()
}

// ReaderSetter functions are those those which are used to instantiate
// a new Reader.
type ReaderSetter func(*Reader) error

// BufferFromReader wraps the specified `io.Reader` using a
// `bufio.Reader` to read from a file.
func BufferFromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = bufio.NewReader(r)
		return nil
	}
}

// FromReader specifies the `io.Reader` to use when reading a file.
func FromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = r
		return nil
	}
}

// Reader structure contains data necessary to read Avro files.
type Reader struct {
	CompressionCodec string
	DataSchema       string
	Sync             []byte
	dataCodec        Codec
	datum            Datum
	deblocked        chan Datum
	done             bool
	err              error
	r                io.Reader
}

// NewReader returns a object to read data from an io.Reader using the
// Avro Object Container Files format.
//
//     func main() {
//         conn, err := net.Dial("tcp", "127.0.0.1:8080")
//         if err != nil {
//             log.Fatal(err)
//         }
//         fr, err := goavro.NewReader(goavro.FromReader(conn))
//         if err != nil {
//             log.Fatal("cannot create Reader: ", err)
//         }
//         defer func() {
//             if err := fr.Close(); err != nil {
//                 log.Fatal(err)
//             }
//         }()
//
//         for fr.Scan() {
//             datum, err := fr.Read()
//             if err != nil {
//                 log.Println("cannot read datum: ", err)
//                 continue
//             }
//             fmt.Println("RECORD: ", datum)
//         }
//     }
func NewReader(setters ...ReaderSetter) (*Reader, error) {
	var err error
	fr := &Reader{}
	for _, setter := range setters {
		err = setter(fr)
		if err != nil {
			return nil, newReaderInitError(err)
		}
	}
	if fr.r == nil {
		return nil, newReaderInitError("must specify io.Reader")
	}
	// read in header information and use it to initialize Reader
	magic := make([]byte, 4)
	_, err = fr.r.Read(magic)
	if err != nil {
		return nil, newReaderInitError("cannot read magic number", err)
	}
	if bytes.Compare(magic, []byte(magicBytes)) != 0 {
		return nil, &ErrReaderInit{Message: "invalid magic number: " + string(magic)}
	}
	meta, err := decodeHeaderMetadata(fr.r)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	fr.CompressionCodec, err = getHeaderString("avro.codec", meta)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	if !IsCompressionCodecSupported(fr.CompressionCodec) {
		return nil, newReaderInitError("unsupported codec: %s", fr.CompressionCodec)
	}
	fr.DataSchema, err = getHeaderString("avro.schema", meta)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	if fr.dataCodec, err = NewCodec(fr.DataSchema); err != nil {
		return nil, newReaderInitError("cannot compile schema", err)
	}
	fr.Sync = make([]byte, syncLength)
	if _, err = fr.r.Read(fr.Sync); err != nil {
		return nil, newReaderInitError("cannot read sync marker", err)
	}
	// setup reading pipeline
	toDecompress := make(chan *readerBlock)
	toDecode := make(chan *readerBlock)
	fr.deblocked = make(chan Datum)
	go read(fr, toDecompress)
	go decompress(fr, toDecompress, toDecode)
	go decode(fr, toDecode)
	return fr, nil
}

// Close releases resources and returns any Reader errors.
func (fr *Reader) Close() error {
	return fr.err
}

// Scan returns true if more data is ready to be read.
func (fr *Reader) Scan() bool {
	fr.datum = <-fr.deblocked
	return !fr.done
}

// Read returns the next element from the Reader.
func (fr *Reader) Read() (interface{}, error) {
	return fr.datum.Value, fr.datum.Err
}

func decodeHeaderMetadata(r io.Reader) (map[string]interface{}, error) {
	md, err := metadataCodec.Decode(r)
	if err != nil {
		return nil, err
	}
	return md.(map[string]interface{}), nil
}

func getHeaderString(someKey string, header map[string]interface{}) (string, error) {
	v, ok := header[someKey]
	if !ok {
		return "", fmt.Errorf("header ought to have %v key", someKey)
	}
	return string(v.([]byte)), nil
}

type readerBlock struct {
	data       []Datum
	datumCount int
	err        error
	r          io.Reader
}

// ErrReader is returned when the reader encounters an error.
type ErrReader struct {
	Message string
	Err     error
}

func (e ErrReader) Error() string {
	if e.Err == nil {
		return "cannot read from reader: " + e.Message
	} else {
		return "cannot read from reader: " + e.Message + ": " + e.Err.Error()
	}
}

func newReaderError(a ...interface{}) *ErrReader {
	var err error
	var format, message string
	var ok bool
	if len(a) == 0 {
		return &ErrReader{"no reason given", nil}
	}
	// if last item is error: save it
	if err, ok = a[len(a)-1].(error); ok {
		a = a[:len(a)-1] // pop it
	}
	// if items left, first ought to be format string
	if len(a) > 0 {
		if format, ok = a[0].(string); ok {
			a = a[1:] // unshift
			message = fmt.Sprintf(format, a...)
		}
	}
	return &ErrReader{message, err}
}

func read(fr *Reader, toDecompress chan<- *readerBlock) {
	// NOTE: these variables created outside loop to reduce churn
	var lr io.Reader
	var bits []byte
	sync := make([]byte, syncLength)

	blockCount, blockSize, err := readBlockCountAndSize(fr.r)
	if err != nil {
		blockCount = 0
	}
	for blockCount != 0 {
		lr = io.LimitReader(fr.r, int64(blockSize))
		if bits, err = ioutil.ReadAll(lr); err != nil {
			err = newReaderError("cannot read block", err)
			break
		}
		toDecompress <- &readerBlock{datumCount: blockCount, r: bytes.NewReader(bits)}
		if _, err = fr.r.Read(sync); err != nil {
			err = newReaderError("cannot read sync marker", err)
			break
		}
		if bytes.Compare(fr.Sync, sync) != 0 {
			err = newReaderError(fmt.Sprintf("sync marker mismatch: %#v != %#v", sync, fr.Sync))
			break
		}
		if blockCount, blockSize, err = readBlockCountAndSize(fr.r); err != nil {
			break
		}
	}
	if err != nil {
		fr.err = err
	}
	close(toDecompress)
}

func readBlockCountAndSize(r io.Reader) (blockCount, blockSize int, err error) {
	bc, err := longCodec.Decode(r)
	if err != nil {
		if ed, ok := err.(*ErrDecoder); ok && ed.Err.Error() == "EOF" {
			return 0, 0, nil // we're done
		}
		return 0, 0, &ErrReaderBlockCount{err}
	}
	bs, err := longCodec.Decode(r)
	if err != nil {
		return 0, 0, &ErrReaderBlockCount{err}
	}
	return int(bc.(int64)), int(bs.(int64)), nil
}

func decompress(fr *Reader, toDecompress <-chan *readerBlock, toDecode chan<- *readerBlock) {
	switch fr.CompressionCodec {
	case CompressionDeflate:
		var rc io.ReadCloser
		var bits []byte
		for block := range toDecompress {
			rc = flate.NewReader(block.r)
			bits, block.err = ioutil.ReadAll(rc)
			if block.err != nil {
				block.err = newReaderError("cannot read from deflate", block.err)
				toDecode <- block
				rc.Close() // already have the read error; ignore the close error
				continue
			}
			block.err = rc.Close()
			if block.err != nil {
				block.err = newReaderError("cannot close deflate", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(bits)
			toDecode <- block
		}
	case CompressionNull:
		for block := range toDecompress {
			toDecode <- block
		}
	case CompressionSnappy:
		var src, dst []byte
		for block := range toDecompress {
			src, block.err = ioutil.ReadAll(block.r)
			if block.err != nil {
				block.err = newReaderError("cannot read", block.err)
				toDecode <- block
				continue
			}
			dst, block.err = snappy.Decode(dst, src)
			if block.err != nil {
				block.err = newReaderError("cannot decompress", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(dst)
			toDecode <- block
		}
	}
	close(toDecode)
}

func decode(fr *Reader, toDecode <-chan *readerBlock) {
decodeLoop:
	for block := range toDecode {
		for i := 0; i < block.datumCount; i++ {
			var datum Datum
			datum.Value, datum.Err = fr.dataCodec.Decode(block.r)
			if datum.Value == nil && datum.Err == nil {
				break decodeLoop
			}
			fr.deblocked <- datum
		}
	}
	fr.done = true
	close(fr.deblocked)
}
<|endoftext|>


<commit_before>// This file is part of bitrot, a bitrot scrubber in Go
// http://github.com/marcopaganini/bitrot
// (C) 2014 by Marco Paganini <paganini@paganini.net>

package main

import (
	"compress/gzip"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"
)

type FileInfo struct {
	Size   int64
	Mode   os.FileMode
	Mtime  time.Time
	Md5sum []byte
}

type DirTree struct {
	Root      string
	ExcludeRe []*string
	Files     map[string]*FileInfo
}

// Return a new DirTree object.
func NewDirTree(root string, excludeRe []*string) *DirTree {
	dt := &DirTree{
		Root:      root,
		ExcludeRe: excludeRe,
	}
	dt.Files = make(map[string]*FileInfo)
	return dt
}

func (d *DirTree) compareFile(fname string) (string, error) {
	var (
		memfi  *FileInfo
		exists bool
	)

	osinfo, err := os.Stat(fname)
	if err != nil {
		return "", err
	}

	md5sum, err := md5sum(fname)
	if err != nil {
		return "", err
	}

	if memfi, exists = d.Files[fname]; !exists {
		// If medatada changed, replace entry silently
		memfi = &FileInfo{
			Size:   osinfo.Size(),
			Mode:   osinfo.Mode(),
			Mtime:  osinfo.ModTime(),
			Md5sum: md5sum,
		}
		d.Files[fname] = memfi
		Log.Verbosef(1, "[New] %s (%x)\n", fname, md5sum)
	} else {
		if osinfo.ModTime() != memfi.Mtime || osinfo.Mode() != memfi.Mode || osinfo.Size() != memfi.Size {
			// Exists: If medatada changed, replace entry silently
			Log.Verbosef(1, "[Metadata changes] %s (%x)\n", fname, md5sum)
			memfi.Size = osinfo.Size()
			memfi.Mode = osinfo.Mode()
			memfi.Mtime = osinfo.ModTime()
			memfi.Md5sum = md5sum
			d.Files[fname] = memfi
		} else {
			Log.Verbosef(1, "[No metadata changes] %s (%x)", fname, md5sum)
			// Exists: No metadata changes. Report md5 differences
			for k, _ := range md5sum {
				if memfi.Md5sum[k] != md5sum[k] {
					return fmt.Sprintf("[MD5 Mismatch] %s (%x -> %x)", fname, memfi.Md5sum, md5sum), nil
				}
			}
		}
	}
	return "", nil
}

// Compare files in the DirTree to the files on disk. Adds new files found on
// disk to DirTree and replace Files with changed attributes. Report in case a
// file exists with the exact same attributes but different checksum.
func (d *DirTree) Compare() error {
	filepath.Walk(d.Root, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if fi.Mode().IsRegular() {
			msg, err := d.compareFile(path)
			if err != nil {
				return (fmt.Errorf("Error comparing file: %s: %q", path, err))
			}
			if msg != "" {
				fmt.Println(msg)
			}
		}
		return nil
	})
	return nil
}

// Write a JSON representation of the current DirTree struct
// to the specified io.Writer.
func (d *DirTree) Save(writer io.Writer) error {
	zwriter := gzip.NewWriter(writer)
	defer zwriter.Close()

	enc := json.NewEncoder(zwriter)
	err := enc.Encode(d)
	if err != nil {
		return err
	}
	return nil
}

// Load a JSON representation of DirTree from the specified io.Reader.
func (d *DirTree) Load(reader io.Reader) error {
	zreader, err := gzip.NewReader(reader)
	if err != nil {
		return (fmt.Errorf("Error uncompressing state DB: %q", err))
	}
	defer zreader.Close()

	dec := json.NewDecoder(zreader)
	err = dec.Decode(d)
	if err != nil {
		return err
	}
	return nil
}

// Calculate the md5sum of a file
func md5sum(fname string) ([]byte, error) {
	fd, err := os.Open(fname)
	if err != nil {
		return []byte{}, err
	}
	defer fd.Close()

	hd := md5.New()
	_, err = io.Copy(hd, fd)
	if err != nil {
		return []byte{}, err
	}
	return hd.Sum(nil), nil
}
<commit_msg>Lint & Comment fixes.<commit_after>// This file is part of bitrot, a bitrot scrubber in Go
// http://github.com/marcopaganini/bitrot
// (C) 2014 by Marco Paganini <paganini@paganini.net>

package main

import (
	"compress/gzip"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"
)

// FileInfo holds file metadata about each file in the filesystem.
type FileInfo struct {
	Size   int64
	Mode   os.FileMode
	Mtime  time.Time
	Md5sum []byte
}

// DirTree contains the metadata information about all files in a given
// mountpoint.
type DirTree struct {
	Root      string
	ExcludeRe []*string
	Files     map[string]*FileInfo
}

// NewDirTree creates a new DirTree struct.
func NewDirTree(root string, excludeRe []*string) *DirTree {
	dt := &DirTree{
		Root:      root,
		ExcludeRe: excludeRe,
	}
	dt.Files = make(map[string]*FileInfo)
	return dt
}

func (d *DirTree) compareFile(fname string) (string, error) {
	var (
		memfi  *FileInfo
		exists bool
	)

	osinfo, err := os.Stat(fname)
	if err != nil {
		return "", err
	}

	md5sum, err := md5sum(fname)
	if err != nil {
		return "", err
	}

	if memfi, exists = d.Files[fname]; !exists {
		// If medatada changed, replace entry silently
		memfi = &FileInfo{
			Size:   osinfo.Size(),
			Mode:   osinfo.Mode(),
			Mtime:  osinfo.ModTime(),
			Md5sum: md5sum,
		}
		d.Files[fname] = memfi
		Log.Verbosef(1, "[New] %s (%x)\n", fname, md5sum)
	} else {
		if osinfo.ModTime() != memfi.Mtime || osinfo.Mode() != memfi.Mode || osinfo.Size() != memfi.Size {
			// Exists: If medatada changed, replace entry silently
			Log.Verbosef(1, "[Metadata changes] %s (%x)\n", fname, md5sum)
			memfi.Size = osinfo.Size()
			memfi.Mode = osinfo.Mode()
			memfi.Mtime = osinfo.ModTime()
			memfi.Md5sum = md5sum
			d.Files[fname] = memfi
		} else {
			Log.Verbosef(1, "[No metadata changes] %s (%x)", fname, md5sum)
			// Exists: No metadata changes. Report md5 differences
			for k := range md5sum {
				if memfi.Md5sum[k] != md5sum[k] {
					return fmt.Sprintf("[MD5 Mismatch] %s (%x -> %x)", fname, memfi.Md5sum, md5sum), nil
				}
			}
		}
	}
	return "", nil
}

// Compare compares all files in the DirTree to the files on disk. Adds new
// files found on disk to DirTree and replace Files with changed attributes.
// Report in case a file exists with the exact same attributes but different
// checksum.
func (d *DirTree) Compare() error {
	filepath.Walk(d.Root, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if fi.Mode().IsRegular() {
			msg, err := d.compareFile(path)
			if err != nil {
				return (fmt.Errorf("Error comparing file: %s: %q", path, err))
			}
			if msg != "" {
				fmt.Println(msg)
			}
		}
		return nil
	})
	return nil
}

// Save writes a JSON representation of the current DirTree struct to the
// specified io.Writer.
func (d *DirTree) Save(writer io.Writer) error {
	zwriter := gzip.NewWriter(writer)
	defer zwriter.Close()

	enc := json.NewEncoder(zwriter)
	err := enc.Encode(d)
	if err != nil {
		return err
	}
	return nil
}

// Load a JSON representation of DirTree from the specified io.Reader.
func (d *DirTree) Load(reader io.Reader) error {
	zreader, err := gzip.NewReader(reader)
	if err != nil {
		return (fmt.Errorf("Error uncompressing state DB: %q", err))
	}
	defer zreader.Close()

	dec := json.NewDecoder(zreader)
	err = dec.Decode(d)
	if err != nil {
		return err
	}
	return nil
}

// Calculate the md5sum of a file
func md5sum(fname string) ([]byte, error) {
	fd, err := os.Open(fname)
	if err != nil {
		return []byte{}, err
	}
	defer fd.Close()

	hd := md5.New()
	_, err = io.Copy(hd, fd)
	if err != nil {
		return []byte{}, err
	}
	return hd.Sum(nil), nil
}
<|endoftext|>


<commit_before><commit_msg>allow messages > 10KB<commit_after><|endoftext|>


<commit_before>package ercaggregate

import (
	"errors"
	"sync"

	"github.com/TIBCOSoftware/flogo-lib/core/activity"
	"github.com/TIBCOSoftware/flogo-lib/core/data"
	"github.com/TIBCOSoftware/flogo-lib/logger"
	"github.com/ecarlier-tibco/flogo/activity/ercaggregate/aggregator"
)

// activityLogger is the default logger for the Aggregate Activity
var activityLogger = logger.GetLogger("activity-tibco-aggregate")

const (
	ivFunction   = "functions"
	ivWindowSize = "windowSize"
	ivValue      = "value"

	ovResult = "result"
	ovReport = "report"
)

func init() {
	activityLogger.SetLogLevel(logger.InfoLevel)
}

// AggregateActivity is an Activity that is used to Aggregate a message to the console
// inputs : {function, windowSize, autoRest, value}
// outputs: {result, report}
type AggregateActivity struct {
	metadata *activity.Metadata
	mutex    *sync.RWMutex

	// aggregators stateful map of aggregators
	aggregators map[string]aggregator.Aggregator
}

// NewActivity creates a new AppActivity
func NewActivity(metadata *activity.Metadata) activity.Activity {
	return &AggregateActivity{metadata: metadata, aggregators: make(map[string]aggregator.Aggregator), mutex: &sync.RWMutex{}}
}

// Metadata returns the activity's metadata
func (a *AggregateActivity) Metadata() *activity.Metadata {
	return a.metadata
}

// Eval implements api.Activity.Eval - Aggregates the Message
func (a *AggregateActivity) Eval(context activity.Context) (done bool, err error) {

	aggregatorKey := context.ActivityHost().Name() + ":" + context.Name()
	aggrNamesList, _ := context.GetInput(ivFunction).([]string)
	resultsList := make(map[string]float64)
	reportsList := make(map[string]bool)

	for _, aggrName := range aggrNamesList {

		a.mutex.RLock()
		//get aggregator for activity, assumes flow & task names are unique
		aggr, ok := a.aggregators[aggregatorKey+":"+aggrName]

		a.mutex.RUnlock()

		//if window not create for this flow, create it

		if !ok {

			//go doesn't have lock upgrades or try, so do same check again

			a.mutex.Lock()
			aggr, ok = a.aggregators[aggregatorKey+":"+aggrName]

			if !ok {
				windowSize, _ := context.GetInput(ivWindowSize).(int)

				factory := aggregator.GetFactory(aggrName)

				if factory == nil {
					return false, errors.New("Unknown aggregator: " + aggrName)
				}

				aggr = factory(windowSize)
				a.aggregators[aggregatorKey+":"+aggrName] = aggr

				activityLogger.Debug("Aggregator created for ", aggregatorKey)
			}

			a.mutex.Unlock()
		}

		value, ok := context.GetInput(ivValue).(float64)

		if !ok {
			value, _ = data.CoerceToNumber(context.GetInput(ivValue))
		}

		reportsList[aggrName], resultsList[aggrName] = aggr.Add(value)
	}

	context.SetOutput(ovReport, reportsList)
	context.SetOutput(ovResult, resultsList)

	return true, nil
}
<commit_msg>added logs<commit_after>package ercaggregate

import (
	"errors"
	"sync"

	"github.com/TIBCOSoftware/flogo-lib/core/activity"
	"github.com/TIBCOSoftware/flogo-lib/core/data"
	"github.com/TIBCOSoftware/flogo-lib/logger"
	"github.com/ecarlier-tibco/flogo/activity/ercaggregate/aggregator"
)

// activityLogger is the default logger for the Aggregate Activity
var activityLogger = logger.GetLogger("activity-tibco-aggregate")

const (
	ivFunction   = "functions"
	ivWindowSize = "windowSize"
	ivValue      = "value"

	ovResult = "result"
	ovReport = "report"
)

func init() {
	activityLogger.SetLogLevel(logger.InfoLevel)
}

// AggregateActivity is an Activity that is used to Aggregate a message to the console
// inputs : {function, windowSize, autoRest, value}
// outputs: {result, report}
type AggregateActivity struct {
	metadata *activity.Metadata
	mutex    *sync.RWMutex

	// aggregators stateful map of aggregators
	aggregators map[string]aggregator.Aggregator
}

// NewActivity creates a new AppActivity
func NewActivity(metadata *activity.Metadata) activity.Activity {
	return &AggregateActivity{metadata: metadata, aggregators: make(map[string]aggregator.Aggregator), mutex: &sync.RWMutex{}}
}

// Metadata returns the activity's metadata
func (a *AggregateActivity) Metadata() *activity.Metadata {
	return a.metadata
}

// Eval implements api.Activity.Eval - Aggregates the Message
func (a *AggregateActivity) Eval(context activity.Context) (done bool, err error) {

	aggregatorKey := context.ActivityHost().Name() + ":" + context.Name()
	aggrNamesList, _ := context.GetInput(ivFunction).([]string)
	resultsList := make(map[string]float64)
	reportsList := make(map[string]bool)

	for _, aggrName := range aggrNamesList {

		a.mutex.RLock()
		//get aggregator for activity, assumes flow & task names are unique
		aggr, ok := a.aggregators[aggregatorKey+":"+aggrName]

		a.mutex.RUnlock()

		//if window not create for this flow, create it

		if !ok {

			//go doesn't have lock upgrades or try, so do same check again

			a.mutex.Lock()
			aggr, ok = a.aggregators[aggregatorKey+":"+aggrName]

			if !ok {
				windowSize, _ := context.GetInput(ivWindowSize).(int)

				factory := aggregator.GetFactory(aggrName)

				if factory == nil {
					return false, errors.New("Unknown aggregator: " + aggrName)
				}

				aggr = factory(windowSize)
				a.aggregators[aggregatorKey+":"+aggrName] = aggr

				activityLogger.Debug("Aggregator created for ", aggregatorKey)
			}

			a.mutex.Unlock()
		}

		value, ok := context.GetInput(ivValue).(float64)

		if !ok {
			value, _ = data.CoerceToNumber(context.GetInput(ivValue))
		}

		reportsList[aggrName], resultsList[aggrName] = aggr.Add(value)
		activityLogger.Infof("RESULTS :\n!%v!\n!%v!", reportsList, resultsList)
	}

	context.SetOutput(ovReport, reportsList)
	context.SetOutput(ovResult, resultsList)
	activityLogger.Infof("END RESULTS :\n!%v!\n!%v!", reportsList, resultsList)

	return true, nil
}
<|endoftext|>