<commit_before>package codegen

import (
	"log"
	"strings"

	"github.com/jkomoros/boardgame"
)

//allValidTypes is an enumeration of all types in order.
var allValidTypes []propertyType

//typeNamesForFieldInfo is where we store the values to power
//fieldInfoForTypeName
var typeNamesToFieldInfo map[string]fieldInfo

//highestProperty is the highest enum in the PropertyType enum.
const highestProperty = boardgame.TypeTimer

func init() {
	//Only need space for highestProperty because we skip TypeIllegal.
	allValidTypes = make([]propertyType, highestProperty)
	//We skip TypeIllegal
	for i := 0; i < int(highestProperty); i++ {
		allValidTypes[i] = propertyType{boardgame.PropertyType(i + 1)}
	}
	typeNamesToFieldInfo = buildTypeNameMap()
}

func buildTypeNameMap() map[string]fieldInfo {
	result := make(map[string]fieldInfo)
	for _, t := range allValidTypes {

		//Non-interface types have the same type string for ImmutableGoType and
		//MutableGoType, and we want the ultimate record to say "mutable". By
		//setting the immutable one first, we'll simply overwrite it just after
		//this with the mutable record if theyr'e the same.
		result[t.ImmutableGoType()] = fieldInfo{
			Type:    t,
			Mutable: false,
		}

		result[t.MutableGoType()] = fieldInfo{
			Type:    t,
			Mutable: true,
		}

		for _, name := range t.ImmutableSubTypes() {
			result[name] = fieldInfo{
				Type:    t,
				Mutable: false,
				SubType: strings.TrimPrefix(name, t.TypePackagePrefix()),
			}
		}

		for _, name := range t.MutableSubTypes() {
			result[name] = fieldInfo{
				Type:    t,
				Mutable: true,
				SubType: strings.TrimPrefix(name, t.TypePackagePrefix()),
			}
		}
	}
	return result
}

//fieldInfo returns a field info for the given field type name (as reported by
//model, which notably does not include "[]" prefix for slices), as well as
//whether it's a slice.
func fieldInfoForTypeName(fieldTypeName string, isSlice bool) fieldInfo {
	key := fieldTypeName
	if isSlice {
		key = "[]" + key
	}
	return typeNamesToFieldInfo[key]
}

//propertyType is a simple wrapper around boardgame.PropertyType that extends it
//with methods that are only useful for codegen and thus don't belong in the
//main package. We can't just alias the type in this package because we need the
//methods. Note one oddity, to compare what type it is, you have to compare
//t.PropertyType == boardgame.TypeInt. Luckily the typechecker will complain if
//you do it wrong.
type propertyType struct {
	boardgame.PropertyType
}

//TypePackagePrefix returns a string representing the package prefix of the go
//type that is represented by this property type. For example, "boardgame." for
//TypeStack, and "" for TypeInt. Using strings.TrimPrefix() with this prefix
//applied to the return value of for example ImmutableGoType and others will
//strip away the package qualifier, if it exists. Most useful for the codegen
//package.
func (t propertyType) TypePackagePrefix() string {
	//Strip away any slices so we have fewer conditions to test for
	base := t.BaseType()
	switch base {
	case boardgame.TypePlayerIndex, boardgame.TypeStack, boardgame.TypeBoard, boardgame.TypeTimer:
		return "boardgame."
	case boardgame.TypeEnum:
		return "enum."
	}
	return ""
}

//ImmutableGoType emits strings like 'bool', 'boardgame.PlayerIndex'. It
//represents the type of this property for the immutable/getter contexts. Most
//useful for codegen package.
func (t propertyType) ImmutableGoType() string {

	if t.IsSlice() {
		return "[]" + propertyType{t.BaseType()}.ImmutableGoType()
	}

	switch t.PropertyType {
	case boardgame.TypeBool:
		return "bool"
	case boardgame.TypeInt:
		return "int"
	case boardgame.TypeString:
		return "string"
	case boardgame.TypePlayerIndex:
		return "boardgame.PlayerIndex"
	case boardgame.TypeEnum:
		return "enum.ImmutableVal"
	case boardgame.TypeStack:
		return "boardgame.ImmutableStack"
	case boardgame.TypeBoard:
		return "boardgame.ImmutableBoard"
	case boardgame.TypeTimer:
		return "boardgame.ImmutableTimer"
	default:
		return ""
	}
}

//MutableGoType emits a string representing the golang type for the property
//when in mutable/setter contexts, e.g 'int', 'boardgame.Stack'. Most useful for
//the codegen package.
func (t propertyType) MutableGoType() string {
	return strings.Replace(t.ImmutableGoType(), "Immutable", "", -1)
}

//Key returns the part of the PropertyReader method signature for this type. For
//example, "Bool" for TypeBool, "Timer" for "TypeTimer". Most useful for the
//codegen package.
func (t propertyType) Key() string {
	return strings.TrimPrefix(t.String(), "Type")
}

//ImmutableSubTypeConverters will return the names of hte converters to convert
//to the various valid subtypes, e.g. "ImmutableSizedStack" for TypeStack.
func (t propertyType) ImmutableSubTypeConverters() []string {
	switch t.PropertyType {
	case boardgame.TypeStack:
		return []string{"MergedStack", "ImmutableSizedStack"}
	case boardgame.TypeEnum:
		return []string{"ImmutableTreeVal", "ImmutableRangeVal"}
	}
	return nil
}

//MutableSubTypeConverters will return the names of hte converters to convert
//to the various valid subtypes, e.g. "ImmutableSizedStack" for TypeStack.
func (t propertyType) MutableSubTypeConverters() []string {
	switch t.PropertyType {
	case boardgame.TypeStack:
		return []string{"SizedStack"}
	case boardgame.TypeEnum:
		return []string{"TreeVal", "RangeVal"}
	}
	return nil
}

//ImmutableSubTypes returns the sub type fully qualitfied type strings for
//sub-types, e.g. "enum.ImmutableRangeVal" for boardgame.TypeEnum.
func (t propertyType) ImmutableSubTypes() []string {
	//Since the convertesr by convention are literally just the name of the type
	//(minus the package qualifier) we can just strip the package name.
	prefix := t.TypePackagePrefix()
	var result []string
	for _, item := range t.ImmutableSubTypeConverters() {
		result = append(result, strings.TrimPrefix(item, prefix))
	}
	return result
}

//MutableSubTypes returns the sub type fully qualitfied type strings for
//sub-types, e.g. "enum.ImmutableRangeVal" for boardgame.TypeEnum.
func (t propertyType) MutableSubTypes() []string {
	//Since the convertesr by convention are literally just the name of the type
	//(minus the package qualifier) we can just strip the package name.
	prefix := t.TypePackagePrefix()
	var result []string
	for _, item := range t.MutableSubTypeConverters() {
		result = append(result, strings.TrimPrefix(item, prefix))
	}
	return result
}

//ZeroValue returns the string representing the zeroValue for this type, e.g.
//"0" for TypeInt and "[]boardgame.PlayerIndex{}" for TypePlayerIndexSlice. Most
//useful for codgen package.
func (t propertyType) ZeroValue() string {

	switch t.PropertyType {
	case boardgame.TypeBool:
		return "false"
	case boardgame.TypeInt:
		return "0"
	case boardgame.TypeString:
		return "\"\""
	case boardgame.TypePlayerIndex:
		return "0"
	case boardgame.TypeIllegal:
		return ""
	}

	if t.IsSlice() {
		return t.ImmutableGoType() + "{}"
	}
	if t.IsInterface() {
		return "nil"
	}

	log.Println("Unexpected type for ZeroValue")
	return ""

}
<commit_msg>Whoops, MutableSubtypes and ImmutableSubTypes were stripping a prefix instead of adding it. Part of #746.<commit_after>package codegen

import (
	"log"
	"strings"

	"github.com/jkomoros/boardgame"
)

//allValidTypes is an enumeration of all types in order.
var allValidTypes []propertyType

//typeNamesForFieldInfo is where we store the values to power
//fieldInfoForTypeName
var typeNamesToFieldInfo map[string]fieldInfo

//highestProperty is the highest enum in the PropertyType enum.
const highestProperty = boardgame.TypeTimer

func init() {
	//Only need space for highestProperty because we skip TypeIllegal.
	allValidTypes = make([]propertyType, highestProperty)
	//We skip TypeIllegal
	for i := 0; i < int(highestProperty); i++ {
		allValidTypes[i] = propertyType{boardgame.PropertyType(i + 1)}
	}
	typeNamesToFieldInfo = buildTypeNameMap()
}

func buildTypeNameMap() map[string]fieldInfo {
	result := make(map[string]fieldInfo)
	for _, t := range allValidTypes {

		//Non-interface types have the same type string for ImmutableGoType and
		//MutableGoType, and we want the ultimate record to say "mutable". By
		//setting the immutable one first, we'll simply overwrite it just after
		//this with the mutable record if theyr'e the same.
		result[t.ImmutableGoType()] = fieldInfo{
			Type:    t,
			Mutable: false,
		}

		result[t.MutableGoType()] = fieldInfo{
			Type:    t,
			Mutable: true,
		}

		for _, name := range t.ImmutableSubTypes() {
			result[name] = fieldInfo{
				Type:    t,
				Mutable: false,
				SubType: strings.TrimPrefix(name, t.TypePackagePrefix()),
			}
		}

		for _, name := range t.MutableSubTypes() {
			result[name] = fieldInfo{
				Type:    t,
				Mutable: true,
				SubType: strings.TrimPrefix(name, t.TypePackagePrefix()),
			}
		}
	}
	return result
}

//fieldInfo returns a field info for the given field type name (as reported by
//model, which notably does not include "[]" prefix for slices), as well as
//whether it's a slice.
func fieldInfoForTypeName(fieldTypeName string, isSlice bool) fieldInfo {
	key := fieldTypeName
	if isSlice {
		key = "[]" + key
	}
	return typeNamesToFieldInfo[key]
}

//propertyType is a simple wrapper around boardgame.PropertyType that extends it
//with methods that are only useful for codegen and thus don't belong in the
//main package. We can't just alias the type in this package because we need the
//methods. Note one oddity, to compare what type it is, you have to compare
//t.PropertyType == boardgame.TypeInt. Luckily the typechecker will complain if
//you do it wrong.
type propertyType struct {
	boardgame.PropertyType
}

//TypePackagePrefix returns a string representing the package prefix of the go
//type that is represented by this property type. For example, "boardgame." for
//TypeStack, and "" for TypeInt. Using strings.TrimPrefix() with this prefix
//applied to the return value of for example ImmutableGoType and others will
//strip away the package qualifier, if it exists. Most useful for the codegen
//package.
func (t propertyType) TypePackagePrefix() string {
	//Strip away any slices so we have fewer conditions to test for
	base := t.BaseType()
	switch base {
	case boardgame.TypePlayerIndex, boardgame.TypeStack, boardgame.TypeBoard, boardgame.TypeTimer:
		return "boardgame."
	case boardgame.TypeEnum:
		return "enum."
	}
	return ""
}

//ImmutableGoType emits strings like 'bool', 'boardgame.PlayerIndex'. It
//represents the type of this property for the immutable/getter contexts. Most
//useful for codegen package.
func (t propertyType) ImmutableGoType() string {

	if t.IsSlice() {
		return "[]" + propertyType{t.BaseType()}.ImmutableGoType()
	}

	switch t.PropertyType {
	case boardgame.TypeBool:
		return "bool"
	case boardgame.TypeInt:
		return "int"
	case boardgame.TypeString:
		return "string"
	case boardgame.TypePlayerIndex:
		return "boardgame.PlayerIndex"
	case boardgame.TypeEnum:
		return "enum.ImmutableVal"
	case boardgame.TypeStack:
		return "boardgame.ImmutableStack"
	case boardgame.TypeBoard:
		return "boardgame.ImmutableBoard"
	case boardgame.TypeTimer:
		return "boardgame.ImmutableTimer"
	default:
		return ""
	}
}

//MutableGoType emits a string representing the golang type for the property
//when in mutable/setter contexts, e.g 'int', 'boardgame.Stack'. Most useful for
//the codegen package.
func (t propertyType) MutableGoType() string {
	return strings.Replace(t.ImmutableGoType(), "Immutable", "", -1)
}

//Key returns the part of the PropertyReader method signature for this type. For
//example, "Bool" for TypeBool, "Timer" for "TypeTimer". Most useful for the
//codegen package.
func (t propertyType) Key() string {
	return strings.TrimPrefix(t.String(), "Type")
}

//ImmutableSubTypeConverters will return the names of hte converters to convert
//to the various valid subtypes, e.g. "ImmutableSizedStack" for TypeStack.
func (t propertyType) ImmutableSubTypeConverters() []string {
	switch t.PropertyType {
	case boardgame.TypeStack:
		return []string{"MergedStack", "ImmutableSizedStack"}
	case boardgame.TypeEnum:
		return []string{"ImmutableTreeVal", "ImmutableRangeVal"}
	}
	return nil
}

//MutableSubTypeConverters will return the names of hte converters to convert
//to the various valid subtypes, e.g. "ImmutableSizedStack" for TypeStack.
func (t propertyType) MutableSubTypeConverters() []string {
	switch t.PropertyType {
	case boardgame.TypeStack:
		return []string{"SizedStack"}
	case boardgame.TypeEnum:
		return []string{"TreeVal", "RangeVal"}
	}
	return nil
}

//ImmutableSubTypes returns the sub type fully qualitfied type strings for
//sub-types, e.g. "enum.ImmutableRangeVal" for boardgame.TypeEnum.
func (t propertyType) ImmutableSubTypes() []string {
	//Since the convertesr by convention are literally just the name of the type
	//(minus the package qualifier) we can just strip the package name.
	prefix := t.TypePackagePrefix()
	var result []string
	for _, item := range t.ImmutableSubTypeConverters() {
		result = append(result, prefix+item)
	}
	return result
}

//MutableSubTypes returns the sub type fully qualitfied type strings for
//sub-types, e.g. "enum.ImmutableRangeVal" for boardgame.TypeEnum.
func (t propertyType) MutableSubTypes() []string {
	//Since the convertesr by convention are literally just the name of the type
	//(minus the package qualifier) we can just strip the package name.
	prefix := t.TypePackagePrefix()
	var result []string
	for _, item := range t.MutableSubTypeConverters() {
		result = append(result, prefix+item)
	}
	return result
}

//ZeroValue returns the string representing the zeroValue for this type, e.g.
//"0" for TypeInt and "[]boardgame.PlayerIndex{}" for TypePlayerIndexSlice. Most
//useful for codgen package.
func (t propertyType) ZeroValue() string {

	switch t.PropertyType {
	case boardgame.TypeBool:
		return "false"
	case boardgame.TypeInt:
		return "0"
	case boardgame.TypeString:
		return "\"\""
	case boardgame.TypePlayerIndex:
		return "0"
	case boardgame.TypeIllegal:
		return ""
	}

	if t.IsSlice() {
		return t.ImmutableGoType() + "{}"
	}
	if t.IsInterface() {
		return "nil"
	}

	log.Println("Unexpected type for ZeroValue")
	return ""

}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	FileFlagName      = "file"
	FileFlagShorthand = "f"
)

// ConfigureFlagSet adds the standard set of WebPA flags to the supplied FlagSet.  Use of this function
// is optional, and necessary only if the standard flags should be supported.  However, this is highly desirable,
// as ConfigureViper can make use of the standard flags to tailor how configuration is loaded.
func ConfigureFlagSet(applicationName string, f *pflag.FlagSet) {
	f.StringP(FileFlagName, FileFlagShorthand, applicationName, "base name of the configuration file")
}

// ConfigureViper configures a Viper instances using the opinionated WebPA settings.  All WebPA servers should
// use this function.
//
// The flagSet is optional.  If supplied, it will be bound to the given Viper instance.  Additionally, if the
// flagSet has a FileFlagName flag, it will be used as the configuration name to hunt for instead of the
// application name.
func ConfigureViper(applicationName string, f *pflag.FlagSet, v *viper.Viper) error {
	v.AddConfigPath(fmt.Sprintf("/etc/%s", applicationName))
	v.AddConfigPath(fmt.Sprintf("$HOME/.%s", applicationName))
	v.AddConfigPath(".")

	v.SetEnvPrefix(applicationName)
	v.AutomaticEnv()

	v.SetDefault("name", applicationName)
	v.SetDefault("address", DefaultAddress)
	v.SetDefault("healthAddress", DefaultHealthAddress)
	v.SetDefault("healthLogInterval", DefaultHealthLogInterval)

	if f != nil {
		if fileFlag := f.Lookup(FileFlagName); fileFlag != nil {
			// use the command-line to specify the base name of the file to be searched for
			v.SetConfigName(fileFlag.Value.String())
		} else {
			v.SetConfigName(applicationName)
		}

		if err := v.BindPFlags(f); err != nil {
			return err
		}
	} else {
		v.SetConfigName(applicationName)
	}

	return nil
}

/*
Configure is a one-stop shopping function for reading in WebPA configuration.  Typical usage is:

    var (
      f = pflag.NewFlagSet()
      v = viper.New()
    )

    if err := server.Configure("petasos", os.Args, f, v); err != nil {
      // deal with the error, possibly just exiting
    }
*/
func Configure(applicationName string, arguments []string, f *pflag.FlagSet, v *viper.Viper) error {
	if f != nil {
		ConfigureFlagSet(applicationName, f)
		if err := f.Parse(arguments); err != nil {
			return err
		}
	}

	if err := ConfigureViper(applicationName, f, v); err != nil {
		return err
	}

	return nil
}
<commit_msg>Cleaned up the Viper API<commit_after>package server

import (
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/logging/golog"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	FileFlagName      = "file"
	FileFlagShorthand = "f"

	LogKey = "log"
)

// ConfigureFlagSet adds the standard set of WebPA flags to the supplied FlagSet.  Use of this function
// is optional, and necessary only if the standard flags should be supported.  However, this is highly desirable,
// as ConfigureViper can make use of the standard flags to tailor how configuration is loaded.
func ConfigureFlagSet(applicationName string, f *pflag.FlagSet) {
	f.StringP(FileFlagName, FileFlagShorthand, applicationName, "base name of the configuration file")
}

// ConfigureViper configures a Viper instances using the opinionated WebPA settings.  All WebPA servers should
// use this function.
//
// The flagSet is optional.  If supplied, it will be bound to the given Viper instance.  Additionally, if the
// flagSet has a FileFlagName flag, it will be used as the configuration name to hunt for instead of the
// application name.
func ConfigureViper(applicationName string, f *pflag.FlagSet, v *viper.Viper) error {
	v.AddConfigPath(fmt.Sprintf("/etc/%s", applicationName))
	v.AddConfigPath(fmt.Sprintf("$HOME/.%s", applicationName))
	v.AddConfigPath(".")

	v.SetEnvPrefix(applicationName)
	v.AutomaticEnv()

	v.SetDefault("name", applicationName)
	v.SetDefault("address", DefaultAddress)
	v.SetDefault("healthAddress", DefaultHealthAddress)
	v.SetDefault("healthLogInterval", DefaultHealthLogInterval)

	if f != nil {
		if fileFlag := f.Lookup(FileFlagName); fileFlag != nil {
			// use the command-line to specify the base name of the file to be searched for
			v.SetConfigName(fileFlag.Value.String())
		} else {
			v.SetConfigName(applicationName)
		}

		if err := v.BindPFlags(f); err != nil {
			return err
		}
	} else {
		v.SetConfigName(applicationName)
	}

	return nil
}

/*
Configure is a one-stop shopping function for reading in WebPA configuration.  Typical usage is:

    var (
      f = pflag.NewFlagSet()
      v = viper.New()
    )

    if err := server.Configure("petasos", os.Args, f, v); err != nil {
      // deal with the error, possibly just exiting
    }
*/
func Configure(applicationName string, arguments []string, f *pflag.FlagSet, v *viper.Viper) error {
	if f != nil {
		ConfigureFlagSet(applicationName, f)
		if err := f.Parse(arguments); err != nil {
			return err
		}
	}

	if err := ConfigureViper(applicationName, f, v); err != nil {
		return err
	}

	return nil
}

// NewWebPA creates a WebPA instance from a Viper configuration
func NewWebPA(v *viper.Viper) (webPA *WebPA, err error) {
	webPA = new(WebPA)
	err = v.Unmarshal(webPA)
	return
}

// NewLoggerFactory creates a LoggerFactory from a Viper configuration
func NewLoggerFactory(v *viper.Viper) (loggerFactory logging.LoggerFactory, err error) {
	loggerFactory = new(golog.LoggerFactory)
	err = v.Unmarshal(loggerFactory)
	return
}
<|endoftext|>


<commit_before>package service

import (
	"bufio"
	"cham/cham"
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"sync"
	"sync/atomic"
)

const (
	GATE_OPEN uint8 = iota
	GATE_KICK
)

var (
	bufioReaderPool sync.Pool
	bufioWriterPool sync.Pool
	GATES           map[cham.Address]*Gate
)

type Conf struct {
	address   string //127.0.0.1:8000
	maxclient uint32 // 0 -> no limit
}

type Gate struct {
	rwmutex   *sync.RWMutex
	source    cham.Address
	clinetnum uint32
	maxclient uint32
	quit      chan struct{}
	sessions  map[uint32]*Session
}

type Session struct {
	sessionid uint32
	conn      net.Conn
	brw       *bufio.ReadWriter
}

func (s *Session) Close() {
	putBufioReader(s.brw.Reader)
	putBufioWriter(s.brw.Writer)
	s.conn.Close()
}

func (s *Session) Write(data []byte) {
	head := make([]byte, 2)
	binary.BigEndian.PutUint16(head, uint16(len(data)))
	s.brw.Write(head)
	s.brw.Write(data)
	s.brw.Flush()
}

func NewConf(address string, maxclient uint32) *Conf {
	return &Conf{address, maxclient}
}

func newSession(sessionid uint32, conn net.Conn) *Session {
	br := newBufioReader(conn)
	bw := newBufioWriter(conn)
	return &Session{sessionid, conn, bufio.NewReadWriter(br, bw)}
}

func (s *Session) ReadFull(buf []byte) error {
	if _, err := io.ReadFull(s.brw, buf); err != nil {
		if e, ok := err.(net.Error); ok && !e.Temporary() {
			return e
		}
	}
	return nil
}

func newGate(source cham.Address) *Gate {
	gate := new(Gate)
	gate.rwmutex = new(sync.RWMutex)
	gate.source = source
	gate.clinetnum = 0
	gate.quit = make(chan struct{})
	gate.sessions = make(map[uint32]*Session)
	return gate
}

func newBufioReader(r io.Reader) *bufio.Reader {
	if v := bufioReaderPool.Get(); v != nil {
		br := v.(*bufio.Reader)
		br.Reset(r)
		return br
	}
	return bufio.NewReader(r)
}

func putBufioReader(r *bufio.Reader) {
	r.Reset(nil)
	bufioReaderPool.Put(r)
}

func newBufioWriter(w io.Writer) *bufio.Writer {
	if v := bufioWriterPool.Get(); v != nil {
		bw := v.(*bufio.Writer)
		bw.Reset(w)
		return bw
	}
	return bufio.NewWriter(w)
}

func putBufioWriter(w *bufio.Writer) {
	w.Reset(nil)
	bufioWriterPool.Put(w)
}

//gate listen
func (g *Gate) open(conf *Conf) bool {
	listen, err := net.Listen("tcp", conf.address)
	if err != nil {
		panic("gate open error:" + err.Error())
	}
	g.maxclient = conf.maxclient
	go g.start(listen)

	return true
}

func (g *Gate) close() {
	close(g.quit)
}

func (g *Gate) start(listen net.Listener) {
	defer listen.Close()
	var sessionId uint32 = 0
	for {
		select {
		case <-g.quit:
			return
		default:
			conn, err := listen.Accept()
			if err != nil {
				continue
			}
			if g.maxclient != 0 && g.clinetnum >= g.maxclient {
				conn.Close()
			}
			g.clinetnum++
			sid := atomic.AddUint32(&sessionId, 1)
			session := newSession(sid, conn)
			g.rwmutex.Lock()
			g.sessions[sid] = session
			g.rwmutex.Unlock()
			go g.serve(session)
		}
	}
}

// bigendian 2byte length+data
func (g *Gate) serve(session *Session) {
	head := make([]byte, 2)
	dest := g.source.GetService()
	for {
		if err := session.ReadFull(head); err != nil {
			g.closeSession(session)
			return
		}

		length := binary.BigEndian.Uint16(head)
		data := make([]byte, length, length)

		if err := session.ReadFull(data); err != nil {
			g.closeSession(session)
			return
		}
		msg := cham.NewMsg(0, 0, cham.PTYPE_CLIENT, cham.Ret(session.sessionid, data))
		dest.Push(msg)
	}
}

func (g *Gate) closeSession(s *Session) {
	g.rwmutex.Lock()
	delete(g.sessions, s.sessionid)
	g.rwmutex.Unlock()
	s.Close()
}

func (g *Gate) kick(sessionid uint32) {
	var session *Session
	var ok bool
	g.rwmutex.Lock()
	if session, ok = g.sessions[sessionid]; ok {
		delete(g.sessions, sessionid)
	}
	g.rwmutex.Unlock()
	if ok {
		session.Close()
	}
}

func (g *Gate) Write(sessionid uint32, data []byte) {
	g.rwmutex.RLock()
	session, ok := g.sessions[sessionid]
	g.rwmutex.RUnlock()
	if ok {
		session.Write(data)
	}
}

func GateResponseDispatch(service *cham.Service, session int32, source cham.Address, ptype uint8, args ...interface{}) []interface{} {
	sessionid := args[0].(uint32)
	data := args[1].([]byte)
	gate := GATES[source]
	gate.Write(sessionid, data)
	return cham.NORET
}

func GateDispatch(service *cham.Service, session int32, source cham.Address, ptype uint8, args ...interface{}) []interface{} {
	gate, ok := GATES[source]
	if !ok {
		service.RegisterProtocol(cham.PTYPE_RESPONSE, GateResponseDispatch)
		gate = newGate(source)
		GATES[source] = gate
	}

	cmd := args[0].(uint8)
	result := cham.NORET
	switch cmd {
	case GATE_OPEN:
		gate.open(args[1].(*Conf))
	case GATE_KICK:
		gate.kick(args[1].(uint32))
	}

	return result
}

//may multi gate
func init() {
	GATES = make(map[cham.Address]*Gate, 1)
}
<commit_msg>v0.0.2<commit_after>package service

import (
	"bufio"
	"cham/cham"
	"encoding/binary"
	// "fmt"
	"io"
	"net"
	"sync"
	"sync/atomic"
)

const (
	GATE_OPEN uint8 = iota
	GATE_KICK
)

var (
	bufioReaderPool sync.Pool
	bufioWriterPool sync.Pool
	GATES           map[cham.Address]*Gate
)

type Conf struct {
	address   string //127.0.0.1:8000
	maxclient uint32 // 0 -> no limit
}

type Gate struct {
	rwmutex   *sync.RWMutex
	source    cham.Address
	clinetnum uint32
	maxclient uint32
	quit      chan struct{}
	sessions  map[uint32]*Session
}

type Session struct {
	sessionid uint32
	conn      net.Conn
	brw       *bufio.ReadWriter
}

func (s *Session) Close() {
	putBufioReader(s.brw.Reader)
	putBufioWriter(s.brw.Writer)
	s.conn.Close()
}

func (s *Session) Write(data []byte) {
	head := make([]byte, 2)
	binary.BigEndian.PutUint16(head, uint16(len(data)))
	s.brw.Write(head)
	s.brw.Write(data)
	s.brw.Flush()
}

func NewConf(address string, maxclient uint32) *Conf {
	return &Conf{address, maxclient}
}

func newSession(sessionid uint32, conn net.Conn) *Session {
	br := newBufioReader(conn)
	bw := newBufioWriter(conn)
	return &Session{sessionid, conn, bufio.NewReadWriter(br, bw)}
}

func (s *Session) ReadFull(buf []byte) error {
	if _, err := io.ReadFull(s.brw, buf); err != nil {
		if e, ok := err.(net.Error); ok && !e.Temporary() {
			return e
		}
	}
	return nil
}

func newGate(source cham.Address) *Gate {
	gate := new(Gate)
	gate.rwmutex = new(sync.RWMutex)
	gate.source = source
	gate.clinetnum = 0
	gate.quit = make(chan struct{})
	gate.sessions = make(map[uint32]*Session)
	return gate
}

func newBufioReader(r io.Reader) *bufio.Reader {
	if v := bufioReaderPool.Get(); v != nil {
		br := v.(*bufio.Reader)
		br.Reset(r)
		return br
	}
	return bufio.NewReader(r)
}

func putBufioReader(r *bufio.Reader) {
	r.Reset(nil)
	bufioReaderPool.Put(r)
}

func newBufioWriter(w io.Writer) *bufio.Writer {
	if v := bufioWriterPool.Get(); v != nil {
		bw := v.(*bufio.Writer)
		bw.Reset(w)
		return bw
	}
	return bufio.NewWriter(w)
}

func putBufioWriter(w *bufio.Writer) {
	w.Reset(nil)
	bufioWriterPool.Put(w)
}

//gate listen
func (g *Gate) open(conf *Conf) bool {
	listen, err := net.Listen("tcp", conf.address)
	if err != nil {
		panic("gate open error:" + err.Error())
	}
	g.maxclient = conf.maxclient
	go g.start(listen)

	return true
}

func (g *Gate) close() {
	close(g.quit)
}

func (g *Gate) start(listen net.Listener) {
	defer listen.Close()
	var sessionId uint32 = 0
	for {
		select {
		case <-g.quit:
			return
		default:
			conn, err := listen.Accept()
			if err != nil {
				continue
			}
			if g.maxclient != 0 && g.clinetnum >= g.maxclient {
				conn.Close()
			}
			g.clinetnum++
			sid := atomic.AddUint32(&sessionId, 1)
			session := newSession(sid, conn)
			g.rwmutex.Lock()
			g.sessions[sid] = session
			g.rwmutex.Unlock()
			go g.serve(session)
		}
	}
}

// bigendian 2byte length+data
func (g *Gate) serve(session *Session) {
	head := make([]byte, 2)
	dest := g.source.GetService()
	for {
		if err := session.ReadFull(head); err != nil {
			g.closeSession(session)
			return
		}

		length := binary.BigEndian.Uint16(head)
		data := make([]byte, length, length)

		if err := session.ReadFull(data); err != nil {
			g.closeSession(session)
			return
		}
		msg := cham.NewMsg(0, 0, cham.PTYPE_CLIENT, cham.Ret(session.sessionid, data))
		dest.Push(msg)
	}
}

func (g *Gate) closeSession(s *Session) {
	g.rwmutex.Lock()
	delete(g.sessions, s.sessionid)
	g.rwmutex.Unlock()
	s.Close()
}

func (g *Gate) kick(sessionid uint32) {
	var session *Session
	var ok bool
	g.rwmutex.Lock()
	if session, ok = g.sessions[sessionid]; ok {
		delete(g.sessions, sessionid)
	}
	g.rwmutex.Unlock()
	if ok {
		session.Close()
	}
}

func (g *Gate) Write(sessionid uint32, data []byte) {
	g.rwmutex.RLock()
	session, ok := g.sessions[sessionid]
	g.rwmutex.RUnlock()
	if ok {
		session.Write(data)
	}
}

func GateResponseDispatch(service *cham.Service, session int32, source cham.Address, ptype uint8, args ...interface{}) []interface{} {
	sessionid := args[0].(uint32)
	data := args[1].([]byte)
	gate := GATES[source]
	gate.Write(sessionid, data)
	return cham.NORET
}

func GateDispatch(service *cham.Service, session int32, source cham.Address, ptype uint8, args ...interface{}) []interface{} {
	gate, ok := GATES[source]
	if !ok {
		service.RegisterProtocol(cham.PTYPE_RESPONSE, GateResponseDispatch)
		gate = newGate(source)
		GATES[source] = gate
	}

	cmd := args[0].(uint8)
	result := cham.NORET
	switch cmd {
	case GATE_OPEN:
		gate.open(args[1].(*Conf))
	case GATE_KICK:
		gate.kick(args[1].(uint32))
	}

	return result
}

//may multi gate
func init() {
	GATES = make(map[cham.Address]*Gate, 1)
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strings"

	"github.com/miekg/dns"
)

var (
	url = flag.String("url",
		"http://www.internic.net/domain/root.zone",
		"URL of the IANA root zone file. If empty, read from stdin")
	whois = flag.String("whois",
		"whois.iana.org",
		"Address of the root whois server to query")
	v = flag.Bool("v", false, "verbose output (to stderr)")
)

func main() {
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	flag.Parse()

	var input io.Reader = os.Stdin

	if *url != "" {
		if *v {
			fmt.Fprintf(os.Stderr, "Fetching %s\n", *url)
		}
		res, err := http.Get(*url)
		if err != nil {
			return err
		}
		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("Bad GET status for %s: %d", *url, res.Status)
		}
		input = res.Body
		defer res.Body.Close()
	}

	zoneMap := make(map[string]string)

	if *v {
		fmt.Fprintf(os.Stderr, "Parsing root.zone\n")
	}
	for token := range dns.ParseZone(input, "", "") {
		if token.Error != nil {
			return token.Error
		}
		header := token.RR.Header()
		if header.Rrtype != dns.TypeNS {
			continue
		}
		domain := strings.TrimSuffix(strings.ToLower(header.Name), ".")
		if domain == "" {
			continue
		}
		zoneMap[domain] = domain
	}

	zones := make([]string, 0, len(zoneMap))
	for zone, _ := range zoneMap {
		zones = append(zones, zone)
	}
	sort.Strings(zones)

	re := regexp.MustCompile("whois:\\s+([a-z0-9\\-\\.]+)")

	for _, zone := range zones {
		if *v {
			fmt.Fprintf(os.Stderr, "whois -h %s %s", *whois, zone)
		}

		res, err := querySocket(*whois, zone)
		if err != nil {
			return err
		}

		matches := re.FindStringSubmatch(res)
		if matches == nil {
			if *v {
				fmt.Fprintf(os.Stderr, "\n")
			}
			continue
		}
		if *v {
			fmt.Fprintf(os.Stderr, "\t\t# %s\n", matches[1])
		}
		zoneMap[zone] = matches[1]
	}

	return nil
}

func querySocket(addr, query string) (string, error) {
	if !strings.Contains(addr, ":") {
		addr = addr + ":43"
	}
	c, err := net.Dial("tcp", addr)
	if err != nil {
		return "", err
	}
	defer c.Close()
	if _, err = fmt.Fprint(c, query, "\r\n"); err != nil {
		return "", err
	}
	res, err := ioutil.ReadAll(c)
	if err != nil {
		return "", err
	}
	return string(res), nil
}
<commit_msg>Parallelize IANA whois lookups.<commit_after>// +build ignore

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/miekg/dns"
)

var (
	url = flag.String("url",
		"http://www.internic.net/domain/root.zone",
		"URL of the IANA root zone file. If empty, read from stdin")
	whois = flag.String("whois",
		"whois.iana.org",
		"Address of the root whois server to query")
	v = flag.Bool("v", false, "verbose output (to stderr)")
)

type ZoneWhois struct {
	zone  string
	whois string
}

func main() {
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	flag.Parse()

	var input io.Reader = os.Stdin

	if *url != "" {
		if *v {
			fmt.Fprintf(os.Stderr, "Fetching %s\n", *url)
		}
		res, err := http.Get(*url)
		if err != nil {
			return err
		}
		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("Bad GET status for %s: %d", *url, res.Status)
		}
		input = res.Body
		defer res.Body.Close()
	}

	zoneMap := make(map[string]string)

	if *v {
		fmt.Fprintf(os.Stderr, "Parsing root.zone\n")
	}
	for token := range dns.ParseZone(input, "", "") {
		if token.Error != nil {
			return token.Error
		}
		header := token.RR.Header()
		if header.Rrtype != dns.TypeNS {
			continue
		}
		domain := strings.TrimSuffix(strings.ToLower(header.Name), ".")
		if domain == "" {
			continue
		}
		zoneMap[domain] = domain
	}

	// Sort zones
	zones := make([]string, 0, len(zoneMap))
	for zone, _ := range zoneMap {
		zones = append(zones, zone)
	}
	sort.Strings(zones)

	// Get whois servers for each zone
	re := regexp.MustCompile("whois:\\s+([a-z0-9\\-\\.]+)")
	c := make(chan ZoneWhois, len(zones))

	if *v {
		fmt.Fprintf(os.Stderr, "Querying whois servers\n")
	}

	// Create 1 goroutine for each zone
	for i, zone := range zones {
		go func(zone string, i int) {
			time.Sleep(time.Duration(i * 100) * time.Millisecond) // Try not to hammer IANA

			res, err := querySocket(*whois, zone)
			if err != nil {
				c <- ZoneWhois{zone, ""}
				return
			}

			matches := re.FindStringSubmatch(res)
			if matches == nil {
				c <- ZoneWhois{zone, ""}
				return
			}
			c <- ZoneWhois{zone, matches[1]}
		}(zone, i)
	}

	// Collect from goroutines
	for i := 0; i < len(zones); i++ {
		select {
		case zw := <-c:
			if *v {
				fmt.Fprintf(os.Stderr, "whois -h %s %s\t\t%s\n", *whois, zw.zone, zw.whois)
			}

		}
	}

	return nil
}

func querySocket(addr, query string) (string, error) {
	if !strings.Contains(addr, ":") {
		addr = addr + ":43"
	}
	c, err := net.Dial("tcp", addr)
	if err != nil {
		return "", err
	}
	defer c.Close()
	if _, err = fmt.Fprint(c, query, "\r\n"); err != nil {
		return "", err
	}
	res, err := ioutil.ReadAll(c)
	if err != nil {
		return "", err
	}
	return string(res), nil
}
<|endoftext|>


<commit_before>package uniter

var NewSettings = newSettings
<commit_msg>Forgot the copyright header for export_test<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.
package uniter

var NewSettings = newSettings
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "sync"
    "os"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

const (
  // How many Keys AWS-S3-Api will return in a single request. Note that S3-API
  // has a hard limit of 1000
  // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
  S3_MAX_KEYS = 1000
  // How many S3-Result-Sets are buffered until the producer blocks
  CHAN_BUFFER_SIZE = 1000
  // How many worker processes should do actual sync from Source-Bucket to Destination-Bucket
  CHAN_UPLOAD_WORKER = 5
)

var wg_list sync.WaitGroup
var wg_upload sync.WaitGroup

// Finds every element in a S3-Bucket
// Is meant to run in parallel. Puts resultsets into channel *s3.ListObjectsOutput
// Closes this channel when done
// First parameter is an *s3.S3 Object (http://docs.aws.amazon.com/sdk-for-go/api/service/s3/S3.html)
// Second is the S3-Bucket-Name (Note not the S3-Path like s3://bucket_name)
// Third is the result channel of type *s3.ListObjectsOutput
func receive_s3_chunks(svc *s3.S3, bucket_name string, chan_s3_chunks chan *s3.ListObjectsOutput) {
  defer wg_list.Done()
  defer close(chan_s3_chunks)

  params := &s3.ListObjectsInput{
    Bucket:       aws.String(bucket_name), // Required
    MaxKeys:      aws.Int64(S3_MAX_KEYS),
  }

  err := svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, lastPage bool) bool {
    chan_s3_chunks <- page
    return !lastPage
  })

  if err != nil {
    // Print the error, cast err to awserr.Error to get the Code and
    // Message from an error.
    fmt.Println(err.Error())
    return
  }

  //fmt.Printf("%s done\n", bucket_name)
}

// Need to convert Chungs of Keys, returned from S3-API into lists
func extract_contents(chan_s3_chunks chan *s3.ListObjectsOutput, output_chan chan *s3.Object) {
  defer wg_list.Done()
  for {
    s3_list_chunk, ok := <- chan_s3_chunks
    if !ok {
      // Channel is closed bei producer
      break
    }

    for _, s3_key := range s3_list_chunk.Contents {
      //fmt.Printf("Key: %v\n", s3_key)
      output_chan <- s3_key
    }

    close(output_chan)

  }
}

// Detects difference between Source- and Destination-Bucket
// s3_contents_src: Input-Channel. Contains every element in Source-Bucket.
// s3_contents_dest: Input-Channel. Contains every element in Destination-Bucket
// s3_contents_4_upload: Output-Channel. Receives every Elemente from Source-Bucket
//                       that does not exist in Destination-Bucket. When done this
//                       Channel is closed.
func calculate_diff(s3_contents_src, s3_contents_dest, s3_contents_4_upload chan *s3.Object) {
  defer wg_list.Done()
  defer close(s3_contents_4_upload)
  var src, dest *s3.Object

  src = <- s3_contents_src
  dest = <- s3_contents_dest

  for {

    if src == nil && dest == nil {
      // Case A: no elements left in both Buckets
      break

    } else if src != nil && dest != nil {
      // Case B: got an element in both Buckets

      if *src.Key < *dest.Key {
        // src-element is ahead of destination-element. Src-element is missing in Destination-Bucket
        fmt.Printf("Adding %v for upload\n", *src.Key)
        s3_contents_4_upload <- src
        // Continue with next element
        src = <- s3_contents_src

      } else if *src.Key > *dest.Key {
        // src-element is beyond destination-element. Destination-element exists only on Destination-Bucket

        // Continue with next element
        dest = <- s3_contents_dest

      } else {  // src.Key == dest.Key
        // Src and Dest are equal
        // Continue with next element
        src = <- s3_contents_src
        // Continue with next element
        dest = <- s3_contents_dest

      }

    } else if dest == nil {
      // Case C: no element left in Destination-Bucket but in Source-Bucket
      // Continue with next element

      fmt.Printf("Adding %v for upload\n", *src.Key)
      s3_contents_4_upload <- src

      src = <- s3_contents_src

    } else if src == nil {
      // Case D: no elements left in Source-Bucket but in Destination-Bucket
      // Continue with next element
      dest = <- s3_contents_dest

    } else {
      // This means not Case A or not Case B or not Case C or not Case D
      fmt.Printf("This case shouldn't appear. However it did. Src: %v Dest: %v", src, dest)
      // TODO: raise an error. Stop execution
      break

    }
  }
}

// Syncronizes elements received from Channel s3_contents_4_upload with given Destination-Bucket
// This function is able to run in parralel with same Channel
func sync_s3_elements(svc *s3.S3, bucket_name_src, bucket_name_dest string, s3_contents_4_upload chan *s3.Object, worker_id int) {
  defer wg_upload.Done()
  var elem *s3.Object
  var ok bool

  for {
    elem, ok = <- s3_contents_4_upload

    if ok && elem != nil {

      fmt.Printf("Sync from Worker %v: %v\n", worker_id, *elem.Key)
      params := &s3.CopyObjectInput{
      	Bucket:                         aws.String(bucket_name_dest), // Required
      	CopySource:                     aws.String(bucket_name_src + "/" +  *elem.Key),  // Required
      	Key:                            aws.String(*elem.Key),        // Required
      }
      _, err := svc.CopyObject(params)

      if err != nil {
      	// Print the error, cast err to awserr.Error to get the Code and
      	// Message from an error.
      	fmt.Println(err.Error())
      	return
      }

      //fmt.Println(resp)

    } else {
      break
    }
  }
  fmt.Printf("Sync done\n")
}

func main() {

    bucket_name_src := os.Args[1]
    bucket_name_dest := os.Args[2]

    // Create an S3 service object in the "eu-west-1" region
    // Note that you can also configure your region globally by
    // exporting the AWS_REGION environment variable
    svc := s3.New(session.New(), &aws.Config{Region: aws.String("eu-west-1")})

    // Channel that contains a list of S3-Chunks
    chan_s3_chunks_src := make(chan *s3.ListObjectsOutput, CHAN_BUFFER_SIZE)
    // Channel that contains a list of S3-Chunks
    chan_s3_chunks_dest := make(chan *s3.ListObjectsOutput, CHAN_BUFFER_SIZE)
    // List that contains a list of S3-Keys
    s3_contents_src := make(chan *s3.Object, CHAN_BUFFER_SIZE)
    // List that contains a list of S3-Keys
    s3_contents_dest := make(chan *s3.Object, CHAN_BUFFER_SIZE)
    // List that contains a list of S3-Keys that are missing in Destination => need upload to Destination
    s3_contents_4_upload := make(chan *s3.Object, CHAN_UPLOAD_WORKER)

    // Task 1: Read Chunks from S3-Buckets.
    // Chunks are a bunch of S3-Keys that the Bucket contains. A S3-Bucket retuns
    // a maximum of 1.000 keys per Chunk. Both Buckets are read asyncroneuous.
    go receive_s3_chunks(svc, bucket_name_src, chan_s3_chunks_src)
    wg_list.Add(1)
    go receive_s3_chunks(svc, bucket_name_dest, chan_s3_chunks_dest)
    wg_list.Add(1)
    // When the first Chunks arrive we extract their keys and write them into a list.
    // Both Buckets have their own list of S3-Keys
    go extract_contents(chan_s3_chunks_src, s3_contents_src)
    wg_list.Add(1)
    go extract_contents(chan_s3_chunks_dest, s3_contents_dest)
    wg_list.Add(1)
    // When the first elements arrive in the list of S3-Keys we calculate the difference
    // between both Bucket. This job starts as soon as there are elements in one of the lists
    go calculate_diff(s3_contents_src, s3_contents_dest, s3_contents_4_upload)
    wg_list.Add(1)

    // Upload files that are missing on Destination-Bucket
    fmt.Printf("Worker-Count: %v\n", CHAN_UPLOAD_WORKER)
    for i := 0; i < CHAN_UPLOAD_WORKER; i++ {
      fmt.Printf("Starting Worker %v\n", i)
      wg_upload.Add(1)
      go sync_s3_elements(svc, bucket_name_src, bucket_name_dest, s3_contents_4_upload, i)
    }

    wg_list.Wait()
    wg_upload.Wait()

}
<commit_msg>Bugfix: closed Channel too early<commit_after>package main

import (
    "fmt"
    "sync"
    "os"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

const (
  // How many Keys AWS-S3-Api will return in a single request. Note that S3-API
  // has a hard limit of 1000
  // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
  S3_MAX_KEYS = 1000
  // How many S3-Result-Sets are buffered until the producer blocks
  CHAN_BUFFER_SIZE = 1000
  // How many worker processes should do actual sync from Source-Bucket to Destination-Bucket
  CHAN_UPLOAD_WORKER = 5
)

var wg_list sync.WaitGroup
var wg_upload sync.WaitGroup

// Finds every element in a S3-Bucket
// Is meant to run in parallel. Puts resultsets into channel *s3.ListObjectsOutput
// Closes this channel when done
// First parameter is an *s3.S3 Object (http://docs.aws.amazon.com/sdk-for-go/api/service/s3/S3.html)
// Second is the S3-Bucket-Name (Note not the S3-Path like s3://bucket_name)
// Third is the result channel of type *s3.ListObjectsOutput
func receive_s3_chunks(svc *s3.S3, bucket_name string, chan_s3_chunks chan *s3.ListObjectsOutput) {
  defer wg_list.Done()
  defer close(chan_s3_chunks)

  params := &s3.ListObjectsInput{
    Bucket:       aws.String(bucket_name), // Required
    MaxKeys:      aws.Int64(S3_MAX_KEYS),
  }

  err := svc.ListObjectsPages(params, func(page *s3.ListObjectsOutput, lastPage bool) bool {
    chan_s3_chunks <- page
    return !lastPage
  })

  if err != nil {
    // Print the error, cast err to awserr.Error to get the Code and
    // Message from an error.
    fmt.Println(err.Error())
    return
  }

  //fmt.Printf("%s done\n", bucket_name)
}

// Need to convert Chungs of Keys, returned from S3-API into lists
func extract_contents(chan_s3_chunks chan *s3.ListObjectsOutput, output_chan chan *s3.Object) {
  defer wg_list.Done()
  for {
    s3_list_chunk, ok := <- chan_s3_chunks
    if !ok {
      // Channel is closed bei producer
      break
    }

    for _, s3_key := range s3_list_chunk.Contents {
      //fmt.Printf("Key: %v\n", s3_key)
      output_chan <- s3_key
    }
  }

  close(output_chan)
}

// Detects difference between Source- and Destination-Bucket
// s3_contents_src: Input-Channel. Contains every element in Source-Bucket.
// s3_contents_dest: Input-Channel. Contains every element in Destination-Bucket
// s3_contents_4_upload: Output-Channel. Receives every Elemente from Source-Bucket
//                       that does not exist in Destination-Bucket. When done this
//                       Channel is closed.
func calculate_diff(s3_contents_src, s3_contents_dest, s3_contents_4_upload chan *s3.Object) {
  defer wg_list.Done()
  defer close(s3_contents_4_upload)
  var src, dest *s3.Object

  src = <- s3_contents_src
  dest = <- s3_contents_dest

  for {

    if src == nil && dest == nil {
      // Case A: no elements left in both Buckets
      break

    } else if src != nil && dest != nil {
      // Case B: got an element in both Buckets

      if *src.Key < *dest.Key {
        // src-element is ahead of destination-element. Src-element is missing in Destination-Bucket
        fmt.Printf("Adding %v for upload\n", *src.Key)
        s3_contents_4_upload <- src
        // Continue with next element
        src = <- s3_contents_src

      } else if *src.Key > *dest.Key {
        // src-element is beyond destination-element. Destination-element exists only on Destination-Bucket

        // Continue with next element
        dest = <- s3_contents_dest

      } else {  // src.Key == dest.Key
        // Src and Dest are equal
        // Continue with next element
        src = <- s3_contents_src
        // Continue with next element
        dest = <- s3_contents_dest

      }

    } else if dest == nil {
      // Case C: no element left in Destination-Bucket but in Source-Bucket
      // Continue with next element

      fmt.Printf("Adding %v for upload\n", *src.Key)
      s3_contents_4_upload <- src

      src = <- s3_contents_src

    } else if src == nil {
      // Case D: no elements left in Source-Bucket but in Destination-Bucket
      // Continue with next element
      dest = <- s3_contents_dest

    } else {
      // This means not Case A or not Case B or not Case C or not Case D
      fmt.Printf("This case shouldn't appear. However it did. Src: %v Dest: %v", src, dest)
      // TODO: raise an error. Stop execution
      break

    }
  }
}

// Syncronizes elements received from Channel s3_contents_4_upload with given Destination-Bucket
// This function is able to run in parralel with same Channel
func sync_s3_elements(svc *s3.S3, bucket_name_src, bucket_name_dest string, s3_contents_4_upload chan *s3.Object, worker_id int) {
  defer wg_upload.Done()
  var elem *s3.Object
  var ok bool

  for {
    elem, ok = <- s3_contents_4_upload

    if ok && elem != nil {

      fmt.Printf("Sync from Worker %v: %v\n", worker_id, *elem.Key)
      params := &s3.CopyObjectInput{
      	Bucket:                         aws.String(bucket_name_dest), // Required
      	CopySource:                     aws.String(bucket_name_src + "/" +  *elem.Key),  // Required
      	Key:                            aws.String(*elem.Key),        // Required
      }
      _, err := svc.CopyObject(params)

      if err != nil {
      	// Print the error, cast err to awserr.Error to get the Code and
      	// Message from an error.
      	fmt.Println(err.Error())
      	return
      }

      //fmt.Println(resp)

    } else {
      break
    }
  }
  fmt.Printf("Sync done\n")
}

func main() {

    bucket_name_src := os.Args[1]
    bucket_name_dest := os.Args[2]

    // Create an S3 service object in the "eu-west-1" region
    // Note that you can also configure your region globally by
    // exporting the AWS_REGION environment variable
    svc := s3.New(session.New(), &aws.Config{Region: aws.String("eu-west-1")})

    // Channel that contains a list of S3-Chunks
    chan_s3_chunks_src := make(chan *s3.ListObjectsOutput, CHAN_BUFFER_SIZE)
    // Channel that contains a list of S3-Chunks
    chan_s3_chunks_dest := make(chan *s3.ListObjectsOutput, CHAN_BUFFER_SIZE)
    // List that contains a list of S3-Keys
    s3_contents_src := make(chan *s3.Object, CHAN_BUFFER_SIZE)
    // List that contains a list of S3-Keys
    s3_contents_dest := make(chan *s3.Object, CHAN_BUFFER_SIZE)
    // List that contains a list of S3-Keys that are missing in Destination => need upload to Destination
    s3_contents_4_upload := make(chan *s3.Object, CHAN_UPLOAD_WORKER)

    // Task 1: Read Chunks from S3-Buckets.
    // Chunks are a bunch of S3-Keys that the Bucket contains. A S3-Bucket retuns
    // a maximum of 1.000 keys per Chunk. Both Buckets are read asyncroneuous.
    go receive_s3_chunks(svc, bucket_name_src, chan_s3_chunks_src)
    wg_list.Add(1)
    go receive_s3_chunks(svc, bucket_name_dest, chan_s3_chunks_dest)
    wg_list.Add(1)
    // When the first Chunks arrive we extract their keys and write them into a list.
    // Both Buckets have their own list of S3-Keys
    go extract_contents(chan_s3_chunks_src, s3_contents_src)
    wg_list.Add(1)
    go extract_contents(chan_s3_chunks_dest, s3_contents_dest)
    wg_list.Add(1)
    // When the first elements arrive in the list of S3-Keys we calculate the difference
    // between both Bucket. This job starts as soon as there are elements in one of the lists
    go calculate_diff(s3_contents_src, s3_contents_dest, s3_contents_4_upload)
    wg_list.Add(1)

    // Upload files that are missing on Destination-Bucket
    fmt.Printf("Worker-Count: %v\n", CHAN_UPLOAD_WORKER)
    for i := 0; i < CHAN_UPLOAD_WORKER; i++ {
      fmt.Printf("Starting Worker %v\n", i)
      wg_upload.Add(1)
      go sync_s3_elements(svc, bucket_name_src, bucket_name_dest, s3_contents_4_upload, i)
    }

    wg_list.Wait()
    wg_upload.Wait()

}
<|endoftext|>


<commit_before>package sansan

import (
	"fmt"
	"sync"
	"sync/atomic"
)

const heapsize = 30000

type heap []int32

type Machine struct {
	mem heap
	wg  *sync.WaitGroup
}

func newMachine() Machine {
	return Machine{
		mem: make(heap, heapsize),
		wg:  &sync.WaitGroup{},
	}
}

type state struct {
	pnt    int16
	atomic bool
}

func (m Machine) run(p Program, s *state) {
	for i := 0; i < len(p); i++ {
		switch ins := p[i]; ins {
		case '>':
			s.pnt++
			s.pnt = (s.pnt%heapsize + heapsize) % heapsize
		case '<':
			s.pnt--
			s.pnt = (s.pnt%heapsize + heapsize) % heapsize
		case '+':
			if s.atomic {
				atomic.AddInt32(&m.mem[s.pnt], 1)
			} else {
				m.mem[s.pnt]++
			}
		case '-':
			if s.atomic {
				atomic.AddInt32(&m.mem[s.pnt], -1)
			} else {
				m.mem[s.pnt]--
			}
		case '[':
			end := i + findClosing(p[i:])

			if (s.atomic && atomic.LoadInt32(&m.mem[s.pnt]) != 0) || (!s.atomic && m.mem[s.pnt] != 0) {
				m.run(p[i+1:end+1], s) // enter loop
			}

			i = end // goto end
		case ']':
			if (s.atomic && atomic.LoadInt32(&m.mem[s.pnt]) == 0) || (!s.atomic && m.mem[s.pnt] == 0) {
				return
			}
			i = -1

		case '{':
			end := i + findClosing(p[i:])

			m.wg.Add(1)
			go m.runThread(p[i+1:end+1], *s)

			i = end // continue parrent thread
		case '}':
			return // kill thread
		case '!':
			// toggle atomic operations on current thread
			s.atomic = s.atomic != true

		case '.':
			var v int32
			if s.atomic {
				v = atomic.LoadInt32(&m.mem[s.pnt])
			} else {
				v = m.mem[s.pnt]
			}

			fmt.Printf("%c", v)
		case ',':
			var n int32
			if _, err := fmt.Scanf("%d\n", &n); err != nil {
				panic(err)
			}

			if s.atomic {
				atomic.SwapInt32(&m.mem[s.pnt], n)
			} else {
				m.mem[s.pnt] = n
			}
		}
	}
}

// runThread runs the given program with a local copy of the
// heap pointer and decrements waitgroup when finished.
func (m Machine) runThread(p Program, s state) {
	defer m.wg.Done()
	m.run(p, &s)
}

func findClosing(prog []byte) int {
	braces := 0
	for i := 0; i < len(prog); i++ {
		switch prog[i] {
		case '[', '{':
			braces++
		case ']', '}':
			braces--
			if braces < 0 {
				panic("invalid program: unbalanced braces")
			}
			if braces == 0 {
				return i
			}
		}
	}
	panic("invalid program: could not find closing ']'")
}
<commit_msg>allow setting per machine io reader/writer<commit_after>package sansan

import (
	"fmt"
	"io"
	"os"
	"sync"
	"sync/atomic"
)

const heapsize = 30000

type heap []int32

type Machine struct {
	mem heap
	wg  *sync.WaitGroup

	stdin  io.Reader
	stdout io.Writer
}

func newMachine() Machine {
	return Machine{
		mem: make(heap, heapsize),
		wg:  &sync.WaitGroup{},

		stdin:  os.Stdin,
		stdout: os.Stdout,
	}
}

type state struct {
	pnt    int16
	atomic bool
}

func (m Machine) run(p Program, s *state) {
	for i := 0; i < len(p); i++ {
		switch ins := p[i]; ins {
		case '>':
			s.pnt++
			s.pnt = (s.pnt%heapsize + heapsize) % heapsize
		case '<':
			s.pnt--
			s.pnt = (s.pnt%heapsize + heapsize) % heapsize
		case '+':
			if s.atomic {
				atomic.AddInt32(&m.mem[s.pnt], 1)
			} else {
				m.mem[s.pnt]++
			}
		case '-':
			if s.atomic {
				atomic.AddInt32(&m.mem[s.pnt], -1)
			} else {
				m.mem[s.pnt]--
			}
		case '[':
			end := i + findClosing(p[i:])

			if (s.atomic && atomic.LoadInt32(&m.mem[s.pnt]) != 0) || (!s.atomic && m.mem[s.pnt] != 0) {
				m.run(p[i+1:end+1], s) // enter loop
			}

			i = end // goto end
		case ']':
			if (s.atomic && atomic.LoadInt32(&m.mem[s.pnt]) == 0) || (!s.atomic && m.mem[s.pnt] == 0) {
				return
			}
			i = -1

		case '{':
			end := i + findClosing(p[i:])

			m.wg.Add(1)
			go m.runThread(p[i+1:end+1], *s)

			i = end // continue parrent thread
		case '}':
			return // kill thread
		case '!':
			// toggle atomic operations on current thread
			s.atomic = s.atomic != true

		case '.':
			var v int32
			if s.atomic {
				v = atomic.LoadInt32(&m.mem[s.pnt])
			} else {
				v = m.mem[s.pnt]
			}

			fmt.Fprintf(m.stdout, "%c", v)
		case ',':
			var n int32
			if _, err := fmt.Fscanf(m.stdin, "%d\n", &n); err != nil {
				panic(err)
			}

			if s.atomic {
				atomic.SwapInt32(&m.mem[s.pnt], n)
			} else {
				m.mem[s.pnt] = n
			}
		}
	}
}

// runThread runs the given program with a local copy of the
// heap pointer and decrements waitgroup when finished.
func (m Machine) runThread(p Program, s state) {
	defer m.wg.Done()
	m.run(p, &s)
}

func (m *Machine) SetInput(r io.Reader) {
	m.stdin = r
}

func (m *Machine) SetOutput(w io.Writer) {
	m.stdout = w
}

func findClosing(prog []byte) int {
	braces := 0
	for i := 0; i < len(prog); i++ {
		switch prog[i] {
		case '[', '{':
			braces++
		case ']', '}':
			braces--
			if braces < 0 {
				panic("invalid program: unbalanced braces")
			}
			if braces == 0 {
				return i
			}
		}
	}
	panic("invalid program: could not find closing ']'")
}
<|endoftext|>


<commit_before>package sbloom

import "hash"

//Filter represents a scalable bloom filter.
type Filter struct {
	bh hash.Hash64
	fs []*filter
	hs []sHash
}

//NewFilter returns a scalable bloom filter with a false negative probaility
//less than 1/2**k. It starts with a default size of 1024 bits per k. If you
//know you will use more/less than that, use NewSizedFilter for a better hint.
//It uses the provided hash to operate.
func NewFilter(h hash.Hash64, k int) *Filter {
	return NewSizedFilter(h, k, 10)
}

//NewSizedFilter returns a scalable bloom filter with a false negative
//probability less than 1/2**k. It start with a default size of 2**log bits per
//k. It uses the provided hash to operate.
func NewSizedFilter(h hash.Hash64, k int, log uint) *Filter {
	f := new(Filter)
	f.bh = h
	f.addNewFilter(log, k+1) //need to start at 1/2 to converge up
	return f
}

//Add adds the slice of bytes to the filter so that it will always return true
//when looked up.
func (f *Filter) Add(p []byte) {
	last := f.fs[len(f.fs)-1]
	last.Add(p, f.hs[:last.K])
	if last.Left == 0 {
		f.addNewFilter(last.Log+1, last.K+1)
	}
}

//addNewFilter creates a new filter of the given size and number of bins, making
//sure we have enough hash functions for the bins.
func (f *Filter) addNewFilter(log uint, k int) {
	//make sure we have up to k hashes
	for len(f.hs) < k {
		f.hs = append(f.hs, newsHash(f.bh))
	}

	//add the new bloom filter
	f.fs = append(f.fs, newFilter(log, k))
}

//Lookup looks for the set of bytes in the bloom filter. If it returns false,
//the set of bytes has definitely not been seen before.
func (f *Filter) Lookup(p []byte) bool {
	for _, subfil := range f.fs {
		if subfil.Lookup(p, f.hs[:subfil.K]) {
			return true
		}
	}
	return false
}
<commit_msg>change docs to indicate that it's a false positive<commit_after>package sbloom

import "hash"

//Filter represents a scalable bloom filter.
type Filter struct {
	bh hash.Hash64
	fs []*filter
	hs []sHash
}

//NewFilter returns a scalable bloom filter with a false positive probaility
//less than 1/2**k. It starts with a default size of 1024 bits per k. If you
//know you will use more/less than that, use NewSizedFilter for a better hint.
//It uses the provided hash to operate.
func NewFilter(h hash.Hash64, k int) *Filter {
	return NewSizedFilter(h, k, 10)
}

//NewSizedFilter returns a scalable bloom filter with a false positive
//probability less than 1/2**k. It start with a default size of 2**log bits per
//k. It uses the provided hash to operate.
func NewSizedFilter(h hash.Hash64, k int, log uint) *Filter {
	f := new(Filter)
	f.bh = h
	f.addNewFilter(log, k+1) //need to start at 1/2 to converge up
	return f
}

//Add adds the slice of bytes to the filter so that it will always return true
//when looked up.
func (f *Filter) Add(p []byte) {
	last := f.fs[len(f.fs)-1]
	last.Add(p, f.hs[:last.K])
	if last.Left == 0 {
		f.addNewFilter(last.Log+1, last.K+1)
	}
}

//addNewFilter creates a new filter of the given size and number of bins, making
//sure we have enough hash functions for the bins.
func (f *Filter) addNewFilter(log uint, k int) {
	//make sure we have up to k hashes
	for len(f.hs) < k {
		f.hs = append(f.hs, newsHash(f.bh))
	}

	//add the new bloom filter
	f.fs = append(f.fs, newFilter(log, k))
}

//Lookup looks for the set of bytes in the bloom filter. If it returns false,
//the set of bytes has definitely not been seen before.
func (f *Filter) Lookup(p []byte) bool {
	for _, subfil := range f.fs {
		if subfil.Lookup(p, f.hs[:subfil.K]) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/i18n"
)

// Lists
const (
	listFormatCSV   = "csv"
	listFormatJSON  = "json"
	listFormatTable = "table"
	listFormatYAML  = "yaml"
)

// Progress tracking
type ProgressRenderer struct {
	Format string

	maxLength int
	wait      time.Time
}

func (p *ProgressRenderer) Done(msg string) {
	if msg != "" {
		msg += "\n"
	}

	if len(msg) > p.maxLength {
		p.maxLength = len(msg)
	} else {
		fmt.Printf("\r%s", strings.Repeat(" ", p.maxLength))
	}

	fmt.Print("\r")
	fmt.Print(msg)
}

func (p *ProgressRenderer) Update(status string) {
	timeout := p.wait.Sub(time.Now())
	if timeout.Seconds() > 0 {
		time.Sleep(timeout)
	}

	msg := "%s"
	if p.Format != "" {
		msg = p.Format
	}

	msg = fmt.Sprintf("\r"+msg, status)

	if len(msg) > p.maxLength {
		p.maxLength = len(msg)
	} else {
		fmt.Printf("\r%s", strings.Repeat(" ", p.maxLength))
	}

	fmt.Print(msg)
}

func (p *ProgressRenderer) Warn(status string, timeout time.Duration) {
	p.wait = time.Now().Add(timeout)
	msg := fmt.Sprintf("\r%s", status)

	if len(msg) > p.maxLength {
		p.maxLength = len(msg)
	} else {
		fmt.Printf("\r%s", strings.Repeat(" ", p.maxLength))
	}

	fmt.Print(msg)
}

func (p *ProgressRenderer) UpdateProgress(progress lxd.ProgressData) {
	p.Update(progress.Text)
}

func (p *ProgressRenderer) UpdateOp(op api.Operation) {
	if op.Metadata == nil {
		return
	}

	for _, key := range []string{"fs_progress", "download_progress"} {
		value, ok := op.Metadata[key]
		if ok {
			p.Update(value.(string))
			break
		}
	}
}

type StringList [][]string

func (a StringList) Len() int {
	return len(a)
}

func (a StringList) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a StringList) Less(i, j int) bool {
	x := 0
	for x = range a[i] {
		if a[i][x] != a[j][x] {
			break
		}
	}

	if a[i][x] == "" {
		return false
	}

	if a[j][x] == "" {
		return true
	}

	return a[i][x] < a[j][x]
}

// Container name sorting
type byName [][]string

func (a byName) Len() int {
	return len(a)
}

func (a byName) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a byName) Less(i, j int) bool {
	if a[i][0] == "" {
		return false
	}

	if a[j][0] == "" {
		return true
	}

	return a[i][0] < a[j][0]
}

// Storage volume sorting
type byNameAndType [][]string

func (a byNameAndType) Len() int {
	return len(a)
}

func (a byNameAndType) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a byNameAndType) Less(i, j int) bool {
	if a[i][0] != a[j][0] {
		return a[i][0] < a[j][0]
	}

	if a[i][1] == "" {
		return false
	}

	if a[j][1] == "" {
		return true
	}

	return a[i][1] < a[j][1]
}

// Batch operations
type batchResult struct {
	err  error
	name string
}

func runBatch(names []string, action func(name string) error) []batchResult {
	chResult := make(chan batchResult, len(names))

	for _, name := range names {
		go func(name string) {
			chResult <- batchResult{action(name), name}
		}(name)
	}

	results := []batchResult{}
	for range names {
		results = append(results, <-chResult)
	}

	return results
}

// summaryLine returns the first line of the help text. Conventionally, this
// should be a one-line command summary, potentially followed by a longer
// explanation.
func summaryLine(usage string) string {
	for _, line := range strings.Split(usage, "\n") {
		if strings.HasPrefix(line, "Usage:") {
			continue
		}

		if len(line) == 0 {
			continue
		}

		return strings.TrimSuffix(line, ".")
	}

	return i18n.G("Missing summary.")
}

// Used to return a user friendly error
func getLocalErr(err error) error {
	t, ok := err.(*url.Error)
	if !ok {
		return nil
	}

	u, ok := t.Err.(*net.OpError)
	if !ok {
		return nil
	}

	if u.Op == "dial" && u.Net == "unix" {
		var lxdErr error

		sysErr, ok := u.Err.(*os.SyscallError)
		if ok {
			lxdErr = sysErr.Err
		} else {
			// syscall.Errno may be returned on some systems, e.g. CentOS
			lxdErr, ok = u.Err.(syscall.Errno)
			if !ok {
				return nil
			}
		}

		switch lxdErr {
		case syscall.ENOENT, syscall.ECONNREFUSED, syscall.EACCES:
			return lxdErr
		}
	}

	return nil
}

// Add a device to a container
func containerDeviceAdd(client lxd.ContainerServer, name string, devName string, dev map[string]string) error {
	// Get the container entry
	container, etag, err := client.GetContainer(name)
	if err != nil {
		return err
	}

	// Check if the device already exists
	_, ok := container.Devices[devName]
	if ok {
		return fmt.Errorf(i18n.G("Device already exists: %s"), devName)
	}

	container.Devices[devName] = dev

	op, err := client.UpdateContainer(name, container.Writable(), etag)
	if err != nil {
		return err
	}

	return op.Wait()
}

// Add a device to a profile
func profileDeviceAdd(client lxd.ContainerServer, name string, devName string, dev map[string]string) error {
	// Get the profile entry
	profile, profileEtag, err := client.GetProfile(name)
	if err != nil {
		return err
	}

	// Check if the device already exists
	_, ok := profile.Devices[devName]
	if ok {
		return fmt.Errorf(i18n.G("Device already exists: %s"), devName)
	}

	// Add the device to the container
	profile.Devices[devName] = dev

	err = client.UpdateProfile(name, profile.Writable(), profileEtag)
	if err != nil {
		return err
	}

	return nil
}

// Wait for an operation and cancel it on SIGINT/SIGTERM
func cancelableWait(op *lxd.RemoteOperation, progress *ProgressRenderer) error {
	// Signal handling
	chSignal := make(chan os.Signal)
	signal.Notify(chSignal, os.Interrupt)

	// Operation handling
	chOperation := make(chan error)
	go func() {
		chOperation <- op.Wait()
		close(chOperation)
	}()

	count := 0
	for {
		select {
		case err := <-chOperation:
			return err
		case <-chSignal:
			err := op.CancelTarget()
			if err == nil {
				return fmt.Errorf(i18n.G("Remote operation canceled by user"))
			} else {
				count++

				if count == 3 {
					return fmt.Errorf(i18n.G("User signaled us three times, exiting. The remote operation will keep running."))
				}

				if progress != nil {
					progress.Warn(fmt.Sprintf(i18n.G("%v (interrupt two more times to force)"), err), time.Second*5)
				}
			}
		}
	}
}
<commit_msg>lxc/utils: Avoid potential progress race condition<commit_after>package main

import (
	"fmt"
	"net"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/i18n"
)

// Lists
const (
	listFormatCSV   = "csv"
	listFormatJSON  = "json"
	listFormatTable = "table"
	listFormatYAML  = "yaml"
)

// Progress tracking
type ProgressRenderer struct {
	Format string

	maxLength int
	wait      time.Time
	done      bool
}

func (p *ProgressRenderer) Done(msg string) {
	p.done = true

	if msg != "" {
		msg += "\n"
	}

	if len(msg) > p.maxLength {
		p.maxLength = len(msg)
	} else {
		fmt.Printf("\r%s", strings.Repeat(" ", p.maxLength))
	}

	fmt.Print("\r")
	fmt.Print(msg)
}

func (p *ProgressRenderer) Update(status string) {
	if p.done {
		return
	}

	timeout := p.wait.Sub(time.Now())
	if timeout.Seconds() > 0 {
		time.Sleep(timeout)
	}

	msg := "%s"
	if p.Format != "" {
		msg = p.Format
	}

	msg = fmt.Sprintf("\r"+msg, status)

	if len(msg) > p.maxLength {
		p.maxLength = len(msg)
	} else {
		fmt.Printf("\r%s", strings.Repeat(" ", p.maxLength))
	}

	fmt.Print(msg)
}

func (p *ProgressRenderer) Warn(status string, timeout time.Duration) {
	p.wait = time.Now().Add(timeout)
	msg := fmt.Sprintf("\r%s", status)

	if len(msg) > p.maxLength {
		p.maxLength = len(msg)
	} else {
		fmt.Printf("\r%s", strings.Repeat(" ", p.maxLength))
	}

	fmt.Print(msg)
}

func (p *ProgressRenderer) UpdateProgress(progress lxd.ProgressData) {
	p.Update(progress.Text)
}

func (p *ProgressRenderer) UpdateOp(op api.Operation) {
	if op.Metadata == nil {
		return
	}

	for _, key := range []string{"fs_progress", "download_progress"} {
		value, ok := op.Metadata[key]
		if ok {
			p.Update(value.(string))
			break
		}
	}
}

type StringList [][]string

func (a StringList) Len() int {
	return len(a)
}

func (a StringList) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a StringList) Less(i, j int) bool {
	x := 0
	for x = range a[i] {
		if a[i][x] != a[j][x] {
			break
		}
	}

	if a[i][x] == "" {
		return false
	}

	if a[j][x] == "" {
		return true
	}

	return a[i][x] < a[j][x]
}

// Container name sorting
type byName [][]string

func (a byName) Len() int {
	return len(a)
}

func (a byName) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a byName) Less(i, j int) bool {
	if a[i][0] == "" {
		return false
	}

	if a[j][0] == "" {
		return true
	}

	return a[i][0] < a[j][0]
}

// Storage volume sorting
type byNameAndType [][]string

func (a byNameAndType) Len() int {
	return len(a)
}

func (a byNameAndType) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a byNameAndType) Less(i, j int) bool {
	if a[i][0] != a[j][0] {
		return a[i][0] < a[j][0]
	}

	if a[i][1] == "" {
		return false
	}

	if a[j][1] == "" {
		return true
	}

	return a[i][1] < a[j][1]
}

// Batch operations
type batchResult struct {
	err  error
	name string
}

func runBatch(names []string, action func(name string) error) []batchResult {
	chResult := make(chan batchResult, len(names))

	for _, name := range names {
		go func(name string) {
			chResult <- batchResult{action(name), name}
		}(name)
	}

	results := []batchResult{}
	for range names {
		results = append(results, <-chResult)
	}

	return results
}

// summaryLine returns the first line of the help text. Conventionally, this
// should be a one-line command summary, potentially followed by a longer
// explanation.
func summaryLine(usage string) string {
	for _, line := range strings.Split(usage, "\n") {
		if strings.HasPrefix(line, "Usage:") {
			continue
		}

		if len(line) == 0 {
			continue
		}

		return strings.TrimSuffix(line, ".")
	}

	return i18n.G("Missing summary.")
}

// Used to return a user friendly error
func getLocalErr(err error) error {
	t, ok := err.(*url.Error)
	if !ok {
		return nil
	}

	u, ok := t.Err.(*net.OpError)
	if !ok {
		return nil
	}

	if u.Op == "dial" && u.Net == "unix" {
		var lxdErr error

		sysErr, ok := u.Err.(*os.SyscallError)
		if ok {
			lxdErr = sysErr.Err
		} else {
			// syscall.Errno may be returned on some systems, e.g. CentOS
			lxdErr, ok = u.Err.(syscall.Errno)
			if !ok {
				return nil
			}
		}

		switch lxdErr {
		case syscall.ENOENT, syscall.ECONNREFUSED, syscall.EACCES:
			return lxdErr
		}
	}

	return nil
}

// Add a device to a container
func containerDeviceAdd(client lxd.ContainerServer, name string, devName string, dev map[string]string) error {
	// Get the container entry
	container, etag, err := client.GetContainer(name)
	if err != nil {
		return err
	}

	// Check if the device already exists
	_, ok := container.Devices[devName]
	if ok {
		return fmt.Errorf(i18n.G("Device already exists: %s"), devName)
	}

	container.Devices[devName] = dev

	op, err := client.UpdateContainer(name, container.Writable(), etag)
	if err != nil {
		return err
	}

	return op.Wait()
}

// Add a device to a profile
func profileDeviceAdd(client lxd.ContainerServer, name string, devName string, dev map[string]string) error {
	// Get the profile entry
	profile, profileEtag, err := client.GetProfile(name)
	if err != nil {
		return err
	}

	// Check if the device already exists
	_, ok := profile.Devices[devName]
	if ok {
		return fmt.Errorf(i18n.G("Device already exists: %s"), devName)
	}

	// Add the device to the container
	profile.Devices[devName] = dev

	err = client.UpdateProfile(name, profile.Writable(), profileEtag)
	if err != nil {
		return err
	}

	return nil
}

// Wait for an operation and cancel it on SIGINT/SIGTERM
func cancelableWait(op *lxd.RemoteOperation, progress *ProgressRenderer) error {
	// Signal handling
	chSignal := make(chan os.Signal)
	signal.Notify(chSignal, os.Interrupt)

	// Operation handling
	chOperation := make(chan error)
	go func() {
		chOperation <- op.Wait()
		close(chOperation)
	}()

	count := 0
	for {
		select {
		case err := <-chOperation:
			return err
		case <-chSignal:
			err := op.CancelTarget()
			if err == nil {
				return fmt.Errorf(i18n.G("Remote operation canceled by user"))
			} else {
				count++

				if count == 3 {
					return fmt.Errorf(i18n.G("User signaled us three times, exiting. The remote operation will keep running."))
				}

				if progress != nil {
					progress.Warn(fmt.Sprintf(i18n.G("%v (interrupt two more times to force)"), err), time.Second*5)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select `long`,`access_count` from `url` where short=?", shorturl).Scan(&longurl, &access_count)

	// TODO: update accessed and access_count

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		log.Printf("%q -> %q", shorturl, longurl)
		http.Redirect(w, r, longurl, http.StatusFound)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@("+*host+")/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>Update access information on successful query<commit_after>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select `long`,`access_count` from `url` where short=?", shorturl).Scan(&longurl, &access_count)

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		log.Printf("%q -> %q", shorturl, longurl)
		access_count++
		access_time := time.Now().UTC()
		_, err := dbConn.Exec("UPDATE `url` SET `access_count`=?,`accessed`=? WHERE short=?", access_count, access_time, shorturl)
		if err != nil {
			log.Printf("Failed to update access information: %v\n", err)
		}
		http.Redirect(w, r, longurl, http.StatusFound)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@("+*host+")/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()
	// TODO: prepare statements

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select `long`,`access_count` from `url` where short=?", shorturl).Scan(&longurl, &access_count)

	// TODO: update accessed and access_count

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		log.Print("%q -> %q", shorturl, longurl)
		http.Redirect(w, r, longurl, 200)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@("+*host+")/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>Redirect, not OK<commit_after>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select `long`,`access_count` from `url` where short=?", shorturl).Scan(&longurl, &access_count)

	// TODO: update accessed and access_count

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		log.Printf("%q -> %q", shorturl, longurl)
		http.Redirect(w, r, longurl, http.StatusFound)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@("+*host+")/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>


<commit_before>package aero

import (
	"context"
	"fmt"
	"mime"
	"net/http"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/aerogo/csp"

	"github.com/aerogo/http/client"
	performance "github.com/aerogo/linter-performance"
	"github.com/aerogo/session"
	memstore "github.com/aerogo/session-store-memory"
	"github.com/fatih/color"
	"github.com/julienschmidt/httprouter"
)

// Application represents a single web service.
type Application struct {
	Config                *Configuration
	Sessions              session.Manager
	Security              ApplicationSecurity
	Servers               [2]*http.Server
	Linters               []Linter
	Router                *httprouter.Router
	ContentSecurityPolicy *csp.ContentSecurityPolicy

	routeTests     map[string][]string
	start          time.Time
	rewrite        func(*RewriteContext)
	middleware     []Middleware
	pushConditions []func(*Context) bool
	onStart        []func()
	onShutdown     []func()
	onPush         []func(*Context)

	routes struct {
		GET  []string
		POST []string
	}
}

// New creates a new application.
func New() *Application {
	app := new(Application)
	app.start = time.Now()
	app.routeTests = make(map[string][]string)
	app.Router = httprouter.New()

	// Default linters
	app.Linters = []Linter{
		performance.New(),
	}

	// Default CSP
	app.ContentSecurityPolicy = csp.New()
	app.ContentSecurityPolicy.SetMap(map[string]string{
		"default-src":  "'none'",
		"img-src":      "https:",
		"media-src":    "https:",
		"script-src":   "'self'",
		"style-src":    "'self'",
		"font-src":     "https:",
		"manifest-src": "'self'",
		"connect-src":  "https: wss:",
		"worker-src":   "'self'",
		"frame-src":    "https:",
		"base-uri":     "'self'",
		"form-action":  "'self'",
	})

	// Configuration
	app.Config = new(Configuration)
	app.Config.Reset()
	app.Load()

	// Default session store: Memory
	app.Sessions.Store = memstore.New()

	// Default style
	// app.SetStyle("")

	// Set mime type for WebP because Go standard library doesn't include it
	mime.AddExtensionType(".webp", "image/webp")

	return app
}

// Get registers your function to be called when a certain GET path has been requested.
func (app *Application) Get(path string, handle Handle) {
	app.routes.GET = append(app.routes.GET, path)
	app.Router.GET(path, app.createRouteHandler(path, handle))
}

// Post registers your function to be called when a certain POST path has been requested.
func (app *Application) Post(path string, handle Handle) {
	app.routes.POST = append(app.routes.POST, path)
	app.Router.POST(path, app.createRouteHandler(path, handle))
}

// createRouteHandler creates a handler function for httprouter.
func (app *Application) createRouteHandler(path string, handle Handle) httprouter.Handle {
	return func(response http.ResponseWriter, request *http.Request, params httprouter.Params) {
		// Create context.
		ctx := Context{
			App:        app,
			StatusCode: http.StatusOK,
			request:    request,
			response:   response,
			params:     params,
		}

		// The last part of the call chain will send the actual response.
		lastPartOfCallChain := func() {
			data := handle(&ctx)
			ctx.respond(data)
		}

		// Declare the type of generateNext so that we can define it recursively in the next part.
		var generateNext func(index int) func()

		// Create a function that returns a bound function next()
		// which can be used as the 2nd parameter in the call chain.
		generateNext = func(index int) func() {
			if index == len(app.middleware) {
				return lastPartOfCallChain
			}

			return func() {
				app.middleware[index](&ctx, generateNext(index+1))
			}
		}

		generateNext(0)()
	}
}

// Run starts your application.
func (app *Application) Run() {
	app.TestManifest()
	app.TestRoutes()
	app.Listen()
	app.Wait()
	app.Shutdown()
}

// Use adds middleware to your middleware chain.
func (app *Application) Use(middlewares ...Middleware) {
	app.middleware = append(app.middleware, middlewares...)
}

// Load loads the application configuration from config.json.
func (app *Application) Load() {
	config, err := LoadConfig("config.json")

	if err != nil {
		// Ignore missing config file, we can perfectly run without one
		return
	}

	app.Config = config
}

// Listen starts the server.
func (app *Application) Listen() {
	if app.Security.Key != "" && app.Security.Certificate != "" {
		go func() {
			app.serveHTTPS(":" + strconv.Itoa(app.Config.Ports.HTTPS))
		}()

		fmt.Println("Server running on:", color.GreenString("https://localhost:"+strconv.Itoa(app.Config.Ports.HTTPS)))
	} else {
		fmt.Println("Server running on:", color.GreenString("http://localhost:"+strconv.Itoa(app.Config.Ports.HTTP)))
	}

	go func() {
		app.serveHTTP(":" + strconv.Itoa(app.Config.Ports.HTTP))
	}()
}

// Wait will make the process wait until it is killed.
func (app *Application) Wait() {
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)

	for _, callback := range app.onStart {
		callback()
	}

	<-stop
}

// Shutdown will gracefully shut down the server.
func (app *Application) Shutdown() {
	for _, server := range app.Servers {
		if server == nil {
			continue
		}

		// Add a timeout to the server shutdown
		ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
		defer cancel()

		// Shut down server
		err := server.Shutdown(ctx)

		if err != nil {
			fmt.Println(err)
		}
	}

	for _, callback := range app.onShutdown {
		callback()
	}
}

// OnStart registers a callback to be executed on server start.
func (app *Application) OnStart(callback func()) {
	app.onStart = append(app.onStart, callback)
}

// OnEnd registers a callback to be executed on server shutdown.
func (app *Application) OnEnd(callback func()) {
	app.onShutdown = append(app.onShutdown, callback)
}

// OnPush registers a callback to be executed when an HTTP/2 push happens.
func (app *Application) OnPush(callback func(*Context)) {
	app.onPush = append(app.onPush, callback)
}

// AddPushCondition registers a callback to be executed when an HTTP/2 push happens.
func (app *Application) AddPushCondition(test func(*Context) bool) {
	app.pushConditions = append(app.pushConditions, test)
}

// Rewrite sets the URL rewrite function.
func (app *Application) Rewrite(rewrite func(*RewriteContext)) {
	app.rewrite = rewrite
}

// Finalize post-processes the HTML (currently a noop).
func (app *Application) Finalize(html string) string {
	return html
}

// StartTime returns the time the application started.
func (app *Application) StartTime() time.Time {
	return app.start
}

// Handler returns the request handler used by the application.
func (app *Application) Handler() http.Handler {
	router := app.Router
	rewrite := app.rewrite

	if rewrite != nil {
		return &rewriteHandler{
			rewrite: rewrite,
			router:  router,
		}
	}

	return router
}

// createServer creates an http server instance.
func (app *Application) createServer(address string) *http.Server {
	return &http.Server{
		Addr:              address,
		Handler:           app.Handler(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      180 * time.Second,
		IdleTimeout:       120 * time.Second,
		TLSConfig:         createTLSConfig(),
	}
}

// serveHTTP serves requests from the given listener.
func (app *Application) serveHTTP(address string) {
	server := app.createServer(address)
	app.Servers[0] = server

	// This will block the calling goroutine until the server shuts down.
	serveError := server.ListenAndServe()

	if serveError != nil && !strings.Contains(serveError.Error(), "closed") {
		panic(serveError)
	}
}

// serveHTTPS serves requests from the given listener.
func (app *Application) serveHTTPS(address string) {
	server := app.createServer(address)
	app.Servers[1] = server

	// This will block the calling goroutine until the server shuts down.
	serveError := server.ListenAndServeTLS(app.Security.Certificate, app.Security.Key)

	if serveError != nil && !strings.Contains(serveError.Error(), "closed") {
		panic(serveError)
	}
}

// Test tests the given URI paths when the application starts.
func (app *Application) Test(route string, paths []string) {
	app.routeTests[route] = paths
}

// TestManifest tests your application's manifest.
func (app *Application) TestManifest() {
	manifest := app.Config.Manifest

	// Warn about short name length (Google Lighthouse)
	// https://developer.chrome.com/apps/manifest/name#short_name
	if len(manifest.ShortName) >= 12 {
		color.Yellow("The short name of your application should have less than 12 characters")
	}
}

// TestRoutes tests your application's routes.
func (app *Application) TestRoutes() {
	fmt.Println(strings.Repeat("-", 80))

	go func() {
		sort.Strings(app.routes.GET)

		for _, route := range app.routes.GET {
			// Skip ajax routes
			if strings.HasPrefix(route, "/_") {
				continue
			}

			// Check if the user defined test routes for the given route
			testRoutes, exists := app.routeTests[route]

			if exists {
				for _, testRoute := range testRoutes {
					app.TestRoute(route, testRoute)
				}

				continue
			}

			// Skip routes with parameters and display a warning to indicate it needs a test route
			if strings.Contains(route, ":") {
				color.Yellow(route)
				continue
			}

			// Test the static route without parameters
			app.TestRoute(route, route)
		}

		// json, _ := Post("https://html5.validator.nu/?out=json").Header("Content-Type", "text/html; charset=utf-8").Header("Content-Encoding", "gzip").Body(body).Send()
		// fmt.Println(json)
	}()
}

// TestRoute tests the given route.
func (app *Application) TestRoute(route string, uri string) {
	for _, linter := range app.Linters {
		linter.Begin(route, uri)
	}

	response, _ := client.Get("http://localhost:" + strconv.Itoa(app.Config.Ports.HTTP) + uri).End()

	for _, linter := range app.Linters {
		linter.End(route, uri, response)
	}
}
<commit_msg>Now passing data race test<commit_after>package aero

import (
	"context"
	"fmt"
	"mime"
	"net/http"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/aerogo/csp"

	"github.com/aerogo/http/client"
	performance "github.com/aerogo/linter-performance"
	"github.com/aerogo/session"
	memstore "github.com/aerogo/session-store-memory"
	"github.com/fatih/color"
	"github.com/julienschmidt/httprouter"
)

// Application represents a single web service.
type Application struct {
	Config                *Configuration
	Sessions              session.Manager
	Security              ApplicationSecurity
	Linters               []Linter
	Router                *httprouter.Router
	ContentSecurityPolicy *csp.ContentSecurityPolicy

	servers        [2]*http.Server
	serversMutex   sync.Mutex
	routeTests     map[string][]string
	start          time.Time
	rewrite        func(*RewriteContext)
	middleware     []Middleware
	pushConditions []func(*Context) bool
	onStart        []func()
	onShutdown     []func()
	onPush         []func(*Context)

	routes struct {
		GET  []string
		POST []string
	}
}

// New creates a new application.
func New() *Application {
	app := new(Application)
	app.start = time.Now()
	app.routeTests = make(map[string][]string)
	app.Router = httprouter.New()

	// Default linters
	app.Linters = []Linter{
		performance.New(),
	}

	// Default CSP
	app.ContentSecurityPolicy = csp.New()
	app.ContentSecurityPolicy.SetMap(map[string]string{
		"default-src":  "'none'",
		"img-src":      "https:",
		"media-src":    "https:",
		"script-src":   "'self'",
		"style-src":    "'self'",
		"font-src":     "https:",
		"manifest-src": "'self'",
		"connect-src":  "https: wss:",
		"worker-src":   "'self'",
		"frame-src":    "https:",
		"base-uri":     "'self'",
		"form-action":  "'self'",
	})

	// Configuration
	app.Config = new(Configuration)
	app.Config.Reset()
	app.Load()

	// Default session store: Memory
	app.Sessions.Store = memstore.New()

	// Default style
	// app.SetStyle("")

	// Set mime type for WebP because Go standard library doesn't include it
	mime.AddExtensionType(".webp", "image/webp")

	return app
}

// Get registers your function to be called when a certain GET path has been requested.
func (app *Application) Get(path string, handle Handle) {
	app.routes.GET = append(app.routes.GET, path)
	app.Router.GET(path, app.createRouteHandler(path, handle))
}

// Post registers your function to be called when a certain POST path has been requested.
func (app *Application) Post(path string, handle Handle) {
	app.routes.POST = append(app.routes.POST, path)
	app.Router.POST(path, app.createRouteHandler(path, handle))
}

// createRouteHandler creates a handler function for httprouter.
func (app *Application) createRouteHandler(path string, handle Handle) httprouter.Handle {
	return func(response http.ResponseWriter, request *http.Request, params httprouter.Params) {
		// Create context.
		ctx := Context{
			App:        app,
			StatusCode: http.StatusOK,
			request:    request,
			response:   response,
			params:     params,
		}

		// The last part of the call chain will send the actual response.
		lastPartOfCallChain := func() {
			data := handle(&ctx)
			ctx.respond(data)
		}

		// Declare the type of generateNext so that we can define it recursively in the next part.
		var generateNext func(index int) func()

		// Create a function that returns a bound function next()
		// which can be used as the 2nd parameter in the call chain.
		generateNext = func(index int) func() {
			if index == len(app.middleware) {
				return lastPartOfCallChain
			}

			return func() {
				app.middleware[index](&ctx, generateNext(index+1))
			}
		}

		generateNext(0)()
	}
}

// Run starts your application.
func (app *Application) Run() {
	app.TestManifest()
	app.TestRoutes()
	app.Listen()
	app.Wait()
	app.Shutdown()
}

// Use adds middleware to your middleware chain.
func (app *Application) Use(middlewares ...Middleware) {
	app.middleware = append(app.middleware, middlewares...)
}

// Load loads the application configuration from config.json.
func (app *Application) Load() {
	config, err := LoadConfig("config.json")

	if err != nil {
		// Ignore missing config file, we can perfectly run without one
		return
	}

	app.Config = config
}

// Listen starts the server.
func (app *Application) Listen() {
	if app.Security.Key != "" && app.Security.Certificate != "" {
		go func() {
			app.serveHTTPS(":" + strconv.Itoa(app.Config.Ports.HTTPS))
		}()

		fmt.Println("Server running on:", color.GreenString("https://localhost:"+strconv.Itoa(app.Config.Ports.HTTPS)))
	} else {
		fmt.Println("Server running on:", color.GreenString("http://localhost:"+strconv.Itoa(app.Config.Ports.HTTP)))
	}

	go func() {
		app.serveHTTP(":" + strconv.Itoa(app.Config.Ports.HTTP))
	}()
}

// Wait will make the process wait until it is killed.
func (app *Application) Wait() {
	stop := make(chan os.Signal, 1)
	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)

	for _, callback := range app.onStart {
		callback()
	}

	<-stop
}

// Shutdown will gracefully shut down all servers.
func (app *Application) Shutdown() {
	app.serversMutex.Lock()
	defer app.serversMutex.Unlock()

	shutdown(app.servers[0])
	shutdown(app.servers[1])

	for _, callback := range app.onShutdown {
		callback()
	}
}

// shutdown will gracefully shut down the server.
func shutdown(server *http.Server) {
	if server == nil {
		return
	}

	// Add a timeout to the server shutdown
	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
	defer cancel()

	// Shut down server
	err := server.Shutdown(ctx)

	if err != nil {
		fmt.Println(err)
	}
}

// OnStart registers a callback to be executed on server start.
func (app *Application) OnStart(callback func()) {
	app.onStart = append(app.onStart, callback)
}

// OnEnd registers a callback to be executed on server shutdown.
func (app *Application) OnEnd(callback func()) {
	app.onShutdown = append(app.onShutdown, callback)
}

// OnPush registers a callback to be executed when an HTTP/2 push happens.
func (app *Application) OnPush(callback func(*Context)) {
	app.onPush = append(app.onPush, callback)
}

// AddPushCondition registers a callback to be executed when an HTTP/2 push happens.
func (app *Application) AddPushCondition(test func(*Context) bool) {
	app.pushConditions = append(app.pushConditions, test)
}

// Rewrite sets the URL rewrite function.
func (app *Application) Rewrite(rewrite func(*RewriteContext)) {
	app.rewrite = rewrite
}

// Finalize post-processes the HTML (currently a noop).
func (app *Application) Finalize(html string) string {
	return html
}

// StartTime returns the time the application started.
func (app *Application) StartTime() time.Time {
	return app.start
}

// Handler returns the request handler used by the application.
func (app *Application) Handler() http.Handler {
	router := app.Router
	rewrite := app.rewrite

	if rewrite != nil {
		return &rewriteHandler{
			rewrite: rewrite,
			router:  router,
		}
	}

	return router
}

// createServer creates an http server instance.
func (app *Application) createServer(address string) *http.Server {
	return &http.Server{
		Addr:              address,
		Handler:           app.Handler(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      180 * time.Second,
		IdleTimeout:       120 * time.Second,
		TLSConfig:         createTLSConfig(),
	}
}

// serveHTTP serves requests from the given listener.
func (app *Application) serveHTTP(address string) {
	server := app.createServer(address)

	app.serversMutex.Lock()
	app.servers[0] = server
	app.serversMutex.Unlock()

	// This will block the calling goroutine until the server shuts down.
	serveError := server.ListenAndServe()

	if serveError != nil && !strings.Contains(serveError.Error(), "closed") {
		panic(serveError)
	}
}

// serveHTTPS serves requests from the given listener.
func (app *Application) serveHTTPS(address string) {
	server := app.createServer(address)

	app.serversMutex.Lock()
	app.servers[1] = server
	app.serversMutex.Unlock()

	// This will block the calling goroutine until the server shuts down.
	serveError := server.ListenAndServeTLS(app.Security.Certificate, app.Security.Key)

	if serveError != nil && !strings.Contains(serveError.Error(), "closed") {
		panic(serveError)
	}
}

// Test tests the given URI paths when the application starts.
func (app *Application) Test(route string, paths []string) {
	app.routeTests[route] = paths
}

// TestManifest tests your application's manifest.
func (app *Application) TestManifest() {
	manifest := app.Config.Manifest

	// Warn about short name length (Google Lighthouse)
	// https://developer.chrome.com/apps/manifest/name#short_name
	if len(manifest.ShortName) >= 12 {
		color.Yellow("The short name of your application should have less than 12 characters")
	}
}

// TestRoutes tests your application's routes.
func (app *Application) TestRoutes() {
	fmt.Println(strings.Repeat("-", 80))

	go func() {
		sort.Strings(app.routes.GET)

		for _, route := range app.routes.GET {
			// Skip ajax routes
			if strings.HasPrefix(route, "/_") {
				continue
			}

			// Check if the user defined test routes for the given route
			testRoutes, exists := app.routeTests[route]

			if exists {
				for _, testRoute := range testRoutes {
					app.TestRoute(route, testRoute)
				}

				continue
			}

			// Skip routes with parameters and display a warning to indicate it needs a test route
			if strings.Contains(route, ":") {
				color.Yellow(route)
				continue
			}

			// Test the static route without parameters
			app.TestRoute(route, route)
		}

		// json, _ := Post("https://html5.validator.nu/?out=json").Header("Content-Type", "text/html; charset=utf-8").Header("Content-Encoding", "gzip").Body(body).Send()
		// fmt.Println(json)
	}()
}

// TestRoute tests the given route.
func (app *Application) TestRoute(route string, uri string) {
	for _, linter := range app.Linters {
		linter.Begin(route, uri)
	}

	response, _ := client.Get("http://localhost:" + strconv.Itoa(app.Config.Ports.HTTP) + uri).End()

	for _, linter := range app.Linters {
		linter.End(route, uri, response)
	}
}
<|endoftext|>


<commit_before>package ego_test

import (
	"bytes"
	"testing"

	"github.com/benbjohnson/ego"
)

// Ensure that a template can be written to a writer.
func TestTemplate_Write(t *testing.T) {
	tmpl := &ego.Template{
		Blocks: []ego.Block{
			&ego.TextBlock{Content: "", Pos: ego.Pos{Path: "foo.ego", LineNo: 4}},
			&ego.CodeBlock{Content: "  for _, num := range nums {"},
			&ego.TextBlock{Content: "
"},
			&ego.RawPrintBlock{Content: "num + 1"},
			&ego.TextBlock{Content: "
"},
			&ego.CodeBlock{Content: "  }"},
			&ego.TextBlock{Content: ""},
		},
	}

	var buf bytes.Buffer
	if _, err := tmpl.WriteTo(&buf); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Fix unit test.<commit_after>package ego_test

import (
	"bytes"
	"testing"

	"github.com/benbjohnson/ego"
)

// Ensure that a template can be written to a writer.
func TestTemplate_Write(t *testing.T) {
	tmpl := &ego.Template{
		Blocks: []ego.Block{
			&ego.CodeBlock{Content: "package foo"},
			&ego.CodeBlock{Content: "func doSomething() {"},
			&ego.TextBlock{Content: "", Pos: ego.Pos{Path: "foo.ego", LineNo: 4}},
			&ego.CodeBlock{Content: "  for _, num := range nums {"},
			&ego.TextBlock{Content: "
"},
			&ego.RawPrintBlock{Content: "num + 1"},
			&ego.TextBlock{Content: "
"},
			&ego.CodeBlock{Content: "  }"},
			&ego.TextBlock{Content: ""},
			&ego.CodeBlock{Content: "}"},
		},
	}

	var buf bytes.Buffer
	if _, err := tmpl.WriteTo(&buf); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>