<commit_before>// Package cycbuf implements cyclic storage.
// Each cycbuf contains a map of cycfiles.
// Each cycfile stores a fixed amount of log lines.
// A cycbuf file is allocated for each md5 sum, i.e.
// Tags, Hostnames, Priority & Regex match.

package cycbuf

import (
	"../syslogd"
	"code.google.com/p/go.net/websocket"
	"crypto/md5"
	"encoding/json"
	"fmt"
	_ "github.com/lib/pq"
	"io"
	"net/http"
	"strconv"
)

// Store up to cycbuflen messages in a cycfile.
const cycbuflen = 1024

type cycfile struct {
	msgs [cycbuflen]*syslogd.Message
	ptr  int
	loop bool
	subs map[*websocket.Conn]chan *syslogd.Message
}

type Cycbuf struct {
	files map[string]*cycfile
	sums  map[string]string
}

func (cf *cycfile) Init() {
	cf.subs = make(map[*websocket.Conn]chan *syslogd.Message)
	cf.loop = false
	cf.ptr = 0
}

func (cf *cycfile) AddMsg(m *syslogd.Message) {
	cf.msgs[cf.ptr] = m
	cf.ptr++
	if cf.ptr > cycbuflen-1 {
		cf.ptr = 0
		cf.loop = true
	}
	cf.publish(m)
}

// Range returns a full range slice of X syslogd messages.
func (cf *cycfile) Range() []*syslogd.Message {
	s1 := cf.msgs[0:cf.ptr]
	if cf.loop {
		s2 := cf.msgs[cf.ptr+1 : len(cf.msgs)]
		return append(s2, s1...)
	}

	return s1
}

func (cf *cycfile) Last(max int) []*syslogd.Message {
	if max == 0 {
		max = cycbuflen
	}

	r := cf.Range()
	if len(r) < max {
		max = len(r)
	}

	newrange := make([]*syslogd.Message, max)
	for i,j := len(r)-1, 0; j<max; i, j = i-1, j+1 {
		newrange[j] = r[i]
	}
	return newrange
}

func (cf *cycfile) publish(m *syslogd.Message) {
	for ws, ch := range cf.subs {
		select {
		case ch <- m:
		default:
			cf.Unsubscribe(ws)
			close(ch)
		}
	}
}

func (cf *cycfile) Subscribe(ws *websocket.Conn, mchan chan *syslogd.Message) error {
	cf.subs[ws] = mchan
	return nil
}

func (cf *cycfile) Unsubscribe(ws *websocket.Conn) error {
	delete(cf.subs, ws)
	return nil
}

func New() *Cycbuf {
	c := new(Cycbuf)
	c.files = make(map[string]*cycfile)
	c.sums = make(map[string]string)
	return c
}

func (cb *Cycbuf) AddString(str string, m *syslogd.Message) {
	sum, x := cb.sums[str]
	if !x {
		h := md5.New()
		io.WriteString(h, str)
		cb.sums[str] = fmt.Sprintf("%x", h.Sum(nil))
		sum = cb.sums[str]
	}

	cf, x := cb.files[sum]
	if !x {
		cb.files[sum] = new(cycfile)
		cb.files[sum].Init()
		cf = cb.files[sum]
	}
	cf.AddMsg(m)
}

func (cb *Cycbuf) Add(sum string, m *syslogd.Message) {
	cf, x := cb.files[sum]
	if !x {
		cb.files[sum] = new(cycfile)
		cb.files[sum].Init()
		cf = cb.files[sum]
	}
	cf.AddMsg(m)
}

// Call on program exit or signal.
func (cb *Cycbuf) Dump() {
}

// Call on program load.
func (cb *Cycbuf) Restore() {
}

// HttpLog returns the last "max" messages of "md5" in json format.
func (cb *Cycbuf) HttpLog(w http.ResponseWriter, r *http.Request) {
	sum := r.FormValue("md5")
	max := r.FormValue("max")

	cf, x := cb.files[sum]
	if !x {
		http.NotFound(w, r)
		return
	}

	var imax int64 = 0
	if max != "" {
		imax, _ = strconv.ParseInt(max, 10, 64)
	}

	lines := cf.Last(int(imax))

	b, err := json.Marshal(&lines)
	if err != nil {
		panic(err)
	}
	w.Write(b)
}

func (cb *Cycbuf) HttpStream(ws *websocket.Conn) {
	c := ws.Config()
	v := c.Location.Query()
	md5 := v.Get("md5")

	cf, x := cb.files[md5]
	if !x {
		fmt.Fprintf(ws, "No such sum %s", md5)
		return
	}

	ch := make(chan *syslogd.Message, 1)
	cf.Subscribe(ws, ch)

	for {
		select {
		case m := <-ch:
			b, err := json.Marshal(&m)
			if err != nil {
				panic(err)
			}
			_, err = ws.Write(b)
			if err != nil {
				cf.Unsubscribe(ws)
				close(ch)
				return
			}
		}
	}
}
<commit_msg>Fix double close of pubsub channel.<commit_after>// Package cycbuf implements cyclic storage.
// Each cycbuf contains a map of cycfiles.
// Each cycfile stores a fixed amount of log lines.
// A cycbuf file is allocated for each md5 sum, i.e.
// Tags, Hostnames, Priority & Regex match.

package cycbuf

import (
	"../syslogd"
	"code.google.com/p/go.net/websocket"
	"crypto/md5"
	"encoding/json"
	"fmt"
	_ "github.com/lib/pq"
	"io"
	"net/http"
	"strconv"
)

// Store up to cycbuflen messages in a cycfile.
const cycbuflen = 1024

type cycfile struct {
	msgs [cycbuflen]*syslogd.Message
	ptr  int
	loop bool
	subs map[*websocket.Conn]chan *syslogd.Message
}

type Cycbuf struct {
	files map[string]*cycfile
	sums  map[string]string
}

func (cf *cycfile) Init() {
	cf.subs = make(map[*websocket.Conn]chan *syslogd.Message)
	cf.loop = false
	cf.ptr = 0
}

func (cf *cycfile) AddMsg(m *syslogd.Message) {
	cf.msgs[cf.ptr] = m
	cf.ptr++
	if cf.ptr > cycbuflen-1 {
		cf.ptr = 0
		cf.loop = true
	}
	cf.publish(m)
}

// Range returns a full range slice of X syslogd messages.
func (cf *cycfile) Range() []*syslogd.Message {
	s1 := cf.msgs[0:cf.ptr]
	if cf.loop {
		s2 := cf.msgs[cf.ptr+1 : len(cf.msgs)]
		return append(s2, s1...)
	}

	return s1
}

func (cf *cycfile) Last(max int) []*syslogd.Message {
	if max == 0 {
		max = cycbuflen
	}

	r := cf.Range()
	if len(r) < max {
		max = len(r)
	}

	newrange := make([]*syslogd.Message, max)
	for i,j := len(r)-1, 0; j<max; i, j = i-1, j+1 {
		newrange[j] = r[i]
	}
	return newrange
}

func (cf *cycfile) publish(m *syslogd.Message) {
	for _, ch := range cf.subs {
		ch <- m
	}
}

func (cf *cycfile) Subscribe(ws *websocket.Conn, mchan chan *syslogd.Message) error {
	cf.subs[ws] = mchan
	return nil
}

func (cf *cycfile) Unsubscribe(ws *websocket.Conn) error {
	delete(cf.subs, ws)
	return nil
}

func New() *Cycbuf {
	c := new(Cycbuf)
	c.files = make(map[string]*cycfile)
	c.sums = make(map[string]string)
	return c
}

func (cb *Cycbuf) AddString(str string, m *syslogd.Message) {
	sum, x := cb.sums[str]
	if !x {
		h := md5.New()
		io.WriteString(h, str)
		cb.sums[str] = fmt.Sprintf("%x", h.Sum(nil))
		sum = cb.sums[str]
	}

	cf, x := cb.files[sum]
	if !x {
		cb.files[sum] = new(cycfile)
		cb.files[sum].Init()
		cf = cb.files[sum]
	}
	cf.AddMsg(m)
}

func (cb *Cycbuf) Add(sum string, m *syslogd.Message) {
	cf, x := cb.files[sum]
	if !x {
		cb.files[sum] = new(cycfile)
		cb.files[sum].Init()
		cf = cb.files[sum]
	}
	cf.AddMsg(m)
}

// Call on program exit or signal.
func (cb *Cycbuf) Dump() {
}

// Call on program load.
func (cb *Cycbuf) Restore() {
}

// HttpLog returns the last "max" messages of "md5" in json format.
func (cb *Cycbuf) HttpLog(w http.ResponseWriter, r *http.Request) {
	sum := r.FormValue("md5")
	max := r.FormValue("max")

	cf, x := cb.files[sum]
	if !x {
		http.NotFound(w, r)
		return
	}

	var imax int64 = 0
	if max != "" {
		imax, _ = strconv.ParseInt(max, 10, 64)
	}

	lines := cf.Last(int(imax))

	b, err := json.Marshal(&lines)
	if err != nil {
		panic(err)
	}
	w.Write(b)
}

func (cb *Cycbuf) HttpStream(ws *websocket.Conn) {
	c := ws.Config()
	v := c.Location.Query()
	md5 := v.Get("md5")

	cf, x := cb.files[md5]
	if !x {
		fmt.Fprintf(ws, "No such sum %s", md5)
		return
	}

	ch := make(chan *syslogd.Message, 1024)
	cf.Subscribe(ws, ch)
	defer close(ch)
	defer cf.Unsubscribe(ws)

	for {
		select {
		case m := <-ch:
			b, err := json.Marshal(&m)
			if err != nil {
				panic(err)
			}
			_, err = ws.Write(b)
			if err != nil {
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>package login

import (
	"github.com/AitorGuerrero/UserGo/user"
)

type Command struct {
	Login user.Login
	UserSource user.UserSource
}

type Request struct {
	Id string
	Passkey string
}

type Response struct {
	SessionToken string
}

func (c Command) Execute(req Request) (Response, error) {
	res := Response{}
	u, err := c.UserSource.Get(user.Id(req.Id))
	if nil != err {
		return res, err
	}
	u2, err2 := c.Login.Try(u, user.Passkey(req.Passkey))
	c.UserSource.Persist(u2)
	res.SessionToken = string(u2.Token().Code)

	return res, err2
}
<commit_msg>Trying functional programming in login command<commit_after>package login

import (
	"github.com/AitorGuerrero/UserGo/user"
)

type Command struct {
	Login user.Login
	UserSource user.UserSource
}

type Request struct {
	Id string
	Passkey string
}

type Response struct {
	SessionToken string
}

func (c Command) Execute(req Request) (res Response, err error) {
	tokenCode, err := c.getTokenCodeFromUserIfCorrectLogin(
		user.Id(req.Id),
		user.Passkey(req.Passkey),
	)
	res.SessionToken = string(tokenCode)

	return res, err
}

func (c Command) getTokenCodeFromUserIfCorrectLogin(uid user.Id, up user.Passkey) (tc user.TokenCode, err error) {
	u, err := c.getUserIfCorrectLogin(uid, up)
	if nil != err {
		return
	}
 	tc = u.Token().Code

	return
}

func (c Command) getUserIfCorrectLogin(uid user.Id, up user.Passkey) (u user.User, err error) {
	u, err = c.tryLogin(uid, up)
	if(nil != err) {
		return
	}
	u, err = c.persistUser(u)

	return
}

func (c Command) persistUser(u user.User) (user.User, error) {
	c.UserSource.Persist(u)
	return u, nil
}

func (c Command) tryLogin(uid user.Id, p user.Passkey) (u user.User, err error) {
	u, err = c.getUser(uid)
	if(nil != err) {
		return
	}
	u, err = c.Login.Try(u, p)

	return
}

func (c Command) getUser(uid user.Id) (user.User, error) {
	return c.UserSource.Get(uid)
}

func (c Command) getTokenCodeFromUser(u user.User) user.TokenCode {
	return u.Token().Code
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Mitchell Cooper
package config

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
)

// parse config
func (conf *Config) Parse() error {

	// open the config
	file, err := os.Open(conf.path)
	if err != nil {
		return err
	}
	defer file.Close()

	// initial state
	state := &parserState{
		line:    1,
		buffers: make([]bufferInfo, 0, 3),
	}
	conf.line = &state.line

	for {
		b := make([]byte, 1)
		_, err := file.Read(b)

		// eof
		if err == io.EOF {
			break
		}

		// some other error
		if err != nil {
			return err
		}

		// handle the character
		err = conf.handleByte(state, b[0])
		state.lastByte = b[0]

		// byte error
		if err != nil {
			err = errors.New(fmt.Sprintf("%s:%d: %s", conf.path, *conf.line, err.Error()))
			return err
		}
	}

	state.line = 0
	return nil
}

// produce a warning
func (conf *Config) Warn(msg string) {
	log.Println(conf.getWarn(msg))
}

func (conf *Config) Warnf(msg string, i ...interface{}) {
	log.Printf(conf.getWarn(msg), i)
}

func (conf *Config) getWarn(msg string) (res string) {
	line := *conf.line
	if line == 0 {
		res = fmt.Sprintf("%s: %s\n", conf.path, msg)
		return
	}
	res = fmt.Sprintf("%s:%d: %s\n", conf.path, line, msg)
	return
}

// handle one byte
func (conf *Config) handleByte(state *parserState, b byte) error {

	// this character is escaped
	if state.escaped {
		b = 0
		state.escaped = false
	}

	if state.lastByte == '/' && b == '*' {

		// comment entrance
		state.increaseCommentLevel()
		return nil

	} else if state.lastByte == '*' && b == '/' {

		// comment closure
		state.decreaseCommentLevel()
		return nil

	} else if state.inComment {

		// we're in a comment currently
		return nil
	}

	switch b {

	// escape
	case '\\':
		state.escaped = true

		// start of a variable
	case '@', '%':

		// we're already in a variable name
		if state.buffType() == VAR_NAME {
			return errors.New("Already in variable name @" + state.endBuffer())
		}

		// this is only allowed at the top level
		if state.buffType() != NO_BUF {
			goto realDefault
		}

		// we're not in a value, so this starts a variable name
		state.varPercent = b == '%'
		state.startBuffer(VAR_NAME)

	// end of variable name, start of string value
	case ':':

		// not in a variable name
		if state.buffType() != VAR_NAME {
			goto realDefault
		}

		// we're in the var name, so terminate it
		state.varName = state.endBuffer()
		state.startBuffer(VAR_VALUE)

		// start of a text format
	case '[':

		// we're already in a text format.
		// this is supported by wikifier but not here
		if state.buffType() == VAR_FORMAT {
			return errors.New("Square brackets in format not yet supported")
		}

		// we aren't in a variable value, or maybe
		// the current variable does not allow interpolation
		if state.buffType() != VAR_VALUE || state.varPercent {
			goto realDefault
		}

		// otherwise, this starts a formatting token
		state.startBuffer(VAR_FORMAT)

	// end of a text format
	case ']':

		// not in a format
		if state.buffType() != VAR_FORMAT {
			goto realDefault
		}

		// otherwise, this terminates a formatting token
		tok := state.endBuffer()

		// parse the formatting token
		err, newVal := conf.getFormattingToken(tok, false)
		if err != nil {
			return err
		}
		if newVal == "" {
			conf.Warn("[" + tok + "] yields empty string")
		}

		// add the value returned by it to the variable value buffer
		state.buffer().WriteString(newVal)

	// end of a variable definition
	case ';':

		if state.buffType() == VAR_NAME {

			// terminating a boolean
			state.endBuffer()
			conf.Set(state.getVariable(), "1")

		} else if state.buffType() == VAR_VALUE {

			// terminating a string
			value := strings.TrimSpace(state.endBuffer())
			conf.Set(state.getVariable(), value)

		} else {
			goto realDefault
		}

	case '\n':
		state.line++
		goto realDefault

	default:
		goto realDefault
	}

	// this is skipped if going to realDefault
	return nil

realDefault:

	// we're in a comment; ignore this
	if state.inComment {
		return nil
	}

	// otherwise, write this to the current buffer
	if state.buffer() != nil {
		state.buffer().WriteByte(b)
	}

	return nil
}

// return the value of a formatting token
func (conf *Config) getFormattingToken(tok string, disableVars bool) (error, string) {

	// normal variable
	if strings.HasPrefix(tok, "@") {
		if disableVars {
			goto badVariable
		}
		return nil, conf.Get(strings.TrimPrefix(tok, "@"))
	}

	// interpolable variable
	if strings.HasPrefix(tok, "%") {
		if disableVars {
			goto badVariable
		}
		val := strings.TrimPrefix("tok", "%")
		return conf.getFormattingToken(val, true)
	}

	return errors.New("Unknown formatting token [" + tok + "]"), ""

badVariable:
	return errors.New("Recursive variable " + tok + " detected"), ""
}

// return the map and attribute name for a variable name
func (conf *Config) getWhere(varName string, createAsNeeded bool) (map[string]interface{}, string) {

	// split up into parts
	var parts = strings.Split(varName, ".")
	if len(parts) == 0 {
		return nil, ""
	}

	// the last one is the final variable name
	lastPart, parts := parts[len(parts)-1], parts[:len(parts)-1]

	// start with the main map
	where := conf.vars

	// for each part, fetch the map inside
	for _, part := range parts {

		// find interface
		iface := where[part]
		if iface == nil {
			if !createAsNeeded {
				return nil, ""
			}

			// maybe create a map
			iface = make(map[string]interface{})
			where[part] = iface
		}

		// find map
		switch aMap := iface.(type) {
		case map[string]interface{}:
			where = aMap

		// nothing there; give up
		default:
			return nil, ""
		}
	}

	return where, lastPart
}
<commit_msg>er only \n in printf<commit_after>// Copyright (c) 2017, Mitchell Cooper
package config

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
)

// parse config
func (conf *Config) Parse() error {

	// open the config
	file, err := os.Open(conf.path)
	if err != nil {
		return err
	}
	defer file.Close()

	// initial state
	state := &parserState{
		line:    1,
		buffers: make([]bufferInfo, 0, 3),
	}
	conf.line = &state.line

	for {
		b := make([]byte, 1)
		_, err := file.Read(b)

		// eof
		if err == io.EOF {
			break
		}

		// some other error
		if err != nil {
			return err
		}

		// handle the character
		err = conf.handleByte(state, b[0])
		state.lastByte = b[0]

		// byte error
		if err != nil {
			err = errors.New(fmt.Sprintf("%s:%d: %s", conf.path, *conf.line, err.Error()))
			return err
		}
	}

	state.line = 0
	return nil
}

// produce a warning
func (conf *Config) Warn(msg string) {
	log.Println(conf.getWarn(msg))
}

func (conf *Config) Warnf(msg string, i ...interface{}) {
	log.Printf(conf.getWarn(msg)+"\n", i)
}

func (conf *Config) getWarn(msg string) (res string) {
	line := *conf.line
	if line == 0 {
		res = fmt.Sprintf("%s: %s", conf.path, msg)
		return
	}
	res = fmt.Sprintf("%s:%d: %s", conf.path, line, msg)
	return
}

// handle one byte
func (conf *Config) handleByte(state *parserState, b byte) error {

	// this character is escaped
	if state.escaped {
		b = 0
		state.escaped = false
	}

	if state.lastByte == '/' && b == '*' {

		// comment entrance
		state.increaseCommentLevel()
		return nil

	} else if state.lastByte == '*' && b == '/' {

		// comment closure
		state.decreaseCommentLevel()
		return nil

	} else if state.inComment {

		// we're in a comment currently
		return nil
	}

	switch b {

	// escape
	case '\\':
		state.escaped = true

		// start of a variable
	case '@', '%':

		// we're already in a variable name
		if state.buffType() == VAR_NAME {
			return errors.New("Already in variable name @" + state.endBuffer())
		}

		// this is only allowed at the top level
		if state.buffType() != NO_BUF {
			goto realDefault
		}

		// we're not in a value, so this starts a variable name
		state.varPercent = b == '%'
		state.startBuffer(VAR_NAME)

	// end of variable name, start of string value
	case ':':

		// not in a variable name
		if state.buffType() != VAR_NAME {
			goto realDefault
		}

		// we're in the var name, so terminate it
		state.varName = state.endBuffer()
		state.startBuffer(VAR_VALUE)

		// start of a text format
	case '[':

		// we're already in a text format.
		// this is supported by wikifier but not here
		if state.buffType() == VAR_FORMAT {
			return errors.New("Square brackets in format not yet supported")
		}

		// we aren't in a variable value, or maybe
		// the current variable does not allow interpolation
		if state.buffType() != VAR_VALUE || state.varPercent {
			goto realDefault
		}

		// otherwise, this starts a formatting token
		state.startBuffer(VAR_FORMAT)

	// end of a text format
	case ']':

		// not in a format
		if state.buffType() != VAR_FORMAT {
			goto realDefault
		}

		// otherwise, this terminates a formatting token
		tok := state.endBuffer()

		// parse the formatting token
		err, newVal := conf.getFormattingToken(tok, false)
		if err != nil {
			return err
		}
		if newVal == "" {
			conf.Warn("[" + tok + "] yields empty string")
		}

		// add the value returned by it to the variable value buffer
		state.buffer().WriteString(newVal)

	// end of a variable definition
	case ';':

		if state.buffType() == VAR_NAME {

			// terminating a boolean
			state.endBuffer()
			conf.Set(state.getVariable(), "1")

		} else if state.buffType() == VAR_VALUE {

			// terminating a string
			value := strings.TrimSpace(state.endBuffer())
			conf.Set(state.getVariable(), value)

		} else {
			goto realDefault
		}

	case '\n':
		state.line++
		goto realDefault

	default:
		goto realDefault
	}

	// this is skipped if going to realDefault
	return nil

realDefault:

	// we're in a comment; ignore this
	if state.inComment {
		return nil
	}

	// otherwise, write this to the current buffer
	if state.buffer() != nil {
		state.buffer().WriteByte(b)
	}

	return nil
}

// return the value of a formatting token
func (conf *Config) getFormattingToken(tok string, disableVars bool) (error, string) {

	// normal variable
	if strings.HasPrefix(tok, "@") {
		if disableVars {
			goto badVariable
		}
		return nil, conf.Get(strings.TrimPrefix(tok, "@"))
	}

	// interpolable variable
	if strings.HasPrefix(tok, "%") {
		if disableVars {
			goto badVariable
		}
		val := strings.TrimPrefix("tok", "%")
		return conf.getFormattingToken(val, true)
	}

	return errors.New("Unknown formatting token [" + tok + "]"), ""

badVariable:
	return errors.New("Recursive variable " + tok + " detected"), ""
}

// return the map and attribute name for a variable name
func (conf *Config) getWhere(varName string, createAsNeeded bool) (map[string]interface{}, string) {

	// split up into parts
	var parts = strings.Split(varName, ".")
	if len(parts) == 0 {
		return nil, ""
	}

	// the last one is the final variable name
	lastPart, parts := parts[len(parts)-1], parts[:len(parts)-1]

	// start with the main map
	where := conf.vars

	// for each part, fetch the map inside
	for _, part := range parts {

		// find interface
		iface := where[part]
		if iface == nil {
			if !createAsNeeded {
				return nil, ""
			}

			// maybe create a map
			iface = make(map[string]interface{})
			where[part] = iface
		}

		// find map
		switch aMap := iface.(type) {
		case map[string]interface{}:
			where = aMap

		// nothing there; give up
		default:
			return nil, ""
		}
	}

	return where, lastPart
}
<|endoftext|>


<commit_before><commit_msg>Add tests for project_service.<commit_after><|endoftext|>


<commit_before>package server

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
)

var apiHost, jsBundleURL, dynoID, twitterAPIToken string
var webPort int

func initEnv() (err error) {
	twitterAPIToken = os.Getenv("TWITTER_TOKEN")
	webPort, err = strconv.Atoi(os.Getenv("PORT"))
	if err != nil || webPort == 0 {
		log.Println("PORT not set, using default")
		webPort = 8080
		err = nil
	}

	jsBundleURL = os.Getenv("JS_BUNDLE")
	if jsBundleURL == "" {
		return errors.New("Could not determing JS bundle URL")
	}

	dynoID = os.Getenv("HEROKU_DYNO_ID")

	apiHost = os.Getenv("API_HOST")
	if apiHost == "" {
		if dynoID == "" {
			return errors.New("Could not determine API URL")
		}
		apiHost = fmt.Sprintf("https://%s.herokuapp.com", dynoID)
		log.Println("API_HOST not set, using default: " + apiHost)
	}

	return
}

func getServeAddress() string {
	return fmt.Sprintf(":%d", webPort)
}
<commit_msg>Rename variable to make more sense given expected value<commit_after>package server

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
)

var apiHost, jsBundleURL, herokuAppName, twitterAPIToken string
var webPort int

func initEnv() (err error) {
	twitterAPIToken = os.Getenv("TWITTER_TOKEN")
	webPort, err = strconv.Atoi(os.Getenv("PORT"))
	if err != nil || webPort == 0 {
		log.Println("PORT not set, using default")
		webPort = 8080
		err = nil
	}

	jsBundleURL = os.Getenv("JS_BUNDLE")
	if jsBundleURL == "" {
		return errors.New("Could not determing JS bundle URL")
	}

	herokuAppName = os.Getenv("HEROKU_APP_NAME")

	apiHost = os.Getenv("API_HOST")
	if apiHost == "" {
		if herokuAppName == "" {
			return errors.New("Could not determine API URL")
		}
		apiHost = fmt.Sprintf("https://%s.herokuapp.com", herokuAppName)
		log.Println("API_HOST not set, using default: " + apiHost)
	}

	return
}

func getServeAddress() string {
	return fmt.Sprintf(":%d", webPort)
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/devices"
)

type UserMapping struct {
	From  int `json:"from"`
	To    int `json:"to"`
	Count int `json:"count"`
}

type Rlimit struct {
	Type int    `json:"type"`
	Hard uint64 `json:"hard"`
	Soft uint64 `json:"soft"`
}

type HugepageLimit struct {
	Pagesize string `json:"pageSize"`
	Limit    int    `json:"limit"`
}

type IfPrioMap struct {
	Interface string `json:"interface"`
	Priority  int64  `json:"priority"`
}

type Resources struct {
	// Memory reservation or soft_limit (in bytes)
	MemoryReservation int64 `json:"memoryReservation"`
	// Total memory usage (memory + swap); set `-1' to disable swap
	MemorySwap int64 `json:"memorySwap"`
	// Kernel memory limit (in bytes)
	KernelMemory int64 `json:"kernelMemory"`
	// CPU shares (relative weight vs. other containers)
	CpuShares int64 `json:"cpuShares"`
	// CPU hardcap limit (in usecs). Allowed cpu time in a given period.
	CpuQuota int64 `json:"cpuQuota"`
	// CPU period to be used for hardcapping (in usecs). 0 to use system default.
	CpuPeriod int64 `json:"cpuPeriod"`
	// How many time CPU will use in realtime scheduling (in usecs).
	CpuRtRuntime int64 `json:"cpuQuota"`
	// CPU period to be used for realtime scheduling (in usecs).
	CpuRtPeriod int64 `json:"cpuPeriod"`
	// CPU to use
	CpusetCpus string `json:"cpusetCpus"`
	// MEM to use
	CpusetMems string `json:"cpusetMems"`
	// IO read rate limit per cgroup per device, bytes per second.
	BlkioThrottleReadBpsDevice string `json:"blkioThrottleReadBpsDevice"`
	// IO write rate limit per cgroup per divice, bytes per second.
	BlkioThrottleWriteBpsDevice string `json:"blkioThrottleWriteBpsDevice"`
	// IO read rate limit per cgroup per device, IO per second.
	BlkioThrottleReadIOpsDevice string `json:"blkioThrottleReadIopsDevice"`
	// IO write rate limit per cgroup per device, IO per second.
	BlkioThrottleWriteIOpsDevice string `json:"blkioThrottleWriteIopsDevice"`
	// Specifies per cgroup weight, range is from 10 to 1000.
	BlkioWeight int64 `json:"blkioWeight"`
	// Weight per cgroup per device, can override BlkioWeight.
	BlkioWeightDevice string `json:"blkioWeightDevice"`
	// Hugetlb limit (in bytes)
	HugetlbLimit []*HugepageLimit `json:"hugetlbLimit"`
	// Whether to disable OOM Killer
	DisableOOMKiller bool `json:"disableOOMKiller"`
	// Set priority of network traffic for container
	NetPrioIfpriomap []*IfPrioMap `json:"netPrioIfpriomap"`
	// Set class identifier for container's network packets
	NetClsClassid string `json:"netClsClassid"`
}

type LinuxSpec struct {
	PortableSpec
	UserMapping      map[string]UserMapping `json:"userMapping"`
	Rlimits          []Rlimit               `json:"rlimits"`
	SystemProperties map[string]string      `json:"systemProperties"`
	Resources        *Resources             `json:"resources"`
}

var namespaceMapping = map[string]configs.NamespaceType{
	"process": configs.NEWPID,
	"network": configs.NEWNET,
	"mount":   configs.NEWNS,
	"user":    configs.NEWUSER,
	"ipc":     configs.NEWIPC,
	"uts":     configs.NEWUTS,
}

// loadSpec loads the specification from the provided path.
// If the path is empty then the default path will be "container.json"
func loadSpec(path string) (*LinuxSpec, error) {
	if path == "" {
		path = "container.json"
	}
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("JSON specification file for %s not found", path)
		}
		return nil, err
	}
	defer f.Close()
	var s *LinuxSpec
	if err := json.NewDecoder(f).Decode(&s); err != nil {
		return nil, err
	}
	return s, nil
}

func createLibcontainerConfig(spec *LinuxSpec) (*configs.Config, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	config := &configs.Config{
		Capabilities: spec.Capabilities,
		Rootfs:       filepath.Join(cwd, spec.Root.Path),
		Readonlyfs:   spec.Root.Readonly,
		Hostname:     spec.Hostname,
		Privatefs:    true,
	}
	for _, ns := range spec.Namespaces {
		t, exists := namespaceMapping[ns.Type]
		if !exists {
			return nil, fmt.Errorf("namespace %q does not exist", ns)
		}
		config.Namespaces.Add(t, ns.Path)
	}
	for _, m := range spec.Mounts {
		config.Mounts = append(config.Mounts, createLibcontianerMount(cwd, m))
	}
	if err := createDevices(spec, config); err != nil {
		return nil, err
	}
	if err := setupUserNamespace(spec, config); err != nil {
		return nil, err
	}
	c, err := createCgroupConfig(spec, config.Devices)
	if err != nil {
		return nil, err
	}
	config.Cgroups = c
	if config.Readonlyfs {
		setReadonly(config)
		config.MaskPaths = []string{
			"/proc/kcore",
		}
		config.ReadonlyPaths = []string{
			"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
		}
	}
	return config, nil
}

func createLibcontianerMount(cwd string, m Mount) *configs.Mount {
	flags, data := parseMountOptions(m.Options)
	source := m.Source
	if m.Type == "bind" {
		if !filepath.IsAbs(source) {
			source = filepath.Join(cwd, m.Source)
		}
	}
	return &configs.Mount{
		Device:      m.Type,
		Source:      source,
		Destination: m.Destination,
		Data:        data,
		Flags:       flags,
	}
}

func createCgroupConfig(spec *LinuxSpec, devices []*configs.Device) (*configs.Cgroup, error) {
	myCgroupPath, err := cgroups.GetThisCgroupDir("devices")
	if err != nil {
		return nil, err
	}
	c := &configs.Cgroup{
		Name:           getDefaultID(),
		Parent:         myCgroupPath,
		AllowedDevices: append(devices, allowedDevices...),
		CpuQuota:       getCPUQuota(spec.Cpus),
		Memory:         spec.Memory * 1024 * 1024,
		MemorySwap:     -1,
	}
	if r := spec.Resources; r != nil {
		c.MemoryReservation = r.MemoryReservation
		c.MemorySwap = r.MemorySwap
		c.KernelMemory = r.KernelMemory
		c.CpuShares = r.CpuShares
		c.CpuQuota = r.CpuQuota
		c.CpuPeriod = r.CpuPeriod
		c.CpuRtRuntime = r.CpuRtRuntime
		c.CpuRtPeriod = r.CpuRtPeriod
		c.CpusetCpus = r.CpusetCpus
		c.CpusetMems = r.CpusetMems
		c.BlkioThrottleReadBpsDevice = r.BlkioThrottleReadBpsDevice
		c.BlkioThrottleWriteBpsDevice = r.BlkioThrottleWriteBpsDevice
		c.BlkioThrottleReadIOpsDevice = r.BlkioThrottleReadIOpsDevice
		c.BlkioThrottleWriteIOpsDevice = r.BlkioThrottleWriteIOpsDevice
		c.BlkioWeight = r.BlkioWeight
		c.BlkioWeightDevice = r.BlkioWeightDevice
		for _, l := range r.HugetlbLimit {
			c.HugetlbLimit = append(c.HugetlbLimit, &configs.HugepageLimit{
				Pagesize: l.Pagesize,
				Limit:    l.Limit,
			})
		}
		c.OomKillDisable = r.DisableOOMKiller
		for _, m := range r.NetPrioIfpriomap {
			c.NetPrioIfpriomap = append(c.NetPrioIfpriomap, &configs.IfPrioMap{
				Interface: m.Interface,
				Priority:  m.Priority,
			})
		}
		c.NetClsClassid = r.NetClsClassid
	}
	return c, nil
}

func createDevices(spec *LinuxSpec, config *configs.Config) error {
	for _, name := range spec.Devices {
		d, err := devices.DeviceFromPath(filepath.Join("/dev", name), "rwm")
		if err != nil {
			return err
		}
		config.Devices = append(config.Devices, d)
	}
	return nil
}

func setReadonly(config *configs.Config) {
	for _, m := range config.Mounts {
		if m.Device == "sysfs" {
			m.Flags |= syscall.MS_RDONLY
		}
	}
}

func getCPUQuota(cpus float64) int64 {
	return int64(cpus * cpuQuotaMultiplyer)
}

func setupUserNamespace(spec *LinuxSpec, config *configs.Config) error {
	if len(spec.UserMapping) == 0 {
		return nil
	}
	config.Namespaces.Add(configs.NEWUSER, "")
	mappings := make(map[string][]configs.IDMap)
	for k, v := range spec.UserMapping {
		mappings[k] = append(mappings[k], configs.IDMap{
			ContainerID: v.From,
			HostID:      v.To,
			Size:        v.Count,
		})
	}
	config.UidMappings = mappings["uid"]
	config.GidMappings = mappings["gid"]
	rootUid, err := config.HostUID()
	if err != nil {
		return err
	}
	rootGid, err := config.HostGID()
	if err != nil {
		return err
	}
	for _, node := range config.Devices {
		node.Uid = uint32(rootUid)
		node.Gid = uint32(rootGid)
	}
	return nil
}

// parseMountOptions parses the string and returns the flags and any mount data that
// it contains.
func parseMountOptions(options string) (int, string) {
	var (
		flag int
		data []string
	)
	flags := map[string]struct {
		clear bool
		flag  int
	}{
		"defaults":      {false, 0},
		"ro":            {false, syscall.MS_RDONLY},
		"rw":            {true, syscall.MS_RDONLY},
		"suid":          {true, syscall.MS_NOSUID},
		"nosuid":        {false, syscall.MS_NOSUID},
		"dev":           {true, syscall.MS_NODEV},
		"nodev":         {false, syscall.MS_NODEV},
		"exec":          {true, syscall.MS_NOEXEC},
		"noexec":        {false, syscall.MS_NOEXEC},
		"sync":          {false, syscall.MS_SYNCHRONOUS},
		"async":         {true, syscall.MS_SYNCHRONOUS},
		"dirsync":       {false, syscall.MS_DIRSYNC},
		"remount":       {false, syscall.MS_REMOUNT},
		"mand":          {false, syscall.MS_MANDLOCK},
		"nomand":        {true, syscall.MS_MANDLOCK},
		"atime":         {true, syscall.MS_NOATIME},
		"noatime":       {false, syscall.MS_NOATIME},
		"diratime":      {true, syscall.MS_NODIRATIME},
		"nodiratime":    {false, syscall.MS_NODIRATIME},
		"bind":          {false, syscall.MS_BIND},
		"rbind":         {false, syscall.MS_BIND | syscall.MS_REC},
		"unbindable":    {false, syscall.MS_UNBINDABLE},
		"runbindable":   {false, syscall.MS_UNBINDABLE | syscall.MS_REC},
		"private":       {false, syscall.MS_PRIVATE},
		"rprivate":      {false, syscall.MS_PRIVATE | syscall.MS_REC},
		"shared":        {false, syscall.MS_SHARED},
		"rshared":       {false, syscall.MS_SHARED | syscall.MS_REC},
		"slave":         {false, syscall.MS_SLAVE},
		"rslave":        {false, syscall.MS_SLAVE | syscall.MS_REC},
		"relatime":      {false, syscall.MS_RELATIME},
		"norelatime":    {true, syscall.MS_RELATIME},
		"strictatime":   {false, syscall.MS_STRICTATIME},
		"nostrictatime": {true, syscall.MS_STRICTATIME},
	}
	for _, o := range strings.Split(options, ",") {
		// If the option does not exist in the flags table or the flag
		// is not supported on the platform,
		// then it is a data value for a specific fs type
		if f, exists := flags[o]; exists && f.flag != 0 {
			if f.clear {
				flag &= ^f.flag
			} else {
				flag |= f.flag
			}
		} else {
			data = append(data, o)
		}
	}
	return flag, strings.Join(data, ",")
}
<commit_msg>Initialize memory.swappiness cgroup to -1<commit_after>// +build linux

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/devices"
)

type UserMapping struct {
	From  int `json:"from"`
	To    int `json:"to"`
	Count int `json:"count"`
}

type Rlimit struct {
	Type int    `json:"type"`
	Hard uint64 `json:"hard"`
	Soft uint64 `json:"soft"`
}

type HugepageLimit struct {
	Pagesize string `json:"pageSize"`
	Limit    int    `json:"limit"`
}

type IfPrioMap struct {
	Interface string `json:"interface"`
	Priority  int64  `json:"priority"`
}

type Resources struct {
	// Memory reservation or soft_limit (in bytes)
	MemoryReservation int64 `json:"memoryReservation"`
	// Total memory usage (memory + swap); set `-1' to disable swap
	MemorySwap int64 `json:"memorySwap"`
	// Kernel memory limit (in bytes)
	KernelMemory int64 `json:"kernelMemory"`
	// CPU shares (relative weight vs. other containers)
	CpuShares int64 `json:"cpuShares"`
	// CPU hardcap limit (in usecs). Allowed cpu time in a given period.
	CpuQuota int64 `json:"cpuQuota"`
	// CPU period to be used for hardcapping (in usecs). 0 to use system default.
	CpuPeriod int64 `json:"cpuPeriod"`
	// How many time CPU will use in realtime scheduling (in usecs).
	CpuRtRuntime int64 `json:"cpuQuota"`
	// CPU period to be used for realtime scheduling (in usecs).
	CpuRtPeriod int64 `json:"cpuPeriod"`
	// CPU to use
	CpusetCpus string `json:"cpusetCpus"`
	// MEM to use
	CpusetMems string `json:"cpusetMems"`
	// IO read rate limit per cgroup per device, bytes per second.
	BlkioThrottleReadBpsDevice string `json:"blkioThrottleReadBpsDevice"`
	// IO write rate limit per cgroup per divice, bytes per second.
	BlkioThrottleWriteBpsDevice string `json:"blkioThrottleWriteBpsDevice"`
	// IO read rate limit per cgroup per device, IO per second.
	BlkioThrottleReadIOpsDevice string `json:"blkioThrottleReadIopsDevice"`
	// IO write rate limit per cgroup per device, IO per second.
	BlkioThrottleWriteIOpsDevice string `json:"blkioThrottleWriteIopsDevice"`
	// Specifies per cgroup weight, range is from 10 to 1000.
	BlkioWeight int64 `json:"blkioWeight"`
	// Weight per cgroup per device, can override BlkioWeight.
	BlkioWeightDevice string `json:"blkioWeightDevice"`
	// Hugetlb limit (in bytes)
	HugetlbLimit []*HugepageLimit `json:"hugetlbLimit"`
	// Whether to disable OOM Killer
	DisableOOMKiller bool `json:"disableOOMKiller"`
	// Set priority of network traffic for container
	NetPrioIfpriomap []*IfPrioMap `json:"netPrioIfpriomap"`
	// Set class identifier for container's network packets
	NetClsClassid string `json:"netClsClassid"`
}

type LinuxSpec struct {
	PortableSpec
	UserMapping      map[string]UserMapping `json:"userMapping"`
	Rlimits          []Rlimit               `json:"rlimits"`
	SystemProperties map[string]string      `json:"systemProperties"`
	Resources        *Resources             `json:"resources"`
}

var namespaceMapping = map[string]configs.NamespaceType{
	"process": configs.NEWPID,
	"network": configs.NEWNET,
	"mount":   configs.NEWNS,
	"user":    configs.NEWUSER,
	"ipc":     configs.NEWIPC,
	"uts":     configs.NEWUTS,
}

// loadSpec loads the specification from the provided path.
// If the path is empty then the default path will be "container.json"
func loadSpec(path string) (*LinuxSpec, error) {
	if path == "" {
		path = "container.json"
	}
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("JSON specification file for %s not found", path)
		}
		return nil, err
	}
	defer f.Close()
	var s *LinuxSpec
	if err := json.NewDecoder(f).Decode(&s); err != nil {
		return nil, err
	}
	return s, nil
}

func createLibcontainerConfig(spec *LinuxSpec) (*configs.Config, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	config := &configs.Config{
		Capabilities: spec.Capabilities,
		Rootfs:       filepath.Join(cwd, spec.Root.Path),
		Readonlyfs:   spec.Root.Readonly,
		Hostname:     spec.Hostname,
		Privatefs:    true,
	}
	for _, ns := range spec.Namespaces {
		t, exists := namespaceMapping[ns.Type]
		if !exists {
			return nil, fmt.Errorf("namespace %q does not exist", ns)
		}
		config.Namespaces.Add(t, ns.Path)
	}
	for _, m := range spec.Mounts {
		config.Mounts = append(config.Mounts, createLibcontianerMount(cwd, m))
	}
	if err := createDevices(spec, config); err != nil {
		return nil, err
	}
	if err := setupUserNamespace(spec, config); err != nil {
		return nil, err
	}
	c, err := createCgroupConfig(spec, config.Devices)
	if err != nil {
		return nil, err
	}
	config.Cgroups = c
	if config.Readonlyfs {
		setReadonly(config)
		config.MaskPaths = []string{
			"/proc/kcore",
		}
		config.ReadonlyPaths = []string{
			"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
		}
	}
	return config, nil
}

func createLibcontianerMount(cwd string, m Mount) *configs.Mount {
	flags, data := parseMountOptions(m.Options)
	source := m.Source
	if m.Type == "bind" {
		if !filepath.IsAbs(source) {
			source = filepath.Join(cwd, m.Source)
		}
	}
	return &configs.Mount{
		Device:      m.Type,
		Source:      source,
		Destination: m.Destination,
		Data:        data,
		Flags:       flags,
	}
}

func createCgroupConfig(spec *LinuxSpec, devices []*configs.Device) (*configs.Cgroup, error) {
	myCgroupPath, err := cgroups.GetThisCgroupDir("devices")
	if err != nil {
		return nil, err
	}
	c := &configs.Cgroup{
		Name:             getDefaultID(),
		Parent:           myCgroupPath,
		AllowedDevices:   append(devices, allowedDevices...),
		CpuQuota:         getCPUQuota(spec.Cpus),
		Memory:           spec.Memory * 1024 * 1024,
		MemorySwap:       -1,
		MemorySwappiness: -1,
	}
	if r := spec.Resources; r != nil {
		c.MemoryReservation = r.MemoryReservation
		c.MemorySwap = r.MemorySwap
		c.KernelMemory = r.KernelMemory
		c.CpuShares = r.CpuShares
		c.CpuQuota = r.CpuQuota
		c.CpuPeriod = r.CpuPeriod
		c.CpuRtRuntime = r.CpuRtRuntime
		c.CpuRtPeriod = r.CpuRtPeriod
		c.CpusetCpus = r.CpusetCpus
		c.CpusetMems = r.CpusetMems
		c.BlkioThrottleReadBpsDevice = r.BlkioThrottleReadBpsDevice
		c.BlkioThrottleWriteBpsDevice = r.BlkioThrottleWriteBpsDevice
		c.BlkioThrottleReadIOpsDevice = r.BlkioThrottleReadIOpsDevice
		c.BlkioThrottleWriteIOpsDevice = r.BlkioThrottleWriteIOpsDevice
		c.BlkioWeight = r.BlkioWeight
		c.BlkioWeightDevice = r.BlkioWeightDevice
		for _, l := range r.HugetlbLimit {
			c.HugetlbLimit = append(c.HugetlbLimit, &configs.HugepageLimit{
				Pagesize: l.Pagesize,
				Limit:    l.Limit,
			})
		}
		c.OomKillDisable = r.DisableOOMKiller
		for _, m := range r.NetPrioIfpriomap {
			c.NetPrioIfpriomap = append(c.NetPrioIfpriomap, &configs.IfPrioMap{
				Interface: m.Interface,
				Priority:  m.Priority,
			})
		}
		c.NetClsClassid = r.NetClsClassid
	}
	return c, nil
}

func createDevices(spec *LinuxSpec, config *configs.Config) error {
	for _, name := range spec.Devices {
		d, err := devices.DeviceFromPath(filepath.Join("/dev", name), "rwm")
		if err != nil {
			return err
		}
		config.Devices = append(config.Devices, d)
	}
	return nil
}

func setReadonly(config *configs.Config) {
	for _, m := range config.Mounts {
		if m.Device == "sysfs" {
			m.Flags |= syscall.MS_RDONLY
		}
	}
}

func getCPUQuota(cpus float64) int64 {
	return int64(cpus * cpuQuotaMultiplyer)
}

func setupUserNamespace(spec *LinuxSpec, config *configs.Config) error {
	if len(spec.UserMapping) == 0 {
		return nil
	}
	config.Namespaces.Add(configs.NEWUSER, "")
	mappings := make(map[string][]configs.IDMap)
	for k, v := range spec.UserMapping {
		mappings[k] = append(mappings[k], configs.IDMap{
			ContainerID: v.From,
			HostID:      v.To,
			Size:        v.Count,
		})
	}
	config.UidMappings = mappings["uid"]
	config.GidMappings = mappings["gid"]
	rootUid, err := config.HostUID()
	if err != nil {
		return err
	}
	rootGid, err := config.HostGID()
	if err != nil {
		return err
	}
	for _, node := range config.Devices {
		node.Uid = uint32(rootUid)
		node.Gid = uint32(rootGid)
	}
	return nil
}

// parseMountOptions parses the string and returns the flags and any mount data that
// it contains.
func parseMountOptions(options string) (int, string) {
	var (
		flag int
		data []string
	)
	flags := map[string]struct {
		clear bool
		flag  int
	}{
		"defaults":      {false, 0},
		"ro":            {false, syscall.MS_RDONLY},
		"rw":            {true, syscall.MS_RDONLY},
		"suid":          {true, syscall.MS_NOSUID},
		"nosuid":        {false, syscall.MS_NOSUID},
		"dev":           {true, syscall.MS_NODEV},
		"nodev":         {false, syscall.MS_NODEV},
		"exec":          {true, syscall.MS_NOEXEC},
		"noexec":        {false, syscall.MS_NOEXEC},
		"sync":          {false, syscall.MS_SYNCHRONOUS},
		"async":         {true, syscall.MS_SYNCHRONOUS},
		"dirsync":       {false, syscall.MS_DIRSYNC},
		"remount":       {false, syscall.MS_REMOUNT},
		"mand":          {false, syscall.MS_MANDLOCK},
		"nomand":        {true, syscall.MS_MANDLOCK},
		"atime":         {true, syscall.MS_NOATIME},
		"noatime":       {false, syscall.MS_NOATIME},
		"diratime":      {true, syscall.MS_NODIRATIME},
		"nodiratime":    {false, syscall.MS_NODIRATIME},
		"bind":          {false, syscall.MS_BIND},
		"rbind":         {false, syscall.MS_BIND | syscall.MS_REC},
		"unbindable":    {false, syscall.MS_UNBINDABLE},
		"runbindable":   {false, syscall.MS_UNBINDABLE | syscall.MS_REC},
		"private":       {false, syscall.MS_PRIVATE},
		"rprivate":      {false, syscall.MS_PRIVATE | syscall.MS_REC},
		"shared":        {false, syscall.MS_SHARED},
		"rshared":       {false, syscall.MS_SHARED | syscall.MS_REC},
		"slave":         {false, syscall.MS_SLAVE},
		"rslave":        {false, syscall.MS_SLAVE | syscall.MS_REC},
		"relatime":      {false, syscall.MS_RELATIME},
		"norelatime":    {true, syscall.MS_RELATIME},
		"strictatime":   {false, syscall.MS_STRICTATIME},
		"nostrictatime": {true, syscall.MS_STRICTATIME},
	}
	for _, o := range strings.Split(options, ",") {
		// If the option does not exist in the flags table or the flag
		// is not supported on the platform,
		// then it is a data value for a specific fs type
		if f, exists := flags[o]; exists && f.flag != 0 {
			if f.clear {
				flag &= ^f.flag
			} else {
				flag |= f.flag
			}
		} else {
			data = append(data, o)
		}
	}
	return flag, strings.Join(data, ",")
}
<|endoftext|>


<commit_before>// +build appengine

package datastore

import (
	"fmt"
	"reflect"
	"strings"

	"gnd.la/config"
	"gnd.la/log"
	"gnd.la/orm/driver"
	"gnd.la/orm/operation"
	"gnd.la/orm/query"
	"gnd.la/util/types"

	"appengine"
	"appengine/datastore"
)

type Driver struct {
	c      appengine.Context
	logger *log.Logger
}

func (d *Driver) Check() error {
	return nil
}

func (d *Driver) Initialize(ms []driver.Model) error {
	// No need to create tables in the datastore. Instead,
	// check that the models can be stored.
	return nil
}

func (d *Driver) Query(m driver.Model, q query.Q, sort []driver.Sort, limit int, offset int) driver.Iter {
	dq, err := d.makeQuery(m, q, sort, limit, offset)
	if err != nil {
		return &Iter{err: err}
	}
	return &Iter{iter: dq.Run(d.c)}
}

func (d *Driver) Count(m driver.Model, q query.Q, limit int, offset int) (uint64, error) {
	dq, err := d.makeQuery(m, q, nil, limit, offset)
	if err != nil {
		return 0, err
	}
	c, err := dq.Count(d.c)
	return uint64(c), err
}

func (d *Driver) Exists(m driver.Model, q query.Q) (bool, error) {
	dq, err := d.makeQuery(m, q, nil, 1, -1)
	if err != nil {
		return false, err
	}
	c, err := dq.Count(d.c)
	return c != 0, err
}

func (d *Driver) Insert(m driver.Model, data interface{}) (driver.Result, error) {
	var id int64
	fields := m.Fields()
	var pkVal *reflect.Value
	// TODO: If the PK is supplied by the user rather than auto-assigned, it
	// might conflict with PKs generated by datastore.AllocateIDs().
	if fields.PrimaryKey >= 0 {
		p := d.primaryKey(fields, data)
		if p.IsValid() && types.Kind(p.Kind()) == types.Int {
			id = p.Int()
			if id == 0 {
				// Must assign PK field value after calling AllocateIDs
				pkVal = &p
			}
		}
	}
	name := m.Table()
	// Make all objects of a given kind ancestors of the same key. While
	// this hurts scalability, it makes all reads strongly consistent.
	parent := d.parentKey(m)
	var err error
	if id == 0 {
		id, _, err = datastore.AllocateIDs(d.c, name, parent, 1)
		if err != nil {
			return nil, err
		}
	}
	if fields.AutoincrementPk && pkVal != nil {
		pkVal.SetInt(int64(id))
	}
	key := datastore.NewKey(d.c, name, "", id, parent)
	log.Debugf("DATASTORE: put %s %v", key, data)
	_, err = datastore.Put(d.c, key, data)
	if err != nil {
		return nil, err
	}
	return &result{key: key, count: 1}, nil
}

func (d *Driver) Operate(m driver.Model, q query.Q, ops []*operation.Operation) (driver.Result, error) {
	return nil, fmt.Errorf("datastore driver does not support Operate")
}

func (d *Driver) Update(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	src := make([]interface{}, len(keys))
	for ii := range src {
		src[ii] = data
	}
	// Multi variants need to be run in transactions, otherwise some
	// might fail while others succeed
	err = datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		_, e := datastore.PutMulti(c, keys, src)
		return e
	}, nil)
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) Upsert(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	return nil, nil
}

func (d *Driver) Delete(m driver.Model, q query.Q) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	// See comment around PutMulti
	err = datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		return datastore.DeleteMulti(c, keys)
	}, nil)
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) getKeys(m driver.Model, q query.Q) ([]*datastore.Key, error) {
	dq, err := d.makeQuery(m, q, nil, -1, -1)
	if err != nil {
		return nil, err
	}
	iter := dq.KeysOnly().Run(d.c)
	var keys []*datastore.Key
	for {
		key, err := iter.Next(nil)
		if err != nil {
			if err == datastore.Done {
				break
			}
			return nil, err
		}
		keys = append(keys, key)
	}
	return keys, nil
}

func (d *Driver) makeQuery(m driver.Model, q query.Q, sort []driver.Sort, limit int, offset int) (*datastore.Query, error) {
	if m.Join() != nil {
		return nil, errJoinNotSupported
	}
	dq := datastore.NewQuery(m.Table()).Ancestor(d.parentKey(m))
	var err error
	if dq, err = d.applyQuery(m, dq, q); err != nil {
		return nil, err
	}
	for _, v := range sort {
		field := v.Field()
		if v.Direction() == driver.DESC {
			field = "-" + field
		}
		dq = dq.Order(field)
	}
	if limit >= 0 {
		dq = dq.Limit(limit)
	}
	if offset > 0 {
		dq = dq.Offset(limit)
	}
	return dq, nil
}

func (d *Driver) parentKey(m driver.Model) *datastore.Key {
	return datastore.NewKey(d.c, m.Table(), "", -1, nil)
}

func (d *Driver) applyQuery(m driver.Model, dq *datastore.Query, q query.Q) (*datastore.Query, error) {
	var field *query.Field
	var op string
	switch x := q.(type) {
	case *query.Eq:
		field = &x.Field
		op = " ="
	case *query.Lt:
		field = &x.Field
		op = " <"
	case *query.Lte:
		field = &x.Field
		op = " <="
	case *query.Gt:
		field = &x.Field
		op = " >"
	case *query.Gte:
		field = &x.Field
		op = " >="
	case *query.And:
		var err error
		for _, v := range x.Conditions {
			dq, err = d.applyQuery(m, dq, v)
			if err != nil {
				return nil, err
			}
		}
	case nil:
	default:
		return nil, fmt.Errorf("datastore does not support %T queries", q)
	}
	if field != nil {
		if _, ok := field.Value.(query.F); ok {
			return nil, fmt.Errorf("datastore queries can't reference other properties (%v)", field.Value)
		}
		name := field.Field
		fields := m.Fields()
		idx, ok := fields.QNameMap[name]
		if !ok {
			return nil, fmt.Errorf("can't map field %q to a datastore name", name)
		}
		if strings.IndexByte(name, '.') >= 0 {
			// GAE flattens embedded fields, so we must remove
			// the parts of the field which refer to a flattened
			// field.
			indexes := fields.Indexes[idx]
			parts := strings.Split(name, ".")
			if len(indexes) == len(parts) {
				var final []string
				typ := fields.Type
				for ii, v := range indexes {
					f := typ.Field(v)
					if !f.Anonymous {
						final = append(final, parts[ii])
					}
					typ = f.Type
				}
				name = strings.Join(final, ".")
			}
		}
		log.Debugf("DATASTORE: filter %s %s %v", m, name+op, field.Value)
		dq = dq.Filter(name+op, field.Value)
	}
	return dq, nil
}

func (d *Driver) Close() error {
	return nil
}

func (d *Driver) Upserts() bool {
	return false
}

func (d *Driver) Tags() []string {
	return []string{"datastore"}
}

func (d *Driver) SetLogger(logger *log.Logger) {
	d.logger = logger
}

func (d *Driver) SetContext(ctx appengine.Context) {
	d.c = ctx
}

func (d *Driver) Begin() (driver.Tx, error) {
	return nil, errTransactionNotSupported
}

func (d *Driver) Commit() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) Rollback() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) Transaction(f func(driver.Driver) error) error {
	return datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		drv := *d
		drv.c = c
		return f(&drv)
	}, nil)
}

func (d *Driver) Capabilities() driver.Capability {
	return driver.CAP_TRANSACTION | driver.CAP_AUTO_ID | driver.CAP_EVENTUAL | driver.CAP_PK
}

func (d *Driver) primaryKey(f *driver.Fields, data interface{}) reflect.Value {
	return fieldByIndex(reflect.ValueOf(data), f.Indexes[f.PrimaryKey])
}

func (d *Driver) HasFunc(fname string, retType reflect.Type) bool {
	return false
}

func (d *Driver) Connection() interface{} {
	return d.c
}

func fieldByIndex(val reflect.Value, indexes []int) reflect.Value {
	for _, v := range indexes {
		if val.Type().Kind() == reflect.Ptr {
			if val.IsNil() {
				return reflect.Value{}
			}
			val = val.Elem()
		}
		val = val.Field(v)
	}
	return val
}

func datastoreOpener(url *config.URL) (driver.Driver, error) {
	return &Driver{}, nil
}

func init() {
	driver.Register("datastore", datastoreOpener)
}
<commit_msg>Datastore: Make all transactions cross-group<commit_after>// +build appengine

package datastore

import (
	"fmt"
	"reflect"
	"strings"

	"gnd.la/config"
	"gnd.la/log"
	"gnd.la/orm/driver"
	"gnd.la/orm/operation"
	"gnd.la/orm/query"
	"gnd.la/util/types"

	"appengine"
	"appengine/datastore"
)

type Driver struct {
	c      appengine.Context
	logger *log.Logger
}

func (d *Driver) Check() error {
	return nil
}

func (d *Driver) Initialize(ms []driver.Model) error {
	// No need to create tables in the datastore. Instead,
	// check that the models can be stored.
	return nil
}

func (d *Driver) Query(m driver.Model, q query.Q, sort []driver.Sort, limit int, offset int) driver.Iter {
	dq, err := d.makeQuery(m, q, sort, limit, offset)
	if err != nil {
		return &Iter{err: err}
	}
	return &Iter{iter: dq.Run(d.c)}
}

func (d *Driver) Count(m driver.Model, q query.Q, limit int, offset int) (uint64, error) {
	dq, err := d.makeQuery(m, q, nil, limit, offset)
	if err != nil {
		return 0, err
	}
	c, err := dq.Count(d.c)
	return uint64(c), err
}

func (d *Driver) Exists(m driver.Model, q query.Q) (bool, error) {
	dq, err := d.makeQuery(m, q, nil, 1, -1)
	if err != nil {
		return false, err
	}
	c, err := dq.Count(d.c)
	return c != 0, err
}

func (d *Driver) Insert(m driver.Model, data interface{}) (driver.Result, error) {
	var id int64
	fields := m.Fields()
	var pkVal *reflect.Value
	// TODO: If the PK is supplied by the user rather than auto-assigned, it
	// might conflict with PKs generated by datastore.AllocateIDs().
	if fields.PrimaryKey >= 0 {
		p := d.primaryKey(fields, data)
		if p.IsValid() && types.Kind(p.Kind()) == types.Int {
			id = p.Int()
			if id == 0 {
				// Must assign PK field value after calling AllocateIDs
				pkVal = &p
			}
		}
	}
	name := m.Table()
	// Make all objects of a given kind ancestors of the same key. While
	// this hurts scalability, it makes all reads strongly consistent.
	parent := d.parentKey(m)
	var err error
	if id == 0 {
		id, _, err = datastore.AllocateIDs(d.c, name, parent, 1)
		if err != nil {
			return nil, err
		}
	}
	if fields.AutoincrementPk && pkVal != nil {
		pkVal.SetInt(int64(id))
	}
	key := datastore.NewKey(d.c, name, "", id, parent)
	log.Debugf("DATASTORE: put %s %v", key, data)
	_, err = datastore.Put(d.c, key, data)
	if err != nil {
		return nil, err
	}
	return &result{key: key, count: 1}, nil
}

func (d *Driver) Operate(m driver.Model, q query.Q, ops []*operation.Operation) (driver.Result, error) {
	return nil, fmt.Errorf("datastore driver does not support Operate")
}

func (d *Driver) Update(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	src := make([]interface{}, len(keys))
	for ii := range src {
		src[ii] = data
	}
	// Multi variants need to be run in transactions, otherwise some
	// might fail while others succeed
	err = datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		_, e := datastore.PutMulti(c, keys, src)
		return e
	}, nil)
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) Upsert(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	return nil, nil
}

func (d *Driver) Delete(m driver.Model, q query.Q) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	// See comment around PutMulti
	err = datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		return datastore.DeleteMulti(c, keys)
	}, nil)
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) getKeys(m driver.Model, q query.Q) ([]*datastore.Key, error) {
	dq, err := d.makeQuery(m, q, nil, -1, -1)
	if err != nil {
		return nil, err
	}
	iter := dq.KeysOnly().Run(d.c)
	var keys []*datastore.Key
	for {
		key, err := iter.Next(nil)
		if err != nil {
			if err == datastore.Done {
				break
			}
			return nil, err
		}
		keys = append(keys, key)
	}
	return keys, nil
}

func (d *Driver) makeQuery(m driver.Model, q query.Q, sort []driver.Sort, limit int, offset int) (*datastore.Query, error) {
	if m.Join() != nil {
		return nil, errJoinNotSupported
	}
	dq := datastore.NewQuery(m.Table()).Ancestor(d.parentKey(m))
	var err error
	if dq, err = d.applyQuery(m, dq, q); err != nil {
		return nil, err
	}
	for _, v := range sort {
		field := v.Field()
		if v.Direction() == driver.DESC {
			field = "-" + field
		}
		dq = dq.Order(field)
	}
	if limit >= 0 {
		dq = dq.Limit(limit)
	}
	if offset > 0 {
		dq = dq.Offset(limit)
	}
	return dq, nil
}

func (d *Driver) parentKey(m driver.Model) *datastore.Key {
	return datastore.NewKey(d.c, m.Table(), "", -1, nil)
}

func (d *Driver) applyQuery(m driver.Model, dq *datastore.Query, q query.Q) (*datastore.Query, error) {
	var field *query.Field
	var op string
	switch x := q.(type) {
	case *query.Eq:
		field = &x.Field
		op = " ="
	case *query.Lt:
		field = &x.Field
		op = " <"
	case *query.Lte:
		field = &x.Field
		op = " <="
	case *query.Gt:
		field = &x.Field
		op = " >"
	case *query.Gte:
		field = &x.Field
		op = " >="
	case *query.And:
		var err error
		for _, v := range x.Conditions {
			dq, err = d.applyQuery(m, dq, v)
			if err != nil {
				return nil, err
			}
		}
	case nil:
	default:
		return nil, fmt.Errorf("datastore does not support %T queries", q)
	}
	if field != nil {
		if _, ok := field.Value.(query.F); ok {
			return nil, fmt.Errorf("datastore queries can't reference other properties (%v)", field.Value)
		}
		name := field.Field
		fields := m.Fields()
		idx, ok := fields.QNameMap[name]
		if !ok {
			return nil, fmt.Errorf("can't map field %q to a datastore name", name)
		}
		if strings.IndexByte(name, '.') >= 0 {
			// GAE flattens embedded fields, so we must remove
			// the parts of the field which refer to a flattened
			// field.
			indexes := fields.Indexes[idx]
			parts := strings.Split(name, ".")
			if len(indexes) == len(parts) {
				var final []string
				typ := fields.Type
				for ii, v := range indexes {
					f := typ.Field(v)
					if !f.Anonymous {
						final = append(final, parts[ii])
					}
					typ = f.Type
				}
				name = strings.Join(final, ".")
			}
		}
		log.Debugf("DATASTORE: filter %s %s %v", m, name+op, field.Value)
		dq = dq.Filter(name+op, field.Value)
	}
	return dq, nil
}

func (d *Driver) Close() error {
	return nil
}

func (d *Driver) Upserts() bool {
	return false
}

func (d *Driver) Tags() []string {
	return []string{"datastore"}
}

func (d *Driver) SetLogger(logger *log.Logger) {
	d.logger = logger
}

func (d *Driver) SetContext(ctx appengine.Context) {
	d.c = ctx
}

func (d *Driver) Begin() (driver.Tx, error) {
	return nil, errTransactionNotSupported
}

func (d *Driver) Commit() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) Rollback() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) Transaction(f func(driver.Driver) error) error {
	// While not optimal, it's valid to request a cross group transaction
	// when the transaction affects a single entity group. Since the ORM
	// framework does not have an option for specifying if the transaction
	// will affect multiple entities (and it's not clear if we should have
	// it, since the datastore driver would be the only one using it), for
	// now we have to make all transactions XG.
	return datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		drv := *d
		drv.c = c
		return f(&drv)
	}, &datastore.TransactionOptions{XG: true})
}

func (d *Driver) Capabilities() driver.Capability {
	return driver.CAP_TRANSACTION | driver.CAP_AUTO_ID | driver.CAP_EVENTUAL | driver.CAP_PK
}

func (d *Driver) primaryKey(f *driver.Fields, data interface{}) reflect.Value {
	return fieldByIndex(reflect.ValueOf(data), f.Indexes[f.PrimaryKey])
}

func (d *Driver) HasFunc(fname string, retType reflect.Type) bool {
	return false
}

func (d *Driver) Connection() interface{} {
	return d.c
}

func fieldByIndex(val reflect.Value, indexes []int) reflect.Value {
	for _, v := range indexes {
		if val.Type().Kind() == reflect.Ptr {
			if val.IsNil() {
				return reflect.Value{}
			}
			val = val.Elem()
		}
		val = val.Field(v)
	}
	return val
}

func datastoreOpener(url *config.URL) (driver.Driver, error) {
	return &Driver{}, nil
}

func init() {
	driver.Register("datastore", datastoreOpener)
}
<|endoftext|>


<commit_before>package irckit

import (
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mattermost/platform/model"
	"github.com/sorcix/irc"
)

// Channel is a representation of a room in our server
type Channel interface {
	Prefixer

	// ID is a normalized unique identifier for the channel
	ID() string

	// Created returns the time when the Channel was created.
	Created() time.Time

	// Names returns a sorted slice of Nicks in the channel
	Names() []string

	// Users returns a slice of Users in the channel.
	Users() []*User

	// HasUser returns whether a User is in the channel.
	HasUser(*User) bool

	// Invite prompts the User to join the Channel on behalf of Prefixer.
	Invite(from Prefixer, u *User) error

	// Join introduces the User to the channel (handler for JOIN).
	Join(u *User) error

	// Part removes the User from the channel (handler for PART).
	Part(u *User, text string)

	// Message transmits a message from a User to the channel (handler for PRIVMSG).
	Message(u *User, text string)

	// Service returns the service that set the channel
	Service() string

	// Topic sets the topic of the channel (handler for TOPIC).
	Topic(from Prefixer, text string)

	// GetTopic gets the topic of the channel
	GetTopic() string

	// Unlink will disassociate the Channel from its Server.
	Unlink()

	// Len returns the number of Users in the channel.
	Len() int

	// String returns the name of the channel
	String() string

	// Spoof message
	SpoofMessage(from string, text string)

	// Spoof notice
	SpoofNotice(from string, text string)
}

type channel struct {
	created time.Time
	name    string
	server  Server
	id      string
	service string

	mu       sync.RWMutex
	topic    string
	usersIdx map[*User]struct{}
}

// NewChannel returns a Channel implementation for a given Server.
func NewChannel(server Server, channelId string, name string, service string) Channel {
	return &channel{
		created:  time.Now(),
		server:   server,
		id:       channelId,
		name:     name,
		service:  service,
		usersIdx: map[*User]struct{}{},
	}
}

func (ch *channel) GetTopic() string {
	return ch.topic
}

func (ch *channel) Prefix() *irc.Prefix {
	return ch.server.Prefix()
}

func (ch *channel) Service() string {
	return ch.service
}

func (ch *channel) String() string {
	return ch.name
}

// Created returns the time when the Channel was created.
func (ch *channel) Created() time.Time {
	return ch.created
}

// ID returns a normalized unique identifier for the channel.
func (ch *channel) ID() string {
	return ID(ch.id)
}

func (ch *channel) Message(from *User, text string) {
	for len(text) > 400 {
		msg := &irc.Message{
			Prefix:   from.Prefix(),
			Command:  irc.PRIVMSG,
			Params:   []string{ch.name},
			Trailing: text[:400] + "\n",
		}
		ch.mu.RLock()
		for to := range ch.usersIdx {
			to.Encode(msg)
		}
		ch.mu.RUnlock()
		text = text[400:]
	}
	msg := &irc.Message{
		Prefix:   from.Prefix(),
		Command:  irc.PRIVMSG,
		Params:   []string{ch.name},
		Trailing: text,
	}
	ch.mu.RLock()
	for to := range ch.usersIdx {
		to.Encode(msg)
	}
	ch.mu.RUnlock()
}

// Quit will remove the user from the channel and emit a PART message.
func (ch *channel) Part(u *User, text string) {
	msg := &irc.Message{
		Prefix:   u.Prefix(),
		Command:  irc.PART,
		Params:   []string{ch.name},
		Trailing: text,
	}
	ch.mu.Lock()
	if _, ok := ch.usersIdx[u]; !ok {
		ch.mu.Unlock()
		u.Encode(&irc.Message{
			Prefix:   ch.Prefix(),
			Command:  irc.ERR_NOTONCHANNEL,
			Params:   []string{ch.name},
			Trailing: "You're not on that channel",
		})
		return
	}
	u.Encode(msg)
	delete(ch.usersIdx, u)
	ch.mu.Unlock()
	u.Lock()
	delete(u.channels, ch)
	u.Unlock()
}

// Unlink will disassociate the Channel from the Server.
func (ch *channel) Unlink() {
	ch.server.UnlinkChannel(ch)
}

// Close will evict all users in the channel.
func (ch *channel) Close() error {
	ch.mu.Lock()
	for to := range ch.usersIdx {
		to.Encode(&irc.Message{
			Prefix:  to.Prefix(),
			Command: irc.PART,
			Params:  []string{ch.name},
		})
	}
	ch.usersIdx = map[*User]struct{}{}
	ch.mu.Unlock()
	return nil
}

// Invite prompts the User to join the Channel on behalf of Prefixer.
func (ch *channel) Invite(from Prefixer, u *User) error {
	return u.Encode(&irc.Message{
		Prefix:  from.Prefix(),
		Command: irc.INVITE,
		Params:  []string{u.Nick, ch.name},
	})
	// TODO: Save state that the user is invited?
}

// Topic sets the topic of the channel (handler for TOPIC).
func (ch *channel) Topic(from Prefixer, text string) {
	ch.mu.RLock()
	ch.topic = text
	// no newlines in topic
	ch.topic = strings.Replace(ch.topic, "\n", " ", -1)

	msg := &irc.Message{
		Prefix:   from.Prefix(),
		Command:  irc.TOPIC,
		Params:   []string{ch.name},
		Trailing: ch.topic,
	}
	for to := range ch.usersIdx {
		to.Encode(msg)
	}

	ch.mu.RUnlock()
}

// Join introduces the User to the channel (sends relevant messages, stores).
func (ch *channel) Join(u *User) error {
	// TODO: Check if user is already here?
	ch.mu.Lock()
	if _, exists := ch.usersIdx[u]; exists {
		ch.mu.Unlock()
		return nil
	}
	topic := ch.topic
	ch.usersIdx[u] = struct{}{}
	ch.mu.Unlock()
	u.Lock()
	u.channels[ch] = struct{}{}
	u.Unlock()

	// speed-up & users join
	if ch.name == "&users" && u.MmGhostUser {
		return nil
	}

	msg := &irc.Message{
		Prefix:  u.Prefix(),
		Command: irc.JOIN,
		Params:  []string{ch.name},
	}

	// only send join messages to real users
	for to := range ch.usersIdx {
		if to.MmGhostUser == false {
			to.Encode(msg)
		}
	}

	msgs := []*irc.Message{}
	if topic != "" {
		msgs = append(msgs, &irc.Message{
			Prefix:   ch.Prefix(),
			Command:  irc.RPL_TOPIC,
			Params:   []string{u.Nick, ch.name},
			Trailing: topic,
		})
	}

	line := ""
	i := 0
	for _, name := range ch.Names() {
		if i+len(name) < 400 {
			line += name + " "
			i += len(name)
		} else {
			msgs = append(msgs, &irc.Message{
				Prefix:   ch.Prefix(),
				Command:  irc.RPL_NAMREPLY,
				Params:   []string{u.Nick, "=", ch.name},
				Trailing: line,
			})
			line = ""
			line += name + " "
			i = len(name)
		}
	}
	msgs = append(msgs, &irc.Message{
		Prefix:   ch.Prefix(),
		Command:  irc.RPL_NAMREPLY,
		Params:   []string{u.Nick, "=", ch.name},
		Trailing: line,
	})

	msgs = append(msgs, &irc.Message{
		Prefix:   ch.Prefix(),
		Params:   []string{u.Nick, ch.name},
		Command:  irc.RPL_ENDOFNAMES,
		Trailing: "End of /NAMES list.",
	},
	)

	return u.Encode(msgs...)
}

func (ch *channel) HasUser(u *User) bool {
	ch.mu.RLock()
	_, ok := ch.usersIdx[u]
	ch.mu.RUnlock()
	return ok
}

// Users returns an unsorted slice of users who are in the channel.
func (ch *channel) Users() []*User {
	ch.mu.RLock()
	users := make([]*User, 0, len(ch.usersIdx))
	for u := range ch.usersIdx {
		users = append(users, u)
	}
	ch.mu.RUnlock()
	return users
}

// Names returns a sorted slice of Nick strings of users who are in the channel.
func (ch *channel) Names() []string {
	users := ch.Users()
	names := make([]string, 0, len(users))
	for _, u := range users {
		if strings.Contains(u.Roles, model.SYSTEM_ADMIN_ROLE_ID) {
			names = append(names, "@"+u.Nick)
		} else {
			names = append(names, u.Nick)
		}
	}
	// TODO: Append in sorted order?
	sort.Strings(names)
	return names
}

// Len returns the number of users in the channel.
func (ch *channel) Len() int {
	ch.mu.RLock()
	defer ch.mu.RUnlock()
	return len(ch.usersIdx)
}
<commit_msg>Sent messages to real users only<commit_after>package irckit

import (
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/mattermost/platform/model"
	"github.com/sorcix/irc"
)

// Channel is a representation of a room in our server
type Channel interface {
	Prefixer

	// ID is a normalized unique identifier for the channel
	ID() string

	// Created returns the time when the Channel was created.
	Created() time.Time

	// Names returns a sorted slice of Nicks in the channel
	Names() []string

	// Users returns a slice of Users in the channel.
	Users() []*User

	// HasUser returns whether a User is in the channel.
	HasUser(*User) bool

	// Invite prompts the User to join the Channel on behalf of Prefixer.
	Invite(from Prefixer, u *User) error

	// Join introduces the User to the channel (handler for JOIN).
	Join(u *User) error

	// Part removes the User from the channel (handler for PART).
	Part(u *User, text string)

	// Message transmits a message from a User to the channel (handler for PRIVMSG).
	Message(u *User, text string)

	// Service returns the service that set the channel
	Service() string

	// Topic sets the topic of the channel (handler for TOPIC).
	Topic(from Prefixer, text string)

	// GetTopic gets the topic of the channel
	GetTopic() string

	// Unlink will disassociate the Channel from its Server.
	Unlink()

	// Len returns the number of Users in the channel.
	Len() int

	// String returns the name of the channel
	String() string

	// Spoof message
	SpoofMessage(from string, text string)

	// Spoof notice
	SpoofNotice(from string, text string)
}

type channel struct {
	created time.Time
	name    string
	server  Server
	id      string
	service string

	mu       sync.RWMutex
	topic    string
	usersIdx map[*User]struct{}
}

// NewChannel returns a Channel implementation for a given Server.
func NewChannel(server Server, channelId string, name string, service string) Channel {
	return &channel{
		created:  time.Now(),
		server:   server,
		id:       channelId,
		name:     name,
		service:  service,
		usersIdx: map[*User]struct{}{},
	}
}

func (ch *channel) GetTopic() string {
	return ch.topic
}

func (ch *channel) Prefix() *irc.Prefix {
	return ch.server.Prefix()
}

func (ch *channel) Service() string {
	return ch.service
}

func (ch *channel) String() string {
	return ch.name
}

// Created returns the time when the Channel was created.
func (ch *channel) Created() time.Time {
	return ch.created
}

// ID returns a normalized unique identifier for the channel.
func (ch *channel) ID() string {
	return ID(ch.id)
}

func (ch *channel) Message(from *User, text string) {
	for len(text) > 400 {
		msg := &irc.Message{
			Prefix:   from.Prefix(),
			Command:  irc.PRIVMSG,
			Params:   []string{ch.name},
			Trailing: text[:400] + "\n",
		}
		ch.mu.RLock()
		for to := range ch.usersIdx {
			to.Encode(msg)
		}
		ch.mu.RUnlock()
		text = text[400:]
	}
	msg := &irc.Message{
		Prefix:   from.Prefix(),
		Command:  irc.PRIVMSG,
		Params:   []string{ch.name},
		Trailing: text,
	}
	ch.mu.RLock()
	for to := range ch.usersIdx {
		to.Encode(msg)
	}
	ch.mu.RUnlock()
}

// Quit will remove the user from the channel and emit a PART message.
func (ch *channel) Part(u *User, text string) {
	msg := &irc.Message{
		Prefix:   u.Prefix(),
		Command:  irc.PART,
		Params:   []string{ch.name},
		Trailing: text,
	}
	ch.mu.Lock()
	if _, ok := ch.usersIdx[u]; !ok {
		ch.mu.Unlock()
		u.Encode(&irc.Message{
			Prefix:   ch.Prefix(),
			Command:  irc.ERR_NOTONCHANNEL,
			Params:   []string{ch.name},
			Trailing: "You're not on that channel",
		})
		return
	}
	u.Encode(msg)
	delete(ch.usersIdx, u)
	ch.mu.Unlock()
	u.Lock()
	delete(u.channels, ch)
	u.Unlock()
}

// Unlink will disassociate the Channel from the Server.
func (ch *channel) Unlink() {
	ch.server.UnlinkChannel(ch)
}

// Close will evict all users in the channel.
func (ch *channel) Close() error {
	ch.mu.Lock()
	for to := range ch.usersIdx {
		to.Encode(&irc.Message{
			Prefix:  to.Prefix(),
			Command: irc.PART,
			Params:  []string{ch.name},
		})
	}
	ch.usersIdx = map[*User]struct{}{}
	ch.mu.Unlock()
	return nil
}

// Invite prompts the User to join the Channel on behalf of Prefixer.
func (ch *channel) Invite(from Prefixer, u *User) error {
	return u.Encode(&irc.Message{
		Prefix:  from.Prefix(),
		Command: irc.INVITE,
		Params:  []string{u.Nick, ch.name},
	})
	// TODO: Save state that the user is invited?
}

// Topic sets the topic of the channel (handler for TOPIC).
func (ch *channel) Topic(from Prefixer, text string) {
	ch.mu.RLock()
	ch.topic = text
	// no newlines in topic
	ch.topic = strings.Replace(ch.topic, "\n", " ", -1)

	msg := &irc.Message{
		Prefix:   from.Prefix(),
		Command:  irc.TOPIC,
		Params:   []string{ch.name},
		Trailing: ch.topic,
	}

	// only send join messages to real users
	for to := range ch.usersIdx {
		if to.MmGhostUser == false {
			to.Encode(msg)
		}
	}

	ch.mu.RUnlock()
}

// Join introduces the User to the channel (sends relevant messages, stores).
func (ch *channel) Join(u *User) error {
	// TODO: Check if user is already here?
	ch.mu.Lock()
	if _, exists := ch.usersIdx[u]; exists {
		ch.mu.Unlock()
		return nil
	}
	topic := ch.topic
	ch.usersIdx[u] = struct{}{}
	ch.mu.Unlock()
	u.Lock()
	u.channels[ch] = struct{}{}
	u.Unlock()

	// speed-up & users join
	if ch.name == "&users" && u.MmGhostUser {
		return nil
	}

	msg := &irc.Message{
		Prefix:  u.Prefix(),
		Command: irc.JOIN,
		Params:  []string{ch.name},
	}

	// only send join messages to real users
	for to := range ch.usersIdx {
		if to.MmGhostUser == false {
			to.Encode(msg)
		}
	}

	msgs := []*irc.Message{}
	if topic != "" {
		msgs = append(msgs, &irc.Message{
			Prefix:   ch.Prefix(),
			Command:  irc.RPL_TOPIC,
			Params:   []string{u.Nick, ch.name},
			Trailing: topic,
		})
	}

	line := ""
	i := 0
	for _, name := range ch.Names() {
		if i+len(name) < 400 {
			line += name + " "
			i += len(name)
		} else {
			msgs = append(msgs, &irc.Message{
				Prefix:   ch.Prefix(),
				Command:  irc.RPL_NAMREPLY,
				Params:   []string{u.Nick, "=", ch.name},
				Trailing: line,
			})
			line = ""
			line += name + " "
			i = len(name)
		}
	}
	msgs = append(msgs, &irc.Message{
		Prefix:   ch.Prefix(),
		Command:  irc.RPL_NAMREPLY,
		Params:   []string{u.Nick, "=", ch.name},
		Trailing: line,
	})

	msgs = append(msgs, &irc.Message{
		Prefix:   ch.Prefix(),
		Params:   []string{u.Nick, ch.name},
		Command:  irc.RPL_ENDOFNAMES,
		Trailing: "End of /NAMES list.",
	},
	)

	return u.Encode(msgs...)
}

func (ch *channel) HasUser(u *User) bool {
	ch.mu.RLock()
	_, ok := ch.usersIdx[u]
	ch.mu.RUnlock()
	return ok
}

// Users returns an unsorted slice of users who are in the channel.
func (ch *channel) Users() []*User {
	ch.mu.RLock()
	users := make([]*User, 0, len(ch.usersIdx))
	for u := range ch.usersIdx {
		users = append(users, u)
	}
	ch.mu.RUnlock()
	return users
}

// Names returns a sorted slice of Nick strings of users who are in the channel.
func (ch *channel) Names() []string {
	users := ch.Users()
	names := make([]string, 0, len(users))
	for _, u := range users {
		if strings.Contains(u.Roles, model.SYSTEM_ADMIN_ROLE_ID) {
			names = append(names, "@"+u.Nick)
		} else {
			names = append(names, u.Nick)
		}
	}
	// TODO: Append in sorted order?
	sort.Strings(names)
	return names
}

// Len returns the number of users in the channel.
func (ch *channel) Len() int {
	ch.mu.RLock()
	defer ch.mu.RUnlock()
	return len(ch.usersIdx)
}
<|endoftext|>


<commit_before>package gol

import (
	"fmt"
	"github.com/juju/errgo"
	"io"
	"io/ioutil"
	"log"
	"runtime"
)

type channel struct {
	writers []io.Writer
	mapper  mapper
	logger  *log.Logger
}

var channels = map[level]*channel{
	LEVEL_CRITICAL: &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "C ", log.LstdFlags)},
	LEVEL_ERROR:    &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "E ", log.LstdFlags)},
	LEVEL_WARNING:  &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "W ", log.LstdFlags)},
	LEVEL_INFO:     &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "I ", log.LstdFlags)},
	LEVEL_DEBUG:    &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "D ", log.LstdFlags)},
	LEVEL_PROTOCOL: &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "P ", log.LstdFlags)},
}

func (c *channel) Printf(format string, values ...interface{}) {
	if c.mapper == nil {
		c.logger.Printf(format, values...)
	} else {
		c.logger.Printf(c.mapper(fmt.Sprintf(format, values...)))
	}
}

// Critical uses format and values to generate a log message on the critical channel.
func Critical(format string, values ...interface{}) {
	channels[LEVEL_CRITICAL].Printf(format, values...)
}

// Error uses format and values to generate a log message on the error channel.
func Error(format string, values ...interface{}) {
	channels[LEVEL_ERROR].Printf(format, values...)
}

// Warning uses format and values to generate a log message on the warning channel.
func Warning(format string, values ...interface{}) {
	channels[LEVEL_WARNING].Printf(format, values...)
}

// Info uses format and values to generate a log message on the info channel.
func Info(format string, values ...interface{}) {
	channels[LEVEL_INFO].Printf(format, values...)
}

// Debug uses format and values to generate a log message on the debug channel.
func Debug(format string, values ...interface{}) {
	channels[LEVEL_DEBUG].Printf(format, values...)
}

// Debug uses format and values to generate a log message on the debug channel.
func Protocol(format string, values ...interface{}) {
	channels[LEVEL_PROTOCOL].Printf(format, values...)
}

// Handle takes the given error and writes it to all channels that correspond to the given levels. If no level is
// given, the error will be published on the error channel.
func Handle(err error, levels ...level) {
	if len(levels) == 0 {
		Handle(err, LEVEL_ERROR)
	} else {
		for _, level := range levels {
			channels[level].Printf(errgo.Details(err))
		}
	}
}

func Panic(v interface{}) {
	buffer := make([]byte, 4096)
	length := runtime.Stack(buffer, false)
	Critical("got panic: %v\n%s", v, string(buffer[:length]))
}
<commit_msg>updated errgo reference<commit_after>package gol

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"runtime"

	"gopkg.in/errgo.v1"
)

type channel struct {
	writers []io.Writer
	mapper  mapper
	logger  *log.Logger
}

var channels = map[level]*channel{
	LEVEL_CRITICAL: &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "C ", log.LstdFlags)},
	LEVEL_ERROR:    &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "E ", log.LstdFlags)},
	LEVEL_WARNING:  &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "W ", log.LstdFlags)},
	LEVEL_INFO:     &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "I ", log.LstdFlags)},
	LEVEL_DEBUG:    &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "D ", log.LstdFlags)},
	LEVEL_PROTOCOL: &channel{writers: make([]io.Writer, 0, 3), logger: log.New(ioutil.Discard, "P ", log.LstdFlags)},
}

func (c *channel) Printf(format string, values ...interface{}) {
	if c.mapper == nil {
		c.logger.Printf(format, values...)
	} else {
		c.logger.Printf(c.mapper(fmt.Sprintf(format, values...)))
	}
}

// Critical uses format and values to generate a log message on the critical channel.
func Critical(format string, values ...interface{}) {
	channels[LEVEL_CRITICAL].Printf(format, values...)
}

// Error uses format and values to generate a log message on the error channel.
func Error(format string, values ...interface{}) {
	channels[LEVEL_ERROR].Printf(format, values...)
}

// Warning uses format and values to generate a log message on the warning channel.
func Warning(format string, values ...interface{}) {
	channels[LEVEL_WARNING].Printf(format, values...)
}

// Info uses format and values to generate a log message on the info channel.
func Info(format string, values ...interface{}) {
	channels[LEVEL_INFO].Printf(format, values...)
}

// Debug uses format and values to generate a log message on the debug channel.
func Debug(format string, values ...interface{}) {
	channels[LEVEL_DEBUG].Printf(format, values...)
}

// Debug uses format and values to generate a log message on the debug channel.
func Protocol(format string, values ...interface{}) {
	channels[LEVEL_PROTOCOL].Printf(format, values...)
}

// Handle takes the given error and writes it to all channels that correspond to the given levels. If no level is
// given, the error will be published on the error channel.
func Handle(err error, levels ...level) {
	if len(levels) == 0 {
		Handle(err, LEVEL_ERROR)
	} else {
		for _, level := range levels {
			channels[level].Printf(errgo.Details(err))
		}
	}
}

func Panic(v interface{}) {
	buffer := make([]byte, 4096)
	length := runtime.Stack(buffer, false)
	Critical("got panic: %v\n%s", v, string(buffer[:length]))
}
<|endoftext|>


<commit_before>package ping_test

import (
	"context"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	bhost "github.com/libp2p/go-libp2p/p2p/host/basic"
	swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing"
	"github.com/libp2p/go-libp2p/p2p/protocol/ping"

	"github.com/libp2p/go-libp2p-core/peer"
)

func TestPing(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	h1, err := bhost.NewHost(swarmt.GenSwarm(t), nil)
	require.NoError(t, err)
	h2, err := bhost.NewHost(swarmt.GenSwarm(t), nil)
	require.NoError(t, err)

	err = h1.Connect(ctx, peer.AddrInfo{
		ID:    h2.ID(),
		Addrs: h2.Addrs(),
	})
	require.NoError(t, err)

	ps1 := ping.NewPingService(h1)
	ps2 := ping.NewPingService(h2)

	testPing(t, ps1, h2.ID())
	testPing(t, ps2, h1.ID())
}

func testPing(t *testing.T, ps *ping.PingService, p peer.ID) {
	pctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	ts := ps.Ping(pctx, p)

	for i := 0; i < 5; i++ {
		select {
		case res := <-ts:
			require.NoError(t, res.Error)
			t.Log("ping took: ", res.RTT)
		case <-time.After(time.Second * 4):
			t.Fatal("failed to receive ping")
		}
	}

}
<commit_msg>ping: fix flaky test (#1617)<commit_after>package ping_test

import (
	"context"
	"testing"
	"time"

	bhost "github.com/libp2p/go-libp2p/p2p/host/basic"
	swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing"
	"github.com/libp2p/go-libp2p/p2p/protocol/ping"

	"github.com/libp2p/go-libp2p-core/peer"

	ma "github.com/multiformats/go-multiaddr"
	"github.com/stretchr/testify/require"
)

func TestPing(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	h1, err := bhost.NewHost(swarmt.GenSwarm(t), nil)
	require.NoError(t, err)
	defer h1.Close()
	h2, err := bhost.NewHost(swarmt.GenSwarm(t), nil)
	require.NoError(t, err)
	defer h2.Close()

	err = h1.Connect(ctx, peer.AddrInfo{
		ID:    h2.ID(),
		Addrs: []ma.Multiaddr{h2.Addrs()[0]},
	})
	require.NoError(t, err)

	ps1 := ping.NewPingService(h1)
	ps2 := ping.NewPingService(h2)

	testPing(t, ps1, h2.ID())
	testPing(t, ps2, h1.ID())
}

func testPing(t *testing.T, ps *ping.PingService, p peer.ID) {
	pctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	ts := ps.Ping(pctx, p)

	for i := 0; i < 5; i++ {
		select {
		case res := <-ts:
			require.NoError(t, res.Error)
			t.Log("ping took: ", res.RTT)
		case <-time.After(time.Second * 4):
			t.Fatal("failed to receive ping")
		}
	}

}
<|endoftext|>


<commit_before>package zannotate

/*
 * ZAnnotate Copyright 2018 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

import (
	"bufio"
	"encoding/json"
	"io"
	"net"
	"os"
	"strings"
	"sync"

	log "github.com/sirupsen/logrus"
)

// struct that is populated by the input reader and passed between types of worker threads
type inProcessIP struct {
	Out map[string]interface{}
	Ip net.IP
}

func jsonToInProcess(line string, ipFieldName string, annotationFieldName string) inProcessIP {
	var inParsed interface{}
	var retv inProcessIP
	if err := json.Unmarshal([]byte(line), &inParsed); err != nil {
		log.Fatal("unable to parse input json record: ", line)
	}
	jsonMap := inParsed.(map[string]interface{})
	if val, ok := jsonMap[ipFieldName]; ok {
		if valS, ok := val.(string); ok {
			retv.Ip = net.ParseIP(valS)
		} else {
			log.Fatal("ip is not a string in JSON for ", line)
		}
	} else {
		log.Fatal("unable to find IP address field in ", line)
	}
	if _, ok := jsonMap[annotationFieldName]; ok {
		log.Fatal("input record already contains annotation key ", line)
	}
	retv.Out = jsonMap
	return retv
}

func ipToInProcess(line string) inProcessIP {
	var retv inProcessIP
	retv.Ip = net.ParseIP(line)
	retv.Out = make(map[string]interface{})
	return retv
}

func AnnotateRead(conf *GlobalConf, path string, in chan<- string) {
	log.Debug("read thread started")
	var f *os.File
	if path == "" || path == "-" {
		log.Debug("reading input from stdin")
		f = os.Stdin
	} else {
		var err error
		f, err = os.Open(path)
		if err != nil {
			log.Fatal("unable to open input file:", err.Error())
		}
		log.Debug("reading input from ", path)
	}
	r := bufio.NewReader(f)
	line, err := r.ReadString('\n')
	// read IPs out of JSON input
	for err == nil {
		in <- line
		line, err = r.ReadString('\n')
	}
	if err != nil && err != io.EOF {
		log.Fatal("input unable to read file", err)
	}
	close(in)
	log.Debug("read thread finished")
}

func AnnotateInputDecode(conf *GlobalConf, out chan<- inProcessIP, inChan <-chan string, wg *sync.WaitGroup, i int) {
	for line := range inChan {
		l := strings.TrimSuffix(line, "\n")
		if conf.InputFileType == "json" {
			out <- jsonToInProcess(l, conf.JSONIPFieldName, conf.JSONAnnotationFieldName)
		} else {
			out <- ipToInProcess(l)
		}
	}
	wg.Done()
}

func AnnotateWrite(path string, out <-chan string, wg *sync.WaitGroup) {
	log.Debug("write thread started")
	var f *os.File
	if path == "" || path == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("unable to open output file:", err.Error())
		}
		defer f.Close()
	}
	for n := range out {
		f.WriteString(n + "\n")
	}
	wg.Done()
	log.Debug("write thread finished")
}

func AnnotateWorker(a Annotator, inChan <-chan inProcessIP, outChan chan<- inProcessIP, wg *sync.WaitGroup, i int) {
	name := a.GetFieldName()
	log.Debug("annotate worker (", name, ") ", i, " started")
	if err := a.Initialize(); err != nil {
		log.Fatal("error initializing annotate worker: ", err)
	}
	for inProcess := range inChan {
		inProcess.Out[name] = a.Annotate(inProcess.Ip)
		outChan <- inProcess
	}
	wg.Done()
}

func DoAnnotation(conf *GlobalConf) {
	// several types of channels/subprocesses
	// [read from file](1) -> [decode input data](n,d=3) -> [annotator 1](n)
	//    -> [annotator 2](n) -> ... -> [annotator n](n) -> [encode output data](n,d=3)
	//    -> [write to file](1)

	inRaw := make(chan string)
	inDecoded := make(chan inProcessIP)
	// read input file
	go AnnotateRead(conf, conf.InputFilePath, inRaw)
	// decode input data
	var decodeWG sync.WaitGroup
	for i := 0; i < conf.InputDecodeThreads; i++ {
		go AnnotateInputDecode(conf, inRaw, inDecoded, decodeWG, i)
		decodeWG.Add(1)
	}
	// spawn threads for each type of annotator
	var annotateWG sync.WaitGroup
	lastChannel := inDecoded
	nextChannel := make(chan inProcessIP)
	for _, annotator := range Annotators {
		if annotator.IsEnabled() {
			for i := 0; i < annotator.GetWorkers(); i++ {
				go AnnotateWorker(annotator.MakeAnnotator(i), lastChannel, nextChannel, annotateWG, i)
			}
			lastChannel = nextChannel
			nextChannel = make(chan inProcessIP)
		}
	}
	// encode raw data
	var encodeWG sync.WaitGroup
	for i := 0; i < conf.OutputEncodeThreads; i++ {
		go AnnotateInputDecode(conf, encodeWG, lastChannel, encodedOut, i)
		encodeWG.Add(1)
	}
	go AnnotateWrite(conf.OutputFilePath, encodedOut, &outputWG)

	annotateWG.Wait()
	close(outChan)
	outputWG.Wait()
	//endTime := time.Now().Format(time.RFC3339)
}
<commit_msg>compilation of the new task runner<commit_after>package zannotate

/*
 * ZAnnotate Copyright 2018 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

import (
	"bufio"
	"encoding/json"
	"io"
	"net"
	"os"
	"strings"
	"sync"

	log "github.com/sirupsen/logrus"
)

// struct that is populated by the input reader and passed between types of worker threads
type inProcessIP struct {
	Out map[string]interface{}
	Ip net.IP
}

func jsonToInProcess(line string, ipFieldName string, annotationFieldName string) inProcessIP {
	var inParsed interface{}
	var retv inProcessIP
	if err := json.Unmarshal([]byte(line), &inParsed); err != nil {
		log.Fatal("unable to parse input json record: ", line)
	}
	jsonMap := inParsed.(map[string]interface{})
	if val, ok := jsonMap[ipFieldName]; ok {
		if valS, ok := val.(string); ok {
			retv.Ip = net.ParseIP(valS)
		} else {
			log.Fatal("ip is not a string in JSON for ", line)
		}
	} else {
		log.Fatal("unable to find IP address field in ", line)
	}
	if _, ok := jsonMap[annotationFieldName]; ok {
		log.Fatal("input record already contains annotation key ", line)
	}
	retv.Out = jsonMap
	return retv
}

func ipToInProcess(line string) inProcessIP {
	var retv inProcessIP
	retv.Ip = net.ParseIP(line)
	retv.Out = make(map[string]interface{})
	return retv
}

// single worker that reads from file and queues raw lines
func AnnotateRead(conf *GlobalConf, path string, in chan<- string) {
	log.Debug("read thread started")
	var f *os.File
	if path == "" || path == "-" {
		log.Debug("reading input from stdin")
		f = os.Stdin
	} else {
		var err error
		f, err = os.Open(path)
		if err != nil {
			log.Fatal("unable to open input file:", err.Error())
		}
		log.Debug("reading input from ", path)
	}
	r := bufio.NewReader(f)
	line, err := r.ReadString('\n')
	// read IPs out of JSON input
	for err == nil {
		in <- line
		line, err = r.ReadString('\n')
	}
	if err != nil && err != io.EOF {
		log.Fatal("input unable to read file", err)
	}
	close(in)
	log.Debug("read thread finished")
}

// multiple workers that decode raw lines from AnnotateRead from JSON/CSV into native golang objects
func AnnotateInputDecode(conf *GlobalConf, inChan <-chan string, outChan chan<- inProcessIP, wg *sync.WaitGroup, i int) {
	for line := range inChan {
		l := strings.TrimSuffix(line, "\n")
		if conf.InputFileType == "json" {
			outChan <- jsonToInProcess(l, conf.JSONIPFieldName, conf.JSONAnnotationFieldName)
		} else {
			outChan <- ipToInProcess(l)
		}
	}
	wg.Done()
}

func AnnotateOutputEncode(conf *GlobalConf, inChan <-chan inProcessIP, outChan chan<- string, wg *sync.WaitGroup, i int) {
	for rec := range inChan {
		jsonRes, err := json.Marshal(rec.Out)
		if err != nil {
			log.Fatal("Unable to marshal JSON result", err)
		}
		outChan <- string(jsonRes)
	}
	wg.Done()
}

func AnnotateWrite(path string, out <-chan string, wg *sync.WaitGroup) {
	log.Debug("write thread started")
	var f *os.File
	if path == "" || path == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("unable to open output file:", err.Error())
		}
		defer f.Close()
	}
	for n := range out {
		f.WriteString(n + "\n")
	}
	wg.Done()
	log.Debug("write thread finished")
}

func AnnotateWorker(a Annotator, inChan <-chan inProcessIP, outChan chan<- inProcessIP, wg *sync.WaitGroup, i int) {
	name := a.GetFieldName()
	log.Debug("annotate worker (", name, ") ", i, " started")
	if err := a.Initialize(); err != nil {
		log.Fatal("error initializing annotate worker: ", err)
	}
	for inProcess := range inChan {
		inProcess.Out[name] = a.Annotate(inProcess.Ip)
		outChan <- inProcess
	}
	wg.Done()
}

func DoAnnotation(conf *GlobalConf) {
	// several types of channels/subprocesses
	// [read from file](1) -> [decode input data](n,d=3) -> [annotator 1](n)
	//    -> [annotator 2](n) -> ... -> [annotator n](n) -> [encode output data](n,d=3)
	//    -> [write to file](1)

	inRaw := make(chan string)
	inDecoded := make(chan inProcessIP)
	// read input file
	go AnnotateRead(conf, conf.InputFilePath, inRaw)
	// decode input data
	var decodeWG sync.WaitGroup
	for i := 0; i < conf.InputDecodeThreads; i++ {
		go AnnotateInputDecode(conf, inRaw, inDecoded, &decodeWG, i)
		decodeWG.Add(1)
	}
	// spawn threads for each type of annotator
	lastChannel := inDecoded
	nextChannel := make(chan inProcessIP)
	var annotateWaitGroups []*sync.WaitGroup
	var annotateChannels []chan inProcessIP

	for _, annotator := range Annotators {
		if annotator.IsEnabled() {
			var annotateWG sync.WaitGroup
			for i := 0; i < annotator.GetWorkers(); i++ {
				go AnnotateWorker(annotator.MakeAnnotator(i), lastChannel, nextChannel, &annotateWG, i)
			}
			lastChannel = nextChannel
			annotateWaitGroups = append(annotateWaitGroups, &annotateWG)
			annotateChannels = append(annotateChannels, lastChannel)
			nextChannel = make(chan inProcessIP)
		}
	}
	// encode raw data
	var encodeWG sync.WaitGroup
	encodedOut := make(chan string)
	for i := 0; i < conf.OutputEncodeThreads; i++ {
		go AnnotateOutputEncode(conf, lastChannel, encodedOut, &encodeWG, i)
		encodeWG.Add(1)
	}
	var writeWG sync.WaitGroup
	go AnnotateWrite(conf.OutputFilePath, encodedOut, &writeWG)
	writeWG.Add(1)
	// all workers started. close out everything in a safe order
	// inRaw: we don't need to wait on this because it'll close its own channel when it finishes
	// until that channel is closed, none of the decoder threads will finish. So, just wait for them
	decodeWG.Wait()
	close(inDecoded)
	// wait on all of the different types of annotation workers
	for i, wg := range annotateWaitGroups {
		wg.Wait()
		close(annotateChannels[i])
	}
	// wait for the encoders
	encodeWG.Wait()
	close(encodedOut)
	// wait on writing to file
	writeWG.Wait()
	//endTime := time.Now().Format(time.RFC3339)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors.
// See https://go.googlesource.com/go/+/master/CONTRIBUTORS
// Licensed under the same terms as Go itself:
// https://go.googlesource.com/go/+/master/LICENSE

package http2

import (
	"flag"
	"net/http"
	"os"
	"testing"
)

var (
	extNet        = flag.Bool("extnet", false, "do external network tests")
	transportHost = flag.String("transporthost", "http2.golang.org", "hostname to use for TestTransport")
)

func TestTransport(t *testing.T) {
	if !*extNet {
		t.Skip("skipping external network test")
	}
	req, _ := http.NewRequest("GET", "https://"+*transportHost+"/", nil)
	rt := &Transport{}
	res, err := rt.RoundTrip(req)
	if err != nil {
		t.Fatalf("%v", err)
	}
	res.Write(os.Stdout)
}
<commit_msg>transport: add an -insecure flag to transport_test<commit_after>// Copyright 2015 The Go Authors.
// See https://go.googlesource.com/go/+/master/CONTRIBUTORS
// Licensed under the same terms as Go itself:
// https://go.googlesource.com/go/+/master/LICENSE

package http2

import (
	"flag"
	"net/http"
	"os"
	"testing"
)

var (
	extNet        = flag.Bool("extnet", false, "do external network tests")
	transportHost = flag.String("transporthost", "http2.golang.org", "hostname to use for TestTransport")
	insecure      = flag.Bool("insecure", false, "insecure TLS dials")
)

func TestTransport(t *testing.T) {
	if !*extNet {
		t.Skip("skipping external network test")
	}
	req, _ := http.NewRequest("GET", "https://"+*transportHost+"/", nil)
	rt := &Transport{
		InsecureTLSDial: *insecure,
	}
	res, err := rt.RoundTrip(req)
	if err != nil {
		t.Fatalf("%v", err)
	}
	res.Write(os.Stdout)
}
<|endoftext|>


<commit_before>// Copyright 2019 Red Hat, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.)

package tree

import (
	"reflect"
	"testing"
)

func TestFixLineColumn(t *testing.T) {
	tests := []struct {
		in  []int64
		src string
		out [][]int64 //list of index, line, col
	}{
		{
			in:  []int64{0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9},
			src: "01\n3\n567\n9",
			out: [][]int64{
				{0, 1, 1},
				{1, 1, 2},
				{1, 1, 2},
				{2, 1, 3},
				{3, 2, 1},
				{4, 2, 2},
				{5, 3, 1},
				{6, 3, 2},
				{7, 3, 3},
				{8, 3, 4},
				{9, 4, 1},
			},
		},
	}

	for i, test := range tests {
		p := make([]*Pos, len(test.in), len(test.in))
		expected := make(map[int64]Pos, len(test.in))
		for j, index := range test.in {
			p[j] = &Pos{Index: index}
			expected[test.out[j][0]] = Pos{
				Index:  test.out[j][0],
				Line:   test.out[j][1],
				Column: test.out[j][2],
			}
		}
		fixLineColumn(p, []byte(test.src))
		for _, pos := range p {
			exp := expected[pos.Index]
			if !reflect.DeepEqual(*pos, exp) {
				t.Errorf("#%d: expected %+v, got %+v:", i, exp, *pos)
			}
		}
	}
}
<commit_msg>tree: fix lint warning<commit_after>// Copyright 2019 Red Hat, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.)

package tree

import (
	"reflect"
	"testing"
)

func TestFixLineColumn(t *testing.T) {
	tests := []struct {
		in  []int64
		src string
		out [][]int64 //list of index, line, col
	}{
		{
			in:  []int64{0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9},
			src: "01\n3\n567\n9",
			out: [][]int64{
				{0, 1, 1},
				{1, 1, 2},
				{1, 1, 2},
				{2, 1, 3},
				{3, 2, 1},
				{4, 2, 2},
				{5, 3, 1},
				{6, 3, 2},
				{7, 3, 3},
				{8, 3, 4},
				{9, 4, 1},
			},
		},
	}

	for i, test := range tests {
		p := make([]*Pos, len(test.in))
		expected := make(map[int64]Pos, len(test.in))
		for j, index := range test.in {
			p[j] = &Pos{Index: index}
			expected[test.out[j][0]] = Pos{
				Index:  test.out[j][0],
				Line:   test.out[j][1],
				Column: test.out[j][2],
			}
		}
		fixLineColumn(p, []byte(test.src))
		for _, pos := range p {
			exp := expected[pos.Index]
			if !reflect.DeepEqual(*pos, exp) {
				t.Errorf("#%d: expected %+v, got %+v:", i, exp, *pos)
			}
		}
	}
}
<|endoftext|>