<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sequenced stores a list of objects that have been sequenced.
package sequenced

import (
	"bytes"
	"database/sql"
	"encoding/gob"
	"errors"
	"fmt"

	"github.com/google/keytransparency/core/sequenced"
	"github.com/google/keytransparency/core/transaction"
)

const (
	insertMapRowExpr = `INSERT INTO Maps (MapID) VALUES (?);`
	countMapRowExpr  = `SELECT COUNT(*) AS count FROM Maps WHERE MapID = ?;`
	insertExpr       = `
	INSERT INTO SMH (MapID, Epoch, Data)
	VALUES (?, ?, ?);`
	readExpr = `
	SELECT Data FROM SMH
	WHERE MapID = ? AND Epoch = ?;`
	latestExpr = `
	SELECT Epoch, Data FROM SMH
	WHERE MapID = ?
	ORDER BY Epoch DESC LIMIT 1;`
)

var (
	createStmt = []string{
		`
	CREATE TABLE IF NOT EXISTS Maps (
		MapID   BIGINT NOT NULL,
		PRIMARY KEY(MapID)
	);`,
		`
	CREATE TABLE IF NOT EXISTS SMH (
		MapID   BIGINT      NOT NULL,
		Epoch   BIGINT      NOT NULL,
		Data    BLOB(1024)  NOT NULL,
		PRIMARY KEY(MapID, Epoch),
		FOREIGN KEY(MapID) REFERENCES Maps(MapID) ON DELETE CASCADE
	);`,
	}
	// ErrNotSupported occurs when performing an operaion that has been disabled.
	ErrNotSupported = errors.New("operation not supported")
)

// Sequenced stores objects in a table.
type Sequenced struct {
	db *sql.DB
}

// New returns an object that can store sequenced items for multiple maps.
func New(db *sql.DB) (sequenced.Sequenced, error) {
	if err := db.Ping(); err != nil {
		return nil, fmt.Errorf("No DB connection: %v", err)
	}

	if err := create(db); err != nil {
		return nil, err
	}
	return &Sequenced{
		db: db,
	}, nil
}

// Create creates a new database.
func create(db *sql.DB) error {
	for _, stmt := range createStmt {
		_, err := db.Exec(stmt)
		if err != nil {
			return fmt.Errorf("Failed to create appender tables: %v", err)
		}
	}
	return nil
}

func (s *Sequenced) insertMapRow(txn transaction.Txn, mapID int64) error {
	// Check if a map row does not exist for the same MapID.
	countStmt, err := txn.Prepare(countMapRowExpr)
	if err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	defer countStmt.Close()
	var count int
	if err := countStmt.QueryRow(mapID).Scan(&count); err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	if count >= 1 {
		return nil
	}

	// Insert a map row if it does not exist already.
	insertStmt, err := txn.Prepare(insertMapRowExpr)
	if err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	defer insertStmt.Close()
	_, err = insertStmt.Exec(mapID)
	if err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	return nil
}

// Append adds an object to the append-only data structure.
func (s *Sequenced) Write(txn transaction.Txn, mapID, epoch int64, obj interface{}) error {
	if err := s.insertMapRow(txn, mapID); err != nil {
		return err
	}

	var data bytes.Buffer
	if err := gob.NewEncoder(&data).Encode(obj); err != nil {
		return err
	}
	writeStmt, err := txn.Prepare(insertExpr)
	if err != nil {
		return fmt.Errorf("DB save failure: %v", err)
	}
	defer writeStmt.Close()
	_, err = writeStmt.Exec(mapID, epoch, data.Bytes())
	if err != nil {
		return fmt.Errorf("DB commit failure: %v", err)
	}
	return nil
}

// Read retrieves a specific object for a map's epoch.
func (s *Sequenced) Read(txn transaction.Txn, mapID, epoch int64, obj interface{}) error {
	readStmt, err := txn.Prepare(readExpr)
	if err != nil {
		return err
	}
	defer readStmt.Close()

	var data []byte
	if err := readStmt.QueryRow(mapID, epoch).Scan(&data); err != nil {
		return err
	}

	err = gob.NewDecoder(bytes.NewBuffer(data)).Decode(obj)
	if err != nil {
		return err
	}
	return nil
}

// Latest returns the latest object.
func (s *Sequenced) Latest(txn transaction.Txn, mapID int64, obj interface{}) (int64, error) {
	readStmt, err := txn.Prepare(latestExpr)
	if err != nil {
		return 0, err
	}
	defer readStmt.Close()

	var epoch int64
	var data []byte
	if err := readStmt.QueryRow(mapID).Scan(&epoch, &data); err != nil {
		return 0, err
	}
	err = gob.NewDecoder(bytes.NewBuffer(data)).Decode(obj)
	if err != nil {
		return 0, err
	}
	return epoch, nil
}
<commit_msg>Use unique table name<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sequenced stores a list of objects that have been sequenced.
package sequenced

import (
	"bytes"
	"database/sql"
	"encoding/gob"
	"errors"
	"fmt"

	"github.com/google/keytransparency/core/sequenced"
	"github.com/google/keytransparency/core/transaction"
)

const (
	insertMapRowExpr = `INSERT INTO Maps (MapID) VALUES (?);`
	countMapRowExpr  = `SELECT COUNT(*) AS count FROM Maps WHERE MapID = ?;`
	insertExpr       = `
	INSERT INTO Sequenced (MapID, Epoch, Data)
	VALUES (?, ?, ?);`
	readExpr = `
	SELECT Data FROM Sequenced
	WHERE MapID = ? AND Epoch = ?;`
	latestExpr = `
	SELECT Epoch, Data FROM Sequenced
	WHERE MapID = ?
	ORDER BY Epoch DESC LIMIT 1;`
)

var (
	createStmt = []string{
		`
	CREATE TABLE IF NOT EXISTS Maps (
		MapID   BIGINT NOT NULL,
		PRIMARY KEY(MapID)
	);`,
		`
	CREATE TABLE IF NOT EXISTS Sequenced (
		MapID   BIGINT      NOT NULL,
		Epoch   BIGINT      NOT NULL,
		Data    BLOB(1024)  NOT NULL,
		PRIMARY KEY(MapID, Epoch),
		FOREIGN KEY(MapID) REFERENCES Maps(MapID) ON DELETE CASCADE
	);`,
	}
	// ErrNotSupported occurs when performing an operaion that has been disabled.
	ErrNotSupported = errors.New("operation not supported")
)

// Sequenced stores objects in a table.
type Sequenced struct {
	db *sql.DB
}

// New returns an object that can store sequenced items for multiple maps.
func New(db *sql.DB) (sequenced.Sequenced, error) {
	if err := db.Ping(); err != nil {
		return nil, fmt.Errorf("No DB connection: %v", err)
	}

	if err := create(db); err != nil {
		return nil, err
	}
	return &Sequenced{
		db: db,
	}, nil
}

// Create creates a new database.
func create(db *sql.DB) error {
	for _, stmt := range createStmt {
		_, err := db.Exec(stmt)
		if err != nil {
			return fmt.Errorf("Failed to create appender tables: %v", err)
		}
	}
	return nil
}

func (s *Sequenced) insertMapRow(txn transaction.Txn, mapID int64) error {
	// Check if a map row does not exist for the same MapID.
	countStmt, err := txn.Prepare(countMapRowExpr)
	if err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	defer countStmt.Close()
	var count int
	if err := countStmt.QueryRow(mapID).Scan(&count); err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	if count >= 1 {
		return nil
	}

	// Insert a map row if it does not exist already.
	insertStmt, err := txn.Prepare(insertMapRowExpr)
	if err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	defer insertStmt.Close()
	_, err = insertStmt.Exec(mapID)
	if err != nil {
		return fmt.Errorf("insertMapRow(): %v", err)
	}
	return nil
}

// Append adds an object to the append-only data structure.
func (s *Sequenced) Write(txn transaction.Txn, mapID, epoch int64, obj interface{}) error {
	if err := s.insertMapRow(txn, mapID); err != nil {
		return err
	}

	var data bytes.Buffer
	if err := gob.NewEncoder(&data).Encode(obj); err != nil {
		return err
	}
	writeStmt, err := txn.Prepare(insertExpr)
	if err != nil {
		return fmt.Errorf("DB save failure: %v", err)
	}
	defer writeStmt.Close()
	_, err = writeStmt.Exec(mapID, epoch, data.Bytes())
	if err != nil {
		return fmt.Errorf("DB commit failure: %v", err)
	}
	return nil
}

// Read retrieves a specific object for a map's epoch.
func (s *Sequenced) Read(txn transaction.Txn, mapID, epoch int64, obj interface{}) error {
	readStmt, err := txn.Prepare(readExpr)
	if err != nil {
		return err
	}
	defer readStmt.Close()

	var data []byte
	if err := readStmt.QueryRow(mapID, epoch).Scan(&data); err != nil {
		return err
	}

	err = gob.NewDecoder(bytes.NewBuffer(data)).Decode(obj)
	if err != nil {
		return err
	}
	return nil
}

// Latest returns the latest object.
func (s *Sequenced) Latest(txn transaction.Txn, mapID int64, obj interface{}) (int64, error) {
	readStmt, err := txn.Prepare(latestExpr)
	if err != nil {
		return 0, err
	}
	defer readStmt.Close()

	var epoch int64
	var data []byte
	if err := readStmt.QueryRow(mapID).Scan(&epoch, &data); err != nil {
		return 0, err
	}
	err = gob.NewDecoder(bytes.NewBuffer(data)).Decode(obj)
	if err != nil {
		return 0, err
	}
	return epoch, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"context"
	"fmt"
	"io"
	"net"
	"strconv"
	"sync"

	"github.com/fatedier/frp/pkg/config"
	"github.com/fatedier/frp/pkg/msg"
	plugin "github.com/fatedier/frp/pkg/plugin/server"
	frpNet "github.com/fatedier/frp/pkg/util/net"
	"github.com/fatedier/frp/pkg/util/xlog"
	"github.com/fatedier/frp/server/controller"
	"github.com/fatedier/frp/server/metrics"

	frpIo "github.com/fatedier/golib/io"
)

type GetWorkConnFn func() (net.Conn, error)

type Proxy interface {
	Context() context.Context
	Run() (remoteAddr string, err error)
	GetName() string
	GetConf() config.ProxyConf
	GetWorkConnFromPool(src, dst net.Addr) (workConn net.Conn, err error)
	GetUsedPortsNum() int
	GetResourceController() *controller.ResourceController
	GetUserInfo() plugin.UserInfo
	Close()
}

type BaseProxy struct {
	name          string
	rc            *controller.ResourceController
	listeners     []net.Listener
	usedPortsNum  int
	poolCount     int
	getWorkConnFn GetWorkConnFn
	serverCfg     config.ServerCommonConf
	userInfo      plugin.UserInfo

	mu  sync.RWMutex
	xl  *xlog.Logger
	ctx context.Context
}

func (pxy *BaseProxy) GetName() string {
	return pxy.name
}

func (pxy *BaseProxy) Context() context.Context {
	return pxy.ctx
}

func (pxy *BaseProxy) GetUsedPortsNum() int {
	return pxy.usedPortsNum
}

func (pxy *BaseProxy) GetResourceController() *controller.ResourceController {
	return pxy.rc
}

func (pxy *BaseProxy) GetUserInfo() plugin.UserInfo {
	return pxy.userInfo
}

func (pxy *BaseProxy) Close() {
	xl := xlog.FromContextSafe(pxy.ctx)
	xl.Info("proxy closing")
	for _, l := range pxy.listeners {
		l.Close()
	}
}

// GetWorkConnFromPool try to get a new work connections from pool
// for quickly response, we immediately send the StartWorkConn message to frpc after take out one from pool
func (pxy *BaseProxy) GetWorkConnFromPool(src, dst net.Addr) (workConn net.Conn, err error) {
	xl := xlog.FromContextSafe(pxy.ctx)
	// try all connections from the pool
	for i := 0; i < pxy.poolCount+1; i++ {
		if workConn, err = pxy.getWorkConnFn(); err != nil {
			xl.Warn("failed to get work connection: %v", err)
			return
		}
		xl.Info("get a new work connection: [%s]", workConn.RemoteAddr().String())
		xl.Spawn().AppendPrefix(pxy.GetName())
		workConn = frpNet.NewContextConn(pxy.ctx, workConn)

		var (
			srcAddr    string
			dstAddr    string
			srcPortStr string
			dstPortStr string
			srcPort    int
			dstPort    int
		)

		if src != nil {
			srcAddr, srcPortStr, _ = net.SplitHostPort(src.String())
			srcPort, _ = strconv.Atoi(srcPortStr)
		}
		if dst != nil {
			dstAddr, dstPortStr, _ = net.SplitHostPort(dst.String())
			dstPort, _ = strconv.Atoi(dstPortStr)
		}
		err := msg.WriteMsg(workConn, &msg.StartWorkConn{
			ProxyName: pxy.GetName(),
			SrcAddr:   srcAddr,
			SrcPort:   uint16(srcPort),
			DstAddr:   dstAddr,
			DstPort:   uint16(dstPort),
			Error:     "",
		})
		if err != nil {
			xl.Warn("failed to send message to work connection from pool: %v, times: %d", err, i)
			workConn.Close()
		} else {
			break
		}
	}

	if err != nil {
		xl.Error("try to get work connection failed in the end")
		return
	}
	return
}

// startListenHandler start a goroutine handler for each listener.
// p: p will just be passed to handler(Proxy, frpNet.Conn).
// handler: each proxy type can set different handler function to deal with connections accepted from listeners.
func (pxy *BaseProxy) startListenHandler(p Proxy, handler func(Proxy, net.Conn, config.ServerCommonConf)) {
	xl := xlog.FromContextSafe(pxy.ctx)
	for _, listener := range pxy.listeners {
		go func(l net.Listener) {
			for {
				// block
				// if listener is closed, err returned
				c, err := l.Accept()
				if err != nil {
					xl.Info("listener is closed")
					return
				}
				xl.Debug("get a user connection [%s]", c.RemoteAddr().String())
				go handler(p, c, pxy.serverCfg)
			}
		}(listener)
	}
}

func NewProxy(ctx context.Context, userInfo plugin.UserInfo, rc *controller.ResourceController, poolCount int,
	getWorkConnFn GetWorkConnFn, pxyConf config.ProxyConf, serverCfg config.ServerCommonConf) (pxy Proxy, err error) {

	xl := xlog.FromContextSafe(ctx).Spawn().AppendPrefix(pxyConf.GetBaseInfo().ProxyName)
	basePxy := BaseProxy{
		name:          pxyConf.GetBaseInfo().ProxyName,
		rc:            rc,
		listeners:     make([]net.Listener, 0),
		poolCount:     poolCount,
		getWorkConnFn: getWorkConnFn,
		serverCfg:     serverCfg,
		xl:            xl,
		ctx:           xlog.NewContext(ctx, xl),
		userInfo:      userInfo,
	}
	switch cfg := pxyConf.(type) {
	case *config.TCPProxyConf:
		basePxy.usedPortsNum = 1
		pxy = &TCPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.TCPMuxProxyConf:
		pxy = &TCPMuxProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.HTTPProxyConf:
		pxy = &HTTPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.HTTPSProxyConf:
		pxy = &HTTPSProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.UDPProxyConf:
		basePxy.usedPortsNum = 1
		pxy = &UDPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.STCPProxyConf:
		pxy = &STCPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.XTCPProxyConf:
		pxy = &XTCPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.SUDPProxyConf:
		pxy = &SUDPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	default:
		return pxy, fmt.Errorf("proxy type not support")
	}
	return
}

// HandleUserTCPConnection is used for incoming user TCP connections.
// It can be used for tcp, http, https type.
func HandleUserTCPConnection(pxy Proxy, userConn net.Conn, serverCfg config.ServerCommonConf) {
	xl := xlog.FromContextSafe(pxy.Context())
	defer userConn.Close()

	// server plugin hook
	rc := pxy.GetResourceController()
	content := &plugin.NewUserConnContent{
		User:       pxy.GetUserInfo(),
		ProxyName:  pxy.GetName(),
		ProxyType:  pxy.GetConf().GetBaseInfo().ProxyType,
		RemoteAddr: userConn.RemoteAddr().String(),
	}
	_, err := rc.PluginManager.NewUserConn(content)
	if err != nil {
		xl.Warn("the user conn [%s] was rejected, err:%v", content.RemoteAddr, err)
		return
	}

	// try all connections from the pool
	workConn, err := pxy.GetWorkConnFromPool(userConn.RemoteAddr(), userConn.LocalAddr())
	if err != nil {
		return
	}
	defer workConn.Close()

	var local io.ReadWriteCloser = workConn
	cfg := pxy.GetConf().GetBaseInfo()
	xl.Trace("handler user tcp connection, use_encryption: %t, use_compression: %t", cfg.UseEncryption, cfg.UseCompression)
	if cfg.UseEncryption {
		local, err = frpIo.WithEncryption(local, []byte(serverCfg.Token))
		if err != nil {
			xl.Error("create encryption stream error: %v", err)
			return
		}
	}
	if cfg.UseCompression {
		local = frpIo.WithCompression(local)
	}
	xl.Debug("join connections, workConn(l[%s] r[%s]) userConn(l[%s] r[%s])", workConn.LocalAddr().String(),
		workConn.RemoteAddr().String(), userConn.LocalAddr().String(), userConn.RemoteAddr().String())

	name := pxy.GetName()
	proxyType := pxy.GetConf().GetBaseInfo().ProxyType
	metrics.Server.OpenConnection(name, proxyType)
	inCount, outCount := frpIo.Join(local, userConn)
	metrics.Server.CloseConnection(name, proxyType)
	metrics.Server.AddTrafficIn(name, proxyType, inCount)
	metrics.Server.AddTrafficOut(name, proxyType, outCount)
	xl.Debug("join connections closed")
}

type Manager struct {
	// proxies indexed by proxy name
	pxys map[string]Proxy

	mu sync.RWMutex
}

func NewManager() *Manager {
	return &Manager{
		pxys: make(map[string]Proxy),
	}
}

func (pm *Manager) Add(name string, pxy Proxy) error {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	if _, ok := pm.pxys[name]; ok {
		return fmt.Errorf("proxy name [%s] is already in use", name)
	}

	pm.pxys[name] = pxy
	return nil
}

func (pm *Manager) Del(name string) {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	delete(pm.pxys, name)
}

func (pm *Manager) GetByName(name string) (pxy Proxy, ok bool) {
	pm.mu.RLock()
	defer pm.mu.RUnlock()
	pxy, ok = pm.pxys[name]
	return
}
<commit_msg>Add user remote address info log (#2184)<commit_after>// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package proxy

import (
	"context"
	"fmt"
	"io"
	"net"
	"strconv"
	"sync"

	"github.com/fatedier/frp/pkg/config"
	"github.com/fatedier/frp/pkg/msg"
	plugin "github.com/fatedier/frp/pkg/plugin/server"
	frpNet "github.com/fatedier/frp/pkg/util/net"
	"github.com/fatedier/frp/pkg/util/xlog"
	"github.com/fatedier/frp/server/controller"
	"github.com/fatedier/frp/server/metrics"

	frpIo "github.com/fatedier/golib/io"
)

type GetWorkConnFn func() (net.Conn, error)

type Proxy interface {
	Context() context.Context
	Run() (remoteAddr string, err error)
	GetName() string
	GetConf() config.ProxyConf
	GetWorkConnFromPool(src, dst net.Addr) (workConn net.Conn, err error)
	GetUsedPortsNum() int
	GetResourceController() *controller.ResourceController
	GetUserInfo() plugin.UserInfo
	Close()
}

type BaseProxy struct {
	name          string
	rc            *controller.ResourceController
	listeners     []net.Listener
	usedPortsNum  int
	poolCount     int
	getWorkConnFn GetWorkConnFn
	serverCfg     config.ServerCommonConf
	userInfo      plugin.UserInfo

	mu  sync.RWMutex
	xl  *xlog.Logger
	ctx context.Context
}

func (pxy *BaseProxy) GetName() string {
	return pxy.name
}

func (pxy *BaseProxy) Context() context.Context {
	return pxy.ctx
}

func (pxy *BaseProxy) GetUsedPortsNum() int {
	return pxy.usedPortsNum
}

func (pxy *BaseProxy) GetResourceController() *controller.ResourceController {
	return pxy.rc
}

func (pxy *BaseProxy) GetUserInfo() plugin.UserInfo {
	return pxy.userInfo
}

func (pxy *BaseProxy) Close() {
	xl := xlog.FromContextSafe(pxy.ctx)
	xl.Info("proxy closing")
	for _, l := range pxy.listeners {
		l.Close()
	}
}

// GetWorkConnFromPool try to get a new work connections from pool
// for quickly response, we immediately send the StartWorkConn message to frpc after take out one from pool
func (pxy *BaseProxy) GetWorkConnFromPool(src, dst net.Addr) (workConn net.Conn, err error) {
	xl := xlog.FromContextSafe(pxy.ctx)
	// try all connections from the pool
	for i := 0; i < pxy.poolCount+1; i++ {
		if workConn, err = pxy.getWorkConnFn(); err != nil {
			xl.Warn("failed to get work connection: %v", err)
			return
		}
		xl.Debug("get a new work connection: [%s]", workConn.RemoteAddr().String())
		xl.Spawn().AppendPrefix(pxy.GetName())
		workConn = frpNet.NewContextConn(pxy.ctx, workConn)

		var (
			srcAddr    string
			dstAddr    string
			srcPortStr string
			dstPortStr string
			srcPort    int
			dstPort    int
		)

		if src != nil {
			srcAddr, srcPortStr, _ = net.SplitHostPort(src.String())
			srcPort, _ = strconv.Atoi(srcPortStr)
		}
		if dst != nil {
			dstAddr, dstPortStr, _ = net.SplitHostPort(dst.String())
			dstPort, _ = strconv.Atoi(dstPortStr)
		}
		err := msg.WriteMsg(workConn, &msg.StartWorkConn{
			ProxyName: pxy.GetName(),
			SrcAddr:   srcAddr,
			SrcPort:   uint16(srcPort),
			DstAddr:   dstAddr,
			DstPort:   uint16(dstPort),
			Error:     "",
		})
		if err != nil {
			xl.Warn("failed to send message to work connection from pool: %v, times: %d", err, i)
			workConn.Close()
		} else {
			break
		}
	}

	if err != nil {
		xl.Error("try to get work connection failed in the end")
		return
	}
	return
}

// startListenHandler start a goroutine handler for each listener.
// p: p will just be passed to handler(Proxy, frpNet.Conn).
// handler: each proxy type can set different handler function to deal with connections accepted from listeners.
func (pxy *BaseProxy) startListenHandler(p Proxy, handler func(Proxy, net.Conn, config.ServerCommonConf)) {
	xl := xlog.FromContextSafe(pxy.ctx)
	for _, listener := range pxy.listeners {
		go func(l net.Listener) {
			for {
				// block
				// if listener is closed, err returned
				c, err := l.Accept()
				if err != nil {
					xl.Info("listener is closed")
					return
				}
				xl.Info("get a user connection [%s]", c.RemoteAddr().String())
				go handler(p, c, pxy.serverCfg)
			}
		}(listener)
	}
}

func NewProxy(ctx context.Context, userInfo plugin.UserInfo, rc *controller.ResourceController, poolCount int,
	getWorkConnFn GetWorkConnFn, pxyConf config.ProxyConf, serverCfg config.ServerCommonConf) (pxy Proxy, err error) {

	xl := xlog.FromContextSafe(ctx).Spawn().AppendPrefix(pxyConf.GetBaseInfo().ProxyName)
	basePxy := BaseProxy{
		name:          pxyConf.GetBaseInfo().ProxyName,
		rc:            rc,
		listeners:     make([]net.Listener, 0),
		poolCount:     poolCount,
		getWorkConnFn: getWorkConnFn,
		serverCfg:     serverCfg,
		xl:            xl,
		ctx:           xlog.NewContext(ctx, xl),
		userInfo:      userInfo,
	}
	switch cfg := pxyConf.(type) {
	case *config.TCPProxyConf:
		basePxy.usedPortsNum = 1
		pxy = &TCPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.TCPMuxProxyConf:
		pxy = &TCPMuxProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.HTTPProxyConf:
		pxy = &HTTPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.HTTPSProxyConf:
		pxy = &HTTPSProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.UDPProxyConf:
		basePxy.usedPortsNum = 1
		pxy = &UDPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.STCPProxyConf:
		pxy = &STCPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.XTCPProxyConf:
		pxy = &XTCPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	case *config.SUDPProxyConf:
		pxy = &SUDPProxy{
			BaseProxy: &basePxy,
			cfg:       cfg,
		}
	default:
		return pxy, fmt.Errorf("proxy type not support")
	}
	return
}

// HandleUserTCPConnection is used for incoming user TCP connections.
// It can be used for tcp, http, https type.
func HandleUserTCPConnection(pxy Proxy, userConn net.Conn, serverCfg config.ServerCommonConf) {
	xl := xlog.FromContextSafe(pxy.Context())
	defer userConn.Close()

	// server plugin hook
	rc := pxy.GetResourceController()
	content := &plugin.NewUserConnContent{
		User:       pxy.GetUserInfo(),
		ProxyName:  pxy.GetName(),
		ProxyType:  pxy.GetConf().GetBaseInfo().ProxyType,
		RemoteAddr: userConn.RemoteAddr().String(),
	}
	_, err := rc.PluginManager.NewUserConn(content)
	if err != nil {
		xl.Warn("the user conn [%s] was rejected, err:%v", content.RemoteAddr, err)
		return
	}

	// try all connections from the pool
	workConn, err := pxy.GetWorkConnFromPool(userConn.RemoteAddr(), userConn.LocalAddr())
	if err != nil {
		return
	}
	defer workConn.Close()

	var local io.ReadWriteCloser = workConn
	cfg := pxy.GetConf().GetBaseInfo()
	xl.Trace("handler user tcp connection, use_encryption: %t, use_compression: %t", cfg.UseEncryption, cfg.UseCompression)
	if cfg.UseEncryption {
		local, err = frpIo.WithEncryption(local, []byte(serverCfg.Token))
		if err != nil {
			xl.Error("create encryption stream error: %v", err)
			return
		}
	}
	if cfg.UseCompression {
		local = frpIo.WithCompression(local)
	}
	xl.Debug("join connections, workConn(l[%s] r[%s]) userConn(l[%s] r[%s])", workConn.LocalAddr().String(),
		workConn.RemoteAddr().String(), userConn.LocalAddr().String(), userConn.RemoteAddr().String())

	name := pxy.GetName()
	proxyType := pxy.GetConf().GetBaseInfo().ProxyType
	metrics.Server.OpenConnection(name, proxyType)
	inCount, outCount := frpIo.Join(local, userConn)
	metrics.Server.CloseConnection(name, proxyType)
	metrics.Server.AddTrafficIn(name, proxyType, inCount)
	metrics.Server.AddTrafficOut(name, proxyType, outCount)
	xl.Debug("join connections closed")
}

type Manager struct {
	// proxies indexed by proxy name
	pxys map[string]Proxy

	mu sync.RWMutex
}

func NewManager() *Manager {
	return &Manager{
		pxys: make(map[string]Proxy),
	}
}

func (pm *Manager) Add(name string, pxy Proxy) error {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	if _, ok := pm.pxys[name]; ok {
		return fmt.Errorf("proxy name [%s] is already in use", name)
	}

	pm.pxys[name] = pxy
	return nil
}

func (pm *Manager) Del(name string) {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	delete(pm.pxys, name)
}

func (pm *Manager) GetByName(name string) (pxy Proxy, ok bool) {
	pm.mu.RLock()
	defer pm.mu.RUnlock()
	pxy, ok = pm.pxys[name]
	return
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/bitly/go-simplejson"
	. "github.com/smartystreets/goconvey/convey"
	"net/http"
	"testing"
)

func TestSearch(t *testing.T) {
	app := App()

	// Uses hardcoded data from the current testing database
	Convey("when a listing is inserted", t, func() {
		Convey("a search on a word in its title returns it", func() {
			keyword := "ignore"
			req, _ := http.NewRequest("GET", "/api/search/"+keyword, nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Header().Get("Content-Type"), ShouldContainSubstring, "application/json")
			So(res.Body.String(), shouldBeJSON)

			result, err := simplejson.NewJson([]byte(res.Body.String()))
			So(err, ShouldBeNil)

			resultAsArray, err := result.Array()
			So(err, ShouldBeNil)
			So(len(resultAsArray), ShouldEqual, 1)

			listing := result.GetIndex(0)
			So(listing.Get("userId").MustInt(), ShouldEqual, 1)
			So(listing.Get("title").MustString(), ShouldContainSubstring, keyword)
		})

		Convey("a search on a word in its description returns it", nil)

		Convey("a search on two words in its title/description returns it exactly once", nil)

		Convey("searching a word that is not in the title/description will not return it", nil)

		Convey("searching a word that is in multiple listings returns both", nil)

	})
}
<commit_msg>case sensitivity tests<commit_after>package server

import (
	"github.com/bitly/go-simplejson"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"net/http"
	"testing"
	"time"
)

func TestSearch(t *testing.T) {
	app := App()

	// Uses hardcoded data from the current testing database
	Convey("Search Functionality", t, func() {
		oldDb := db
		defer func() {
			db.Close()
			db = oldDb
		}()

		var mock sqlmock.Sqlmock
		var err error
		db, mock, err = sqlmock.New()
		if err != nil {
			t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
		}

		Convey("a search on a word in its title returns it", func() {
			mock.ExpectQuery("SELECT DISTINCT .* FROM listings .* WHERE .* ").
				WillReturnRows(sqlmock.NewRows([]string{
					"listings.key_id", "listings.creation_date", "listings.last_modification_date",
					"title", "description", "user_id",
					"price", "status", "expiration_date", "thumbnails.url",
				}).AddRow(
					1, time.Now(), time.Now(),
					"SampleValue", "Sampleish Value!",
					1, 1001, "For Sale", time.Now(), "http://example.com/asf.gif",
				))

			req, _ := http.NewRequest("GET", "/api/search/SampleValue", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Header().Get("Content-Type"), ShouldContainSubstring, "application/json")
			So(res.Body.String(), shouldBeJSON)

			result, err := simplejson.NewJson([]byte(res.Body.String()))
			So(err, ShouldBeNil)

			resultAsArray, err := result.Array()
			So(err, ShouldBeNil)
			So(len(resultAsArray), ShouldEqual, 1)

			listing := result.GetIndex(0)
			So(listing.Get("userId").MustInt(), ShouldEqual, 1)
			So(listing.Get("title").MustString(), ShouldContainSubstring, "SampleValue")

			So(mock.ExpectationsWereMet(), ShouldBeNil)
		})

		Convey("is case insensitive", func() {
			mock.ExpectQuery("SELECT DISTINCT .* FROM listings .* WHERE .* ").
				WillReturnRows(sqlmock.NewRows([]string{
					"listings.key_id", "listings.creation_date", "listings.last_modification_date",
					"title", "description", "user_id",
					"price", "status", "expiration_date", "thumbnails.url",
				}).AddRow(
					1, time.Now(), time.Now(),
					"SampleValue", "Sampleish Value!",
					1, 1001, "For Sale", time.Now(), "http://example.com/asf.gif",
				))

			req, _ := http.NewRequest("GET", "/api/search/sAmPleVaLue", nil)
			res := executeRequest(app, req)

			So(res.Code, ShouldEqual, http.StatusOK)
			So(res.Header().Get("Content-Type"), ShouldContainSubstring, "application/json")
			So(res.Body.String(), shouldBeJSON)

			result, err := simplejson.NewJson([]byte(res.Body.String()))
			So(err, ShouldBeNil)

			resultAsArray, err := result.Array()
			So(err, ShouldBeNil)
			So(len(resultAsArray), ShouldEqual, 1)

			listing := result.GetIndex(0)
			So(listing.Get("userId").MustInt(), ShouldEqual, 1)
			So(listing.Get("title").MustString(), ShouldContainSubstring, "SampleValue")

			So(mock.ExpectationsWereMet(), ShouldBeNil)
		})

		Convey("a search on a word in its description returns it", nil)

		Convey("a search on two words in its title/description returns it exactly once", nil)

		Convey("searching a word that is not in the title/description will not return it", nil)

		Convey("searching a word that is in multiple listings returns both", nil)

	})
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/albertyw/reaction-pics/tumblr"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"go.uber.org/zap"
)

type HandlerTestSuite struct {
	suite.Suite
	deps handlerDeps
}

func TestHandlerTestSuite(t *testing.T) {
	suite.Run(t, new(HandlerTestSuite))
}

func (s *HandlerTestSuite) SetupTest() {
	logger := zap.NewNop().Sugar()
	board := tumblr.NewBoard([]tumblr.Post{})
	s.deps = handlerDeps{
		logger:         logger,
		board:          &board,
		appCacheString: appCacheString(logger),
	}
}

func (s *HandlerTestSuite) TestIndexFile() {
	request, err := http.NewRequest("GET", "/", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)

	assert.Contains(s.T(), response.Body.String(), s.deps.appCacheString)
}

func (s *HandlerTestSuite) TestOnlyIndexFile() {
	request, err := http.NewRequest("GET", "/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestReadFile() {
	request, err := http.NewRequest("GET", "/static/favicon/manifest.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}

func (s *HandlerTestSuite) TestNoExactURL() {
	request, err := http.NewRequest("GET", "/static/asdf.js", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)

	response = httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestSearchHandler() {
	request, err := http.NewRequest("GET", "/search", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerOffset() {
	request, err := http.NewRequest("GET", "/search?offset=1", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":1,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerMalformedOffset() {
	request, err := http.NewRequest("GET", "/search?offset=asdf", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestPostHandlerMalformed() {
	request, err := http.NewRequest("GET", "/post/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandlerNotFound() {
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataPercentHandler() {
	post := tumblr.Post{ID: 1234, Title: `asdf% qwer`}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	var data map[string][]map[string]interface{}
	json.Unmarshal(response.Body.Bytes(), &data)
	title := data["data"][0]["title"].(string)
	assert.Equal(s.T(), `asdf% qwer`, title)
}

func (s *HandlerTestSuite) TestPostDataHandlerMalformed() {
	request, err := http.NewRequest("GET", "/postdata/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostDataHandlerUnknown() {
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestStatsHandler() {
	request, err := http.NewRequest("GET", "/stats.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	statsHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"keywords\":[],\"postCount\":\"0\"}")
}

func (s *HandlerTestSuite) TestSitemapHandler() {
	request, err := http.NewRequest("GET", "/sitemap.xml", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	sitemapHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}
<commit_msg>Add test<commit_after>package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/albertyw/reaction-pics/tumblr"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"go.uber.org/zap"
)

type HandlerTestSuite struct {
	suite.Suite
	deps handlerDeps
}

func TestHandlerTestSuite(t *testing.T) {
	suite.Run(t, new(HandlerTestSuite))
}

func (s *HandlerTestSuite) SetupTest() {
	logger := zap.NewNop().Sugar()
	board := tumblr.NewBoard([]tumblr.Post{})
	s.deps = handlerDeps{
		logger:         logger,
		board:          &board,
		appCacheString: appCacheString(logger),
	}
}

func (s *HandlerTestSuite) TestIndexFile() {
	request, err := http.NewRequest("GET", "/", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)

	assert.Contains(s.T(), response.Body.String(), s.deps.appCacheString)
}

func (s *HandlerTestSuite) TestOnlyIndexFile() {
	request, err := http.NewRequest("GET", "/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestReadFile() {
	request, err := http.NewRequest("GET", "/static/favicon/manifest.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}

func (s *HandlerTestSuite) TestNoExactURL() {
	request, err := http.NewRequest("GET", "/static/asdf.js", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)

	response = httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestSearchHandler() {
	request, err := http.NewRequest("GET", "/search", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerOffset() {
	request, err := http.NewRequest("GET", "/search?offset=1", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":1,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerMalformedOffset() {
	request, err := http.NewRequest("GET", "/search?offset=asdf", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestPostHandlerMalformed() {
	request, err := http.NewRequest("GET", "/post/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandlerNotFound() {
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataPercentHandler() {
	post := tumblr.Post{ID: 1234, Title: `asdf% qwer`}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	var data map[string][]map[string]interface{}
	json.Unmarshal(response.Body.Bytes(), &data)
	title := data["data"][0]["title"].(string)
	assert.Equal(s.T(), `asdf% qwer`, title)
}

func (s *HandlerTestSuite) TestPostDataHandlerMalformed() {
	request, err := http.NewRequest("GET", "/postdata/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostDataHandlerUnknown() {
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestStatsHandler() {
	request, err := http.NewRequest("GET", "/stats.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	statsHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"keywords\":[],\"postCount\":\"0\"}")
}

func (s *HandlerTestSuite) TestSitemapHandler() {
	request, err := http.NewRequest("GET", "/sitemap.xml", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	sitemapHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}

func (s *HandlerTestSuite) TestTimeHandler() {
	request, err := http.NewRequest("POST", "/time/", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	timeHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	var data map[string]int
	json.Unmarshal(response.Body.Bytes(), &data)
	unixTime, found := data["unixtime"]
	assert.True(s.T(), found)
	assert.True(s.T(), unixTime > 0)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/lib/pq"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func importJson(c *cli.Context) {

}

func connect(connStr string, importSchema string) *sql.DB {
	db, err := sql.Open("postgres", connStr)
	failOnError(err, "Could not prepare connection to database")

	err = db.Ping()
	failOnError(err, "Could not reach the database")

	createSchema, err := db.Prepare("CREATE SCHEMA IF NOT EXISTS ?")
	failOnError(err, "Could not create schema statement")

	_, err = createSchema.Exec(importSchema)
	failOnError(err, fmt.Sprintf("Could not create schema %s", importSchema))

	return db
}

func createConnStr(c *cli.Context) string {
	otherParams := "sslmode=disable connect_timeout=5"
	return fmt.Sprintf("user=%s dbname=%s password='%s' host=%s port=%s %s",
		c.GlobalString("username"),
		c.GlobalString("dbname"),
		c.GlobalString("pass"),
		c.GlobalString("host"),
		c.GlobalString("port"),
		otherParams,
	)
}

func createTableStatement(db *sql.DB, schema string, tableName string, columns []string) *sql.Stmt {
	columnTypes := make([]string, len(columns))
	for i, col := range columns {
		columnTypes[i] = fmt.Sprintf("%s TEXT", col)
	}
	columnDefinitions := strings.Join(columnTypes, ",")
	fullyQualifiedTable := fmt.Sprintf("%s.%s", schema, tableName)
	statement, err := db.Prepare(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", fullyQualifiedTable, columnDefinitions))
	failOnError(err, "Could not create statement")
	return statement
}

func importCsv(c *cli.Context) {
	filename := c.Args().First()
	if filename == "" {
		fmt.Println("Please provide name of file to import")
		os.Exit(1)
	}

	db := connect(createConnStr(c), c.GlobalString("schema"))
	defer db.Close()

	file, err := os.Open(filename)
	failOnError(err, "Cannot open file")
	defer file.Close()

	reader := csv.NewReader(file)
	reader.Comma = rune(c.String("delimiter")[0])
	reader.LazyQuotes = true

	// Find out header fields
	var columns []string
	if c.Bool("skip-header") {
		columns = strings.Split(c.String("fields"), ",")
		reader.FieldsPerRecord = len(columns)
	} else {
		columns, err = reader.Read()
		failOnError(err, "Could not read header row")
	}

	schema := c.GlobalString("schema")
	tableName := "impowimpi"

	createTable := createTableStatement(db, schema, tableName, columns)
	_, err = createTable.Exec()
	failOnError(err, "Could not create table")

	txn, err := db.Begin()
	failOnError(err, "Could not start transaction")

	stmt, err := txn.Prepare(pq.CopyInSchema(schema, tableName, columns...))
	failOnError(err, "Could not prepare copy in statement")

	for {
		cols := make([]interface{}, len(columns))
		record, err := reader.Read()
		for i, col := range record {
			cols[i] = col
		}

		if err == io.EOF {
			break
		}
		failOnError(err, "Could not read csv")
		_, err = stmt.Exec(cols...)
		failOnError(err, "Could add bulk insert")
	}

	_, err = stmt.Exec()
	failOnError(err, "Could not exec the bulk copy")

	err = stmt.Close()
	failOnError(err, "Could not close")

	err = txn.Commit()
	failOnError(err, "Could not commit transaction")
}

func main() {
	app := cli.NewApp()
	app.Name = "pgfutter"
	app.Usage = "Imports anything into PostgreSQL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "dbname, db",
			Value:  "postgres",
			Usage:  "database to connect to",
			EnvVar: "DB_NAME",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "host name",
			EnvVar: "DB_HOST",
		},
		cli.StringFlag{
			Name:   "port",
			Value:  "5432",
			Usage:  "port",
			EnvVar: "DB_PORT",
		},
		cli.StringFlag{
			Name:   "username, user",
			Value:  "postgres",
			Usage:  "username",
			EnvVar: "DB_USER",
		},
		cli.StringFlag{
			Name:   "pass, pw",
			Value:  "",
			Usage:  "password",
			EnvVar: "DB_PASS",
		},
		cli.StringFlag{
			Name:   "schema",
			Value:  "import",
			Usage:  "database schema",
			EnvVar: "DB_SCHEMA",
		},
		cli.BoolFlag{
			Name:  "abort",
			Usage: "halt transaction on inconsistencies",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:   "json",
			Usage:  "Import lines of JSON objects into database",
			Action: importJson,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "flatten-graph, flatten",
					Usage: "flatten fields into columns",
				},
			},
		},
		{
			Name:   "csv",
			Usage:  "Import CSV into database",
			Action: importCsv,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "skip-header",
					Usage: "skip header row",
				},
				cli.StringFlag{
					Name:  "fields",
					Usage: "comma separated field names if no header row",
				},
				cli.StringFlag{
					Name:  "delimiter, d",
					Value: ",",
					Usage: "field delimiter",
				},
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Refactored parseColumns<commit_after>package main

import (
	"database/sql"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/lib/pq"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func importJson(c *cli.Context) {

}

func connect(connStr string, importSchema string) *sql.DB {
	db, err := sql.Open("postgres", connStr)
	failOnError(err, "Could not prepare connection to database")

	err = db.Ping()
	failOnError(err, "Could not reach the database")

	createSchema, err := db.Prepare(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", importSchema))
	failOnError(err, "Could not create schema statement")

	_, err = createSchema.Exec()
	failOnError(err, fmt.Sprintf("Could not create schema %s", importSchema))

	return db
}

func createConnStr(c *cli.Context) string {
	otherParams := "sslmode=disable connect_timeout=5"
	return fmt.Sprintf("user=%s dbname=%s password='%s' host=%s port=%s %s",
		c.GlobalString("username"),
		c.GlobalString("dbname"),
		c.GlobalString("pass"),
		c.GlobalString("host"),
		c.GlobalString("port"),
		otherParams,
	)
}

func createTableStatement(db *sql.DB, schema string, tableName string, columns []string) *sql.Stmt {
	columnTypes := make([]string, len(columns))
	for i, col := range columns {
		columnTypes[i] = fmt.Sprintf("%s TEXT", col)
	}
	columnDefinitions := strings.Join(columnTypes, ",")
	fullyQualifiedTable := fmt.Sprintf("%s.%s", schema, tableName)
	tableSchema := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", fullyQualifiedTable, columnDefinitions)

	statement, err := db.Prepare(tableSchema)
	failOnError(err, "Could not create statement")

	return statement
}

func parseColumns(c *cli.Context, reader *csv.Reader) []string {
	var err error
	var columns []string
	if c.Bool("skip-header") {
		columns = strings.Split(c.String("fields"), ",")
	} else {
		columns, err = reader.Read()
		failOnError(err, "Could not read header row")
	}

	for i, column := range columns {
		columns[i] = strings.ToLower(column)
	}

	return columns
}

func importCsv(c *cli.Context) {
	filename := c.Args().First()
	if filename == "" {
		fmt.Println("Please provide name of file to import")
		os.Exit(1)
	}

	db := connect(createConnStr(c), c.GlobalString("schema"))
	defer db.Close()

	file, err := os.Open(filename)
	failOnError(err, "Cannot open file")
	defer file.Close()

	reader := csv.NewReader(file)
	reader.Comma = rune(c.String("delimiter")[0])
	reader.LazyQuotes = true

	// Find out header fields

	columns := parseColumns(c, reader)
	reader.FieldsPerRecord = len(columns)

	schema := c.GlobalString("schema")
	tableName := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename))
	tableName = strings.ToLower(tableName)

	createTable := createTableStatement(db, schema, tableName, columns)
	_, err = createTable.Exec()
	failOnError(err, "Could not create table")

	txn, err := db.Begin()
	failOnError(err, "Could not start transaction")

	stmt, err := txn.Prepare(pq.CopyInSchema(schema, tableName, columns...))
	failOnError(err, "Could not prepare copy in statement")

	for {
		cols := make([]interface{}, len(columns))
		record, err := reader.Read()
		for i, col := range record {
			cols[i] = col
		}

		if err == io.EOF {
			break
		}
		failOnError(err, "Could not read csv")
		_, err = stmt.Exec(cols...)
		failOnError(err, "Could add bulk insert")
	}

	_, err = stmt.Exec()
	failOnError(err, "Could not exec the bulk copy")

	err = stmt.Close()
	failOnError(err, "Could not close")

	err = txn.Commit()
	failOnError(err, "Could not commit transaction")
}

func main() {
	app := cli.NewApp()
	app.Name = "pgfutter"
	app.Usage = "Imports anything into PostgreSQL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "dbname, db",
			Value:  "postgres",
			Usage:  "database to connect to",
			EnvVar: "DB_NAME",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "host name",
			EnvVar: "DB_HOST",
		},
		cli.StringFlag{
			Name:   "port",
			Value:  "5432",
			Usage:  "port",
			EnvVar: "DB_PORT",
		},
		cli.StringFlag{
			Name:   "username, user",
			Value:  "postgres",
			Usage:  "username",
			EnvVar: "DB_USER",
		},
		cli.StringFlag{
			Name:   "pass, pw",
			Value:  "",
			Usage:  "password",
			EnvVar: "DB_PASS",
		},
		cli.StringFlag{
			Name:   "schema",
			Value:  "import",
			Usage:  "database schema",
			EnvVar: "DB_SCHEMA",
		},
		cli.BoolFlag{
			Name:  "abort",
			Usage: "halt transaction on inconsistencies",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:   "json",
			Usage:  "Import lines of JSON objects into database",
			Action: importJson,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "flatten-graph, flatten",
					Usage: "flatten fields into columns",
				},
			},
		},
		{
			Name:   "csv",
			Usage:  "Import CSV into database",
			Action: importCsv,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "skip-header",
					Usage: "skip header row",
				},
				cli.StringFlag{
					Name:  "fields",
					Usage: "comma separated field names if no header row",
				},
				cli.StringFlag{
					Name:  "delimiter, d",
					Value: ",",
					Usage: "field delimiter",
				},
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>/*

static server is a development server for hosting your static client-side
files for the boardgame app. When you deploy, you just upload the bundled
output and set the ErrorPage to return index.html, and no server is necessary.

static server does a bit of magic during development. It presents a consistent
view of the world, but it actually shadows your local /webapp folder on top of
the package default /webapp folder. So if there's a hit in your /webapp, it
returns that. Otherwise, it defaults to the package /webapp.

The other magic it does is /static/config-src/boardgame-config.html is actually
fetched from /static/config-src/boardgame-config-dev.html, so you can have
different endpoints configured in production and in dev.

*/
package static

import (
	"errors"
	"github.com/gin-gonic/gin"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

type Server struct {
	fs       http.FileSystem
	prefixes []string
}

/*
NewServer returns a new server. Get it to run by calling Start().

Use it like so:

	func main() {
		static.NewServer().Start()
	}

*/
func NewServer() *Server {
	return &Server{}
}

//TODO: figure out a more dynamic way to figure out where the other resources are.
const (
	pathToLib = "$GOPATH/src/github.com/jkomoros/boardgame/server/static/"
)

func (s *Server) staticHandler(c *gin.Context) {
	request := c.Request
	url := request.URL.String()

	if strings.HasSuffix(url, "/") {
		c.HTML(http.StatusOK, "index.html", nil)
		return
	}

	file, _ := s.fs.Open(url)

	if file != nil {

		contents, _ := ioutil.ReadAll(file)

		mimeType := "text/plain"

		//TODO: it seems brittle to roll our own here...

		if strings.HasSuffix(url, ".js") {
			mimeType = "text/javascript"
		} else if strings.HasSuffix(url, ".svg") {
			mimeType = "image/svg+xml"
		} else if strings.HasSuffix(url, ".html") {
			mimeType = "text/html"
		}

		c.Data(http.StatusOK, mimeType, contents)

		return

	}

	for _, prefix := range s.prefixes {
		if strings.HasPrefix(url, prefix) {
			//We expected to ahve this file but didn't!
			c.AbortWithError(http.StatusNotFound, errors.New("Not found"))
			return
		}
	}

	c.HTML(http.StatusOK, "index.html", nil)
}

//ShadowedFS is a simple FileSystem that tries the first FS and if that fails falls back on the Secondary.
type shadowedFS struct {
	Primary   http.FileSystem
	Secondary http.FileSystem
	Redirects map[string]string
}

func (s *shadowedFS) Open(name string) (http.File, error) {

	for from, to := range s.Redirects {
		if name == from {
			log.Println("Found redirect for", name, "to", to)
			return s.Open(to)
		}
	}

	if file, err := s.Primary.Open(name); err == nil {
		log.Println("Serving", name, "from primary")
		return file, nil
	}
	log.Println("Attempting to serve", name, "from secondary")
	return s.Secondary.Open(name)
}

func newShadowedFS(primary http.FileSystem, secondary http.FileSystem) *shadowedFS {
	return &shadowedFS{
		Primary:   primary,
		Secondary: secondary,
		Redirects: make(map[string]string),
	}
}

//AddRedirect adds a redirect so whenever from is fetched, we'll actually
//return the result for to. Take care to not create loops!
func (s *shadowedFS) AddRedirect(from string, to string) {
	s.Redirects[from] = to
}

func (s *Server) ExpectPrefix(prefix string) {
	s.prefixes = append(s.prefixes, prefix)
}

//Start is where you start the server, and it never returns until it's time to shut down.
func (s *Server) Start() {

	router := gin.Default()

	expandedPathToLib := os.ExpandEnv(pathToLib)

	router.NoRoute(s.staticHandler)

	router.LoadHTMLFiles(expandedPathToLib + "webapp/index.html")

	fs := newShadowedFS(http.Dir("webapp"), http.Dir(expandedPathToLib+"webapp"))

	s.fs = fs

	//Tell the server the prefixes for URLs that we do expect to be there, so
	//it can serve a 404 (insted of index.html) if they're not there.
	s.ExpectPrefix("/service-worker.js")
	s.ExpectPrefix("/manifest.json")
	s.ExpectPrefix("/src")
	s.ExpectPrefix("/bower_components")
	s.ExpectPrefix("/config-src")
	s.ExpectPrefix("/game-src")

	router.Run(":8080")

}
<commit_msg>Remove the old static server binary, as build.SimpleStaticServer is also prefered. Part of #655.<commit_after><|endoftext|>


<commit_before>package main

import "github.com/kybin/tor/cell"

type Selection struct {
	on  bool
	rng cell.Range

	text *Text
}

func NewSelection(text *Text) *Selection {
	return &Selection{text: text}
}

func (s *Selection) SetStart(p cell.Pt) {
	s.rng.Start = p
}

func (s *Selection) SetEnd(p cell.Pt) {
	s.rng.End = p
}

// Lines return selected line numbers as int slice.
// Note it will not return last line number if last cursor's offset is 0.
func (s *Selection) Lines() []int {
	if !s.on {
		return nil
	}
	return s.rng.Lines()
}

func (s *Selection) Min() cell.Pt {
	return s.rng.Min()
}

func (s *Selection) Max() cell.Pt {
	return s.rng.Max()
}

func (s *Selection) MinMax() (cell.Pt, cell.Pt) {
	return s.rng.MinMax()
}

func (s *Selection) Contains(p cell.Pt) bool {
	return s.rng.Contains(p)
}

func (s *Selection) Data() string {
	if !s.on {
		return ""
	}
	return s.text.DataInside(s.MinMax())
}
<commit_msg>add verification to all Methods of Selection<commit_after>package main

import "github.com/kybin/tor/cell"

// Selection is selection for text.
// When it is off, rng is invalid and treated as [(-1,-1):(-1,-1)].
type Selection struct {
	on  bool
	rng cell.Range

	text *Text
}

func NewSelection(text *Text) *Selection {
	return &Selection{text: text}
}

func (s *Selection) SetStart(p cell.Pt) {
	s.rng.Start = p
}

func (s *Selection) SetEnd(p cell.Pt) {
	s.rng.End = p
}

// Lines return selected line numbers as int slice.
// Note it will not return last line number if last cursor's offset is 0.
func (s *Selection) Lines() []int {
	if !s.on {
		return nil
	}
	return s.rng.Lines()
}

func (s *Selection) Min() cell.Pt {
	if !s.on {
		return cell.Pt{-1, -1}
	}
	return s.rng.Min()
}

func (s *Selection) Max() cell.Pt {
	if !s.on {
		return cell.Pt{-1, -1}
	}
	return s.rng.Max()
}

func (s *Selection) MinMax() (cell.Pt, cell.Pt) {
	if !s.on {
		return cell.Pt{-1, -1}, cell.Pt{-1, -1}
	}
	return s.rng.MinMax()
}

func (s *Selection) Contains(p cell.Pt) bool {
	if !s.on {
		return false
	}
	return s.rng.Contains(p)
}

func (s *Selection) Data() string {
	if !s.on {
		return ""
	}
	return s.text.DataInside(s.MinMax())
}
<|endoftext|>


<commit_before>package irmaclient

import (
	"crypto/rand"
	"encoding/binary"

	"fmt"
	"math"
	"sync"
	"time"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi/revocation"
	irma "github.com/privacybydesign/irmago"
	"github.com/sirupsen/logrus"
)

func (client *Client) initRevocation() {
	// For every credential supporting revocation, compute nonrevocation caches in async jobs
	for id, attrsets := range client.attributes {
		for i, attrs := range attrsets {
			if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
				continue
			}
			id := id // make copy of same name to capture the value for closure below
			i := i   // see https://golang.org/doc/faq#closures_and_goroutines
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}

	// Of each credential supporting revocation, we periodically update its nonrevocation witness
	// by fetching updates from the issuer's server, such that:
	// - The time interval between two updates is random so that the server cannot recognize us
	//   using the update interval,
	// - Updating happens regularly even if the app is rarely used.
	// We do this by every 10 seconds updating the credential with a low probability, which
	// increases over time since the last update.
	client.Configuration.Scheduler.Every(irma.RevocationParameters.ClientUpdateInterval).Seconds().Do(func() {
		for id, attrsets := range client.attributes {
			for i, attrs := range attrsets {
				if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
					continue
				}
				cred, err := client.credential(id, i)
				if err != nil {
					client.reportError(err)
					continue
				}
				if cred.NonRevocationWitness == nil {
					continue
				}
				r, err := randomfloat()
				if err != nil {
					client.reportError(err)
					break
				}
				speed := attrs.CredentialType().RevocationUpdateSpeed * 60 * 60
				p := probability(cred.NonRevocationWitness.Updated, speed)
				if r < p {
					irma.Logger.Debugf("scheduling nonrevocation witness remote update for %s-%s", id, attrs.Hash())
					client.jobs <- func() {
						if err = client.NonrevUpdateFromServer(id); err != nil {
							client.reportError(err)
							return
						}
					}
				}
			}
		}
	})
}

// NonrevPrepare updates the revocation state for each credential in the request
// requiring a nonrevocation proof, using the updates included in the request, or the remote
// revocation server if those do not suffice.
func (client *Client) NonrevPrepare(request irma.SessionRequest) error {
	base := request.Base()
	var err error
	var wg sync.WaitGroup
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		if !base.RequestsRevocation(id) {
			continue
		}
		irma.Logger.WithField("credtype", id).Debug("updating witnesses")
		wg.Add(1)
		go func() {
			if e := client.nonrevUpdate(id, base.Revocation[id].Updates); e != nil {
				err = e // overwrites err from previously finished call, if any
			}
			wg.Done()
		}()
	}
	wg.Wait()
	return err
}

// nonrevUpdate updates all contained instances of the specified type, using the specified
// updates if present and if they suffice, and contacting the issuer's server to download updates
// otherwise.
func (client *Client) nonrevUpdate(id irma.CredentialTypeIdentifier, updates map[uint]*revocation.Update) error {
	lowest := map[uint]uint64{}
	attrs := client.attrs(id)

	// Per credential and issuer key counter we may posess multiple credential instances.
	// Of the nonrevocation witnesses of these, take the lowest index.
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil {
			continue
		}
		pkid := cred.Pk.Counter
		_, present := lowest[pkid]
		if !present || cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index < lowest[pkid] {
			lowest[pkid] = cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index
		}
	}

	// For each key counter, get an update message starting at the lowest index computed above,
	// that can update all of our credential instance of the given type and key counter,
	// using the specified update messags if they suffice, or the issuer's server otherwise.
	u := map[uint]*revocation.Update{}
	for counter, l := range lowest {
		update := updates[counter]
		if updates != nil && (update == nil || len(update.Events) == 0) {
			return errors.Errorf("missing revocation update for %s-%d", id, counter)
		}
		if update != nil && update.Events[0].Index <= l+1 {
			u[counter] = update
		} else {
			var err error
			u[counter], err = irma.RevocationClient{Conf: client.Configuration}.
				FetchUpdateFrom(id, counter, l+1)
			if err != nil {
				return err
			}
		}
	}

	// Apply the update messages to all instances of the given type and key counter
	for counter, update := range u {
		if err := client.nonrevApplyUpdates(id, counter, update); err != nil {
			return err
		}
	}
	return nil
}

func (client *Client) nonrevApplyUpdates(id irma.CredentialTypeIdentifier, counter uint, update *revocation.Update) error {
	client.credMutex.Lock()
	defer client.credMutex.Unlock()

	attrs := client.attrs(id)
	var save bool
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil || cred.Pk.Counter != counter {
			continue
		}
		updated, err := cred.nonrevApplyUpdates(update, irma.RevocationKeys{Conf: client.Configuration})
		if updated {
			save = true
		}
		if err == revocation.ErrorRevoked {
			id := cred.CredentialType().Identifier()
			hash := cred.attrs.Hash()
			irma.Logger.Warnf("credential %s %s revoked", id, hash)
			attrs[i].Revoked = true
			cred.attrs.Revoked = true
			save = true
			client.handler.Revoked(&irma.CredentialIdentifier{Type: id, Hash: hash})
			// Even if this credential is revoked during a session, we may have
			// other instances that can satisfy the request. So don't return an
			// error which would halt the session.
			continue
		}
		if err != nil {
			return err
		}
		// Asynchroniously update nonrevocation proof cache from updated witness
		irma.Logger.WithField("credtype", id).Debug("scheduling nonrevocation cache update")
		go func(cred *credential) {
			if err := cred.NonrevPrepareCache(); err != nil {
				client.reportError(err)
			}
		}(cred)
	}
	if save {
		if err := client.storage.StoreAttributes(id, client.attributes[id]); err != nil {
			client.reportError(err)
			return err
		}
	}
	return nil
}

func (client *Client) NonrevUpdateFromServer(id irma.CredentialTypeIdentifier) error {
	return client.nonrevUpdate(id, nil)
}

func (client *Client) nonrevPrepareCache(id irma.CredentialTypeIdentifier, index int) error {
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": id, "index": index})
	logger.Debug("preparing cache")
	defer logger.Debug("Preparing cache done")
	cred, err := client.credential(id, index)
	if err != nil {
		return err
	}
	return cred.NonrevPrepareCache()
}

// nonrevRepopulateCaches repopulates the consumed nonrevocation caches of the credentials involved
// in the request, in background jobs, after the request has finished.
func (client *Client) nonrevRepopulateCaches(request irma.SessionRequest) {
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		for i := range client.attrs(id) {
			id := id
			i := i
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}
}

// nonrevApplyUpdates updates the credential's nonrevocation witness using the specified messages,
// if they all verify and if their indices are ahead and adjacent to that of our witness.
func (cred *credential) nonrevApplyUpdates(update *revocation.Update, keys irma.RevocationKeys) (bool, error) {
	t := cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time

	pk, err := keys.PublicKey(cred.CredentialType().IssuerIdentifier(), update.SignedAccumulator.PKCounter)
	if err != nil {
		return false, err
	}
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": cred.CredentialType().Identifier(), "hash": cred.attrs.Hash()})
	logger.Debugf("updating witness")
	defer logger.Debug("updating witness done")
	if err = cred.NonRevocationWitness.Update(pk, update); err != nil {
		return false, err
	}

	return cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time != t, nil
}

// probability returns a float between 0 and asymptote, representing a probability
// that asymptotically increases to the asymptote, reaching
// a reference probability at a reference index.
func probability(lastUpdate time.Time, refindex uint64) float64 {
	const (
		asymptote      = 1.0 / 3          // max probability
		refprobability = 0.75 * asymptote // probability after one week
	)
	f := math.Tan(math.Pi * refprobability / (2 * asymptote))
	i := time.Now().Sub(lastUpdate).Seconds()
	return 2 * asymptote / math.Pi * math.Atan(i/float64(refindex)*f)
}

// randomfloat between 0 and 1
func randomfloat() (float64, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		fmt.Println("error:", err)
		return 0, err
	}
	c := float64(binary.BigEndian.Uint32(b)) / float64(^uint32(0)) // random int / max int
	return c, nil
}
<commit_msg>fix: nil deref in irmaclient revocation code in case of nonexisting credential types<commit_after>package irmaclient

import (
	"crypto/rand"
	"encoding/binary"

	"fmt"
	"math"
	"sync"
	"time"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi/revocation"
	irma "github.com/privacybydesign/irmago"
	"github.com/sirupsen/logrus"
)

func (client *Client) initRevocation() {
	// For every credential supporting revocation, compute nonrevocation caches in async jobs
	for id, attrsets := range client.attributes {
		for i, attrs := range attrsets {
			if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
				continue
			}
			id := id // make copy of same name to capture the value for closure below
			i := i   // see https://golang.org/doc/faq#closures_and_goroutines
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}

	// Of each credential supporting revocation, we periodically update its nonrevocation witness
	// by fetching updates from the issuer's server, such that:
	// - The time interval between two updates is random so that the server cannot recognize us
	//   using the update interval,
	// - Updating happens regularly even if the app is rarely used.
	// We do this by every 10 seconds updating the credential with a low probability, which
	// increases over time since the last update.
	client.Configuration.Scheduler.Every(irma.RevocationParameters.ClientUpdateInterval).Seconds().Do(func() {
		for id, attrsets := range client.attributes {
			for i, attrs := range attrsets {
				if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
					continue
				}
				cred, err := client.credential(id, i)
				if err != nil {
					client.reportError(err)
					continue
				}
				if cred.NonRevocationWitness == nil {
					continue
				}
				r, err := randomfloat()
				if err != nil {
					client.reportError(err)
					break
				}
				speed := attrs.CredentialType().RevocationUpdateSpeed * 60 * 60
				p := probability(cred.NonRevocationWitness.Updated, speed)
				if r < p {
					irma.Logger.Debugf("scheduling nonrevocation witness remote update for %s-%s", id, attrs.Hash())
					client.jobs <- func() {
						if err = client.NonrevUpdateFromServer(id); err != nil {
							client.reportError(err)
							return
						}
					}
				}
			}
		}
	})
}

// NonrevPrepare updates the revocation state for each credential in the request
// requiring a nonrevocation proof, using the updates included in the request, or the remote
// revocation server if those do not suffice.
func (client *Client) NonrevPrepare(request irma.SessionRequest) error {
	base := request.Base()
	var err error
	var wg sync.WaitGroup
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		if !base.RequestsRevocation(id) {
			continue
		}
		irma.Logger.WithField("credtype", id).Debug("updating witnesses")
		wg.Add(1)
		go func() {
			if e := client.nonrevUpdate(id, base.Revocation[id].Updates); e != nil {
				err = e // overwrites err from previously finished call, if any
			}
			wg.Done()
		}()
	}
	wg.Wait()
	return err
}

// nonrevUpdate updates all contained instances of the specified type, using the specified
// updates if present and if they suffice, and contacting the issuer's server to download updates
// otherwise.
func (client *Client) nonrevUpdate(id irma.CredentialTypeIdentifier, updates map[uint]*revocation.Update) error {
	lowest := map[uint]uint64{}
	attrs := client.attrs(id)

	// Per credential and issuer key counter we may posess multiple credential instances.
	// Of the nonrevocation witnesses of these, take the lowest index.
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil {
			continue
		}
		pkid := cred.Pk.Counter
		_, present := lowest[pkid]
		if !present || cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index < lowest[pkid] {
			lowest[pkid] = cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index
		}
	}

	// For each key counter, get an update message starting at the lowest index computed above,
	// that can update all of our credential instance of the given type and key counter,
	// using the specified update messags if they suffice, or the issuer's server otherwise.
	u := map[uint]*revocation.Update{}
	for counter, l := range lowest {
		update := updates[counter]
		if updates != nil && (update == nil || len(update.Events) == 0) {
			return errors.Errorf("missing revocation update for %s-%d", id, counter)
		}
		if update != nil && update.Events[0].Index <= l+1 {
			u[counter] = update
		} else {
			var err error
			u[counter], err = irma.RevocationClient{Conf: client.Configuration}.
				FetchUpdateFrom(id, counter, l+1)
			if err != nil {
				return err
			}
		}
	}

	// Apply the update messages to all instances of the given type and key counter
	for counter, update := range u {
		if err := client.nonrevApplyUpdates(id, counter, update); err != nil {
			return err
		}
	}
	return nil
}

func (client *Client) nonrevApplyUpdates(id irma.CredentialTypeIdentifier, counter uint, update *revocation.Update) error {
	client.credMutex.Lock()
	defer client.credMutex.Unlock()

	attrs := client.attrs(id)
	var save bool
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil || cred.Pk.Counter != counter {
			continue
		}
		updated, err := cred.nonrevApplyUpdates(update, irma.RevocationKeys{Conf: client.Configuration})
		if updated {
			save = true
		}
		if err == revocation.ErrorRevoked {
			id := cred.CredentialType().Identifier()
			hash := cred.attrs.Hash()
			irma.Logger.Warnf("credential %s %s revoked", id, hash)
			attrs[i].Revoked = true
			cred.attrs.Revoked = true
			save = true
			client.handler.Revoked(&irma.CredentialIdentifier{Type: id, Hash: hash})
			// Even if this credential is revoked during a session, we may have
			// other instances that can satisfy the request. So don't return an
			// error which would halt the session.
			continue
		}
		if err != nil {
			return err
		}
		// Asynchroniously update nonrevocation proof cache from updated witness
		irma.Logger.WithField("credtype", id).Debug("scheduling nonrevocation cache update")
		go func(cred *credential) {
			if err := cred.NonrevPrepareCache(); err != nil {
				client.reportError(err)
			}
		}(cred)
	}
	if save {
		if err := client.storage.StoreAttributes(id, client.attributes[id]); err != nil {
			client.reportError(err)
			return err
		}
	}
	return nil
}

func (client *Client) NonrevUpdateFromServer(id irma.CredentialTypeIdentifier) error {
	return client.nonrevUpdate(id, nil)
}

func (client *Client) nonrevPrepareCache(id irma.CredentialTypeIdentifier, index int) error {
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": id, "index": index})
	logger.Debug("preparing cache")
	defer logger.Debug("Preparing cache done")
	cred, err := client.credential(id, index)
	if err != nil {
		return err
	}
	return cred.NonrevPrepareCache()
}

// nonrevRepopulateCaches repopulates the consumed nonrevocation caches of the credentials involved
// in the request, in background jobs, after the request has finished.
func (client *Client) nonrevRepopulateCaches(request irma.SessionRequest) {
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if credtype == nil || !credtype.RevocationSupported() {
			continue
		}
		for i := range client.attrs(id) {
			id := id
			i := i
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}
}

// nonrevApplyUpdates updates the credential's nonrevocation witness using the specified messages,
// if they all verify and if their indices are ahead and adjacent to that of our witness.
func (cred *credential) nonrevApplyUpdates(update *revocation.Update, keys irma.RevocationKeys) (bool, error) {
	t := cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time

	pk, err := keys.PublicKey(cred.CredentialType().IssuerIdentifier(), update.SignedAccumulator.PKCounter)
	if err != nil {
		return false, err
	}
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": cred.CredentialType().Identifier(), "hash": cred.attrs.Hash()})
	logger.Debugf("updating witness")
	defer logger.Debug("updating witness done")
	if err = cred.NonRevocationWitness.Update(pk, update); err != nil {
		return false, err
	}

	return cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time != t, nil
}

// probability returns a float between 0 and asymptote, representing a probability
// that asymptotically increases to the asymptote, reaching
// a reference probability at a reference index.
func probability(lastUpdate time.Time, refindex uint64) float64 {
	const (
		asymptote      = 1.0 / 3          // max probability
		refprobability = 0.75 * asymptote // probability after one week
	)
	f := math.Tan(math.Pi * refprobability / (2 * asymptote))
	i := time.Now().Sub(lastUpdate).Seconds()
	return 2 * asymptote / math.Pi * math.Atan(i/float64(refindex)*f)
}

// randomfloat between 0 and 1
func randomfloat() (float64, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		fmt.Println("error:", err)
		return 0, err
	}
	c := float64(binary.BigEndian.Uint32(b)) / float64(^uint32(0)) // random int / max int
	return c, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"golang.org/x/crypto/acme/autocert"
)

func main() {
	r := gin.Default()

	gin.AutoTLSManager.Cache = autocert.DirCache("/var/www/.cache")

	// Ping handler
	r.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong")
	})

	// Listen and Server in 0.0.0.0:443
	r.RunAutoTLS("example.com")
}
<commit_msg>docs: update example.<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"golang.org/x/crypto/acme/autocert"
)

func main() {
	r := gin.Default()

	gin.AutoTLSManager.Cache = autocert.DirCache("/var/www/.cache")

	// Ping handler
	r.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong")
	})

	// Listen and Server in 0.0.0.0:443
	r.RunAutoTLS("example1.com", "example2.com")
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"fmt"
	"os"
	"github.com/dotcloud/docker/pkg/dockerscript"
	"github.com/dotcloud/docker/pkg/beam"
	"github.com/dotcloud/docker/pkg/beam/data"
	"github.com/dotcloud/docker/pkg/term"
	"strings"
	"sync"
	"net"
	"path"
	"bufio"
	"crypto/rand"
	"encoding/hex"
)

func main() {
	devnull, err := Devnull()
	if err != nil {
		Fatal(err)
	}
	defer devnull.Close()
	if term.IsTerminal(0) {
		input := bufio.NewScanner(os.Stdin)
		for {
			os.Stdout.Write([]byte("beamsh> "))
			if !input.Scan() {
				break
			}
			line := input.Text()
			if len(line) != 0 {
				cmd, err := dockerscript.Parse(strings.NewReader(line))
				if err != nil {
					fmt.Fprintf(os.Stderr, "error: %v\n", err)
					continue
				}
				executeScript(devnull, cmd)
			}
			if err := input.Err(); err == io.EOF {
				break
			} else if err != nil {
				Fatal(err)
			}
		}
	} else {
		script, err := dockerscript.Parse(os.Stdin)
		if err != nil {
			Fatal("parse error: %v\n", err)
		}
		executeScript(devnull, script)
	}
}

func beamCopy(dst *net.UnixConn, src *net.UnixConn) error {
	for {
		payload, attachment, err := beam.Receive(src)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
		if err := beam.Send(dst, payload, attachment); err != nil {
			if attachment != nil {
				attachment.Close()
			}
			return err
		}
	}
	panic("impossibru!")
	return nil
}

type Handler func([]string, *net.UnixConn, *net.UnixConn)

func Devnull() (*net.UnixConn, error) {
	priv, pub, err := beam.USocketPair()
	if err != nil {
		return nil, err
	}
	go func() {
		defer priv.Close()
		for {
			payload, attachment, err := beam.Receive(priv)
			if err != nil {
				return
			}
			fmt.Fprintf(os.Stderr, "[devnull] discarding '%s'\n", payload)
			if attachment != nil {
				attachment.Close()
			}
		}
	}()
	return pub, nil
}

func scriptString(script []*dockerscript.Command) string {
	lines := make([]string, 0, len(script))
	for _, cmd := range script {
		line := strings.Join(cmd.Args, " ")
		if len(cmd.Children) > 0 {
			line += fmt.Sprintf(" { %s }", scriptString(cmd.Children))
		} else {
			line += " {}"
		}
		lines = append(lines, line)
	}
	return fmt.Sprintf("'%s'", strings.Join(lines, "; "))
}

func executeScript(client *net.UnixConn, script []*dockerscript.Command) error {
	Debugf("executeScript(%s)\n", scriptString(script))
	defer Debugf("executeScript(%s) DONE\n", scriptString(script))
	for _, cmd := range script {
		if err := executeCommand(client, cmd); err != nil {
			return err
		}
	}
	return nil
}

//	1) Find a handler for the command (if no handler, fail)
//	2) Attach new in & out pair to the handler
//	3) [in the background] Copy handler output to our own output
//	4) [in the background] Run the handler
//	5) Recursively executeScript() all children commands and wait for them to complete
//	6) Wait for handler to return and (shortly afterwards) output copy to complete
//	7)
func executeCommand(client *net.UnixConn, cmd *dockerscript.Command) error {
	Debugf("executeCommand(%s)\n", strings.Join(cmd.Args, " "))
	defer Debugf("executeCommand(%s) DONE\n", strings.Join(cmd.Args, " "))
	handler := GetHandler(cmd.Args[0])
	if handler == nil {
		return fmt.Errorf("no such command: %s", cmd.Args[0])
	}
	inPub, inPriv, err := beam.USocketPair()
	if err != nil {
		return err
	}
	// Don't close inPub here. We close it to signify the end of input once
	// all children are completed (guaranteeing that no more input will be sent
	// by children).
	// Otherwise we get a deadlock.
	defer inPriv.Close()
	outPub, outPriv, err := beam.USocketPair()
	if err != nil {
		return err
	}
	defer outPub.Close()
	// don't close outPriv here. It must be closed after the handler is called,
	// but before the copy tasks associated with it completes.
	// Otherwise we get a deadlock.
	var tasks sync.WaitGroup
	tasks.Add(2)
	go func() {
		handler(cmd.Args, inPriv, outPriv)
		// FIXME: do we need to outPriv.sync before closing it?
		Debugf("[%s] handler returned, closing output\n", strings.Join(cmd.Args, " "))
		outPriv.Close()
		tasks.Done()
	}()
	go func() {
		Debugf("[%s] copy start...\n", strings.Join(cmd.Args, " "))
		beamCopy(client, outPub)
		Debugf("[%s] copy done\n", strings.Join(cmd.Args, " "))
		tasks.Done()
	}()
	// depth-first execution of children commands
	// executeScript() blocks until all commands are completed
	executeScript(inPub, cmd.Children)
	inPub.Close()
	Debugf("[%s] waiting for handler and output copy to complete...\n", strings.Join(cmd.Args, " "))
	tasks.Wait()
	Debugf("[%s] handler and output copy complete!\n", strings.Join(cmd.Args, " "))
	return nil
}

func randomId() string {
	id := make([]byte, 4)
	io.ReadFull(rand.Reader, id)
	return hex.EncodeToString(id)
}

func GetHandler(name string) Handler {
	if name == "trace" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			for {
				p, a, err := beam.Receive(in)
				if err != nil {
					return
				}
				var msg string
				if pretty := data.Message(string(p)).Pretty(); pretty != "" {
					msg = pretty
				} else {
					msg = string(p)
				}
				if a != nil {
					msg = fmt.Sprintf("%s [%d]", msg, a.Fd())
				}
				fmt.Printf("===> %s\n", msg)
				beam.Send(out, p, a)
			}
		}
	} else if name == "emit" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			beam.Send(out, data.Empty().Set("foo", args[1:]...).Bytes(), nil)
		}
	} else if name == "print" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			for {
				_, a, err := beam.Receive(in)
				if err != nil {
					return
				}
				if a != nil {
					io.Copy(os.Stdout, a)
				}
			}
		}
	} else if name == "openfile" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			for _, name := range args {
				f, err := os.Open(name)
				if err != nil {
					continue
				}
				if err := beam.Send(out, data.Empty().Set("path", name).Set("type", "file").Bytes(), f); err != nil {
					f.Close()
				}
			}
		}
	}
	return nil
}

// 'status' is a notification of a job's status.
//
func parseEnv(args []string) ([]string, map[string]string) {
	var argsOut []string
	env := make(map[string]string)
	for _, word := range args[1:] {
		if strings.Contains(word, "=") {
			kv := strings.SplitN(word, "=", 2)
			key := kv[0]
			var val string
			if len(kv) == 2 {
				val = kv[1]
			}
			env[key] = val
		} else {
			argsOut = append(argsOut, word)
		}
	}
	return argsOut, env
}

type Msg struct {
	payload		[]byte
	attachment	*os.File
}

func Logf(msg string, args ...interface{}) (int, error) {
	if len(msg) == 0 || msg[len(msg) - 1] != '\n' {
		msg = msg + "\n"
	}
	msg = fmt.Sprintf("[%v] [%v] %s", os.Getpid(), path.Base(os.Args[0]), msg)
	return fmt.Printf(msg, args...)
}

func Debugf(msg string, args ...interface{}) {
	if os.Getenv("BEAMDEBUG") != "" {
		Logf(msg, args...)
	}
}

func Fatalf(msg string, args ...interface{})  {
	Logf(msg, args)
	os.Exit(1)
}

func Fatal(args ...interface{}) {
	Fatalf("%v", args[0])
}
<commit_msg>beam/examples/beamsh: prettier devnull<commit_after>package main

import (
	"io"
	"fmt"
	"os"
	"github.com/dotcloud/docker/pkg/dockerscript"
	"github.com/dotcloud/docker/pkg/beam"
	"github.com/dotcloud/docker/pkg/beam/data"
	"github.com/dotcloud/docker/pkg/term"
	"strings"
	"sync"
	"net"
	"path"
	"bufio"
	"crypto/rand"
	"encoding/hex"
)

func main() {
	devnull, err := Devnull()
	if err != nil {
		Fatal(err)
	}
	defer devnull.Close()
	if term.IsTerminal(0) {
		input := bufio.NewScanner(os.Stdin)
		for {
			os.Stdout.Write([]byte("beamsh> "))
			if !input.Scan() {
				break
			}
			line := input.Text()
			if len(line) != 0 {
				cmd, err := dockerscript.Parse(strings.NewReader(line))
				if err != nil {
					fmt.Fprintf(os.Stderr, "error: %v\n", err)
					continue
				}
				executeScript(devnull, cmd)
			}
			if err := input.Err(); err == io.EOF {
				break
			} else if err != nil {
				Fatal(err)
			}
		}
	} else {
		script, err := dockerscript.Parse(os.Stdin)
		if err != nil {
			Fatal("parse error: %v\n", err)
		}
		executeScript(devnull, script)
	}
}

func beamCopy(dst *net.UnixConn, src *net.UnixConn) error {
	for {
		payload, attachment, err := beam.Receive(src)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
		if err := beam.Send(dst, payload, attachment); err != nil {
			if attachment != nil {
				attachment.Close()
			}
			return err
		}
	}
	panic("impossibru!")
	return nil
}

type Handler func([]string, *net.UnixConn, *net.UnixConn)

func Devnull() (*net.UnixConn, error) {
	priv, pub, err := beam.USocketPair()
	if err != nil {
		return nil, err
	}
	go func() {
		defer priv.Close()
		for {
			payload, attachment, err := beam.Receive(priv)
			if err != nil {
				return
			}
			fmt.Fprintf(os.Stderr, "[devnull] discarding '%s'\n", data.Message(string(payload)).Pretty())
			if attachment != nil {
				attachment.Close()
			}
		}
	}()
	return pub, nil
}

func scriptString(script []*dockerscript.Command) string {
	lines := make([]string, 0, len(script))
	for _, cmd := range script {
		line := strings.Join(cmd.Args, " ")
		if len(cmd.Children) > 0 {
			line += fmt.Sprintf(" { %s }", scriptString(cmd.Children))
		} else {
			line += " {}"
		}
		lines = append(lines, line)
	}
	return fmt.Sprintf("'%s'", strings.Join(lines, "; "))
}

func executeScript(client *net.UnixConn, script []*dockerscript.Command) error {
	Debugf("executeScript(%s)\n", scriptString(script))
	defer Debugf("executeScript(%s) DONE\n", scriptString(script))
	for _, cmd := range script {
		if err := executeCommand(client, cmd); err != nil {
			return err
		}
	}
	return nil
}

//	1) Find a handler for the command (if no handler, fail)
//	2) Attach new in & out pair to the handler
//	3) [in the background] Copy handler output to our own output
//	4) [in the background] Run the handler
//	5) Recursively executeScript() all children commands and wait for them to complete
//	6) Wait for handler to return and (shortly afterwards) output copy to complete
//	7)
func executeCommand(client *net.UnixConn, cmd *dockerscript.Command) error {
	Debugf("executeCommand(%s)\n", strings.Join(cmd.Args, " "))
	defer Debugf("executeCommand(%s) DONE\n", strings.Join(cmd.Args, " "))
	handler := GetHandler(cmd.Args[0])
	if handler == nil {
		return fmt.Errorf("no such command: %s", cmd.Args[0])
	}
	inPub, inPriv, err := beam.USocketPair()
	if err != nil {
		return err
	}
	// Don't close inPub here. We close it to signify the end of input once
	// all children are completed (guaranteeing that no more input will be sent
	// by children).
	// Otherwise we get a deadlock.
	defer inPriv.Close()
	outPub, outPriv, err := beam.USocketPair()
	if err != nil {
		return err
	}
	defer outPub.Close()
	// don't close outPriv here. It must be closed after the handler is called,
	// but before the copy tasks associated with it completes.
	// Otherwise we get a deadlock.
	var tasks sync.WaitGroup
	tasks.Add(2)
	go func() {
		handler(cmd.Args, inPriv, outPriv)
		// FIXME: do we need to outPriv.sync before closing it?
		Debugf("[%s] handler returned, closing output\n", strings.Join(cmd.Args, " "))
		outPriv.Close()
		tasks.Done()
	}()
	go func() {
		Debugf("[%s] copy start...\n", strings.Join(cmd.Args, " "))
		beamCopy(client, outPub)
		Debugf("[%s] copy done\n", strings.Join(cmd.Args, " "))
		tasks.Done()
	}()
	// depth-first execution of children commands
	// executeScript() blocks until all commands are completed
	executeScript(inPub, cmd.Children)
	inPub.Close()
	Debugf("[%s] waiting for handler and output copy to complete...\n", strings.Join(cmd.Args, " "))
	tasks.Wait()
	Debugf("[%s] handler and output copy complete!\n", strings.Join(cmd.Args, " "))
	return nil
}

func randomId() string {
	id := make([]byte, 4)
	io.ReadFull(rand.Reader, id)
	return hex.EncodeToString(id)
}

func GetHandler(name string) Handler {
	if name == "trace" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			for {
				p, a, err := beam.Receive(in)
				if err != nil {
					return
				}
				var msg string
				if pretty := data.Message(string(p)).Pretty(); pretty != "" {
					msg = pretty
				} else {
					msg = string(p)
				}
				if a != nil {
					msg = fmt.Sprintf("%s [%d]", msg, a.Fd())
				}
				fmt.Printf("===> %s\n", msg)
				beam.Send(out, p, a)
			}
		}
	} else if name == "emit" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			beam.Send(out, data.Empty().Set("foo", args[1:]...).Bytes(), nil)
		}
	} else if name == "print" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			for {
				_, a, err := beam.Receive(in)
				if err != nil {
					return
				}
				if a != nil {
					io.Copy(os.Stdout, a)
				}
			}
		}
	} else if name == "openfile" {
		return func(args []string, in *net.UnixConn, out *net.UnixConn) {
			for _, name := range args {
				f, err := os.Open(name)
				if err != nil {
					continue
				}
				if err := beam.Send(out, data.Empty().Set("path", name).Set("type", "file").Bytes(), f); err != nil {
					f.Close()
				}
			}
		}
	}
	return nil
}

// 'status' is a notification of a job's status.
//
func parseEnv(args []string) ([]string, map[string]string) {
	var argsOut []string
	env := make(map[string]string)
	for _, word := range args[1:] {
		if strings.Contains(word, "=") {
			kv := strings.SplitN(word, "=", 2)
			key := kv[0]
			var val string
			if len(kv) == 2 {
				val = kv[1]
			}
			env[key] = val
		} else {
			argsOut = append(argsOut, word)
		}
	}
	return argsOut, env
}

type Msg struct {
	payload		[]byte
	attachment	*os.File
}

func Logf(msg string, args ...interface{}) (int, error) {
	if len(msg) == 0 || msg[len(msg) - 1] != '\n' {
		msg = msg + "\n"
	}
	msg = fmt.Sprintf("[%v] [%v] %s", os.Getpid(), path.Base(os.Args[0]), msg)
	return fmt.Printf(msg, args...)
}

func Debugf(msg string, args ...interface{}) {
	if os.Getenv("BEAMDEBUG") != "" {
		Logf(msg, args...)
	}
}

func Fatalf(msg string, args ...interface{})  {
	Logf(msg, args)
	os.Exit(1)
}

func Fatal(args ...interface{}) {
	Fatalf("%v", args[0])
}
<|endoftext|>


<commit_before>package raft

import (
	"log"

	"code.google.com/p/go.net/context"
)

func applyToStore(ents []Entry)        {}
func sendMessages(msgs []Message)      {}
func saveStateToDisk(st State)         {}
func saveToDisk(ents []Entry)          {}

func Example_Node() {
	n := Start(context.Background(), "", 0, 0)

	// stuff to n happens in other gorotines

	// the last known state
	var prev State
	for {
		// ReadState blocks until there is new state ready.
		st, ents, cents, msgs, err := n.ReadState()
		if err != nil {
			log.Fatal(err)
		}

		if prev.Equal(st) {
			saveStateToDisk(st)
			prev = st
		}

		saveToDisk(ents)
		applyToStore(cents)
		sendMessages(msgs)
	}
}
<commit_msg>raft: whoops<commit_after>package raft

import (
	"log"

	"code.google.com/p/go.net/context"
)

func applyToStore(ents []Entry)        {}
func sendMessages(msgs []Message)      {}
func saveStateToDisk(st State)         {}
func saveToDisk(ents []Entry)          {}

func Example_Node() {
	n := Start(context.Background(), "", 0, 0)

	// stuff to n happens in other gorotines

	// the last known state
	var prev State
	for {
		// ReadState blocks until there is new state ready.
		st, ents, cents, msgs, err := n.ReadState()
		if err != nil {
			log.Fatal(err)
		}

		if !prev.Equal(st) {
			saveStateToDisk(st)
			prev = st
		}

		saveToDisk(ents)
		applyToStore(cents)
		sendMessages(msgs)
	}
}
<|endoftext|>