<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package setup

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	"github.com/admpub/log"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	"github.com/admpub/nging/application/model"
)

type ProgressInfo struct {
	Finished  int64
	TotalSize int64
	Summary   string
	Timestamp int64
}

var (
	installProgress   *ProgressInfo
	installedProgress = &ProgressInfo{
		Finished:  1,
		TotalSize: 1,
	}
	uninstallProgress = &ProgressInfo{
		Finished:  0,
		TotalSize: 1,
	}

	onInstalled []func(ctx echo.Context) error
)

func OnInstalled(cb func(ctx echo.Context) error) {
	if cb == nil {
		return
	}
	onInstalled = append(onInstalled, cb)
}

func init() {
	handler.Register(func(e echo.RouteRegister) {
		e.Route("GET,POST", `/setup`, Setup)
		e.Route("GET", `/progress`, Progress)
		e.Route("GET,POST", `/license`, License)
	})
}

func Progress(ctx echo.Context) error {
	data := ctx.Data()
	if config.IsInstalled() {
		data.SetInfo(ctx.T(`已经安装过了`), 0)
		data.SetData(installedProgress)
	} else {
		if installProgress == nil {
			data.SetInfo(ctx.T(`尚未开始`), 1)
			data.SetData(uninstallProgress)
		} else {
			data.SetInfo(ctx.T(`安装中`), 1)
			data.SetData(installProgress)
		}
	}
	return ctx.JSON(data)
}

func install(ctx echo.Context, sqlFile string, installer func(string) error) (err error) {
	installProgress = &ProgressInfo{
		Timestamp: time.Now().Local().Unix(),
	}
	defer func() {
		if err != nil {
			installProgress = nil
		}
	}()
	var sqlStr string
	installProgress.TotalSize, err = com.FileSize(sqlFile)
	if err != nil {
		return
	}
	installProgress.TotalSize += int64(len(handler.OfficialSQL))
	installFunction := func(line string) (rErr error) {
		installProgress.Finished += int64(len(line)) + 1
		if strings.HasPrefix(line, `--`) {
			return nil
		}
		if strings.HasPrefix(line, `/*`) && strings.HasSuffix(line, `*/;`) {
			return nil
		}
		line = strings.TrimSpace(line)
		sqlStr += line
		if strings.HasSuffix(line, `;`) && len(sqlStr) > 0 {
			//installProgress.Summary = sqlStr
			defer func() {
				sqlStr = ``
			}()
			return installer(sqlStr)
		}
		return nil
	}
	err = com.SeekFileLines(sqlFile, installFunction)
	if err != nil {
		return
	}
	for _, line := range strings.Split(handler.OfficialSQL, "\n") {
		err = installFunction(line)
		if err != nil {
			return
		}
	}
	return
}

func Setup(ctx echo.Context) error {
	var err error
	lockFile := filepath.Join(echo.Wd(), `installed.lock`)
	if info, err := os.Stat(lockFile); err == nil && info.IsDir() == false {
		msg := ctx.T(`已经安装过了。如要重新安装，请先删除%s`, lockFile)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	sqlFiles, err := config.GetSQLInstallFiles()
	if err != nil {
		msg := ctx.T(`找不到文件%s，无法安装`, `config/install.sql`)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	if ctx.IsPost() && installProgress == nil {

		err = ctx.MustBind(&config.DefaultConfig.DB)
		if err != nil {
			return err
		}
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "'", "", -1)
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "`", "", -1)
		if config.DefaultConfig.DB.Type == `sqlite` {
			config.DefaultConfig.DB.User = ``
			config.DefaultConfig.DB.Password = ``
			config.DefaultConfig.DB.Host = ``
			if strings.HasSuffix(config.DefaultConfig.DB.Database, `.db`) == false {
				config.DefaultConfig.DB.Database += `.db`
			}
		}
		//连接数据库
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			err = createDatabase(err)
		}
		if err != nil {
			return err
		}
		//创建数据库数据
		installer, ok := config.DBInstallers[config.DefaultConfig.DB.Type]
		if !ok {
			err = ctx.E(`不支持安装到%s`, config.DefaultConfig.DB.Type)
			return err
		}

		adminUser := ctx.Form(`adminUser`)
		adminPass := ctx.Form(`adminPass`)
		adminEmail := ctx.Form(`adminEmail`)
		if len(adminUser) == 0 {
			err = ctx.E(`管理员用户名不能为空`)
			return err
		}
		if !com.IsUsername(adminUser) {
			err = ctx.E(`管理员名不能包含特殊字符(只能由字母、数字、下划线和汉字组成)`)
			return err
		}
		if len(adminPass) < 8 {
			err = ctx.E(`管理员密码不能少于8个字符`)
			return err
		}
		if len(adminEmail) == 0 {
			err = ctx.E(`管理员邮箱不能为空`)
			return err
		}
		if !ctx.Validate(`adminEmail`, adminEmail, `email`).Ok() {
			err = ctx.E(`管理员邮箱格式不正确`)
			return err
		}
		data := ctx.Data()
		for _, sqlFile := range sqlFiles {
			err = install(ctx, sqlFile, installer)
			if err != nil {
				break
			}
		}

		// 重新连接数据库
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			return err
		}

		// 添加创始人
		m := model.NewUser(ctx)
		err = m.Register(adminUser, adminPass, adminEmail)
		if err != nil {
			return err
		}

		// 生成安全密钥
		config.DefaultConfig.InitSecretKey()

		// 保存数据库账号到配置文件
		err = config.DefaultConfig.SaveToFile()
		if err != nil {
			return err
		}

		for _, cb := range onInstalled {
			if err = cb(ctx); err != nil {
				return err
			}
		}

		// 生成锁文件
		err = config.SetInstalled(lockFile)
		if err != nil {
			return err
		}

		time.Sleep(1 * time.Second) // 等1秒

		// 启动
		config.DefaultCLIConfig.RunStartup()

		// 升级
		if err := Upgrade(); err != nil {
			log.Error(err)
		}

		if ctx.IsAjax() {
			if err != nil {
				data.SetError(err)
			} else {
				data.SetInfo(ctx.T(`安装成功`)).SetData(installProgress)
			}
			return ctx.JSON(data)
		}
		if err != nil {
			goto DIE
		}
		handler.SendOk(ctx, ctx.T(`安装成功`))
		return ctx.Redirect(handler.URLFor(`/`))
	}

DIE:
	ctx.Set(`dbEngines`, config.DBEngines.Slice())
	return ctx.Render(`setup`, handler.Err(ctx, err))
}

func createDatabase(err error) error {
	if fn, ok := config.DBCreaters[config.DefaultConfig.DB.Type]; ok {
		return fn(err, config.DefaultConfig)
	}
	return err
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package setup

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	"github.com/admpub/log"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	"github.com/admpub/nging/application/model"
)

type ProgressInfo struct {
	Finished  int64
	TotalSize int64
	Summary   string
	Timestamp int64
}

var (
	installProgress   *ProgressInfo
	installedProgress = &ProgressInfo{
		Finished:  1,
		TotalSize: 1,
	}
	uninstallProgress = &ProgressInfo{
		Finished:  0,
		TotalSize: 1,
	}

	onInstalled []func(ctx echo.Context) error
)

func OnInstalled(cb func(ctx echo.Context) error) {
	if cb == nil {
		return
	}
	onInstalled = append(onInstalled, cb)
}

func init() {
	handler.Register(func(e echo.RouteRegister) {
		e.Route("GET,POST", `/setup`, Setup)
		e.Route("GET", `/progress`, Progress)
		e.Route("GET,POST", `/license`, License)
	})
}

func Progress(ctx echo.Context) error {
	data := ctx.Data()
	if config.IsInstalled() {
		data.SetInfo(ctx.T(`已经安装过了`), 0)
		data.SetData(installedProgress)
	} else {
		if installProgress == nil {
			data.SetInfo(ctx.T(`尚未开始`), 1)
			data.SetData(uninstallProgress)
		} else {
			data.SetInfo(ctx.T(`安装中`), 1)
			data.SetData(installProgress)
		}
	}
	return ctx.JSON(data)
}

func install(ctx echo.Context, sqlFile string, installer func(string) error) (err error) {
	installProgress = &ProgressInfo{
		Timestamp: time.Now().Local().Unix(),
	}
	defer func() {
		if err != nil {
			installProgress = nil
		}
	}()
	var sqlStr string
	installProgress.TotalSize, err = com.FileSize(sqlFile)
	if err != nil {
		return
	}
	installProgress.TotalSize += int64(len(handler.OfficialSQL))
	installFunction := func(line string) (rErr error) {
		installProgress.Finished += int64(len(line)) + 1
		if strings.HasPrefix(line, `--`) {
			return nil
		}
		if strings.HasPrefix(line, `/*`) && strings.HasSuffix(line, `*/;`) {
			return nil
		}
		line = strings.TrimSpace(line)
		sqlStr += line
		if strings.HasSuffix(line, `;`) && len(sqlStr) > 0 {
			//installProgress.Summary = sqlStr
			defer func() {
				sqlStr = ``
			}()
			return installer(sqlStr)
		}
		return nil
	}
	err = com.SeekFileLines(sqlFile, installFunction)
	if err != nil {
		return
	}
	for _, line := range strings.Split(handler.OfficialSQL, "\n") {
		err = installFunction(line)
		if err != nil {
			return
		}
	}
	return
}

func Setup(ctx echo.Context) error {
	var err error
	lockFile := filepath.Join(echo.Wd(), `installed.lock`)
	if info, err := os.Stat(lockFile); err == nil && info.IsDir() == false {
		msg := ctx.T(`已经安装过了。如要重新安装，请先删除%s`, lockFile)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	sqlFiles, err := config.GetSQLInstallFiles()
	if err != nil {
		msg := ctx.T(`找不到文件%s，无法安装`, `config/install.sql`)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	if ctx.IsPost() && installProgress == nil {

		err = ctx.MustBind(&config.DefaultConfig.DB)
		if err != nil {
			return err
		}
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "'", "", -1)
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "`", "", -1)
		if config.DefaultConfig.DB.Type == `sqlite` {
			config.DefaultConfig.DB.User = ``
			config.DefaultConfig.DB.Password = ``
			config.DefaultConfig.DB.Host = ``
			if strings.HasSuffix(config.DefaultConfig.DB.Database, `.db`) == false {
				config.DefaultConfig.DB.Database += `.db`
			}
		}
		//连接数据库
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			err = createDatabase(err)
		}
		if err != nil {
			return err
		}
		//创建数据库数据
		installer, ok := config.DBInstallers[config.DefaultConfig.DB.Type]
		if !ok {
			err = ctx.E(`不支持安装到%s`, config.DefaultConfig.DB.Type)
			return err
		}

		adminUser := ctx.Form(`adminUser`)
		adminPass := ctx.Form(`adminPass`)
		adminEmail := ctx.Form(`adminEmail`)
		if len(adminUser) == 0 {
			err = ctx.E(`管理员用户名不能为空`)
			return err
		}
		if !com.IsUsername(adminUser) {
			err = ctx.E(`管理员名不能包含特殊字符(只能由字母、数字、下划线和汉字组成)`)
			return err
		}
		if len(adminPass) < 8 {
			err = ctx.E(`管理员密码不能少于8个字符`)
			return err
		}
		if len(adminEmail) == 0 {
			err = ctx.E(`管理员邮箱不能为空`)
			return err
		}
		if !ctx.Validate(`adminEmail`, adminEmail, `email`).Ok() {
			err = ctx.E(`管理员邮箱格式不正确`)
			return err
		}
		data := ctx.Data()
		for _, sqlFile := range sqlFiles {
			err = install(ctx, sqlFile, installer)
			if err != nil {
				return err
			}
		}

		// 重新连接数据库
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			return err
		}

		// 添加创始人
		m := model.NewUser(ctx)
		err = m.Register(adminUser, adminPass, adminEmail)
		if err != nil {
			return err
		}

		// 生成安全密钥
		config.DefaultConfig.InitSecretKey()

		// 保存数据库账号到配置文件
		err = config.DefaultConfig.SaveToFile()
		if err != nil {
			return err
		}

		for _, cb := range onInstalled {
			if err = cb(ctx); err != nil {
				return err
			}
		}

		// 生成锁文件
		err = config.SetInstalled(lockFile)
		if err != nil {
			return err
		}

		time.Sleep(1 * time.Second) // 等1秒

		// 启动
		config.DefaultCLIConfig.RunStartup()

		// 升级
		if err := Upgrade(); err != nil {
			log.Error(err)
		}

		if ctx.IsAjax() {
			if err != nil {
				data.SetError(err)
			} else {
				data.SetInfo(ctx.T(`安装成功`)).SetData(installProgress)
			}
			return ctx.JSON(data)
		}
		if err != nil {
			goto DIE
		}
		handler.SendOk(ctx, ctx.T(`安装成功`))
		return ctx.Redirect(handler.URLFor(`/`))
	}

DIE:
	ctx.Set(`dbEngines`, config.DBEngines.Slice())
	return ctx.Render(`setup`, handler.Err(ctx, err))
}

func createDatabase(err error) error {
	if fn, ok := config.DBCreaters[config.DefaultConfig.DB.Type]; ok {
		return fn(err, config.DefaultConfig)
	}
	return err
}
<|endoftext|>


<commit_before>package quiclatest

import (
	"encoding/binary"
	"fmt"

	"github.com/ami-GS/gQUIC/latest/qtype"
)

type LongHeaderPacketType byte

const (
	ZeroRTTProtectedPacketType LongHeaderPacketType = iota + 0x7c
	HandshakePacketType
	RetryPacketType
	InitialPacketType
)

func (lht LongHeaderPacketType) String() string {
	return map[LongHeaderPacketType]string{
		ZeroRTTProtectedPacketType: "0 RTT Protected Packet",
		HandshakePacketType:        "Handshake Packet",
		RetryPacketType:            "Retry Packet",
		InitialPacketType:          "Initial Pakcet",
	}[lht]
}

type ShortHeaderFlagType byte

const (
	KeyPhaseFlag           ShortHeaderFlagType = 0x40
	GQUICDemultipexingFlag                     = 0x08
	//Reserved                                   = 0x04
)

type ShortHeaderPacketType byte

const (
	OneOctetType ShortHeaderPacketType = iota
	TwoOctetsType
	FourOctetsType
	ShortHeaderPacketTypeMask
	ShortHeaderReservedBits ShortHeaderPacketType = 0x30
	KeyPhase                ShortHeaderPacketType = 0x40
)

func (sht ShortHeaderPacketType) String() string {
	keyPhase := sht&KeyPhase == KeyPhase
	octet := map[ShortHeaderPacketType]byte{
		OneOctetType:   1,
		TwoOctetsType:  2,
		FourOctetsType: 4,
	}[sht&ShortHeaderPacketTypeMask]
	return fmt.Sprintf("KeyPhrase:%v, Octet:%d", keyPhase, octet)
}

type PacketHeaderType byte

const (
	LongHeaderType  PacketHeaderType = 0x80
	ShortHeaderType PacketHeaderType = 0x00
)

type PacketHeader interface {
	// TODO: can be defined as different interface, like WireObject?
	GetWire() []byte
	GetWireSize() int
	genWire() ([]byte, error)
	String() string
	GetConnectionIDPair() (qtype.ConnectionID, qtype.ConnectionID) // srcID, destID
	getPacketNumber() qtype.PacketNumber
}

type BasePacketHeader struct {
	DestConnID   qtype.ConnectionID
	SrcConnID    qtype.ConnectionID
	PacketNumber qtype.PacketNumber
	wire         []byte
}

func (ph *BasePacketHeader) GetConnectionIDPair() (qtype.ConnectionID, qtype.ConnectionID) {
	return ph.SrcConnID, ph.DestConnID
}

func (ph *BasePacketHeader) getPacketNumber() qtype.PacketNumber {
	return ph.PacketNumber
}

func (ph *BasePacketHeader) GetWire() []byte {
	return ph.wire
}

func (ph *BasePacketHeader) GetWireSize() int {
	return len(ph.wire)
}

func (ph *BasePacketHeader) String() string {
	return fmt.Sprintf("DstConnID:%v\nSrcConnID:%v\nPacketNum:%d", ph.DestConnID.Bytes(), ph.SrcConnID.Bytes(), ph.PacketNumber)
}

// before passing data, need to check whether data[1:5] == 0x00 or not
// data[1:5] == 0x00 means version negotiation packet
type PacketHeaderPerser func(data []byte) (p PacketHeader, length int, err error)

var PacketHeaderParserMap = map[PacketHeaderType]PacketHeaderPerser{
	LongHeaderType:  ParseLongHeader,
	ShortHeaderType: ParseShortHeader,
}

// Long Header
/*
    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
   +-+-+-+-+-+-+-+-+
   |1|   Type (7)  |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                         Version (32)                          |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |DCIL(4)|SCIL(4)|
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |               Destination Connection ID (0/32..144)         ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                 Source Connection ID (0/32..144)            ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                           Length (i)                        ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                       Packet Number (8/16/32)                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                          Payload (*)                        ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/
type LongHeader struct {
	*BasePacketHeader
	PacketType LongHeaderPacketType
	Version    qtype.Version
	DCIL       byte
	SCIL       byte
	Length     qtype.QuicInt
}

func NewLongHeader(packetType LongHeaderPacketType, version qtype.Version, destConnID, srcConnID qtype.ConnectionID, packetNumber qtype.PacketNumber, length qtype.QuicInt) *LongHeader {
	dcil := 0
	if destConnID != nil {
		dcil = len(destConnID) - 3
	}
	scil := 0
	if srcConnID != nil {
		scil = len(srcConnID) - 3
	}

	lh := &LongHeader{
		BasePacketHeader: &BasePacketHeader{
			DestConnID:   destConnID,
			SrcConnID:    srcConnID,
			PacketNumber: packetNumber,
		},
		PacketType: packetType,
		Version:    version,
		DCIL:       byte(dcil),
		SCIL:       byte(scil),
		Length:     length,
	}
	var err error
	lh.wire, err = lh.genWire()
	if err != nil {
		// error
		return nil
	}
	return lh
}

func ParseLongHeader(data []byte) (PacketHeader, int, error) {
	var err error
	idx := 0
	lh := NewLongHeader(0, 0, nil, nil, 0, 0)
	lh.PacketType = LongHeaderPacketType(data[idx] & 0x7f)
	idx++
	lh.Version = qtype.Version(binary.BigEndian.Uint32(data[idx:]))
	idx += 4
	lh.DCIL = data[idx] >> 4
	lh.SCIL = data[idx] & 0x0f
	idx++
	if lh.DCIL != 0 {
		dcil := int(lh.DCIL + 3)
		lh.DestConnID, err = qtype.ReadConnectionID(data[idx:], dcil)
		if err != nil {
			return nil, 0, err
		}
		idx += dcil
	}
	if lh.SCIL != 0 {
		scil := int(lh.SCIL + 3)
		lh.SrcConnID, err = qtype.ReadConnectionID(data[idx:], scil)
		if err != nil {
			return nil, 0, err
		}
		idx += scil
	}
	lh.Length = qtype.DecodeQuicInt(data[idx:])
	idx += lh.Length.GetByteLen()
	lh.PacketNumber = qtype.DecodePacketNumber(data[idx:])
	idx += lh.PacketNumber.GetByteLen()
	lh.wire = data[:idx]
	return lh, idx, nil
}

func (lh LongHeader) String() string {
	return fmt.Sprintf("LongHeader:%s\tVer:%s\nDCIL:%d,SCIL:%d\n%s\nLength:%d", lh.PacketType, lh.Version, lh.DCIL, lh.SCIL, lh.BasePacketHeader, lh.Length)
}

func (lh LongHeader) genWire() (wire []byte, err error) {
	wireLen := int(6 + lh.Length.GetByteLen() + lh.PacketNumber.GetByteLen())
	if lh.DCIL != 0 {
		wireLen += int(lh.DCIL + 3)
	}
	if lh.SCIL != 0 {
		wireLen += int(lh.SCIL + 3)
	}
	wire = make([]byte, wireLen)
	wire[0] = 0x80 | byte(lh.PacketType)
	binary.BigEndian.PutUint32(wire[1:], uint32(lh.Version))
	wire[5] = (lh.DCIL << 4) | lh.SCIL
	idx := 6
	if lh.DCIL != 0 {
		copy(wire[idx:], lh.DestConnID.Bytes())
		idx += int(lh.DCIL + 3)
	}
	if lh.SCIL != 0 {
		copy(wire[idx:], lh.SrcConnID.Bytes())
		idx += int(lh.SCIL + 3)
	}
	lh.Length.PutWire(wire[idx:])
	idx += lh.Length.GetByteLen()
	lh.PacketNumber.PutWire(wire[idx:])
	return
}

// Short Header
/*
    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
   +-+-+-+-+-+-+-+-+
   |0|K|1|1|0|R R R|
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                Destination Connection ID (0..144)           ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                      Packet Number (8/16/32)                ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                     Protected Payload (*)                   ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/

type ShortHeader struct {
	*BasePacketHeader
	PacketType ShortHeaderPacketType
}

func NewShortHeader(key bool, destConnID qtype.ConnectionID, packetNumber qtype.PacketNumber) *ShortHeader {
	packetType := ShortHeaderPacketType(ShortHeaderType) | 0x30

	if key {
		packetType |= KeyPhase
	}

	sh := &ShortHeader{
		BasePacketHeader: &BasePacketHeader{
			DestConnID:   destConnID,
			SrcConnID:    nil,
			PacketNumber: packetNumber,
		},
		PacketType: packetType,
	}
	var err error
	sh.wire, err = sh.genWire()
	if err != nil {
		// error
	}
	return sh
}

func ParseShortHeader(data []byte) (PacketHeader, int, error) {
	var err error
	idx := 0
	sh := NewShortHeader(false, nil, 0)
	sh.PacketType = ShortHeaderPacketType(data[idx])
	if sh.PacketType&ShortHeaderPacketTypeMask == ShortHeaderPacketTypeMask {
		// TODO: must be 0,1,2, but the error is not defined
		return nil, 0, qtype.ProtocolViolation
	}
	idx++
	sh.DestConnID, err = qtype.ReadConnectionID(data[idx:], qtype.ConnectionIDLen)
	if err != nil {
		return nil, 0, err
	}
	idx += qtype.ConnectionIDLen
	sh.PacketNumber = qtype.DecodePacketNumber(data[idx:])
	idx += sh.PacketNumber.GetByteLen()
	sh.wire = data[:idx]
	return sh, idx, nil
}

func (sh ShortHeader) String() string {
	return fmt.Sprintf("ShortHeader:%s\n%s", sh.PacketType, sh.BasePacketHeader)
}

func (sh ShortHeader) genWire() (wire []byte, err error) {
	connIDLen := len(sh.DestConnID)
	wire = make([]byte, 1+connIDLen+sh.PacketNumber.GetByteLen())
	wire[0] = byte(sh.PacketType)
	idx := 1
	if connIDLen != 0 {
		copy(wire[idx:idx+connIDLen], sh.DestConnID)
		idx += connIDLen
	}
	sh.PacketNumber.PutWire(wire[idx:])
	return
}
<commit_msg>remove old format of Short Header<commit_after>package quiclatest

import (
	"encoding/binary"
	"fmt"

	"github.com/ami-GS/gQUIC/latest/qtype"
)

type LongHeaderPacketType byte

const (
	ZeroRTTProtectedPacketType LongHeaderPacketType = iota + 0x7c
	HandshakePacketType
	RetryPacketType
	InitialPacketType
)

func (lht LongHeaderPacketType) String() string {
	return map[LongHeaderPacketType]string{
		ZeroRTTProtectedPacketType: "0 RTT Protected Packet",
		HandshakePacketType:        "Handshake Packet",
		RetryPacketType:            "Retry Packet",
		InitialPacketType:          "Initial Pakcet",
	}[lht]
}

type ShortHeaderFlagType byte

const (
	KeyPhaseFlag           ShortHeaderFlagType = 0x40
	GQUICDemultipexingFlag                     = 0x08
	//Reserved                                   = 0x04
)

type ShortHeaderPacketType byte

const (
	ShortHeaderReservedBits ShortHeaderPacketType = 0x30
	KeyPhase                ShortHeaderPacketType = 0x40
)

func (sht ShortHeaderPacketType) String() string {
	keyPhase := sht&KeyPhase == KeyPhase
	return fmt.Sprintf("KeyPhrase:%v", keyPhase)
}

type PacketHeaderType byte

const (
	LongHeaderType  PacketHeaderType = 0x80
	ShortHeaderType PacketHeaderType = 0x00
)

type PacketHeader interface {
	// TODO: can be defined as different interface, like WireObject?
	GetWire() []byte
	GetWireSize() int
	genWire() ([]byte, error)
	String() string
	GetConnectionIDPair() (qtype.ConnectionID, qtype.ConnectionID) // srcID, destID
	getPacketNumber() qtype.PacketNumber
}

type BasePacketHeader struct {
	DestConnID   qtype.ConnectionID
	SrcConnID    qtype.ConnectionID
	PacketNumber qtype.PacketNumber
	wire         []byte
}

func (ph *BasePacketHeader) GetConnectionIDPair() (qtype.ConnectionID, qtype.ConnectionID) {
	return ph.SrcConnID, ph.DestConnID
}

func (ph *BasePacketHeader) getPacketNumber() qtype.PacketNumber {
	return ph.PacketNumber
}

func (ph *BasePacketHeader) GetWire() []byte {
	return ph.wire
}

func (ph *BasePacketHeader) GetWireSize() int {
	return len(ph.wire)
}

func (ph *BasePacketHeader) String() string {
	return fmt.Sprintf("DstConnID:%v\nSrcConnID:%v\nPacketNum:%d", ph.DestConnID.Bytes(), ph.SrcConnID.Bytes(), ph.PacketNumber)
}

// before passing data, need to check whether data[1:5] == 0x00 or not
// data[1:5] == 0x00 means version negotiation packet
type PacketHeaderPerser func(data []byte) (p PacketHeader, length int, err error)

var PacketHeaderParserMap = map[PacketHeaderType]PacketHeaderPerser{
	LongHeaderType:  ParseLongHeader,
	ShortHeaderType: ParseShortHeader,
}

// Long Header
/*
    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
   +-+-+-+-+-+-+-+-+
   |1|   Type (7)  |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                         Version (32)                          |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |DCIL(4)|SCIL(4)|
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |               Destination Connection ID (0/32..144)         ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                 Source Connection ID (0/32..144)            ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                           Length (i)                        ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                       Packet Number (8/16/32)                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                          Payload (*)                        ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/
type LongHeader struct {
	*BasePacketHeader
	PacketType LongHeaderPacketType
	Version    qtype.Version
	DCIL       byte
	SCIL       byte
	Length     qtype.QuicInt
}

func NewLongHeader(packetType LongHeaderPacketType, version qtype.Version, destConnID, srcConnID qtype.ConnectionID, packetNumber qtype.PacketNumber, length qtype.QuicInt) *LongHeader {
	dcil := 0
	if destConnID != nil {
		dcil = len(destConnID) - 3
	}
	scil := 0
	if srcConnID != nil {
		scil = len(srcConnID) - 3
	}

	lh := &LongHeader{
		BasePacketHeader: &BasePacketHeader{
			DestConnID:   destConnID,
			SrcConnID:    srcConnID,
			PacketNumber: packetNumber,
		},
		PacketType: packetType,
		Version:    version,
		DCIL:       byte(dcil),
		SCIL:       byte(scil),
		Length:     length,
	}
	var err error
	lh.wire, err = lh.genWire()
	if err != nil {
		// error
		return nil
	}
	return lh
}

func ParseLongHeader(data []byte) (PacketHeader, int, error) {
	var err error
	idx := 0
	lh := NewLongHeader(0, 0, nil, nil, 0, 0)
	lh.PacketType = LongHeaderPacketType(data[idx] & 0x7f)
	idx++
	lh.Version = qtype.Version(binary.BigEndian.Uint32(data[idx:]))
	idx += 4
	lh.DCIL = data[idx] >> 4
	lh.SCIL = data[idx] & 0x0f
	idx++
	if lh.DCIL != 0 {
		dcil := int(lh.DCIL + 3)
		lh.DestConnID, err = qtype.ReadConnectionID(data[idx:], dcil)
		if err != nil {
			return nil, 0, err
		}
		idx += dcil
	}
	if lh.SCIL != 0 {
		scil := int(lh.SCIL + 3)
		lh.SrcConnID, err = qtype.ReadConnectionID(data[idx:], scil)
		if err != nil {
			return nil, 0, err
		}
		idx += scil
	}
	lh.Length = qtype.DecodeQuicInt(data[idx:])
	idx += lh.Length.GetByteLen()
	lh.PacketNumber = qtype.DecodePacketNumber(data[idx:])
	idx += lh.PacketNumber.GetByteLen()
	lh.wire = data[:idx]
	return lh, idx, nil
}

func (lh LongHeader) String() string {
	return fmt.Sprintf("LongHeader:%s\tVer:%s\nDCIL:%d,SCIL:%d\n%s\nLength:%d", lh.PacketType, lh.Version, lh.DCIL, lh.SCIL, lh.BasePacketHeader, lh.Length)
}

func (lh LongHeader) genWire() (wire []byte, err error) {
	wireLen := int(6 + lh.Length.GetByteLen() + lh.PacketNumber.GetByteLen())
	if lh.DCIL != 0 {
		wireLen += int(lh.DCIL + 3)
	}
	if lh.SCIL != 0 {
		wireLen += int(lh.SCIL + 3)
	}
	wire = make([]byte, wireLen)
	wire[0] = 0x80 | byte(lh.PacketType)
	binary.BigEndian.PutUint32(wire[1:], uint32(lh.Version))
	wire[5] = (lh.DCIL << 4) | lh.SCIL
	idx := 6
	if lh.DCIL != 0 {
		copy(wire[idx:], lh.DestConnID.Bytes())
		idx += int(lh.DCIL + 3)
	}
	if lh.SCIL != 0 {
		copy(wire[idx:], lh.SrcConnID.Bytes())
		idx += int(lh.SCIL + 3)
	}
	lh.Length.PutWire(wire[idx:])
	idx += lh.Length.GetByteLen()
	lh.PacketNumber.PutWire(wire[idx:])
	return
}

// Short Header
/*
    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
   +-+-+-+-+-+-+-+-+
   |0|K|1|1|0|R R R|
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                Destination Connection ID (0..144)           ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                      Packet Number (8/16/32)                ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                     Protected Payload (*)                   ...
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/

type ShortHeader struct {
	*BasePacketHeader
	PacketType ShortHeaderPacketType
}

func NewShortHeader(key bool, destConnID qtype.ConnectionID, packetNumber qtype.PacketNumber) *ShortHeader {
	packetType := ShortHeaderPacketType(ShortHeaderType) | 0x30

	if key {
		packetType |= KeyPhase
	}

	sh := &ShortHeader{
		BasePacketHeader: &BasePacketHeader{
			DestConnID:   destConnID,
			SrcConnID:    nil,
			PacketNumber: packetNumber,
		},
		PacketType: packetType,
	}
	var err error
	sh.wire, err = sh.genWire()
	if err != nil {
		// error
	}
	return sh
}

func ParseShortHeader(data []byte) (PacketHeader, int, error) {
	var err error
	idx := 0
	sh := NewShortHeader(false, nil, 0)
	sh.PacketType = ShortHeaderPacketType(data[idx])
	idx++
	sh.DestConnID, err = qtype.ReadConnectionID(data[idx:], qtype.ConnectionIDLen)
	if err != nil {
		return nil, 0, err
	}
	idx += qtype.ConnectionIDLen
	sh.PacketNumber = qtype.DecodePacketNumber(data[idx:])
	idx += sh.PacketNumber.GetByteLen()
	sh.wire = data[:idx]
	return sh, idx, nil
}

func (sh ShortHeader) String() string {
	return fmt.Sprintf("ShortHeader:%s\n%s", sh.PacketType, sh.BasePacketHeader)
}

func (sh ShortHeader) genWire() (wire []byte, err error) {
	connIDLen := len(sh.DestConnID)
	wire = make([]byte, 1+connIDLen+sh.PacketNumber.GetByteLen())
	wire[0] = byte(sh.PacketType)
	idx := 1
	if connIDLen != 0 {
		copy(wire[idx:idx+connIDLen], sh.DestConnID)
		idx += connIDLen
	}
	sh.PacketNumber.PutWire(wire[idx:])
	return
}
<|endoftext|>


<commit_before>package executor

import (
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/go-ps"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"

	"github.com/hashicorp/nomad/client/stats"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// The statistics the executor exposes when using cgroups
	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
	ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
)

// configureIsolation configures chroot and creates cgroups
func (e *UniversalExecutor) configureIsolation() error {
	if e.command.FSIsolation {
		if err := e.configureChroot(); err != nil {
			return err
		}
	}

	if e.command.ResourceLimits || e.command.Cgroup {
		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
			return fmt.Errorf("error creating cgroups: %v", err)
		}
	}
	return nil
}

// applyLimits puts a process in a pre-configured cgroup
func (e *UniversalExecutor) applyLimits(pid int) error {
	if !(e.command.ResourceLimits || e.command.Cgroup) {
		return nil
	}

	// Entering the process in the cgroup
	manager := getCgroupManager(e.resConCtx.groups, nil)
	if err := manager.Apply(pid); err != nil {
		e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err)
		return err
	}
	e.resConCtx.cgPaths = manager.GetPaths()
	cgConfig := cgroupConfig.Config{Cgroups: e.resConCtx.groups}
	if err := manager.Set(&cgConfig); err != nil {
		e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err)
		if er := DestroyCgroup(e.resConCtx.groups, e.resConCtx.cgPaths, os.Getpid()); er != nil {
			e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er)
		}
		return err
	}
	return nil
}

// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
	e.resConCtx.groups = &cgroupConfig.Cgroup{}
	e.resConCtx.groups.Resources = &cgroupConfig.Resources{}
	cgroupName := uuid.Generate()
	e.resConCtx.groups.Path = filepath.Join("/nomad", cgroupName)

	// Allow access to /dev/
	e.resConCtx.groups.Resources.AllowAllDevices = true

	// Use a cgroup but don't apply limits
	if !e.command.ResourceLimits {
		return nil
	}

	if resources.MemoryMB > 0 {
		// Total amount of memory allowed to consume
		e.resConCtx.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
		// Disable swap to avoid issues on the machine
		var memSwappiness int64 = 0
		e.resConCtx.groups.Resources.MemorySwappiness = &memSwappiness
	}

	if resources.CPU < 2 {
		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
	}

	// Set the relative CPU shares for this cgroup.
	e.resConCtx.groups.Resources.CpuShares = int64(resources.CPU)

	if resources.IOPS != 0 {
		// Validate it is in an acceptable range.
		if resources.IOPS < 10 || resources.IOPS > 1000 {
			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
		}

		e.resConCtx.groups.Resources.BlkioWeight = uint16(resources.IOPS)
	}

	return nil
}

// Stats reports the resource utilization of the cgroup. If there is no resource
// isolation we aggregate the resource utilization of all the pids launched by
// the executor.
func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
	if !(e.command.ResourceLimits || e.command.Cgroup) {
		pidStats, err := e.pidStats()
		if err != nil {
			return nil, err
		}
		return e.aggregatedResourceUsage(pidStats), nil
	}
	ts := time.Now()
	manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
	stats, err := manager.GetStats()
	if err != nil {
		return nil, err
	}

	// Memory Related Stats
	swap := stats.MemoryStats.SwapUsage
	maxUsage := stats.MemoryStats.Usage.MaxUsage
	rss := stats.MemoryStats.Stats["rss"]
	cache := stats.MemoryStats.Stats["cache"]
	ms := &cstructs.MemoryStats{
		RSS:            rss,
		Cache:          cache,
		Swap:           swap.Usage,
		MaxUsage:       maxUsage,
		KernelUsage:    stats.MemoryStats.KernelUsage.Usage,
		KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
		Measured:       ExecutorCgroupMeasuredMemStats,
	}

	// CPU Related Stats
	totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
	userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
	kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)

	totalPercent := e.totalCpuStats.Percent(totalProcessCPUUsage)
	cs := &cstructs.CpuStats{
		SystemMode:       e.systemCpuStats.Percent(kernelModeTime),
		UserMode:         e.userCpuStats.Percent(userModeTime),
		Percent:          totalPercent,
		ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
		ThrottledTime:    stats.CpuStats.ThrottlingData.ThrottledTime,
		TotalTicks:       e.systemCpuStats.TicksConsumed(totalPercent),
		Measured:         ExecutorCgroupMeasuredCpuStats,
	}
	taskResUsage := cstructs.TaskResourceUsage{
		ResourceUsage: &cstructs.ResourceUsage{
			MemoryStats: ms,
			CpuStats:    cs,
		},
		Timestamp: ts.UTC().UnixNano(),
	}
	if pidStats, err := e.pidStats(); err == nil {
		taskResUsage.Pids = pidStats
	}
	return &taskResUsage, nil
}

// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
	u, err := user.Lookup(userid)
	if err != nil {
		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
	}

	// Get the groups the user is a part of
	gidStrings, err := u.GroupIds()
	if err != nil {
		return fmt.Errorf("Unable to lookup user's group membership: %v", err)
	}

	gids := make([]uint32, len(gidStrings))
	for _, gidString := range gidStrings {
		u, err := strconv.Atoi(gidString)
		if err != nil {
			return fmt.Errorf("Unable to convert user's group to int %s: %v", gidString, err)
		}

		gids = append(gids, uint32(u))
	}

	// Convert the uid and gid
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
	}
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
	}

	// Set the command to run as that user and group.
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	if e.cmd.SysProcAttr.Credential == nil {
		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
	}
	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
	e.cmd.SysProcAttr.Credential.Groups = gids

	e.logger.Printf("[DEBUG] executor: running as user:group %d:%d with group membership in %v", uid, gid, gids)

	return nil
}

// configureChroot configures a chroot
func (e *UniversalExecutor) configureChroot() error {
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	e.cmd.SysProcAttr.Chroot = e.ctx.TaskDir
	e.cmd.Dir = "/"

	e.fsIsolationEnforced = true
	return nil
}

// getAllPids returns the pids of all the processes spun up by the executor. We
// use the libcontainer apis to get the pids when the user is using cgroup
// isolation and we scan the entire process table if the user is not using any
// isolation
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
	if e.command.ResourceLimits || e.command.Cgroup {
		manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
		pids, err := manager.GetAllPids()
		if err != nil {
			return nil, err
		}
		np := make(map[int]*nomadPid, len(pids))
		for _, pid := range pids {
			np[pid] = &nomadPid{
				pid:           pid,
				cpuStatsTotal: stats.NewCpuStats(),
				cpuStatsSys:   stats.NewCpuStats(),
				cpuStatsUser:  stats.NewCpuStats(),
			}
		}
		return np, nil
	}
	allProcesses, err := ps.Processes()
	if err != nil {
		return nil, err
	}
	return e.scanPids(os.Getpid(), allProcesses)
}

// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host. This function is idempotent.
func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error {
	mErrs := new(multierror.Error)
	if groups == nil {
		return fmt.Errorf("Can't destroy: cgroup configuration empty")
	}

	// Move the executor into the global cgroup so that the task specific
	// cgroup can be destroyed.
	nilGroup := &cgroupConfig.Cgroup{}
	nilGroup.Path = "/"
	nilGroup.Resources = groups.Resources
	nilManager := getCgroupManager(nilGroup, nil)
	err := nilManager.Apply(executorPid)
	if err != nil && !strings.Contains(err.Error(), "no such process") {
		return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err)
	}

	// Freeze the Cgroup so that it can not continue to fork/exec.
	manager := getCgroupManager(groups, cgPaths)
	err = manager.Freeze(cgroupConfig.Frozen)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		return fmt.Errorf("failed to freeze cgroup: %v", err)
	}

	var procs []*os.Process
	pids, err := manager.GetAllPids()
	if err != nil {
		multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))

		// Unfreeze the cgroup.
		err = manager.Freeze(cgroupConfig.Thawed)
		if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
			multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
		}
		return mErrs.ErrorOrNil()
	}

	// Kill the processes in the cgroup
	for _, pid := range pids {
		proc, err := os.FindProcess(pid)
		if err != nil {
			multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
			continue
		}

		procs = append(procs, proc)
		if e := proc.Kill(); e != nil {
			multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
		}
	}

	// Unfreeze the cgroug so we can wait.
	err = manager.Freeze(cgroupConfig.Thawed)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
	}

	// Wait on the killed processes to ensure they are cleaned up.
	for _, proc := range procs {
		// Don't capture the error because we expect this to fail for
		// processes we didn't fork.
		proc.Wait()
	}

	// Remove the cgroup.
	if err := manager.Destroy(); err != nil {
		multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
	}
	return mErrs.ErrorOrNil()
}

// getCgroupManager returns the correct libcontainer cgroup manager.
func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager {
	return &cgroupFs.Manager{Cgroups: groups, Paths: paths}
}
<commit_msg>Only use freezer/devices when in the basic cgroup only<commit_after>package executor

import (
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/go-ps"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"

	"github.com/hashicorp/nomad/client/stats"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// The statistics the executor exposes when using cgroups
	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
	ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
)

// configureIsolation configures chroot and creates cgroups
func (e *UniversalExecutor) configureIsolation() error {
	if e.command.FSIsolation {
		if err := e.configureChroot(); err != nil {
			return err
		}
	}

	if e.command.ResourceLimits || e.command.Cgroup {
		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
			return fmt.Errorf("error creating cgroups: %v", err)
		}
	}
	return nil
}

// applyLimits puts a process in a pre-configured cgroup
func (e *UniversalExecutor) applyLimits(pid int) error {
	if !(e.command.ResourceLimits || e.command.Cgroup) {
		return nil
	}

	// Entering the process in the cgroup
	manager := getCgroupManager(e.resConCtx.groups, nil)
	if err := manager.Apply(pid); err != nil {
		e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err)
		return err
	}

	e.resConCtx.cgPaths = manager.GetPaths()

	// Don't enter all the cgroups since we will inherit resources limits. Only
	// use devices (required by libcontainer) and freezer. Freezer allows us to
	// capture all pids and stop any fork/execs from happening while we are
	// cleaning up.
	if !e.command.ResourceLimits {
		// Move the executor into the global cgroup so that the task specific
		// cgroup can be destroyed.
		nilGroup := &cgroupConfig.Cgroup{}
		nilGroup.Path = "/"
		nilGroup.Resources = e.resConCtx.groups.Resources
		nilManager := getCgroupManager(nilGroup, nil)
		err := nilManager.Apply(pid)
		if err != nil {
			return fmt.Errorf("failed to remove executor pid %d: %v", pid, err)
		}

		// Grab the freezer and devices cgroup paths. We do this from the old
		// manager after the executor pid has been applied since there is no
		// other way to determine what the proper cgroup paths would be.
		freezer := &cgroupFs.FreezerGroup{}
		devices := &cgroupFs.DevicesGroup{}
		freezerName, devicesName := freezer.Name(), devices.Name()
		newPath := map[string]string{
			freezerName: e.resConCtx.cgPaths[freezerName],
			devicesName: e.resConCtx.cgPaths[devicesName],
		}

		// Clear the cgroups paths so that everything is properly cleaned except
		// the groups we want our process to stay in. This will delete the
		// directories from disk.
		manager.Cgroups.Paths = nil
		delete(manager.Paths, freezerName)
		delete(manager.Paths, devicesName)
		if err := manager.Destroy(); err != nil {
			e.logger.Printf("[ERR] executor: failed to destroy original: %v", err)
			return err
		}

		// Update our context such that the new cgroup manager only is tracking
		// the paths we care about now.
		e.resConCtx.cgPaths = newPath
		e.resConCtx.groups.Paths = newPath

		// Apply just the freezer and devices now
		manager = getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
		if err := manager.Apply(pid); err != nil {
			e.logger.Printf("[ERR] executor: error applying pid to cgroup subset %v: %v", e.resConCtx.cgPaths, err)
			return err
		}
	}

	cgConfig := cgroupConfig.Config{Cgroups: e.resConCtx.groups}
	if err := manager.Set(&cgConfig); err != nil {
		e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err)
		if er := DestroyCgroup(e.resConCtx.groups, e.resConCtx.cgPaths, os.Getpid()); er != nil {
			e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er)
		}
		return err
	}
	return nil
}

// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
	e.resConCtx.groups = &cgroupConfig.Cgroup{}
	e.resConCtx.groups.Resources = &cgroupConfig.Resources{}
	cgroupName := uuid.Generate()
	e.resConCtx.groups.Path = filepath.Join("/nomad", cgroupName)

	// Allow access to /dev/
	e.resConCtx.groups.Resources.AllowAllDevices = true

	// Use a cgroup but don't apply limits
	if !e.command.ResourceLimits {
		return nil
	}

	if resources.MemoryMB > 0 {
		// Total amount of memory allowed to consume
		e.resConCtx.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
		// Disable swap to avoid issues on the machine
		var memSwappiness int64 = 0
		e.resConCtx.groups.Resources.MemorySwappiness = &memSwappiness
	}

	if resources.CPU < 2 {
		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
	}

	// Set the relative CPU shares for this cgroup.
	e.resConCtx.groups.Resources.CpuShares = int64(resources.CPU)

	if resources.IOPS != 0 {
		// Validate it is in an acceptable range.
		if resources.IOPS < 10 || resources.IOPS > 1000 {
			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
		}

		e.resConCtx.groups.Resources.BlkioWeight = uint16(resources.IOPS)
	}

	return nil
}

// Stats reports the resource utilization of the cgroup. If there is no resource
// isolation we aggregate the resource utilization of all the pids launched by
// the executor.
func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
	// If we don't use full resource limits fallback to normal collection. It is
	// not enough to be in the Cgroup since you must be in the memory, cpu, and
	// cpuacct cgroup to gather the correct statistics.
	if !e.command.ResourceLimits {
		pidStats, err := e.pidStats()
		if err != nil {
			return nil, err
		}
		return e.aggregatedResourceUsage(pidStats), nil
	}
	ts := time.Now()
	manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
	stats, err := manager.GetStats()
	if err != nil {
		return nil, err
	}

	// Memory Related Stats
	swap := stats.MemoryStats.SwapUsage
	maxUsage := stats.MemoryStats.Usage.MaxUsage
	rss := stats.MemoryStats.Stats["rss"]
	cache := stats.MemoryStats.Stats["cache"]
	ms := &cstructs.MemoryStats{
		RSS:            rss,
		Cache:          cache,
		Swap:           swap.Usage,
		MaxUsage:       maxUsage,
		KernelUsage:    stats.MemoryStats.KernelUsage.Usage,
		KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
		Measured:       ExecutorCgroupMeasuredMemStats,
	}

	// CPU Related Stats
	totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
	userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
	kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)

	totalPercent := e.totalCpuStats.Percent(totalProcessCPUUsage)
	cs := &cstructs.CpuStats{
		SystemMode:       e.systemCpuStats.Percent(kernelModeTime),
		UserMode:         e.userCpuStats.Percent(userModeTime),
		Percent:          totalPercent,
		ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
		ThrottledTime:    stats.CpuStats.ThrottlingData.ThrottledTime,
		TotalTicks:       e.systemCpuStats.TicksConsumed(totalPercent),
		Measured:         ExecutorCgroupMeasuredCpuStats,
	}
	taskResUsage := cstructs.TaskResourceUsage{
		ResourceUsage: &cstructs.ResourceUsage{
			MemoryStats: ms,
			CpuStats:    cs,
		},
		Timestamp: ts.UTC().UnixNano(),
	}
	if pidStats, err := e.pidStats(); err == nil {
		taskResUsage.Pids = pidStats
	}
	return &taskResUsage, nil
}

// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
	u, err := user.Lookup(userid)
	if err != nil {
		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
	}

	// Get the groups the user is a part of
	gidStrings, err := u.GroupIds()
	if err != nil {
		return fmt.Errorf("Unable to lookup user's group membership: %v", err)
	}

	gids := make([]uint32, len(gidStrings))
	for _, gidString := range gidStrings {
		u, err := strconv.Atoi(gidString)
		if err != nil {
			return fmt.Errorf("Unable to convert user's group to int %s: %v", gidString, err)
		}

		gids = append(gids, uint32(u))
	}

	// Convert the uid and gid
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
	}
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
	}

	// Set the command to run as that user and group.
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	if e.cmd.SysProcAttr.Credential == nil {
		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
	}
	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
	e.cmd.SysProcAttr.Credential.Groups = gids

	e.logger.Printf("[DEBUG] executor: running as user:group %d:%d with group membership in %v", uid, gid, gids)

	return nil
}

// configureChroot configures a chroot
func (e *UniversalExecutor) configureChroot() error {
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	e.cmd.SysProcAttr.Chroot = e.ctx.TaskDir
	e.cmd.Dir = "/"

	e.fsIsolationEnforced = true
	return nil
}

// getAllPids returns the pids of all the processes spun up by the executor. We
// use the libcontainer apis to get the pids when the user is using cgroup
// isolation and we scan the entire process table if the user is not using any
// isolation
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
	if e.command.ResourceLimits || e.command.Cgroup {
		manager := getCgroupManager(e.resConCtx.groups, e.resConCtx.cgPaths)
		pids, err := manager.GetAllPids()
		if err != nil {
			return nil, err
		}
		np := make(map[int]*nomadPid, len(pids))
		for _, pid := range pids {
			np[pid] = &nomadPid{
				pid:           pid,
				cpuStatsTotal: stats.NewCpuStats(),
				cpuStatsSys:   stats.NewCpuStats(),
				cpuStatsUser:  stats.NewCpuStats(),
			}
		}
		return np, nil
	}
	allProcesses, err := ps.Processes()
	if err != nil {
		return nil, err
	}
	return e.scanPids(os.Getpid(), allProcesses)
}

// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host. This function is idempotent.
func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error {
	mErrs := new(multierror.Error)
	if groups == nil {
		return fmt.Errorf("Can't destroy: cgroup configuration empty")
	}

	// Move the executor into the global cgroup so that the task specific
	// cgroup can be destroyed.
	nilGroup := &cgroupConfig.Cgroup{}
	nilGroup.Path = "/"
	nilGroup.Resources = groups.Resources
	nilManager := getCgroupManager(nilGroup, nil)
	err := nilManager.Apply(executorPid)
	if err != nil && !strings.Contains(err.Error(), "no such process") {
		return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err)
	}

	// Freeze the Cgroup so that it can not continue to fork/exec.
	manager := getCgroupManager(groups, cgPaths)
	err = manager.Freeze(cgroupConfig.Frozen)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		return fmt.Errorf("failed to freeze cgroup: %v", err)
	}

	var procs []*os.Process
	pids, err := manager.GetAllPids()
	if err != nil {
		multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))

		// Unfreeze the cgroup.
		err = manager.Freeze(cgroupConfig.Thawed)
		if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
			multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
		}
		return mErrs.ErrorOrNil()
	}

	// Kill the processes in the cgroup
	for _, pid := range pids {
		proc, err := os.FindProcess(pid)
		if err != nil {
			multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
			continue
		}

		procs = append(procs, proc)
		if e := proc.Kill(); e != nil {
			multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
		}
	}

	// Unfreeze the cgroug so we can wait.
	err = manager.Freeze(cgroupConfig.Thawed)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
	}

	// Wait on the killed processes to ensure they are cleaned up.
	for _, proc := range procs {
		// Don't capture the error because we expect this to fail for
		// processes we didn't fork.
		proc.Wait()
	}

	// Clear the cgroups paths so that everything is properly cleaned
	manager.Cgroups.Paths = nil

	// Remove the cgroup.
	if err := manager.Destroy(); err != nil {
		multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
	}
	return mErrs.ErrorOrNil()
}

// getCgroupManager returns the correct libcontainer cgroup manager.
func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) *cgroupFs.Manager {
	return &cgroupFs.Manager{Cgroups: groups, Paths: paths}
}
<|endoftext|>


<commit_before>package transaction

import (
	"errors"
	"strconv"
	"time"

	"github.com/evalphobia/go-paypal-classic/client"
)

// TransactionSearch is struct for TransactionSearch API
// see: https://developer.paypal.com/docs/classic/api/merchant/TransactionSearch_API_Operation_NVP/
type TransactionSearch struct {
	client.BaseRequest `url:",squash"`
	ParamStartDate     string `url:"STARTDATE"`
	ParamEndDate       string `url:"ENDDATE,omitempty"`

	// Required
	StartDate time.Time `url:"-"`

	// Optional
	EndDate time.Time `url:"-"`

	ProfileID         string `url:"PROFILEID,omitempty"`
	TransactionID     string `url:"TRANSACTIONID,omitempty"`
	RceiptID          string `url:"RECEIPTID,omitempty"`
	InvNum            string `url:"INVNUM,omitempty"`
	AuctionItemNumber string `url:"AUCTIONITEMNUMBER,omitempty"`

	Email    string `url:"EMAIL,omitempty"`
	Receiver string `url:"RECEIVER,omitempty"`

	Status           string `url:"STATUS,omitempty"`
	TransactionClass string `url:"TRANSACTIONCLASS,omitempty"`
}

// Do executes TransactionSearch operation
func (svc *TransactionSearch) Do(cli client.Client) (*TransactionSearchResponse, error) {
	const method = "TransactionSearch"
	svc.BaseRequest.Method = method

	if !svc.StartDate.IsZero() {
		svc.ParamStartDate = timeToString(svc.StartDate)
	}
	if !svc.EndDate.IsZero() {
		svc.ParamEndDate = timeToString(svc.EndDate)
	}

	result := &TransactionSearchResponse{}
	err := cli.Call(svc, result)
	return result, err
}

// TransactionSearchResponse is struct for response of GetRecurringPaymentsProfileDetails API
type TransactionSearchResponse struct {
	client.BaseResponse `url:",squash"`

	Items []TransactionSearchItem
}

// IsSuccess checks the request is success or not
func (r *TransactionSearchResponse) IsSuccess() bool {
	return r.IsRequestSuccess()
}

// Error returns error text
func (r *TransactionSearchResponse) Error() string {
	return r.BaseResponse.Error()
}

// Unmarshal assigns results from map.
func (r *TransactionSearchResponse) Unmarshal(mapData map[string]interface{}) error {
	if _, ok := mapData["L_TRANSACTIONID0"]; !ok {
		return errors.New("Cannot unmarshal result")
	}

	max := 1
	for {
		_, ok := mapData["L_TRANSACTIONID"+strconv.Itoa(max)]
		if !ok {
			break
		}
		max++
	}

	r.Items = make([]TransactionSearchItem, max)
	for i := 0; i < max; i++ {
		r.Items[i] = createTransactionSearchItem(mapData, i)
	}
	return nil
}

// TransactionSearchItem is single transaction.
type TransactionSearchItem struct {
	Timestamp     time.Time
	Timezone      string
	Type          string
	Email         string
	Name          string
	TransactionID string
	Status        string
	Amount        float64
	Currency      string
	FeeAmount     float64
	NetAmount     float64
}

func createTransactionSearchItem(mapData map[string]interface{}, i int) TransactionSearchItem {
	result := TransactionSearchItem{}
	num := strconv.Itoa(i)

	if v, ok := mapData["L_TIMESTAMP"+num].(string); ok {
		result.Timestamp, _ = stringToTime(v)
	}
	if v, ok := mapData["L_TIMEZONE"+num].(string); ok {
		result.Timezone = v
	}
	if v, ok := mapData["L_TYPE"+num].(string); ok {
		result.Type = v
	}
	if v, ok := mapData["L_EMAIL"+num].(string); ok {
		result.Email = v
	}
	if v, ok := mapData["L_NAME"+num].(string); ok {
		result.Name = v
	}
	if v, ok := mapData["L_TRANSACTIONID"+num].(string); ok {
		result.TransactionID = v
	}
	if v, ok := mapData["L_STATUS"+num].(string); ok {
		result.Status = v
	}
	if v, ok := mapData["L_AMT"+num].(float64); ok {
		result.Amount = v
	}
	if v, ok := mapData["L_CURRENCYCODE"+num].(string); ok {
		result.Currency = v
	}
	if v, ok := mapData["L_FEEAMT"+num].(float64); ok {
		result.FeeAmount = v
	}
	if v, ok := mapData["L_NETAMT"+num].(float64); ok {
		result.NetAmount = v
	}
	return result
}
<commit_msg>Add helper methods for TransactionSearch<commit_after>package transaction

import (
	"errors"
	"strconv"
	"strings"
	"time"

	"github.com/evalphobia/go-paypal-classic/client"
)

// TransactionSearch is struct for TransactionSearch API
// see: https://developer.paypal.com/docs/classic/api/merchant/TransactionSearch_API_Operation_NVP/
type TransactionSearch struct {
	client.BaseRequest `url:",squash"`
	ParamStartDate     string `url:"STARTDATE"`
	ParamEndDate       string `url:"ENDDATE,omitempty"`

	// Required
	StartDate time.Time `url:"-"`

	// Optional
	EndDate time.Time `url:"-"`

	ProfileID         string `url:"PROFILEID,omitempty"`
	TransactionID     string `url:"TRANSACTIONID,omitempty"`
	RceiptID          string `url:"RECEIPTID,omitempty"`
	InvNum            string `url:"INVNUM,omitempty"`
	AuctionItemNumber string `url:"AUCTIONITEMNUMBER,omitempty"`

	Email    string `url:"EMAIL,omitempty"`
	Receiver string `url:"RECEIVER,omitempty"`

	Status           string `url:"STATUS,omitempty"`
	TransactionClass string `url:"TRANSACTIONCLASS,omitempty"`
}

// Do executes TransactionSearch operation
func (svc *TransactionSearch) Do(cli client.Client) (*TransactionSearchResponse, error) {
	const method = "TransactionSearch"
	svc.BaseRequest.Method = method

	if !svc.StartDate.IsZero() {
		svc.ParamStartDate = timeToString(svc.StartDate)
	}
	if !svc.EndDate.IsZero() {
		svc.ParamEndDate = timeToString(svc.EndDate)
	}

	result := &TransactionSearchResponse{}
	err := cli.Call(svc, result)
	return result, err
}

// TransactionSearchResponse is struct for response of GetRecurringPaymentsProfileDetails API
type TransactionSearchResponse struct {
	client.BaseResponse `url:",squash"`

	Items []TransactionSearchItem
}

// IsSuccess checks the request is success or not
func (r *TransactionSearchResponse) IsSuccess() bool {
	return r.IsRequestSuccess()
}

// Error returns error text
func (r *TransactionSearchResponse) Error() string {
	return r.BaseResponse.Error()
}

// Unmarshal assigns results from map.
func (r *TransactionSearchResponse) Unmarshal(mapData map[string]interface{}) error {
	if _, ok := mapData["L_TRANSACTIONID0"]; !ok {
		return errors.New("Cannot unmarshal result")
	}

	max := 1
	for {
		_, ok := mapData["L_TRANSACTIONID"+strconv.Itoa(max)]
		if !ok {
			break
		}
		max++
	}

	r.Items = make([]TransactionSearchItem, max)
	for i := 0; i < max; i++ {
		r.Items[i] = createTransactionSearchItem(mapData, i)
	}
	return nil
}

// Transactions returns list of transactions.
func (r *TransactionSearchResponse) Transactions() (list []TransactionSearchItem) {
	for _, item := range r.Items {
		if item.IsTransaction() {
			list = append(list, item)
		}
	}
	return
}

// TransactionIDList returns list of transaction id for payment.
func (r *TransactionSearchResponse) TransactionIDList() (list []string) {
	for _, item := range r.Items {
		if item.IsTransaction() {
			list = append(list, item.TransactionID)
		}
	}
	return
}

// GetCancelDate returns time of canceled.
func (r *TransactionSearchResponse) GetCancelDate() (time.Time, bool) {
	for i := len(r.Items) - 1; i >= 0; i-- {
		item := r.Items[i]
		if item.IsProfile() && item.IsCanceled() {
			return item.Timestamp, true
		}
	}
	return time.Time{}, false
}

// TransactionSearchItem is single transaction.
type TransactionSearchItem struct {
	Timestamp     time.Time
	Timezone      string
	Type          string
	Email         string
	Name          string
	TransactionID string
	Status        string
	Amount        float64
	Currency      string
	FeeAmount     float64
	NetAmount     float64
}

func createTransactionSearchItem(mapData map[string]interface{}, i int) TransactionSearchItem {
	result := TransactionSearchItem{}
	num := strconv.Itoa(i)

	if v, ok := mapData["L_TIMESTAMP"+num].(string); ok {
		result.Timestamp, _ = stringToTime(v)
	}
	if v, ok := mapData["L_TIMEZONE"+num].(string); ok {
		result.Timezone = v
	}
	if v, ok := mapData["L_TYPE"+num].(string); ok {
		result.Type = v
	}
	if v, ok := mapData["L_EMAIL"+num].(string); ok {
		result.Email = v
	}
	if v, ok := mapData["L_NAME"+num].(string); ok {
		result.Name = v
	}
	if v, ok := mapData["L_TRANSACTIONID"+num].(string); ok {
		result.TransactionID = v
	}
	if v, ok := mapData["L_STATUS"+num].(string); ok {
		result.Status = v
	}
	if v, ok := mapData["L_AMT"+num].(float64); ok {
		result.Amount = v
	}
	if v, ok := mapData["L_CURRENCYCODE"+num].(string); ok {
		result.Currency = v
	}
	if v, ok := mapData["L_FEEAMT"+num].(float64); ok {
		result.FeeAmount = v
	}
	if v, ok := mapData["L_NETAMT"+num].(float64); ok {
		result.NetAmount = v
	}
	return result
}

// IsProfile checks the TransactionID is Profile or not.
func (item TransactionSearchItem) IsProfile() bool {
	return strings.HasPrefix(item.TransactionID, "I-")
}

// IsTransaction checks the TransactionID is Transaction or not.
func (item TransactionSearchItem) IsTransaction() bool {
	return !item.IsProfile()
}

// IsRecurringPayment checks the transaction is Recurring Payment or not.
func (item TransactionSearchItem) IsRecurringPayment() bool {
	return item.Type == "Recurring Payment"
}

// IsPayment checks the transaction is Payment or not.
func (item TransactionSearchItem) IsPayment() bool {
	return item.Type == "Payment"
}

// IsCreated checks the transaction is created or not.
func (item TransactionSearchItem) IsCreated() bool {
	return item.Status == "Created"
}

// IsCanceled checks the transaction is canceled or not.
func (item TransactionSearchItem) IsCanceled() bool {
	return item.Status == "Canceled"
}

// IsCompleted checks the transaction is completed or not.
func (item TransactionSearchItem) IsCompleted() bool {
	return item.Status == "Completed"
}
<|endoftext|>


<commit_before><commit_msg>Tolerate columns added to shema before all old rows are processed<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os/user"
	"strconv"
	"strings"
)

var uidMap = map[string]user.User{}

//This does the heavy lifting of making auditd messages into json.
//It is NOT thread (goroutine) safe, as it uses two shared global maps
//It can be made threadsafe by locking these if performance becomes an issue.
func makeJsonString(evBuf map[int]map[string]string, dstring string, c chan<- string) {
	data := strings.Fields(dstring)
	_, seq := parseAuditHeader(data[0])
	if _, ok := evBuf[seq]; ok == false {
		evBuf[seq] = make(map[string]string)
	}
	//End of the event - send to channel to process
	//This is a brutal hack and we need to detect the EOE properly
	if len(data) == 1 {
		c <- mapToJsonString(evBuf[seq])
		delete(evBuf, seq)
	}
	splitz := []string{}

	//Divide into key value pairs for conversion to json
	for i := 0; i < len(data); i++ {
		a := data
		splitz = strings.SplitN(a[i], "=", 2)
		switch len(splitz) {
		//This means we found a key/value pair
		case 2:
			evBuf[seq][splitz[0]] = splitz[1]
			switch splitz[0] {
			//Get arg count and generate complete "command" element.
			case "argc":
				defer parseArgs(evBuf[seq])
			//Add contextual info per host if we can look up the uid
			case "uid":
				parseUid(evBuf[seq], "uid")
			case "auid":
				parseUid(evBuf[seq], "auid")
			}
		//Empty case here because this is what the header looks like and we don't want to error
		case 1:
		//This shouldn't happen
		default:
			log.Fatal("unexpected split: ", splitz)
		}
	}

}

//Parses the auditd headers. Example: 'audit(1446832550.594:2767175):'
//Where the first number in parens is the timestamp
//and the second is the sequence number of the event
func parseAuditHeader(data string) (time string, seq int) {
	if len(data) > 6 && data[:6] == "audit(" {
		timeAndSeq := strings.Split(strings.Trim(data, "audit():"), ":")
		seq, _ := strconv.Atoi(timeAndSeq[1])
		return timeAndSeq[0], seq
	} else {
		return
	}
}

//Iterates over argc(int) and combines individual a[num] into a single "command" key
func parseArgs(m map[string]string) {
	argc, _ := strconv.Atoi(m["argc"])
	if _, ok := m["argc"]; ok {
		command := []byte{}
		for i := 0; i < argc; i++ {
			element := fmt.Sprintf("a%d", i)
			//If surrounded by quotes, it is a literal. Just remove the double quotes.
			//If not surrounded by quotes, it is hex encoded. Decode the text.
			if m[element][0] == '"' {
				m[element] = strings.Trim(m[element], "\"")
				command = append(command, ' ')
				command = append(command, m[element]...)

			} else {
				n, _ := hex.DecodeString(m[element])
				command = append(command, ' ')
				command = append(command, n...)
				m[element] = string(n)
			}
		}
		//Store assembled command text into "command" key in passed map
		//Drop the first character, which is always a space
		m["command"] = string(command)[1:]
	}
}

//Simple wrapper to make the map into json
func mapToJsonString(m map[string]string) string {
	s, err := json.Marshal(m)
	if err != nil {
		log.Fatal(err)
	}
	out := fmt.Sprintf("%s", s)
	return out
}

//This takes the map to modify and a key name and adds the username to a new key with "_username"
func parseUid(m map[string]string, uidKey string) {
	//Make sure we have a uid element to work with.
	if uid, ok := m[uidKey]; ok {
		//Give a default value in case we don't find something.
		m[uidKey+"_username"] = "UNKNOWN_USER"
		if luser, ok := uidMap[uid]; ok {
			m[uidKey+"_username"] = luser.Username
		} else {
			luser, err := user.LookupId(uid)
			if err == nil {
				m[uidKey+"_username"] = luser.Username
				uidMap[uid] = *luser
				//Probably redundant. FIX
			} else {
				uidMap[uid] = user.User{Username: "UNKNOWN_USER"}
			}
		}
	}
}
<commit_msg>sometimes a weird things happens. we now handle that<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os/user"
	"strconv"
	"strings"
)

var uidMap = map[string]user.User{}

//This does the heavy lifting of making auditd messages into json.
//It is NOT thread (goroutine) safe, as it uses two shared global maps
//It can be made threadsafe by locking these if performance becomes an issue.
func makeJsonString(evBuf map[int]map[string]string, dstring string, c chan<- string) {
	data := strings.Fields(dstring)
	_, seq := parseAuditHeader(data[0])
	if _, ok := evBuf[seq]; ok == false {
		evBuf[seq] = make(map[string]string)
	}
	//End of the event - send to channel to process
	//This is a brutal hack and we need to detect the EOE properly
	if len(data) == 1 {
		c <- mapToJsonString(evBuf[seq])
		delete(evBuf, seq)
	}
	splitz := []string{}

	//Divide into key value pairs for conversion to json
	for i := 0; i < len(data); i++ {
		a := data
		splitz = strings.SplitN(a[i], "=", 2)
		switch len(splitz) {
		//This means we found a key/value pair
		case 2:
			evBuf[seq][splitz[0]] = splitz[1]
			switch splitz[0] {
			//Get arg count and generate complete "command" element.
			case "argc":
				defer parseArgs(evBuf[seq])
			//Add contextual info per host if we can look up the uid
			case "uid":
				parseUid(evBuf[seq], "uid")
			case "auid":
				parseUid(evBuf[seq], "auid")
			}
		//Empty case here because this is what the header looks like and we don't want to error
		case 1:
		//This shouldn't happen
		default:
			log.Fatal("unexpected split: ", splitz)
		}
	}

}

//Parses the auditd headers. Example: 'audit(1446832550.594:2767175):'
//Where the first number in parens is the timestamp
//and the second is the sequence number of the event
func parseAuditHeader(data string) (time string, seq int) {
	if len(data) > 6 && data[:6] == "audit(" {
		timeAndSeq := strings.Split(strings.Trim(data, "audit():"), ":")
		seq, _ := strconv.Atoi(timeAndSeq[1])
		return timeAndSeq[0], seq
	} else {
		return
	}
}

//Iterates over argc(int) and combines individual a[num] into a single "command" key
func parseArgs(m map[string]string) {
	argc, _ := strconv.Atoi(m["argc"])
	if _, ok := m["argc"]; ok {
		command := []byte{}
		for i := 0; i < argc; i++ {
			element := fmt.Sprintf("a%d", i)
			//If surrounded by quotes, it is a literal. Just remove the double quotes.
			//If not surrounded by quotes, it is hex encoded. Decode the text.
			if len(m[element]) > 0 {
				if m[element][0] == '"' {
					m[element] = strings.Trim(m[element], "\"")
					command = append(command, ' ')
					command = append(command, m[element]...)

				} else {
					n, _ := hex.DecodeString(m[element])
					command = append(command, ' ')
					command = append(command, n...)
					m[element] = string(n)
				}
			} else {
				log.Output(1, string("Failed to parse args in "+fmt.Sprintf("%s", m)))
			}

		}
		//Store assembled command text into "command" key in passed map
		//Drop the first character, which is always a space
		m["command"] = string(command)[1:]
	}
}

//Simple wrapper to make the map into json
func mapToJsonString(m map[string]string) string {
	s, err := json.Marshal(m)
	if err != nil {
		log.Fatal(err)
	}
	out := fmt.Sprintf("%s", s)
	return out
}

//This takes the map to modify and a key name and adds the username to a new key with "_username"
func parseUid(m map[string]string, uidKey string) {
	//Make sure we have a uid element to work with.
	if uid, ok := m[uidKey]; ok {
		//Give a default value in case we don't find something.
		m[uidKey+"_username"] = "UNKNOWN_USER"
		if luser, ok := uidMap[uid]; ok {
			m[uidKey+"_username"] = luser.Username
		} else {
			luser, err := user.LookupId(uid)
			if err == nil {
				m[uidKey+"_username"] = luser.Username
				uidMap[uid] = *luser
				//Probably redundant. FIX
			} else {
				uidMap[uid] = user.User{Username: "UNKNOWN_USER"}
			}
		}
	}
}
<|endoftext|>


<commit_before>package render

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"net/http"
	"reflect"
)

var Respond = DefaultRespond

func DefaultRespond(w http.ResponseWriter, r *http.Request, v interface{}) {
	switch reflect.TypeOf(v).Kind() {
	case reflect.Chan:
		switch getContentType(r) {
		case ContentTypeEventStream:
			EventStream(w, r, v)
			return
		default:
			channelIntoSlice(w, r, v)
			return
		}
	}

	// Present the object.
	if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
		v = presenter.Present(r, v)
	} else {
		v = DefaultPresenter.Present(r, v)
	}

	// Format data based on Content-Type.
	switch getContentType(r) {
	case ContentTypeJSON:
		JSON(w, r, v)
	case ContentTypeXML:
		XML(w, r, v)
	default:
		JSON(w, r, v)
	}
}

func PlainText(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	w.Write([]byte(v))
}

func HTML(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	w.Write([]byte(v))
}

func JSON(w http.ResponseWriter, r *http.Request, v interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	enc := json.NewEncoder(w)
	enc.SetEscapeHTML(true)
	if err := enc.Encode(v); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func XML(w http.ResponseWriter, r *http.Request, v interface{}) {
	b, err := xml.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	// Try to find <?xml header in first 100 bytes (just in case there're some XML comments).
	findHeaderUntil := len(b)
	if findHeaderUntil > 100 {
		findHeaderUntil = 100
	}
	if bytes.Index(b[:findHeaderUntil], []byte("<?xml")) == -1 {
		// No header found. Print it out first.
		w.Write([]byte(xml.Header))
	}

	w.Write(b)
}

func NoContent(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(204)
}

func EventStream(w http.ResponseWriter, r *http.Request, v interface{}) {
	if reflect.TypeOf(v).Kind() != reflect.Chan {
		panic(fmt.Sprintf("render.EventStream() expects channel, not %v", reflect.TypeOf(v).Kind()))
	}

	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")
	w.WriteHeader(200)

	ctx := r.Context()
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(v)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			w.Write([]byte("event: error\ndata: {\"error\":\"Server Timeout\"}\n\n"))
			return

		default: // equivalent to: case recv, ok := <-stream
			if !ok {
				w.Write([]byte("event: EOF\n\n"))
				return
			}

			// TODO: Can't use json Encoder - it panics on bufio.Flush(). Why?!
			// enc := json.NewEncoder(w)
			// enc.Encode(recv.Interface())
			bytes, err := json.Marshal(recv.Interface())
			if err != nil {
				w.Write([]byte(fmt.Sprintf("event: error\ndata: {\"error\":\"%v\"}\n\n", err)))
				if f, ok := w.(http.Flusher); ok {
					f.Flush()
				}
				continue
			}
			w.Write([]byte(fmt.Sprintf("event: data\ndata: %s\n\n", bytes)))
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}
}

// channelIntoSlice buffers channel data and responds with complete slice.
func channelIntoSlice(w http.ResponseWriter, r *http.Request, v interface{}) {
	ctx := r.Context()

	var resp []interface{}
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(v)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			http.Error(w, "Server Timeout", 504)
			return

		default: // equivalent to: case recv, ok := <-stream
			if !ok {
				DefaultRespond(w, r, resp)
				return
			}
			resp = append(resp, recv.Interface())
		}
	}
	panic("unreachable")
}
<commit_msg>Enable Presenter for EventStream<commit_after>package render

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"net/http"
	"reflect"
)

var Respond = DefaultRespond

func DefaultRespond(w http.ResponseWriter, r *http.Request, v interface{}) {
	switch reflect.TypeOf(v).Kind() {
	case reflect.Chan:
		switch getContentType(r) {
		case ContentTypeEventStream:
			EventStream(w, r, v)
			return
		default:
			channelIntoSlice(w, r, v)
			return
		}
	}

	// Present the object.
	if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
		v = presenter.Present(r, v)
	} else {
		v = DefaultPresenter.Present(r, v)
	}

	// Format data based on Content-Type.
	switch getContentType(r) {
	case ContentTypeJSON:
		JSON(w, r, v)
	case ContentTypeXML:
		XML(w, r, v)
	default:
		JSON(w, r, v)
	}
}

func PlainText(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	w.Write([]byte(v))
}

func HTML(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	w.Write([]byte(v))
}

func JSON(w http.ResponseWriter, r *http.Request, v interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	enc := json.NewEncoder(w)
	enc.SetEscapeHTML(true)
	if err := enc.Encode(v); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func XML(w http.ResponseWriter, r *http.Request, v interface{}) {
	b, err := xml.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	// Try to find <?xml header in first 100 bytes (just in case there're some XML comments).
	findHeaderUntil := len(b)
	if findHeaderUntil > 100 {
		findHeaderUntil = 100
	}
	if bytes.Index(b[:findHeaderUntil], []byte("<?xml")) == -1 {
		// No header found. Print it out first.
		w.Write([]byte(xml.Header))
	}

	w.Write(b)
}

func NoContent(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(204)
}

func EventStream(w http.ResponseWriter, r *http.Request, v interface{}) {
	if reflect.TypeOf(v).Kind() != reflect.Chan {
		panic(fmt.Sprintf("render.EventStream() expects channel, not %v", reflect.TypeOf(v).Kind()))
	}

	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")
	w.WriteHeader(200)

	ctx := r.Context()
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(v)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			w.Write([]byte("event: error\ndata: {\"error\":\"Server Timeout\"}\n\n"))
			return

		default: // equivalent to: case v, ok := <-stream
			if !ok {
				w.Write([]byte("event: EOF\n\n"))
				return
			}
			v := recv.Interface()

			// Present the object.
			if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
				v = presenter.Present(r, v)
			} else {
				v = DefaultPresenter.Present(r, v)
			}

			// TODO: Can't use json Encoder - it panics on bufio.Flush(). Why?!
			// enc := json.NewEncoder(w)
			// enc.Encode(v.Interface())
			bytes, err := json.Marshal(v)
			if err != nil {
				w.Write([]byte(fmt.Sprintf("event: error\ndata: {\"error\":\"%v\"}\n\n", err)))
				if f, ok := w.(http.Flusher); ok {
					f.Flush()
				}
				continue
			}
			w.Write([]byte(fmt.Sprintf("event: data\ndata: %s\n\n", bytes)))
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}
}

// channelIntoSlice buffers channel data and responds with complete slice.
func channelIntoSlice(w http.ResponseWriter, r *http.Request, v interface{}) {
	ctx := r.Context()

	var resp []interface{}
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(v)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			http.Error(w, "Server Timeout", 504)
			return

		default: // equivalent to: case v, ok := <-stream
			if !ok {
				DefaultRespond(w, r, resp)
				return
			}
			v := recv.Interface()

			// Present the object.
			if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
				v = presenter.Present(r, v)
			} else {
				v = DefaultPresenter.Present(r, v)
			}

			resp = append(resp, v)
		}
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>package devstep

import (
	"bytes"
	"errors"
	"gopkg.in/yaml.v1"
	"os"
	"path/filepath"
	"text/template"
)

type ConfigLoader interface {
	Load() (*ProjectConfig, error)
}

type configLoader struct {
	client        DockerClient
	homeDirectory string
	projectRoot   string
}

type yamlConfig struct {
	RepositoryName string                  `yaml:"repository"`
	SourceImage    string                  `yaml:"source_image"`
	CacheDir       string                  `yaml:"cache_dir"`
	GuestDir       string                  `yaml:"working_dir"`
	Privileged     bool                    `yaml:"privileged"`
	Links          []string                `yaml:"links"`
	Volumes        []string                `yaml:"volumes"`
	Env            map[string]string       `yaml:"environment"`
	Hack           *yamlConfig             `yaml:"hack"`
	Commands       map[string]*yamlCommand `yaml:"commands"`
}

type yamlCommand struct {
	Name       string            `yaml:"name"`
	Cmd        []string          `yaml:"cmd"`
	Privileged bool              `yaml:"privileged"`
	Links      []string          `yaml:"links"`
	Volumes    []string          `yaml:"volumes"`
	Publish    []string          `yaml:"publish"`
	Env        map[string]string `yaml:"environment"`
}

func NewConfigLoader(client DockerClient, homeDirectory, projectRoot string) ConfigLoader {
	return &configLoader{
		client:        client,
		homeDirectory: homeDirectory,
		projectRoot:   projectRoot,
	}
}

func (l *configLoader) Load() (*ProjectConfig, error) {
	log.Info("Loading configuration for %s", l.projectRoot)

	config, err := l.buildDefaultConfig()
	if err != nil {
		return nil, err
	}

	yamlConf, err := parseYaml(l.homeDirectory + "/devstep.yml")
	if err != nil {
		return nil, err
	}
	if yamlConf != nil {
		log.Info("Loaded config from home dir")
		log.Debug("Home dir config: %+v", yamlConf)
		if yamlConf.RepositoryName != "" {
			return nil, errors.New("Repository name can't be set globally")
		}
		if yamlConf.Privileged {
			return nil, errors.New("Privileged name can't be set globally")
		}
		assignYamlValues(yamlConf, config)
	}
	yamlConf, err = parseYaml(l.projectRoot + "/devstep.yml")
	if err != nil {
		return nil, err
	}
	if yamlConf != nil {
		log.Info("Loaded config from project dir")
		log.Debug("Project dir config: %+v", yamlConf)
		assignYamlValues(yamlConf, config)
		config.Defaults.Privileged = yamlConf.Privileged
	}

	tags, err := l.client.ListTags(config.RepositoryName)
	if err != nil {
		return nil, err
	}
	if len(tags) > 0 {
		config.BaseImage = config.RepositoryName + ":" + tags[0]
	} else {
		config.BaseImage = config.SourceImage
	}

	log.Info("Config loaded")
	log.Debug("Final config: %+v", config)

	return config, nil
}

func (l *configLoader) buildDefaultConfig() (*ProjectConfig, error) {
	projectDirName := filepath.Base(l.projectRoot)
	repositoryName := "devstep/" + projectDirName
	config := &ProjectConfig{
		SourceImage:    "fgrehm/devstep:v0.1.0",
		RepositoryName: repositoryName,
		HostDir:        l.projectRoot,
		GuestDir:       "/workspace",
		CacheDir:       "/tmp/devstep/cache",
		Defaults: &DockerRunOpts{
			Env:      make(map[string]string),
			Hostname: projectDirName,
		},
		HackOpts: &DockerRunOpts{Env: make(map[string]string)},
		Commands: make(map[string]*ProjectCommand),
	}

	return config, nil
}

func parseYaml(configPath string) (*yamlConfig, error) {
	configInfo, err := os.Stat(configPath)
	// File does not exist or is a directory
	if err != nil || configInfo.IsDir() {
		return nil, nil
	}

	// Parse yaml
	file, err := os.Open(configPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	data := make([]byte, configInfo.Size())
	_, err = file.Read(data)

	funcMap := template.FuncMap{
		"env": os.Getenv,
	}

	tmpl, err := template.New("config").Funcs(funcMap).Parse(string(data))
	if err != nil {
		return nil, errors.New("Error parsing '" + configPath + "'\n  " + err.Error())
	}

	var b bytes.Buffer
	tmpl.ExecuteTemplate(&b, "config", struct{}{})

	c := &yamlConfig{}
	err = yaml.Unmarshal(b.Bytes(), &c)

	if err != nil {
		return nil, errors.New("Error parsing '" + configPath + "'\n  " + err.Error())
	}

	return c, nil
}

func assignYamlValues(yamlConf *yamlConfig, config *ProjectConfig) {
	if yamlConf.RepositoryName != "" {
		config.RepositoryName = yamlConf.RepositoryName
	}
	if yamlConf.SourceImage != "" {
		config.SourceImage = yamlConf.SourceImage
	}
	if yamlConf.CacheDir != "" {
		config.CacheDir = yamlConf.CacheDir
	}
	if yamlConf.GuestDir != "" {
		config.GuestDir = yamlConf.GuestDir
	}
	if yamlConf.Links != nil {
		config.Defaults.Links = append(config.Defaults.Links, yamlConf.Links...)
	}
	if yamlConf.Volumes != nil {
		config.Defaults.Volumes = append(config.Defaults.Volumes, yamlConf.Volumes...)
	}
	if yamlConf.Env != nil {
		for k, v := range yamlConf.Env {
			config.Defaults.Env[k] = v
		}
	}

	if yamlConf.Hack != nil {
		if yamlConf.Hack.Links != nil {
			config.HackOpts.Links = append(config.HackOpts.Links, yamlConf.Hack.Links...)
		}
		if yamlConf.Hack.Volumes != nil {
			config.HackOpts.Volumes = append(config.HackOpts.Volumes, yamlConf.Hack.Volumes...)
		}
		if yamlConf.Hack.Env != nil {
			for k, v := range yamlConf.Hack.Env {
				config.HackOpts.Env[k] = v
			}
		}
	}

	if yamlConf.Commands != nil && len(yamlConf.Commands) > 0 {
		for cmdName, yamlCmd := range yamlConf.Commands {
			if yamlCmd == nil {
				yamlCmd = &yamlCommand{}
			}

			cmd := &ProjectCommand{
				cmdName,
				DockerRunOpts{
					Privileged: yamlCmd.Privileged,
					Links:      yamlCmd.Links,
					Volumes:    yamlCmd.Volumes,
					Env:        yamlCmd.Env,
					Publish:    yamlCmd.Publish,
				},
			}

			if len(yamlCmd.Cmd) > 0 {
				cmd.Cmd = yamlCmd.Cmd
			} else {
				cmd.Cmd = []string{cmdName}
			}

			config.Commands[cmdName] = cmd
		}
	}
}
<commit_msg>Bump default base image to v0.2.0<commit_after>package devstep

import (
	"bytes"
	"errors"
	"gopkg.in/yaml.v1"
	"os"
	"path/filepath"
	"text/template"
)

type ConfigLoader interface {
	Load() (*ProjectConfig, error)
}

type configLoader struct {
	client        DockerClient
	homeDirectory string
	projectRoot   string
}

type yamlConfig struct {
	RepositoryName string                  `yaml:"repository"`
	SourceImage    string                  `yaml:"source_image"`
	CacheDir       string                  `yaml:"cache_dir"`
	GuestDir       string                  `yaml:"working_dir"`
	Privileged     bool                    `yaml:"privileged"`
	Links          []string                `yaml:"links"`
	Volumes        []string                `yaml:"volumes"`
	Env            map[string]string       `yaml:"environment"`
	Hack           *yamlConfig             `yaml:"hack"`
	Commands       map[string]*yamlCommand `yaml:"commands"`
}

type yamlCommand struct {
	Name       string            `yaml:"name"`
	Cmd        []string          `yaml:"cmd"`
	Privileged bool              `yaml:"privileged"`
	Links      []string          `yaml:"links"`
	Volumes    []string          `yaml:"volumes"`
	Publish    []string          `yaml:"publish"`
	Env        map[string]string `yaml:"environment"`
}

func NewConfigLoader(client DockerClient, homeDirectory, projectRoot string) ConfigLoader {
	return &configLoader{
		client:        client,
		homeDirectory: homeDirectory,
		projectRoot:   projectRoot,
	}
}

func (l *configLoader) Load() (*ProjectConfig, error) {
	log.Info("Loading configuration for %s", l.projectRoot)

	config, err := l.buildDefaultConfig()
	if err != nil {
		return nil, err
	}

	yamlConf, err := parseYaml(l.homeDirectory + "/devstep.yml")
	if err != nil {
		return nil, err
	}
	if yamlConf != nil {
		log.Info("Loaded config from home dir")
		log.Debug("Home dir config: %+v", yamlConf)
		if yamlConf.RepositoryName != "" {
			return nil, errors.New("Repository name can't be set globally")
		}
		if yamlConf.Privileged {
			return nil, errors.New("Privileged name can't be set globally")
		}
		assignYamlValues(yamlConf, config)
	}
	yamlConf, err = parseYaml(l.projectRoot + "/devstep.yml")
	if err != nil {
		return nil, err
	}
	if yamlConf != nil {
		log.Info("Loaded config from project dir")
		log.Debug("Project dir config: %+v", yamlConf)
		assignYamlValues(yamlConf, config)
		config.Defaults.Privileged = yamlConf.Privileged
	}

	tags, err := l.client.ListTags(config.RepositoryName)
	if err != nil {
		return nil, err
	}
	if len(tags) > 0 {
		config.BaseImage = config.RepositoryName + ":" + tags[0]
	} else {
		config.BaseImage = config.SourceImage
	}

	log.Info("Config loaded")
	log.Debug("Final config: %+v", config)

	return config, nil
}

func (l *configLoader) buildDefaultConfig() (*ProjectConfig, error) {
	projectDirName := filepath.Base(l.projectRoot)
	repositoryName := "devstep/" + projectDirName
	config := &ProjectConfig{
		SourceImage:    "fgrehm/devstep:v0.2.0",
		RepositoryName: repositoryName,
		HostDir:        l.projectRoot,
		GuestDir:       "/workspace",
		CacheDir:       "/tmp/devstep/cache",
		Defaults: &DockerRunOpts{
			Env:      make(map[string]string),
			Hostname: projectDirName,
		},
		HackOpts: &DockerRunOpts{Env: make(map[string]string)},
		Commands: make(map[string]*ProjectCommand),
	}

	return config, nil
}

func parseYaml(configPath string) (*yamlConfig, error) {
	configInfo, err := os.Stat(configPath)
	// File does not exist or is a directory
	if err != nil || configInfo.IsDir() {
		return nil, nil
	}

	// Parse yaml
	file, err := os.Open(configPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	data := make([]byte, configInfo.Size())
	_, err = file.Read(data)

	funcMap := template.FuncMap{
		"env": os.Getenv,
	}

	tmpl, err := template.New("config").Funcs(funcMap).Parse(string(data))
	if err != nil {
		return nil, errors.New("Error parsing '" + configPath + "'\n  " + err.Error())
	}

	var b bytes.Buffer
	tmpl.ExecuteTemplate(&b, "config", struct{}{})

	c := &yamlConfig{}
	err = yaml.Unmarshal(b.Bytes(), &c)

	if err != nil {
		return nil, errors.New("Error parsing '" + configPath + "'\n  " + err.Error())
	}

	return c, nil
}

func assignYamlValues(yamlConf *yamlConfig, config *ProjectConfig) {
	if yamlConf.RepositoryName != "" {
		config.RepositoryName = yamlConf.RepositoryName
	}
	if yamlConf.SourceImage != "" {
		config.SourceImage = yamlConf.SourceImage
	}
	if yamlConf.CacheDir != "" {
		config.CacheDir = yamlConf.CacheDir
	}
	if yamlConf.GuestDir != "" {
		config.GuestDir = yamlConf.GuestDir
	}
	if yamlConf.Links != nil {
		config.Defaults.Links = append(config.Defaults.Links, yamlConf.Links...)
	}
	if yamlConf.Volumes != nil {
		config.Defaults.Volumes = append(config.Defaults.Volumes, yamlConf.Volumes...)
	}
	if yamlConf.Env != nil {
		for k, v := range yamlConf.Env {
			config.Defaults.Env[k] = v
		}
	}

	if yamlConf.Hack != nil {
		if yamlConf.Hack.Links != nil {
			config.HackOpts.Links = append(config.HackOpts.Links, yamlConf.Hack.Links...)
		}
		if yamlConf.Hack.Volumes != nil {
			config.HackOpts.Volumes = append(config.HackOpts.Volumes, yamlConf.Hack.Volumes...)
		}
		if yamlConf.Hack.Env != nil {
			for k, v := range yamlConf.Hack.Env {
				config.HackOpts.Env[k] = v
			}
		}
	}

	if yamlConf.Commands != nil && len(yamlConf.Commands) > 0 {
		for cmdName, yamlCmd := range yamlConf.Commands {
			if yamlCmd == nil {
				yamlCmd = &yamlCommand{}
			}

			cmd := &ProjectCommand{
				cmdName,
				DockerRunOpts{
					Privileged: yamlCmd.Privileged,
					Links:      yamlCmd.Links,
					Volumes:    yamlCmd.Volumes,
					Env:        yamlCmd.Env,
					Publish:    yamlCmd.Publish,
				},
			}

			if len(yamlCmd.Cmd) > 0 {
				cmd.Cmd = yamlCmd.Cmd
			} else {
				cmd.Cmd = []string{cmdName}
			}

			config.Commands[cmdName] = cmd
		}
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/op/go-logging"
	"strconv"
)

// This is the one and only Database-object.
var db *sql.DB

// Opens a new connection-pool to the database using the given databaseConfiguration.
func OpenDatabase(config databaseConfiguration) {
	logging.MustGetLogger("logger").Info("Opening Database...")
	var err error = nil

	// username:password@protocol(address)/dbname
	db, err = sql.Open("mysql", config.User+":"+config.Password+"@tcp("+config.Host+":"+strconv.Itoa(config.Port)+")/"+config.Database)
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to open Database-Connection: ", err)
	}
	db.SetMaxOpenConns(config.ConnectionLimit)

	err = db.Ping()
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to reach Database: ", err)
	}

	prepareDatabase()
}

// Creates the needed tables in the database.
func prepareDatabase() {
	logging.MustGetLogger("logger").Debug("Preparing Database...")

	_, err := db.Exec("CREATE TABLE IF NOT EXISTS `website` (`id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(50) NOT NULL, `enabled` int(1) NOT NULL DEFAULT '1', `visible` int(1) NOT NULL DEFAULT '1', `protocol` varchar(8) NOT NULL, `url` varchar(100) NOT NULL, `checkMethod` VARCHAR(10) NOT NULL DEFAULT 'HEAD', `status` varchar(50) NOT NULL DEFAULT 'unknown', `time` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `lastFailStatus` varchar(50) NOT NULL DEFAULT 'unknown', `lastFailTime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `ups` int(11) NOT NULL DEFAULT '0', `downs` int(11) NOT NULL DEFAULT '0', `totalChecks` int(11) NOT NULL DEFAULT '0', `avgAvail` float NOT NULL DEFAULT '100', PRIMARY KEY (`id`), UNIQUE KEY `url` (`url`)) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;")
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to create table 'website': ", err)
	}

	_, err = db.Exec("ALTER TABLE `website` ADD `checkMethod` VARCHAR(10) NOT NULL DEFAULT 'HEAD' AFTER `url`;")
	if err != nil {
		logging.MustGetLogger("logger").Debug("Unable to add 'checkMethod'-column: ", err)
	}

	_, err = db.Exec("CREATE TABLE IF NOT EXISTS `settings` (`id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL, `value` varchar(1024) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `name` (`name`)) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;")
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to create table 'settings': ", err)
	}
}

// Returns the current Database-object.
func GetDatabase() *sql.DB {
	if db == nil {
		logging.MustGetLogger("logger").Fatal("No active Database found.")
	} else {
		return db
	}
	return nil
}
<commit_msg>Modified database-preparation<commit_after>package lib

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/op/go-logging"
	"strconv"
)

// This is the one and only Database-object.
var db *sql.DB

// Opens a new connection-pool to the database using the given databaseConfiguration.
func OpenDatabase(config databaseConfiguration) {
	logging.MustGetLogger("logger").Info("Opening Database...")
	var err error = nil

	// username:password@protocol(address)/dbname
	db, err = sql.Open("mysql", config.User+":"+config.Password+"@tcp("+config.Host+":"+strconv.Itoa(config.Port)+")/"+config.Database)
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to open Database-Connection: ", err)
	}
	db.SetMaxOpenConns(config.ConnectionLimit)

	err = db.Ping()
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to reach Database: ", err)
	}

	prepareDatabase()
}

// Creates the needed tables in the database.
func prepareDatabase() {
	logging.MustGetLogger("logger").Debug("Preparing Database...")

	_, err := db.Exec("CREATE TABLE IF NOT EXISTS `website` (`id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(50) NOT NULL, `enabled` int(1) NOT NULL DEFAULT '1', `visible` int(1) NOT NULL DEFAULT '1', `protocol` varchar(8) NOT NULL DEFAULT 'https', `url` varchar(100) NOT NULL, `checkMethod` VARCHAR(10) NOT NULL DEFAULT 'HEAD', `status` varchar(50) NOT NULL DEFAULT 'unknown', `time` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `lastFailStatus` varchar(50) NOT NULL DEFAULT 'unknown', `lastFailTime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `ups` int(11) NOT NULL DEFAULT '0', `downs` int(11) NOT NULL DEFAULT '0', `totalChecks` int(11) NOT NULL DEFAULT '0', `avgAvail` float NOT NULL DEFAULT '100', PRIMARY KEY (`id`), UNIQUE KEY `url` (`url`)) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;")
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to create table 'website': ", err)
	}

	// v2.0.0 Beta
	_, err = db.Exec("ALTER TABLE `website` ADD `checkMethod` VARCHAR(10) NOT NULL DEFAULT 'HEAD' AFTER `url`;")
	if err != nil {
		logging.MustGetLogger("logger").Debug("Unable to add 'checkMethod'-column: ", err)
	}

	_, err = db.Exec("CREATE TABLE IF NOT EXISTS `settings` (`id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL, `value` varchar(1024) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `name` (`name`)) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;")
	if err != nil {
		logging.MustGetLogger("logger").Fatal("Unable to create table 'settings': ", err)
	}
}

// Returns the current Database-object.
func GetDatabase() *sql.DB {
	if db == nil {
		logging.MustGetLogger("logger").Fatal("No active Database found.")
	} else {
		return db
	}
	return nil
}
<|endoftext|>