<commit_before>// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// Package gcfg provides reading, caching and managing for configuration files.
//
// 配置管理,
// 配置文件格式支持：json, xml, toml, yaml/yml
package gcfg

import (
    "bytes"
    "errors"
    "fmt"
    "github.com/gogf/gf/g/container/garray"
    "github.com/gogf/gf/g/container/gmap"
    "github.com/gogf/gf/g/container/gtype"
    "github.com/gogf/gf/g/container/gvar"
    "github.com/gogf/gf/g/encoding/gjson"
    "github.com/gogf/gf/g/os/gfile"
    "github.com/gogf/gf/g/os/gfsnotify"
    "github.com/gogf/gf/g/os/glog"
    "github.com/gogf/gf/g/os/gspath"
)

const (
    // 默认的配置管理文件名称
    DEFAULT_CONFIG_FILE = "config.toml"
)

// 配置管理对象
type Config struct {
    name   *gtype.String            // 默认配置文件名称
    paths  *garray.StringArray      // 搜索目录路径
    jsons  *gmap.StringInterfaceMap // 配置文件对象
    vc     *gtype.Bool              // 层级检索是否执行分隔符冲突检测(默认为false，检测会比较影响检索效率)
}

// New returns a new configuration management object.
//
// 生成一个配置管理对象
func New(path string, file...string) *Config {
    name := DEFAULT_CONFIG_FILE
    if len(file) > 0 {
        name = file[0]
    }
    c := &Config {
        name   : gtype.NewString(name),
        paths  : garray.NewStringArray(),
        jsons  : gmap.NewStringInterfaceMap(),
        vc     : gtype.NewBool(),
    }
    if len(path) > 0 {
        c.SetPath(path)
    }
    return c
}

// filePath returns the absolute configuration file path for the given filename by <file>.
//
// 判断从哪个配置文件中获取内容，返回配置文件的绝对路径
func (c *Config) filePath(file...string) (path string) {
    name := c.name.Val()
    if len(file) > 0 {
        name = file[0]
    }
    path = c.GetFilePath(name)
    if path == "" {
        buffer := bytes.NewBuffer(nil)
        if c.paths.Len() > 0 {
            buffer.WriteString(fmt.Sprintf("[gcfg] cannot find config file \"%s\" in following paths:", name))
            c.paths.RLockFunc(func(array []string) {
                for k, v := range array {
                    buffer.WriteString(fmt.Sprintf("\n%d. %s",k + 1,  v))
                }
            })
        } else {
            buffer.WriteString(fmt.Sprintf("[gcfg] cannot find config file \"%s\" with no path set/add", name))
        }
        glog.Error(buffer.String())
    }
    return path
}

// 设置配置管理器的配置文件存放目录绝对路径
func (c *Config) SetPath(path string) error {
    realPath := gfile.RealPath(path)
    if realPath == "" {
        err := errors.New(fmt.Sprintf(`path "%s" does not exist`, path))
        glog.Error(fmt.Sprintf(`[gcfg] SetPath failed: %s`, err.Error()))
        return err
    }
    // 重复判断
    if c.paths.Search(realPath) != -1 {
        return nil
    }
    c.jsons.Clear()
    c.paths.Clear()
    c.paths.Append(realPath)
    //glog.Debug("[gcfg] SetPath:", realPath)
    return nil
}

// 设置是否执行层级冲突检查，当键名中存在层级符号时需要开启该特性，默认为关闭。
// 开启比较耗性能，也不建议允许键名中存在分隔符，最好在应用端避免这种情况。
func (c *Config) SetViolenceCheck(check bool) {
    c.vc.Set(check)
    c.Reload()
}

// 添加配置管理器的配置文件搜索路径
func (c *Config) AddPath(path string) error {
    realPath := gfile.RealPath(path)
    if realPath == "" {
        err := errors.New(fmt.Sprintf(`path "%s" does not exist`, path))
        glog.Error(fmt.Sprintf(`[gcfg] AddPath failed: %s`, err.Error()))
        return err
    }
    // 重复判断
    if c.paths.Search(realPath) != -1 {
        return nil
    }
    c.paths.Append(realPath)
    //glog.Debug("[gcfg] AddPath:", realPath)
    return nil
}

// 查找配置文件，获取指定配置文件的绝对路径，默认获取默认的配置文件路径；
// 当指定的配置文件不存在时，返回空字符串，并且不会报错。
func (c *Config) GetFilePath(file...string) (path string) {
    name := c.name.Val()
    if len(file) > 0 {
        name = file[0]
    }
    c.paths.RLockFunc(func(array []string) {
        for _, v := range array {
            // 查找当前目录
            if path, _ = gspath.Search(v, name); path != "" {
                break
            }
            // 查找当前目录下的config子目录
            if path, _ = gspath.Search(v, "config" + gfile.Separator + name); path != "" {
                break
            }
        }
    })
    return
}

// 设置配置管理对象的默认文件名称
func (c *Config) SetFileName(name string) {
    //glog.Debug("[gcfg] SetFileName:", name)
    c.name.Set(name)
}

// 获取配置管理对象的默认文件名称
func (c *Config) GetFileName() string {
    return c.name.Val()
}

// 添加配置文件到配置管理器中，第二个参数为非必须，如果不输入表示添加进入默认的配置名称中
// 内部带缓存控制功能。
func (c *Config) getJson(file...string) *gjson.Json {
    name := c.name.Val()
    if len(file) > 0 {
        name = file[0]
    }
    r := c.jsons.GetOrSetFuncLock(name, func() interface{} {
        filePath := c.filePath(file...)
        if filePath == "" {
            return nil
        }
        if j, err := gjson.Load(filePath); err == nil {
            j.SetViolenceCheck(c.vc.Val())
            // 添加配置文件监听，如果有任何变化，删除文件内容缓存，下一次查询会自动更新
            gfsnotify.Add(filePath, func(event *gfsnotify.Event) {
                c.jsons.Remove(event.Path)
            })
            return j
        } else {
            glog.Errorfln(`[gcfg] Load config file "%s" failed: %s`, filePath, err.Error())
        }
        return nil
    })
    if r != nil {
        return r.(*gjson.Json)
    }
    return nil
}

// 获取配置项，当不存在时返回nil
func (c *Config) Get(pattern string, file...string) interface{} {
    if j := c.getJson(file...); j != nil {
        return j.Get(pattern)
    }
    return nil
}

// 获得配置项，返回动态变量
func (c *Config) GetVar(pattern string, file...string) gvar.VarRead {
    if j := c.getJson(file...); j != nil {
        return gvar.New(j.Get(pattern), true)
    }
    return gvar.New(nil, true)
}

// 判断指定的配置项是否存在
func (c *Config) Contains(pattern string, file...string) bool {
    if j := c.getJson(file...); j != nil {
        return j.Contains(pattern)
    }
    return false
}

// 获得一个键值对关联数组/哈希表，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetMap(pattern string, file...string)  map[string]interface{} {
    if j := c.getJson(file...); j != nil {
        return j.GetMap(pattern)
    }
    return nil
}

// 获得一个数组[]interface{}，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetArray(pattern string, file...string)  []interface{} {
    if j := c.getJson(file...); j != nil {
        return j.GetArray(pattern)
    }
    return nil
}

// 返回指定json中的string
func (c *Config) GetString(pattern string, file...string) string {
    if j := c.getJson(file...); j != nil {
        return j.GetString(pattern)
    }
    return ""
}

func (c *Config) GetStrings(pattern string, file...string) []string {
    if j := c.getJson(file...); j != nil {
        return j.GetStrings(pattern)
    }
    return nil
}

func (c *Config) GetInterfaces(pattern string, file...string) []interface{} {
    if j := c.getJson(file...); j != nil {
        return j.GetInterfaces(pattern)
    }
    return nil
}

// 返回指定json中的bool
func (c *Config) GetBool(pattern string, file...string) bool {
    if j := c.getJson(file...); j != nil {
        return j.GetBool(pattern)
    }
    return false
}

// 返回指定json中的float32
func (c *Config) GetFloat32(pattern string, file...string) float32 {
    if j := c.getJson(file...); j != nil {
        return j.GetFloat32(pattern)
    }
    return 0
}

// 返回指定json中的float64
func (c *Config) GetFloat64(pattern string, file...string) float64 {
    if j := c.getJson(file...); j != nil {
        return j.GetFloat64(pattern)
    }
    return 0
}

func (c *Config) GetFloats(pattern string, file...string) []float64 {
    if j := c.getJson(file...); j != nil {
        return j.GetFloats(pattern)
    }
    return nil
}

// 返回指定json中的float64->int
func (c *Config) GetInt(pattern string, file...string)  int {
    if j := c.getJson(file...); j != nil {
        return j.GetInt(pattern)
    }
    return 0
}

func (c *Config) GetInt8(pattern string, file...string)  int8 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt8(pattern)
    }
    return 0
}

func (c *Config) GetInt16(pattern string, file...string)  int16 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt16(pattern)
    }
    return 0
}

func (c *Config) GetInt32(pattern string, file...string)  int32 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt32(pattern)
    }
    return 0
}

func (c *Config) GetInt64(pattern string, file...string)  int64 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt64(pattern)
    }
    return 0
}

func (c *Config) GetInts(pattern string, file...string) []int {
    if j := c.getJson(file...); j != nil {
        return j.GetInts(pattern)
    }
    return nil
}

// 返回指定json中的float64->uint
func (c *Config) GetUint(pattern string, file...string)  uint {
    if j := c.getJson(file...); j != nil {
        return j.GetUint(pattern)
    }
    return 0
}

func (c *Config) GetUint8(pattern string, file...string)  uint8 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint8(pattern)
    }
    return 0
}

func (c *Config) GetUint16(pattern string, file...string)  uint16 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint16(pattern)
    }
    return 0
}

func (c *Config) GetUint32(pattern string, file...string)  uint32 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint32(pattern)
    }
    return 0
}

func (c *Config) GetUint64(pattern string, file...string)  uint64 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint64(pattern)
    }
    return 0
}

func (c *Config) GetToStruct(pattern string, objPointer interface{}, file...string) error {
    if j := c.getJson(file...); j != nil {
        return j.GetToStruct(pattern, objPointer)
    }
    return errors.New("config file not found")
}

// 清空当前配置文件缓存，强制重新从磁盘文件读取配置文件内容
func (c *Config) Reload() {
    c.jsons.Clear()
}

<commit_msg>fix issue in config auto reload of gcfg<commit_after>// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// Package gcfg provides reading, caching and managing for configuration files.
//
// 配置管理,
// 配置文件格式支持：json, xml, toml, yaml/yml
package gcfg

import (
    "bytes"
    "errors"
    "fmt"
    "github.com/gogf/gf/g/container/garray"
    "github.com/gogf/gf/g/container/gmap"
    "github.com/gogf/gf/g/container/gtype"
    "github.com/gogf/gf/g/container/gvar"
    "github.com/gogf/gf/g/encoding/gjson"
    "github.com/gogf/gf/g/os/gfile"
    "github.com/gogf/gf/g/os/gfsnotify"
    "github.com/gogf/gf/g/os/glog"
    "github.com/gogf/gf/g/os/gspath"
)

const (
    // 默认的配置管理文件名称
    DEFAULT_CONFIG_FILE = "config.toml"
)

// 配置管理对象
type Config struct {
    name   *gtype.String            // 默认配置文件名称
    paths  *garray.StringArray      // 搜索目录路径
    jsons  *gmap.StringInterfaceMap // 配置文件对象
    vc     *gtype.Bool              // 层级检索是否执行分隔符冲突检测(默认为false，检测会比较影响检索效率)
}

// New returns a new configuration management object.
//
// 生成一个配置管理对象
func New(path string, file...string) *Config {
    name := DEFAULT_CONFIG_FILE
    if len(file) > 0 {
        name = file[0]
    }
    c := &Config {
        name   : gtype.NewString(name),
        paths  : garray.NewStringArray(),
        jsons  : gmap.NewStringInterfaceMap(),
        vc     : gtype.NewBool(),
    }
    if len(path) > 0 {
        c.SetPath(path)
    }
    return c
}

// filePath returns the absolute configuration file path for the given filename by <file>.
//
// 判断从哪个配置文件中获取内容，返回配置文件的绝对路径
func (c *Config) filePath(file...string) (path string) {
    name := c.name.Val()
    if len(file) > 0 {
        name = file[0]
    }
    path = c.GetFilePath(name)
    if path == "" {
        buffer := bytes.NewBuffer(nil)
        if c.paths.Len() > 0 {
            buffer.WriteString(fmt.Sprintf("[gcfg] cannot find config file \"%s\" in following paths:", name))
            c.paths.RLockFunc(func(array []string) {
                for k, v := range array {
                    buffer.WriteString(fmt.Sprintf("\n%d. %s",k + 1,  v))
                }
            })
        } else {
            buffer.WriteString(fmt.Sprintf("[gcfg] cannot find config file \"%s\" with no path set/add", name))
        }
        glog.Error(buffer.String())
    }
    return path
}

// 设置配置管理器的配置文件存放目录绝对路径
func (c *Config) SetPath(path string) error {
    realPath := gfile.RealPath(path)
    if realPath == "" {
        err := errors.New(fmt.Sprintf(`path "%s" does not exist`, path))
        glog.Error(fmt.Sprintf(`[gcfg] SetPath failed: %s`, err.Error()))
        return err
    }
    // 重复判断
    if c.paths.Search(realPath) != -1 {
        return nil
    }
    c.jsons.Clear()
    c.paths.Clear()
    c.paths.Append(realPath)
    //glog.Debug("[gcfg] SetPath:", realPath)
    return nil
}

// 设置是否执行层级冲突检查，当键名中存在层级符号时需要开启该特性，默认为关闭。
// 开启比较耗性能，也不建议允许键名中存在分隔符，最好在应用端避免这种情况。
func (c *Config) SetViolenceCheck(check bool) {
    c.vc.Set(check)
    c.Reload()
}

// 添加配置管理器的配置文件搜索路径
func (c *Config) AddPath(path string) error {
    realPath := gfile.RealPath(path)
    if realPath == "" {
        err := errors.New(fmt.Sprintf(`path "%s" does not exist`, path))
        glog.Error(fmt.Sprintf(`[gcfg] AddPath failed: %s`, err.Error()))
        return err
    }
    // 重复判断
    if c.paths.Search(realPath) != -1 {
        return nil
    }
    c.paths.Append(realPath)
    //glog.Debug("[gcfg] AddPath:", realPath)
    return nil
}

// 查找配置文件，获取指定配置文件的绝对路径，默认获取默认的配置文件路径；
// 当指定的配置文件不存在时，返回空字符串，并且不会报错。
func (c *Config) GetFilePath(file...string) (path string) {
    name := c.name.Val()
    if len(file) > 0 {
        name = file[0]
    }
    c.paths.RLockFunc(func(array []string) {
        for _, v := range array {
            // 查找当前目录
            if path, _ = gspath.Search(v, name); path != "" {
                break
            }
            // 查找当前目录下的config子目录
            if path, _ = gspath.Search(v, "config" + gfile.Separator + name); path != "" {
                break
            }
        }
    })
    return
}

// 设置配置管理对象的默认文件名称
func (c *Config) SetFileName(name string) {
    //glog.Debug("[gcfg] SetFileName:", name)
    c.name.Set(name)
}

// 获取配置管理对象的默认文件名称
func (c *Config) GetFileName() string {
    return c.name.Val()
}

// 添加配置文件到配置管理器中，第二个参数为非必须，如果不输入表示添加进入默认的配置名称中
// 内部带缓存控制功能。
func (c *Config) getJson(file...string) *gjson.Json {
    name := c.name.Val()
    if len(file) > 0 {
        name = file[0]
    }
    r := c.jsons.GetOrSetFuncLock(name, func() interface{} {
        filePath := c.filePath(file...)
        if filePath == "" {
            return nil
        }
        if j, err := gjson.Load(filePath); err == nil {
            j.SetViolenceCheck(c.vc.Val())
            // 添加配置文件监听，如果有任何变化，删除文件内容缓存，下一次查询会自动更新
            gfsnotify.Add(filePath, func(event *gfsnotify.Event) {
                c.jsons.Remove(name)
            })
            return j
        } else {
            glog.Criticalfln(`[gcfg] Load config file "%s" failed: %s`, filePath, err.Error())
        }
        return nil
    })
    if r != nil {
        return r.(*gjson.Json)
    }
    return nil
}

// 获取配置项，当不存在时返回nil
func (c *Config) Get(pattern string, file...string) interface{} {
    if j := c.getJson(file...); j != nil {
        return j.Get(pattern)
    }
    return nil
}

// 获得配置项，返回动态变量
func (c *Config) GetVar(pattern string, file...string) gvar.VarRead {
    if j := c.getJson(file...); j != nil {
        return gvar.New(j.Get(pattern), true)
    }
    return gvar.New(nil, true)
}

// 判断指定的配置项是否存在
func (c *Config) Contains(pattern string, file...string) bool {
    if j := c.getJson(file...); j != nil {
        return j.Contains(pattern)
    }
    return false
}

// 获得一个键值对关联数组/哈希表，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetMap(pattern string, file...string)  map[string]interface{} {
    if j := c.getJson(file...); j != nil {
        return j.GetMap(pattern)
    }
    return nil
}

// 获得一个数组[]interface{}，方便操作，不需要自己做类型转换
// 注意，如果获取的值不存在，或者类型与json类型不匹配，那么将会返回nil
func (c *Config) GetArray(pattern string, file...string)  []interface{} {
    if j := c.getJson(file...); j != nil {
        return j.GetArray(pattern)
    }
    return nil
}

// 返回指定json中的string
func (c *Config) GetString(pattern string, file...string) string {
    if j := c.getJson(file...); j != nil {
        return j.GetString(pattern)
    }
    return ""
}

func (c *Config) GetStrings(pattern string, file...string) []string {
    if j := c.getJson(file...); j != nil {
        return j.GetStrings(pattern)
    }
    return nil
}

func (c *Config) GetInterfaces(pattern string, file...string) []interface{} {
    if j := c.getJson(file...); j != nil {
        return j.GetInterfaces(pattern)
    }
    return nil
}

// 返回指定json中的bool
func (c *Config) GetBool(pattern string, file...string) bool {
    if j := c.getJson(file...); j != nil {
        return j.GetBool(pattern)
    }
    return false
}

// 返回指定json中的float32
func (c *Config) GetFloat32(pattern string, file...string) float32 {
    if j := c.getJson(file...); j != nil {
        return j.GetFloat32(pattern)
    }
    return 0
}

// 返回指定json中的float64
func (c *Config) GetFloat64(pattern string, file...string) float64 {
    if j := c.getJson(file...); j != nil {
        return j.GetFloat64(pattern)
    }
    return 0
}

func (c *Config) GetFloats(pattern string, file...string) []float64 {
    if j := c.getJson(file...); j != nil {
        return j.GetFloats(pattern)
    }
    return nil
}

// 返回指定json中的float64->int
func (c *Config) GetInt(pattern string, file...string)  int {
    if j := c.getJson(file...); j != nil {
        return j.GetInt(pattern)
    }
    return 0
}

func (c *Config) GetInt8(pattern string, file...string)  int8 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt8(pattern)
    }
    return 0
}

func (c *Config) GetInt16(pattern string, file...string)  int16 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt16(pattern)
    }
    return 0
}

func (c *Config) GetInt32(pattern string, file...string)  int32 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt32(pattern)
    }
    return 0
}

func (c *Config) GetInt64(pattern string, file...string)  int64 {
    if j := c.getJson(file...); j != nil {
        return j.GetInt64(pattern)
    }
    return 0
}

func (c *Config) GetInts(pattern string, file...string) []int {
    if j := c.getJson(file...); j != nil {
        return j.GetInts(pattern)
    }
    return nil
}

// 返回指定json中的float64->uint
func (c *Config) GetUint(pattern string, file...string)  uint {
    if j := c.getJson(file...); j != nil {
        return j.GetUint(pattern)
    }
    return 0
}

func (c *Config) GetUint8(pattern string, file...string)  uint8 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint8(pattern)
    }
    return 0
}

func (c *Config) GetUint16(pattern string, file...string)  uint16 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint16(pattern)
    }
    return 0
}

func (c *Config) GetUint32(pattern string, file...string)  uint32 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint32(pattern)
    }
    return 0
}

func (c *Config) GetUint64(pattern string, file...string)  uint64 {
    if j := c.getJson(file...); j != nil {
        return j.GetUint64(pattern)
    }
    return 0
}

func (c *Config) GetToStruct(pattern string, objPointer interface{}, file...string) error {
    if j := c.getJson(file...); j != nil {
        return j.GetToStruct(pattern, objPointer)
    }
    return errors.New("config file not found")
}

// 清空当前配置文件缓存，强制重新从磁盘文件读取配置文件内容
func (c *Config) Reload() {
    c.jsons.Clear()
}

<|endoftext|>


<commit_before>package container

import (
	"bytes"
	"fmt"
	"html/template"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/webvariants/susi-dev/components"
)

func runScript(script string) {
	cmd := exec.Command("/bin/bash", "-c", script)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
}

func runScriptWithSudo(script string) {
	cmd := exec.Command("sudo", "/bin/bash", "-c", script)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
}

// BuildAlpineBuilder creates an container for building alpine susi binaries
func BuildAlpineBuilder(gpgpass string) {
	if _, err := os.Stat(".containers/susi-builder-alpine-latest-linux-amd64.aci"); err != nil && gpgpass == "" {
		log.Fatal("please specify --gpgpass")
	}
	script := `
  if ! test -f .containers/susi-builder-alpine-latest-linux-amd64.aci; then
    set -e

		mkdir -p .containers
		chmod 777 .containers

    if [ "$EUID" -ne 0 ]; then
      echo "This script uses functionality which requires root privileges"
      exit 1
    fi

    # Start the build with an empty ACI
    acbuild --debug begin

    # In the event of the script exiting, end the build
    trap "{ export EXT=$?; acbuild --debug end && exit $EXT; }" EXIT

    # Name the ACI
    acbuild --debug set-name susi.io/alpine-builder

    # Based on alpine
    acbuild --debug dep add quay.io/coreos/alpine-sh
    acbuild --debug run -- mkdir -p /etc/apk
    acbuild --debug run -- /bin/sh -c "echo -en 'http://dl-4.alpinelinux.org/alpine/v3.3/main\n@community http://dl-4.alpinelinux.org/alpine/v3.3/community\n@testing http://dl-4.alpinelinux.org/alpine/edge/testing\n' > /etc/apk/repositories"
    acbuild --debug run -- apk update
    acbuild --debug run -- apk add gcc g++ make cmake git perl python py-lxml openssl-dev linux-headers boost-dev mosquitto-dev leveldb-dev@testing go@community

    acbuild --debug mount add susi /susi
    acbuild --debug mount add out /out

    # Run build
    acbuild --debug set-exec -- /bin/sh -c "cd /out && cmake /susi && make -j8 && GOPATH=/out go get github.com/webvariants/susi-gowebstack"

    # Write the result
    acbuild --debug write --overwrite .containers/susi-builder-alpine-latest-linux-amd64.aci
  fi

  if ! test -d .susi-src; then
    git clone --recursive https://github.com/webvariants/susi.git .susi-src
  fi
  `
	fmt.Println("Preparing alpine build container...")
	runScriptWithSudo(script)
	if gpgpass != "" {
		fmt.Println("Signing alpine build container...")
		signScript := fmt.Sprintf(`
			if ! test -f .containers/susi-builder-alpine-latest-linux-amd64.aci.asc; then
				gpg --batch --passphrase %v --sign --detach-sign --armor .containers/susi-builder-alpine-latest-linux-amd64.aci
			fi
			`, gpgpass)
		runScript(signScript)
	}
}

// RunAlpineBuilder executes the build container
func RunAlpineBuilder() {
	script := fmt.Sprintf(`
  mkdir -p .build/alpine
  sudo rkt run \
	--trust-keys-from-https \
  --volume susi,kind=host,source=$(pwd)/.susi-src \
  --volume out,kind=host,source=$(pwd)/.build/alpine \
  .containers/susi-builder-alpine-latest-linux-amd64.aci
  `)
	fmt.Println("Running alpine build...")
	runScriptWithSudo(script)
}

// BuildAlpineBaseContainer builds a base container for susi service containers
func BuildAlpineBaseContainer() {
	script := `
  if ! test -f .containers/susi-base-latest-linux-amd64.aci; then
		mkdir -p .containers
		chmod 777 .containers

    acbuild --debug begin
    # Name the ACI
    acbuild --debug set-name susi.io/susi-base
    # Based on alpine
    acbuild --debug dep add quay.io/coreos/alpine-sh
    acbuild --debug run -- /bin/sh -c "echo -en 'http://dl-4.alpinelinux.org/alpine/v3.3/main\n@testing http://dl-4.alpinelinux.org/alpine/edge/testing\n' > /etc/apk/repositories"
    acbuild --debug run -- apk update
    acbuild --debug run -- apk add libssl1.0 boost-system boost-program_options mosquitto-dev leveldb@testing

    for lib in .build/alpine/lib/*.so; do
      acbuild --debug copy $lib /lib/$(basename $lib)
    done

    # Write the result
    acbuild --debug write --overwrite .containers/susi-base-latest-linux-amd64.aci
    acbuild --debug end
  fi`

	fmt.Println("Building alpine base container...")
	runScriptWithSudo(script)
}

// BuildAlpineContainer builds a susi service container
func BuildAlpineContainer(node, component, gpgpass string) {
	container := fmt.Sprintf("%v/containers/%v-latest-linux-amd64.aci", node, component)
	if _, err := os.Stat(container); err != nil && gpgpass == "" {
		log.Fatal("please specify --gpgpass")
	}
	templateString := `
  acbuild --debug begin .containers/susi-base-latest-linux-amd64.aci

  acbuild --debug set-name susi.io/{{.Component}}

  acbuild --debug copy .build/alpine/bin/{{.Component}} /usr/local/bin/{{.Component}}
  acbuild --debug copy {{.Node}}/pki/pki/issued/{{.Component}}.crt /etc/susi/keys/{{.Component}}.crt
  acbuild --debug copy {{.Node}}/pki/pki/private/{{.Component}}.key /etc/susi/keys/{{.Component}}.key
  acbuild --debug copy {{.Node}}/configs/{{.Component}}.json /etc/susi/{{.Component}}.json || true
  for asset in $(find {{.Node}}/assets -type f); do
    acbuild --debug copy $asset /usr/share/susi/$(echo $asset|cut -d\/ -f 3,4,5,6,7,8,9)
  done
  for key in $(find {{.Node}}/foreignKeys -type f); do
    acbuild --debug copy $key /etc/susi/keys/$(echo $key|cut -d\/ -f 3,4,5,6,7,8,9)
  done
	cp nodes.txt .hosts
	echo "127.0.0.1 localhost" >> .hosts
	acbuild --debug copy .hosts /etc/hosts

  {{.Extra}}

  acbuild --debug set-exec -- {{.Start}}

  # Write the result
  acbuild --debug write --overwrite {{.Node}}/containers/{{.Component}}-latest-linux-amd64.aci
	if test -f {{.Node}}/containers/{{.Component}}-latest-linux-amd64.aci.asc; then
		rm {{.Node}}/containers/{{.Component}}-latest-linux-amd64.aci.asc
	fi
  acbuild --debug end
  `
	type templateData struct {
		Node      string
		Component string
		Start     string
		Extra     string
	}
	extra := ""
	if component == "susi-gowebstack" {
		extra = "acbuild --debug port add http tcp 80"
	}
	template := template.Must(template.New("").Parse(templateString))
	buff := bytes.Buffer{}
	template.Execute(&buff, templateData{node, component, components.GetStartCommand(component), extra})

	fmt.Printf("Building %v container...\n", component)
	runScriptWithSudo(buff.String())

	if gpgpass != "" {
		fmt.Printf("Signing %v container...\n", component)
		signScript := fmt.Sprintf(`
			if ! test -f %v/containers/%v-latest-linux-amd64.aci.asc; then
				gpg --batch --passphrase %v --sign --detach-sign --armor %v/containers/%v-latest-linux-amd64.aci
			fi
			`, node, component, gpgpass, node, component)
		runScript(signScript)
	}
}

//BuildDebianBuilder builds a susi builder on debian stable
func BuildDebianBuilder(version, gpgpass string) {
	container := fmt.Sprintf(".containers/susi-builder-debian-%v-latest-linux-amd64.aci", version)
	if _, err := os.Stat(container); err != nil && gpgpass == "" {
		log.Fatal("please specify --gpgpass")
	}
	script := fmt.Sprintf(`
  if ! test -f .containers/susi-builder-debian-%v-latest-linux-amd64.aci; then
    set -e

		mkdir -p .containers
		chmod 777 .containers

    if [ "$EUID" -ne 0 ]; then
      echo "This script uses functionality which requires root privileges"
      exit 1
    fi

		if ! test -f .containers/debian-%v.aci; then
			docker2aci docker://debian:%v
			mv library-debian-%v.aci .containers/debian-%v.aci
		fi

    # Start the build with debian base
    acbuild begin .containers/debian-%v.aci

    # In the event of the script exiting, end the build
    trap "{ export EXT=$?; acbuild --debug end && exit $EXT; }" EXIT

    # Name the ACI
    acbuild set-name susi.io/debian-%v-builder

    # Based on debian
    acbuild --debug run -- apt-get --yes update
    acbuild --debug run -- apt-get --yes install cmake make gcc g++ git libssl-dev libboost-all-dev libmosquitto-dev libmosquittopp-dev libleveldb-dev golang
		acbuild --debug run -- apt-get clean

    acbuild mount add susi /susi
    acbuild mount add out /out

    # Run build
    acbuild set-exec -- /bin/sh -c "cd /out && cmake /susi && make -j8 package && GOPATH=/out go get github.com/webvariants/susi-gowebstack"

    # Write the result
    acbuild --debug write --overwrite .containers/susi-builder-debian-%v-latest-linux-amd64.aci
  fi

  if ! test -d .susi-src; then
    git clone --recursive https://github.com/webvariants/susi.git .susi-src
  fi
  `, version, version, version, version, version, version, version, version)

	fmt.Printf("Preparing debian %v build container...\n", version)
	runScriptWithSudo(script)
	if gpgpass != "" {
		fmt.Printf("Signing debian %v build container...\n", version)
		signScript := fmt.Sprintf(`
			if ! test -f .containers/susi-builder-debian-%v-latest-linux-amd64.aci.asc; then
				gpg --batch --passphrase %v --sign --detach-sign --armor .containers/susi-builder-debian-%v-latest-linux-amd64.aci
			fi
			`, version, gpgpass, version)
		runScript(signScript)
	}
}

//RunDebianBuilder runs the susi-on-debian-builder
func RunDebianBuilder(version string) {
	script := fmt.Sprintf(`
  mkdir -p .build/debian-%v
  sudo rkt run \
		--trust-keys-from-https \
  	--volume susi,kind=host,source=$(pwd)/.susi-src \
  	--volume out,kind=host,source=$(pwd)/.build/debian-%v \
  	.containers/susi-builder-debian-%v-latest-linux-amd64.aci
	cp .build/debian-%v/*.deb ./susi-debian-%v.deb
	`, version, version, version, version, version)
	fmt.Printf("Running debian %v build...\n", version)
	runScriptWithSudo(script)
}

//RunNativeBuilder runs the susi-on-debian-builder
func RunNativeBuilder() {
	script := `
		mkdir -p .build/native
		cd .build/native
		cmake ../../.susi-src
		make -j8 package
		cp *.deb ../../susi-native-build.deb
	`
	fmt.Printf("Running native build...\n")
	runScript(script)
}

//Prepare prepares a pod
func Prepare(node string) (uuid string) {
	script := fmt.Sprintf("sudo rkt prepare %v/containers/*.aci ", node)
	cmd := exec.Command("/bin/bash", "-c", script)
	var out bytes.Buffer
	cmd.Stderr = os.Stderr
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
	uuid = strings.Trim(out.String(), "\n")
	return uuid
}

// Run starts a pod for a node
func Run(uuid, ip string) (systemdID string) {
	script := fmt.Sprintf("sudo systemd-run rkt run-prepared --net=\"default:ip=%v;\" %v", ip, uuid)
	cmd := exec.Command("/bin/bash", "-c", script)
	var out bytes.Buffer
	cmd.Stderr = &out
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
	text := out.String()
	words := strings.Split(text, " ")
	last := words[3]
	words = strings.Split(last, ".")
	systemdID = words[0]
	return systemdID
}
<commit_msg>make ip assignment work;<commit_after>package container

import (
	"bytes"
	"fmt"
	"html/template"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/webvariants/susi-dev/components"
)

func runScript(script string) {
	cmd := exec.Command("/bin/bash", "-c", script)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
}

func runScriptWithSudo(script string) {
	cmd := exec.Command("sudo", "/bin/bash", "-c", script)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
}

// BuildAlpineBuilder creates an container for building alpine susi binaries
func BuildAlpineBuilder(gpgpass string) {
	if _, err := os.Stat(".containers/susi-builder-alpine-latest-linux-amd64.aci"); err != nil && gpgpass == "" {
		log.Fatal("please specify --gpgpass")
	}
	script := `
  if ! test -f .containers/susi-builder-alpine-latest-linux-amd64.aci; then
    set -e

		mkdir -p .containers
		chmod 777 .containers

    if [ "$EUID" -ne 0 ]; then
      echo "This script uses functionality which requires root privileges"
      exit 1
    fi

    # Start the build with an empty ACI
    acbuild --debug begin

    # In the event of the script exiting, end the build
    trap "{ export EXT=$?; acbuild --debug end && exit $EXT; }" EXIT

    # Name the ACI
    acbuild --debug set-name susi.io/alpine-builder

    # Based on alpine
    acbuild --debug dep add quay.io/coreos/alpine-sh
    acbuild --debug run -- mkdir -p /etc/apk
    acbuild --debug run -- /bin/sh -c "echo -en 'http://dl-4.alpinelinux.org/alpine/v3.3/main\n@community http://dl-4.alpinelinux.org/alpine/v3.3/community\n@testing http://dl-4.alpinelinux.org/alpine/edge/testing\n' > /etc/apk/repositories"
    acbuild --debug run -- apk update
    acbuild --debug run -- apk add gcc g++ make cmake git perl python py-lxml openssl-dev linux-headers boost-dev mosquitto-dev leveldb-dev@testing go@community

    acbuild --debug mount add susi /susi
    acbuild --debug mount add out /out

    # Run build
    acbuild --debug set-exec -- /bin/sh -c "cd /out && cmake /susi && make -j8 && GOPATH=/out go get github.com/webvariants/susi-gowebstack"

    # Write the result
    acbuild --debug write --overwrite .containers/susi-builder-alpine-latest-linux-amd64.aci
  fi

  if ! test -d .susi-src; then
    git clone --recursive https://github.com/webvariants/susi.git .susi-src
  fi
  `
	fmt.Println("Preparing alpine build container...")
	runScriptWithSudo(script)
	if gpgpass != "" {
		fmt.Println("Signing alpine build container...")
		signScript := fmt.Sprintf(`
			if ! test -f .containers/susi-builder-alpine-latest-linux-amd64.aci.asc; then
				gpg --batch --passphrase %v --sign --detach-sign --armor .containers/susi-builder-alpine-latest-linux-amd64.aci
			fi
			`, gpgpass)
		runScript(signScript)
	}
}

// RunAlpineBuilder executes the build container
func RunAlpineBuilder() {
	script := fmt.Sprintf(`
  mkdir -p .build/alpine
  sudo rkt run \
	--trust-keys-from-https \
  --volume susi,kind=host,source=$(pwd)/.susi-src \
  --volume out,kind=host,source=$(pwd)/.build/alpine \
  .containers/susi-builder-alpine-latest-linux-amd64.aci
  `)
	fmt.Println("Running alpine build...")
	runScriptWithSudo(script)
}

// BuildAlpineBaseContainer builds a base container for susi service containers
func BuildAlpineBaseContainer() {
	script := `
  if ! test -f .containers/susi-base-latest-linux-amd64.aci; then
		mkdir -p .containers
		chmod 777 .containers

    acbuild --debug begin
    # Name the ACI
    acbuild --debug set-name susi.io/susi-base
    # Based on alpine
    acbuild --debug dep add quay.io/coreos/alpine-sh
    acbuild --debug run -- /bin/sh -c "echo -en 'http://dl-4.alpinelinux.org/alpine/v3.3/main\n@testing http://dl-4.alpinelinux.org/alpine/edge/testing\n' > /etc/apk/repositories"
    acbuild --debug run -- apk update
    acbuild --debug run -- apk add libssl1.0 boost-system boost-program_options mosquitto-dev leveldb@testing

    for lib in .build/alpine/lib/*.so; do
      acbuild --debug copy $lib /lib/$(basename $lib)
    done

    # Write the result
    acbuild --debug write --overwrite .containers/susi-base-latest-linux-amd64.aci
    acbuild --debug end
  fi`

	fmt.Println("Building alpine base container...")
	runScriptWithSudo(script)
}

// BuildAlpineContainer builds a susi service container
func BuildAlpineContainer(node, component, gpgpass string) {
	container := fmt.Sprintf("%v/containers/%v-latest-linux-amd64.aci", node, component)
	if _, err := os.Stat(container); err != nil && gpgpass == "" {
		log.Fatal("please specify --gpgpass")
	}
	templateString := `
  acbuild --debug begin .containers/susi-base-latest-linux-amd64.aci

  acbuild --debug set-name susi.io/{{.Component}}

  acbuild --debug copy .build/alpine/bin/{{.Component}} /usr/local/bin/{{.Component}}
  acbuild --debug copy {{.Node}}/pki/pki/issued/{{.Component}}.crt /etc/susi/keys/{{.Component}}.crt
  acbuild --debug copy {{.Node}}/pki/pki/private/{{.Component}}.key /etc/susi/keys/{{.Component}}.key
  acbuild --debug copy {{.Node}}/configs/{{.Component}}.json /etc/susi/{{.Component}}.json || true
  for asset in $(find {{.Node}}/assets -type f); do
    acbuild --debug copy $asset /usr/share/susi/$(echo $asset|cut -d\/ -f 3,4,5,6,7,8,9)
  done
  for key in $(find {{.Node}}/foreignKeys -type f); do
    acbuild --debug copy $key /etc/susi/keys/$(echo $key|cut -d\/ -f 3,4,5,6,7,8,9)
  done
	cp nodes.txt .hosts
	echo "127.0.0.1 localhost" >> .hosts
	acbuild --debug copy .hosts /etc/hosts

  {{.Extra}}

  acbuild --debug set-exec -- {{.Start}}

  # Write the result
  acbuild --debug write --overwrite {{.Node}}/containers/{{.Component}}-latest-linux-amd64.aci
	if test -f {{.Node}}/containers/{{.Component}}-latest-linux-amd64.aci.asc; then
		rm {{.Node}}/containers/{{.Component}}-latest-linux-amd64.aci.asc
	fi
  acbuild --debug end
  `
	type templateData struct {
		Node      string
		Component string
		Start     string
		Extra     string
	}
	extra := ""
	if component == "susi-gowebstack" {
		extra = "acbuild --debug port add http tcp 80"
	}
	template := template.Must(template.New("").Parse(templateString))
	buff := bytes.Buffer{}
	template.Execute(&buff, templateData{node, component, components.GetStartCommand(component), extra})

	fmt.Printf("Building %v container...\n", component)
	runScriptWithSudo(buff.String())

	if gpgpass != "" {
		fmt.Printf("Signing %v container...\n", component)
		signScript := fmt.Sprintf(`
			if ! test -f %v/containers/%v-latest-linux-amd64.aci.asc; then
				gpg --batch --passphrase %v --sign --detach-sign --armor %v/containers/%v-latest-linux-amd64.aci
			fi
			`, node, component, gpgpass, node, component)
		runScript(signScript)
	}
}

//BuildDebianBuilder builds a susi builder on debian stable
func BuildDebianBuilder(version, gpgpass string) {
	container := fmt.Sprintf(".containers/susi-builder-debian-%v-latest-linux-amd64.aci", version)
	if _, err := os.Stat(container); err != nil && gpgpass == "" {
		log.Fatal("please specify --gpgpass")
	}
	script := fmt.Sprintf(`
  if ! test -f .containers/susi-builder-debian-%v-latest-linux-amd64.aci; then
    set -e

		mkdir -p .containers
		chmod 777 .containers

    if [ "$EUID" -ne 0 ]; then
      echo "This script uses functionality which requires root privileges"
      exit 1
    fi

		if ! test -f .containers/debian-%v.aci; then
			docker2aci docker://debian:%v
			mv library-debian-%v.aci .containers/debian-%v.aci
		fi

    # Start the build with debian base
    acbuild begin .containers/debian-%v.aci

    # In the event of the script exiting, end the build
    trap "{ export EXT=$?; acbuild --debug end && exit $EXT; }" EXIT

    # Name the ACI
    acbuild set-name susi.io/debian-%v-builder

    # Based on debian
    acbuild --debug run -- apt-get --yes update
    acbuild --debug run -- apt-get --yes install cmake make gcc g++ git libssl-dev libboost-all-dev libmosquitto-dev libmosquittopp-dev libleveldb-dev golang
		acbuild --debug run -- apt-get clean

    acbuild mount add susi /susi
    acbuild mount add out /out

    # Run build
    acbuild set-exec -- /bin/sh -c "cd /out && cmake /susi && make -j8 package && GOPATH=/out go get github.com/webvariants/susi-gowebstack"

    # Write the result
    acbuild --debug write --overwrite .containers/susi-builder-debian-%v-latest-linux-amd64.aci
  fi

  if ! test -d .susi-src; then
    git clone --recursive https://github.com/webvariants/susi.git .susi-src
  fi
  `, version, version, version, version, version, version, version, version)

	fmt.Printf("Preparing debian %v build container...\n", version)
	runScriptWithSudo(script)
	if gpgpass != "" {
		fmt.Printf("Signing debian %v build container...\n", version)
		signScript := fmt.Sprintf(`
			if ! test -f .containers/susi-builder-debian-%v-latest-linux-amd64.aci.asc; then
				gpg --batch --passphrase %v --sign --detach-sign --armor .containers/susi-builder-debian-%v-latest-linux-amd64.aci
			fi
			`, version, gpgpass, version)
		runScript(signScript)
	}
}

//RunDebianBuilder runs the susi-on-debian-builder
func RunDebianBuilder(version string) {
	script := fmt.Sprintf(`
  mkdir -p .build/debian-%v
  sudo rkt run \
		--trust-keys-from-https \
  	--volume susi,kind=host,source=$(pwd)/.susi-src \
  	--volume out,kind=host,source=$(pwd)/.build/debian-%v \
  	.containers/susi-builder-debian-%v-latest-linux-amd64.aci
	cp .build/debian-%v/*.deb ./susi-debian-%v.deb
	`, version, version, version, version, version)
	fmt.Printf("Running debian %v build...\n", version)
	runScriptWithSudo(script)
}

//RunNativeBuilder runs the susi-on-debian-builder
func RunNativeBuilder() {
	script := `
		mkdir -p .build/native
		cd .build/native
		cmake ../../.susi-src
		make -j8 package
		cp *.deb ../../susi-native-build.deb
	`
	fmt.Printf("Running native build...\n")
	runScript(script)
}

//Prepare prepares a pod
func Prepare(node string) (uuid string) {
	script := fmt.Sprintf("sudo rkt prepare %v/containers/*.aci ", node)
	cmd := exec.Command("/bin/bash", "-c", script)
	var out bytes.Buffer
	cmd.Stderr = os.Stderr
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
	uuid = strings.Trim(out.String(), "\n")
	return uuid
}

// Run starts a pod for a node
func Run(uuid, ip string) (systemdID string) {
	script := fmt.Sprintf("sudo systemd-run -p Environment=CNI_ARGS=IP=%v rkt run-prepared %v", ip, uuid)
	cmd := exec.Command("/bin/bash", "-c", script)
	var out bytes.Buffer
	cmd.Stderr = &out
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		log.Println("Error: ", err)
	}
	text := out.String()
	words := strings.Split(text, " ")
	last := words[3]
	words = strings.Split(last, ".")
	systemdID = words[0]
	return systemdID
}
<|endoftext|>


<commit_before>package consumer

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/stealthly/siesta"
)

type PartitionConsumer struct {
	config       PartitionConsumerConfig
	kafkaClient  siesta.Connector
	fetchers     map[string]map[int32]*FetcherState
	fetchersLock sync.Mutex
}

type PartitionConsumerConfig struct {
	// Consumer group
	Group string

	//Interval to commit offsets at
	CommitInterval time.Duration

	// BrokerList is a bootstrap list to discover other brokers in a cluster. At least one broker is required.
	BrokerList []string

	// ReadTimeout is a timeout to read the response from a TCP socket.
	ReadTimeout time.Duration

	// WriteTimeout is a timeout to write the request to a TCP socket.
	WriteTimeout time.Duration

	// ConnectTimeout is a timeout to connect to a TCP socket.
	ConnectTimeout time.Duration

	// Sets whether the connection should be kept alive.
	KeepAlive bool

	// A keep alive period for a TCP connection.
	KeepAliveTimeout time.Duration

	// Maximum number of open connections for a connector.
	MaxConnections int

	// Maximum number of open connections for a single broker for a connector.
	MaxConnectionsPerBroker int

	// Maximum fetch size in bytes which will be used in all Consume() calls.
	FetchSize int32

	// The minimum amount of data the server should return for a fetch request. If insufficient data is available the request will block
	FetchMinBytes int32

	// The maximum amount of time the server will block before answering the fetch request if there isn't sufficient data to immediately satisfy FetchMinBytes
	FetchMaxWaitTime int32

	// Number of retries to get topic metadata.
	MetadataRetries int

	// Backoff value between topic metadata requests.
	MetadataBackoff time.Duration

	// Number of retries to commit an offset.
	CommitOffsetRetries int

	// Backoff value between commit offset requests.
	CommitOffsetBackoff time.Duration

	// Number of retries to get consumer metadata.
	ConsumerMetadataRetries int

	// Backoff value between consumer metadata requests.
	ConsumerMetadataBackoff time.Duration

	// ClientID that will be used by a connector to identify client requests by broker.
	ClientID string
}

func NewPartitionConsumerConfig(group string) *PartitionConsumerConfig {
	return &PartitionConsumerConfig{
		Group:                   group,
		CommitInterval:          1 * time.Second,
		ReadTimeout:             5 * time.Second,
		WriteTimeout:            5 * time.Second,
		ConnectTimeout:          5 * time.Second,
		KeepAlive:               true,
		KeepAliveTimeout:        1 * time.Minute,
		MaxConnections:          5,
		MaxConnectionsPerBroker: 5,
		FetchSize:               1024000,
		FetchMaxWaitTime:        1000,
		MetadataRetries:         5,
		MetadataBackoff:         200 * time.Millisecond,
		CommitOffsetRetries:     5,
		CommitOffsetBackoff:     200 * time.Millisecond,
		ConsumerMetadataRetries: 15,
		ConsumerMetadataBackoff: 500 * time.Millisecond,
		ClientID:                "partition-consumer",
	}
}

func NewPartitionConsumer(consumerConfig PartitionConsumerConfig) *PartitionConsumer {
	connectorConfig := siesta.NewConnectorConfig()
	connectorConfig.BrokerList = consumerConfig.BrokerList
	connectorConfig.ClientID = consumerConfig.ClientID
	connectorConfig.CommitOffsetBackoff = consumerConfig.CommitOffsetBackoff
	connectorConfig.CommitOffsetRetries = consumerConfig.CommitOffsetRetries
	connectorConfig.ConnectTimeout = consumerConfig.ConnectTimeout
	connectorConfig.ConsumerMetadataBackoff = consumerConfig.ConsumerMetadataBackoff
	connectorConfig.ConsumerMetadataRetries = consumerConfig.ConsumerMetadataRetries
	connectorConfig.FetchMaxWaitTime = consumerConfig.FetchMaxWaitTime
	connectorConfig.FetchMinBytes = consumerConfig.FetchMinBytes
	connectorConfig.FetchSize = consumerConfig.FetchSize
	connectorConfig.KeepAlive = consumerConfig.KeepAlive
	connectorConfig.KeepAliveTimeout = consumerConfig.KeepAliveTimeout
	connectorConfig.MaxConnections = consumerConfig.MaxConnections
	connectorConfig.MaxConnectionsPerBroker = consumerConfig.MaxConnectionsPerBroker
	connectorConfig.MetadataBackoff = consumerConfig.MetadataBackoff
	connectorConfig.MetadataRetries = consumerConfig.MetadataRetries
	connectorConfig.ReadTimeout = consumerConfig.ReadTimeout
	connectorConfig.WriteTimeout = consumerConfig.WriteTimeout
	kafkaClient, err := siesta.NewDefaultConnector(connectorConfig)
	if err != nil {
		panic(err)
	}

	consumer := &PartitionConsumer{
		config:      consumerConfig,
		kafkaClient: kafkaClient,
		fetchers:    make(map[string]map[int32]*FetcherState),
	}

	commitTimer := time.NewTimer(consumerConfig.CommitInterval)
	go func() {
		for {
			select {
			case <-commitTimer.C:
				{

					for topic, partitions := range consumer.fetchers {
						for partition, fetcherState := range partitions {
							offsetToCommit := fetcherState.GetOffset()
							if offsetToCommit > fetcherState.LastCommitted {
								err := consumer.kafkaClient.CommitOffset(consumer.config.Group, topic, partition, offsetToCommit)
								if err != nil {
									fmt.Printf("Failed to commit offset: %s\n", err.Error())
								}
							}
							if fetcherState.Removed {
								inLock(&consumer.fetchersLock, func() {
									if consumer.fetchers[topic][partition].Removed {
										delete(consumer.fetchers[topic], partition)
									}
								})
							}
						}
					}
					commitTimer.Reset(consumerConfig.CommitInterval)
				}
			}
		}
	}()

	return consumer
}

func (this *PartitionConsumer) Add(topic string, partition int32, strategy Strategy) error {
	fmt.Printf("Adding new topic partition: %s, %d\n", topic, partition)
	if _, exists := this.fetchers[topic]; !exists {
		fmt.Printf("Creating partition hash this.fetchers[%s]\n", topic)
		this.fetchers[topic] = make(map[int32]*FetcherState)
	}
	var fetcherState *FetcherState
	inLock(&this.fetchersLock, func() {
		fmt.Println("We are in lock!")
		if _, exists := this.fetchers[topic][partition]; !exists || this.fetchers[topic][partition].Removed {
			fmt.Printf("Not exists or removed (%s)", exists)
			if !exists {
				fmt.Println("Not exists! Getting offset from kafka...")
				offset, err := this.kafkaClient.GetOffset(this.config.Group, topic, partition)
				fmt.Printf("Offset received: %d\n", offset)
				if err != nil {
					//It's not critical, since offsets have not been committed yet
					fmt.Printf("Error fetching topic metadata: %s\n", err.Error())
				}
				fetcherState = NewFetcherState(offset)
				fmt.Printf("Fetcher state received: %v\n", fetcherState)
				this.fetchers[topic][partition] = fetcherState
			} else {
				fmt.Println("Was Removed, now setting Removed to false")
				this.fetchers[topic][partition].Removed = false
			}
		}
	})

	if fetcherState == nil {
		fmt.Println("Fetcher state is nil! Yikes! Returning...")
		return nil
	}

	go func() {
		fmt.Printf("Start fetching cycle for %s, %d\n", topic, partition)
		for {
			response, err := this.kafkaClient.Fetch(topic, partition, fetcherState.GetOffset()+1)
			if err != nil {
				fmt.Printf("Kafka error: %s\n", err.Error())
				continue
			}

			select {
			case fetcherState.Removed = <-fetcherState.stopChannel:
				{
					fmt.Printf("Stop fetching cycle for %s, %d\n", topic, partition)
					break
				}
			default:
				{
					if _, exists := response.Data[topic]; !exists {
						continue
					}
					if _, exists := response.Data[topic][partition]; !exists {
						continue
					}
					if response.Data[topic][partition].Error != siesta.ErrNoError {
						fmt.Printf("Got error for topic %s and partition %d: %s", topic, partition, response.Data[topic][partition].Error)
						continue
					}

					if len(response.Data[topic][partition].Messages) == 0 {
						continue
					}

					fmt.Printf("Sending messages to strategy %s, %d, %v\n", topic, partition, response.Data[topic][partition].Messages)
					err = strategy(topic, partition, response.Data[topic][partition].Messages)
					if err != nil {
						fmt.Printf("Strategy error: %s\n", err.Error())
					}

					offsetIndex := len(response.Data[topic][partition].Messages) - 1
					offsetValue := response.Data[topic][partition].Messages[offsetIndex].Offset
					fetcherState.SetOffset(offsetValue)
				}
			}
		}
	}()

	return nil
}

func (this *PartitionConsumer) Remove(topic string, partition int32) {
	if topicFetchers, exists := this.fetchers[topic]; exists {
		if fetcherState, exists := topicFetchers[partition]; exists {
			fetcherState.GetStopChannel() <- true
		}
	}
}

func (this *PartitionConsumer) GetTopicPartitions() *TopicAndPartitionSet {
	tpSet := NewTopicAndPartitionSet()
	for topic, partitions := range this.fetchers {
		for partition, _ := range partitions {
			tpSet.Add(TopicAndPartition{topic, partition})
		}
	}

	return tpSet
}

type FetcherState struct {
	LastCommitted int64
	Removed       bool
	offset        int64
	stopChannel   chan bool
}

func NewFetcherState(initialOffset int64) *FetcherState {
	return &FetcherState{
		LastCommitted: initialOffset,
		offset:        initialOffset,
		stopChannel:   make(chan bool),
	}
}

func (this *FetcherState) GetStopChannel() chan<- bool {
	return this.stopChannel
}

func (this *FetcherState) GetOffset() int64 {
	return atomic.LoadInt64(&this.offset)
}

func (this *FetcherState) SetOffset(offset int64) {
	atomic.StoreInt64(&this.offset, offset)
}

type Strategy func(topic string, partition int32, messages []*siesta.MessageAndOffset) error
<commit_msg>Added offset out of range handling<commit_after>package consumer

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/stealthly/siesta"
)

type PartitionConsumer struct {
	config       PartitionConsumerConfig
	kafkaClient  siesta.Connector
	fetchers     map[string]map[int32]*FetcherState
	fetchersLock sync.Mutex
}

type PartitionConsumerConfig struct {
	// Consumer group
	Group string

	//Interval to commit offsets at
	CommitInterval time.Duration

	// BrokerList is a bootstrap list to discover other brokers in a cluster. At least one broker is required.
	BrokerList []string

	// ReadTimeout is a timeout to read the response from a TCP socket.
	ReadTimeout time.Duration

	// WriteTimeout is a timeout to write the request to a TCP socket.
	WriteTimeout time.Duration

	// ConnectTimeout is a timeout to connect to a TCP socket.
	ConnectTimeout time.Duration

	// Sets whether the connection should be kept alive.
	KeepAlive bool

	// A keep alive period for a TCP connection.
	KeepAliveTimeout time.Duration

	// Maximum number of open connections for a connector.
	MaxConnections int

	// Maximum number of open connections for a single broker for a connector.
	MaxConnectionsPerBroker int

	// Maximum fetch size in bytes which will be used in all Consume() calls.
	FetchSize int32

	// The minimum amount of data the server should return for a fetch request. If insufficient data is available the request will block
	FetchMinBytes int32

	// The maximum amount of time the server will block before answering the fetch request if there isn't sufficient data to immediately satisfy FetchMinBytes
	FetchMaxWaitTime int32

	// Number of retries to get topic metadata.
	MetadataRetries int

	// Backoff value between topic metadata requests.
	MetadataBackoff time.Duration

	// Number of retries to commit an offset.
	CommitOffsetRetries int

	// Backoff value between commit offset requests.
	CommitOffsetBackoff time.Duration

	// Number of retries to get consumer metadata.
	ConsumerMetadataRetries int

	// Backoff value between consumer metadata requests.
	ConsumerMetadataBackoff time.Duration

	// ClientID that will be used by a connector to identify client requests by broker.
	ClientID string
}

func NewPartitionConsumerConfig(group string) *PartitionConsumerConfig {
	return &PartitionConsumerConfig{
		Group:                   group,
		CommitInterval:          1 * time.Second,
		ReadTimeout:             5 * time.Second,
		WriteTimeout:            5 * time.Second,
		ConnectTimeout:          5 * time.Second,
		KeepAlive:               true,
		KeepAliveTimeout:        1 * time.Minute,
		MaxConnections:          5,
		MaxConnectionsPerBroker: 5,
		FetchSize:               1024000,
		FetchMaxWaitTime:        1000,
		MetadataRetries:         5,
		MetadataBackoff:         200 * time.Millisecond,
		CommitOffsetRetries:     5,
		CommitOffsetBackoff:     200 * time.Millisecond,
		ConsumerMetadataRetries: 15,
		ConsumerMetadataBackoff: 500 * time.Millisecond,
		ClientID:                "partition-consumer",
	}
}

func NewPartitionConsumer(consumerConfig PartitionConsumerConfig) *PartitionConsumer {
	connectorConfig := siesta.NewConnectorConfig()
	connectorConfig.BrokerList = consumerConfig.BrokerList
	connectorConfig.ClientID = consumerConfig.ClientID
	connectorConfig.CommitOffsetBackoff = consumerConfig.CommitOffsetBackoff
	connectorConfig.CommitOffsetRetries = consumerConfig.CommitOffsetRetries
	connectorConfig.ConnectTimeout = consumerConfig.ConnectTimeout
	connectorConfig.ConsumerMetadataBackoff = consumerConfig.ConsumerMetadataBackoff
	connectorConfig.ConsumerMetadataRetries = consumerConfig.ConsumerMetadataRetries
	connectorConfig.FetchMaxWaitTime = consumerConfig.FetchMaxWaitTime
	connectorConfig.FetchMinBytes = consumerConfig.FetchMinBytes
	connectorConfig.FetchSize = consumerConfig.FetchSize
	connectorConfig.KeepAlive = consumerConfig.KeepAlive
	connectorConfig.KeepAliveTimeout = consumerConfig.KeepAliveTimeout
	connectorConfig.MaxConnections = consumerConfig.MaxConnections
	connectorConfig.MaxConnectionsPerBroker = consumerConfig.MaxConnectionsPerBroker
	connectorConfig.MetadataBackoff = consumerConfig.MetadataBackoff
	connectorConfig.MetadataRetries = consumerConfig.MetadataRetries
	connectorConfig.ReadTimeout = consumerConfig.ReadTimeout
	connectorConfig.WriteTimeout = consumerConfig.WriteTimeout
	kafkaClient, err := siesta.NewDefaultConnector(connectorConfig)
	if err != nil {
		panic(err)
	}

	consumer := &PartitionConsumer{
		config:      consumerConfig,
		kafkaClient: kafkaClient,
		fetchers:    make(map[string]map[int32]*FetcherState),
	}

	commitTimer := time.NewTimer(consumerConfig.CommitInterval)
	go func() {
		for {
			select {
			case <-commitTimer.C:
				{

					for topic, partitions := range consumer.fetchers {
						for partition, fetcherState := range partitions {
							offsetToCommit := fetcherState.GetOffset()
							if offsetToCommit > fetcherState.LastCommitted {
								err := consumer.kafkaClient.CommitOffset(consumer.config.Group, topic, partition, offsetToCommit)
								if err != nil {
									fmt.Printf("Failed to commit offset: %s\n", err.Error())
								}
							}
							if fetcherState.Removed {
								inLock(&consumer.fetchersLock, func() {
									if consumer.fetchers[topic][partition].Removed {
										delete(consumer.fetchers[topic], partition)
									}
								})
							}
						}
					}
					commitTimer.Reset(consumerConfig.CommitInterval)
				}
			}
		}
	}()

	return consumer
}

func (this *PartitionConsumer) Add(topic string, partition int32, strategy Strategy) error {
	fmt.Printf("Adding new topic partition: %s, %d\n", topic, partition)
	if _, exists := this.fetchers[topic]; !exists {
		fmt.Printf("Creating partition hash this.fetchers[%s]\n", topic)
		this.fetchers[topic] = make(map[int32]*FetcherState)
	}
	var fetcherState *FetcherState
	inLock(&this.fetchersLock, func() {
		fmt.Println("We are in lock!")
		if _, exists := this.fetchers[topic][partition]; !exists || this.fetchers[topic][partition].Removed {
			fmt.Printf("Not exists or removed (%s)", exists)
			if !exists {
				fmt.Println("Not exists! Getting offset from kafka...")
				offset, err := this.kafkaClient.GetOffset(this.config.Group, topic, partition)
				fmt.Printf("Offset received: %d\n", offset)
				if err != nil {
					//It's not critical, since offsets have not been committed yet
					fmt.Printf("Error fetching topic metadata: %s\n", err.Error())

					offset, err = this.kafkaClient.GetAvailableOffset(topic, partition, siesta.EarliestTime)
					if err != nil {
						fmt.Printf("Error getting available offset for topic %s, partition %d: %s\n", topic, partition, err.Error())
					}
				}
				fetcherState = NewFetcherState(offset)
				fmt.Printf("Fetcher state received: %v\n", fetcherState)
				this.fetchers[topic][partition] = fetcherState
			} else {
				fmt.Println("Was Removed, now setting Removed to false")
				this.fetchers[topic][partition].Removed = false
			}
		}
	})

	if fetcherState == nil {
		fmt.Println("Fetcher state is nil! Yikes! Returning...")
		return nil
	}

	go func() {
		fmt.Printf("Start fetching cycle for %s, %d\n", topic, partition)
		for {
			response, err := this.kafkaClient.Fetch(topic, partition, fetcherState.GetOffset()+1)
			if err != nil {
				fmt.Printf("Kafka error: %s\n", err.Error())
				continue
			}

			select {
			case fetcherState.Removed = <-fetcherState.stopChannel:
				{
					fmt.Printf("Stop fetching cycle for %s, %d\n", topic, partition)
					break
				}
			default:
				{
					if _, exists := response.Data[topic]; !exists {
						continue
					}
					if _, exists := response.Data[topic][partition]; !exists {
						continue
					}
					err = response.Data[topic][partition].Error
					if err != siesta.ErrNoError {
						fmt.Printf("Got error for topic %s and partition %d: %s", topic, partition, err)
						if err == siesta.ErrOffsetOutOfRange {
							offset, err := this.kafkaClient.GetAvailableOffset(topic, partition, siesta.EarliestTime)
							if err != nil {
								fmt.Printf("Error getting available offset for topic %s, partition %d: %s\n", topic, partition, err.Error())
							} else {
								fetcherState.SetOffset(offset)
							}
						}
						continue
					}

					if len(response.Data[topic][partition].Messages) == 0 {
						continue
					}

					fmt.Printf("Sending messages to strategy %s, %d, %v\n", topic, partition, response.Data[topic][partition].Messages)
					err = strategy(topic, partition, response.Data[topic][partition].Messages)
					if err != nil {
						fmt.Printf("Strategy error: %s\n", err.Error())
					}

					offsetIndex := len(response.Data[topic][partition].Messages) - 1
					offsetValue := response.Data[topic][partition].Messages[offsetIndex].Offset
					fetcherState.SetOffset(offsetValue)
				}
			}
		}
	}()

	return nil
}

func (this *PartitionConsumer) Remove(topic string, partition int32) {
	if topicFetchers, exists := this.fetchers[topic]; exists {
		if fetcherState, exists := topicFetchers[partition]; exists {
			fetcherState.GetStopChannel() <- true
		}
	}
}

func (this *PartitionConsumer) GetTopicPartitions() *TopicAndPartitionSet {
	tpSet := NewTopicAndPartitionSet()
	for topic, partitions := range this.fetchers {
		for partition, _ := range partitions {
			tpSet.Add(TopicAndPartition{topic, partition})
		}
	}

	return tpSet
}

type FetcherState struct {
	LastCommitted int64
	Removed       bool
	offset        int64
	stopChannel   chan bool
}

func NewFetcherState(initialOffset int64) *FetcherState {
	return &FetcherState{
		LastCommitted: initialOffset,
		offset:        initialOffset,
		stopChannel:   make(chan bool),
	}
}

func (this *FetcherState) GetStopChannel() chan<- bool {
	return this.stopChannel
}

func (this *FetcherState) GetOffset() int64 {
	return atomic.LoadInt64(&this.offset)
}

func (this *FetcherState) SetOffset(offset int64) {
	atomic.StoreInt64(&this.offset, offset)
}

type Strategy func(topic string, partition int32, messages []*siesta.MessageAndOffset) error
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package lxc

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/symlink"
	"launchpad.net/golxc"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/container"
	"github.com/juju/juju/environs/cloudinit"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/version"
)

var logger = loggo.GetLogger("juju.container.lxc")

var (
	defaultTemplate  = "ubuntu-cloud"
	LxcContainerDir  = "/var/lib/lxc"
	LxcRestartDir    = "/etc/lxc/auto"
	LxcObjectFactory = golxc.Factory()
)

const (
	// DefaultLxcBridge is the package created container bridge
	DefaultLxcBridge = "lxcbr0"
	// Btrfs is special as we treat it differently for create and clone.
	Btrfs = "btrfs"
)

// DefaultNetworkConfig returns a valid NetworkConfig to use the
// defaultLxcBridge that is created by the lxc package.
func DefaultNetworkConfig() *container.NetworkConfig {
	return container.BridgeNetworkConfig(DefaultLxcBridge)
}

// FsCommandOutput calls cmd.Output, this is used as an overloading point so
// we can test what *would* be run without actually executing another program
var FsCommandOutput = (*exec.Cmd).CombinedOutput

func containerDirFilesystem() (string, error) {
	cmd := exec.Command("df", "--output=fstype", LxcContainerDir)
	out, err := FsCommandOutput(cmd)
	if err != nil {
		return "", err
	}
	// The filesystem is the second line.
	lines := strings.Split(string(out), "\n")
	if len(lines) < 2 {
		logger.Errorf("unexpected output: %q", out)
		return "", fmt.Errorf("could not determine filesystem type")
	}
	return lines[1], nil
}

type containerManager struct {
	name              string
	logdir            string
	createWithClone   bool
	useAUFS           bool
	backingFilesystem string
}

// containerManager implements container.Manager.
var _ container.Manager = (*containerManager)(nil)

// NewContainerManager returns a manager object that can start and stop lxc
// containers. The containers that are created are namespaced by the name
// parameter.
func NewContainerManager(conf container.ManagerConfig) (container.Manager, error) {
	name := conf.PopValue(container.ConfigName)
	if name == "" {
		return nil, fmt.Errorf("name is required")
	}
	logDir := conf.PopValue(container.ConfigLogDir)
	if logDir == "" {
		logDir = agent.DefaultLogDir
	}
	var useClone bool
	useCloneVal := conf.PopValue("use-clone")
	if useCloneVal != "" {
		// Explicitly ignore the error result from ParseBool.
		// If it fails to parse, the value is false, and this suits
		// us fine.
		useClone, _ = strconv.ParseBool(useCloneVal)
	} else {
		// If no lxc-clone value is explicitly set in config, then
		// see if the Ubuntu series we are running on supports it
		// and if it does, we will use clone.
		useClone = preferFastLXC(releaseVersion())
	}
	useAUFS, _ := strconv.ParseBool(conf.PopValue("use-aufs"))
	backingFS, err := containerDirFilesystem()
	if err != nil {
		// Especially in tests, or a bot, the lxc dir may not exist
		// causing the test to fail. Since we only really care if the
		// backingFS is 'btrfs' and we treat the rest the same, just
		// call it 'unknown'.
		backingFS = "unknown"
	}
	logger.Tracef("backing filesystem: %q", backingFS)
	conf.WarnAboutUnused()
	return &containerManager{
		name:              name,
		logdir:            logDir,
		createWithClone:   useClone,
		useAUFS:           useAUFS,
		backingFilesystem: backingFS,
	}, nil
}

// releaseVersion is a function that returns a string representing the
// DISTRIB_RELEASE from the /etc/lsb-release file.
var releaseVersion = version.ReleaseVersion

// preferFastLXC returns true if the host is capable of
// LXC cloning from a template.
func preferFastLXC(release string) bool {
	if release == "" {
		return false
	}
	value, err := strconv.ParseFloat(release, 64)
	if err != nil {
		return false
	}
	return value >= 14.04
}

func (manager *containerManager) CreateContainer(
	machineConfig *cloudinit.MachineConfig,
	series string,
	network *container.NetworkConfig,
) (instance.Instance, *instance.HardwareCharacteristics, error) {
	start := time.Now()
	name := names.NewMachineTag(machineConfig.MachineId).String()
	if manager.name != "" {
		name = fmt.Sprintf("%s-%s", manager.name, name)
	}
	// Create the cloud-init.
	directory, err := container.NewDirectory(name)
	if err != nil {
		return nil, nil, err
	}
	logger.Tracef("write cloud-init")
	if manager.createWithClone {
		// If we are using clone, disable the apt-get steps
		machineConfig.DisablePackageCommands = true
	}
	userDataFilename, err := container.WriteUserData(machineConfig, directory)
	if err != nil {
		logger.Errorf("failed to write user data: %v", err)
		return nil, nil, err
	}
	logger.Tracef("write the lxc.conf file")
	configFile, err := writeLxcConfig(network, directory)
	if err != nil {
		logger.Errorf("failed to write config file: %v", err)
		return nil, nil, err
	}

	var lxcContainer golxc.Container
	if manager.createWithClone {
		templateContainer, err := EnsureCloneTemplate(
			manager.backingFilesystem,
			series,
			network,
			machineConfig.AuthorizedKeys,
			machineConfig.AptProxySettings,
		)
		if err != nil {
			return nil, nil, err
		}
		templateParams := []string{
			"--debug",                      // Debug errors in the cloud image
			"--userdata", userDataFilename, // Our groovey cloud-init
			"--hostid", name, // Use the container name as the hostid
		}
		var extraCloneArgs []string
		if manager.backingFilesystem == Btrfs || manager.useAUFS {
			extraCloneArgs = append(extraCloneArgs, "--snapshot")
		}
		if manager.backingFilesystem != Btrfs && manager.useAUFS {
			extraCloneArgs = append(extraCloneArgs, "--backingstore", "aufs")
		}

		lock, err := AcquireTemplateLock(templateContainer.Name(), "clone")
		if err != nil {
			return nil, nil, fmt.Errorf("failed to acquire lock on template: %v", err)
		}
		defer lock.Unlock()
		lxcContainer, err = templateContainer.Clone(name, extraCloneArgs, templateParams)
		if err != nil {
			logger.Errorf("lxc container cloning failed: %v", err)
			return nil, nil, err
		}
	} else {
		// Note here that the lxcObjectFacotry only returns a valid container
		// object, and doesn't actually construct the underlying lxc container on
		// disk.
		lxcContainer = LxcObjectFactory.New(name)
		templateParams := []string{
			"--debug",                      // Debug errors in the cloud image
			"--userdata", userDataFilename, // Our groovey cloud-init
			"--hostid", name, // Use the container name as the hostid
			"-r", series,
		}
		// Create the container.
		logger.Tracef("create the container")
		if err := lxcContainer.Create(configFile, defaultTemplate, nil, templateParams); err != nil {
			logger.Errorf("lxc container creation failed: %v", err)
			return nil, nil, err
		}
		logger.Tracef("lxc container created")
	}
	if err := autostartContainer(name); err != nil {
		return nil, nil, err
	}
	if err := mountHostLogDir(name, manager.logdir); err != nil {
		return nil, nil, err
	}
	// Start the lxc container with the appropriate settings for grabbing the
	// console output and a log file.
	consoleFile := filepath.Join(directory, "console.log")
	lxcContainer.SetLogFile(filepath.Join(directory, "container.log"), golxc.LogDebug)
	logger.Tracef("start the container")
	// We explicitly don't pass through the config file to the container.Start
	// method as we have passed it through at container creation time.  This
	// is necessary to get the appropriate rootfs reference without explicitly
	// setting it ourselves.
	if err = lxcContainer.Start("", consoleFile); err != nil {
		logger.Errorf("container failed to start: %v", err)
		return nil, nil, err
	}
	arch := version.Current.Arch
	hardware := &instance.HardwareCharacteristics{
		Arch: &arch,
	}
	logger.Tracef("container %q started: %v", name, time.Now().Sub(start))
	return &lxcInstance{lxcContainer, name}, hardware, nil
}

func appendToContainerConfig(name, line string) error {
	file, err := os.OpenFile(
		containerConfigFilename(name), os.O_RDWR|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = file.WriteString(line)
	return err
}

func autostartContainer(name string) error {
	// Now symlink the config file into the restart directory, if it exists.
	// This is for backwards compatiblity. From Trusty onwards, the auto start
	// option should be set in the LXC config file, this is done in the networkConfigTemplate
	// function below.
	if useRestartDir() {
		if err := symlink.New(
			containerConfigFilename(name),
			restartSymlink(name),
		); err != nil {
			return err
		}
		logger.Tracef("auto-restart link created")
	} else {
		logger.Tracef("Setting auto start to true in lxc config.")
		return appendToContainerConfig(name, "lxc.start.auto = 1\n")
	}
	return nil
}

func mountHostLogDir(name, logDir string) error {
	// Make sure that the mount dir has been created.
	logger.Tracef("make the mount dir for the shared logs")
	if err := os.MkdirAll(internalLogDir(name), 0755); err != nil {
		logger.Errorf("failed to create internal /var/log/juju mount dir: %v", err)
		return err
	}
	line := fmt.Sprintf(
		"lxc.mount.entry=%s var/log/juju none defaults,bind 0 0\n",
		logDir)
	return appendToContainerConfig(name, line)
}

func (manager *containerManager) DestroyContainer(id instance.Id) error {
	start := time.Now()
	name := string(id)
	lxcContainer := LxcObjectFactory.New(name)
	if useRestartDir() {
		// Remove the autostart link.
		if err := os.Remove(restartSymlink(name)); err != nil {
			logger.Errorf("failed to remove restart symlink: %v", err)
			return err
		}
	}
	if err := lxcContainer.Destroy(); err != nil {
		logger.Errorf("failed to destroy lxc container: %v", err)
		return err
	}

	err := container.RemoveDirectory(name)
	logger.Tracef("container %q stopped: %v", name, time.Now().Sub(start))
	return err
}

func (manager *containerManager) ListContainers() (result []instance.Instance, err error) {
	containers, err := LxcObjectFactory.List()
	if err != nil {
		logger.Errorf("failed getting all instances: %v", err)
		return
	}
	managerPrefix := ""
	if manager.name != "" {
		managerPrefix = fmt.Sprintf("%s-", manager.name)
	}

	for _, container := range containers {
		// Filter out those not starting with our name.
		name := container.Name()
		if !strings.HasPrefix(name, managerPrefix) {
			continue
		}
		if container.IsRunning() {
			result = append(result, &lxcInstance{container, name})
		}
	}
	return
}

const internalLogDirTemplate = "%s/%s/rootfs/var/log/juju"

func internalLogDir(containerName string) string {
	return fmt.Sprintf(internalLogDirTemplate, LxcContainerDir, containerName)
}

func restartSymlink(name string) string {
	return filepath.Join(LxcRestartDir, name+".conf")
}

func containerConfigFilename(name string) string {
	return filepath.Join(LxcContainerDir, name, "config")
}

const networkTemplate = `
lxc.network.type = %s
lxc.network.link = %s
lxc.network.flags = up
`

func networkConfigTemplate(networkType, networkLink string) string {
	return fmt.Sprintf(networkTemplate, networkType, networkLink)
}

func generateNetworkConfig(network *container.NetworkConfig) string {
	var lxcConfig string
	if network == nil {
		logger.Warningf("network unspecified, using default networking config")
		network = DefaultNetworkConfig()
	}
	switch network.NetworkType {
	case container.PhysicalNetwork:
		lxcConfig = networkConfigTemplate("phys", network.Device)
	default:
		logger.Warningf("Unknown network config type %q: using bridge", network.NetworkType)
		fallthrough
	case container.BridgeNetwork:
		lxcConfig = networkConfigTemplate("veth", network.Device)
	}

	return lxcConfig
}

func writeLxcConfig(network *container.NetworkConfig, directory string) (string, error) {
	networkConfig := generateNetworkConfig(network)
	configFilename := filepath.Join(directory, "lxc.conf")
	if err := ioutil.WriteFile(configFilename, []byte(networkConfig), 0644); err != nil {
		return "", err
	}
	return configFilename, nil
}

// useRestartDir is used to determine whether or not to use a symlink to the
// container config as the restart mechanism.  Older versions of LXC had the
// /etc/lxc/auto directory that would indicate that a container shoud auto-
// restart when the machine boots by having a symlink to the lxc.conf file.
// Newer versions don't do this, but instead have a config value inside the
// lxc.conf file.
func useRestartDir() bool {
	if _, err := os.Stat(LxcRestartDir); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}
<commit_msg>- Use the recently introduced function golxc.GetDefaultLXCContainerDir() for getting the default LXC path. 	- This commit fixes LP: #1329049<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package lxc

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/symlink"
	"launchpad.net/golxc"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/container"
	"github.com/juju/juju/environs/cloudinit"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/version"
)

var logger = loggo.GetLogger("juju.container.lxc")

var (
	defaultTemplate  = "ubuntu-cloud"
	LxcContainerDir  = golxc.GetDefaultLXCContainerDir()
	LxcRestartDir    = "/etc/lxc/auto"
	LxcObjectFactory = golxc.Factory()
)

const (
	// DefaultLxcBridge is the package created container bridge
	DefaultLxcBridge = "lxcbr0"
	// Btrfs is special as we treat it differently for create and clone.
	Btrfs = "btrfs"
)

// DefaultNetworkConfig returns a valid NetworkConfig to use the
// defaultLxcBridge that is created by the lxc package.
func DefaultNetworkConfig() *container.NetworkConfig {
	return container.BridgeNetworkConfig(DefaultLxcBridge)
}

// FsCommandOutput calls cmd.Output, this is used as an overloading point so
// we can test what *would* be run without actually executing another program
var FsCommandOutput = (*exec.Cmd).CombinedOutput

func containerDirFilesystem() (string, error) {
	cmd := exec.Command("df", "--output=fstype", LxcContainerDir)
	out, err := FsCommandOutput(cmd)
	if err != nil {
		return "", err
	}
	// The filesystem is the second line.
	lines := strings.Split(string(out), "\n")
	if len(lines) < 2 {
		logger.Errorf("unexpected output: %q", out)
		return "", fmt.Errorf("could not determine filesystem type")
	}
	return lines[1], nil
}

type containerManager struct {
	name              string
	logdir            string
	createWithClone   bool
	useAUFS           bool
	backingFilesystem string
}

// containerManager implements container.Manager.
var _ container.Manager = (*containerManager)(nil)

// NewContainerManager returns a manager object that can start and stop lxc
// containers. The containers that are created are namespaced by the name
// parameter.
func NewContainerManager(conf container.ManagerConfig) (container.Manager, error) {
	name := conf.PopValue(container.ConfigName)
	if name == "" {
		return nil, fmt.Errorf("name is required")
	}
	logDir := conf.PopValue(container.ConfigLogDir)
	if logDir == "" {
		logDir = agent.DefaultLogDir
	}
	var useClone bool
	useCloneVal := conf.PopValue("use-clone")
	if useCloneVal != "" {
		// Explicitly ignore the error result from ParseBool.
		// If it fails to parse, the value is false, and this suits
		// us fine.
		useClone, _ = strconv.ParseBool(useCloneVal)
	} else {
		// If no lxc-clone value is explicitly set in config, then
		// see if the Ubuntu series we are running on supports it
		// and if it does, we will use clone.
		useClone = preferFastLXC(releaseVersion())
	}
	useAUFS, _ := strconv.ParseBool(conf.PopValue("use-aufs"))
	backingFS, err := containerDirFilesystem()
	if err != nil {
		// Especially in tests, or a bot, the lxc dir may not exist
		// causing the test to fail. Since we only really care if the
		// backingFS is 'btrfs' and we treat the rest the same, just
		// call it 'unknown'.
		backingFS = "unknown"
	}
	logger.Tracef("backing filesystem: %q", backingFS)
	conf.WarnAboutUnused()
	return &containerManager{
		name:              name,
		logdir:            logDir,
		createWithClone:   useClone,
		useAUFS:           useAUFS,
		backingFilesystem: backingFS,
	}, nil
}

// releaseVersion is a function that returns a string representing the
// DISTRIB_RELEASE from the /etc/lsb-release file.
var releaseVersion = version.ReleaseVersion

// preferFastLXC returns true if the host is capable of
// LXC cloning from a template.
func preferFastLXC(release string) bool {
	if release == "" {
		return false
	}
	value, err := strconv.ParseFloat(release, 64)
	if err != nil {
		return false
	}
	return value >= 14.04
}

func (manager *containerManager) CreateContainer(
	machineConfig *cloudinit.MachineConfig,
	series string,
	network *container.NetworkConfig,
) (instance.Instance, *instance.HardwareCharacteristics, error) {
	start := time.Now()
	name := names.NewMachineTag(machineConfig.MachineId).String()
	if manager.name != "" {
		name = fmt.Sprintf("%s-%s", manager.name, name)
	}
	// Create the cloud-init.
	directory, err := container.NewDirectory(name)
	if err != nil {
		return nil, nil, err
	}
	logger.Tracef("write cloud-init")
	if manager.createWithClone {
		// If we are using clone, disable the apt-get steps
		machineConfig.DisablePackageCommands = true
	}
	userDataFilename, err := container.WriteUserData(machineConfig, directory)
	if err != nil {
		logger.Errorf("failed to write user data: %v", err)
		return nil, nil, err
	}
	logger.Tracef("write the lxc.conf file")
	configFile, err := writeLxcConfig(network, directory)
	if err != nil {
		logger.Errorf("failed to write config file: %v", err)
		return nil, nil, err
	}

	var lxcContainer golxc.Container
	if manager.createWithClone {
		templateContainer, err := EnsureCloneTemplate(
			manager.backingFilesystem,
			series,
			network,
			machineConfig.AuthorizedKeys,
			machineConfig.AptProxySettings,
		)
		if err != nil {
			return nil, nil, err
		}
		templateParams := []string{
			"--debug",                      // Debug errors in the cloud image
			"--userdata", userDataFilename, // Our groovey cloud-init
			"--hostid", name, // Use the container name as the hostid
		}
		var extraCloneArgs []string
		if manager.backingFilesystem == Btrfs || manager.useAUFS {
			extraCloneArgs = append(extraCloneArgs, "--snapshot")
		}
		if manager.backingFilesystem != Btrfs && manager.useAUFS {
			extraCloneArgs = append(extraCloneArgs, "--backingstore", "aufs")
		}

		lock, err := AcquireTemplateLock(templateContainer.Name(), "clone")
		if err != nil {
			return nil, nil, fmt.Errorf("failed to acquire lock on template: %v", err)
		}
		defer lock.Unlock()
		lxcContainer, err = templateContainer.Clone(name, extraCloneArgs, templateParams)
		if err != nil {
			logger.Errorf("lxc container cloning failed: %v", err)
			return nil, nil, err
		}
	} else {
		// Note here that the lxcObjectFacotry only returns a valid container
		// object, and doesn't actually construct the underlying lxc container on
		// disk.
		lxcContainer = LxcObjectFactory.New(name)
		templateParams := []string{
			"--debug",                      // Debug errors in the cloud image
			"--userdata", userDataFilename, // Our groovey cloud-init
			"--hostid", name, // Use the container name as the hostid
			"-r", series,
		}
		// Create the container.
		logger.Tracef("create the container")
		if err := lxcContainer.Create(configFile, defaultTemplate, nil, templateParams); err != nil {
			logger.Errorf("lxc container creation failed: %v", err)
			return nil, nil, err
		}
		logger.Tracef("lxc container created")
	}
	if err := autostartContainer(name); err != nil {
		return nil, nil, err
	}
	if err := mountHostLogDir(name, manager.logdir); err != nil {
		return nil, nil, err
	}
	// Start the lxc container with the appropriate settings for grabbing the
	// console output and a log file.
	consoleFile := filepath.Join(directory, "console.log")
	lxcContainer.SetLogFile(filepath.Join(directory, "container.log"), golxc.LogDebug)
	logger.Tracef("start the container")
	// We explicitly don't pass through the config file to the container.Start
	// method as we have passed it through at container creation time.  This
	// is necessary to get the appropriate rootfs reference without explicitly
	// setting it ourselves.
	if err = lxcContainer.Start("", consoleFile); err != nil {
		logger.Errorf("container failed to start: %v", err)
		return nil, nil, err
	}
	arch := version.Current.Arch
	hardware := &instance.HardwareCharacteristics{
		Arch: &arch,
	}
	logger.Tracef("container %q started: %v", name, time.Now().Sub(start))
	return &lxcInstance{lxcContainer, name}, hardware, nil
}

func appendToContainerConfig(name, line string) error {
	file, err := os.OpenFile(
		containerConfigFilename(name), os.O_RDWR|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = file.WriteString(line)
	return err
}

func autostartContainer(name string) error {
	// Now symlink the config file into the restart directory, if it exists.
	// This is for backwards compatiblity. From Trusty onwards, the auto start
	// option should be set in the LXC config file, this is done in the networkConfigTemplate
	// function below.
	if useRestartDir() {
		if err := symlink.New(
			containerConfigFilename(name),
			restartSymlink(name),
		); err != nil {
			return err
		}
		logger.Tracef("auto-restart link created")
	} else {
		logger.Tracef("Setting auto start to true in lxc config.")
		return appendToContainerConfig(name, "lxc.start.auto = 1\n")
	}
	return nil
}

func mountHostLogDir(name, logDir string) error {
	// Make sure that the mount dir has been created.
	logger.Tracef("make the mount dir for the shared logs")
	if err := os.MkdirAll(internalLogDir(name), 0755); err != nil {
		logger.Errorf("failed to create internal /var/log/juju mount dir: %v", err)
		return err
	}
	line := fmt.Sprintf(
		"lxc.mount.entry=%s var/log/juju none defaults,bind 0 0\n",
		logDir)
	return appendToContainerConfig(name, line)
}

func (manager *containerManager) DestroyContainer(id instance.Id) error {
	start := time.Now()
	name := string(id)
	lxcContainer := LxcObjectFactory.New(name)
	if useRestartDir() {
		// Remove the autostart link.
		if err := os.Remove(restartSymlink(name)); err != nil {
			logger.Errorf("failed to remove restart symlink: %v", err)
			return err
		}
	}
	if err := lxcContainer.Destroy(); err != nil {
		logger.Errorf("failed to destroy lxc container: %v", err)
		return err
	}

	err := container.RemoveDirectory(name)
	logger.Tracef("container %q stopped: %v", name, time.Now().Sub(start))
	return err
}

func (manager *containerManager) ListContainers() (result []instance.Instance, err error) {
	containers, err := LxcObjectFactory.List()
	if err != nil {
		logger.Errorf("failed getting all instances: %v", err)
		return
	}
	managerPrefix := ""
	if manager.name != "" {
		managerPrefix = fmt.Sprintf("%s-", manager.name)
	}

	for _, container := range containers {
		// Filter out those not starting with our name.
		name := container.Name()
		if !strings.HasPrefix(name, managerPrefix) {
			continue
		}
		if container.IsRunning() {
			result = append(result, &lxcInstance{container, name})
		}
	}
	return
}

const internalLogDirTemplate = "%s/%s/rootfs/var/log/juju"

func internalLogDir(containerName string) string {
	return fmt.Sprintf(internalLogDirTemplate, LxcContainerDir, containerName)
}

func restartSymlink(name string) string {
	return filepath.Join(LxcRestartDir, name+".conf")
}

func containerConfigFilename(name string) string {
	return filepath.Join(LxcContainerDir, name, "config")
}

const networkTemplate = `
lxc.network.type = %s
lxc.network.link = %s
lxc.network.flags = up
`

func networkConfigTemplate(networkType, networkLink string) string {
	return fmt.Sprintf(networkTemplate, networkType, networkLink)
}

func generateNetworkConfig(network *container.NetworkConfig) string {
	var lxcConfig string
	if network == nil {
		logger.Warningf("network unspecified, using default networking config")
		network = DefaultNetworkConfig()
	}
	switch network.NetworkType {
	case container.PhysicalNetwork:
		lxcConfig = networkConfigTemplate("phys", network.Device)
	default:
		logger.Warningf("Unknown network config type %q: using bridge", network.NetworkType)
		fallthrough
	case container.BridgeNetwork:
		lxcConfig = networkConfigTemplate("veth", network.Device)
	}

	return lxcConfig
}

func writeLxcConfig(network *container.NetworkConfig, directory string) (string, error) {
	networkConfig := generateNetworkConfig(network)
	configFilename := filepath.Join(directory, "lxc.conf")
	if err := ioutil.WriteFile(configFilename, []byte(networkConfig), 0644); err != nil {
		return "", err
	}
	return configFilename, nil
}

// useRestartDir is used to determine whether or not to use a symlink to the
// container config as the restart mechanism.  Older versions of LXC had the
// /etc/lxc/auto directory that would indicate that a container shoud auto-
// restart when the machine boots by having a symlink to the lxc.conf file.
// Newer versions don't do this, but instead have a config value inside the
// lxc.conf file.
func useRestartDir() bool {
	if _, err := os.Stat(LxcRestartDir); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package contract

type SendChatCommand struct {
	To		string
	Text		string
}

type ChatReceivedEvent struct {
	From		string
	Text		string
}<commit_msg>Go fmt<commit_after>package contract

type SendChatCommand struct {
	To   string
	Text string
}

type ChatReceivedEvent struct {
	From string
	Text string
}
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"time"

	"github.com/chzyer/flow"
	"github.com/chzyer/next/packet"
)

type CliDelegate interface {
	OnNewDC(port []int)
}

type Client struct {
	*Controller
	toTun    chan<- []byte
	delegate CliDelegate
}

func NewClient(f *flow.Flow, delegate CliDelegate, toDC chan<- *packet.Packet, fromDC <-chan *packet.Packet, toTun chan<- []byte) *Client {
	ctl := NewController(f, toDC, fromDC)
	cli := &Client{
		Controller: ctl,
		toTun:      toTun,
		delegate:   delegate,
	}
	go cli.loop()
	return cli
}

func (c *Client) RequestNewDC() {
	ok := c.SendTimeout(packet.New(nil, packet.NEWDC), time.Second)
	if !ok {
		// fail
	}
}

func (c *Client) loop() {
	c.flow.Add(1)
	defer c.flow.DoneAndClose()
	out := c.GetOutChan()
loop:
	for {
		select {
		case pRecv := <-out:
			switch pRecv.Type {
			case packet.DATA:
				select {
				case c.toTun <- pRecv.Data():
				case <-c.flow.IsClose():
					break loop
				}
			case packet.NEWDC_R:
				var port []int
				json.Unmarshal(pRecv.Payload, &port)
				if len(port) > 0 {
					c.delegate.OnNewDC(port)
				}
			}
			if pRecv.Type.IsReq() {
				c.Send(pRecv.Reply(nil))
			}
		case <-c.flow.IsClose():
			break loop
		}
	}
}
<commit_msg>[controller] add request new dc timeout log<commit_after>package controller

import (
	"encoding/json"
	"time"

	"gopkg.in/logex.v1"

	"github.com/chzyer/flow"
	"github.com/chzyer/next/packet"
)

type CliDelegate interface {
	OnNewDC(port []int)
}

type Client struct {
	*Controller
	toTun    chan<- []byte
	delegate CliDelegate
}

func NewClient(f *flow.Flow, delegate CliDelegate, toDC chan<- *packet.Packet, fromDC <-chan *packet.Packet, toTun chan<- []byte) *Client {
	ctl := NewController(f, toDC, fromDC)
	cli := &Client{
		Controller: ctl,
		toTun:      toTun,
		delegate:   delegate,
	}
	go cli.loop()
	return cli
}

func (c *Client) RequestNewDC() {
	ok := c.SendTimeout(packet.New(nil, packet.NEWDC), time.Second)
	if !ok {
		// fail
		logex.Info("request new dc timed out")
	}
}

func (c *Client) loop() {
	c.flow.Add(1)
	defer c.flow.DoneAndClose()
	out := c.GetOutChan()
loop:
	for {
		select {
		case pRecv := <-out:
			switch pRecv.Type {
			case packet.DATA:
				select {
				case c.toTun <- pRecv.Data():
				case <-c.flow.IsClose():
					break loop
				}
			case packet.NEWDC_R:
				var port []int
				json.Unmarshal(pRecv.Payload, &port)
				if len(port) > 0 {
					c.delegate.OnNewDC(port)
				}
			}
			if pRecv.Type.IsReq() {
				c.Send(pRecv.Reply(nil))
			}
		case <-c.flow.IsClose():
			break loop
		}
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"time"

	"github.com/ninjasphere/app-scheduler/model"
)

type window struct {
	after  Event
	before Event
}

func (w *window) init(m *model.Window) error {
	var err error
	if m == nil {
		return fmt.Errorf("illegal argument: m == nil")
	}
	w.after, err = newEvent(m.After, false)
	if err == nil {
		w.before, err = newEvent(m.Before, true)
	}
	return err
}

// Answer true if the window is permanently closed with respect to the specified time.
// That is: the close event is a non-recurring event has already occurred.
//
// This method is never true for recurring close events provided the open
// event has not yet occurred or itself recurring
func (w *window) isPermanentlyClosed(ref time.Time) bool {

	return !w.isOpen(ref, ref) &&
		(w.after.hasFinalEventOccurred(ref) || (w.after.hasEventOccurred(ref, ref) && w.before.hasFinalEventOccurred(ref)))
}

// Answer true if the window is open with respect to the specified time.
func (w *window) isOpen(scheduledAt time.Time, ref time.Time) bool {
	openWaitsForTime := w.after.hasTimestamp()

	if openWaitsForTime {

		// when both events are timestamp based, check
		// that the reference timestamp is within the boundaries
		// of those timestamp

		return w.after.hasEventOccurred(scheduledAt, ref) &&
			!w.before.hasEventOccurred(w.after.asTimestamp(scheduledAt), ref)
	} else {

		// when neither events are timestamp based, we have to
		// wait to wait for the open event to know we are open

		return w.after.hasEventOccurred(scheduledAt, ref) &&
			!w.before.hasEventOccurred(scheduledAt, ref)
	}
}

// Answer a channel that will receive an event when the next open event occurs.
func (w *window) whenOpen(ref time.Time) chan time.Time {
	return w.after.waiter(ref)
}

// Answer a channel that will receive an event when the next close event after the specified open event occurs.
func (w *window) whenClosed(opened time.Time) chan time.Time {
	return w.before.waiter(opened)
}

func (w *window) StringAt(ref time.Time) string {
	return fmt.Sprintf("window[after: %s, before:%s]", dump(w.after, ref), dump(w.before, ref))
}

func (w *window) String() string {
	return fmt.Sprintf("[%s, %s]", w.after.String(), w.before.String())
}
<commit_msg>apply golint.<commit_after>package controller

import (
	"fmt"
	"time"

	"github.com/ninjasphere/app-scheduler/model"
)

type window struct {
	after  Event
	before Event
}

func (w *window) init(m *model.Window) error {
	var err error
	if m == nil {
		return fmt.Errorf("illegal argument: m == nil")
	}
	w.after, err = newEvent(m.After, false)
	if err == nil {
		w.before, err = newEvent(m.Before, true)
	}
	return err
}

// Answer true if the window is permanently closed with respect to the specified time.
// That is: the close event is a non-recurring event has already occurred.
//
// This method is never true for recurring close events provided the open
// event has not yet occurred or itself recurring
func (w *window) isPermanentlyClosed(ref time.Time) bool {

	return !w.isOpen(ref, ref) &&
		(w.after.hasFinalEventOccurred(ref) || (w.after.hasEventOccurred(ref, ref) && w.before.hasFinalEventOccurred(ref)))
}

// Answer true if the window is open with respect to the specified time.
func (w *window) isOpen(scheduledAt time.Time, ref time.Time) bool {
	openWaitsForTime := w.after.hasTimestamp()

	if openWaitsForTime {

		// when both events are timestamp based, check
		// that the reference timestamp is within the boundaries
		// of those timestamp

		return w.after.hasEventOccurred(scheduledAt, ref) &&
			!w.before.hasEventOccurred(w.after.asTimestamp(scheduledAt), ref)
	}

	// when neither events are timestamp based, we have to
	// wait to wait for the open event to know we are open

	return w.after.hasEventOccurred(scheduledAt, ref) &&
		!w.before.hasEventOccurred(scheduledAt, ref)
}

// Answer a channel that will receive an event when the next open event occurs.
func (w *window) whenOpen(ref time.Time) chan time.Time {
	return w.after.waiter(ref)
}

// Answer a channel that will receive an event when the next close event after the specified open event occurs.
func (w *window) whenClosed(opened time.Time) chan time.Time {
	return w.before.waiter(opened)
}

func (w *window) StringAt(ref time.Time) string {
	return fmt.Sprintf("window[after: %s, before:%s]", dump(w.after, ref), dump(w.before, ref))
}

func (w *window) String() string {
	return fmt.Sprintf("[%s, %s]", w.after.String(), w.before.String())
}
<|endoftext|>


<commit_before>package conv

import (
	"testing"
)

func assertInflate(t *testing.T, c *Converter, input string, expected []string) {
	var actual []string
	ch := c.Inflate(input)
	for s := range ch {
		actual = append(actual, s)
	}

	if len(actual) != len(expected) {
		t.Errorf("length not match, expected=%d actual=%d",
			len(expected), len(actual))
		t.Logf("  expected=%v", expected)
		return
	}
	for i, v := range actual {
		if v != expected[i] {
			t.Errorf("item[%d]=%v is not match with %v", i, v, expected[i])
			t.Logf("  expected=%v", expected)
			break
		}
	}
}

func TestInflate(t *testing.T) {
	c := New()
	c.Add("a", "あ", "")
	c.Add("i", "い", "")
	c.Add("u", "う", "")
	c.Add("e", "え", "")
	c.Add("o", "お", "")
	c.Add("ka", "か", "")
	c.Add("ki", "き", "")
	c.Add("ku", "く", "")
	c.Add("ke", "け", "")
	c.Add("ko", "こ", "")
	c.Add("kk", "っ", "k")

	assertInflate(t, c, "a", []string{"あ"})
	assertInflate(t, c, "ak", []string{
		"あか", "あけ", "あき", "あっ", "あこ", "あく",
	})
}
<commit_msg>more test data<commit_after>package conv

import (
	"testing"
)

func assertInflate(t *testing.T, c *Converter, input string, expected []string) {
	var actual []string
	ch := c.Inflate(input)
	for s := range ch {
		actual = append(actual, s)
	}

	if len(actual) != len(expected) {
		t.Errorf("length not match, expected=%d actual=%d",
			len(expected), len(actual))
		t.Logf("  expected=%v", expected)
		return
	}
	for i, v := range actual {
		if v != expected[i] {
			t.Errorf("item[%d]=%v is not match with %v", i, v, expected[i])
			t.Logf("  expected=%v", expected)
			break
		}
	}
}

func TestInflate(t *testing.T) {
	c := New()
	c.Add("a", "あ", "")
	c.Add("i", "い", "")
	c.Add("u", "う", "")
	c.Add("e", "え", "")
	c.Add("o", "お", "")
	c.Add("ka", "か", "")
	c.Add("ki", "き", "")
	c.Add("ku", "く", "")
	c.Add("ke", "け", "")
	c.Add("ko", "こ", "")
	c.Add("kk", "っ", "k")

	assertInflate(t, c, "a", []string{"あ"})
	assertInflate(t, c, "ak", []string{
		"あか", "あけ", "あき", "あっ", "あこ", "あく",
	})
	assertInflate(t, c, "ik", []string{
		"いか", "いけ", "いき", "いっ", "いこ", "いく",
	})
	assertInflate(t, c, "uk", []string{
		"うか", "うけ", "うき", "うっ", "うこ", "うく",
	})
	assertInflate(t, c, "ek", []string{
		"えか", "えけ", "えき", "えっ", "えこ", "えく",
	})
	assertInflate(t, c, "ok", []string{
		"おか", "おけ", "おき", "おっ", "おこ", "おく",
	})
}
<|endoftext|>


<commit_before>// Package link provides a measured transport.Socket link
package link

import (
	"errors"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/transport"
)

// Link is a layer ontop of a transport socket with the
// buffering send and recv queue's with the ability to
// measure the actual transport link and reconnect.
type Link interface {
	// provides the transport.Socket interface
	transport.Socket
	// Connect connects the link. It must be called first.
	Connect() error
	// Id of the link likely a uuid if not specified
	Id() string
	// Status of the link
	Status() string
	// Depth of the buffers
	Weight() int
	// Rate of the link
	Length() int
}

var (
	ErrLinkClosed = errors.New("link closed")
)

// NewLink creates a new link on top of a socket
func NewLink(opts ...options.Option) Link {
	return newLink(options.NewOptions(opts...))
}

// Sets the link id
func Id(id string) options.Option {
	return options.WithValue("link.id", id)
}

// The address to use for the link
func Address(a string) options.Option {
	return options.WithValue("link.address", a)
}

// The transport to use for the link
func Transport(t transport.Transport) options.Option {
	return options.WithValue("link.transport", t)
}
<commit_msg>Update link Id comment<commit_after>// Package link provides a measured transport.Socket link
package link

import (
	"errors"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/transport"
)

// Link is a layer ontop of a transport socket with the
// buffering send and recv queue's with the ability to
// measure the actual transport link and reconnect.
type Link interface {
	// provides the transport.Socket interface
	transport.Socket
	// Connect connects the link. It must be called first.
	Connect() error
	// Id of the link is "local" if not specified
	Id() string
	// Status of the link
	Status() string
	// Depth of the buffers
	Weight() int
	// Rate of the link
	Length() int
}

var (
	ErrLinkClosed = errors.New("link closed")
)

// NewLink creates a new link on top of a socket
func NewLink(opts ...options.Option) Link {
	return newLink(options.NewOptions(opts...))
}

// Sets the link id
func Id(id string) options.Option {
	return options.WithValue("link.id", id)
}

// The address to use for the link
func Address(a string) options.Option {
	return options.WithValue("link.address", a)
}

// The transport to use for the link
func Transport(t transport.Transport) options.Option {
	return options.WithValue("link.transport", t)
}
<|endoftext|>