<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cm

import (
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/util/wait"

	"k8s.io/kubernetes/pkg/api/v1"
	v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos"
	"k8s.io/kubernetes/pkg/api/v1/resource"
)

const (
	// how often the qos cgroup manager will perform periodic update
	// of the qos level cgroup resource constraints
	periodicQOSCgroupUpdateInterval = 1 * time.Minute
)

type QOSContainerManager interface {
	Start(func() v1.ResourceList, ActivePodsFunc) error
	GetQOSContainersInfo() QOSContainersInfo
	UpdateCgroups() error
}

type qosContainerManagerImpl struct {
	sync.Mutex
	nodeInfo           *v1.Node
	qosContainersInfo  QOSContainersInfo
	subsystems         *CgroupSubsystems
	cgroupManager      CgroupManager
	activePods         ActivePodsFunc
	getNodeAllocatable func() v1.ResourceList
	cgroupRoot         string
	qosReserved        map[v1.ResourceName]int64
}

func NewQOSContainerManager(subsystems *CgroupSubsystems, cgroupRoot string, nodeConfig NodeConfig) (QOSContainerManager, error) {
	if !nodeConfig.CgroupsPerQOS {
		return &qosContainerManagerNoop{
			cgroupRoot: CgroupName(nodeConfig.CgroupRoot),
		}, nil
	}

	return &qosContainerManagerImpl{
		subsystems:    subsystems,
		cgroupManager: NewCgroupManager(subsystems, nodeConfig.CgroupDriver),
		cgroupRoot:    cgroupRoot,
		qosReserved:   nodeConfig.ExperimentalQOSReserved,
	}, nil
}

func (m *qosContainerManagerImpl) GetQOSContainersInfo() QOSContainersInfo {
	return m.qosContainersInfo
}

func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceList, activePods ActivePodsFunc) error {
	cm := m.cgroupManager
	rootContainer := m.cgroupRoot
	if !cm.Exists(CgroupName(rootContainer)) {
		return fmt.Errorf("root container %s doesn't exist", rootContainer)
	}

	// Top level for Qos containers are created only for Burstable
	// and Best Effort classes
	qosClasses := map[v1.PodQOSClass]string{
		v1.PodQOSBurstable:  path.Join(rootContainer, strings.ToLower(string(v1.PodQOSBurstable))),
		v1.PodQOSBestEffort: path.Join(rootContainer, strings.ToLower(string(v1.PodQOSBestEffort))),
	}

	// Create containers for both qos classes
	for qosClass, containerName := range qosClasses {
		// get the container's absolute name
		absoluteContainerName := CgroupName(containerName)

		resourceParameters := &ResourceConfig{}
		// the BestEffort QoS class has a statically configured minShares value
		if qosClass == v1.PodQOSBestEffort {
			minShares := int64(MinShares)
			resourceParameters.CpuShares = &minShares
		}
		// containerConfig object stores the cgroup specifications
		containerConfig := &CgroupConfig{
			Name:               absoluteContainerName,
			ResourceParameters: resourceParameters,
		}
		// check if it exists
		if !cm.Exists(absoluteContainerName) {
			if err := cm.Create(containerConfig); err != nil {
				return fmt.Errorf("failed to create top level %v QOS cgroup : %v", qosClass, err)
			}
		} else {
			// to ensure we actually have the right state, we update the config on startup
			if err := cm.Update(containerConfig); err != nil {
				return fmt.Errorf("failed to update top level %v QOS cgroup : %v", qosClass, err)
			}
		}
	}
	// Store the top level qos container names
	m.qosContainersInfo = QOSContainersInfo{
		Guaranteed: rootContainer,
		Burstable:  qosClasses[v1.PodQOSBurstable],
		BestEffort: qosClasses[v1.PodQOSBestEffort],
	}
	m.getNodeAllocatable = getNodeAllocatable
	m.activePods = activePods

	// update qos cgroup tiers on startup and in periodic intervals
	// to ensure desired state is in synch with actual state.
	go wait.Until(func() {
		err := m.UpdateCgroups()
		if err != nil {
			glog.Warningf("[ContainerManager] Failed to reserve QoS requests: %v", err)
		}
	}, periodicQOSCgroupUpdateInterval, wait.NeverStop)

	return nil
}

func (m *qosContainerManagerImpl) setCPUCgroupConfig(configs map[v1.PodQOSClass]*CgroupConfig) error {
	pods := m.activePods()
	burstablePodCPURequest := int64(0)
	for i := range pods {
		pod := pods[i]
		qosClass := v1qos.GetPodQOS(pod)
		if qosClass != v1.PodQOSBurstable {
			// we only care about the burstable qos tier
			continue
		}
		req, _, err := resource.PodRequestsAndLimits(pod)
		if err != nil {
			return err
		}
		if request, found := req[v1.ResourceCPU]; found {
			burstablePodCPURequest += request.MilliValue()
		}
	}

	// make sure best effort is always 2 shares
	bestEffortCPUShares := int64(MinShares)
	configs[v1.PodQOSBestEffort].ResourceParameters.CpuShares = &bestEffortCPUShares

	// set burstable shares based on current observe state
	burstableCPUShares := MilliCPUToShares(burstablePodCPURequest)
	if burstableCPUShares < int64(MinShares) {
		burstableCPUShares = int64(MinShares)
	}
	configs[v1.PodQOSBurstable].ResourceParameters.CpuShares = &burstableCPUShares
	return nil
}

// setMemoryReserve sums the memory limits of all pods in a QOS class,
// calculates QOS class memory limits, and set those limits in the
// CgroupConfig for each QOS class.
func (m *qosContainerManagerImpl) setMemoryReserve(configs map[v1.PodQOSClass]*CgroupConfig, percentReserve int64) {
	qosMemoryRequests := map[v1.PodQOSClass]int64{
		v1.PodQOSGuaranteed: 0,
		v1.PodQOSBurstable:  0,
	}

	// Sum the pod limits for pods in each QOS class
	pods := m.activePods()
	for _, pod := range pods {
		podMemoryRequest := int64(0)
		qosClass := v1qos.GetPodQOS(pod)
		if qosClass == v1.PodQOSBestEffort {
			// limits are not set for Best Effort pods
			continue
		}
		req, _, err := resource.PodRequestsAndLimits(pod)
		if err != nil {
			glog.V(2).Infof("[Container Manager] Pod resource requests/limits could not be determined.  Not setting QOS memory limts.")
			return
		}
		if request, found := req[v1.ResourceMemory]; found {
			podMemoryRequest += request.Value()
		}
		qosMemoryRequests[qosClass] += podMemoryRequest
	}

	resources := m.getNodeAllocatable()
	allocatableResource, ok := resources[v1.ResourceMemory]
	if !ok {
		glog.V(2).Infof("[Container Manager] Allocatable memory value could not be determined.  Not setting QOS memory limts.")
		return
	}
	allocatable := allocatableResource.Value()
	if allocatable == 0 {
		glog.V(2).Infof("[Container Manager] Memory allocatable reported as 0, might be in standalone mode.  Not setting QOS memory limts.")
		return
	}

	for qos, limits := range qosMemoryRequests {
		glog.V(2).Infof("[Container Manager] %s pod requests total %d bytes (reserve %d%%)", qos, limits, percentReserve)
	}

	// Calculate QOS memory limits
	burstableLimit := allocatable - (qosMemoryRequests[v1.PodQOSGuaranteed] * percentReserve / 100)
	bestEffortLimit := burstableLimit - (qosMemoryRequests[v1.PodQOSBurstable] * percentReserve / 100)
	configs[v1.PodQOSBurstable].ResourceParameters.Memory = &burstableLimit
	configs[v1.PodQOSBestEffort].ResourceParameters.Memory = &bestEffortLimit
}

// retrySetMemoryReserve checks for any QoS cgroups over the limit
// that was attempted to be set in the first Update() and adjusts
// their memory limit to the usage to prevent further growth.
func (m *qosContainerManagerImpl) retrySetMemoryReserve(configs map[v1.PodQOSClass]*CgroupConfig, percentReserve int64) {
	// Unreclaimable memory usage may already exceeded the desired limit
	// Attempt to set the limit near the current usage to put pressure
	// on the cgroup and prevent further growth.
	for qos, config := range configs {
		stats, err := m.cgroupManager.GetResourceStats(config.Name)
		if err != nil {
			glog.V(2).Infof("[Container Manager] %v", err)
			return
		}
		usage := stats.MemoryStats.Usage

		// Because there is no good way to determine of the original Update()
		// on the memory resource was successful, we determine failure of the
		// first attempt by checking if the usage is above the limit we attempt
		// to set.  If it is, we assume the first attempt to set the limit failed
		// and try again setting the limit to the usage.  Otherwise we leave
		// the CgroupConfig as is.
		if configs[qos].ResourceParameters.Memory != nil && usage > *configs[qos].ResourceParameters.Memory {
			configs[qos].ResourceParameters.Memory = &usage
		}
	}
}

func (m *qosContainerManagerImpl) UpdateCgroups() error {
	m.Lock()
	defer m.Unlock()

	qosConfigs := map[v1.PodQOSClass]*CgroupConfig{
		v1.PodQOSBurstable: {
			Name:               CgroupName(m.qosContainersInfo.Burstable),
			ResourceParameters: &ResourceConfig{},
		},
		v1.PodQOSBestEffort: {
			Name:               CgroupName(m.qosContainersInfo.BestEffort),
			ResourceParameters: &ResourceConfig{},
		},
	}

	// update the qos level cgroup settings for cpu shares
	if err := m.setCPUCgroupConfig(qosConfigs); err != nil {
		return err
	}

	for resource, percentReserve := range m.qosReserved {
		switch resource {
		case v1.ResourceMemory:
			m.setMemoryReserve(qosConfigs, percentReserve)
		}
	}
	updateSuccess := true
	for _, config := range qosConfigs {
		err := m.cgroupManager.Update(config)
		if err != nil {
			updateSuccess = false
		}
	}
	if updateSuccess {
		glog.V(2).Infof("[ContainerManager]: Updated QoS cgroup configuration")
		return nil
	}

	// If the resource can adjust the ResourceConfig to increase likelihood of
	// success, call the adjustment function here.  Otherwise, the Update() will
	// be called again with the same values.
	for resource, percentReserve := range m.qosReserved {
		switch resource {
		case v1.ResourceMemory:
			m.retrySetMemoryReserve(qosConfigs, percentReserve)
		}
	}

	for _, config := range qosConfigs {
		err := m.cgroupManager.Update(config)
		if err != nil {
			glog.V(2).Infof("[ContainerManager]: Failed to update QoS cgroup configuration")
			return err
		}
	}

	glog.V(2).Infof("[ContainerManager]: Updated QoS cgroup configuration on retry")
	return nil
}

type qosContainerManagerNoop struct {
	cgroupRoot CgroupName
}

var _ QOSContainerManager = &qosContainerManagerNoop{}

func (m *qosContainerManagerNoop) GetQOSContainersInfo() QOSContainersInfo {
	return QOSContainersInfo{}
}

func (m *qosContainerManagerNoop) Start(_ func() v1.ResourceList, _ ActivePodsFunc) error {
	return nil
}

func (m *qosContainerManagerNoop) UpdateCgroups() error {
	return nil
}
<commit_msg>UPSTREAM: 53753: Reduce log spam in qos container manager<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cm

import (
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/util/wait"

	"k8s.io/kubernetes/pkg/api/v1"
	v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos"
	"k8s.io/kubernetes/pkg/api/v1/resource"
)

const (
	// how often the qos cgroup manager will perform periodic update
	// of the qos level cgroup resource constraints
	periodicQOSCgroupUpdateInterval = 1 * time.Minute
)

type QOSContainerManager interface {
	Start(func() v1.ResourceList, ActivePodsFunc) error
	GetQOSContainersInfo() QOSContainersInfo
	UpdateCgroups() error
}

type qosContainerManagerImpl struct {
	sync.Mutex
	nodeInfo           *v1.Node
	qosContainersInfo  QOSContainersInfo
	subsystems         *CgroupSubsystems
	cgroupManager      CgroupManager
	activePods         ActivePodsFunc
	getNodeAllocatable func() v1.ResourceList
	cgroupRoot         string
	qosReserved        map[v1.ResourceName]int64
}

func NewQOSContainerManager(subsystems *CgroupSubsystems, cgroupRoot string, nodeConfig NodeConfig) (QOSContainerManager, error) {
	if !nodeConfig.CgroupsPerQOS {
		return &qosContainerManagerNoop{
			cgroupRoot: CgroupName(nodeConfig.CgroupRoot),
		}, nil
	}

	return &qosContainerManagerImpl{
		subsystems:    subsystems,
		cgroupManager: NewCgroupManager(subsystems, nodeConfig.CgroupDriver),
		cgroupRoot:    cgroupRoot,
		qosReserved:   nodeConfig.ExperimentalQOSReserved,
	}, nil
}

func (m *qosContainerManagerImpl) GetQOSContainersInfo() QOSContainersInfo {
	return m.qosContainersInfo
}

func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceList, activePods ActivePodsFunc) error {
	cm := m.cgroupManager
	rootContainer := m.cgroupRoot
	if !cm.Exists(CgroupName(rootContainer)) {
		return fmt.Errorf("root container %s doesn't exist", rootContainer)
	}

	// Top level for Qos containers are created only for Burstable
	// and Best Effort classes
	qosClasses := map[v1.PodQOSClass]string{
		v1.PodQOSBurstable:  path.Join(rootContainer, strings.ToLower(string(v1.PodQOSBurstable))),
		v1.PodQOSBestEffort: path.Join(rootContainer, strings.ToLower(string(v1.PodQOSBestEffort))),
	}

	// Create containers for both qos classes
	for qosClass, containerName := range qosClasses {
		// get the container's absolute name
		absoluteContainerName := CgroupName(containerName)

		resourceParameters := &ResourceConfig{}
		// the BestEffort QoS class has a statically configured minShares value
		if qosClass == v1.PodQOSBestEffort {
			minShares := int64(MinShares)
			resourceParameters.CpuShares = &minShares
		}
		// containerConfig object stores the cgroup specifications
		containerConfig := &CgroupConfig{
			Name:               absoluteContainerName,
			ResourceParameters: resourceParameters,
		}
		// check if it exists
		if !cm.Exists(absoluteContainerName) {
			if err := cm.Create(containerConfig); err != nil {
				return fmt.Errorf("failed to create top level %v QOS cgroup : %v", qosClass, err)
			}
		} else {
			// to ensure we actually have the right state, we update the config on startup
			if err := cm.Update(containerConfig); err != nil {
				return fmt.Errorf("failed to update top level %v QOS cgroup : %v", qosClass, err)
			}
		}
	}
	// Store the top level qos container names
	m.qosContainersInfo = QOSContainersInfo{
		Guaranteed: rootContainer,
		Burstable:  qosClasses[v1.PodQOSBurstable],
		BestEffort: qosClasses[v1.PodQOSBestEffort],
	}
	m.getNodeAllocatable = getNodeAllocatable
	m.activePods = activePods

	// update qos cgroup tiers on startup and in periodic intervals
	// to ensure desired state is in synch with actual state.
	go wait.Until(func() {
		err := m.UpdateCgroups()
		if err != nil {
			glog.Warningf("[ContainerManager] Failed to reserve QoS requests: %v", err)
		}
	}, periodicQOSCgroupUpdateInterval, wait.NeverStop)

	return nil
}

func (m *qosContainerManagerImpl) setCPUCgroupConfig(configs map[v1.PodQOSClass]*CgroupConfig) error {
	pods := m.activePods()
	burstablePodCPURequest := int64(0)
	for i := range pods {
		pod := pods[i]
		qosClass := v1qos.GetPodQOS(pod)
		if qosClass != v1.PodQOSBurstable {
			// we only care about the burstable qos tier
			continue
		}
		req, _, err := resource.PodRequestsAndLimits(pod)
		if err != nil {
			return err
		}
		if request, found := req[v1.ResourceCPU]; found {
			burstablePodCPURequest += request.MilliValue()
		}
	}

	// make sure best effort is always 2 shares
	bestEffortCPUShares := int64(MinShares)
	configs[v1.PodQOSBestEffort].ResourceParameters.CpuShares = &bestEffortCPUShares

	// set burstable shares based on current observe state
	burstableCPUShares := MilliCPUToShares(burstablePodCPURequest)
	if burstableCPUShares < int64(MinShares) {
		burstableCPUShares = int64(MinShares)
	}
	configs[v1.PodQOSBurstable].ResourceParameters.CpuShares = &burstableCPUShares
	return nil
}

// setMemoryReserve sums the memory limits of all pods in a QOS class,
// calculates QOS class memory limits, and set those limits in the
// CgroupConfig for each QOS class.
func (m *qosContainerManagerImpl) setMemoryReserve(configs map[v1.PodQOSClass]*CgroupConfig, percentReserve int64) {
	qosMemoryRequests := map[v1.PodQOSClass]int64{
		v1.PodQOSGuaranteed: 0,
		v1.PodQOSBurstable:  0,
	}

	// Sum the pod limits for pods in each QOS class
	pods := m.activePods()
	for _, pod := range pods {
		podMemoryRequest := int64(0)
		qosClass := v1qos.GetPodQOS(pod)
		if qosClass == v1.PodQOSBestEffort {
			// limits are not set for Best Effort pods
			continue
		}
		req, _, err := resource.PodRequestsAndLimits(pod)
		if err != nil {
			glog.V(2).Infof("[Container Manager] Pod resource requests/limits could not be determined.  Not setting QOS memory limts.")
			return
		}
		if request, found := req[v1.ResourceMemory]; found {
			podMemoryRequest += request.Value()
		}
		qosMemoryRequests[qosClass] += podMemoryRequest
	}

	resources := m.getNodeAllocatable()
	allocatableResource, ok := resources[v1.ResourceMemory]
	if !ok {
		glog.V(2).Infof("[Container Manager] Allocatable memory value could not be determined.  Not setting QOS memory limts.")
		return
	}
	allocatable := allocatableResource.Value()
	if allocatable == 0 {
		glog.V(2).Infof("[Container Manager] Memory allocatable reported as 0, might be in standalone mode.  Not setting QOS memory limts.")
		return
	}

	for qos, limits := range qosMemoryRequests {
		glog.V(2).Infof("[Container Manager] %s pod requests total %d bytes (reserve %d%%)", qos, limits, percentReserve)
	}

	// Calculate QOS memory limits
	burstableLimit := allocatable - (qosMemoryRequests[v1.PodQOSGuaranteed] * percentReserve / 100)
	bestEffortLimit := burstableLimit - (qosMemoryRequests[v1.PodQOSBurstable] * percentReserve / 100)
	configs[v1.PodQOSBurstable].ResourceParameters.Memory = &burstableLimit
	configs[v1.PodQOSBestEffort].ResourceParameters.Memory = &bestEffortLimit
}

// retrySetMemoryReserve checks for any QoS cgroups over the limit
// that was attempted to be set in the first Update() and adjusts
// their memory limit to the usage to prevent further growth.
func (m *qosContainerManagerImpl) retrySetMemoryReserve(configs map[v1.PodQOSClass]*CgroupConfig, percentReserve int64) {
	// Unreclaimable memory usage may already exceeded the desired limit
	// Attempt to set the limit near the current usage to put pressure
	// on the cgroup and prevent further growth.
	for qos, config := range configs {
		stats, err := m.cgroupManager.GetResourceStats(config.Name)
		if err != nil {
			glog.V(2).Infof("[Container Manager] %v", err)
			return
		}
		usage := stats.MemoryStats.Usage

		// Because there is no good way to determine of the original Update()
		// on the memory resource was successful, we determine failure of the
		// first attempt by checking if the usage is above the limit we attempt
		// to set.  If it is, we assume the first attempt to set the limit failed
		// and try again setting the limit to the usage.  Otherwise we leave
		// the CgroupConfig as is.
		if configs[qos].ResourceParameters.Memory != nil && usage > *configs[qos].ResourceParameters.Memory {
			configs[qos].ResourceParameters.Memory = &usage
		}
	}
}

func (m *qosContainerManagerImpl) UpdateCgroups() error {
	m.Lock()
	defer m.Unlock()

	qosConfigs := map[v1.PodQOSClass]*CgroupConfig{
		v1.PodQOSBurstable: {
			Name:               CgroupName(m.qosContainersInfo.Burstable),
			ResourceParameters: &ResourceConfig{},
		},
		v1.PodQOSBestEffort: {
			Name:               CgroupName(m.qosContainersInfo.BestEffort),
			ResourceParameters: &ResourceConfig{},
		},
	}

	// update the qos level cgroup settings for cpu shares
	if err := m.setCPUCgroupConfig(qosConfigs); err != nil {
		return err
	}

	for resource, percentReserve := range m.qosReserved {
		switch resource {
		case v1.ResourceMemory:
			m.setMemoryReserve(qosConfigs, percentReserve)
		}
	}
	updateSuccess := true
	for _, config := range qosConfigs {
		err := m.cgroupManager.Update(config)
		if err != nil {
			updateSuccess = false
		}
	}
	if updateSuccess {
		glog.V(4).Infof("[ContainerManager]: Updated QoS cgroup configuration")
		return nil
	}

	// If the resource can adjust the ResourceConfig to increase likelihood of
	// success, call the adjustment function here.  Otherwise, the Update() will
	// be called again with the same values.
	for resource, percentReserve := range m.qosReserved {
		switch resource {
		case v1.ResourceMemory:
			m.retrySetMemoryReserve(qosConfigs, percentReserve)
		}
	}

	for _, config := range qosConfigs {
		err := m.cgroupManager.Update(config)
		if err != nil {
			glog.Errorf("[ContainerManager]: Failed to update QoS cgroup configuration")
			return err
		}
	}

	glog.V(4).Infof("[ContainerManager]: Updated QoS cgroup configuration on retry")
	return nil
}

type qosContainerManagerNoop struct {
	cgroupRoot CgroupName
}

var _ QOSContainerManager = &qosContainerManagerNoop{}

func (m *qosContainerManagerNoop) GetQOSContainersInfo() QOSContainersInfo {
	return QOSContainersInfo{}
}

func (m *qosContainerManagerNoop) Start(_ func() v1.ResourceList, _ ActivePodsFunc) error {
	return nil
}

func (m *qosContainerManagerNoop) UpdateCgroups() error {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Dmitry Vyukov. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package sqlparser

import (
	"fmt"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
	"github.com/youtube/vitess/go/vt/sqlparser"
)

func Fuzz(data []byte) int {
	stmt, err := sqlparser.Parse(string(data))
	if err != nil {
		if stmt != nil {
			panic("stmt is not nil on error")
		}
		return 0
	}
	if true {
		data1 := sqlparser.String(stmt)
		stmt1, err := sqlparser.Parse(data1)
		if err != nil {
			fmt.Printf("data0: %q\n", data)
			fmt.Printf("data1: %q\n", data1)
			panic(err)
		}
		if !fuzz.DeepEqual(stmt, stmt1) {
			fmt.Printf("data0: %q\n", data)
			fmt.Printf("data1: %q\n", data1)
			panic("not equal")
		}
	} else {
		sqlparser.String(stmt)
	}
	if sel, ok := stmt.(*sqlparser.Select); ok {
		var nodes []sqlparser.SQLNode
		for _, x := range sel.From {
			nodes = append(nodes, x)
		}
		for _, x := range sel.SelectExprs {
			nodes = append(nodes, x)
		}
		for _, x := range sel.GroupBy {
			nodes = append(nodes, x)
		}
		for _, x := range sel.OrderBy {
			nodes = append(nodes, x)
		}
		nodes = append(nodes, sel.Where)
		nodes = append(nodes, sel.Having)
		nodes = append(nodes, sel.Limit)
		for _, n := range nodes {
			if n == nil {
				continue
			}
			if x, ok := n.(sqlparser.SimpleTableExpr); ok {
				sqlparser.GetTableName(x)
			}
			if x, ok := n.(sqlparser.Expr); ok {
				sqlparser.GetColName(x)
			}
			if x, ok := n.(sqlparser.ValExpr); ok {
				sqlparser.IsValue(x)
			}
			if x, ok := n.(sqlparser.ValExpr); ok {
				sqlparser.IsColName(x)
			}
			if x, ok := n.(sqlparser.ValExpr); ok {
				sqlparser.IsSimpleTuple(x)
			}
			if x, ok := n.(sqlparser.ValExpr); ok {
				sqlparser.AsInterface(x)
			}
			if x, ok := n.(sqlparser.BoolExpr); ok {
				sqlparser.HasINClause([]sqlparser.BoolExpr{x})
			}
		}
	}
	buf := sqlparser.NewTrackedBuffer(nil)
	stmt.Format(buf)
	pq := buf.ParsedQuery()
	vars := map[string]interface{}{
		"A": 42,
		"B": 123123123,
		"C": "",
		"D": "a",
		"E": "foobar",
		"F": 1.1,
	}
	pq.GenerateQuery(vars)
	return 1
}
<commit_msg>Updated the sqlparser fuzzer to use the latest version of the sqlparser library.<commit_after>// Copyright 2015 Dmitry Vyukov. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package sqlparser

import (
	"bytes"
	"fmt"
	"io"

	"github.com/youtube/vitess/go/sqltypes"
	querypb "github.com/youtube/vitess/go/vt/proto/query"
	"github.com/youtube/vitess/go/vt/sqlparser"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
)

// shortReader is a io.Reader that forces all reads to only be
// a few bytes at a time. This helps force more code paths in
// the parser which makes heavy use of an internal buffer.
type shortReader struct {
	r io.Reader
	n int
}

func (r *shortReader) Read(p []byte) (n int, err error) {
	if len(p) > r.n {
		p = p[:r.n]
	}
	return r.r.Read(p)
}

func parseAll(data []byte) ([]sqlparser.Statement, error) {
	r := &shortReader{
		r: bytes.NewReader(data),
		n: 3,
	}

	tokens := sqlparser.NewTokenizer(r)

	var statements []sqlparser.Statement
	for i := 0; i < 1000; i++ { // Only allow 1000 statements
		if stmt, err := sqlparser.ParseNext(tokens); err != nil {
			if stmt != nil {
				panic("stmt is not nil on error")
			}
			if err == io.EOF {
				err = nil
			}
			return statements, err
		} else {
			statements = append(statements, stmt)
		}
	}

	panic("ParseNext loop")
}

// stringAndParse turns the Statement into a SQL string, re-parses
// that string, and checks the result matches the original.
func stringAndParse(data []byte, stmt sqlparser.Statement) {
	data1 := sqlparser.String(stmt)
	stmt1, err := sqlparser.Parse(data1)
	if err != nil {
		fmt.Printf("data0: %q\n", data)
		fmt.Printf("data1: %q\n", data1)
		panic(err)
	}
	if !fuzz.DeepEqual(stmt, stmt1) {
		fmt.Printf("data0: %q\n", data)
		fmt.Printf("data1: %q\n", data1)
		panic("not equal")
	}
}

func Fuzz(data []byte) int {
	stmts, err := parseAll(data)
	if err != nil {
		return 0
	}
	for _, stmt := range stmts {
		stringAndParse(data, stmt)

		if sel, ok := stmt.(*sqlparser.Select); ok {
			var nodes []sqlparser.SQLNode
			for _, x := range sel.From {
				nodes = append(nodes, x)
			}
			for _, x := range sel.SelectExprs {
				nodes = append(nodes, x)
			}
			for _, x := range sel.GroupBy {
				nodes = append(nodes, x)
			}
			for _, x := range sel.OrderBy {
				nodes = append(nodes, x)
			}
			nodes = append(nodes, sel.Where)
			nodes = append(nodes, sel.Having)
			nodes = append(nodes, sel.Limit)
			for _, n := range nodes {
				if n == nil {
					continue
				}
				if x, ok := n.(sqlparser.SimpleTableExpr); ok {
					sqlparser.GetTableName(x)
				}
				if x, ok := n.(sqlparser.Expr); ok {
					sqlparser.IsColName(x)
					sqlparser.IsValue(x)
					sqlparser.IsNull(x)
					sqlparser.IsSimpleTuple(x)
				}
			}
		}
		pq := sqlparser.NewParsedQuery(stmt)
		vars := map[string]*querypb.BindVariable{
			"A": sqltypes.Int64BindVariable(42),
			"B": sqltypes.Uint64BindVariable(123123123),
			"C": sqltypes.StringBindVariable("aa"),
			"D": sqltypes.BytesBindVariable([]byte("a")),
			"E": sqltypes.StringBindVariable("foobar"),
			"F": sqltypes.Float64BindVariable(1.1),
		}
		pq.GenerateQuery(vars, nil)
	}
	return 1
}
<|endoftext|>


<commit_before>package pt

import "math"

type Cube struct {
	Min      Vector
	Max      Vector
	Material Material
	Box      Box
}

func NewCube(min, max Vector, material Material) Shape {
	box := Box{min, max}
	return &Cube{min, max, material, box}
}

func (c *Cube) Compile() {
}

func (c *Cube) BoundingBox() Box {
	return c.Box
}

func (c *Cube) Intersect(r Ray) Hit {
	n := c.Min.Sub(r.Origin).Div(r.Direction)
	f := c.Max.Sub(r.Origin).Div(r.Direction)
	n, f = n.Min(f), n.Max(f)
	t0 := math.Max(math.Max(n.X, n.Y), n.Z)
	t1 := math.Min(math.Min(f.X, f.Y), f.Z)
	if t0 > 0 && t0 < t1 {
		return Hit{c, t0, nil}
	}
	return NoHit
}

func (c *Cube) UV(p Vector) Vector {
	p = p.Sub(c.Min).Div(c.Max.Sub(c.Min))
	return Vector{p.X, p.Z, 0}
}

func (c *Cube) MaterialAt(p Vector) Material {
	return c.Material
}

func (c *Cube) NormalAt(p Vector) Vector {
	switch {
	case p.X < c.Min.X+EPS:
		return Vector{-1, 0, 0}
	case p.X > c.Max.X-EPS:
		return Vector{1, 0, 0}
	case p.Y < c.Min.Y+EPS:
		return Vector{0, -1, 0}
	case p.Y > c.Max.Y-EPS:
		return Vector{0, 1, 0}
	case p.Z < c.Min.Z+EPS:
		return Vector{0, 0, -1}
	case p.Z > c.Max.Z-EPS:
		return Vector{0, 0, 1}
	}
	return Vector{0, 1, 0}
}
<commit_msg>Cube.Mesh<commit_after>package pt

import "math"

type Cube struct {
	Min      Vector
	Max      Vector
	Material Material
	Box      Box
}

func NewCube(min, max Vector, material Material) *Cube {
	box := Box{min, max}
	return &Cube{min, max, material, box}
}

func (c *Cube) Compile() {
}

func (c *Cube) BoundingBox() Box {
	return c.Box
}

func (c *Cube) Intersect(r Ray) Hit {
	n := c.Min.Sub(r.Origin).Div(r.Direction)
	f := c.Max.Sub(r.Origin).Div(r.Direction)
	n, f = n.Min(f), n.Max(f)
	t0 := math.Max(math.Max(n.X, n.Y), n.Z)
	t1 := math.Min(math.Min(f.X, f.Y), f.Z)
	if t0 > 0 && t0 < t1 {
		return Hit{c, t0, nil}
	}
	return NoHit
}

func (c *Cube) UV(p Vector) Vector {
	p = p.Sub(c.Min).Div(c.Max.Sub(c.Min))
	return Vector{p.X, p.Z, 0}
}

func (c *Cube) MaterialAt(p Vector) Material {
	return c.Material
}

func (c *Cube) NormalAt(p Vector) Vector {
	switch {
	case p.X < c.Min.X+EPS:
		return Vector{-1, 0, 0}
	case p.X > c.Max.X-EPS:
		return Vector{1, 0, 0}
	case p.Y < c.Min.Y+EPS:
		return Vector{0, -1, 0}
	case p.Y > c.Max.Y-EPS:
		return Vector{0, 1, 0}
	case p.Z < c.Min.Z+EPS:
		return Vector{0, 0, -1}
	case p.Z > c.Max.Z-EPS:
		return Vector{0, 0, 1}
	}
	return Vector{0, 1, 0}
}

func (c *Cube) Mesh() *Mesh {
	a := c.Min
	b := c.Max
	z := Vector{}
	m := c.Material
	v000 := Vector{a.X, a.Y, a.Z}
	v001 := Vector{a.X, a.Y, b.Z}
	v010 := Vector{a.X, b.Y, a.Z}
	v011 := Vector{a.X, b.Y, b.Z}
	v100 := Vector{b.X, a.Y, a.Z}
	v101 := Vector{b.X, a.Y, b.Z}
	v110 := Vector{b.X, b.Y, a.Z}
	v111 := Vector{b.X, b.Y, b.Z}
	triangles := []*Triangle{
		NewTriangle(v000, v100, v110, z, z, z, m),
		NewTriangle(v000, v110, v010, z, z, z, m),
		NewTriangle(v001, v101, v111, z, z, z, m),
		NewTriangle(v001, v111, v011, z, z, z, m),
		NewTriangle(v000, v100, v101, z, z, z, m),
		NewTriangle(v000, v101, v001, z, z, z, m),
		NewTriangle(v010, v110, v111, z, z, z, m),
		NewTriangle(v010, v111, v011, z, z, z, m),
		NewTriangle(v000, v010, v011, z, z, z, m),
		NewTriangle(v000, v011, v001, z, z, z, m),
		NewTriangle(v100, v110, v111, z, z, z, m),
		NewTriangle(v100, v111, v101, z, z, z, m),
	}
	return NewMesh(triangles)
}
<|endoftext|>


<commit_before>package atlas

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/atlas-go/archive"
	"github.com/hashicorp/atlas-go/v1"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
)

const BuildEnvKey = "ATLAS_BUILD_ID"

// Artifacts can return a string for this state key and the post-processor
// will use automatically use this as the type. The user's value overrides
// this if `artifact_type_override` is set to true.
const ArtifactStateType = "atlas.artifact.type"

// Artifacts can return a map[string]string for this state key and this
// post-processor will automatically merge it into the metadata for any
// uploaded artifact versions.
const ArtifactStateMetadata = "atlas.artifact.metadata"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	Artifact     string
	Type         string `mapstructure:"artifact_type"`
	TypeOverride bool   `mapstructure:"artifact_type_override"`
	Metadata     map[string]string

	ServerAddr string `mapstructure:"server_address"`
	Token      string

	// This shouldn't ever be set outside of unit tests.
	Test bool `mapstructure:"test"`

	tpl        *packer.ConfigTemplate
	user, name string
	buildId    int
}

type PostProcessor struct {
	config Config
	client *atlas.Client
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	_, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	templates := map[string]*string{
		"artifact":       &p.config.Artifact,
		"type":           &p.config.Type,
		"server_address": &p.config.ServerAddr,
		"token":          &p.config.Token,
	}

	errs := new(packer.MultiError)
	for key, ptr := range templates {
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", key, err))
		}
	}

	required := map[string]*string{
		"artifact":      &p.config.Artifact,
		"artifact_type": &p.config.Type,
	}

	for key, ptr := range required {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("%s must be set", key))
		}
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	p.config.user, p.config.name, err = atlas.ParseSlug(p.config.Artifact)
	if err != nil {
		return err
	}

	// If we have a build ID, save it
	if v := os.Getenv(BuildEnvKey); v != "" {
		raw, err := strconv.ParseInt(v, 0, 0)
		if err != nil {
			return fmt.Errorf(
				"Error parsing build ID: %s", err)
		}

		p.config.buildId = int(raw)
	}

	// Build the client
	p.client = atlas.DefaultClient()
	if p.config.ServerAddr != "" {
		p.client, err = atlas.NewClient(p.config.ServerAddr)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}
	if p.config.Token != "" {
		p.client.Token = p.config.Token
	}

	if !p.config.Test {
		// Verify the client
		if err := p.client.Verify(); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	if _, err := p.client.Artifact(p.config.user, p.config.name); err != nil {
		if err != atlas.ErrNotFound {
			return nil, false, fmt.Errorf(
				"Error finding artifact: %s", err)
		}

		// Artifact doesn't exist, create it
		ui.Message(fmt.Sprintf("Creating artifact: %s", p.config.Artifact))
		_, err = p.client.CreateArtifact(p.config.user, p.config.name)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error creating artifact: %s", err)
		}
	}

	opts := &atlas.UploadArtifactOpts{
		User:     p.config.user,
		Name:     p.config.name,
		Type:     p.config.Type,
		ID:       artifact.Id(),
		Metadata: p.metadata(artifact),
		BuildID:  p.config.buildId,
	}

	if fs := artifact.Files(); len(fs) > 0 {
		var archiveOpts archive.ArchiveOpts

		// We have files. We want to compress/upload them. If we have just
		// one file, then we use it as-is. Otherwise, we compress all of
		// them into a single file.
		var path string
		if len(fs) == 1 {
			path = fs[0]
		} else {
			path = longestCommonPrefix(fs)
			if path == "" {
				return nil, false, fmt.Errorf(
					"No common prefix for achiving files: %v", fs)
			}

			// Modify the archive options to only include the files
			// that are in our file list.
			include := make([]string, 0, len(fs))
			for i, f := range fs {
				include[i] = strings.Replace(f, path, "", 1)
			}
			archiveOpts.Include = include
		}

		r, err := archive.CreateArchive(path, &archiveOpts)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error archiving artifact: %s", err)
		}
		defer r.Close()

		opts.File = r
		opts.FileSize = r.Size
	}

	ui.Message("Uploading artifact version...")
	var av *atlas.ArtifactVersion
	doneCh := make(chan struct{})
	errCh := make(chan error, 1)
	go func() {
		var err error
		av, err = p.client.UploadArtifact(opts)
		if err != nil {
			errCh <- err
			return
		}
		close(doneCh)
	}()

	select {
	case err := <-errCh:
		return nil, false, fmt.Errorf("Error uploading: %s", err)
	case <-doneCh:
	}

	return &Artifact{
		Name:    p.config.Artifact,
		Type:    p.config.Type,
		Version: av.Version,
	}, true, nil
}

func (p *PostProcessor) metadata(artifact packer.Artifact) map[string]string {
	var metadata map[string]string
	metadataRaw := artifact.State(ArtifactStateMetadata)
	if metadataRaw != nil {
		if err := mapstructure.Decode(metadataRaw, &metadata); err != nil {
			panic(err)
		}
	}

	if p.config.Metadata != nil {
		// If we have no extra metadata, just return as-is
		if metadata == nil {
			return p.config.Metadata
		}

		// Merge the metadata
		for k, v := range p.config.Metadata {
			metadata[k] = v
		}
	}

	return metadata
}

func (p *PostProcessor) artifactType(artifact packer.Artifact) string {
	if !p.config.TypeOverride {
		if v := artifact.State(ArtifactStateType); v != nil {
			return v.(string)
		}
	}

	return p.config.Type
}
<commit_msg>post-processor/atlas: fix index out of range panic when artifacts are present<commit_after>package atlas

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/atlas-go/archive"
	"github.com/hashicorp/atlas-go/v1"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
)

const BuildEnvKey = "ATLAS_BUILD_ID"

// Artifacts can return a string for this state key and the post-processor
// will use automatically use this as the type. The user's value overrides
// this if `artifact_type_override` is set to true.
const ArtifactStateType = "atlas.artifact.type"

// Artifacts can return a map[string]string for this state key and this
// post-processor will automatically merge it into the metadata for any
// uploaded artifact versions.
const ArtifactStateMetadata = "atlas.artifact.metadata"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	Artifact     string
	Type         string `mapstructure:"artifact_type"`
	TypeOverride bool   `mapstructure:"artifact_type_override"`
	Metadata     map[string]string

	ServerAddr string `mapstructure:"server_address"`
	Token      string

	// This shouldn't ever be set outside of unit tests.
	Test bool `mapstructure:"test"`

	tpl        *packer.ConfigTemplate
	user, name string
	buildId    int
}

type PostProcessor struct {
	config Config
	client *atlas.Client
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	_, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	templates := map[string]*string{
		"artifact":       &p.config.Artifact,
		"type":           &p.config.Type,
		"server_address": &p.config.ServerAddr,
		"token":          &p.config.Token,
	}

	errs := new(packer.MultiError)
	for key, ptr := range templates {
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", key, err))
		}
	}

	required := map[string]*string{
		"artifact":      &p.config.Artifact,
		"artifact_type": &p.config.Type,
	}

	for key, ptr := range required {
		if *ptr == "" {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("%s must be set", key))
		}
	}

	if len(errs.Errors) > 0 {
		return errs
	}

	p.config.user, p.config.name, err = atlas.ParseSlug(p.config.Artifact)
	if err != nil {
		return err
	}

	// If we have a build ID, save it
	if v := os.Getenv(BuildEnvKey); v != "" {
		raw, err := strconv.ParseInt(v, 0, 0)
		if err != nil {
			return fmt.Errorf(
				"Error parsing build ID: %s", err)
		}

		p.config.buildId = int(raw)
	}

	// Build the client
	p.client = atlas.DefaultClient()
	if p.config.ServerAddr != "" {
		p.client, err = atlas.NewClient(p.config.ServerAddr)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}
	if p.config.Token != "" {
		p.client.Token = p.config.Token
	}

	if !p.config.Test {
		// Verify the client
		if err := p.client.Verify(); err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error initializing client: %s", err))
			return errs
		}
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	if _, err := p.client.Artifact(p.config.user, p.config.name); err != nil {
		if err != atlas.ErrNotFound {
			return nil, false, fmt.Errorf(
				"Error finding artifact: %s", err)
		}

		// Artifact doesn't exist, create it
		ui.Message(fmt.Sprintf("Creating artifact: %s", p.config.Artifact))
		_, err = p.client.CreateArtifact(p.config.user, p.config.name)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error creating artifact: %s", err)
		}
	}

	opts := &atlas.UploadArtifactOpts{
		User:     p.config.user,
		Name:     p.config.name,
		Type:     p.config.Type,
		ID:       artifact.Id(),
		Metadata: p.metadata(artifact),
		BuildID:  p.config.buildId,
	}

	if fs := artifact.Files(); len(fs) > 0 {
		var archiveOpts archive.ArchiveOpts

		// We have files. We want to compress/upload them. If we have just
		// one file, then we use it as-is. Otherwise, we compress all of
		// them into a single file.
		var path string
		if len(fs) == 1 {
			path = fs[0]
		} else {
			path = longestCommonPrefix(fs)
			if path == "" {
				return nil, false, fmt.Errorf(
					"No common prefix for achiving files: %v", fs)
			}

			// Modify the archive options to only include the files
			// that are in our file list.
			include := make([]string, len(fs))
			for i, f := range fs {
				include[i] = strings.Replace(f, path, "", 1)
			}
			archiveOpts.Include = include
		}

		r, err := archive.CreateArchive(path, &archiveOpts)
		if err != nil {
			return nil, false, fmt.Errorf(
				"Error archiving artifact: %s", err)
		}
		defer r.Close()

		opts.File = r
		opts.FileSize = r.Size
	}

	ui.Message("Uploading artifact version...")
	var av *atlas.ArtifactVersion
	doneCh := make(chan struct{})
	errCh := make(chan error, 1)
	go func() {
		var err error
		av, err = p.client.UploadArtifact(opts)
		if err != nil {
			errCh <- err
			return
		}
		close(doneCh)
	}()

	select {
	case err := <-errCh:
		return nil, false, fmt.Errorf("Error uploading: %s", err)
	case <-doneCh:
	}

	return &Artifact{
		Name:    p.config.Artifact,
		Type:    p.config.Type,
		Version: av.Version,
	}, true, nil
}

func (p *PostProcessor) metadata(artifact packer.Artifact) map[string]string {
	var metadata map[string]string
	metadataRaw := artifact.State(ArtifactStateMetadata)
	if metadataRaw != nil {
		if err := mapstructure.Decode(metadataRaw, &metadata); err != nil {
			panic(err)
		}
	}

	if p.config.Metadata != nil {
		// If we have no extra metadata, just return as-is
		if metadata == nil {
			return p.config.Metadata
		}

		// Merge the metadata
		for k, v := range p.config.Metadata {
			metadata[k] = v
		}
	}

	return metadata
}

func (p *PostProcessor) artifactType(artifact packer.Artifact) string {
	if !p.config.TypeOverride {
		if v := artifact.State(ArtifactStateType); v != nil {
			return v.(string)
		}
	}

	return p.config.Type
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path/filepath"

	"fmt"

	"net/http"
	"strings"

	"github.com/braintree/manners"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/data/templates"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/migration"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/routing"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/pufferpanel/pufferd/shutdown"
	"github.com/pufferpanel/pufferd/uninstaller"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	GITHASH      = "unknown"
)

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var regenerate bool
	var migrate bool
	var uninstall bool
	var configPath string
	var pid int
	var installService bool
	var runDaemon bool
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.BoolVar(&regenerate, "regenerate", false, "Regenerate pufferd templates")
	flag.BoolVar(&migrate, "migrate", false, "Migrate Scales data to pufferd")
	flag.BoolVar(&uninstall, "uninstall", false, "Uninstall pufferd")
	flag.StringVar(&configPath, "config", "config.json", "Path to pufferd config.json")
	flag.IntVar(&pid, "shutdown", 0, "PID to shut down")
	flag.BoolVar(&installService, "installService", false, "Installs the pufferd service file")
	flag.BoolVar(&runDaemon, "daemon", false, "Runs the daemon")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s)", VERSION, GITHASH)

	if pid != 0 {
		logging.Info("Shutting down")
		shutdown.Command(pid)
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			shutdown.CompleteShutdown()
		}
	}

	if uninstall {
		fmt.Println("This option will UNINSTALL pufferd, are you sure? Please enter \"yes\" to proceed [no]")
		var response string
		fmt.Scanln(&response)
		if strings.ToLower(response) == "yes" || strings.ToLower(response) == "y" {
			if os.Geteuid() != 0 {
				logging.Error("To uninstall pufferd you need to have sudo or root privileges")
			} else {
				config.Load(configPath)
				uninstaller.StartProcess()
				logging.Info("pufferd is now uninstalled.")
			}
		} else {
			logging.Info("Uninstall process aborted")
			logging.Info("Exiting")
		}
		return
	}

	if version || !daemon {
		os.Stdout.WriteString(versionString + "\r\n")
	}

	if license {
		os.Stdout.WriteString(data.LICENSE + "\r\n")
	}

	if regenerate {
		config.Load(configPath)
		programs.Initialize()

		if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
			logging.Info("No template directory found, creating")
			err = os.MkdirAll(programs.TemplateFolder, 0755)
			if err != nil {
				logging.Error("Error creating template folder", err)
			}
		}
		// Overwrite existing templates
		templates.CopyTemplates()
		logging.Info("Templates regenerated")
	}

	if migrate {
		config.Load(configPath)
		migration.MigrateFromScales()
	}

	if installService {
		install.InstallService()
	}

	if license || version || regenerate || migrate || pid != 0 {
		return
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	var logPath = config.GetOrDefault("logPath", "logs")
	logging.SetLogFolder(logPath)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || installService || !runDaemon {
		return
	}

	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}
	if files, _ := ioutil.ReadDir(programs.TemplateFolder); len(files) == 0 {
		logging.Info("Templates being copied to " + programs.TemplateFolder)
		templates.CopyTemplates()
	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		os.MkdirAll(programs.ServerFolder, 0755)
	}

	programs.LoadFromFolder()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() && element.IsAutoStart() {
			logging.Info("Starting server " + element.Id())
			element.Start()
		}
	}

	r := routing.ConfigureWeb()

	useHttps := false

	dataFolder := config.GetOrDefault("datafolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	//check if there's an update
	if config.GetOrDefault("update-check", "true") == "true" {
		go func() {
			url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
			logging.Debug("Checking for updates using " + url)
			resp, err := http.Get(url)
			if err != nil {
				return
			}
			defer resp.Body.Close()
			onlineVersion, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return
			}
			if string(onlineVersion) != GITHASH {
				logging.Infof("DL server reports a different hash than this version, an update may be available")
				logging.Infof("Installed: %s", GITHASH)
				logging.Infof("Online: %s", onlineVersion)
			}
		}()
	}

	web := config.GetOrDefault("web", config.GetOrDefault("webhost", "0.0.0.0")+":"+config.GetOrDefault("webport", "5656"))

	shutdown.CreateHook()

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
	shutdown.Shutdown()
}
<commit_msg>Fix variable<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path/filepath"

	"fmt"

	"net/http"
	"strings"

	"github.com/braintree/manners"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/data/templates"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/migration"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/routing"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/pufferpanel/pufferd/shutdown"
	"github.com/pufferpanel/pufferd/uninstaller"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	GITHASH      = "unknown"
)

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var regenerate bool
	var migrate bool
	var uninstall bool
	var configPath string
	var pid int
	var installService bool
	var runDaemon bool
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.BoolVar(&regenerate, "regenerate", false, "Regenerate pufferd templates")
	flag.BoolVar(&migrate, "migrate", false, "Migrate Scales data to pufferd")
	flag.BoolVar(&uninstall, "uninstall", false, "Uninstall pufferd")
	flag.StringVar(&configPath, "config", "config.json", "Path to pufferd config.json")
	flag.IntVar(&pid, "shutdown", 0, "PID to shut down")
	flag.BoolVar(&installService, "installService", false, "Installs the pufferd service file")
	flag.BoolVar(&runDaemon, "daemon", false, "Runs the daemon")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s)", VERSION, GITHASH)

	if pid != 0 {
		logging.Info("Shutting down")
		shutdown.Command(pid)
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			shutdown.CompleteShutdown()
		}
	}

	if uninstall {
		fmt.Println("This option will UNINSTALL pufferd, are you sure? Please enter \"yes\" to proceed [no]")
		var response string
		fmt.Scanln(&response)
		if strings.ToLower(response) == "yes" || strings.ToLower(response) == "y" {
			if os.Geteuid() != 0 {
				logging.Error("To uninstall pufferd you need to have sudo or root privileges")
			} else {
				config.Load(configPath)
				uninstaller.StartProcess()
				logging.Info("pufferd is now uninstalled.")
			}
		} else {
			logging.Info("Uninstall process aborted")
			logging.Info("Exiting")
		}
		return
	}

	if version || !runDaemon {
		os.Stdout.WriteString(versionString + "\r\n")
	}

	if license {
		os.Stdout.WriteString(data.LICENSE + "\r\n")
	}

	if regenerate {
		config.Load(configPath)
		programs.Initialize()

		if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
			logging.Info("No template directory found, creating")
			err = os.MkdirAll(programs.TemplateFolder, 0755)
			if err != nil {
				logging.Error("Error creating template folder", err)
			}
		}
		// Overwrite existing templates
		templates.CopyTemplates()
		logging.Info("Templates regenerated")
	}

	if migrate {
		config.Load(configPath)
		migration.MigrateFromScales()
	}

	if installService {
		install.InstallService()
	}

	if license || version || regenerate || migrate || pid != 0 {
		return
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	var logPath = config.GetOrDefault("logPath", "logs")
	logging.SetLogFolder(logPath)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || installService || !runDaemon {
		return
	}

	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}
	if files, _ := ioutil.ReadDir(programs.TemplateFolder); len(files) == 0 {
		logging.Info("Templates being copied to " + programs.TemplateFolder)
		templates.CopyTemplates()
	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		os.MkdirAll(programs.ServerFolder, 0755)
	}

	programs.LoadFromFolder()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() && element.IsAutoStart() {
			logging.Info("Starting server " + element.Id())
			element.Start()
		}
	}

	r := routing.ConfigureWeb()

	useHttps := false

	dataFolder := config.GetOrDefault("datafolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	//check if there's an update
	if config.GetOrDefault("update-check", "true") == "true" {
		go func() {
			url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
			logging.Debug("Checking for updates using " + url)
			resp, err := http.Get(url)
			if err != nil {
				return
			}
			defer resp.Body.Close()
			onlineVersion, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return
			}
			if string(onlineVersion) != GITHASH {
				logging.Infof("DL server reports a different hash than this version, an update may be available")
				logging.Infof("Installed: %s", GITHASH)
				logging.Infof("Online: %s", onlineVersion)
			}
		}()
	}

	web := config.GetOrDefault("web", config.GetOrDefault("webhost", "0.0.0.0")+":"+config.GetOrDefault("webport", "5656"))

	shutdown.CreateHook()

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
	shutdown.Shutdown()
}
<|endoftext|>


<commit_before>// Dict and StringDict type
//
// The idea is that most dicts just have strings for keys so we use
// the simpler StringDict and promote it into a Dict when necessary

package py

var StringDictType = NewType("dict", "dict() -> new empty dictionary\ndict(mapping) -> new dictionary initialized from a mapping object's\n    (key, value) pairs\ndict(iterable) -> new dictionary initialized as if via:\n    d = {}\n    for k, v in iterable:\n        d[k] = v\ndict(**kwargs) -> new dictionary initialized with the name=value pairs\n    in the keyword argument list.  For example:  dict(one=1, two=2)")

var DictType = NewType("dict", "dict() -> new empty dictionary\ndict(mapping) -> new dictionary initialized from a mapping object's\n    (key, value) pairs\ndict(iterable) -> new dictionary initialized as if via:\n    d = {}\n    for k, v in iterable:\n        d[k] = v\ndict(**kwargs) -> new dictionary initialized with the name=value pairs\n    in the keyword argument list.  For example:  dict(one=1, two=2)")

// String to object dictionary
//
// Used for variables etc where the keys can only be strings
type StringDict map[string]Object

// Type of this StringDict object
func (o StringDict) Type() *Type {
	return StringDictType
}

// Make a new dictionary
func NewStringDict() StringDict {
	return make(StringDict)
}

// Make a new dictionary with reservation for n entries
func NewStringDictSized(n int) StringDict {
	return make(StringDict, n)
}

// Copy a dictionary
func (d StringDict) Copy() StringDict {
	e := make(StringDict, len(d))
	for k, v := range d {
		e[k] = v
	}
	return e
}
<commit_msg>py: dict.__setitem__ and dict.__getitem__<commit_after>// Dict and StringDict type
//
// The idea is that most dicts just have strings for keys so we use
// the simpler StringDict and promote it into a Dict when necessary

package py

var StringDictType = NewType("dict", "dict() -> new empty dictionary\ndict(mapping) -> new dictionary initialized from a mapping object's\n    (key, value) pairs\ndict(iterable) -> new dictionary initialized as if via:\n    d = {}\n    for k, v in iterable:\n        d[k] = v\ndict(**kwargs) -> new dictionary initialized with the name=value pairs\n    in the keyword argument list.  For example:  dict(one=1, two=2)")

var DictType = NewType("dict", "dict() -> new empty dictionary\ndict(mapping) -> new dictionary initialized from a mapping object's\n    (key, value) pairs\ndict(iterable) -> new dictionary initialized as if via:\n    d = {}\n    for k, v in iterable:\n        d[k] = v\ndict(**kwargs) -> new dictionary initialized with the name=value pairs\n    in the keyword argument list.  For example:  dict(one=1, two=2)")

// String to object dictionary
//
// Used for variables etc where the keys can only be strings
type StringDict map[string]Object

// Type of this StringDict object
func (o StringDict) Type() *Type {
	return StringDictType
}

// Make a new dictionary
func NewStringDict() StringDict {
	return make(StringDict)
}

// Make a new dictionary with reservation for n entries
func NewStringDictSized(n int) StringDict {
	return make(StringDict, n)
}

// Copy a dictionary
func (d StringDict) Copy() StringDict {
	e := make(StringDict, len(d))
	for k, v := range d {
		e[k] = v
	}
	return e
}

func (d StringDict) M__getitem__(key Object) Object {
	str, ok := key.(String)
	if ok {
		res, ok := d[string(str)]
		if ok {
			return res
		}
	}
	panic(ExceptionNewf(KeyError, "%v", key))
}

func (d StringDict) M__setitem__(key, value Object) Object {
	str, ok := key.(String)
	if !ok {
		panic("FIXME can only have string keys!")
	}
	d[string(str)] = value
	return None
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/alexflint/go-arg"

	"github.com/smancke/guble/client"
	"github.com/smancke/guble/guble"
)

type Args struct {
	Exit     bool     `arg:"-x,help: Exit after sending the commands"`
	Commands []string `arg:"positional,help: The commands to send after startup"`
	Verbose  bool     `arg:"-v,help: Display verbose server communication"`
	Url      string   `arg:"help: The websocket url to connect (ws://localhost:8080/stream/)"`
	User     string   `arg:"help: The user name to connect with (guble-cli)"`
	LogInfo  bool     `arg:"--log-info,help: Log on INFO level (false)" env:"GUBLE_LOG_INFO"`
	LogDebug bool     `arg:"--log-debug,help: Log on DEBUG level (false)" env:"GUBLE_LOG_DEBUG"`
}

var args Args

// This is a minimal commandline client to connect through a websocket
func main() {
	guble.LogLevel = guble.LEVEL_ERR

	args = loadArgs()
	if args.LogInfo {
		guble.LogLevel = guble.LEVEL_INFO
	}
	if args.LogDebug {
		guble.LogLevel = guble.LEVEL_DEBUG
	}

	origin := "http://localhost/"
	url := fmt.Sprintf("%v/user/%v", removeTrailingSlash(args.Url), args.User)
	client, err := client.Open(url, origin, 100, true)
	if err != nil {
		log.Fatal(err)
	}

	go writeLoop(client)
	go readLoop(client)

	for _, cmd := range args.Commands {
		client.WriteRawMessage([]byte(cmd))
	}
	if args.Exit {
		return
	}
	waitForTermination(func() {})
}

func loadArgs() Args {
	args := Args{
		Verbose: false,
		Url:     "ws://localhost:8080/stream/",
		User:    "guble-cli",
	}

	arg.MustParse(&args)
	return args
}

func readLoop(client *client.Client) {
	for {
		select {
		case incomingMessage := <-client.Messages():
			if args.Verbose {
				fmt.Println(string(incomingMessage.Bytes()))
			} else {
				fmt.Printf("%v: %v\n", incomingMessage.PublisherUserId, incomingMessage.BodyAsString())
			}
		case error := <-client.Errors():
			fmt.Println("ERROR: " + string(error.Bytes()))
		case status := <-client.StatusMessages():
			fmt.Println(string(status.Bytes()))
			fmt.Println()
		}
	}
}

func writeLoop(client *client.Client) {
	shouldStop := false
	for !shouldStop {
		func() {
			defer guble.PanicLogger()
			reader := bufio.NewReader(os.Stdin)
			text, _ := reader.ReadString('\n')
			if strings.TrimSpace(text) == "" {
				return
			}

			if strings.TrimSpace(text) == "?" || strings.TrimSpace(text) == "help" {
				printHelp()
				return
			}

			if strings.HasPrefix(text, ">") {
				fmt.Print("header: ")
				header, _ := reader.ReadString('\n')
				text += header
				fmt.Print("body: ")
				body, _ := reader.ReadString('\n')
				text += strings.TrimSpace(body)
			}

			if args.Verbose {
				log.Printf("Sending: %v\n", text)
			}
			if err := client.WriteRawMessage([]byte(text)); err != nil {
				shouldStop = true
				guble.Err(err.Error())
			}
		}()
	}
}

func waitForTermination(callback func()) {
	sigc := make(chan os.Signal)
	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
	log.Printf("%q", <-sigc)
	callback()
	os.Exit(0)
}

func printHelp() {
	fmt.Println(`
## Commands
?           # print this info

+ /foo/bar  # subscribe to the topic /foo/bar
+ /foo 0    # read from message 0 and subscribe to the topic /foo
+ /foo 0 5  # read messages 0-5 from /foo
+ /foo -5   # read the last 5 messages and subscribe to the topic /foo

- /foo      # cancel the subscription for /foo

> /foo         # send a message to /foo
> /foo/bar 42  # send a message to /foo/bar with publisherid 42
`)
}

func removeTrailingSlash(path string) string {
	if len(path) > 0 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<commit_msg>renamed "error"<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/alexflint/go-arg"

	"github.com/smancke/guble/client"
	"github.com/smancke/guble/guble"
)

type Args struct {
	Exit     bool     `arg:"-x,help: Exit after sending the commands"`
	Commands []string `arg:"positional,help: The commands to send after startup"`
	Verbose  bool     `arg:"-v,help: Display verbose server communication"`
	Url      string   `arg:"help: The websocket url to connect (ws://localhost:8080/stream/)"`
	User     string   `arg:"help: The user name to connect with (guble-cli)"`
	LogInfo  bool     `arg:"--log-info,help: Log on INFO level (false)" env:"GUBLE_LOG_INFO"`
	LogDebug bool     `arg:"--log-debug,help: Log on DEBUG level (false)" env:"GUBLE_LOG_DEBUG"`
}

var args Args

// This is a minimal commandline client to connect through a websocket
func main() {
	guble.LogLevel = guble.LEVEL_ERR

	args = loadArgs()
	if args.LogInfo {
		guble.LogLevel = guble.LEVEL_INFO
	}
	if args.LogDebug {
		guble.LogLevel = guble.LEVEL_DEBUG
	}

	origin := "http://localhost/"
	url := fmt.Sprintf("%v/user/%v", removeTrailingSlash(args.Url), args.User)
	client, err := client.Open(url, origin, 100, true)
	if err != nil {
		log.Fatal(err)
	}

	go writeLoop(client)
	go readLoop(client)

	for _, cmd := range args.Commands {
		client.WriteRawMessage([]byte(cmd))
	}
	if args.Exit {
		return
	}
	waitForTermination(func() {})
}

func loadArgs() Args {
	args := Args{
		Verbose: false,
		Url:     "ws://localhost:8080/stream/",
		User:    "guble-cli",
	}

	arg.MustParse(&args)
	return args
}

func readLoop(client *client.Client) {
	for {
		select {
		case incomingMessage := <-client.Messages():
			if args.Verbose {
				fmt.Println(string(incomingMessage.Bytes()))
			} else {
				fmt.Printf("%v: %v\n", incomingMessage.PublisherUserId, incomingMessage.BodyAsString())
			}
		case e := <-client.Errors():
			fmt.Println("ERROR: " + string(e.Bytes()))
		case status := <-client.StatusMessages():
			fmt.Println(string(status.Bytes()))
			fmt.Println()
		}
	}
}

func writeLoop(client *client.Client) {
	shouldStop := false
	for !shouldStop {
		func() {
			defer guble.PanicLogger()
			reader := bufio.NewReader(os.Stdin)
			text, _ := reader.ReadString('\n')
			if strings.TrimSpace(text) == "" {
				return
			}

			if strings.TrimSpace(text) == "?" || strings.TrimSpace(text) == "help" {
				printHelp()
				return
			}

			if strings.HasPrefix(text, ">") {
				fmt.Print("header: ")
				header, _ := reader.ReadString('\n')
				text += header
				fmt.Print("body: ")
				body, _ := reader.ReadString('\n')
				text += strings.TrimSpace(body)
			}

			if args.Verbose {
				log.Printf("Sending: %v\n", text)
			}
			if err := client.WriteRawMessage([]byte(text)); err != nil {
				shouldStop = true
				guble.Err(err.Error())
			}
		}()
	}
}

func waitForTermination(callback func()) {
	sigc := make(chan os.Signal)
	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
	log.Printf("%q", <-sigc)
	callback()
	os.Exit(0)
}

func printHelp() {
	fmt.Println(`
## Commands
?           # print this info

+ /foo/bar  # subscribe to the topic /foo/bar
+ /foo 0    # read from message 0 and subscribe to the topic /foo
+ /foo 0 5  # read messages 0-5 from /foo
+ /foo -5   # read the last 5 messages and subscribe to the topic /foo

- /foo      # cancel the subscription for /foo

> /foo         # send a message to /foo
> /foo/bar 42  # send a message to /foo/bar with publisherid 42
`)
}

func removeTrailingSlash(path string) string {
	if len(path) > 0 && path[len(path)-1] == '/' {
		return path[:len(path)-1]
	}
	return path
}
<|endoftext|>


<commit_before>package filterlist

import (
	"io"
	"bufio"
	"strings"
	"log"
)

type ABPFilterParser struct { }

func NewABPFilterParser() (*ABPFilterParser) {
	return &ABPFilterParser{}
}

func (parser *ABPFilterParser) Parse(reader io.Reader) (whitelist []Filter, blacklist []Filter, err error) {
	scanner := bufio.NewScanner(reader)

	for scanner.Scan() {
		rule, isException := parser.isExceptionRule(scanner.Text())
		if rule, domainRule := parser.isSimpleDomainRule(rule); domainRule {
			filter, err := NewABPFilter(rule)

			if err {
				log.Fatalf("Could not create ABPFilter on rule %s. Error %t", rule, err)
				break
			}

			if isException {
				whitelist = append(whitelist, *filter)
			} else {
				blacklist = append(blacklist, *filter)
			}
		}
	}

	return whitelist, blacklist, err
}

const (
	exceptionPrefix = "@@"
	domainPrefix = "||"
	domainSuffix = "^"
)

func (parser *ABPFilterParser) isSimpleDomainRule(rule string) (string, bool) {
	if cleanedRule, matchesPrefix := parser.checkRulePrefixAndRemove(rule, domainPrefix); matchesPrefix {
		return parser.checkRuleSuffixAndRemove(cleanedRule, domainSuffix)
	}
	return rule, false
}

func (parser *ABPFilterParser) isExceptionRule(rule string) (string, bool) {

	return parser.checkRulePrefixAndRemove(rule, exceptionPrefix)
}

func (*ABPFilterParser) checkRulePrefixAndRemove(rule string, prefix string) (string, bool) {
	if strings.HasPrefix(rule, prefix) {
		return strings.TrimPrefix(rule, prefix), true
	}
	return rule, false
}

func (*ABPFilterParser) checkRuleSuffixAndRemove(rule string, suffix string) (string, bool) {
	if strings.HasSuffix(rule, suffix) {
		return strings.TrimSuffix(rule, suffix), true
	}
	return rule, false
}<commit_msg>Use clearer names for ABPFilterParser helper methods<commit_after>package filterlist

import (
	"io"
	"bufio"
	"strings"
	"log"
)

type ABPFilterParser struct { }

func NewABPFilterParser() (*ABPFilterParser) {
	return &ABPFilterParser{}
}

func (parser *ABPFilterParser) Parse(reader io.Reader) (whitelist []Filter, blacklist []Filter, err error) {
	scanner := bufio.NewScanner(reader)

	for scanner.Scan() {
		rule, isException := parser.checkAndCleanIfExceptionRule(scanner.Text())
		if rule, domainRule := parser.checkAndCleanIfSimpleDomainRule(rule); domainRule {
			filter, err := NewABPFilter(rule)

			if err {
				log.Fatalf("Could not create ABPFilter on rule %s. Error %t", rule, err)
				break
			}

			if isException {
				whitelist = append(whitelist, *filter)
			} else {
				blacklist = append(blacklist, *filter)
			}
		}
	}

	return whitelist, blacklist, err
}

const (
	exceptionPrefix = "@@"
	domainPrefix = "||"
	domainSuffix = "^"
)

func (parser *ABPFilterParser) checkAndCleanIfSimpleDomainRule(rule string) (string, bool) {
	if cleanedRule, matchesPrefix := parser.checkRulePrefixAndRemove(rule, domainPrefix); matchesPrefix {
		return parser.checkRuleSuffixAndRemove(cleanedRule, domainSuffix)
	}
	return rule, false
}

func (parser *ABPFilterParser) checkAndCleanIfExceptionRule(rule string) (string, bool) {

	return parser.checkRulePrefixAndRemove(rule, exceptionPrefix)
}

func (*ABPFilterParser) checkRulePrefixAndRemove(rule string, prefix string) (string, bool) {
	if strings.HasPrefix(rule, prefix) {
		return strings.TrimPrefix(rule, prefix), true
	}
	return rule, false
}

func (*ABPFilterParser) checkRuleSuffixAndRemove(rule string, suffix string) (string, bool) {
	if strings.HasSuffix(rule, suffix) {
		return strings.TrimSuffix(rule, suffix), true
	}
	return rule, false
}<|endoftext|>


<commit_before>package gobrake

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"
)

type QueryInfo struct {
	Method string
	Route  string
	Query  string
	Start  time.Time
	End    time.Time
}

type queryKey struct {
	Method string
	Route  string
	Query  string
	Time   time.Time
}

type queryKeyStat struct {
	queryKey
	*routeStat
}

type QueryStats struct {
	opt    *NotifierOptions
	apiURL string

	flushTimer *time.Timer
	addWG      *sync.WaitGroup

	mu sync.Mutex
	m  map[queryKey]*routeStat
}

func newQueryStats(opt *NotifierOptions) *QueryStats {
	return &QueryStats{
		opt: opt,
		apiURL: fmt.Sprintf("%s/api/v5/projects/%d/queries-stats",
			opt.Host, opt.ProjectId),
	}
}

func (s *QueryStats) init() {
	if s.flushTimer == nil {
		s.flushTimer = time.AfterFunc(flushPeriod, s.flush)
		s.addWG = new(sync.WaitGroup)
		s.m = make(map[queryKey]*routeStat)
	}
}

func (s *QueryStats) flush() {
	s.mu.Lock()

	s.flushTimer = nil
	addWG := s.addWG
	s.addWG = nil
	m := s.m
	s.m = nil

	s.mu.Unlock()

	addWG.Wait()
	err := s.send(m)
	if err != nil {
		logger.Printf("queryStats.send failed: %s", err)
	}
}

type queriesStatsJSONRequest struct {
	Queries []queryKeyStat `json:"queries"`
}

func (s *QueryStats) send(m map[queryKey]*routeStat) error {
	var queries []queryKeyStat
	for k, v := range m {
		err := v.td.Compress()
		if err != nil {
			return err
		}

		b, err := v.td.AsBytes()
		if err != nil {
			return err
		}
		v.TDigest = b

		queries = append(queries, queryKeyStat{
			queryKey:  k,
			routeStat: v,
		})
	}

	jsonReq := queriesStatsJSONRequest{
		Queries: queries,
	}

	buf := buffers.Get().(*bytes.Buffer)
	defer buffers.Put(buf)

	buf.Reset()
	err := json.NewEncoder(buf).Encode(jsonReq)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", s.apiURL, buf)
	if err != nil {
		return err
	}

	req.Header.Set("Authorization", "Bearer "+s.opt.ProjectKey)
	req.Header.Set("Content-Type", "application/json")
	resp, err := s.opt.HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	buf.Reset()
	_, err = buf.ReadFrom(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return nil
	}

	switch resp.StatusCode {
	case http.StatusUnauthorized:
		return errUnauthorized
	}

	err = fmt.Errorf("got unexpected response status=%q", resp.Status)
	return err
}

func (s *QueryStats) NotifyQuery(q *QueryInfo) error {
	key := queryKey{
		Method: q.Method,
		Route:  q.Route,
		Query:  q.Query,
		Time:   q.Start.UTC().Truncate(time.Minute),
	}

	s.mu.Lock()
	s.init()
	stat, ok := s.m[key]
	if !ok {
		stat = &routeStat{}
		s.m[key] = stat
	}
	addWG := s.addWG
	s.addWG.Add(1)
	s.mu.Unlock()

	ms := float64(q.End.Sub(q.Start)) / float64(time.Millisecond)

	stat.mu.Lock()
	err := stat.Add(ms)
	addWG.Done()
	stat.mu.Unlock()

	return err
}
<commit_msg>NotifyQuery to Notify<commit_after>package gobrake

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"
)

type QueryInfo struct {
	Method string
	Route  string
	Query  string
	Start  time.Time
	End    time.Time
}

type queryKey struct {
	Method string
	Route  string
	Query  string
	Time   time.Time
}

type queryKeyStat struct {
	queryKey
	*routeStat
}

type QueryStats struct {
	opt    *NotifierOptions
	apiURL string

	flushTimer *time.Timer
	addWG      *sync.WaitGroup

	mu sync.Mutex
	m  map[queryKey]*routeStat
}

func newQueryStats(opt *NotifierOptions) *QueryStats {
	return &QueryStats{
		opt: opt,
		apiURL: fmt.Sprintf("%s/api/v5/projects/%d/queries-stats",
			opt.Host, opt.ProjectId),
	}
}

func (s *QueryStats) init() {
	if s.flushTimer == nil {
		s.flushTimer = time.AfterFunc(flushPeriod, s.flush)
		s.addWG = new(sync.WaitGroup)
		s.m = make(map[queryKey]*routeStat)
	}
}

func (s *QueryStats) flush() {
	s.mu.Lock()

	s.flushTimer = nil
	addWG := s.addWG
	s.addWG = nil
	m := s.m
	s.m = nil

	s.mu.Unlock()

	addWG.Wait()
	err := s.send(m)
	if err != nil {
		logger.Printf("queryStats.send failed: %s", err)
	}
}

type queriesStatsJSONRequest struct {
	Queries []queryKeyStat `json:"queries"`
}

func (s *QueryStats) send(m map[queryKey]*routeStat) error {
	var queries []queryKeyStat
	for k, v := range m {
		err := v.td.Compress()
		if err != nil {
			return err
		}

		b, err := v.td.AsBytes()
		if err != nil {
			return err
		}
		v.TDigest = b

		queries = append(queries, queryKeyStat{
			queryKey:  k,
			routeStat: v,
		})
	}

	jsonReq := queriesStatsJSONRequest{
		Queries: queries,
	}

	buf := buffers.Get().(*bytes.Buffer)
	defer buffers.Put(buf)

	buf.Reset()
	err := json.NewEncoder(buf).Encode(jsonReq)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", s.apiURL, buf)
	if err != nil {
		return err
	}

	req.Header.Set("Authorization", "Bearer "+s.opt.ProjectKey)
	req.Header.Set("Content-Type", "application/json")
	resp, err := s.opt.HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	buf.Reset()
	_, err = buf.ReadFrom(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return nil
	}

	switch resp.StatusCode {
	case http.StatusUnauthorized:
		return errUnauthorized
	}

	err = fmt.Errorf("got unexpected response status=%q", resp.Status)
	return err
}

func (s *QueryStats) Notify(q *QueryInfo) error {
	key := queryKey{
		Method: q.Method,
		Route:  q.Route,
		Query:  q.Query,
		Time:   q.Start.UTC().Truncate(time.Minute),
	}

	s.mu.Lock()
	s.init()
	stat, ok := s.m[key]
	if !ok {
		stat = &routeStat{}
		s.m[key] = stat
	}
	addWG := s.addWG
	s.addWG.Add(1)
	s.mu.Unlock()

	ms := float64(q.End.Sub(q.Start)) / float64(time.Millisecond)

	stat.mu.Lock()
	err := stat.Add(ms)
	addWG.Done()
	stat.mu.Unlock()

	return err
}
<|endoftext|>


<commit_before>package cloudformation

import "encoding/json"

// IAMPolicyDocument represents an IAM policy document
type IAMPolicyDocument struct {
	Version   string `json:",omitempty"`
	Statement []IAMPolicyStatement
}

// IAMPrincipal represents a principal in an IAM policy
type IAMPrincipal struct {
	AWS           *StringListExpr `json:",omitempty"`
	CanonicalUser *StringListExpr `json:",omitempty"`
	Federated     *StringListExpr `json:",omitempty"`
	Service       *StringListExpr `json:",omitempty"`
}

// IAMPolicyStatement represents an IAM policy statement
type IAMPolicyStatement struct {
	Sid          string          `json:",omitempty"`
	Effect       string          `json:",omitempty"`
	Principal    *IAMPrincipal   `json:",omitempty"`
	NotPrincipal *IAMPrincipal   `json:",omitempty"`
	Action       *StringListExpr `json:",omitempty"`
	NotAction    *StringListExpr `json:",omitempty"`
	Resource     *StringListExpr `json:",omitempty"`
	Condition    interface{}     `json:",omitempty"`
}

// Avoid infinite loops when we just want to marshal the struct normally
type iamPrincipalCopy IAMPrincipal

// MarshalJSON returns a JSON representation of the object. This has been added
// to handle the special case of "*" as the Principal value.
func (i IAMPrincipal) MarshalJSON() ([]byte, error) {
	// Special case for "*"
	if i.AWS != nil && len(i.AWS.Literal) == 1 && i.AWS.Literal[0].Literal == "*" {
		return json.Marshal(i.AWS.Literal[0].Literal)
	}

	c := iamPrincipalCopy(i)

	return json.Marshal(c)
}

// UnmarshalJSON sets the object from the provided JSON representation. This has
// been added to handle the special case of "*" as the Principal value.
func (i *IAMPrincipal) UnmarshalJSON(data []byte) error {
	// Handle single string values like "*"
	var v string
	err := json.Unmarshal(data, &v)
	if err == nil {
		i.AWS = StringList(String(v))
		i.CanonicalUser = nil
		i.Federated = nil
		i.Service = nil
		return nil
	}

	// Handle all other values
	var v2 iamPrincipalCopy
	err = json.Unmarshal(data, &v2)
	if err != nil {
		return err
	}

	i.AWS = v2.AWS
	i.CanonicalUser = v2.CanonicalUser
	i.Federated = v2.Federated
	i.Service = v2.Service

	return nil
}
<commit_msg>add a ToJSON method to IAMPolicyDocument<commit_after>package cloudformation

import "encoding/json"

// IAMPolicyDocument represents an IAM policy document
type IAMPolicyDocument struct {
	Version   string `json:",omitempty"`
	Statement []IAMPolicyStatement
}

// ToJSON returns the JSON representation of the policy document or
// panics if the object cannot be marshaled.
func (i IAMPolicyDocument) ToJSON() string {
	buf, err := json.Marshal(i)
	if err != nil {
		panic(err)
	}
	return string(buf)
}

// IAMPrincipal represents a principal in an IAM policy
type IAMPrincipal struct {
	AWS           *StringListExpr `json:",omitempty"`
	CanonicalUser *StringListExpr `json:",omitempty"`
	Federated     *StringListExpr `json:",omitempty"`
	Service       *StringListExpr `json:",omitempty"`
}

// IAMPolicyStatement represents an IAM policy statement
type IAMPolicyStatement struct {
	Sid          string          `json:",omitempty"`
	Effect       string          `json:",omitempty"`
	Principal    *IAMPrincipal   `json:",omitempty"`
	NotPrincipal *IAMPrincipal   `json:",omitempty"`
	Action       *StringListExpr `json:",omitempty"`
	NotAction    *StringListExpr `json:",omitempty"`
	Resource     *StringListExpr `json:",omitempty"`
	Condition    interface{}     `json:",omitempty"`
}

// Avoid infinite loops when we just want to marshal the struct normally
type iamPrincipalCopy IAMPrincipal

// MarshalJSON returns a JSON representation of the object. This has been added
// to handle the special case of "*" as the Principal value.
func (i IAMPrincipal) MarshalJSON() ([]byte, error) {
	// Special case for "*"
	if i.AWS != nil && len(i.AWS.Literal) == 1 && i.AWS.Literal[0].Literal == "*" {
		return json.Marshal(i.AWS.Literal[0].Literal)
	}

	c := iamPrincipalCopy(i)

	return json.Marshal(c)
}

// UnmarshalJSON sets the object from the provided JSON representation. This has
// been added to handle the special case of "*" as the Principal value.
func (i *IAMPrincipal) UnmarshalJSON(data []byte) error {
	// Handle single string values like "*"
	var v string
	err := json.Unmarshal(data, &v)
	if err == nil {
		i.AWS = StringList(String(v))
		i.CanonicalUser = nil
		i.Federated = nil
		i.Service = nil
		return nil
	}

	// Handle all other values
	var v2 iamPrincipalCopy
	err = json.Unmarshal(data, &v2)
	if err != nil {
		return err
	}

	i.AWS = v2.AWS
	i.CanonicalUser = v2.CanonicalUser
	i.Federated = v2.Federated
	i.Service = v2.Service

	return nil
}
<|endoftext|>


<commit_before>package idea

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

// Used to manage idea storage in a directory
type DirectoryStore struct {
	root string
}

// Returned if a directory structure doesn't match
// the required format of an idea storage directory
type InvalidDirectoryStoreError struct {
	Err error
}

func (e InvalidDirectoryStoreError) Error() string {
	return fmt.Sprintf("invalid directory store: %v", e.Err)
}

func IsInvalidDirectoryStoreError(err error) bool {
	_, ok := err.(InvalidDirectoryStoreError)
	return ok
}

func isAnDirectoryStore(d string) error {
	nextIdPath := filepath.Join(d, "nextid")

	data, err := ioutil.ReadFile(nextIdPath)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	var nextAvailableId uint
	n, err := fmt.Fscanf(bytes.NewReader(data), "%d\n", &nextAvailableId)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	if n != 1 {
		return InvalidDirectoryStoreError{errors.New("next available id wasn't found")}
	}

	return nil
}

// Checks that the directory contains the correct files
// to be a DirectoryStore.
// If the directory doesn't contain the require files
// with the expected format this function will return an InvalidDirectoryStoreError.
func NewDirectoryStore(directory string) (*DirectoryStore, error) {
	err := isAnDirectoryStore(directory)
	if err != nil {
		return nil, err
	}

	return &DirectoryStore{directory}, nil
}

// Returned if InitDirectoryStore is called on a directory
// that has already been initialized
var ErrInitOnExistingDirectoryStore = errors.New("init on existing directory store")

// Check that the directory is empty
// and if it is then it initializes an empty
// idea directory store.
func InitDirectoryStore(directory string) (*DirectoryStore, git.Commitable, error) {
	err := isAnDirectoryStore(directory)
	if err == nil {
		return nil, nil, ErrInitOnExistingDirectoryStore
	}

	nextIdCounter := filepath.Join(directory, "nextid")
	err = ioutil.WriteFile(nextIdCounter, []byte("1\n"), 0600)
	if err != nil {
		return nil, nil, err
	}

	activeIndex := filepath.Join(directory, "active")
	err = ioutil.WriteFile(activeIndex, []byte(""), 0600)
	if err != nil {
		return nil, nil, err
	}

	changes := git.NewChangesIn(directory)
	changes.Add(git.ChangedFile("nextid"))
	changes.Add(git.ChangedFile("active"))
	changes.Msg = "directory store initialized"

	return &DirectoryStore{directory}, changes, nil
}

// Saves an idea to the directory store and
// returns a commitable containing all changes.
// If the idea does not have an id it will be assigned one.
// If the idea does have an id it will be updated.
func (d DirectoryStore) SaveIdea(idea *Idea) (git.Commitable, error) {
	return nil, nil
}

var ErrIdeaExists = errors.New("cannot save a new idea because it already exists")

// Saves an idea that doesn't have an id to the directory and
// returns a commitable containing all changes.
// If the idea is already assigned an id this method will
// return ErrIdeaExists
func (d DirectoryStore) SaveNewIdea(idea *Idea) (git.Commitable, error) {
	return d.saveNewIdea(idea)
}

// Does not check if the idea has an id
func (d DirectoryStore) saveNewIdea(idea *Idea) (git.Commitable, error) {
	changes := git.NewChangesIn(d.root)

	// Retrieve nextid
	data, err := ioutil.ReadFile(filepath.Join(d.root, "nextid"))
	if err != nil {
		return nil, err
	}

	var nextId uint
	_, err = fmt.Fscan(bytes.NewReader(data), &nextId)
	if err != nil {
		return nil, err
	}

	idea.Id = nextId

	// Increment nextid
	nextId++

	err = ioutil.WriteFile(filepath.Join(d.root, "nextid"), []byte(fmt.Sprintf("%d\n", nextId)), 0600)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile("nextid"))

	// write to file
	r, err := NewIdeaReader(*idea)
	if err != nil {
		return nil, err
	}

	ideaFile, err := os.OpenFile(filepath.Join(d.root, fmt.Sprint(idea.Id)), os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer ideaFile.Close()

	_, err = io.Copy(ideaFile, r)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile(filepath.Base(ideaFile.Name())))

	// If Active, append to active index
	if idea.Status == IS_Active {
		activeIndexFile, err := os.OpenFile(filepath.Join(d.root, "active"), os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			return nil, err
		}
		defer activeIndexFile.Close()

		_, err = fmt.Fprintln(activeIndexFile, idea.Id)
		if err != nil {
			return nil, err
		}
		changes.Add(git.ChangedFile("active"))
	}

	changes.Msg = fmt.Sprintf("IDEA - %d - Created", idea.Id)

	return changes, nil
}

var ErrIdeaNotModified = errors.New("the idea was not modified")

// Updates an idea that has already been assigned an id and
// exists in the directory already and
// returns a commitable containing all changes.
// If the idea body wasn't modified this method will
// return ErrIdeaNotModified
func (d DirectoryStore) UpdateIdea(idea Idea) (git.Commitable, error) {
	return nil, nil
}
<commit_msg>Implemented some unspecified simple behavior<commit_after>package idea

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

// Used to manage idea storage in a directory
type DirectoryStore struct {
	root string
}

// Returned if a directory structure doesn't match
// the required format of an idea storage directory
type InvalidDirectoryStoreError struct {
	Err error
}

func (e InvalidDirectoryStoreError) Error() string {
	return fmt.Sprintf("invalid directory store: %v", e.Err)
}

func IsInvalidDirectoryStoreError(err error) bool {
	_, ok := err.(InvalidDirectoryStoreError)
	return ok
}

func isAnDirectoryStore(d string) error {
	nextIdPath := filepath.Join(d, "nextid")

	data, err := ioutil.ReadFile(nextIdPath)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	var nextAvailableId uint
	n, err := fmt.Fscanf(bytes.NewReader(data), "%d\n", &nextAvailableId)
	if err != nil {
		return InvalidDirectoryStoreError{err}
	}

	if n != 1 {
		return InvalidDirectoryStoreError{errors.New("next available id wasn't found")}
	}

	return nil
}

// Checks that the directory contains the correct files
// to be a DirectoryStore.
// If the directory doesn't contain the require files
// with the expected format this function will return an InvalidDirectoryStoreError.
func NewDirectoryStore(directory string) (*DirectoryStore, error) {
	err := isAnDirectoryStore(directory)
	if err != nil {
		return nil, err
	}

	return &DirectoryStore{directory}, nil
}

// Returned if InitDirectoryStore is called on a directory
// that has already been initialized
var ErrInitOnExistingDirectoryStore = errors.New("init on existing directory store")

// Check that the directory is empty
// and if it is then it initializes an empty
// idea directory store.
func InitDirectoryStore(directory string) (*DirectoryStore, git.Commitable, error) {
	err := isAnDirectoryStore(directory)
	if err == nil {
		return nil, nil, ErrInitOnExistingDirectoryStore
	}

	nextIdCounter := filepath.Join(directory, "nextid")
	err = ioutil.WriteFile(nextIdCounter, []byte("1\n"), 0600)
	if err != nil {
		return nil, nil, err
	}

	activeIndex := filepath.Join(directory, "active")
	err = ioutil.WriteFile(activeIndex, []byte(""), 0600)
	if err != nil {
		return nil, nil, err
	}

	changes := git.NewChangesIn(directory)
	changes.Add(git.ChangedFile("nextid"))
	changes.Add(git.ChangedFile("active"))
	changes.Msg = "directory store initialized"

	return &DirectoryStore{directory}, changes, nil
}

// Saves an idea to the directory store and
// returns a commitable containing all changes.
// If the idea does not have an id it will be assigned one.
// If the idea does have an id it will be updated.
func (d DirectoryStore) SaveIdea(idea *Idea) (git.Commitable, error) {
	if idea.Id == 0 {
		return d.saveNewIdea(idea)
	}

	return d.UpdateIdea(*idea)
}

var ErrIdeaExists = errors.New("cannot save a new idea because it already exists")

// Saves an idea that doesn't have an id to the directory and
// returns a commitable containing all changes.
// If the idea is already assigned an id this method will
// return ErrIdeaExists
func (d DirectoryStore) SaveNewIdea(idea *Idea) (git.Commitable, error) {
	if idea.Id != 0 {
		return nil, ErrIdeaExists
	}
	return d.saveNewIdea(idea)
}

// Does not check if the idea has an id
func (d DirectoryStore) saveNewIdea(idea *Idea) (git.Commitable, error) {
	changes := git.NewChangesIn(d.root)

	// Retrieve nextid
	data, err := ioutil.ReadFile(filepath.Join(d.root, "nextid"))
	if err != nil {
		return nil, err
	}

	var nextId uint
	_, err = fmt.Fscan(bytes.NewReader(data), &nextId)
	if err != nil {
		return nil, err
	}

	idea.Id = nextId

	// Increment nextid
	nextId++

	err = ioutil.WriteFile(filepath.Join(d.root, "nextid"), []byte(fmt.Sprintf("%d\n", nextId)), 0600)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile("nextid"))

	// write to file
	r, err := NewIdeaReader(*idea)
	if err != nil {
		return nil, err
	}

	ideaFile, err := os.OpenFile(filepath.Join(d.root, fmt.Sprint(idea.Id)), os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return nil, err
	}
	defer ideaFile.Close()

	_, err = io.Copy(ideaFile, r)
	if err != nil {
		return nil, err
	}
	changes.Add(git.ChangedFile(filepath.Base(ideaFile.Name())))

	// If Active, append to active index
	if idea.Status == IS_Active {
		activeIndexFile, err := os.OpenFile(filepath.Join(d.root, "active"), os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			return nil, err
		}
		defer activeIndexFile.Close()

		_, err = fmt.Fprintln(activeIndexFile, idea.Id)
		if err != nil {
			return nil, err
		}
		changes.Add(git.ChangedFile("active"))
	}

	changes.Msg = fmt.Sprintf("IDEA - %d - Created", idea.Id)

	return changes, nil
}

var ErrIdeaNotModified = errors.New("the idea was not modified")

// Updates an idea that has already been assigned an id and
// exists in the directory already and
// returns a commitable containing all changes.
// If the idea body wasn't modified this method will
// return ErrIdeaNotModified
func (d DirectoryStore) UpdateIdea(idea Idea) (git.Commitable, error) {
	return nil, nil
}
<|endoftext|>