<commit_before>package i2c

const (
	ADDR          = 0x1E
	REGISTER_XOUT = 0x06
	REGISTER_YOUT = 0x07
	REGISTER_ZOUT = 0x08
)

type HMC5883L struct {
	bus *I2CBus
}

func New() (dev *HMC5883L, err error) {
	dev = new(HMC5883L)
	dev.bus, err = Bus(1)

	err = dev.Write(0, 0x10)
	err = dev.Write(1, 0x20)
	err = dev.Write(2, 0x00)

	return
}

func (bp *HMC5883L) Read(reg byte) (int8) {
	var bytes []byte
	bytes, _ = bp.bus.ReadByteBlock(ADDR, reg, 1)
	return int8(bytes[0])
}

func (bp *HMC5883L) Write(reg byte, value int8) (err error) {
	err = bp.bus.WriteByte(0x1e, 2, 1)
	if err != nil {
		return
	}
	return
}<commit_msg>DISPLAY: other magnetometer integration<commit_after>package i2c

const (
	ADDR          = 0x1E
	REGISTER_XOUT = 0x06
	REGISTER_YOUT = 0x07
	REGISTER_ZOUT = 0x08
)

type HMC5883L struct {
	bus *I2CBus
}

func New() (dev *HMC5883L, err error) {
	dev = new(HMC5883L)
	dev.bus, err = Bus(1)
	err = dev.Write(0, 0x10)
	err = dev.Write(1, 0x20)
	err = dev.Write(2, 0x00)
	return
}

func (dev *HMC5883L) Read(reg byte) (int8) {
	var bytes []byte
	bytes, _ = dev.bus.ReadByteBlock(ADDR, reg, 1)
	return int8(bytes[0])
}

func (dev *HMC5883L) Write(reg byte, value int8) (err error) {
	err = dev.bus.WriteByte(0x1e, byte, value)
	if err != nil {
		return
	}
	return
}<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	commontypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/controller/volume/expand/util"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume/util/types"
)

// VolumeResizeMap defines an interface that serves as a cache for holding pending resizing requests
type VolumeResizeMap interface {
	// AddPVCUpdate adds pvc for resizing
	AddPVCUpdate(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume)
	// DeletePVC deletes pvc that is scheduled for resizing
	DeletePVC(pvc *v1.PersistentVolumeClaim)
	// GetPVCsWithResizeRequest returns all pending pvc resize requests
	GetPVCsWithResizeRequest() []*PVCWithResizeRequest
	// MarkAsResized marks a pvc as fully resized
	MarkAsResized(*PVCWithResizeRequest, resource.Quantity) error
	// UpdatePVSize updates just pv size after cloudprovider resizing is successful
	UpdatePVSize(*PVCWithResizeRequest, resource.Quantity) error
}

type volumeResizeMap struct {
	// map of unique pvc name and resize requests that are pending or inflight
	pvcrs map[types.UniquePVCName]*PVCWithResizeRequest
	// kube client for making API calls
	kubeClient clientset.Interface
	// for guarding access to pvcrs map
	sync.RWMutex
}

// PVCWithResizeRequest struct defines data structure that stores state needed for
// performing file system resize
type PVCWithResizeRequest struct {
	// PVC that needs to be resized
	PVC *v1.PersistentVolumeClaim
	// persistentvolume
	PersistentVolume *v1.PersistentVolume
	// Current volume size
	CurrentSize resource.Quantity
	// Expended volume size
	ExpectedSize resource.Quantity
}

// UniquePVCKey returns unique key of the PVC based on its UID
func (pvcr *PVCWithResizeRequest) UniquePVCKey() types.UniquePVCName {
	return types.UniquePVCName(pvcr.PVC.UID)
}

// QualifiedName returns namespace and name combination of the PVC
func (pvcr *PVCWithResizeRequest) QualifiedName() string {
	return strings.JoinQualifiedName(pvcr.PVC.Namespace, pvcr.PVC.Name)
}

// NewVolumeResizeMap returns new VolumeResizeMap which acts as a cache
// for holding pending resize requests.
func NewVolumeResizeMap(kubeClient clientset.Interface) VolumeResizeMap {
	resizeMap := &volumeResizeMap{}
	resizeMap.pvcrs = make(map[types.UniquePVCName]*PVCWithResizeRequest)
	resizeMap.kubeClient = kubeClient
	return resizeMap
}

// AddPVCUpdate adds pvc for resizing
// This function intentionally allows addition of PVCs for which pv.Spec.Size >= pvc.Spec.Size,
// the reason being - lack of transaction in k8s means after successful resize, we can't guarantee that when we update PV,
// pvc update will be successful too and after resize we alyways update PV first.
// If for some reason we weren't able to update PVC after successful resize, then we are going to reprocess
// the PVC and hopefully after a no-op resize in volume plugin, PVC will be updated with right values as well.
func (resizeMap *volumeResizeMap) AddPVCUpdate(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
	if pv.Spec.ClaimRef == nil || pvc.Namespace != pv.Spec.ClaimRef.Namespace || pvc.Name != pv.Spec.ClaimRef.Name {
		glog.V(4).Infof("Persistent Volume is not bound to PVC being updated : %s", util.ClaimToClaimKey(pvc))
		return
	}

	if pvc.Status.Phase != v1.ClaimBound {
		return
	}

	resizeMap.Lock()
	defer resizeMap.Unlock()

	pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
	pvcStatusSize := pvc.Status.Capacity[v1.ResourceStorage]

	if pvcStatusSize.Cmp(pvcSize) >= 0 {
		return
	}

	glog.V(4).Infof("Adding pvc %s with Size %s/%s for resizing", util.ClaimToClaimKey(pvc), pvcSize.String(), pvcStatusSize.String())

	pvcRequest := &PVCWithResizeRequest{
		PVC:              pvc,
		CurrentSize:      pvcStatusSize,
		ExpectedSize:     pvcSize,
		PersistentVolume: pv,
	}
	resizeMap.pvcrs[types.UniquePVCName(pvc.UID)] = pvcRequest
}

// GetPVCsWithResizeRequest returns all pending pvc resize requests
func (resizeMap *volumeResizeMap) GetPVCsWithResizeRequest() []*PVCWithResizeRequest {
	resizeMap.Lock()
	defer resizeMap.Unlock()

	pvcrs := []*PVCWithResizeRequest{}
	for _, pvcr := range resizeMap.pvcrs {
		pvcrs = append(pvcrs, pvcr)
	}
	// Empty out pvcrs map, we will add back failed resize requests later
	resizeMap.pvcrs = map[types.UniquePVCName]*PVCWithResizeRequest{}
	return pvcrs
}

// DeletePVC removes given pvc object from list of pvcs that needs resizing.
// deleting a pvc in this map doesn't affect operations that are already inflight.
func (resizeMap *volumeResizeMap) DeletePVC(pvc *v1.PersistentVolumeClaim) {
	resizeMap.Lock()
	defer resizeMap.Unlock()
	pvcUniqueName := types.UniquePVCName(pvc.UID)
	glog.V(5).Infof("Removing PVC %v from resize map", pvcUniqueName)
	delete(resizeMap.pvcrs, pvcUniqueName)
}

// MarkAsResized marks a pvc as fully resized
func (resizeMap *volumeResizeMap) MarkAsResized(pvcr *PVCWithResizeRequest, newSize resource.Quantity) error {
	resizeMap.Lock()
	defer resizeMap.Unlock()

	emptyCondition := []v1.PersistentVolumeClaimCondition{}

	err := resizeMap.updatePVCCapacityAndConditions(pvcr, newSize, emptyCondition)
	if err != nil {
		glog.V(4).Infof("Error updating PV spec capacity for volume %q with : %v", pvcr.QualifiedName(), err)
		return err
	}
	return nil
}

// UpdatePVSize updates just pv size after cloudprovider resizing is successful
func (resizeMap *volumeResizeMap) UpdatePVSize(pvcr *PVCWithResizeRequest, newSize resource.Quantity) error {
	resizeMap.Lock()
	defer resizeMap.Unlock()

	oldPv := pvcr.PersistentVolume
	pvClone := oldPv.DeepCopy()

	oldData, err := json.Marshal(pvClone)

	if err != nil {
		return fmt.Errorf("Unexpected error marshaling PV : %q with error %v", pvClone.Name, err)
	}

	pvClone.Spec.Capacity[v1.ResourceStorage] = newSize

	newData, err := json.Marshal(pvClone)

	if err != nil {
		return fmt.Errorf("Unexpected error marshaling PV : %q with error %v", pvClone.Name, err)
	}

	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, pvClone)

	if err != nil {
		return fmt.Errorf("Error Creating two way merge patch for  PV : %q with error %v", pvClone.Name, err)
	}

	_, updateErr := resizeMap.kubeClient.CoreV1().PersistentVolumes().Patch(pvClone.Name, commontypes.StrategicMergePatchType, patchBytes)

	if updateErr != nil {
		glog.V(4).Infof("Error updating pv %q with error : %v", pvClone.Name, updateErr)
		return updateErr
	}
	return nil
}

func (resizeMap *volumeResizeMap) updatePVCCapacityAndConditions(pvcr *PVCWithResizeRequest, newSize resource.Quantity, pvcConditions []v1.PersistentVolumeClaimCondition) error {

	claimClone := pvcr.PVC.DeepCopy()

	claimClone.Status.Capacity[v1.ResourceStorage] = newSize
	claimClone.Status.Conditions = pvcConditions

	_, updateErr := resizeMap.kubeClient.CoreV1().PersistentVolumeClaims(claimClone.Namespace).UpdateStatus(claimClone)
	if updateErr != nil {
		glog.V(4).Infof("updating PersistentVolumeClaim[%s] status: failed: %v", pvcr.QualifiedName(), updateErr)
		return updateErr
	}
	return nil
}
<commit_msg>optimize volumeResizeMap lock<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	commontypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/controller/volume/expand/util"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume/util/types"
)

// VolumeResizeMap defines an interface that serves as a cache for holding pending resizing requests
type VolumeResizeMap interface {
	// AddPVCUpdate adds pvc for resizing
	AddPVCUpdate(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume)
	// DeletePVC deletes pvc that is scheduled for resizing
	DeletePVC(pvc *v1.PersistentVolumeClaim)
	// GetPVCsWithResizeRequest returns all pending pvc resize requests
	GetPVCsWithResizeRequest() []*PVCWithResizeRequest
	// MarkAsResized marks a pvc as fully resized
	MarkAsResized(*PVCWithResizeRequest, resource.Quantity) error
	// UpdatePVSize updates just pv size after cloudprovider resizing is successful
	UpdatePVSize(*PVCWithResizeRequest, resource.Quantity) error
}

type volumeResizeMap struct {
	// map of unique pvc name and resize requests that are pending or inflight
	pvcrs map[types.UniquePVCName]*PVCWithResizeRequest
	// kube client for making API calls
	kubeClient clientset.Interface
	// for guarding access to pvcrs map
	sync.Mutex
}

// PVCWithResizeRequest struct defines data structure that stores state needed for
// performing file system resize
type PVCWithResizeRequest struct {
	// PVC that needs to be resized
	PVC *v1.PersistentVolumeClaim
	// persistentvolume
	PersistentVolume *v1.PersistentVolume
	// Current volume size
	CurrentSize resource.Quantity
	// Expended volume size
	ExpectedSize resource.Quantity
}

// UniquePVCKey returns unique key of the PVC based on its UID
func (pvcr *PVCWithResizeRequest) UniquePVCKey() types.UniquePVCName {
	return types.UniquePVCName(pvcr.PVC.UID)
}

// QualifiedName returns namespace and name combination of the PVC
func (pvcr *PVCWithResizeRequest) QualifiedName() string {
	return strings.JoinQualifiedName(pvcr.PVC.Namespace, pvcr.PVC.Name)
}

// NewVolumeResizeMap returns new VolumeResizeMap which acts as a cache
// for holding pending resize requests.
func NewVolumeResizeMap(kubeClient clientset.Interface) VolumeResizeMap {
	resizeMap := &volumeResizeMap{}
	resizeMap.pvcrs = make(map[types.UniquePVCName]*PVCWithResizeRequest)
	resizeMap.kubeClient = kubeClient
	return resizeMap
}

// AddPVCUpdate adds pvc for resizing
// This function intentionally allows addition of PVCs for which pv.Spec.Size >= pvc.Spec.Size,
// the reason being - lack of transaction in k8s means after successful resize, we can't guarantee that when we update PV,
// pvc update will be successful too and after resize we alyways update PV first.
// If for some reason we weren't able to update PVC after successful resize, then we are going to reprocess
// the PVC and hopefully after a no-op resize in volume plugin, PVC will be updated with right values as well.
func (resizeMap *volumeResizeMap) AddPVCUpdate(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
	if pv.Spec.ClaimRef == nil || pvc.Namespace != pv.Spec.ClaimRef.Namespace || pvc.Name != pv.Spec.ClaimRef.Name {
		glog.V(4).Infof("Persistent Volume is not bound to PVC being updated : %s", util.ClaimToClaimKey(pvc))
		return
	}

	if pvc.Status.Phase != v1.ClaimBound {
		return
	}

	pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
	pvcStatusSize := pvc.Status.Capacity[v1.ResourceStorage]

	if pvcStatusSize.Cmp(pvcSize) >= 0 {
		return
	}

	glog.V(4).Infof("Adding pvc %s with Size %s/%s for resizing", util.ClaimToClaimKey(pvc), pvcSize.String(), pvcStatusSize.String())

	pvcRequest := &PVCWithResizeRequest{
		PVC:              pvc,
		CurrentSize:      pvcStatusSize,
		ExpectedSize:     pvcSize,
		PersistentVolume: pv,
	}

	resizeMap.Lock()
	defer resizeMap.Unlock()
	resizeMap.pvcrs[types.UniquePVCName(pvc.UID)] = pvcRequest
}

// GetPVCsWithResizeRequest returns all pending pvc resize requests
func (resizeMap *volumeResizeMap) GetPVCsWithResizeRequest() []*PVCWithResizeRequest {
	resizeMap.Lock()
	defer resizeMap.Unlock()

	pvcrs := []*PVCWithResizeRequest{}
	for _, pvcr := range resizeMap.pvcrs {
		pvcrs = append(pvcrs, pvcr)
	}
	// Empty out pvcrs map, we will add back failed resize requests later
	resizeMap.pvcrs = map[types.UniquePVCName]*PVCWithResizeRequest{}
	return pvcrs
}

// DeletePVC removes given pvc object from list of pvcs that needs resizing.
// deleting a pvc in this map doesn't affect operations that are already inflight.
func (resizeMap *volumeResizeMap) DeletePVC(pvc *v1.PersistentVolumeClaim) {
	pvcUniqueName := types.UniquePVCName(pvc.UID)
	glog.V(5).Infof("Removing PVC %v from resize map", pvcUniqueName)
	resizeMap.Lock()
	defer resizeMap.Unlock()
	delete(resizeMap.pvcrs, pvcUniqueName)
}

// MarkAsResized marks a pvc as fully resized
func (resizeMap *volumeResizeMap) MarkAsResized(pvcr *PVCWithResizeRequest, newSize resource.Quantity) error {
	emptyCondition := []v1.PersistentVolumeClaimCondition{}

	err := resizeMap.updatePVCCapacityAndConditions(pvcr, newSize, emptyCondition)
	if err != nil {
		glog.V(4).Infof("Error updating PV spec capacity for volume %q with : %v", pvcr.QualifiedName(), err)
		return err
	}
	return nil
}

// UpdatePVSize updates just pv size after cloudprovider resizing is successful
func (resizeMap *volumeResizeMap) UpdatePVSize(pvcr *PVCWithResizeRequest, newSize resource.Quantity) error {
	oldPv := pvcr.PersistentVolume
	pvClone := oldPv.DeepCopy()

	oldData, err := json.Marshal(pvClone)

	if err != nil {
		return fmt.Errorf("Unexpected error marshaling PV : %q with error %v", pvClone.Name, err)
	}

	pvClone.Spec.Capacity[v1.ResourceStorage] = newSize

	newData, err := json.Marshal(pvClone)

	if err != nil {
		return fmt.Errorf("Unexpected error marshaling PV : %q with error %v", pvClone.Name, err)
	}

	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, pvClone)

	if err != nil {
		return fmt.Errorf("Error Creating two way merge patch for  PV : %q with error %v", pvClone.Name, err)
	}

	_, updateErr := resizeMap.kubeClient.CoreV1().PersistentVolumes().Patch(pvClone.Name, commontypes.StrategicMergePatchType, patchBytes)

	if updateErr != nil {
		glog.V(4).Infof("Error updating pv %q with error : %v", pvClone.Name, updateErr)
		return updateErr
	}
	return nil
}

func (resizeMap *volumeResizeMap) updatePVCCapacityAndConditions(pvcr *PVCWithResizeRequest, newSize resource.Quantity, pvcConditions []v1.PersistentVolumeClaimCondition) error {
	claimClone := pvcr.PVC.DeepCopy()

	claimClone.Status.Capacity[v1.ResourceStorage] = newSize
	claimClone.Status.Conditions = pvcConditions

	_, updateErr := resizeMap.kubeClient.CoreV1().PersistentVolumeClaims(claimClone.Namespace).UpdateStatus(claimClone)
	if updateErr != nil {
		glog.V(4).Infof("updating PersistentVolumeClaim[%s] status: failed: %v", pvcr.QualifiedName(), updateErr)
		return updateErr
	}
	return nil
}
<|endoftext|>


<commit_before>package disk

import (
	"container/list"
)

type sizedItem interface {
	Size() int64
}

type Key interface{}

// EvictCallback is the type of callbacks that are invoked when items are evicted.
type EvictCallback func(key Key, value sizedItem)

// SizedLRU is an LRU cache that will keep its total size below maxSize by evicting
// items.
// SizedLRU is not thread-safe.
type SizedLRU interface {
	Add(key Key, value sizedItem) (ok bool)
	Get(key Key) (value sizedItem, ok bool)
	Remove(key Key)
	Len() int
	CurrentSize() int64
	MaxSize() int64
}

type sizedLRU struct {
	// Eviction double-linked list. Most recently accessed elements are at the front.
	ll *list.List
	// Map to access the items in O(1) time
	cache       map[interface{}]*list.Element
	currentSize int64
	// SizedLRU will evict items as needed to maintain the total size of the cache
	// below maxSize.
	maxSize int64
	onEvict EvictCallback
}

type entry struct {
	key   Key
	value sizedItem
}

// NewSizedLRU returns a new sizedLRU cache
func NewSizedLRU(maxSize int64, onEvict EvictCallback) SizedLRU {
	return &sizedLRU{
		maxSize: maxSize,
		ll:      list.New(),
		cache:   make(map[interface{}]*list.Element),
		onEvict: onEvict,
	}
}

// Add adds a (key, value) to the cache, evicting items as necessary. Add returns false (
// and does not add the item) if the item size is larger than the maximum size of the cache.
func (c *sizedLRU) Add(key Key, value sizedItem) (ok bool) {
	if value.Size() > c.maxSize {
		return false
	}

	var sizeDelta int64
	if ee, ok := c.cache[key]; ok {
		c.ll.MoveToFront(ee)
		sizeDelta = value.Size() - ee.Value.(*entry).value.Size()
		ee.Value.(*entry).value = value
	} else {
		ele := c.ll.PushFront(&entry{key, value})
		c.cache[key] = ele
		sizeDelta = value.Size()
	}

	// Eviction. This is needed even if the key was already present, since the size of the
	// value might have changed, pushing the total size over maxSize.
	for c.currentSize+sizeDelta > c.maxSize {
		ele := c.ll.Back()
		if ele != nil {
			c.removeElement(ele)
		}
	}

	c.currentSize += sizeDelta

	return true
}

// Get looks up a key in the cache
func (c *sizedLRU) Get(key Key) (value sizedItem, ok bool) {
	if ele, hit := c.cache[key]; hit {
		c.ll.MoveToFront(ele)
		return ele.Value.(*entry).value, true
	}

	return
}

// Remove removes a (key, value) from the cache
func (c *sizedLRU) Remove(key Key) {
	if ele, hit := c.cache[key]; hit {
		c.removeElement(ele)
	}
}

// Len returns the number of items in the cache
func (c *sizedLRU) Len() int {
	return len(c.cache)
}

func (c *sizedLRU) CurrentSize() int64 {
	return c.currentSize
}

func (c *sizedLRU) MaxSize() int64 {
	return c.maxSize
}

func (c *sizedLRU) removeElement(e *list.Element) {
	c.ll.Remove(e)
	kv := e.Value.(*entry)
	delete(c.cache, kv.key)
	c.currentSize -= kv.value.Size()

	if c.onEvict != nil {
		c.onEvict(kv.key, kv.value)
	}
}
<commit_msg>cache/disk/lru: add godoc style comment for Key<commit_after>package disk

import (
	"container/list"
)

type sizedItem interface {
	Size() int64
}

// Key is the type used for identifying cache items. For non-test code,
// this is a string of the form "<keyspace>/<hash>". Some test code uses
// ints for simplicity.
//
// TODO: update the test code to use strings too, then drop all the
// type assertions.
type Key interface{}

// EvictCallback is the type of callbacks that are invoked when items are evicted.
type EvictCallback func(key Key, value sizedItem)

// SizedLRU is an LRU cache that will keep its total size below maxSize by evicting
// items.
// SizedLRU is not thread-safe.
type SizedLRU interface {
	Add(key Key, value sizedItem) (ok bool)
	Get(key Key) (value sizedItem, ok bool)
	Remove(key Key)
	Len() int
	CurrentSize() int64
	MaxSize() int64
}

type sizedLRU struct {
	// Eviction double-linked list. Most recently accessed elements are at the front.
	ll *list.List
	// Map to access the items in O(1) time
	cache       map[interface{}]*list.Element
	currentSize int64
	// SizedLRU will evict items as needed to maintain the total size of the cache
	// below maxSize.
	maxSize int64
	onEvict EvictCallback
}

type entry struct {
	key   Key
	value sizedItem
}

// NewSizedLRU returns a new sizedLRU cache
func NewSizedLRU(maxSize int64, onEvict EvictCallback) SizedLRU {
	return &sizedLRU{
		maxSize: maxSize,
		ll:      list.New(),
		cache:   make(map[interface{}]*list.Element),
		onEvict: onEvict,
	}
}

// Add adds a (key, value) to the cache, evicting items as necessary. Add returns false (
// and does not add the item) if the item size is larger than the maximum size of the cache.
func (c *sizedLRU) Add(key Key, value sizedItem) (ok bool) {
	if value.Size() > c.maxSize {
		return false
	}

	var sizeDelta int64
	if ee, ok := c.cache[key]; ok {
		c.ll.MoveToFront(ee)
		sizeDelta = value.Size() - ee.Value.(*entry).value.Size()
		ee.Value.(*entry).value = value
	} else {
		ele := c.ll.PushFront(&entry{key, value})
		c.cache[key] = ele
		sizeDelta = value.Size()
	}

	// Eviction. This is needed even if the key was already present, since the size of the
	// value might have changed, pushing the total size over maxSize.
	for c.currentSize+sizeDelta > c.maxSize {
		ele := c.ll.Back()
		if ele != nil {
			c.removeElement(ele)
		}
	}

	c.currentSize += sizeDelta

	return true
}

// Get looks up a key in the cache
func (c *sizedLRU) Get(key Key) (value sizedItem, ok bool) {
	if ele, hit := c.cache[key]; hit {
		c.ll.MoveToFront(ele)
		return ele.Value.(*entry).value, true
	}

	return
}

// Remove removes a (key, value) from the cache
func (c *sizedLRU) Remove(key Key) {
	if ele, hit := c.cache[key]; hit {
		c.removeElement(ele)
	}
}

// Len returns the number of items in the cache
func (c *sizedLRU) Len() int {
	return len(c.cache)
}

func (c *sizedLRU) CurrentSize() int64 {
	return c.currentSize
}

func (c *sizedLRU) MaxSize() int64 {
	return c.maxSize
}

func (c *sizedLRU) removeElement(e *list.Element) {
	c.ll.Remove(e)
	kv := e.Value.(*entry)
	delete(c.cache, kv.key)
	c.currentSize -= kv.value.Size()

	if c.onEvict != nil {
		c.onEvict(kv.key, kv.value)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package context_test

import (
	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/process"
	"github.com/juju/juju/process/context"
	jujuctesting "github.com/juju/juju/worker/uniter/runner/jujuc/testing"
)

type baseSuite struct {
	jujuctesting.ContextSuite
	proc    *process.Info
	compCtx *context.Context
	//compCtx *jujuctesting.ContextComponent
}

func (s *baseSuite) SetUpTest(c *gc.C) {
	s.ContextSuite.SetUpTest(c)

	proc := process.NewInfo("proc A", "docker")
	compCtx := context.NewContext(proc)
	//compCtx := &jujuctesting.ContextComponent{Stub: s.Stub}

	s.Ctx = s.HookContext("u/0", nil)
	s.Ctx.SetComponent(process.ComponentName, compCtx)

	s.proc = proc
	s.compCtx = compCtx
	//s.compCtx = compCtx
}

type contextSuite struct {
	baseSuite
}

var _ = gc.Suite(&contextSuite{})

func (s *contextSuite) TestNewContextEmpty(c *gc.C) {
	ctx := context.NewContext()
	procs := ctx.Processes()

	c.Check(procs, gc.HasLen, 0)
}

func (s *contextSuite) TestNewContextPrePopulated(c *gc.C) {
	expected := []*process.Info{{}, {}}
	expected[0].Name = "A"
	expected[1].Name = "B"

	ctx := context.NewContext(expected...)
	procs := ctx.Processes()

	c.Assert(procs, gc.HasLen, 2)
	if procs[0].Name == "A" {
		c.Check(procs[0], jc.DeepEquals, expected[0])
		c.Check(procs[1], jc.DeepEquals, expected[1])
	} else {
		c.Check(procs[0], jc.DeepEquals, expected[1])
		c.Check(procs[1], jc.DeepEquals, expected[0])
	}
}

func (s *contextSuite) TestContextComponentOkay(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	expected := context.NewContext()
	ctx.SetComponent(process.ComponentName, expected)

	compCtx, err := context.ContextComponent(ctx.Context)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(compCtx, gc.Equals, expected)
	s.Stub.CheckCallNames(c, "Component")
	s.Stub.CheckCall(c, 0, "Component", "process")
}

func (s *contextSuite) TestContextComponentMissing(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	_, err := context.ContextComponent(ctx.Context)

	c.Check(err, gc.ErrorMatches, `component "process" not registered`)
	s.Stub.CheckCallNames(c, "Component")
}

func (s *contextSuite) TestContextComponentWrong(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	compCtx := &jujuctesting.ContextComponent{}
	ctx.SetComponent(process.ComponentName, compCtx)

	_, err := context.ContextComponent(ctx.Context)

	c.Check(err, gc.ErrorMatches, "wrong component context type registered: .*")
	s.Stub.CheckCallNames(c, "Component")
}

func (s *contextSuite) TestContextComponentDisabled(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	ctx.SetComponent(process.ComponentName, nil)

	_, err := context.ContextComponent(ctx.Context)

	c.Check(err, gc.ErrorMatches, `component "process" disabled`)
	s.Stub.CheckCallNames(c, "Component")
}

func (s *contextSuite) TestGetOkay(c *gc.C) {
	var info, expected, extra process.Info
	expected.Name = "A"

	ctx := context.NewContext(&expected, &extra)
	err := ctx.Get("A", &info)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(info, jc.DeepEquals, expected)
}

func (s *contextSuite) TestGetWrongType(c *gc.C) {
	ctx := context.NewContext()
	err := ctx.Get("A", nil)

	c.Check(err, gc.ErrorMatches, "invalid type: expected process.Info, got .*")
}

func (s *contextSuite) TestGetNotFound(c *gc.C) {
	var info process.Info
	ctx := context.NewContext()
	err := ctx.Get("A", &info)

	c.Check(err, jc.Satisfies, errors.IsNotFound)
}

func (s *contextSuite) TestSetOkay(c *gc.C) {
	var info process.Info
	info.Name = "A"
	ctx := context.NewContext()
	before := ctx.Processes()
	err := ctx.Set("A", &info)
	c.Assert(err, jc.ErrorIsNil)
	after := ctx.Processes()

	c.Check(before, gc.HasLen, 0)
	c.Check(after, jc.DeepEquals, []*process.Info{&info})
}

func (s *contextSuite) TestSetOverwrite(c *gc.C) {
	var info, other process.Info
	info.Name = "A"
	other.Status = process.StatusFailed
	other.Name = "A"
	other.Status = process.StatusPending
	ctx := context.NewContext(&other)
	before := ctx.Processes()
	err := ctx.Set("A", &info)
	c.Assert(err, jc.ErrorIsNil)
	after := ctx.Processes()

	c.Check(before, jc.DeepEquals, []*process.Info{&other})
	c.Check(after, jc.DeepEquals, []*process.Info{&info})
}

func (s *contextSuite) TestSetWrongType(c *gc.C) {
	ctx := context.NewContext()
	before := ctx.Processes()
	err := ctx.Set("A", nil)
	after := ctx.Processes()

	c.Check(err, gc.ErrorMatches, "invalid type: expected process.Info, got .*")
	c.Check(before, gc.HasLen, 0)
	c.Check(after, gc.HasLen, 0)
}

func (s *contextSuite) TestSetNameMismatch(c *gc.C) {
	var info, other process.Info
	info.Name = "B"
	other.Name = "A"
	ctx := context.NewContext(&other)
	before := ctx.Processes()
	err := ctx.Set("A", &info)
	after := ctx.Processes()

	c.Check(err, gc.ErrorMatches, "mismatch on name: A != B")
	c.Check(before, jc.DeepEquals, []*process.Info{&other})
	c.Check(after, jc.DeepEquals, []*process.Info{&other})
}

func (s *contextSuite) TestFlushDirty(c *gc.C) {
	ctx := context.NewContext()
	ctx.Set("A", nil)

	err := ctx.Flush()
	c.Assert(err, gc.ErrorMatches, "not finished")
}

func (s *contextSuite) TestFlushNotDirty(c *gc.C) {
	var info process.Info
	info.Name = "flush-not-dirty"
	ctx := context.NewContext(&info)

	err := ctx.Flush()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c)
}

func (s *contextSuite) TestFlushEmpty(c *gc.C) {
	ctx := context.NewContext()
	err := ctx.Flush()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c)
}
<commit_msg>Remove dead code.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package context_test

import (
	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/process"
	"github.com/juju/juju/process/context"
	jujuctesting "github.com/juju/juju/worker/uniter/runner/jujuc/testing"
)

type baseSuite struct {
	jujuctesting.ContextSuite
	proc    *process.Info
	compCtx *context.Context
}

func (s *baseSuite) SetUpTest(c *gc.C) {
	s.ContextSuite.SetUpTest(c)

	proc := process.NewInfo("proc A", "docker")
	compCtx := context.NewContext(proc)

	s.Ctx = s.HookContext("u/0", nil)
	s.Ctx.SetComponent(process.ComponentName, compCtx)

	s.proc = proc
	s.compCtx = compCtx
}

type contextSuite struct {
	baseSuite
}

var _ = gc.Suite(&contextSuite{})

func (s *contextSuite) TestNewContextEmpty(c *gc.C) {
	ctx := context.NewContext()
	procs := ctx.Processes()

	c.Check(procs, gc.HasLen, 0)
}

func (s *contextSuite) TestNewContextPrePopulated(c *gc.C) {
	expected := []*process.Info{{}, {}}
	expected[0].Name = "A"
	expected[1].Name = "B"

	ctx := context.NewContext(expected...)
	procs := ctx.Processes()

	c.Assert(procs, gc.HasLen, 2)
	if procs[0].Name == "A" {
		c.Check(procs[0], jc.DeepEquals, expected[0])
		c.Check(procs[1], jc.DeepEquals, expected[1])
	} else {
		c.Check(procs[0], jc.DeepEquals, expected[1])
		c.Check(procs[1], jc.DeepEquals, expected[0])
	}
}

func (s *contextSuite) TestContextComponentOkay(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	expected := context.NewContext()
	ctx.SetComponent(process.ComponentName, expected)

	compCtx, err := context.ContextComponent(ctx.Context)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(compCtx, gc.Equals, expected)
	s.Stub.CheckCallNames(c, "Component")
	s.Stub.CheckCall(c, 0, "Component", "process")
}

func (s *contextSuite) TestContextComponentMissing(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	_, err := context.ContextComponent(ctx.Context)

	c.Check(err, gc.ErrorMatches, `component "process" not registered`)
	s.Stub.CheckCallNames(c, "Component")
}

func (s *contextSuite) TestContextComponentWrong(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	compCtx := &jujuctesting.ContextComponent{}
	ctx.SetComponent(process.ComponentName, compCtx)

	_, err := context.ContextComponent(ctx.Context)

	c.Check(err, gc.ErrorMatches, "wrong component context type registered: .*")
	s.Stub.CheckCallNames(c, "Component")
}

func (s *contextSuite) TestContextComponentDisabled(c *gc.C) {
	ctx := s.HookContext("u/1", nil)
	ctx.SetComponent(process.ComponentName, nil)

	_, err := context.ContextComponent(ctx.Context)

	c.Check(err, gc.ErrorMatches, `component "process" disabled`)
	s.Stub.CheckCallNames(c, "Component")
}

func (s *contextSuite) TestGetOkay(c *gc.C) {
	var info, expected, extra process.Info
	expected.Name = "A"

	ctx := context.NewContext(&expected, &extra)
	err := ctx.Get("A", &info)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(info, jc.DeepEquals, expected)
}

func (s *contextSuite) TestGetWrongType(c *gc.C) {
	ctx := context.NewContext()
	err := ctx.Get("A", nil)

	c.Check(err, gc.ErrorMatches, "invalid type: expected process.Info, got .*")
}

func (s *contextSuite) TestGetNotFound(c *gc.C) {
	var info process.Info
	ctx := context.NewContext()
	err := ctx.Get("A", &info)

	c.Check(err, jc.Satisfies, errors.IsNotFound)
}

func (s *contextSuite) TestSetOkay(c *gc.C) {
	var info process.Info
	info.Name = "A"
	ctx := context.NewContext()
	before := ctx.Processes()
	err := ctx.Set("A", &info)
	c.Assert(err, jc.ErrorIsNil)
	after := ctx.Processes()

	c.Check(before, gc.HasLen, 0)
	c.Check(after, jc.DeepEquals, []*process.Info{&info})
}

func (s *contextSuite) TestSetOverwrite(c *gc.C) {
	var info, other process.Info
	info.Name = "A"
	other.Status = process.StatusFailed
	other.Name = "A"
	other.Status = process.StatusPending
	ctx := context.NewContext(&other)
	before := ctx.Processes()
	err := ctx.Set("A", &info)
	c.Assert(err, jc.ErrorIsNil)
	after := ctx.Processes()

	c.Check(before, jc.DeepEquals, []*process.Info{&other})
	c.Check(after, jc.DeepEquals, []*process.Info{&info})
}

func (s *contextSuite) TestSetWrongType(c *gc.C) {
	ctx := context.NewContext()
	before := ctx.Processes()
	err := ctx.Set("A", nil)
	after := ctx.Processes()

	c.Check(err, gc.ErrorMatches, "invalid type: expected process.Info, got .*")
	c.Check(before, gc.HasLen, 0)
	c.Check(after, gc.HasLen, 0)
}

func (s *contextSuite) TestSetNameMismatch(c *gc.C) {
	var info, other process.Info
	info.Name = "B"
	other.Name = "A"
	ctx := context.NewContext(&other)
	before := ctx.Processes()
	err := ctx.Set("A", &info)
	after := ctx.Processes()

	c.Check(err, gc.ErrorMatches, "mismatch on name: A != B")
	c.Check(before, jc.DeepEquals, []*process.Info{&other})
	c.Check(after, jc.DeepEquals, []*process.Info{&other})
}

func (s *contextSuite) TestFlushDirty(c *gc.C) {
	ctx := context.NewContext()
	ctx.Set("A", nil)

	err := ctx.Flush()
	c.Assert(err, gc.ErrorMatches, "not finished")
}

func (s *contextSuite) TestFlushNotDirty(c *gc.C) {
	var info process.Info
	info.Name = "flush-not-dirty"
	ctx := context.NewContext(&info)

	err := ctx.Flush()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c)
}

func (s *contextSuite) TestFlushEmpty(c *gc.C) {
	ctx := context.NewContext()
	err := ctx.Flush()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c)
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/docker-deploy/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)
var commandSplit = regexp.MustCompile(`["']([^"']+)["']|(\S+)`)

type dockerComposeService struct {
	Image       string
	Command     string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, loggedUser *user, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = loggedUser.username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if service.Command != `` {
		config.Cmd = strslice.StrSlice{}
		for _, args := range commandSplit.FindAllStringSubmatch(service.Command, -1) {
			config.Cmd = append(config.Cmd, args[1]+args[2])
		}
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := &deployedServices[linkParts[0]]; ok {
			target = linkedService.Name
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			`traefik`: &traefikConfig,
		},
	}
}

func pullImage(image string, loggedUser *user) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(loggedUser.username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(loggedUser.username + ` ends pulling for ` + image)
	return nil
}

func cleanContainers(containers *[]types.Container, loggedUser *user) {
	for _, container := range *containers {
		log.Print(loggedUser.username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(loggedUser.username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, strings.TrimSuffix(service.Name, deploySuffix)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(appName string, serviceName string) string {
	return appName + `_` + serviceName + deploySuffix
}

func createAppHandler(w http.ResponseWriter, loggedUser *user, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(loggedUser.username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(loggedUser, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, loggedUser); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Print(loggedUser.username + ` starts ` + serviceFullName)

		id, err := docker.ContainerCreate(context.Background(), getConfig(&service, loggedUser, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			return
		}

		startContainer(id.ID)
		deployedServices[serviceName] = deployedService{ID: id.ID, Name: serviceFullName}
	}

	log.Print(`Waiting 5 seconds for containers to start...`)
	time.Sleep(5 * time.Second)

	cleanContainers(&ownerContainers, loggedUser)
	if err := renameDeployedContainers(&deployedServices); err != nil {
		errorHandler(w, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<commit_msg>Update compose.go<commit_after>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/docker-deploy/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)
var commandSplit = regexp.MustCompile(`["']([^"']+)["']|(\S+)`)

type dockerComposeService struct {
	Image       string
	Command     string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, loggedUser *user, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = loggedUser.username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if service.Command != `` {
		config.Cmd = strslice.StrSlice{}
		for _, args := range commandSplit.FindAllStringSubmatch(service.Command, -1) {
			config.Cmd = append(config.Cmd, args[1]+args[2])
		}
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (&deployedServices)[target]; ok {
			target = linkedService.Name
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			`traefik`: &traefikConfig,
		},
	}
}

func pullImage(image string, loggedUser *user) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(loggedUser.username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(loggedUser.username + ` ends pulling for ` + image)
	return nil
}

func cleanContainers(containers *[]types.Container, loggedUser *user) {
	for _, container := range *containers {
		log.Print(loggedUser.username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(loggedUser.username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, strings.TrimSuffix(service.Name, deploySuffix)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(appName string, serviceName string) string {
	return appName + `_` + serviceName + deploySuffix
}

func createAppHandler(w http.ResponseWriter, loggedUser *user, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(loggedUser.username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(loggedUser, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, loggedUser); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Print(loggedUser.username + ` starts ` + serviceFullName)

		id, err := docker.ContainerCreate(context.Background(), getConfig(&service, loggedUser, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			return
		}

		startContainer(id.ID)
		deployedServices[serviceName] = deployedService{ID: id.ID, Name: serviceFullName}
	}

	log.Print(`Waiting 5 seconds for containers to start...`)
	time.Sleep(5 * time.Second)

	cleanContainers(&ownerContainers, loggedUser)
	if err := renameDeployedContainers(&deployedServices); err != nil {
		errorHandler(w, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<|endoftext|>


<commit_before>package carbon

import (
	"github.com/vektra/errors"
)

// CarbonInterval represents an interval between two carbons.
type CarbonInterval struct {
	Start *Carbon
	End *Carbon
}

// NewCarbonInterval returns a pointer to a new CarbonInterval instance
func NewCarbonInterval(start, end *Carbon) (*CarbonInterval, error) {
	if start.Gte(end) {
		return nil, errors.New("The end date must be greater than the start date.")
	}

	return &CarbonInterval{
		Start: start,
		End: end,
	}, nil
}

// DiffInHours return the difference in hours between start and end date
func (ci *CarbonInterval) DiffInHours() int64 {
	return ci.End.DiffInHours(ci.Start, true)
}
<commit_msg>using std error<commit_after>package carbon

import (
	"errors"
)

// CarbonInterval represents an interval between two carbons.
type CarbonInterval struct {
	Start *Carbon
	End *Carbon
}

// NewCarbonInterval returns a pointer to a new CarbonInterval instance
func NewCarbonInterval(start, end *Carbon) (*CarbonInterval, error) {
	if start.Gte(end) {
		return nil, errors.New("The end date must be greater than the start date.")
	}

	return &CarbonInterval{
		Start: start,
		End: end,
	}, nil
}

// DiffInHours return the difference in hours between start and end date
func (ci *CarbonInterval) DiffInHours() int64 {
	return ci.End.DiffInHours(ci.Start, true)
}
<|endoftext|>


<commit_before>package span

import (
	"regexp"
	"strings"
	"unicode"
)

type Detector interface {
	Detect(*Splitter) (consumed int)
}

var DefaultDetectors = []Detector{
	EscapedChar{},
	LinkTags{},
	EmphasisTags{},
}

type EscapedChar struct{}

func (EscapedChar) Detect(s *Splitter) (consumed int) {
	rest := s.Buf[s.Pos:]
	if len(rest) >= 2 && rest[0] == '\\' {
		return 2
	} else {
		return 0
	}
}

type LinkTags struct{}

func (LinkTags) Detect(s *Splitter) (consumed int) {
	// [#procedure-for-identifying-link-tags]
	c := s.Buf[s.Pos]
	if c != '[' && c != ']' {
		return 0
	}
	// "opening link tag"?
	if c == '[' {
		s.Openings.Push(MaybeOpening{
			Tag:       s.Buf[s.Pos : s.Pos+1],
			NodeType:  LinkNode,
			LinkStart: s.Pos + 1,
		})
		return 1
	}
	// "closing link tag", c==']'
	return LinkTags{}.closingLinkTag(s)
}

var (
	// e.g.: "] [ref id]"
	reClosingTagRef = regexp.MustCompile(`^\]\s*\[(([^\\\[\]\` + "`" + `]|\\.)+)\]`)
	// e.g.: "] (http://www.example.net"...
	reClosingTagWithoutAngle = regexp.MustCompile(`^\]\s*\(\s*([^\(\)<>\` + "`" + `\s]+)([\)\s].*)$`)
	// e.g.: "] ( <http://example.net/?q=)>"...
	reClosingTagWithAngle = regexp.MustCompile(`^\]\s*\(\s*<([^<>\` + "`" + `]*)>([\)\s].*)$`)

	reJustClosingParen     = regexp.MustCompile(`^\s*\)`)
	reTitleAndClosingParen = regexp.MustCompile(`^\s*("(([^\\"\` + "`" + `]|\\.)*)"|'(([^\\'\` + "`" + `]|\\.)*)')\s*\)`)

	reEmptyRef = regexp.MustCompile(`^(\]\s*\[\s*\])`)
)

func (LinkTags) closingLinkTag(s *Splitter) (consumed int) {
	if s.Openings.NullTopmostOfType(LinkNode) {
		return 1 // consume the ']'
	}
	rest := s.Buf[s.Pos:]

	// e.g.: "] [ref id]" ?
	m := reClosingTagRef.FindSubmatch(rest)
	if m != nil {
		// cancel all unclosed spans inside the link
		for s.Openings.Peek().NodeType != LinkNode {
			s.Openings.Pop()
		}
		// emit a link
		s.Emit(s.Openings.Peek().Tag, LinkBegin{
			ReferenceID: Simplify(string(m[1])),
		})
		s.Emit(m[0], LinkEnd{})
		s.Openings.Pop()
		// cancel all unclosed links
		s.Openings.deleteLinks()
		return len(m[0])
	}

	// e.g.: "] (http://www.example.net"... ?
	m = reClosingTagWithoutAngle.FindSubmatch(rest)
	if m == nil {
		// e.g.: "] ( <http://example.net/?q=)>"... ?
		m = reClosingTagWithAngle.FindSubmatch(rest)
	}
	if m != nil {
		linkURL := DelWhitespace(string(m[1]))
		residual := m[2]
		title := ""
		t := reJustClosingParen.FindSubmatch(residual)
		if t == nil {
			t = reTitleAndClosingParen.FindSubmatch(residual)
		}
		if t != nil {
			if len(t) > 1 {
				attribs := t[1]
				unquoted := attribs[1 : len(attribs)-2]
				title = strings.Replace(string(unquoted), "\u000a", "", -1)
			}
			// cancel all unclosed spans inside the link
			for s.Openings.Peek().NodeType != LinkNode {
				s.Openings.Pop()
			}
			// emit a link
			s.Emit(s.Openings.Peek().Tag, LinkBegin{
				URL:   linkURL,
				Title: title,
			})
			s.Emit(t[0], LinkEnd{})
			s.Openings.Pop()
			// cancel all unclosed links
			s.Openings.deleteLinks()
			return len(rest) - len(residual) + len(t[0])
		}
	}

	// e.g.: "] []" ?
	m = reEmptyRef.FindSubmatch(rest)
	if m == nil {
		// just: "]"
		m = [][]byte{rest[:1]}
	}
	// cancel all unclosed spans inside the link
	for s.Openings.Peek().NodeType != LinkNode {
		s.Openings.Pop()
	}
	// emit a link
	begin := s.Openings.Peek()
	s.Emit(begin.Tag, LinkBegin{
		ReferenceID: Simplify(string(s.Buf[begin.LinkStart:s.Pos])),
	})
	s.Emit(m[0], LinkEnd{})
	s.Openings.Pop()
	// cancel all unclosed links
	s.Openings.deleteLinks()
	return len(m[0])
}

type LinkBegin struct{ ReferenceID, URL, Title string }
type LinkEnd struct{}

type EmphasisTags struct{}

func (EmphasisTags) Detect(s *Splitter) (consumed int) {
	rest := s.Buf[s.Pos:]
	if rest[0] != '*' && rest[0] != '_' {
		return 0
	}
	panic("NIY")
}

func emphasisFringeRank(r rune) int {
	switch {
	case unicode.In(r, unicode.Zs, unicode.Zl, unicode.Zp, unicode.Cc, unicode.Cf):
		return 0
	case unicode.In(r, unicode.Pc, unicode.Pd, unicode.Ps, unicode.Pe, unicode.Pi, unicode.Pf, unicode.Po, unicode.Sc, unicode.Sk, unicode.Sm, unicode.So):
		return 1
	default:
		return 2
	}
}
<commit_msg>WIP emphasis span detection<commit_after>package span

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Detector interface {
	Detect(*Splitter) (consumed int)
}

var DefaultDetectors = []Detector{
	EscapedChar{},
	LinkTags{},
	EmphasisTags{},
}

type EscapedChar struct{}

func (EscapedChar) Detect(s *Splitter) (consumed int) {
	rest := s.Buf[s.Pos:]
	if len(rest) >= 2 && rest[0] == '\\' {
		return 2
	} else {
		return 0
	}
}

type LinkTags struct{}

func (LinkTags) Detect(s *Splitter) (consumed int) {
	// [#procedure-for-identifying-link-tags]
	c := s.Buf[s.Pos]
	if c != '[' && c != ']' {
		return 0
	}
	// "opening link tag"?
	if c == '[' {
		s.Openings.Push(MaybeOpening{
			Tag:       s.Buf[s.Pos : s.Pos+1],
			NodeType:  LinkNode,
			LinkStart: s.Pos + 1,
		})
		return 1
	}
	// "closing link tag", c==']'
	return LinkTags{}.closingLinkTag(s)
}

var (
	// e.g.: "] [ref id]"
	reClosingTagRef = regexp.MustCompile(`^\]\s*\[(([^\\\[\]\` + "`" + `]|\\.)+)\]`)
	// e.g.: "] (http://www.example.net"...
	reClosingTagWithoutAngle = regexp.MustCompile(`^\]\s*\(\s*([^\(\)<>\` + "`" + `\s]+)([\)\s].*)$`)
	// e.g.: "] ( <http://example.net/?q=)>"...
	reClosingTagWithAngle = regexp.MustCompile(`^\]\s*\(\s*<([^<>\` + "`" + `]*)>([\)\s].*)$`)

	reJustClosingParen     = regexp.MustCompile(`^\s*\)`)
	reTitleAndClosingParen = regexp.MustCompile(`^\s*("(([^\\"\` + "`" + `]|\\.)*)"|'(([^\\'\` + "`" + `]|\\.)*)')\s*\)`)

	reEmptyRef = regexp.MustCompile(`^(\]\s*\[\s*\])`)
)

func (LinkTags) closingLinkTag(s *Splitter) (consumed int) {
	if s.Openings.NullTopmostOfType(LinkNode) {
		return 1 // consume the ']'
	}
	rest := s.Buf[s.Pos:]

	// e.g.: "] [ref id]" ?
	m := reClosingTagRef.FindSubmatch(rest)
	if m != nil {
		// cancel all unclosed spans inside the link
		for s.Openings.Peek().NodeType != LinkNode {
			s.Openings.Pop()
		}
		// emit a link
		s.Emit(s.Openings.Peek().Tag, LinkBegin{
			ReferenceID: Simplify(string(m[1])),
		})
		s.Emit(m[0], LinkEnd{})
		s.Openings.Pop()
		// cancel all unclosed links
		s.Openings.deleteLinks()
		return len(m[0])
	}

	// e.g.: "] (http://www.example.net"... ?
	m = reClosingTagWithoutAngle.FindSubmatch(rest)
	if m == nil {
		// e.g.: "] ( <http://example.net/?q=)>"... ?
		m = reClosingTagWithAngle.FindSubmatch(rest)
	}
	if m != nil {
		linkURL := DelWhitespace(string(m[1]))
		residual := m[2]
		title := ""
		t := reJustClosingParen.FindSubmatch(residual)
		if t == nil {
			t = reTitleAndClosingParen.FindSubmatch(residual)
		}
		if t != nil {
			if len(t) > 1 {
				attribs := t[1]
				unquoted := attribs[1 : len(attribs)-2]
				title = strings.Replace(string(unquoted), "\u000a", "", -1)
			}
			// cancel all unclosed spans inside the link
			for s.Openings.Peek().NodeType != LinkNode {
				s.Openings.Pop()
			}
			// emit a link
			s.Emit(s.Openings.Peek().Tag, LinkBegin{
				URL:   linkURL,
				Title: title,
			})
			s.Emit(t[0], LinkEnd{})
			s.Openings.Pop()
			// cancel all unclosed links
			s.Openings.deleteLinks()
			return len(rest) - len(residual) + len(t[0])
		}
	}

	// e.g.: "] []" ?
	m = reEmptyRef.FindSubmatch(rest)
	if m == nil {
		// just: "]"
		m = [][]byte{rest[:1]}
	}
	// cancel all unclosed spans inside the link
	for s.Openings.Peek().NodeType != LinkNode {
		s.Openings.Pop()
	}
	// emit a link
	begin := s.Openings.Peek()
	s.Emit(begin.Tag, LinkBegin{
		ReferenceID: Simplify(string(s.Buf[begin.LinkStart:s.Pos])),
	})
	s.Emit(m[0], LinkEnd{})
	s.Openings.Pop()
	// cancel all unclosed links
	s.Openings.deleteLinks()
	return len(m[0])
}

type LinkBegin struct{ ReferenceID, URL, Title string }
type LinkEnd struct{}

type EmphasisTags struct{}

func (EmphasisTags) Detect(s *Splitter) (consumed int) {
	rest := s.Buf[s.Pos:]
	if !isEmph(rest[0]) {
		return 0
	}
	// find substring composed solely of '*' and '_'
	indicator := rest[:1]
	for i := len(indicator); i < len(rest); i++ {
		if !isEmph(rest[i]) {
			break
		}
		indicator = rest[:i+1]
	}
	// "right-fringe-mark"
	r, _ := utf8.DecodeRune(rest[len(indicator):])
	rightFringe := emphasisFringeRank(r)
	r, _ = utf8.DecodeLastRune(s.Buf[:s.Pos])
	leftFringe := emphasisFringeRank(r)
	// <0 means "left-flanking", >0 "right-flanking", 0 "non-flanking"
	flanking := leftFringe - rightFringe
	if flanking == 0 {
		return len(indicator)
	}
	// split into "emphasis-tag-strings" - subslices of the same char
	tags := [][]byte{}
	prev := 0
	for curr := 1; curr <= len(indicator); curr++ {
		if curr == len(indicator) || indicator[curr] != indicator[prev] {
			tags = append(tags, indicator[prev:curr])
			prev = curr
		}
	}
	// left-flanking? if yes, add some openings
	if flanking < 0 {
		for _, tag := range tags {
			// TODO(akavel): leave just one EmphasisNode value and compare specific subtypes based on Tag[0]
			typ := AsteriskEmphasisNode
			if tag[0] == '_' {
				typ = UnderscoreEmphasisNode
			}
			s.Openings.Push(MaybeOpening{
				Tag:      tag,
				NodeType: typ,
			})
		}
		return len(indicator)
	}

	panic("NIY")
}

func isEmph(c byte) { return c == '*' || c == '_' }

func emphasisFringeRank(r rune) int {
	switch {
	case r == utf8.RuneError:
		// NOTE(akavel): not sure if that's really correct
		return 0
	case unicode.In(r, unicode.Zs, unicode.Zl, unicode.Zp, unicode.Cc, unicode.Cf):
		return 0
	case unicode.In(r, unicode.Pc, unicode.Pd, unicode.Ps, unicode.Pe, unicode.Pi, unicode.Pf, unicode.Po, unicode.Sc, unicode.Sk, unicode.Sm, unicode.So):
		return 1
	default:
		return 2
	}
}
<|endoftext|>


<commit_before>package canal

import (
	"regexp"
	"time"

	"golang.org/x/net/context"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

var (
	expAlterTable = regexp.MustCompile("(?i)^ALTER\\sTABLE\\s.*?`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s.*")
)

func (c *Canal) startSyncBinlog() error {
	pos := mysql.Position{c.master.Name, c.master.Position}

	log.Infof("start sync binlog at %v", pos)

	s, err := c.syncer.StartSync(pos)
	if err != nil {
		return errors.Errorf("start sync replication at %v error %v", pos, err)
	}

	timeout := time.Second
	forceSavePos := false
	for {
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		ev, err := s.GetEvent(ctx)
		cancel()

		if err == context.DeadlineExceeded {
			timeout = 2 * timeout
			continue
		}

		if err != nil {
			return errors.Trace(err)
		}

		timeout = time.Second

		//next binlog pos
		pos.Pos = ev.Header.LogPos

		forceSavePos = false

		// We only save position with RotateEvent and XIDEvent.
		// For RowsEvent, we can't save the position until meeting XIDEvent
		// which tells the whole transaction is over.
		// TODO: If we meet any DDL query, we must save too.
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			pos.Name = string(e.NextLogName)
			pos.Pos = uint32(e.Position)
			// r.ev <- pos
			forceSavePos = true
			log.Infof("rotate binlog to %v", pos)
		case *replication.RowsEvent:
			// we only focus row based event
			if err = c.handleRowsEvent(ev); err != nil {
				log.Errorf("handle rows event error %v", err)
				return errors.Trace(err)
			}
			continue
		case *replication.XIDEvent:
			// try to save the position later
		case *replication.QueryEvent:
			// handle alert table query
			if mb := expAlterTable.FindSubmatch(e.Query); mb != nil {
				if len(mb[1]) == 0 {
					mb[1] = e.Schema
				}
				c.ClearTableCache(mb[1], mb[2])
				log.Infof("table structure changed, clear table cache: %s.%s\n", mb[1], mb[2])
				forceSavePos = true
			} else {
				// skip others
				continue
			}
		default:
			continue
		}

		c.master.Update(pos.Name, pos.Pos)
		c.master.Save(forceSavePos)
	}

	return nil
}

func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
	ev := e.Event.(*replication.RowsEvent)

	// Caveat: table may be altered at runtime.
	schema := string(ev.Table.Schema)
	table := string(ev.Table.Table)

	t, err := c.GetTable(schema, table)
	if err != nil {
		return errors.Trace(err)
	}
	var action string
	switch e.Header.EventType {
	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
		action = InsertAction
	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
		action = DeleteAction
	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
		action = UpdateAction
	default:
		return errors.Errorf("%s not supported now", e.Header.EventType)
	}
	events := newRowsEvent(t, action, ev.Rows)
	return c.travelRowsEventHandler(events)
}

func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error {
	if timeout <= 0 {
		timeout = 60
	}

	timer := time.NewTimer(time.Duration(timeout) * time.Second)
	for {
		select {
		case <-timer.C:
			return errors.Errorf("wait position %v err", pos)
		default:
			curpos := c.master.Pos()
			if curpos.Compare(pos) >= 0 {
				return nil
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		}
	}

	return nil
}

func (c *Canal) CatchMasterPos(timeout int) error {
	rr, err := c.Execute("SHOW MASTER STATUS")
	if err != nil {
		return errors.Trace(err)
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)

	return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout)
}
<commit_msg>canal add ClearTableCache when table structure changed<commit_after>package canal

import (
	"regexp"
	"time"

	"golang.org/x/net/context"

	"bytes"
	"regexp"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

var (
	expAlterTable = regexp.MustCompile("(?i)^ALTER\\sTABLE\\s.*?`{0,1}(.*?)`{0,1}\\.{0,1}`{0,1}([^`\\.]+?)`{0,1}\\s.*")
)

func (c *Canal) startSyncBinlog() error {
	pos := mysql.Position{c.master.Name, c.master.Position}

	log.Infof("start sync binlog at %v", pos)

	s, err := c.syncer.StartSync(pos)
	if err != nil {
		return errors.Errorf("start sync replication at %v error %v", pos, err)
	}

	timeout := time.Second
	forceSavePos := false
	for {
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
		ev, err := s.GetEvent(ctx)
		cancel()

		if err == context.DeadlineExceeded {
			timeout = 2 * timeout
			continue
		}

		if err != nil {
			return errors.Trace(err)
		}

		timeout = time.Second

		//next binlog pos
		pos.Pos = ev.Header.LogPos

		forceSavePos = false

		// We only save position with RotateEvent and XIDEvent.
		// For RowsEvent, we can't save the position until meeting XIDEvent
		// which tells the whole transaction is over.
		// TODO: If we meet any DDL query, we must save too.
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			pos.Name = string(e.NextLogName)
			pos.Pos = uint32(e.Position)
			// r.ev <- pos
			forceSavePos = true
			log.Infof("rotate binlog to %v", pos)
		case *replication.RowsEvent:
			// we only focus row based event
			if err = c.handleRowsEvent(ev); err != nil {
				log.Errorf("handle rows event error %v", err)
				return errors.Trace(err)
			}
			continue
		case *replication.XIDEvent:
			// try to save the position later
		case *replication.QueryEvent:
			// handle alert table query
			if mb := expAlterTable.FindSubmatch(e.Query); mb != nil {
				if len(mb[1]) == 0 {
					mb[1] = e.Schema
				}
				c.ClearTableCache(mb[1], mb[2])
				log.Infof("table structure changed, clear table cache: %s.%s\n", mb[1], mb[2])
				forceSavePos = true
			} else {
				// skip others
				continue
			}
		default:
			continue
		}

		c.master.Update(pos.Name, pos.Pos)
		c.master.Save(forceSavePos)
	}

	return nil
}

func (c *Canal) handleRowsEvent(e *replication.BinlogEvent) error {
	ev := e.Event.(*replication.RowsEvent)

	// Caveat: table may be altered at runtime.
	schema := string(ev.Table.Schema)
	table := string(ev.Table.Table)

	t, err := c.GetTable(schema, table)
	if err != nil {
		return errors.Trace(err)
	}
	var action string
	switch e.Header.EventType {
	case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2:
		action = InsertAction
	case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2:
		action = DeleteAction
	case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2:
		action = UpdateAction
	default:
		return errors.Errorf("%s not supported now", e.Header.EventType)
	}
	events := newRowsEvent(t, action, ev.Rows)
	return c.travelRowsEventHandler(events)
}

func (c *Canal) WaitUntilPos(pos mysql.Position, timeout int) error {
	if timeout <= 0 {
		timeout = 60
	}

	timer := time.NewTimer(time.Duration(timeout) * time.Second)
	for {
		select {
		case <-timer.C:
			return errors.Errorf("wait position %v err", pos)
		default:
			curpos := c.master.Pos()
			if curpos.Compare(pos) >= 0 {
				return nil
			} else {
				time.Sleep(100 * time.Millisecond)
			}
		}
	}

	return nil
}

func (c *Canal) CatchMasterPos(timeout int) error {
	rr, err := c.Execute("SHOW MASTER STATUS")
	if err != nil {
		return errors.Trace(err)
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)

	return c.WaitUntilPos(mysql.Position{name, uint32(pos)}, timeout)
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"fmt"
	"math"

	"github.com/square/metrics/api"
	"github.com/square/metrics/function"
)

var Timeshift = function.MetricFunction{
	Name:         "transform.timeshift",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		value, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		duration, err := value.ToDuration()
		if err != nil {
			return nil, err
		}
		newContext := context
		newContext.Timerange = newContext.Timerange.Shift(duration)

		result, err := arguments[0].Evaluate(newContext)
		if err != nil {
			return nil, err
		}

		if seriesValue, ok := result.(api.SeriesList); ok {
			seriesValue.Timerange = context.Timerange
			return seriesValue, nil
		}
		return result, nil
	},
}

var MovingAverage = function.MetricFunction{
	Name:         "transform.moving_average",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// Applying a similar trick as did TimeshiftFunction. It fetches data prior to the start of the timerange.

		sizeValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		size, err := sizeValue.ToDuration()
		if err != nil {
			return nil, err
		}
		limit := int(float64(size)/float64(context.Timerange.Resolution()) + 0.5) // Limit is the number of items to include in the average
		if limit < 1 {
			// At least one value must be included at all times
			limit = 1
		}

		newContext := context.Copy()
		timerange := context.Timerange
		newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-int64(limit-1)*timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
		if err != nil {
			return nil, err
		}
		// The new context has a timerange which is extended beyond the query's.
		listValue, err := arguments[0].Evaluate(&newContext)
		if err != nil {
			return nil, err
		}

		// This value must be a SeriesList.
		list, err := listValue.ToSeriesList(newContext.Timerange)
		if err != nil {
			return nil, err
		}

		// The timerange must be reverted.
		list.Timerange = context.Timerange
		context.CopyNotesFrom(&newContext)
		newContext.Invalidate() //Prevent this from leaking or getting used.

		// Update each series in the list.
		for index, series := range list.Series {
			// The series will be given a (shorter) replaced list of values.
			results := make([]float64, context.Timerange.Slots())
			count := 0
			sum := 0.0
			for i := range series.Values {
				// Add the new element, if it isn't NaN.
				if !math.IsNaN(series.Values[i]) {
					sum += series.Values[i]
					count++
				}
				// Remove the oldest element, if it isn't NaN, and it's in range.
				// (e.g., if limit = 1, then this removes the previous element from the sum).
				if i >= limit && !math.IsNaN(series.Values[i-limit]) {
					sum -= series.Values[i-limit]
					count--
				}
				// Numerical error could (possibly) cause count == 0 but sum != 0.
				if i-limit+1 >= 0 {
					if count == 0 {
						results[i-limit+1] = math.NaN()
					} else {
						results[i-limit+1] = sum / float64(count)
					}
				}
			}
			list.Series[index].Values = results
		}
		return list, nil
	},
}

var ExponentialMovingAverage = function.MetricFunction{
	Name:         "transform.exponential_moving_average",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// Applying a similar trick as did TimeshiftFunction. It fetches data prior to the start of the timerange.

		sizeValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		size, err := sizeValue.ToDuration()
		if err != nil {
			return nil, err
		}
		limit := int(float64(size)/float64(context.Timerange.Resolution()) + 0.5) // Limit is the number of items to include in the average
		if limit < 1 {
			// At least one value must be included at all times
			limit = 1
		}

		newContext := context.Copy()
		timerange := context.Timerange
		newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-int64(limit-1)*timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
		if err != nil {
			return nil, err
		}
		// The new context has a timerange which is extended beyond the query's.
		listValue, err := arguments[0].Evaluate(&newContext)
		if err != nil {
			return nil, err
		}

		// This value must be a SeriesList.
		list, err := listValue.ToSeriesList(newContext.Timerange)
		if err != nil {
			return nil, err
		}

		// How many "ticks" are there in "size"?
		// size / resolution
		// alpha is a parameter such that
		// alpha^ticks = 1/2
		// so, alpha = exp(log(1/2) / ticks)
		alpha := math.Exp(math.Log(0.5) * float64(context.Timerange.Resolution()) / float64(size))

		// The timerange must be reverted.
		list.Timerange = context.Timerange
		context.CopyNotesFrom(&newContext)
		newContext.Invalidate() //Prevent this from leaking or getting used.

		// Update each series in the list.
		for index, series := range list.Series {
			// The series will be given a (shorter) replaced list of values.
			results := make([]float64, context.Timerange.Slots())
			weight := 0.0
			sum := 0.0
			for i := range series.Values {
				weight *= alpha
				sum *= alpha
				if !math.IsNaN(series.Values[i]) {
					weight += 1
					sum += series.Values[i]
				}
				results[i-limit+1] = sum / weight
			}
			list.Series[index].Values = results
		}
		list.Query = fmt.Sprintf("transform.exponential_moving_average(%s, %s)", listValue.GetName(), sizeValue.GetName())
		list.Name = list.Query
		return list, nil
	},
}

var Alias = function.MetricFunction{
	Name:         "transform.alias",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// TODO: delete this function
		// also, this operation is not thread-safe, is it?
		context.EvaluationNotes = append(context.EvaluationNotes, "transform.alias is deprecated")
		return arguments[0].Evaluate(context)
	},
}

// Derivative is special because it needs to get one extra data point to the left
// This transform estimates the "change per second" between the two samples (scaled consecutive difference)
var Derivative = newDerivativeBasedTransform("derivative", derivative)

func derivative(ctx *function.EvaluationContext, series api.Timeseries, parameters []function.Value, scale float64) ([]float64, error) {
	values := series.Values
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
	}
	return result, nil
}

// Rate is special because it needs to get one extra data point to the left.
// This transform functions mostly like Derivative but bounds the result to be positive.
// Specifically this function is designed for strictly increasing counters that
// only decrease when reset to zero. That is, thie function returns consecutive
// differences which are at least 0, or math.Max of the newly reported value and 0
var Rate = newDerivativeBasedTransform("rate", rate)

func rate(ctx *function.EvaluationContext, series api.Timeseries, parameters []function.Value, scale float64) ([]float64, error) {
	values := series.Values
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
		if result[i-1] < 0 {
			result[i-1] = 0
		}
		if i+1 < len(values) && values[i-1] > values[i] && values[i] <= values[i+1] {
			// Downsampling may cause a drop from 1000 to 0 to look like [1000, 500, 0] instead of [1000, 1001, 0].
			// So we check the next, in addition to the previous.
			ctx.AddNote(fmt.Sprintf("Rate(%v): The underlying counter reset between %f, %f\n", series.TagSet, values[i-1], values[i]))
			// values[i] is our best approximatation of the delta between i-1 and i
			// Why? This should only be used on counters, so if v[i] - v[i-1] < 0 then
			// the counter has reset, and we know *at least* v[i] increments have happened
			result[i-1] = math.Max(values[i], 0) / scale
		}
	}
	return result, nil
}

// newDerivativeBasedTransform returns a function.MetricFunction that performs
// a delta between two data points. The transform parameter is a function of type
// transform is expected to return an array of values whose length is 1 less
// than the given series
func newDerivativeBasedTransform(name string, transformer transform) function.MetricFunction {
	return function.MetricFunction{
		Name:         "transform." + name,
		MinArguments: 1,
		MaxArguments: 1,
		Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
			var err error
			// Calcuate the new timerange to include one extra point to the left
			newContext := context.Copy()
			timerange := context.Timerange
			newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
			if err != nil {
				return nil, err
			}

			// The new context has a timerange which is extended beyond the query's.
			listValue, err := arguments[0].Evaluate(&newContext)
			if err != nil {
				return nil, err
			}

			// This value must be a SeriesList.
			list, err := listValue.ToSeriesList(newContext.Timerange)
			if err != nil {
				return nil, err
			}

			// Reset the timerange
			list.Timerange = context.Timerange
			context.CopyNotesFrom(&newContext)
			newContext.Invalidate() // Prevent leaking this around.

			//Apply the original context to the transform even though the list
			//will include one additional data point.
			result, err := ApplyTransform(context, list, transformer, []function.Value{})
			if err != nil {
				return nil, err
			}

			// Validate our series are the correct length
			for i := range result.Series {
				if len(result.Series[i].Values) != len(list.Series[i].Values)-1 {
					panic(fmt.Sprintf("Expected transform to return %d values, received %d", len(list.Series[i].Values)-1, len(result.Series[i].Values)))
				}
			}
			return result, nil
		},
	}
}
<commit_msg>update against master<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"fmt"
	"math"

	"github.com/square/metrics/api"
	"github.com/square/metrics/function"
)

var Timeshift = function.MetricFunction{
	Name:         "transform.timeshift",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		value, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		duration, err := value.ToDuration()
		if err != nil {
			return nil, err
		}
		newContext := context
		newContext.Timerange = newContext.Timerange.Shift(duration)

		result, err := arguments[0].Evaluate(newContext)
		if err != nil {
			return nil, err
		}

		if seriesValue, ok := result.(api.SeriesList); ok {
			seriesValue.Timerange = context.Timerange
			return seriesValue, nil
		}
		return result, nil
	},
}

var MovingAverage = function.MetricFunction{
	Name:         "transform.moving_average",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// Applying a similar trick as did TimeshiftFunction. It fetches data prior to the start of the timerange.

		sizeValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		size, err := sizeValue.ToDuration()
		if err != nil {
			return nil, err
		}
		limit := int(float64(size)/float64(context.Timerange.Resolution()) + 0.5) // Limit is the number of items to include in the average
		if limit < 1 {
			// At least one value must be included at all times
			limit = 1
		}

		newContext := context.Copy()
		timerange := context.Timerange
		newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-int64(limit-1)*timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
		if err != nil {
			return nil, err
		}
		// The new context has a timerange which is extended beyond the query's.
		listValue, err := arguments[0].Evaluate(&newContext)
		if err != nil {
			return nil, err
		}

		// This value must be a SeriesList.
		list, err := listValue.ToSeriesList(newContext.Timerange)
		if err != nil {
			return nil, err
		}

		// The timerange must be reverted.
		list.Timerange = context.Timerange
		context.CopyNotesFrom(&newContext)
		newContext.Invalidate() //Prevent this from leaking or getting used.

		// Update each series in the list.
		for index, series := range list.Series {
			// The series will be given a (shorter) replaced list of values.
			results := make([]float64, context.Timerange.Slots())
			count := 0
			sum := 0.0
			for i := range series.Values {
				// Add the new element, if it isn't NaN.
				if !math.IsNaN(series.Values[i]) {
					sum += series.Values[i]
					count++
				}
				// Remove the oldest element, if it isn't NaN, and it's in range.
				// (e.g., if limit = 1, then this removes the previous element from the sum).
				if i >= limit && !math.IsNaN(series.Values[i-limit]) {
					sum -= series.Values[i-limit]
					count--
				}
				// Numerical error could (possibly) cause count == 0 but sum != 0.
				if i-limit+1 >= 0 {
					if count == 0 {
						results[i-limit+1] = math.NaN()
					} else {
						results[i-limit+1] = sum / float64(count)
					}
				}
			}
			list.Series[index].Values = results
		}
		return list, nil
	},
}

var ExponentialMovingAverage = function.MetricFunction{
	Name:         "transform.exponential_moving_average",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// Applying a similar trick as did TimeshiftFunction. It fetches data prior to the start of the timerange.

		sizeValue, err := arguments[1].Evaluate(context)
		if err != nil {
			return nil, err
		}
		size, err := sizeValue.ToDuration()
		if err != nil {
			return nil, err
		}
		limit := int(float64(size)/float64(context.Timerange.Resolution()) + 0.5) // Limit is the number of items to include in the average
		if limit < 1 {
			// At least one value must be included at all times
			limit = 1
		}

		newContext := context.Copy()
		timerange := context.Timerange
		newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-int64(limit-1)*timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
		if err != nil {
			return nil, err
		}
		// The new context has a timerange which is extended beyond the query's.
		listValue, err := arguments[0].Evaluate(&newContext)
		if err != nil {
			return nil, err
		}

		// This value must be a SeriesList.
		list, err := listValue.ToSeriesList(newContext.Timerange)
		if err != nil {
			return nil, err
		}

		// How many "ticks" are there in "size"?
		// size / resolution
		// alpha is a parameter such that
		// alpha^ticks = 1/2
		// so, alpha = exp(log(1/2) / ticks)
		alpha := math.Exp(math.Log(0.5) * float64(context.Timerange.Resolution()) / float64(size))

		// The timerange must be reverted.
		list.Timerange = context.Timerange
		context.CopyNotesFrom(&newContext)
		newContext.Invalidate() //Prevent this from leaking or getting used.

		// Update each series in the list.
		for index, series := range list.Series {
			// The series will be given a (shorter) replaced list of values.
			results := make([]float64, context.Timerange.Slots())
			weight := 0.0
			sum := 0.0
			for i := range series.Values {
				weight *= alpha
				sum *= alpha
				if !math.IsNaN(series.Values[i]) {
					weight += 1
					sum += series.Values[i]
				}
				results[i-limit+1] = sum / weight
			}
			list.Series[index].Values = results
		}
		return list, nil
	},
}

var Alias = function.MetricFunction{
	Name:         "transform.alias",
	MinArguments: 2,
	MaxArguments: 2,
	Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
		// TODO: delete this function
		// also, this operation is not thread-safe, is it?
		context.EvaluationNotes = append(context.EvaluationNotes, "transform.alias is deprecated")
		return arguments[0].Evaluate(context)
	},
}

// Derivative is special because it needs to get one extra data point to the left
// This transform estimates the "change per second" between the two samples (scaled consecutive difference)
var Derivative = newDerivativeBasedTransform("derivative", derivative)

func derivative(ctx *function.EvaluationContext, series api.Timeseries, parameters []function.Value, scale float64) ([]float64, error) {
	values := series.Values
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
	}
	return result, nil
}

// Rate is special because it needs to get one extra data point to the left.
// This transform functions mostly like Derivative but bounds the result to be positive.
// Specifically this function is designed for strictly increasing counters that
// only decrease when reset to zero. That is, thie function returns consecutive
// differences which are at least 0, or math.Max of the newly reported value and 0
var Rate = newDerivativeBasedTransform("rate", rate)

func rate(ctx *function.EvaluationContext, series api.Timeseries, parameters []function.Value, scale float64) ([]float64, error) {
	values := series.Values
	result := make([]float64, len(values)-1)
	for i := range values {
		if i == 0 {
			continue
		}
		// Scaled difference
		result[i-1] = (values[i] - values[i-1]) / scale
		if result[i-1] < 0 {
			result[i-1] = 0
		}
		if i+1 < len(values) && values[i-1] > values[i] && values[i] <= values[i+1] {
			// Downsampling may cause a drop from 1000 to 0 to look like [1000, 500, 0] instead of [1000, 1001, 0].
			// So we check the next, in addition to the previous.
			ctx.AddNote(fmt.Sprintf("Rate(%v): The underlying counter reset between %f, %f\n", series.TagSet, values[i-1], values[i]))
			// values[i] is our best approximatation of the delta between i-1 and i
			// Why? This should only be used on counters, so if v[i] - v[i-1] < 0 then
			// the counter has reset, and we know *at least* v[i] increments have happened
			result[i-1] = math.Max(values[i], 0) / scale
		}
	}
	return result, nil
}

// newDerivativeBasedTransform returns a function.MetricFunction that performs
// a delta between two data points. The transform parameter is a function of type
// transform is expected to return an array of values whose length is 1 less
// than the given series
func newDerivativeBasedTransform(name string, transformer transform) function.MetricFunction {
	return function.MetricFunction{
		Name:         "transform." + name,
		MinArguments: 1,
		MaxArguments: 1,
		Compute: func(context *function.EvaluationContext, arguments []function.Expression, groups function.Groups) (function.Value, error) {
			var err error
			// Calcuate the new timerange to include one extra point to the left
			newContext := context.Copy()
			timerange := context.Timerange
			newContext.Timerange, err = api.NewSnappedTimerange(timerange.Start()-timerange.ResolutionMillis(), timerange.End(), timerange.ResolutionMillis())
			if err != nil {
				return nil, err
			}

			// The new context has a timerange which is extended beyond the query's.
			listValue, err := arguments[0].Evaluate(&newContext)
			if err != nil {
				return nil, err
			}

			// This value must be a SeriesList.
			list, err := listValue.ToSeriesList(newContext.Timerange)
			if err != nil {
				return nil, err
			}

			// Reset the timerange
			list.Timerange = context.Timerange
			context.CopyNotesFrom(&newContext)
			newContext.Invalidate() // Prevent leaking this around.

			//Apply the original context to the transform even though the list
			//will include one additional data point.
			result, err := ApplyTransform(context, list, transformer, []function.Value{})
			if err != nil {
				return nil, err
			}

			// Validate our series are the correct length
			for i := range result.Series {
				if len(result.Series[i].Values) != len(list.Series[i].Values)-1 {
					panic(fmt.Sprintf("Expected transform to return %d values, received %d", len(list.Series[i].Values)-1, len(result.Series[i].Values)))
				}
			}
			return result, nil
		},
	}
}
<|endoftext|>