<commit_before>/**
 * (C) Copyright 2013 Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package deftlabsutil

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"errors"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	SocketTimeout = 40
	HttpPostMethod = "POST"
	ContentTypeHeader = "Content-Type"
	ContentTypeTextPlain = "text/plain; charset=utf-8"
)

func HttpPostJson(url string, value interface{}) ([]byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil {
		return nil, err
	}

	httpClient, httpTransport := getDefaultHttpClient()
	defer httpTransport.Close()

	request, err := http.NewRequest("POST", url, bytes.NewReader(rawJson))
	if err != nil {
		return nil, err
	}
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	response, err := httpClient.Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	data, err := ioutil.ReadAll(response.Body)

		/*
	if err != nil {

	}
	*/

	return data, nil
}

func HttpPostBson(url string, bsonDoc interface{}) ([]byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil {
		return nil, err
	}

	httpClient, httpTransport := getDefaultHttpClient()
	defer httpTransport.Close()

	request, err := http.NewRequest("POST", url, bytes.NewReader(rawBson))
	if err != nil {
		return nil, err
	}
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	response, err := httpClient.Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil {
		return nil, err
	} else {
		return data, nil
	}
}

func getDefaultHttpClient() (*http.Client, *httpclient.Transport) {
	transport := getDefaultHttpTransport()
	return &http.Client{ Transport: transport }, transport
}

func getDefaultHttpTransport() *httpclient.Transport {
	return &httpclient.Transport {
		ConnectTimeout:        SocketTimeout * time.Second,
		RequestTimeout:        SocketTimeout * time.Second,
		ResponseHeaderTimeout: SocketTimeout * time.Second,
	}
}

func HttpGetBson(url string) (bson.M, error) {

	httpClient, httpTransport := getDefaultHttpClient()
	defer httpTransport.Close()

	request, requestErr := http.NewRequest("GET", url, nil)
	if requestErr != nil {
		return nil, requestErr
	}

	response, err := httpClient.Do(request)

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil {
		return nil, err
	}

	return bsonDoc, nil
}

// This method returns true if the http request method is a HTTP post. If the
// field missing or incorrect, false is returned. This method will panic if the request
// is nil.
func IsHttpMethodPost(request *http.Request) bool {
	if request == nil {
		panic("request param is nil")
	}
	return len(request.Method) > 0 && strings.ToUpper(request.Method) == HttpPostMethod
}

// Write an http ok response string. The content type is text/plain.
func WriteOkResponseString(response http.ResponseWriter, msg string) error {
	if response == nil {
		panic("response param is nil")
	}

	msgLength := len(msg)

	if msgLength == 0 {
		panic("do not write an empty string to the response")
	}

	response.Header().Set(ContentTypeHeader, ContentTypeTextPlain)

	written, err := response.Write([]byte(msg))

	if err != nil {
		return err
	}

	if written != msgLength {
		return errors.New(fmt.Sprintf("Did not write full message - bytes written %d - expected %d", written, msgLength))
	}

	return nil
}

<commit_msg>fixed post method<commit_after>/**
 * (C) Copyright 2013 Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package deftlabsutil

import (
	"fmt"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"errors"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	SocketTimeout = 40
	HttpPostMethod = "POST"
	ContentTypeHeader = "Content-Type"
	ContentTypeTextPlain = "text/plain; charset=utf-8"
)

func HttpPostJson(url string, value interface{}) ([]byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil {
		return nil, err
	}

	httpClient, httpTransport := getDefaultHttpClient()
	defer httpTransport.Close()

	request, err := http.NewRequest("POST", url, bytes.NewReader(rawJson))
	if err != nil {
		return nil, err
	}
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	response, err := httpClient.Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil {
		return nil, err
	} else {
		return data, nil
	}
}

func HttpPostBson(url string, bsonDoc interface{}) ([]byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil {
		return nil, err
	}

	httpClient, httpTransport := getDefaultHttpClient()
	defer httpTransport.Close()

	request, err := http.NewRequest("POST", url, bytes.NewReader(rawBson))
	if err != nil {
		return nil, err
	}
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	response, err := httpClient.Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil {
		return nil, err
	} else {
		return data, nil
	}
}

func getDefaultHttpClient() (*http.Client, *httpclient.Transport) {
	transport := getDefaultHttpTransport()
	return &http.Client{ Transport: transport }, transport
}

func getDefaultHttpTransport() *httpclient.Transport {
	return &httpclient.Transport {
		ConnectTimeout:        SocketTimeout * time.Second,
		RequestTimeout:        SocketTimeout * time.Second,
		ResponseHeaderTimeout: SocketTimeout * time.Second,
	}
}

func HttpGetBson(url string) (bson.M, error) {

	httpClient, httpTransport := getDefaultHttpClient()
	defer httpTransport.Close()

	request, requestErr := http.NewRequest("GET", url, nil)
	if requestErr != nil {
		return nil, requestErr
	}

	response, err := httpClient.Do(request)

	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil {
		return nil, err
	}

	return bsonDoc, nil
}

// This method returns true if the http request method is a HTTP post. If the
// field missing or incorrect, false is returned. This method will panic if the request
// is nil.
func IsHttpMethodPost(request *http.Request) bool {
	if request == nil {
		panic("request param is nil")
	}
	return len(request.Method) > 0 && strings.ToUpper(request.Method) == HttpPostMethod
}

// Write an http ok response string. The content type is text/plain.
func WriteOkResponseString(response http.ResponseWriter, msg string) error {
	if response == nil {
		panic("response param is nil")
	}

	msgLength := len(msg)

	if msgLength == 0 {
		panic("do not write an empty string to the response")
	}

	response.Header().Set(ContentTypeHeader, ContentTypeTextPlain)

	written, err := response.Write([]byte(msg))

	if err != nil {
		return err
	}

	if written != msgLength {
		return errors.New(fmt.Sprintf("Did not write full message - bytes written %d - expected %d", written, msgLength))
	}

	return nil
}

<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"fmt"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
	"k8s.io/klog"

	"k8s.io/apimachinery/pkg/types"
)

// AttachDisk attaches a vhd to vm
// the vhd must exist, can be identified by diskName, diskURI, and lun.
func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
	vmName := mapNodeNameToVMName(nodeName)
	ssName, instanceID, vm, err := ss.getVmssVM(vmName)
	if err != nil {
		return err
	}

	nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName)
	if err != nil {
		return err
	}

	disks := []compute.DataDisk{}
	if vm.StorageProfile != nil && vm.StorageProfile.DataDisks != nil {
		disks = *vm.StorageProfile.DataDisks
	}
	if isManagedDisk {
		disks = append(disks,
			compute.DataDisk{
				Name:         &diskName,
				Lun:          &lun,
				Caching:      compute.CachingTypes(cachingMode),
				CreateOption: "attach",
				ManagedDisk: &compute.ManagedDiskParameters{
					ID: &diskURI,
				},
			})
	} else {
		disks = append(disks,
			compute.DataDisk{
				Name: &diskName,
				Vhd: &compute.VirtualHardDisk{
					URI: &diskURI,
				},
				Lun:          &lun,
				Caching:      compute.CachingTypes(cachingMode),
				CreateOption: "attach",
			})
	}
	vm.StorageProfile.DataDisks = &disks

	ctx, cancel := getContextWithCancel()
	defer cancel()

	// Invalidate the cache right after updating
	key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
	defer ss.vmssVMCache.Delete(key)

	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, nodeName, diskName)
	_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, vm)
	if err != nil {
		detail := err.Error()
		if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) {
			// if lease cannot be acquired or disk not found, immediately detach the disk and return the original error
			klog.Infof("azureDisk - err %s, try detach disk(%s)", detail, diskName)
			ss.DetachDiskByName(diskName, diskURI, nodeName)
		}
	} else {
		klog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName)
	}
	return err
}

// DetachDiskByName detaches a vhd from host
// the vhd can be identified by diskName or diskURI
func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
	vmName := mapNodeNameToVMName(nodeName)
	ssName, instanceID, vm, err := ss.getVmssVM(vmName)
	if err != nil {
		return err
	}

	nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName)
	if err != nil {
		return err
	}

	disks := []compute.DataDisk{}
	if vm.StorageProfile != nil && vm.StorageProfile.DataDisks != nil {
		disks = *vm.StorageProfile.DataDisks
	}
	bFoundDisk := false
	for i, disk := range disks {
		if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
			(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
			(disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) {
			// found the disk
			klog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI)
			disks = append(disks[:i], disks[i+1:]...)
			bFoundDisk = true
			break
		}
	}

	if !bFoundDisk {
		return fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI)
	}

	vm.StorageProfile.DataDisks = &disks
	ctx, cancel := getContextWithCancel()
	defer cancel()

	// Invalidate the cache right after updating
	key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
	defer ss.vmssVMCache.Delete(key)

	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName)
	_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, vm)
	if err != nil {
		klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err)
	} else {
		klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName)
	}

	return err
}

// GetDataDisks gets a list of data disks attached to the node.
func (ss *scaleSet) GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error) {
	_, _, vm, err := ss.getVmssVM(string(nodeName))
	if err != nil {
		return nil, err
	}

	if vm.StorageProfile == nil || vm.StorageProfile.DataDisks == nil {
		return nil, nil
	}

	return *vm.StorageProfile.DataDisks, nil
}
<commit_msg>fix race condition when attach azure disk in vmss<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
	"fmt"
	"strings"

	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
	"k8s.io/klog"

	"k8s.io/apimachinery/pkg/types"
)

// AttachDisk attaches a vhd to vm
// the vhd must exist, can be identified by diskName, diskURI, and lun.
func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
	vmName := mapNodeNameToVMName(nodeName)
	ssName, instanceID, vm, err := ss.getVmssVM(vmName)
	if err != nil {
		return err
	}

	nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName)
	if err != nil {
		return err
	}

	disks := []compute.DataDisk{}
	if vm.StorageProfile != nil && vm.StorageProfile.DataDisks != nil {
		disks = *vm.StorageProfile.DataDisks
	}
	if isManagedDisk {
		disks = append(disks,
			compute.DataDisk{
				Name:         &diskName,
				Lun:          &lun,
				Caching:      compute.CachingTypes(cachingMode),
				CreateOption: "attach",
				ManagedDisk: &compute.ManagedDiskParameters{
					ID: &diskURI,
				},
			})
	} else {
		disks = append(disks,
			compute.DataDisk{
				Name: &diskName,
				Vhd: &compute.VirtualHardDisk{
					URI: &diskURI,
				},
				Lun:          &lun,
				Caching:      compute.CachingTypes(cachingMode),
				CreateOption: "attach",
			})
	}
	newVM := compute.VirtualMachineScaleSetVM{
		Sku:      vm.Sku,
		Location: vm.Location,
		VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
			HardwareProfile: vm.HardwareProfile,
			StorageProfile: &compute.StorageProfile{
				OsDisk:    vm.StorageProfile.OsDisk,
				DataDisks: &disks,
			},
		},
	}

	ctx, cancel := getContextWithCancel()
	defer cancel()

	// Invalidate the cache right after updating
	key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
	defer ss.vmssVMCache.Delete(key)

	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk(%s)", nodeResourceGroup, nodeName, diskName)
	_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM)
	if err != nil {
		detail := err.Error()
		if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) {
			// if lease cannot be acquired or disk not found, immediately detach the disk and return the original error
			klog.Infof("azureDisk - err %s, try detach disk(%s)", detail, diskName)
			ss.DetachDiskByName(diskName, diskURI, nodeName)
		}
	} else {
		klog.V(2).Infof("azureDisk - attach disk(%s) succeeded", diskName)
	}
	return err
}

// DetachDiskByName detaches a vhd from host
// the vhd can be identified by diskName or diskURI
func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
	vmName := mapNodeNameToVMName(nodeName)
	ssName, instanceID, vm, err := ss.getVmssVM(vmName)
	if err != nil {
		return err
	}

	nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName)
	if err != nil {
		return err
	}

	disks := []compute.DataDisk{}
	if vm.StorageProfile != nil && vm.StorageProfile.DataDisks != nil {
		disks = *vm.StorageProfile.DataDisks
	}
	bFoundDisk := false
	for i, disk := range disks {
		if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
			(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
			(disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) {
			// found the disk
			klog.V(2).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI)
			disks = append(disks[:i], disks[i+1:]...)
			bFoundDisk = true
			break
		}
	}

	if !bFoundDisk {
		return fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI)
	}

	newVM := compute.VirtualMachineScaleSetVM{
		Sku:      vm.Sku,
		Location: vm.Location,
		VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
			HardwareProfile: vm.HardwareProfile,
			StorageProfile: &compute.StorageProfile{
				OsDisk:    vm.StorageProfile.OsDisk,
				DataDisks: &disks,
			},
		},
	}

	ctx, cancel := getContextWithCancel()
	defer cancel()

	// Invalidate the cache right after updating
	key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID))
	defer ss.vmssVMCache.Delete(key)

	klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName)
	_, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM)
	if err != nil {
		klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err)
	} else {
		klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName)
	}

	return err
}

// GetDataDisks gets a list of data disks attached to the node.
func (ss *scaleSet) GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error) {
	_, _, vm, err := ss.getVmssVM(string(nodeName))
	if err != nil {
		return nil, err
	}

	if vm.StorageProfile == nil || vm.StorageProfile.DataDisks == nil {
		return nil, nil
	}

	return *vm.StorageProfile.DataDisks, nil
}
<|endoftext|>


<commit_before>package tabular

import (
	"fmt"
	"log"
	"math"
	"regexp"
)

// This is a lot of helper functions. To keep things straight, here's a diagram
// of where they're used.
// mean -> sqrDiff -> variance -> stdDev -> normal -> chauvenet -> ProbabalisticSplit
// [fork]          -> chauvenet -> ProbabalisticSplit

// meanInt uses sum to find the mean of a list of ints.
func meanInt(data []int) float64 {
	// sumInt finds the sum of a list of ints. Exciting, I know.
	sumInt := func(data []int) (sum int) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return float64(sumInt(data)) / float64(len(data))
}

// meanFloat uses sum to find the mean of a list of float.
func meanFloat(data []float64) float64 {
	// sumFloat finds the sum of a list of floats. Exciting, I know.
	sumFloat := func(data []float64) (sum float64) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return sumFloat(data) / float64(len(data))
}

// getSquaredDifferences returns a list of the squared differences between each
// point and the mean. Used in variance and chauvenet->highestVarianceIndex
func getSquaredDifferences(data []int) (sqrDiffs []float64) {
	xbar := meanInt(data)
	for _, xn := range data {
		sqrDiff := math.Pow((xbar - float64(xn)), 2)
		sqrDiffs = append(sqrDiffs, sqrDiff)
	}
	return sqrDiffs
}

// variance is the mean of the squared differences between each observed
// point and the mean (xbar)
func variance(data []int) float64 {
	return meanFloat(getSquaredDifferences(data))
}

// stdDev returns the standard deviation of a data set of integers
func stdDev(data []int) float64 {
	return math.Sqrt(variance(data))
}

// normalDistribution returns the probability that the data point x
// lands where it does, based on the mean (mu) and standard deviation (sigma)
func normalDistribution(x float64, mu float64, sigma float64) float64 {
	coefficient := 1 / (sigma * math.Sqrt(2*math.Pi))
	exponent := -(math.Pow(x-mu, 2) / (2 * math.Pow(sigma, 2)))
	return coefficient * math.Pow(math.E, exponent)
}

// compare is a function that compares to floats. It is used for sorting and
// finding - see extremaIndex
type compare func(x float64, y float64) bool

// maxFunc is an instance of compare that can be used to find the greater
// of two values
var maxFunc compare = func(x float64, y float64) bool { return x > y }

// minFunc is like maxFunc, but wiht the lesser of the two values
var minFunc compare = func(x float64, y float64) bool { return x < y }

// extremaIndex finds the index of a value in a list that when compared with
// the any of other data with comparisonFunc will return true. It is most easily
// applicable in finding maxes and mins
func extremaIndex(comparisonFunc compare, data []float64) (index int) {
	if len(data) < 1 {
		return 0
	}
	extrema := data[0]
	extremaIndex := 0
	for i, datum := range data {
		if comparisonFunc(datum, extrema) {
			extrema = datum
			extremaIndex = i
		}
	}
	return extremaIndex
}

// chauvenet simply takes a slice of integers, applies Chauvenet's Criterion,
// and potentially discards a single outlier. Not necessarily, though!
// https://en.wikipedia.org/wiki/Chauvenet%27s_criterion
func chauvenet(data []int) (result []int) {
	// isOutlier applies chauvenet's criterion to determine whether or not
	// x is an outlier
	isOutlier := func(x float64, data []int) bool {
		xbar := meanInt(data)
		sigma := stdDev(data)
		probability := normalDistribution(x, xbar, sigma)
		if probability < float64(1/(2*len(data))) {
			return true
		}
		return false
	}
	// if its an outlier, cut it out. If not, leave it in.
	// find the index with the highest variance
	index := extremaIndex(maxFunc, getSquaredDifferences(data))
	potentialOutlier := float64(data[index])
	// test if that datum is an outlier
	if isOutlier(potentialOutlier, data) {
		return append(data[:index], data[index+1:]...)
	}
	return data
}

// ProbabalisticSplit splits a string based on the regexp that gives the most
// consistent line length (potentially discarding one outlier line length).
func ProbabalisticSplit(str string) (output Table) {
	// allEqual checks to see if all of the given list of integers are the same
	allEqual := func(ints []int) bool {
		if len(ints) < 1 {
			return true
		}
		sentinel := ints[0]
		for _, i := range ints {
			if i != sentinel {
				return false
			}
		}
		return true
	}
	// matchesMost is used to ensure that our regexp actually is splitting the
	// lines of a table, instead of just returning them whole.
	matchesMost := func(re *regexp.Regexp, rows []string) bool {
		count := 0
		for _, row := range rows {
			if re.MatchString(row) {
				count++
			}
		}
		return count > (len(rows) / 2)
	}
	// getRowLengths returns row length counts for each table
	getRowLengths := func(tables []Table) (rowLengths [][]int) {
		for _, table := range tables {
			var lengths []int
			for _, row := range table {
				lengths = append(lengths, len(row))
			}
			rowLengths = append(rowLengths, lengths)
		}
		return rowLengths
	}

	// getColumnRegex is the core of the logic. It determines which regex most
	// accurately splits the data into columns by testing the deviation in the
	// row lengths using different regexps.
	getColumnRegex := func(str string, rowSep *regexp.Regexp) *regexp.Regexp {
		// different column separators to try out
		initialColSeps := []*regexp.Regexp{
			regexp.MustCompile("\\s+"),    // any whitespace
			regexp.MustCompile("\\s{2,}"), // two+ whitespace (spaces in cols)
			regexp.MustCompile("\\s{4}"),  // exactly four whitespaces
			regexp.MustCompile("\\t+"),    // tabs
		}
		// filter regexps that have no matches at all - they will always return
		// rows of even length (length 1).
		colSeps := []*regexp.Regexp{}
		rows := rowSep.Split(str, -1)
		for _, re := range initialColSeps {
			if matchesMost(re, rows) {
				colSeps = append(colSeps, re)
			}
		}
		if len(colSeps) < 1 {
			msg := "ProbabalisticSplit couldn't divide the table."
			msg += "\n\tAttempted regexps: " + fmt.Sprint(initialColSeps)
			log.Fatal(msg)
		}
		// separate the data based on the above column regexps
		var tables []Table
		for _, colSep := range colSeps {
			table := SeparateString(rowSep, colSep, str)
			tables = append(tables, table)
		}
		// see if any of them had utterly consistent row lengths
		rowLengths := getRowLengths(tables)
		if len(rowLengths) != len(tables) {
			log.Fatal("Internal error: len(rowLengths) != len(tables)")
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) {
				return colSeps[i]
			}
		}
		// if not, cast out outliers and try again
		for i, lengths := range rowLengths {
			rowLengths[i] = chauvenet(lengths)
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) {
				return colSeps[i]
			}
		}
		// if still not done, just pick the one with the lowest variance
		var variances []float64
		for _, lengths := range rowLengths {
			variances = append(variances, variance(lengths))
		}
		// ensure that index can be found in tables
		minVarianceIndex := extremaIndex(minFunc, variances)
		if len(tables) <= minVarianceIndex {
			log.Fatal("Internal error: minVarianceIndex couldn't be found in tables")
		}
		return colSeps[minVarianceIndex]
	}
	rowSep := regexp.MustCompile("\n+")
	colSep := getColumnRegex(str, rowSep)
	//fmt.Println("THE CHOSEN REGEX: " + colSep.String())
	return SeparateString(rowSep, colSep, str)
}
<commit_msg>Replaced log with logrus in tabular<commit_after>package tabular

import (
	log "github.com/Sirupsen/logrus"
	"math"
	"regexp"
)

// This is a lot of helper functions. To keep things straight, here's a diagram
// of where they're used.
// mean -> sqrDiff -> variance -> stdDev -> normal -> chauvenet -> ProbabalisticSplit
// [fork]          -> chauvenet -> ProbabalisticSplit

// meanInt uses sum to find the mean of a list of ints.
func meanInt(data []int) float64 {
	// sumInt finds the sum of a list of ints. Exciting, I know.
	sumInt := func(data []int) (sum int) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return float64(sumInt(data)) / float64(len(data))
}

// meanFloat uses sum to find the mean of a list of float.
func meanFloat(data []float64) float64 {
	// sumFloat finds the sum of a list of floats. Exciting, I know.
	sumFloat := func(data []float64) (sum float64) {
		for _, i := range data {
			sum += i
		}
		return sum
	}
	return sumFloat(data) / float64(len(data))
}

// getSquaredDifferences returns a list of the squared differences between each
// point and the mean. Used in variance and chauvenet->highestVarianceIndex
func getSquaredDifferences(data []int) (sqrDiffs []float64) {
	xbar := meanInt(data)
	for _, xn := range data {
		sqrDiff := math.Pow((xbar - float64(xn)), 2)
		sqrDiffs = append(sqrDiffs, sqrDiff)
	}
	return sqrDiffs
}

// variance is the mean of the squared differences between each observed
// point and the mean (xbar)
func variance(data []int) float64 {
	return meanFloat(getSquaredDifferences(data))
}

// stdDev returns the standard deviation of a data set of integers
func stdDev(data []int) float64 {
	return math.Sqrt(variance(data))
}

// normalDistribution returns the probability that the data point x
// lands where it does, based on the mean (mu) and standard deviation (sigma)
func normalDistribution(x float64, mu float64, sigma float64) float64 {
	coefficient := 1 / (sigma * math.Sqrt(2*math.Pi))
	exponent := -(math.Pow(x-mu, 2) / (2 * math.Pow(sigma, 2)))
	return coefficient * math.Pow(math.E, exponent)
}

// compare is a function that compares to floats. It is used for sorting and
// finding - see extremaIndex
type compare func(x float64, y float64) bool

// maxFunc is an instance of compare that can be used to find the greater
// of two values
var maxFunc compare = func(x float64, y float64) bool { return x > y }

// minFunc is like maxFunc, but wiht the lesser of the two values
var minFunc compare = func(x float64, y float64) bool { return x < y }

// extremaIndex finds the index of a value in a list that when compared with
// the any of other data with comparisonFunc will return true. It is most easily
// applicable in finding maxes and mins
func extremaIndex(comparisonFunc compare, data []float64) (index int) {
	if len(data) < 1 {
		return 0
	}
	extrema := data[0]
	extremaIndex := 0
	for i, datum := range data {
		if comparisonFunc(datum, extrema) {
			extrema = datum
			extremaIndex = i
		}
	}
	return extremaIndex
}

// chauvenet simply takes a slice of integers, applies Chauvenet's Criterion,
// and potentially discards a single outlier. Not necessarily, though!
// https://en.wikipedia.org/wiki/Chauvenet%27s_criterion
func chauvenet(data []int) (result []int) {
	// isOutlier applies chauvenet's criterion to determine whether or not
	// x is an outlier
	isOutlier := func(x float64, data []int) bool {
		xbar := meanInt(data)
		sigma := stdDev(data)
		probability := normalDistribution(x, xbar, sigma)
		if probability < float64(1/(2*len(data))) {
			return true
		}
		return false
	}
	// if its an outlier, cut it out. If not, leave it in.
	// find the index with the highest variance
	index := extremaIndex(maxFunc, getSquaredDifferences(data))
	potentialOutlier := float64(data[index])
	// test if that datum is an outlier
	if isOutlier(potentialOutlier, data) {
		return append(data[:index], data[index+1:]...)
	}
	return data
}

// ProbabalisticSplit splits a string based on the regexp that gives the most
// consistent line length (potentially discarding one outlier line length).
func ProbabalisticSplit(str string) (output Table) {
	// allEqual checks to see if all of the given list of integers are the same
	allEqual := func(ints []int) bool {
		if len(ints) < 1 {
			return true
		}
		sentinel := ints[0]
		for _, i := range ints {
			if i != sentinel {
				return false
			}
		}
		return true
	}
	// matchesMost is used to ensure that our regexp actually is splitting the
	// lines of a table, instead of just returning them whole.
	matchesMost := func(re *regexp.Regexp, rows []string) bool {
		count := 0
		for _, row := range rows {
			if re.MatchString(row) {
				count++
			}
		}
		return count > (len(rows) / 2)
	}
	// getRowLengths returns row length counts for each table
	getRowLengths := func(tables []Table) (rowLengths [][]int) {
		for _, table := range tables {
			var lengths []int
			for _, row := range table {
				lengths = append(lengths, len(row))
			}
			rowLengths = append(rowLengths, lengths)
		}
		return rowLengths
	}

	// getColumnRegex is the core of the logic. It determines which regex most
	// accurately splits the data into columns by testing the deviation in the
	// row lengths using different regexps.
	getColumnRegex := func(str string, rowSep *regexp.Regexp) *regexp.Regexp {
		// different column separators to try out
		initialColSeps := []*regexp.Regexp{
			regexp.MustCompile("\\s+"),    // any whitespace
			regexp.MustCompile("\\s{2,}"), // two+ whitespace (spaces in cols)
			regexp.MustCompile("\\s{4}"),  // exactly four whitespaces
			regexp.MustCompile("\\t+"),    // tabs
		}
		// filter regexps that have no matches at all - they will always return
		// rows of even length (length 1).
		colSeps := []*regexp.Regexp{}
		rows := rowSep.Split(str, -1)
		for _, re := range initialColSeps {
			if matchesMost(re, rows) {
				colSeps = append(colSeps, re)
			}
		}
		if len(colSeps) < 1 {
			log.WithFields(log.Fields{
				"attempted": initialColSeps,
			}).Fatal("ProbabalisticSplit couldn't divide the table.")
		}
		// separate the data based on the above column regexps
		var tables []Table
		for _, colSep := range colSeps {
			table := SeparateString(rowSep, colSep, str)
			tables = append(tables, table)
		}
		// see if any of them had utterly consistent row lengths
		rowLengths := getRowLengths(tables)
		if len(rowLengths) != len(tables) {
			log.WithFields(log.Fields{
				"rowLengths": rowLengths,
				"tables":     tables,
			}).Fatal("Internal error: len(rowLengths) != len(tables)")
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) {
				return colSeps[i]
			}
		}
		// if not, cast out outliers and try again
		for i, lengths := range rowLengths {
			rowLengths[i] = chauvenet(lengths)
		}
		for i, lengths := range rowLengths {
			if allEqual(lengths) {
				return colSeps[i]
			}
		}
		// if still not done, just pick the one with the lowest variance
		var variances []float64
		for _, lengths := range rowLengths {
			variances = append(variances, variance(lengths))
		}
		// ensure that index can be found in tables
		minVarianceIndex := extremaIndex(minFunc, variances)
		if len(tables) <= minVarianceIndex {
			log.Fatal("Internal error: minVarianceIndex couldn't be found in tables")
		}
		return colSeps[minVarianceIndex]
	}
	rowSep := regexp.MustCompile("\n+")
	colSep := getColumnRegex(str, rowSep)
	//fmt.Println("THE CHOSEN REGEX: " + colSep.String())
	return SeparateString(rowSep, colSep, str)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vppcalls

import (
	"fmt"

	"time"

	"github.com/ligato/cn-infra/logging/measure"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/bin_api/interfaces"
)

// InterfaceAdminDown calls binary API SwInterfaceSetFlagsReply with AdminUpDown=0.
func InterfaceAdminDown(ifIdx uint32, vppChan VPPChannel, timeLog *measure.TimeLog) error {
	// SwInterfaceSetFlags time measurement
	start := time.Now()
	defer func() {
		if timeLog != nil {
			timeLog.LogTimeEntry(time.Since(start))
		}
	}()

	// Prepare the message.
	req := &interfaces.SwInterfaceSetFlags{}
	req.SwIfIndex = ifIdx
	req.AdminUpDown = 0

	reply := &interfaces.SwInterfaceSetFlagsReply{}
	err := vppChan.SendRequest(req).ReceiveReply(reply)
	if err != nil {
		return err
	}
	if 0 != reply.Retval {
		return fmt.Errorf("setting of interface flags returned %d", reply.Retval)
	}

	return nil
}

// InterfaceAdminUp calls binary API SwInterfaceSetFlagsReply with AdminUpDown=1.
func InterfaceAdminUp(ifIdx uint32, vppChan VPPChannel, timeLog measure.StopWatchEntry) error {
	// SwInterfaceSetFlags time measurement
	start := time.Now()
	defer func() {
		if timeLog != nil {
			timeLog.LogTimeEntry(time.Since(start))
		}
	}()

	// Prepare the message.
	req := &interfaces.SwInterfaceSetFlags{}
	req.SwIfIndex = ifIdx
	req.AdminUpDown = 1

	reply := &interfaces.SwInterfaceSetFlagsReply{}
	err := vppChan.SendRequest(req).ReceiveReply(reply)
	if err != nil {
		return err
	}
	if 0 != reply.Retval {
		return fmt.Errorf("setting of interface flags returned %d", reply.Retval)
	}

	return nil

}
<commit_msg>Include relevant request info in error message<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vppcalls

import (
	"fmt"

	"time"

	"github.com/ligato/cn-infra/logging/measure"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/bin_api/interfaces"
)

// InterfaceAdminDown calls binary API SwInterfaceSetFlagsReply with AdminUpDown=0.
func InterfaceAdminDown(ifIdx uint32, vppChan VPPChannel, timeLog measure.StopWatchEntry) error {
	// SwInterfaceSetFlags time measurement
	start := time.Now()
	defer func() {
		if timeLog != nil {
			timeLog.LogTimeEntry(time.Since(start))
		}
	}()

	return interfaceSetFlags(ifIdx, false, vppChan)
}

// InterfaceAdminUp calls binary API SwInterfaceSetFlagsReply with AdminUpDown=1.
func InterfaceAdminUp(ifIdx uint32, vppChan VPPChannel, timeLog measure.StopWatchEntry) error {
	// SwInterfaceSetFlags time measurement
	start := time.Now()
	defer func() {
		if timeLog != nil {
			timeLog.LogTimeEntry(time.Since(start))
		}
	}()

	return interfaceSetFlags(ifIdx, true, vppChan)
}

func interfaceSetFlags(ifIdx uint32, adminUp bool, vppChan VPPChannel) error {
	// Prepare the message.
	req := &interfaces.SwInterfaceSetFlags{
		SwIfIndex: ifIdx,
	}
	if adminUp {
		req.AdminUpDown = 1
	} else {
		req.AdminUpDown = 0
	}

	reply := &interfaces.SwInterfaceSetFlagsReply{}
	err := vppChan.SendRequest(req).ReceiveReply(reply)
	if err != nil {
		return err
	}
	if reply.Retval != 0 {
		return fmt.Errorf("setting of interface flags (%+v) returned %d", req, reply.Retval)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gc

// This file contains utility routines and harness infrastructure used
// by the ABI tests in "abiutils_test.go".

import (
	"cmd/compile/internal/ir"
	"cmd/compile/internal/types"
	"cmd/internal/src"
	"fmt"
	"strings"
	"testing"
	"text/scanner"
)

func mkParamResultField(t *types.Type, s *types.Sym, which ir.Class) *types.Field {
	field := types.NewField(src.NoXPos, s, t)
	n := NewName(s)
	n.SetClass(which)
	field.Nname = n
	n.SetType(t)
	return field
}

// mkstruct is a helper routine to create a struct type with fields
// of the types specified in 'fieldtypes'.
func mkstruct(fieldtypes []*types.Type) *types.Type {
	fields := make([]*types.Field, len(fieldtypes))
	for k, t := range fieldtypes {
		if t == nil {
			panic("bad -- field has no type")
		}
		f := types.NewField(src.NoXPos, nil, t)
		fields[k] = f
	}
	s := types.NewStruct(types.LocalPkg, fields)
	return s
}

func mkFuncType(rcvr *types.Type, ins []*types.Type, outs []*types.Type) *types.Type {
	q := lookup("?")
	inf := []*types.Field{}
	for _, it := range ins {
		inf = append(inf, mkParamResultField(it, q, ir.PPARAM))
	}
	outf := []*types.Field{}
	for _, ot := range outs {
		outf = append(outf, mkParamResultField(ot, q, ir.PPARAMOUT))
	}
	var rf *types.Field
	if rcvr != nil {
		rf = mkParamResultField(rcvr, q, ir.PPARAM)
	}
	return types.NewSignature(types.LocalPkg, rf, inf, outf)
}

type expectedDump struct {
	dump string
	file string
	line int
}

func tokenize(src string) []string {
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	res := []string{}
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		res = append(res, s.TokenText())
	}
	return res
}

func verifyParamResultOffset(t *testing.T, f *types.Field, r ABIParamAssignment, which string, idx int) int {
	n := ir.AsNode(f.Nname)
	if n == nil {
		panic("not expected")
	}
	if n.Offset() != int64(r.Offset) {
		t.Errorf("%s %d: got offset %d wanted %d t=%v",
			which, idx, r.Offset, n.Offset(), f.Type)
		return 1
	}
	return 0
}

func makeExpectedDump(e string) expectedDump {
	return expectedDump{dump: e}
}

func difftokens(atoks []string, etoks []string) string {
	if len(atoks) != len(etoks) {
		return fmt.Sprintf("expected %d tokens got %d",
			len(etoks), len(atoks))
	}
	for i := 0; i < len(etoks); i++ {
		if etoks[i] == atoks[i] {
			continue
		}

		return fmt.Sprintf("diff at token %d: expected %q got %q",
			i, etoks[i], atoks[i])
	}
	return ""
}

func abitest(t *testing.T, ft *types.Type, exp expectedDump) {

	dowidth(ft)

	// Analyze with full set of registers.
	regRes := ABIAnalyze(ft, configAMD64)
	regResString := strings.TrimSpace(regRes.String())

	// Check results.
	reason := difftokens(tokenize(regResString), tokenize(exp.dump))
	if reason != "" {
		t.Errorf("\nexpected:\n%s\ngot:\n%s\nreason: %s",
			strings.TrimSpace(exp.dump), regResString, reason)
	}

	// Analyze again with empty register set.
	empty := ABIConfig{}
	emptyRes := ABIAnalyze(ft, empty)
	emptyResString := emptyRes.String()

	// Walk the results and make sure the offsets assigned match
	// up with those assiged by dowidth. This checks to make sure that
	// when we have no available registers the ABI assignment degenerates
	// back to the original ABI0.

	// receiver
	failed := 0
	rfsl := ft.Recvs().Fields().Slice()
	poff := 0
	if len(rfsl) != 0 {
		failed |= verifyParamResultOffset(t, rfsl[0], emptyRes.inparams[0], "receiver", 0)
		poff = 1
	}
	// params
	pfsl := ft.Params().Fields().Slice()
	for k, f := range pfsl {
		verifyParamResultOffset(t, f, emptyRes.inparams[k+poff], "param", k)
	}
	// results
	ofsl := ft.Results().Fields().Slice()
	for k, f := range ofsl {
		failed |= verifyParamResultOffset(t, f, emptyRes.outparams[k], "result", k)
	}

	if failed != 0 {
		t.Logf("emptyres:\n%s\n", emptyResString)
	}
}
<commit_msg>[dev.regabi] cmd/compile: add type assertion in regabi test<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gc

// This file contains utility routines and harness infrastructure used
// by the ABI tests in "abiutils_test.go".

import (
	"cmd/compile/internal/ir"
	"cmd/compile/internal/types"
	"cmd/internal/src"
	"fmt"
	"strings"
	"testing"
	"text/scanner"
)

func mkParamResultField(t *types.Type, s *types.Sym, which ir.Class) *types.Field {
	field := types.NewField(src.NoXPos, s, t)
	n := NewName(s)
	n.SetClass(which)
	field.Nname = n
	n.SetType(t)
	return field
}

// mkstruct is a helper routine to create a struct type with fields
// of the types specified in 'fieldtypes'.
func mkstruct(fieldtypes []*types.Type) *types.Type {
	fields := make([]*types.Field, len(fieldtypes))
	for k, t := range fieldtypes {
		if t == nil {
			panic("bad -- field has no type")
		}
		f := types.NewField(src.NoXPos, nil, t)
		fields[k] = f
	}
	s := types.NewStruct(types.LocalPkg, fields)
	return s
}

func mkFuncType(rcvr *types.Type, ins []*types.Type, outs []*types.Type) *types.Type {
	q := lookup("?")
	inf := []*types.Field{}
	for _, it := range ins {
		inf = append(inf, mkParamResultField(it, q, ir.PPARAM))
	}
	outf := []*types.Field{}
	for _, ot := range outs {
		outf = append(outf, mkParamResultField(ot, q, ir.PPARAMOUT))
	}
	var rf *types.Field
	if rcvr != nil {
		rf = mkParamResultField(rcvr, q, ir.PPARAM)
	}
	return types.NewSignature(types.LocalPkg, rf, inf, outf)
}

type expectedDump struct {
	dump string
	file string
	line int
}

func tokenize(src string) []string {
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	res := []string{}
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		res = append(res, s.TokenText())
	}
	return res
}

func verifyParamResultOffset(t *testing.T, f *types.Field, r ABIParamAssignment, which string, idx int) int {
	n := ir.AsNode(f.Nname).(*ir.Name)
	if n.Offset() != int64(r.Offset) {
		t.Errorf("%s %d: got offset %d wanted %d t=%v",
			which, idx, r.Offset, n.Offset(), f.Type)
		return 1
	}
	return 0
}

func makeExpectedDump(e string) expectedDump {
	return expectedDump{dump: e}
}

func difftokens(atoks []string, etoks []string) string {
	if len(atoks) != len(etoks) {
		return fmt.Sprintf("expected %d tokens got %d",
			len(etoks), len(atoks))
	}
	for i := 0; i < len(etoks); i++ {
		if etoks[i] == atoks[i] {
			continue
		}

		return fmt.Sprintf("diff at token %d: expected %q got %q",
			i, etoks[i], atoks[i])
	}
	return ""
}

func abitest(t *testing.T, ft *types.Type, exp expectedDump) {

	dowidth(ft)

	// Analyze with full set of registers.
	regRes := ABIAnalyze(ft, configAMD64)
	regResString := strings.TrimSpace(regRes.String())

	// Check results.
	reason := difftokens(tokenize(regResString), tokenize(exp.dump))
	if reason != "" {
		t.Errorf("\nexpected:\n%s\ngot:\n%s\nreason: %s",
			strings.TrimSpace(exp.dump), regResString, reason)
	}

	// Analyze again with empty register set.
	empty := ABIConfig{}
	emptyRes := ABIAnalyze(ft, empty)
	emptyResString := emptyRes.String()

	// Walk the results and make sure the offsets assigned match
	// up with those assiged by dowidth. This checks to make sure that
	// when we have no available registers the ABI assignment degenerates
	// back to the original ABI0.

	// receiver
	failed := 0
	rfsl := ft.Recvs().Fields().Slice()
	poff := 0
	if len(rfsl) != 0 {
		failed |= verifyParamResultOffset(t, rfsl[0], emptyRes.inparams[0], "receiver", 0)
		poff = 1
	}
	// params
	pfsl := ft.Params().Fields().Slice()
	for k, f := range pfsl {
		verifyParamResultOffset(t, f, emptyRes.inparams[k+poff], "param", k)
	}
	// results
	ofsl := ft.Results().Fields().Slice()
	for k, f := range ofsl {
		failed |= verifyParamResultOffset(t, f, emptyRes.outparams[k], "result", k)
	}

	if failed != 0 {
		t.Logf("emptyres:\n%s\n", emptyResString)
	}
}
<|endoftext|>


<commit_before>// +build windows

package enforcer

import (
	"fmt"
	"sync"

	"github.com/gentlemanautomaton/winsession"
	"github.com/gentlemanautomaton/winsession/connstate"
)

// SessionManager manages communication with any number of windows sessions.
//
// Its zero value is ready for use.
type SessionManager struct {
	command Command
	logger  Logger

	mutex   sync.RWMutex
	managed map[SessionID]*Session
}

// NewSessionManager returns a new session manager that is ready for use.
//
// The given command describes what process to launch within each session.
func NewSessionManager(cmd Command, logger Logger) *SessionManager {
	return &SessionManager{
		command: cmd,
		logger:  logger,
		managed: make(map[SessionID]*Session, 8),
	}
}

// Session returns the managed session with id, if one exists.
func (m *SessionManager) Session(id SessionID) *Session {
	m.mutex.RLock()
	defer m.mutex.RUnlock()
	return m.managed[id]
}

// Scan causes the service to rescan the current sessions.
func (m *SessionManager) Scan() error {
	sessions, err := winsession.Local.Sessions(
		winsession.Exclude(winsession.MatchID(0)),
		winsession.Include(winsession.MatchState(connstate.Active, connstate.Disconnected)),
		winsession.CollectSessionInfo,
	)
	if err != nil {
		return nil
	}

	scanned := make(map[winsession.ID]struct{}, len(sessions))

	m.mutex.Lock()
	defer m.mutex.Unlock()

	var pending []SessionData
	for _, session := range sessions {
		id := session.ID
		scanned[id] = struct{}{} // Record the ID in the map of scanned sessions

		// Don't re-process sessions that are already managed
		if _, exists := m.managed[id]; exists {
			continue
		}

		pending = append(pending, session)
	}

	// Bookkeeping for dead sessions
	for id, session := range m.managed {
		if _, exists := scanned[id]; !exists || !session.Connected() {
			go session.Disconnect()
			delete(m.managed, id)
			m.log("Stopped management of session %d", id)
		}
	}

	// Establish a connection with newly discovered sessions
	for _, data := range pending {
		data := data
		session := NewSession(data, m.command, 64, m.logger)
		m.managed[data.ID] = session
		go func() {
			if err := session.Connect(); err != nil {
				// TODO: Try again?
				m.mutex.Lock()
				defer m.mutex.Unlock()
				if m.managed[data.ID] == session {
					delete(m.managed, data.ID)
				}
			}

			// Send the current policy set to the session
			/*
				mgr.Send(enforcerui.Message{
					Type: "policy.change",
					PolicyChange: enforcerui.PolicyChange{
						New: m.Policies(),
					},
				})
			*/
		}()
	}

	return nil
}

/*
// Send sends the given message to the ui process running in each session.
func (s *Service) Send(msg enforcerui.Message) {
	s.sessionMutex.Lock()
	defer s.sessionMutex.Unlock()
	for _, session := range s.sessions {
		session.Send(msg)
	}
}
*/

// Stop causes the session manager to stop all session management.
func (m *SessionManager) Stop() {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	for id, session := range m.managed {
		session.Disconnect()
		delete(m.managed, id)
		m.log("Stopped management of session %d", session.data.ID)
	}
}

func (m *SessionManager) log(format string, v ...interface{}) {
	if m.logger == nil {
		return
	}
	m.logger.Log(ServiceEvent{
		Msg: fmt.Sprintf(format, v...),
	})
}

func (m *SessionManager) debug(format string, v ...interface{}) {
	if m.logger == nil {
		return
	}
	m.logger.Log(ServiceEvent{
		Msg:   fmt.Sprintf(format, v...),
		Debug: true,
	})
}
<commit_msg>enforcer: SessionManager now backs off after failed connection attempts<commit_after>// +build windows

package enforcer

import (
	"fmt"
	"sync"
	"time"

	"github.com/gentlemanautomaton/winsession"
	"github.com/gentlemanautomaton/winsession/connstate"
	"github.com/scjalliance/resourceful/enforcerui"
)

// sessionAttempt records information about the last time a connection
// was attempted with a session
type sessionAttempt struct {
	Attempt time.Time
	Wait    time.Duration
}

// SessionManager manages communication with any number of windows sessions.
//
// Its zero value is ready for use.
type SessionManager struct {
	command Command
	logger  Logger

	mutex     sync.RWMutex
	managed   map[SessionID]*Session
	attempted map[SessionID]sessionAttempt
}

// NewSessionManager returns a new session manager that is ready for use.
//
// The given command describes what process to launch within each session.
func NewSessionManager(cmd Command, logger Logger) *SessionManager {
	return &SessionManager{
		command:   cmd,
		logger:    logger,
		managed:   make(map[SessionID]*Session, 8),
		attempted: make(map[SessionID]sessionAttempt, 8),
	}
}

// Session returns the managed session with id, if one exists.
func (m *SessionManager) Session(id SessionID) *Session {
	m.mutex.RLock()
	defer m.mutex.RUnlock()
	return m.managed[id]
}

// Scan causes the service to rescan the current sessions.
func (m *SessionManager) Scan() error {
	sessions, err := winsession.Local.Sessions(
		winsession.Exclude(winsession.MatchID(0)),
		winsession.Include(winsession.MatchState(connstate.Active, connstate.Disconnected)),
		winsession.CollectSessionInfo,
	)
	if err != nil {
		return nil
	}

	now := time.Now()
	scanned := make(map[winsession.ID]struct{}, len(sessions))

	m.mutex.Lock()
	defer m.mutex.Unlock()

	var pending []SessionData
	for _, session := range sessions {
		id := session.ID
		scanned[id] = struct{}{} // Record the ID in the map of scanned sessions

		// Don't re-process sessions that are already managed
		if _, exists := m.managed[id]; exists {
			continue
		}

		// Wait before retrying connections that previously failed
		if attempt, ok := m.attempted[id]; ok {
			if now.Sub(attempt.Attempt) < attempt.Wait {
				continue
			}
		}

		pending = append(pending, session)
	}

	// Bookkeeping for dead sessions
	for id, session := range m.managed {
		if _, exists := scanned[id]; !exists || !session.Connected() {
			go session.Disconnect()
			delete(m.managed, id)
			m.log("Stopped management of session %d", id)
		}
	}

	for id := range m.attempted {
		if _, exists := scanned[id]; !exists {
			delete(m.attempted, id)
		}
	}

	// Establish a connection with newly discovered sessions
	for _, data := range pending {
		data := data
		id := data.ID
		session := NewSession(data, m.command, 64, m.logger)
		m.managed[id] = session
		go func() {
			now := time.Now()
			err := session.Connect()

			m.mutex.Lock()
			if err != nil {
				if m.managed[id] == session {
					delete(m.managed, id)
					m.attempted[id] = nextSessionAttempt(m.attempted[id], now)
				}
			} else {
				delete(m.attempted, id)
			}
			m.mutex.Unlock()

			if err != nil {
				return
			}

			// Send the current policy set to the session
			/*
				session.Send(enforcerui.Message{
					Type: "policy.change",
					PolicyChange: enforcerui.PolicyChange{
						New: m.Policies(),
					},
				})
			*/
		}()
	}

	return nil
}

// Send sends the given message to the ui process running in each session.
func (m *SessionManager) Send(msg enforcerui.Message) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	for _, session := range m.managed {
		session.Send(msg)
	}
}

// Stop causes the session manager to stop all session management.
func (m *SessionManager) Stop() {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	for id, session := range m.managed {
		session.Disconnect()
		delete(m.managed, id)
		m.log("Stopped management of session %d", session.data.ID)
	}
}

func (m *SessionManager) log(format string, v ...interface{}) {
	if m.logger == nil {
		return
	}
	m.logger.Log(ServiceEvent{
		Msg: fmt.Sprintf(format, v...),
	})
}

func (m *SessionManager) debug(format string, v ...interface{}) {
	if m.logger == nil {
		return
	}
	m.logger.Log(ServiceEvent{
		Msg:   fmt.Sprintf(format, v...),
		Debug: true,
	})
}

// nextSessionAttempt calculates the retry backoff after failed session
// connection attempts.
func nextSessionAttempt(last sessionAttempt, now time.Time) (next sessionAttempt) {
	const (
		minWait = 30 * time.Second
		maxWait = 5 * time.Minute
	)

	if last.Attempt.IsZero() {
		return sessionAttempt{
			Attempt: now,
			Wait:    minWait,
		}
	}

	wait := last.Wait
	wait *= 2
	if wait > maxWait {
		wait = maxWait
	}
	return sessionAttempt{
		Attempt: now,
		Wait:    wait,
	}
}
<|endoftext|>


<commit_before>package information

import (
	"strings"

	"github.com/Necroforger/Fantasia/system"
	"github.com/Necroforger/discordgo"
	"github.com/Necroforger/dream"
)

// Module ...
type Module struct{}

// Build adds this modules commands to the system's router
func (m *Module) Build(s *system.System) {
	r := s.CommandRouter
	r.SetCategory("information")

	r.On("help", m.Help).Set("", "help", "displays a help menu with the available commands")
}

// Help maps a list of available commands and descends into subrouters.
func (m *Module) Help(ctx *system.Context) {

	_, err := ctx.ReplyEmbed(depthcharge(ctx.System.CommandRouter, nil, 0).
		SetColor(system.StatusNotify).
		SetThumbnail(ctx.Ses.DG.State.User.AvatarURL("2048")).
		InlineAllFields().
		SetDescription("subcommands are represented by indentation.").
		MessageEmbed)
	if err != nil {
		ctx.ReplyError(err)
	}
}

func depthcharge(r *system.CommandRouter, embed *dream.Embed, depth int) *dream.Embed {
	if embed == nil {
		embed = dream.NewEmbed()
	}

	depthString := func(text string, depth int, subrouter bool) string {
		quote := ""
		if subrouter {
			quote = "`"
		}
		return strings.Repeat("  ", depth) + quote + text + quote + "\n"
	}

	getField := func(name string) *discordgo.MessageEmbedField {
		for _, v := range embed.Fields {
			if v.Name == name {
				return v
			}
		}
		if name == "" {
			name = "undefined"
		}
		field := &discordgo.MessageEmbedField{Name: name}
		embed.Fields = append(embed.Fields, field)
		return field
	}

	for _, v := range r.Routes {
		field := getField(v.Category)
		field.Value += depthString(v.Name, depth, false)
	}

	for _, v := range r.Subrouters {
		field := getField(v.Category())
		field.Value += depthString(v.Name, depth, true)
		embed = depthcharge(v.Router, embed, depth+1)
	}

	return embed
}
<commit_msg>fix help<commit_after>package information

import (
	"strings"

	"github.com/Necroforger/Fantasia/system"
	"github.com/Necroforger/discordgo"
	"github.com/Necroforger/dream"
)

// Module ...
type Module struct{}

// Build adds this modules commands to the system's router
func (m *Module) Build(s *system.System) {
	r := s.CommandRouter
	r.SetCategory("information")

	r.On("help", m.Help).Set("", "Displays a help menu with the available commands")
}

// Help maps a list of available commands and descends into subrouters.
func (m *Module) Help(ctx *system.Context) {

	_, err := ctx.ReplyEmbed(depthcharge(ctx.System.CommandRouter, nil, 0).
		SetColor(system.StatusNotify).
		SetThumbnail(ctx.Ses.DG.State.User.AvatarURL("2048")).
		InlineAllFields().
		SetDescription("subcommands are represented by indentation.").
		MessageEmbed)
	if err != nil {
		ctx.ReplyError(err)
	}
}

func depthcharge(r *system.CommandRouter, embed *dream.Embed, depth int) *dream.Embed {
	if embed == nil {
		embed = dream.NewEmbed()
	}

	depthString := func(text string, depth int, subrouter bool) string {
		quote := ""
		if subrouter {
			quote = "`"
		}
		return strings.Repeat("  ", depth) + quote + text + quote + "\n"
	}

	getField := func(name string) *discordgo.MessageEmbedField {
		for _, v := range embed.Fields {
			if v.Name == name {
				return v
			}
		}
		if name == "" {
			name = "undefined"
		}
		field := &discordgo.MessageEmbedField{Name: name}
		embed.Fields = append(embed.Fields, field)
		return field
	}

	for _, v := range r.Routes {
		field := getField(v.Category)
		field.Value += depthString(v.Name, depth, false)
	}

	for _, v := range r.Subrouters {
		field := getField(v.Category())
		field.Value += depthString(v.Name, depth, true)
		embed = depthcharge(v.Router, embed, depth+1)
	}

	return embed
}
<|endoftext|>


<commit_before>package http

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/keel-hq/keel/types"
)

func (s *TriggerServer) approvalsHandler(resp http.ResponseWriter, req *http.Request) {
	// unknown lists all
	approvals, err := s.approvalsManager.List()
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	if len(approvals) == 0 {
		approvals = make([]*types.Approval, 0)
	}

	bts, err := json.Marshal(&approvals)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.Write(bts)
}

func (s *TriggerServer) approvalDeleteHandler(resp http.ResponseWriter, req *http.Request) {
	identifier := getID(req)

	err := s.approvalsManager.Delete(identifier)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.WriteHeader(http.StatusOK)
	fmt.Fprintf(resp, identifier)
}
<commit_msg>http endpoint to accept approvals<commit_after>package http

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/keel-hq/keel/cache"
	"github.com/keel-hq/keel/types"
)

type approveRequest struct {
	Identifier string `json:"identifier"`
	Voter      string `json:"voter"`
}

func (s *TriggerServer) approvalsHandler(resp http.ResponseWriter, req *http.Request) {
	// unknown lists all
	approvals, err := s.approvalsManager.List()
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	if len(approvals) == 0 {
		approvals = make([]*types.Approval, 0)
	}

	bts, err := json.Marshal(&approvals)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.Write(bts)
}

func (s *TriggerServer) approvalApproveHandler(resp http.ResponseWriter, req *http.Request) {

	var ar approveRequest
	dec := json.NewDecoder(req.Body)
	defer req.Body.Close()

	err := dec.Decode(&ar)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusBadRequest)
		return
	}

	if ar.Identifier == "" {
		http.Error(resp, "identifier not supplied", http.StatusBadRequest)
		return
	}

	approval, err := s.approvalsManager.Approve(ar.Identifier, ar.Voter)
	if err != nil {
		if err == cache.ErrNotFound {
			http.Error(resp, fmt.Sprintf("approval '%s' not found", ar.Identifier), http.StatusNotFound)
			return
		}

		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	bts, err := json.Marshal(&approval)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.Write(bts)
}

func (s *TriggerServer) approvalDeleteHandler(resp http.ResponseWriter, req *http.Request) {
	identifier := getID(req)

	err := s.approvalsManager.Delete(identifier)
	if err != nil {
		fmt.Fprintf(resp, "%s", err)
		resp.WriteHeader(http.StatusInternalServerError)
		return
	}

	resp.WriteHeader(http.StatusOK)
	fmt.Fprintf(resp, identifier)
}
<|endoftext|>


<commit_before>package fetcher

import (
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"net/url"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	"github.com/syou6162/GoOse"
)

type Article struct {
	Url           string
	Title         string
	Description   string
	OgDescription string
	OgType        string
	OgImage       string
	Body          string
	StatusCode    int
	Favicon       string
}

var articleFetcher = http.Client{
	Transport: &http.Transport{
		MaxIdleConns:        0,
		MaxIdleConnsPerHost: 100,
	},
	Timeout: time.Duration(5 * time.Second),
}

func GetArticle(origUrl string) Article {
	g := goose.New()
	resp, err := articleFetcher.Get(origUrl)
	if err != nil {
		return Article{}
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Article{StatusCode: resp.StatusCode}
	}

	if !utf8.Valid(html) {
		return Article{Url: resp.Request.URL.String(), StatusCode: resp.StatusCode}
	}

	article, err := g.ExtractFromRawHTML(resp.Request.URL.String(), string(html))
	if err != nil {
		return Article{StatusCode: resp.StatusCode}
	}

	finalUrl := article.CanonicalLink
	if finalUrl == "" {
		finalUrl = resp.Request.URL.String()
	}

	arxivUrl := "https://arxiv.org/abs/"
	if strings.Contains(origUrl, arxivUrl) || strings.Contains(finalUrl, arxivUrl) {
		// article.Docでもいけそうだが、gooseが中で書き換えていてダメ。Documentを作りなおす
		doc, _ := goquery.NewDocumentFromReader(strings.NewReader(string(html)))
		article.MetaDescription = doc.Find(".abstract").Text()
	}

	favicon := ""
	if u, err := url.Parse(article.MetaFavicon); err == nil {
		if u.IsAbs() {
			favicon = article.MetaFavicon
		}
	}

	return Article{
		finalUrl,
		article.Title,
		article.MetaDescription,
		article.MetaOgDescription,
		article.MetaOgType,
		article.MetaOgImage,
		article.CleanedText,
		resp.StatusCode,
		favicon,
	}
}
<commit_msg>バグりやすいのでフィールド名を付けておく<commit_after>package fetcher

import (
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"net/url"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	"github.com/syou6162/GoOse"
)

type Article struct {
	Url           string
	Title         string
	Description   string
	OgDescription string
	OgType        string
	OgImage       string
	Body          string
	StatusCode    int
	Favicon       string
}

var articleFetcher = http.Client{
	Transport: &http.Transport{
		MaxIdleConns:        0,
		MaxIdleConnsPerHost: 100,
	},
	Timeout: time.Duration(5 * time.Second),
}

func GetArticle(origUrl string) Article {
	g := goose.New()
	resp, err := articleFetcher.Get(origUrl)
	if err != nil {
		return Article{}
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return Article{StatusCode: resp.StatusCode}
	}

	if !utf8.Valid(html) {
		return Article{Url: resp.Request.URL.String(), StatusCode: resp.StatusCode}
	}

	article, err := g.ExtractFromRawHTML(resp.Request.URL.String(), string(html))
	if err != nil {
		return Article{StatusCode: resp.StatusCode}
	}

	finalUrl := article.CanonicalLink
	if finalUrl == "" {
		finalUrl = resp.Request.URL.String()
	}

	arxivUrl := "https://arxiv.org/abs/"
	if strings.Contains(origUrl, arxivUrl) || strings.Contains(finalUrl, arxivUrl) {
		// article.Docでもいけそうだが、gooseが中で書き換えていてダメ。Documentを作りなおす
		doc, _ := goquery.NewDocumentFromReader(strings.NewReader(string(html)))
		article.MetaDescription = doc.Find(".abstract").Text()
	}

	favicon := ""
	if u, err := url.Parse(article.MetaFavicon); err == nil {
		if u.IsAbs() {
			favicon = article.MetaFavicon
		}
	}

	return Article{
		Url: finalUrl,
		Title: article.Title,
		Description: article.MetaDescription,
		OgDescription: article.MetaOgDescription,
		OgType: article.MetaOgType,
		OgImage: article.MetaOgImage,
		Body: article.CleanedText,
		StatusCode: resp.StatusCode,
		Favicon: favicon,
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Keep in sync with ../base64/example_test.go.

package base32_test

import (
	"encoding/base32"
	"fmt"
	"os"
)

func ExampleEncoding_EncodeToString() {
	data := []byte("any + old & data")
	str := base32.StdEncoding.EncodeToString(data)
	fmt.Println(str)
	// Output:
	// MFXHSIBLEBXWYZBAEYQGIYLUME======
}

func ExampleEncoding_DecodeString() {
	str := "ONXW2ZJAMRQXIYJAO5UXI2BAAAQGC3TEEDX3XPY="
	data, err := base32.StdEncoding.DecodeString(str)
	if err != nil {
		fmt.Println("error:", err)
		return
	}
	fmt.Printf("%q\n", data)
	// Output:
	// "some data with \x00 and \ufeff"
}

func ExampleNewEncoder() {
	input := []byte("foo\x00bar")
	encoder := base32.NewEncoder(base32.StdEncoding, os.Stdout)
	encoder.Write(input)
	// Must close the encoder when finished to flush any partial blocks.
	// If you comment out the following line, the last partial block "r"
	// won't be encoded.
	encoder.Close()
	// Output:
	// MZXW6ADCMFZA====
}
<commit_msg>encoding/base32: Add examples for Encode/Decode<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Keep in sync with ../base64/example_test.go.

package base32_test

import (
	"encoding/base32"
	"fmt"
	"os"
)

func ExampleEncoding_EncodeToString() {
	data := []byte("any + old & data")
	str := base32.StdEncoding.EncodeToString(data)
	fmt.Println(str)
	// Output:
	// MFXHSIBLEBXWYZBAEYQGIYLUME======
}

func ExampleEncoding_Encode() {
	data := []byte("Hello, world!")
	dst := make([]byte, base32.StdEncoding.EncodedLen(len(data)))
	base32.StdEncoding.Encode(dst, data)
	fmt.Println(string(dst))
	// Output:
	// JBSWY3DPFQQHO33SNRSCC===
}

func ExampleEncoding_DecodeString() {
	str := "ONXW2ZJAMRQXIYJAO5UXI2BAAAQGC3TEEDX3XPY="
	data, err := base32.StdEncoding.DecodeString(str)
	if err != nil {
		fmt.Println("error:", err)
		return
	}
	fmt.Printf("%q\n", data)
	// Output:
	// "some data with \x00 and \ufeff"
}

func ExampleEncoding_Decode() {
	str := "JBSWY3DPFQQHO33SNRSCC==="
	dst := make([]byte, base32.StdEncoding.DecodedLen(len(str)))
	n, err := base32.StdEncoding.Decode(dst, []byte(str))
	if err != nil {
		fmt.Println("decode error:", err)
		return
	}
	dst = dst[:n]
	fmt.Printf("%q\n", dst)
	// Output:
	// "Hello, world!"
}

func ExampleNewEncoder() {
	input := []byte("foo\x00bar")
	encoder := base32.NewEncoder(base32.StdEncoding, os.Stdout)
	encoder.Write(input)
	// Must close the encoder when finished to flush any partial blocks.
	// If you comment out the following line, the last partial block "r"
	// won't be encoded.
	encoder.Close()
	// Output:
	// MZXW6ADCMFZA====
}
<|endoftext|>


<commit_before>package python

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srclib/graph"
	"sourcegraph.com/sourcegraph/srclib/grapher"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

type GraphContext struct {
	Unit *unit.SourceUnit
	Reqs []*requirement
}

func NewGraphContext(unit *unit.SourceUnit) *GraphContext {
	var g GraphContext
	g.Unit = unit
	for _, dep := range unit.Dependencies {
		if req, err := asRequirement(dep); err == nil {
			g.Reqs = append(g.Reqs, req)
		}
	}
	return &g
}

// Graphs the Python source unit. If run outside of a Docker container, this assumes that the source unit has already
// been installed (via pip or `python setup.py install`).
func (c *GraphContext) Graph() (*grapher.Output, error) {
	if os.Getenv("IN_DOCKER_CONTAINER") != "" {
		// NOTE: this may cause an error when graphing any source unit that depends
		// on jedi (or any other dependency of the graph code)
		requirementFiles, err := filepath.Glob(filepath.Join(c.Unit.Dir, "*requirements.txt"))
		if err != nil {
			return nil, err
		}
		for _, requirementFile := range requirementFiles {
			exec.Command("pip", "install", "-r", requirementFile)
		}
		exec.Command("pip", "install", "-I", c.Unit.Dir)
	}

	cmd := exec.Command("python", "-m", "grapher.graph", c.Unit.Dir, "--verbose")
	cmd.Stderr = os.Stderr
	b, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	var raw RawOutput
	if err := json.Unmarshal(b, &raw); err != nil {
		return nil, err
	}

	out := c.transform(&raw, c.Unit)
	return out, nil
}

func (c *GraphContext) transform(raw *RawOutput, unit *unit.SourceUnit) *grapher.Output {
	var out grapher.Output

	for _, def := range raw.Defs {
		out.Defs = append(out.Defs, c.transformDef(def))
		if doc := c.transformDefDoc(def); doc != nil {
			out.Docs = append(out.Docs, doc)
		}
	}
	for _, ref := range raw.Refs {
		if outRef, err := c.transformRef(ref); err == nil {
			out.Refs = append(out.Refs, outRef)
		} else {
			log.Printf("Could not transform ref %v: %s", ref, err)
		}
	}

	return &out
}

var jediKindToDefKind = map[string]graph.DefKind{
	"statement":        graph.Var,
	"statementelement": graph.Var,
	"param":            graph.Var,
	"module":           graph.Module,
	"submodule":        graph.Module,
	"class":            graph.Type,
	"function":         graph.Func,
	"lambda":           graph.Func,
	"import":           graph.Var,
}

func (c *GraphContext) transformDef(rawDef *RawDef) *graph.Def {
	return &graph.Def{
		DefKey: graph.DefKey{
			Repo:     c.Unit.Repo,
			Unit:     c.Unit.Name,
			UnitType: c.Unit.Type,
			Path:     graph.DefPath(rawDef.Path),
		},
		TreePath: graph.TreePath(rawDef.Path), // TODO: make this consistent w/ old way
		Kind:     jediKindToDefKind[rawDef.Kind],
		Name:     rawDef.Name,
		File:     rawDef.File,
		DefStart: rawDef.DefStart,
		DefEnd:   rawDef.DefEnd,
		Exported: rawDef.Exported,
		Data:     nil, // TODO
	}
}

func (c *GraphContext) transformRef(rawRef *RawRef) (*graph.Ref, error) {
	defUnit, err := c.inferSourceUnit(rawRef, c.Reqs)
	if err != nil {
		return nil, err
	}

	defPath := graph.DefPath(rawRef.DefPath)
	if defPath == "" {
		defPath = "."
	}

	return &graph.Ref{
		DefRepo:     defUnit.Repo,
		DefUnitType: defUnit.Type,
		DefUnit:     defUnit.Name,
		DefPath:     defPath,

		Repo:     c.Unit.Repo,
		Unit:     c.Unit.Name,
		UnitType: c.Unit.Type,

		File:  rawRef.File,
		Start: rawRef.Start,
		End:   rawRef.End,
	}, nil
}

func (c *GraphContext) transformDefDoc(rawDef *RawDef) *graph.Doc {
	return nil
}

func (c *GraphContext) inferSourceUnit(rawRef *RawRef, reqs []*requirement) (*unit.SourceUnit, error) {
	if rawRef.ToBuiltin {
		return stdLibPkg.SourceUnit(), nil
	}
	return c.inferSourceUnitFromFile(rawRef.DefFile, reqs)
}

// Note: file is expected to be an absolute path
func (c *GraphContext) inferSourceUnitFromFile(file string, reqs []*requirement) (*unit.SourceUnit, error) {
	// Case: in current source unit (u)
	pwd, _ := os.Getwd()
	if isSubPath(pwd, file) {
		return c.Unit, nil
	}

	// Case: in dependent source unit(depUnits)
	fileCmps := strings.Split(file, string(filepath.Separator))
	pkgsDirIdx := -1
	for i, cmp := range fileCmps {
		if cmp == "site-packages" || cmp == "dist-packages" {
			pkgsDirIdx = i
			break
		}
	}
	if pkgsDirIdx != -1 {
		fileSubCmps := fileCmps[pkgsDirIdx+1:]
		fileSubPath := filepath.Join(fileSubCmps...)

		var foundReq *requirement = nil
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				if isSubPath(moduleToFilepath(pkg, true), fileSubPath) {
					foundReq = req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				if moduleToFilepath(mod, false) == fileSubPath {
					foundReq = req
					break FindReq
				}
			}
		}

		if foundReq == nil {
			var candidatesStr string
			if len(reqs) <= 7 {
				candidatesStr = fmt.Sprintf("%v", reqs)
			} else {
				candidatesStr = fmt.Sprintf("%v...", reqs[:7])
			}
			return nil, fmt.Errorf("Could not find requirement that contains file %s. Candidates were: %s",
				file, candidatesStr)
		}

		return foundReq.SourceUnit(), nil
	}

	// Case 3: in std lib
	pythonDirIdx := -1
	for i, cmp := range fileCmps {
		if strings.HasPrefix(cmp, "python") {
			pythonDirIdx = i
			break
		}
	}
	if pythonDirIdx != -1 {
		return stdLibPkg.SourceUnit(), nil
	}

	return nil, fmt.Errorf("Cannot infer source unit for file %s", file)
}

func isSubPath(parent, child string) bool {
	relpath, err := filepath.Rel(parent, child)
	return err == nil && !strings.HasPrefix(relpath, "..")
}

func moduleToFilepath(moduleName string, isPackage bool) string {
	moduleName = strings.Replace(moduleName, ".", "/", -1)
	if !isPackage {
		moduleName += ".py"
	}
	return moduleName
}

type RawOutput struct {
	Defs []*RawDef
	Refs []*RawRef
}

type RawDef struct {
	Path      string
	Kind      string
	Name      string
	File      string // relative path (to source unit directory)
	DefStart  int
	DefEnd    int
	Exported  bool
	Docstring string
	Data      interface{}
}

type RawRef struct {
	DefPath   string
	Def       bool
	DefFile   string // absolute path
	File      string // relative path (to source unit directory)
	Start     int
	End       int
	ToBuiltin bool
}
<commit_msg>emit whether ref is a defref<commit_after>package python

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srclib/graph"
	"sourcegraph.com/sourcegraph/srclib/grapher"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

type GraphContext struct {
	Unit *unit.SourceUnit
	Reqs []*requirement
}

func NewGraphContext(unit *unit.SourceUnit) *GraphContext {
	var g GraphContext
	g.Unit = unit
	for _, dep := range unit.Dependencies {
		if req, err := asRequirement(dep); err == nil {
			g.Reqs = append(g.Reqs, req)
		}
	}
	return &g
}

// Graphs the Python source unit. If run outside of a Docker container, this assumes that the source unit has already
// been installed (via pip or `python setup.py install`).
func (c *GraphContext) Graph() (*grapher.Output, error) {
	if os.Getenv("IN_DOCKER_CONTAINER") != "" {
		// NOTE: this may cause an error when graphing any source unit that depends
		// on jedi (or any other dependency of the graph code)
		requirementFiles, err := filepath.Glob(filepath.Join(c.Unit.Dir, "*requirements.txt"))
		if err != nil {
			return nil, err
		}
		for _, requirementFile := range requirementFiles {
			exec.Command("pip", "install", "-r", requirementFile)
		}
		exec.Command("pip", "install", "-I", c.Unit.Dir)
	}

	cmd := exec.Command("python", "-m", "grapher.graph", c.Unit.Dir, "--verbose")
	cmd.Stderr = os.Stderr
	b, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	var raw RawOutput
	if err := json.Unmarshal(b, &raw); err != nil {
		return nil, err
	}

	out := c.transform(&raw, c.Unit)
	return out, nil
}

func (c *GraphContext) transform(raw *RawOutput, unit *unit.SourceUnit) *grapher.Output {
	var out grapher.Output

	for _, def := range raw.Defs {
		out.Defs = append(out.Defs, c.transformDef(def))
		if doc := c.transformDefDoc(def); doc != nil {
			out.Docs = append(out.Docs, doc)
		}
	}
	for _, ref := range raw.Refs {
		if outRef, err := c.transformRef(ref); err == nil {
			out.Refs = append(out.Refs, outRef)
		} else {
			log.Printf("Could not transform ref %v: %s", ref, err)
		}
	}

	return &out
}

var jediKindToDefKind = map[string]graph.DefKind{
	"statement":        graph.Var,
	"statementelement": graph.Var,
	"param":            graph.Var,
	"module":           graph.Module,
	"submodule":        graph.Module,
	"class":            graph.Type,
	"function":         graph.Func,
	"lambda":           graph.Func,
	"import":           graph.Var,
}

func (c *GraphContext) transformDef(rawDef *RawDef) *graph.Def {
	return &graph.Def{
		DefKey: graph.DefKey{
			Repo:     c.Unit.Repo,
			Unit:     c.Unit.Name,
			UnitType: c.Unit.Type,
			Path:     graph.DefPath(rawDef.Path),
		},
		TreePath: graph.TreePath(rawDef.Path), // TODO: make this consistent w/ old way
		Kind:     jediKindToDefKind[rawDef.Kind],
		Name:     rawDef.Name,
		File:     rawDef.File,
		DefStart: rawDef.DefStart,
		DefEnd:   rawDef.DefEnd,
		Exported: rawDef.Exported,
		Data:     nil, // TODO
	}
}

func (c *GraphContext) transformRef(rawRef *RawRef) (*graph.Ref, error) {
	defUnit, err := c.inferSourceUnit(rawRef, c.Reqs)
	if err != nil {
		return nil, err
	}

	defPath := graph.DefPath(rawRef.DefPath)
	if defPath == "" {
		defPath = "."
	}

	return &graph.Ref{
		DefRepo:     defUnit.Repo,
		DefUnitType: defUnit.Type,
		DefUnit:     defUnit.Name,
		DefPath:     defPath,

		Repo:     c.Unit.Repo,
		Unit:     c.Unit.Name,
		UnitType: c.Unit.Type,

		File:  rawRef.File,
		Start: rawRef.Start,
		End:   rawRef.End,
		Def:   rawRef.Def,
	}, nil
}

func (c *GraphContext) transformDefDoc(rawDef *RawDef) *graph.Doc {
	return nil
}

func (c *GraphContext) inferSourceUnit(rawRef *RawRef, reqs []*requirement) (*unit.SourceUnit, error) {
	if rawRef.ToBuiltin {
		return stdLibPkg.SourceUnit(), nil
	}
	return c.inferSourceUnitFromFile(rawRef.DefFile, reqs)
}

// Note: file is expected to be an absolute path
func (c *GraphContext) inferSourceUnitFromFile(file string, reqs []*requirement) (*unit.SourceUnit, error) {
	// Case: in current source unit (u)
	pwd, _ := os.Getwd()
	if isSubPath(pwd, file) {
		return c.Unit, nil
	}

	// Case: in dependent source unit(depUnits)
	fileCmps := strings.Split(file, string(filepath.Separator))
	pkgsDirIdx := -1
	for i, cmp := range fileCmps {
		if cmp == "site-packages" || cmp == "dist-packages" {
			pkgsDirIdx = i
			break
		}
	}
	if pkgsDirIdx != -1 {
		fileSubCmps := fileCmps[pkgsDirIdx+1:]
		fileSubPath := filepath.Join(fileSubCmps...)

		var foundReq *requirement = nil
	FindReq:
		for _, req := range reqs {
			for _, pkg := range req.Packages {
				if isSubPath(moduleToFilepath(pkg, true), fileSubPath) {
					foundReq = req
					break FindReq
				}
			}
			for _, mod := range req.Modules {
				if moduleToFilepath(mod, false) == fileSubPath {
					foundReq = req
					break FindReq
				}
			}
		}

		if foundReq == nil {
			var candidatesStr string
			if len(reqs) <= 7 {
				candidatesStr = fmt.Sprintf("%v", reqs)
			} else {
				candidatesStr = fmt.Sprintf("%v...", reqs[:7])
			}
			return nil, fmt.Errorf("Could not find requirement that contains file %s. Candidates were: %s",
				file, candidatesStr)
		}

		return foundReq.SourceUnit(), nil
	}

	// Case 3: in std lib
	pythonDirIdx := -1
	for i, cmp := range fileCmps {
		if strings.HasPrefix(cmp, "python") {
			pythonDirIdx = i
			break
		}
	}
	if pythonDirIdx != -1 {
		return stdLibPkg.SourceUnit(), nil
	}

	return nil, fmt.Errorf("Cannot infer source unit for file %s", file)
}

func isSubPath(parent, child string) bool {
	relpath, err := filepath.Rel(parent, child)
	return err == nil && !strings.HasPrefix(relpath, "..")
}

func moduleToFilepath(moduleName string, isPackage bool) string {
	moduleName = strings.Replace(moduleName, ".", "/", -1)
	if !isPackage {
		moduleName += ".py"
	}
	return moduleName
}

type RawOutput struct {
	Defs []*RawDef
	Refs []*RawRef
}

type RawDef struct {
	Path      string
	Kind      string
	Name      string
	File      string // relative path (to source unit directory)
	DefStart  int
	DefEnd    int
	Exported  bool
	Docstring string
	Data      interface{}
}

type RawRef struct {
	DefPath   string
	Def       bool
	DefFile   string // absolute path
	File      string // relative path (to source unit directory)
	Start     int
	End       int
	ToBuiltin bool
}
<|endoftext|>


<commit_before>package main_test

//go:generate v23 test generate .

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"v.io/core/veyron/lib/testutil"
	"v.io/core/veyron/lib/testutil/v23tests"
)

func V23TestTunneld(t *v23tests.T) {
	v23tests.RunRootMT(t, "--veyron.tcp.address=127.0.0.1:0")

	tunneldBin := t.BuildGoPkg("v.io/apps/tunnel/tunneld")
	vsh := t.BuildGoPkg("v.io/apps/tunnel/vsh")
	mounttableBin := t.BuildGoPkg("v.io/core/veyron/tools/mounttable")

	port, err := testutil.FindUnusedPort()
	if err != nil {
		t.Fatalf("FindUnusedPort failed: %v", err)
	}

	tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
	tunnelEndpoint := "/" + tunnelAddress

	// Start tunneld with a known endpoint.
	tunneldBin.Start("--veyron.tcp.address=" + tunnelAddress)

	// Run remote command with the endpoint.
	if want, got := "HELLO ENDPOINT\n", vsh.Start(tunnelEndpoint, "echo", "HELLO", "ENDPOINT").Output(); want != got {
		t.Fatalf("unexpected output, got %s, want %s", got, want)
	}

	// Run remote command with the object name.
	hostname, err := os.Hostname()
	if err != nil {
		t.Fatalf("Hostname() failed: %v", err)
	}

	if want, got := "HELLO NAME\n", vsh.Start("tunnel/hostname/"+hostname, "echo", "HELLO", "NAME").Output(); want != got {
		t.Fatalf("unexpected output, got %s, want %s", got, want)
	}

	// Send input to remote command.
	want := "HELLO SERVER"
	if got := vsh.WithStdin(bytes.NewBufferString(want)).Start(tunnelEndpoint, "cat").Output(); want != got {
		t.Fatalf("unexpected output, got %s, want %s", got, want)
	}

	// And again with a file redirection this time.
	outDir := t.TempDir()
	outPath := filepath.Join(outDir, "hello.txt")

	// TODO(sjr): instead of using Output() here, we'd really rather do
	// WaitOrDie(os.Stdout, os.Stderr). There is currently a race caused by
	// WithStdin that makes this flaky.
	vsh.WithStdin(bytes.NewBufferString(want)).Start(tunnelEndpoint, "cat > "+outPath).Output()
	if got, err := ioutil.ReadFile(outPath); err != nil || string(got) != want {
		if err != nil {
			t.Fatalf("ReadFile(%v) failed: %v", outPath, err)
		} else {
			t.Fatalf("unexpected output, got %s, want %s", got, want)
		}
	}

	// Verify that all published names are there.
	root, _ := t.GetVar("NAMESPACE_ROOT")
	inv := mounttableBin.Start("glob", root, "tunnel/*/*")

	// Expect two entries: one for the tunnel hostname and one for its hwaddr.
	matches := inv.ExpectSetEventuallyRE(
		"tunnel/hostname/"+regexp.QuoteMeta(hostname)+" (.*) \\(TTL .*\\)",
		"tunnel/hwaddr/.* (.*) \\(TTL .*\\)")

	// The full endpoint should contain the address we initially specified for the tunnel.
	if want = "@" + tunnelAddress + "@"; !strings.Contains(matches[0][1], want) {
		t.Fatalf("expected tunnel endpoint %s to contain %s, but it did not", matches[0][1], want)
	}

	// The hwaddr endpoint should be the same as the hostname endpoint.
	if matches[0][1] != matches[1][1] {
		t.Fatalf("expected hwaddr and hostname tunnel endpoints to match, but they did not (%s != %s)", matches[0][1], matches[1][1])
	}
}
<commit_msg>Needed for: https://vanadium-review.googlesource.com/#/c/4910/ MultiPart: 2/2<commit_after>package main_test

//go:generate v23 test generate .

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"v.io/core/veyron/lib/testutil"
	"v.io/core/veyron/lib/testutil/v23tests"
)

func V23TestTunneld(t *v23tests.T) {
	v23tests.RunRootMT(t, "--veyron.tcp.address=127.0.0.1:0")

	tunneldBin := t.BuildGoPkg("v.io/apps/tunnel/tunneld")
	vsh := t.BuildGoPkg("v.io/apps/tunnel/vsh")
	mounttableBin := t.BuildGoPkg("v.io/core/veyron/tools/mounttable")

	port, err := testutil.FindUnusedPort()
	if err != nil {
		t.Fatalf("FindUnusedPort failed: %v", err)
	}

	tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
	tunnelEndpoint := "/" + tunnelAddress

	// Start tunneld with a known endpoint.
	tunneldBin.Start("--veyron.tcp.address=" + tunnelAddress)

	// Run remote command with the endpoint.
	if want, got := "HELLO ENDPOINT\n", vsh.Start(tunnelEndpoint, "echo", "HELLO", "ENDPOINT").Output(); want != got {
		t.Fatalf("unexpected output, got %s, want %s", got, want)
	}

	// Run remote command with the object name.
	hostname, err := os.Hostname()
	if err != nil {
		t.Fatalf("Hostname() failed: %v", err)
	}

	if want, got := "HELLO NAME\n", vsh.Start("tunnel/hostname/"+hostname, "echo", "HELLO", "NAME").Output(); want != got {
		t.Fatalf("unexpected output, got %s, want %s", got, want)
	}

	// Send input to remote command.
	want := "HELLO SERVER"
	if got := vsh.WithStdin(bytes.NewBufferString(want)).Start(tunnelEndpoint, "cat").Output(); want != got {
		t.Fatalf("unexpected output, got %s, want %s", got, want)
	}

	// And again with a file redirection this time.
	outDir := t.NewTempDir()
	outPath := filepath.Join(outDir, "hello.txt")

	// TODO(sjr): instead of using Output() here, we'd really rather do
	// WaitOrDie(os.Stdout, os.Stderr). There is currently a race caused by
	// WithStdin that makes this flaky.
	vsh.WithStdin(bytes.NewBufferString(want)).Start(tunnelEndpoint, "cat > "+outPath).Output()
	if got, err := ioutil.ReadFile(outPath); err != nil || string(got) != want {
		if err != nil {
			t.Fatalf("ReadFile(%v) failed: %v", outPath, err)
		} else {
			t.Fatalf("unexpected output, got %s, want %s", got, want)
		}
	}

	// Verify that all published names are there.
	root, _ := t.GetVar("NAMESPACE_ROOT")
	inv := mounttableBin.Start("glob", root, "tunnel/*/*")

	// Expect two entries: one for the tunnel hostname and one for its hwaddr.
	matches := inv.ExpectSetEventuallyRE(
		"tunnel/hostname/"+regexp.QuoteMeta(hostname)+" (.*) \\(TTL .*\\)",
		"tunnel/hwaddr/.* (.*) \\(TTL .*\\)")

	// The full endpoint should contain the address we initially specified for the tunnel.
	if want = "@" + tunnelAddress + "@"; !strings.Contains(matches[0][1], want) {
		t.Fatalf("expected tunnel endpoint %s to contain %s, but it did not", matches[0][1], want)
	}

	// The hwaddr endpoint should be the same as the hostname endpoint.
	if matches[0][1] != matches[1][1] {
		t.Fatalf("expected hwaddr and hostname tunnel endpoints to match, but they did not (%s != %s)", matches[0][1], matches[1][1])
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationmaster_test

import (
	"time"

	"github.com/juju/errors"
	jujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	apitesting "github.com/juju/juju/api/base/testing"
	"github.com/juju/juju/api/migrationmaster"
	"github.com/juju/juju/apiserver/params"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/worker"
)

type ClientSuite struct {
	jujutesting.IsolationSuite
}

var _ = gc.Suite(&ClientSuite{})

func (s *ClientSuite) TestWatch(c *gc.C) {
	var stub jujutesting.Stub
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		stub.AddCall("call", objType, id, request, arg)
		switch request {
		case "Watch":
			*(result.(*params.NotifyWatchResult)) = params.NotifyWatchResult{
				NotifyWatcherId: "abc",
			}
		case "Next":
			// The full success case is tested in api/watcher.
			return errors.New("boom")
		case "Stop":
		}
		return nil
	})

	client := migrationmaster.NewClient(apiCaller)
	w, err := client.Watch()
	c.Assert(err, jc.ErrorIsNil)
	defer worker.Stop(w)

	errC := make(chan error)
	go func() {
		errC <- w.Wait()
	}()

	select {
	case err := <-errC:
		c.Assert(err, gc.ErrorMatches, "boom")
		stub.CheckCalls(c, []jujutesting.StubCall{
			{"call", []interface{}{"MigrationMaster", "", "Watch", nil}},
			{"call", []interface{}{"MigrationMasterWatcher", "abc", "Next", nil}},
			{"call", []interface{}{"MigrationMasterWatcher", "abc", "Stop", nil}},
		})
	case <-time.After(coretesting.LongWait):
		c.Fatal("timed out waiting for watcher to die")
	}
}

func (s *ClientSuite) TestWatchErr(c *gc.C) {
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		return errors.New("boom")
	})

	client := migrationmaster.NewClient(apiCaller)
	_, err := client.Watch()
	c.Assert(err, gc.ErrorMatches, "boom")
}
<commit_msg>api/migrationmaster: Make stub call records easier to debug<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationmaster_test

import (
	"time"

	"github.com/juju/errors"
	jujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	apitesting "github.com/juju/juju/api/base/testing"
	"github.com/juju/juju/api/migrationmaster"
	"github.com/juju/juju/apiserver/params"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/worker"
)

type ClientSuite struct {
	jujutesting.IsolationSuite
}

var _ = gc.Suite(&ClientSuite{})

func (s *ClientSuite) TestWatch(c *gc.C) {
	var stub jujutesting.Stub
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		stub.AddCall(objType+"."+request, id, arg)
		switch request {
		case "Watch":
			*(result.(*params.NotifyWatchResult)) = params.NotifyWatchResult{
				NotifyWatcherId: "abc",
			}
		case "Next":
			// The full success case is tested in api/watcher.
			return errors.New("boom")
		case "Stop":
		}
		return nil
	})

	client := migrationmaster.NewClient(apiCaller)
	w, err := client.Watch()
	c.Assert(err, jc.ErrorIsNil)
	defer worker.Stop(w)

	errC := make(chan error)
	go func() {
		errC <- w.Wait()
	}()

	select {
	case err := <-errC:
		c.Assert(err, gc.ErrorMatches, "boom")
		stub.CheckCalls(c, []jujutesting.StubCall{
			{"MigrationMaster.Watch", []interface{}{"", nil}},
			{"MigrationMasterWatcher.Next", []interface{}{"abc", nil}},
			{"MigrationMasterWatcher.Stop", []interface{}{"abc", nil}},
		})
	case <-time.After(coretesting.LongWait):
		c.Fatal("timed out waiting for watcher to die")
	}
}

func (s *ClientSuite) TestWatchErr(c *gc.C) {
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		return errors.New("boom")
	})

	client := migrationmaster.NewClient(apiCaller)
	_, err := client.Watch()
	c.Assert(err, gc.ErrorMatches, "boom")
}
<|endoftext|>


<commit_before>/*
* @Author: souravray
* @Date:   2014-11-08 00:57:48
* @Last Modified by:   souravray
* @Last Modified time: 2014-11-17 09:04:34
 */

package db

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"sync"
)

type Model struct {
	TableName string
	DB        *sql.DB
	Tx        *sql.Tx
	RWLock    sync.RWMutex
}

type QueueIteam struct {
	key  sql.NullString
	task sql.NullString
}

func NewModel(connectionString, name string) (model *Model, err error) {
	var db *sql.DB
	db, err = sql.Open("sqlite3", connectionString)
	if err != nil {
		return
	}
	model = &Model{TableName: name, DB: db}
	query := `
		PRAGMA automatic_index = OFF;
		PRAGMA cache_size = 32768;
		PRAGMA cache_spill = OFF;
		PRAGMA foreign_keys = OFF;
		PRAGMA journal_size_limit = 67110000;
		PRAGMA locking_mode = FULL;
		PRAGMA page_size = 4096;
		PRAGMA recursive_triggers = OFF;
		PRAGMA secure_delete = OFF;
		PRAGMA synchronous = NORMAL;
		PRAGMA temp_store = MEMORY;
		PRAGMA journal_mode = WAL;
		PRAGMA wal_autocheckpoint = 16384;

		CREATE TABLE IF NOT EXISTS queue (
        key text not null primary key,
        task blob
    );
    `
	_, err = model.DB.Exec(query)
	if err != nil {
		log.Fatal(err)
		return
	}
	return
}

func (m *Model) begin() {
	m.Tx, _ = m.DB.Begin()
}

func (m *Model) BatchTransaction() {
	m.RWLock.Lock()
	if m.Tx != nil {
		oldTx := m.Tx
		defer oldTx.Commit()
	}
	m.begin()
	m.RWLock.Unlock()
}

func (m *Model) TransactionEnd() {
	if m.Tx != nil {
		m.RWLock.Lock()
		m.Tx.Commit()
		m.RWLock.Unlock()
	}
}

func (m *Model) Add(key string, task []byte) (err error) {
	m.RWLock.RLock()
	stmt, err := m.Tx.Prepare("INSERT INTO queue (key, task) VALUES (?, ?)")
	defer stmt.Close()
	if err != nil {
		log.Fatal(err)
		return
	}
	_, err = stmt.Exec(key, task)
	m.RWLock.RUnlock()
	if err != nil {
		log.Fatal(err)
		return
	}
	return
}

func (m *Model) Update(key string, task []byte) (err error) {
	m.RWLock.RLock()
	stmt, err := m.Tx.Prepare("UPDATE queue SET task = ? WHERE key = ?")
	defer stmt.Close()
	if err != nil {
		log.Fatal(err)
		return
	}
	_, err = stmt.Exec(task, key)
	if err != nil {
		log.Fatal(err)
		return
	}
	m.RWLock.RUnlock()
	return
}

func (m *Model) Delete(key string) (err error) {
	m.RWLock.RLock()
	stmt, err := m.Tx.Prepare("DELETE FROM queue WHERE key = ?")
	defer stmt.Close()
	if err != nil {
		log.Fatal(err)
		return
	}
	_, err = stmt.Exec(key)
	if err != nil {
		log.Fatal(err)
		return
	}
	m.RWLock.RUnlock()
	return
}

// func (m *Model) Read() chan *bytes.Buffer {
// }
<commit_msg>Change sqlite configuration<commit_after>/*
* @Author: souravray
* @Date:   2014-11-08 00:57:48
* @Last Modified by:   souravray
* @Last Modified time: 2014-11-18 00:11:05
 */

package db

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"sync"
)

type Model struct {
	TableName string
	DB        *sql.DB
	Tx        *sql.Tx
	RWLock    sync.RWMutex
}

type QueueIteam struct {
	key  sql.NullString
	task sql.NullString
}

func NewModel(connectionString, name string) (model *Model, err error) {
	var db *sql.DB
	db, err = sql.Open("sqlite3", connectionString)
	if err != nil {
		return
	}
	model = &Model{TableName: name, DB: db}
	query := `
		PRAGMA automatic_index = OFF;
		PRAGMA cache_size = 32768;
		PRAGMA cache_spill = OFF;
		PRAGMA foreign_keys = OFF;
		PRAGMA journal_size_limit = 67110000;
		PRAGMA locking_mode = NORMAL;
		PRAGMA page_size = 4096;
		PRAGMA recursive_triggers = OFF;
		PRAGMA secure_delete = OFF;
		PRAGMA synchronous = FULL;
		PRAGMA temp_store = MEMORY;
		PRAGMA journal_mode = WAL;
		PRAGMA wal_autocheckpoint = 16384;

		CREATE TABLE IF NOT EXISTS queue (
        key text not null primary key,
        task blob
    );
    `
	_, err = model.DB.Exec(query)
	if err != nil {
		log.Fatal(err)
		return
	}
	return
}

func (m *Model) begin() {
	m.Tx, _ = m.DB.Begin()
}

func (m *Model) BatchTransaction() {
	m.RWLock.Lock()
	if m.Tx != nil {
		oldTx := m.Tx
		defer oldTx.Commit()
	}
	m.begin()
	m.RWLock.Unlock()
}

func (m *Model) TransactionEnd() {
	if m.Tx != nil {
		m.RWLock.Lock()
		m.Tx.Commit()
		m.RWLock.Unlock()
	}
}

func (m *Model) Add(key string, task []byte) (err error) {
	m.RWLock.RLock()
	stmt, err := m.Tx.Prepare("INSERT INTO queue (key, task) VALUES (?, ?)")
	defer stmt.Close()
	if err != nil {
		log.Fatal(err)
		return
	}
	_, err = stmt.Exec(key, task)
	m.RWLock.RUnlock()
	if err != nil {
		log.Fatal(err)
		return
	}
	return
}

func (m *Model) Update(key string, task []byte) (err error) {
	m.RWLock.RLock()
	stmt, err := m.Tx.Prepare("UPDATE queue SET task = ? WHERE key = ?")
	defer stmt.Close()
	if err != nil {
		log.Fatal(err)
		return
	}
	_, err = stmt.Exec(task, key)
	if err != nil {
		log.Fatal(err)
		return
	}
	m.RWLock.RUnlock()
	return
}

func (m *Model) Delete(key string) (err error) {
	m.RWLock.RLock()
	stmt, err := m.Tx.Prepare("DELETE FROM queue WHERE key = ?")
	defer stmt.Close()
	if err != nil {
		log.Fatal(err)
		return
	}
	_, err = stmt.Exec(key)
	if err != nil {
		log.Fatal(err)
		return
	}
	m.RWLock.RUnlock()
	return
}

// func (m *Model) Read() chan *bytes.Buffer {
// }
<|endoftext|>