<commit_before>package cl11

import (
	"fmt"
	"reflect"
	"strings"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// A kernel is a function declared in a program.
type Kernel struct {
	id clw.Kernel

	// The number of arguments to kernel.
	Arguments int

	// The kernel function name.
	FunctionName string

	// The context associated with the kernel.
	Context *Context

	// The program associated with the kernel.
	Program *Program

	// Information about the kernel object that may be specific to a device.
	WorkGroupInfo []KernelWorkGroupInfo

	// Scratch space to store arugment.
	argScratch [][scratchSize]byte
}

// Information about the kernel object specific to a device.
type KernelWorkGroupInfo struct {

	// The device associated with the kernel.
	Device *Device

	// The maximum workgroup size that can be used by the kernel to execute on
	// the device.
	WorkGroupSize int

	// The work group size specified in the function qualifiers or all zeros.
	CompileWorkGroupSize [3]int

	// The amount of local memory used by the kernel.
	LocalMemSize int

	// The preferred multiple of workgroup size for launch. This is a
	// performance hint.
	PreferredWorkGroupSizeMultiple int

	// The minimum amount of private memory, in bytes, used by each workitem in
	// the kernel.
	PrivateMemSize int
}

type LocalSpaceArg int

// Creates a kernal object.
//
// A kernel is a function declared in a program. A kernel is identified by the
// __kernel qualifier applied to any function in a program. A kernel object
// encapsulates the specific __kernel function declared in a program and the
// argument values to be used when executing this __kernel function.
func (p *Program) CreateKernel(name string) (*Kernel, error) {

	kernel, err := clw.CreateKernel(p.id, name)
	if err != nil {
		return nil, err
	}

	k := &Kernel{
		id:            kernel,
		Context:       p.Context,
		Program:       p,
		WorkGroupInfo: make([]KernelWorkGroupInfo, len(p.Devices)),
	}
	for i := range p.Devices {
		k.WorkGroupInfo[i].Device = p.Devices[i]
	}

	err = k.getAllInfo()
	if err != nil {
		return nil, err
	}

	return k, nil
}

func (k *Kernel) getAllInfo() (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	k.FunctionName = k.getString(clw.KernelFunctionName)
	k.Arguments = k.getUint(clw.KernelNumArgs)
	k.argScratch = make([][8]byte, k.Arguments)

	for i := range k.WorkGroupInfo {
		wgi := &k.WorkGroupInfo[i]

		wgi.WorkGroupSize = k.getWorkGroupSize(wgi.Device, clw.KernelWorkGroupSize)
		wgi.LocalMemSize = k.getWorkGroupUlong(wgi.Device, clw.KernelLocalMemSize)
		wgi.PreferredWorkGroupSizeMultiple =
			k.getWorkGroupSize(wgi.Device, clw.KernelPreferredWorkGroupSizeMultiple)
		wgi.PrivateMemSize = k.getWorkGroupUlong(wgi.Device, clw.KernelPrivateMemSize)
		wgi.CompileWorkGroupSize = k.getWorkGroupSize3(wgi.Device, clw.KernelCompileWorkGroupSize)
	}

	return
}

func (k *Kernel) getUint(paramName clw.KernelInfo) int {
	var param clw.Uint
	err := clw.GetKernelInfo(k.id, clw.KernelNumArgs, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param), nil)
	if err != nil {
		panic(err)
	}
	return int(param)
}

func (k *Kernel) getWorkGroupSize(d *Device, paramName clw.KernelWorkGroupInfo) int {
	var param clw.Size
	err := clw.GetKernelWorkGroupInfo(k.id, d.id, paramName, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param),
		nil)
	if err != nil {
		panic(err)
	}
	return int(param)
}

func (k *Kernel) getWorkGroupSize3(d *Device, paramName clw.KernelWorkGroupInfo) [3]int {
	var param [3]clw.Size
	err := clw.GetKernelWorkGroupInfo(k.id, d.id, paramName, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param),
		nil)
	if err != nil {
		panic(err)
	}
	return [3]int{int(param[0]), int(param[1]), int(param[2])}
}

func (k *Kernel) getWorkGroupUlong(d *Device, paramName clw.KernelWorkGroupInfo) int {
	var param clw.Ulong
	err := clw.GetKernelWorkGroupInfo(k.id, d.id, paramName, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param),
		nil)
	if err != nil {
		panic(err)
	}
	return int(param)
}

func (k *Kernel) getString(paramName clw.KernelInfo) string {

	var paramValueSize clw.Size
	err := clw.GetKernelInfo(k.id, paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	buffer := make([]byte, paramValueSize)
	err = clw.GetKernelInfo(k.id, paramName, paramValueSize, unsafe.Pointer(&buffer[0]), nil)
	if err != nil {
		panic(err)
	}

	// Trim space and trailing \0.
	return strings.TrimSpace(string(buffer[:len(buffer)-1]))
}

func (k *Kernel) SetArgument(index int, arg interface{}) error {

	var size uintptr
	var pointer unsafe.Pointer

	switch v := arg.(type) {

	case *Buffer:
		pointer = unsafe.Pointer(&v.id)
		size = unsafe.Sizeof(v.id)

	case *Image:
		pointer = unsafe.Pointer(&v.id)
		size = unsafe.Sizeof(v.id)

	case LocalSpaceArg:
		pointer = nil
		size = uintptr(v)

	default:
		value := reflect.ValueOf(arg)
		kind := value.Kind()
		for kind == reflect.Ptr || kind == reflect.Interface {
			value = value.Elem()
			kind = value.Kind()
		}

		pointer = unsafe.Pointer(&k.argScratch[index][0])

		switch kind {

		case reflect.Bool:

			localCopy := reflect.NewAt(int32Type, pointer).Elem()
			if value.Bool() {
				localCopy.SetInt(int64(clw.True))
			} else {
				localCopy.SetInt(int64(clw.False))
			}

			size = int32Size

		case reflect.Int:

			localCopy := reflect.NewAt(int32Type, pointer).Elem()
			localCopy.SetInt(value.Int())

			size = int32Size

		case reflect.Uint:

			localCopy := reflect.NewAt(uint32Type, pointer).Elem()
			localCopy.SetUint(value.Uint())

			size = uint32Size

		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32,
			reflect.Uint64, reflect.Float32, reflect.Float64:

			localType := value.Type()
			localCopy := reflect.NewAt(localType, pointer).Elem()
			localCopy.Set(value)

			size = localType.Size()

		default:
			return wrapError(fmt.Errorf("invaild argument kind: %s", kind.String()))
		}
	}

	return clw.SetKernelArg(k.id, clw.Uint(index), clw.Size(size), pointer)
}

func (k *Kernel) SetArguments(args ...interface{}) error {

	if len(args) != k.Arguments {
		return wrapError(fmt.Errorf("invalid number of arguments: expecting %d, got %d", k.Arguments, len(args)))
	}

	for i := range args {
		err := k.SetArgument(i, args[i])
		if err != nil {
			return wrapError(fmt.Errorf("setting argument %d: %s", i, err.Error()))
		}
	}

	return nil
}

func (cq *CommandQueue) EnqueueNDRangeKernel(k *Kernel, globalOffset, globalSize, localSize []int,
	waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandNDRangeKernel
		e.CommandQueue = cq
	}

	dims := len(globalSize)
	sizes := make([]clw.Size, dims*3)
	if globalOffset != nil {
		for i := 0; i < dims; i++ {
			sizes[i] = clw.Size(globalOffset[i])
		}
	}
	for i := 0; i < dims; i++ {
		sizes[dims+i] = clw.Size(globalSize[i])
		sizes[2*dims+i] = clw.Size(localSize[i])
	}

	return clw.EnqueueNDRangeKernel(cq.id, k.id, sizes[:dims], sizes[dims:2*dims], sizes[2*dims:],
		cq.toEvents(waitList), event)
}
<commit_msg>Add CreateKernelsInProgram.<commit_after>package cl11

import (
	"fmt"
	"reflect"
	"strings"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// A kernel is a function declared in a program.
type Kernel struct {
	id clw.Kernel

	// The number of arguments to kernel.
	Arguments int

	// The kernel function name.
	FunctionName string

	// The context associated with the kernel.
	Context *Context

	// The program associated with the kernel.
	Program *Program

	// Information about the kernel object that may be specific to a device.
	WorkGroupInfo []KernelWorkGroupInfo

	// Scratch space to store arugment.
	argScratch [][scratchSize]byte
}

// Information about the kernel object specific to a device.
type KernelWorkGroupInfo struct {

	// The device associated with the kernel.
	Device *Device

	// The maximum workgroup size that can be used by the kernel to execute on
	// the device.
	WorkGroupSize int

	// The work group size specified in the function qualifiers or all zeros.
	CompileWorkGroupSize [3]int

	// The amount of local memory used by the kernel.
	LocalMemSize int

	// The preferred multiple of workgroup size for launch. This is a
	// performance hint.
	PreferredWorkGroupSizeMultiple int

	// The minimum amount of private memory, in bytes, used by each workitem in
	// the kernel.
	PrivateMemSize int
}

type LocalSpaceArg int

// Creates a kernal object.
//
// A kernel is a function declared in a program. A kernel is identified by the
// __kernel qualifier applied to any function in a program. A kernel object
// encapsulates the specific __kernel function declared in a program and the
// argument values to be used when executing this __kernel function.
func (p *Program) CreateKernel(name string) (*Kernel, error) {

	kernel, err := clw.CreateKernel(p.id, name)
	if err != nil {
		return nil, err
	}

	k := &Kernel{
		id:            kernel,
		Context:       p.Context,
		Program:       p,
		WorkGroupInfo: make([]KernelWorkGroupInfo, len(p.Devices)),
	}
	for i := range p.Devices {
		k.WorkGroupInfo[i].Device = p.Devices[i]
	}

	err = k.getAllInfo()
	if err != nil {
		return nil, err
	}

	return k, nil
}

// Creates kernel objects for all kernel functions in a program object.
//
// Creates kernel objects for all kernel functions in program. Kernel objects
// are not created for any __kernel functions in program that do not have the
// same function definition across all devices for which a program executable
// has been successfully built.
func (p *Program) CreateKernelsInProgram() ([]*Kernel, error) {

	var numKernels clw.Uint
	err := clw.CreateKernelsInProgram(p.id, nil, &numKernels)
	if err != nil {
		return nil, err
	}

	kernelIDs := make([]clw.Kernel, int(numKernels))
	err = clw.CreateKernelsInProgram(p.id, kernelIDs, nil)
	if err != nil {
		return nil, err
	}

	kernels := make([]*Kernel, int(numKernels))
	for i := range kernels {

		kernels[i] = &Kernel{
			id:            kernelIDs[i],
			Context:       p.Context,
			Program:       p,
			WorkGroupInfo: make([]KernelWorkGroupInfo, len(p.Devices)),
		}
		for j := range p.Devices {
			kernels[i].WorkGroupInfo[j].Device = p.Devices[j]
		}

		err = kernels[i].getAllInfo()
		if err != nil {
			return nil, err
		}
	}

	return kernels, nil
}

func (k *Kernel) getAllInfo() (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	k.FunctionName = k.getString(clw.KernelFunctionName)
	k.Arguments = k.getUint(clw.KernelNumArgs)
	k.argScratch = make([][8]byte, k.Arguments)

	for i := range k.WorkGroupInfo {
		wgi := &k.WorkGroupInfo[i]

		wgi.WorkGroupSize = k.getWorkGroupSize(wgi.Device, clw.KernelWorkGroupSize)
		wgi.LocalMemSize = k.getWorkGroupUlong(wgi.Device, clw.KernelLocalMemSize)
		wgi.PreferredWorkGroupSizeMultiple =
			k.getWorkGroupSize(wgi.Device, clw.KernelPreferredWorkGroupSizeMultiple)
		wgi.PrivateMemSize = k.getWorkGroupUlong(wgi.Device, clw.KernelPrivateMemSize)
		wgi.CompileWorkGroupSize = k.getWorkGroupSize3(wgi.Device, clw.KernelCompileWorkGroupSize)
	}

	return
}

func (k *Kernel) getUint(paramName clw.KernelInfo) int {
	var param clw.Uint
	err := clw.GetKernelInfo(k.id, clw.KernelNumArgs, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param), nil)
	if err != nil {
		panic(err)
	}
	return int(param)
}

func (k *Kernel) getWorkGroupSize(d *Device, paramName clw.KernelWorkGroupInfo) int {
	var param clw.Size
	err := clw.GetKernelWorkGroupInfo(k.id, d.id, paramName, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param),
		nil)
	if err != nil {
		panic(err)
	}
	return int(param)
}

func (k *Kernel) getWorkGroupSize3(d *Device, paramName clw.KernelWorkGroupInfo) [3]int {
	var param [3]clw.Size
	err := clw.GetKernelWorkGroupInfo(k.id, d.id, paramName, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param),
		nil)
	if err != nil {
		panic(err)
	}
	return [3]int{int(param[0]), int(param[1]), int(param[2])}
}

func (k *Kernel) getWorkGroupUlong(d *Device, paramName clw.KernelWorkGroupInfo) int {
	var param clw.Ulong
	err := clw.GetKernelWorkGroupInfo(k.id, d.id, paramName, clw.Size(unsafe.Sizeof(param)), unsafe.Pointer(&param),
		nil)
	if err != nil {
		panic(err)
	}
	return int(param)
}

func (k *Kernel) getString(paramName clw.KernelInfo) string {

	var paramValueSize clw.Size
	err := clw.GetKernelInfo(k.id, paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	buffer := make([]byte, paramValueSize)
	err = clw.GetKernelInfo(k.id, paramName, paramValueSize, unsafe.Pointer(&buffer[0]), nil)
	if err != nil {
		panic(err)
	}

	// Trim space and trailing \0.
	return strings.TrimSpace(string(buffer[:len(buffer)-1]))
}

func (k *Kernel) SetArgument(index int, arg interface{}) error {

	var size uintptr
	var pointer unsafe.Pointer

	switch v := arg.(type) {

	case *Buffer:
		pointer = unsafe.Pointer(&v.id)
		size = unsafe.Sizeof(v.id)

	case *Image:
		pointer = unsafe.Pointer(&v.id)
		size = unsafe.Sizeof(v.id)

	case LocalSpaceArg:
		pointer = nil
		size = uintptr(v)

	default:
		value := reflect.ValueOf(arg)
		kind := value.Kind()
		for kind == reflect.Ptr || kind == reflect.Interface {
			value = value.Elem()
			kind = value.Kind()
		}

		pointer = unsafe.Pointer(&k.argScratch[index][0])

		switch kind {

		case reflect.Bool:

			localCopy := reflect.NewAt(int32Type, pointer).Elem()
			if value.Bool() {
				localCopy.SetInt(int64(clw.True))
			} else {
				localCopy.SetInt(int64(clw.False))
			}

			size = int32Size

		case reflect.Int:

			localCopy := reflect.NewAt(int32Type, pointer).Elem()
			localCopy.SetInt(value.Int())

			size = int32Size

		case reflect.Uint:

			localCopy := reflect.NewAt(uint32Type, pointer).Elem()
			localCopy.SetUint(value.Uint())

			size = uint32Size

		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32,
			reflect.Uint64, reflect.Float32, reflect.Float64:

			localType := value.Type()
			localCopy := reflect.NewAt(localType, pointer).Elem()
			localCopy.Set(value)

			size = localType.Size()

		default:
			return wrapError(fmt.Errorf("invaild argument kind: %s", kind.String()))
		}
	}

	return clw.SetKernelArg(k.id, clw.Uint(index), clw.Size(size), pointer)
}

func (k *Kernel) SetArguments(args ...interface{}) error {

	if len(args) != k.Arguments {
		return wrapError(fmt.Errorf("invalid number of arguments: expecting %d, got %d", k.Arguments, len(args)))
	}

	for i := range args {
		err := k.SetArgument(i, args[i])
		if err != nil {
			return wrapError(fmt.Errorf("setting argument %d: %s", i, err.Error()))
		}
	}

	return nil
}

func (cq *CommandQueue) EnqueueNDRangeKernel(k *Kernel, globalOffset, globalSize, localSize []int,
	waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandNDRangeKernel
		e.CommandQueue = cq
	}

	dims := len(globalSize)
	sizes := make([]clw.Size, dims*3)
	if globalOffset != nil {
		for i := 0; i < dims; i++ {
			sizes[i] = clw.Size(globalOffset[i])
		}
	}
	for i := 0; i < dims; i++ {
		sizes[dims+i] = clw.Size(globalSize[i])
		sizes[2*dims+i] = clw.Size(localSize[i])
	}

	return clw.EnqueueNDRangeKernel(cq.id, k.id, sizes[:dims], sizes[dims:2*dims], sizes[2*dims:],
		cq.toEvents(waitList), event)
}
<|endoftext|>


<commit_before>package ketama

import (
	"crypto/sha1"
	"sort"
	"strconv"
)

type node struct {
	node string
	hash uint
}

type tickArray []node

func (p tickArray) Len() int           { return len(p) }
func (p tickArray) Less(i, j int) bool { return p[i].hash < p[j].hash }
func (p tickArray) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p tickArray) Sort()              { sort.Sort(p) }

type hashRing struct {
	defaultSpots int
	ticks        tickArray
	length       int
}

func NewRing(n int) *hashRing {
	h := new(hashRing)
	h.defaultSpots = n
	return h
}

// Adds a new node to a hash ring
// n: name of the server
// s: multiplier for default number of ticks (useful when one cache node has more resources, like RAM, than another)
func (h *hashRing) AddNode(n string, s int) {
	tSpots := h.defaultSpots * s
	hash := sha1.New()
	for i := 1; i <= tSpots; i++ {
		hash.Write([]byte(n + ":" + strconv.Itoa(i)))
		hashBytes := hash.Sum(nil)

		n := &node{
			node: n,
			hash: uint(hashBytes[19]) | uint(hashBytes[18])<<8 | uint(hashBytes[17])<<16 | uint(hashBytes[16])<<24,
		}

		h.ticks = append(h.ticks, *n)
		hash.Reset()
	}
}

func (h *hashRing) Bake() {
	h.ticks.Sort()
	h.length = len(h.ticks)
}

func (h *hashRing) Hash(s string) string {
	hash := sha1.New()
	hash.Write([]byte(s))
	hashBytes := hash.Sum(nil)
	v := uint(hashBytes[19]) | uint(hashBytes[18])<<8 | uint(hashBytes[17])<<16 | uint(hashBytes[16])<<24
	i := sort.Search(h.length, func(i int) bool { return h.ticks[i].hash >= v })

	if i == h.length {
		i = 0
	}

	return h.ticks[i].node
}
<commit_msg>Trying out the unadorned return thingamajig<commit_after>package ketama

import (
	"crypto/sha1"
	"sort"
	"strconv"
)

type node struct {
	node string
	hash uint
}

type tickArray []node

func (p tickArray) Len() int           { return len(p) }
func (p tickArray) Less(i, j int) bool { return p[i].hash < p[j].hash }
func (p tickArray) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p tickArray) Sort()              { sort.Sort(p) }

type hashRing struct {
	defaultSpots int
	ticks        tickArray
	length       int
}

func NewRing(n int) (h *hashRing) {
	h = new(hashRing)
	h.defaultSpots = n
	return
}

// Adds a new node to a hash ring
// n: name of the server
// s: multiplier for default number of ticks (useful when one cache node has more resources, like RAM, than another)
func (h *hashRing) AddNode(n string, s int) {
	tSpots := h.defaultSpots * s
	hash := sha1.New()
	for i := 1; i <= tSpots; i++ {
		hash.Write([]byte(n + ":" + strconv.Itoa(i)))
		hashBytes := hash.Sum(nil)

		n := &node{
			node: n,
			hash: uint(hashBytes[19]) | uint(hashBytes[18])<<8 | uint(hashBytes[17])<<16 | uint(hashBytes[16])<<24,
		}

		h.ticks = append(h.ticks, *n)
		hash.Reset()
	}
}

func (h *hashRing) Bake() {
	h.ticks.Sort()
	h.length = len(h.ticks)
}

func (h *hashRing) Hash(s string) string {
	hash := sha1.New()
	hash.Write([]byte(s))
	hashBytes := hash.Sum(nil)
	v := uint(hashBytes[19]) | uint(hashBytes[18])<<8 | uint(hashBytes[17])<<16 | uint(hashBytes[16])<<24
	i := sort.Search(h.length, func(i int) bool { return h.ticks[i].hash >= v })

	if i == h.length {
		i = 0
	}

	return h.ticks[i].node
}
<|endoftext|>


<commit_before>package accurev

import (
	"fmt"
	"github.com/aryann/difflib"
	"html"
	"strings"
)

type ElementDiff struct {
	diffRecords []*difflib.DiffRecord
}

func (e *ElementDiff) ToHTML() string {
	htmlLines := []string{}
	for _, dr := range e.diffRecords {
		htmlLines = append(htmlLines, getHtmlLineFromDiffRecord(dr))
	}
	return strings.Join(htmlLines, " ")
}

func (e *ElementDiff) ToHTML_HideInsignificantLines(contextLinesAbove, contextLinesBelow int) string {
	htmlLines := []string{}

	mustAddPlaceholderLine := false
	for i, dr := range e.diffRecords {
		if !e.isDiffRecordSignificant(i, contextLinesAbove, contextLinesBelow) {
			if mustAddPlaceholderLine {
				mustAddPlaceholderLine = false
				htmlLines = append(htmlLines, getHtmlForCodeLine(``, "placeholder"))
			}
			continue
		}

		mustAddPlaceholderLine = true
		htmlLines = append(htmlLines, getHtmlLineFromDiffRecord(dr))
	}
	return strings.Join(htmlLines, " ")
}

func (e *ElementDiff) isDiffRecordSignificant(index, contextLinesAbove, contextLinesBelow int) bool {
	for _, modIndex := range e.getModifiedLineIndexes() {
		if index >= modIndex-contextLinesAbove && index <= modIndex+contextLinesBelow {
			return true
		}
	}
	return false
}

func (e *ElementDiff) getModifiedLineIndexes() (indexes []int) {
	for i, dr := range e.diffRecords {
		if dr.Delta.String() != " " {
			indexes = append(indexes, i)
		}
	}
	return
}

func getHtmlForCodeLine(escapedLine, cssClass string) string {
	return fmt.Sprintf(`<div class="codeline %s">%s</div>`, cssClass, escapedLine)
}

func getHtmlLineFromDiffRecord(dr *difflib.DiffRecord) string {
	deltaSymbol := dr.Delta.String()

	cssClass := ""
	switch deltaSymbol {
	case " ":
		cssClass = "normal"
		break
	case "-":
		cssClass = "deleted"
		break
	case "+":
		cssClass = "added"
		break
	default:
		panic("Invalid delta symbol for 'github.com/aryann/difflib', delta symbol is '" + deltaSymbol + "'")
	}

	return getHtmlForCodeLine(html.EscapeString(dr.Payload), cssClass)
}

func GetElementDiffBetweenVersions(elementId, oldVersion, newVersion string) *ElementDiff {
	oldContent := Cat_WithElementIDAndVersion(elementId, oldVersion)
	newContent := Cat_WithElementIDAndVersion(elementId, newVersion)
	diffRecords := difflib.Diff(strings.Split(oldContent, "\n"), strings.Split(newContent, "\n"))

	diffRecordPointers := []*difflib.DiffRecord{}
	for ind, _ := range diffRecords {
		diffRecordPointers = append(diffRecordPointers, &diffRecords[ind])
	}

	return &ElementDiff{diffRecordPointers}
}
<commit_msg>Added `TotalLineCount`, `TotalLinesDeletedCount` and `TotalLinesAddedCount` methods.<commit_after>package accurev

import (
	"fmt"
	"github.com/aryann/difflib"
	"html"
	"strings"
)

type ElementDiff struct {
	diffRecords []*difflib.DiffRecord
}

func (e *ElementDiff) TotalLineCount() int {
	return len(e.diffRecords)
}

func (e *ElementDiff) TotalLinesDeletedCount() int {
	count := 0
	for _, dr := range e.diffRecords {
		if dr.Delta.String() == "-" {
			count++
		}
	}
	return count
}

func (e *ElementDiff) TotalLinesAddedCount() int {
	count := 0
	for _, dr := range e.diffRecords {
		if dr.Delta.String() == "+" {
			count++
		}
	}
	return count
}

func (e *ElementDiff) ToHTML() string {
	htmlLines := []string{}
	for _, dr := range e.diffRecords {
		htmlLines = append(htmlLines, getHtmlLineFromDiffRecord(dr))
	}
	return strings.Join(htmlLines, " ")
}

func (e *ElementDiff) ToHTML_HideInsignificantLines(contextLinesAbove, contextLinesBelow int) string {
	htmlLines := []string{}

	mustAddPlaceholderLine := false
	for i, dr := range e.diffRecords {
		if !e.isDiffRecordSignificant(i, contextLinesAbove, contextLinesBelow) {
			if mustAddPlaceholderLine {
				mustAddPlaceholderLine = false
				htmlLines = append(htmlLines, getHtmlForCodeLine(``, "placeholder"))
			}
			continue
		}

		mustAddPlaceholderLine = true
		htmlLines = append(htmlLines, getHtmlLineFromDiffRecord(dr))
	}
	return strings.Join(htmlLines, " ")
}

func (e *ElementDiff) isDiffRecordSignificant(index, contextLinesAbove, contextLinesBelow int) bool {
	for _, modIndex := range e.getModifiedLineIndexes() {
		if index >= modIndex-contextLinesAbove && index <= modIndex+contextLinesBelow {
			return true
		}
	}
	return false
}

func (e *ElementDiff) getModifiedLineIndexes() (indexes []int) {
	for i, dr := range e.diffRecords {
		if dr.Delta.String() != " " {
			indexes = append(indexes, i)
		}
	}
	return
}

func getHtmlForCodeLine(escapedLine, cssClass string) string {
	return fmt.Sprintf(`<div class="codeline %s">%s</div>`, cssClass, escapedLine)
}

func getHtmlLineFromDiffRecord(dr *difflib.DiffRecord) string {
	deltaSymbol := dr.Delta.String()

	cssClass := ""
	switch deltaSymbol {
	case " ":
		cssClass = "normal"
		break
	case "-":
		cssClass = "deleted"
		break
	case "+":
		cssClass = "added"
		break
	default:
		panic("Invalid delta symbol for 'github.com/aryann/difflib', delta symbol is '" + deltaSymbol + "'")
	}

	return getHtmlForCodeLine(html.EscapeString(dr.Payload), cssClass)
}

func GetElementDiffBetweenVersions(elementId, oldVersion, newVersion string) *ElementDiff {
	oldContent := Cat_WithElementIDAndVersion(elementId, oldVersion)
	newContent := Cat_WithElementIDAndVersion(elementId, newVersion)
	diffRecords := difflib.Diff(strings.Split(oldContent, "\n"), strings.Split(newContent, "\n"))

	diffRecordPointers := []*difflib.DiffRecord{}
	for ind, _ := range diffRecords {
		diffRecordPointers = append(diffRecordPointers, &diffRecords[ind])
	}

	return &ElementDiff{diffRecordPointers}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/elazarl/goproxy"
	"log"
	"net/http"
	"os"
	"os/exec"
)

func checkError(err error) {
	if err != nil {
		log.Fatalf("Error: %s", err)
	}
}

func main() {

	port := os.Getenv("PORT")

	if port == "" {
		port = "3000"
	}

	fmt.Println(port)
	binary, err := exec.LookPath("node")
	fmt.Println(binary)

	cmd := exec.Command(binary, "--expose_gc", "index.js")
	env := os.Environ()
	env = append([]string{"PORT=3001"}, env...)
	cmd.Env = env
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	checkError(err)
	defer func() {
		err = cmd.Wait()
		checkError(err)
	}()

	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = true

	proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		req.Host = "bower.herokuapp.com"
		req.URL.Scheme = "http"
		req.URL.Host = "localhost:3001"
		proxy.ServeHTTP(w, req)
	})

	fmt.Println("Starting web server at port", port)

	log.Fatal(http.ListenAndServe(":"+port, proxy))
}
<commit_msg>Serve package metadata directly through go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/jackc/pgx"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

func checkError(err error) {
	if err != nil {
		log.Fatalf("Error: %s", err)
	}
}

func UrlHasPrefix(prefix string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, prefix)
	}
}

var pool *pgx.ConnPool

func afterConnect(conn *pgx.Conn) (err error) {
	_, err = conn.Prepare("getPackage", `
		SELECT name, url FROM packages WHERE name = $1
  `)
	if err != nil {
		return
	}

	return
}

func main() {

	var err error
	pgxcfg, err := pgx.ParseURI(os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Fatalf("Parse URI error: %s", err)
	}
	pool, err = pgx.NewConnPool(pgx.ConnPoolConfig{
		ConnConfig:     pgxcfg,
		MaxConnections: 10,
		AfterConnect:   afterConnect,
	})
	if err != nil {
		log.Fatalf("Connection error: %s", err)
	}
	defer pool.Close()

	port := os.Getenv("PORT")

	if port == "" {
		port = "3000"
	}

	binary, err := exec.LookPath("node")
	checkError(err)
	cmd := exec.Command(binary, "--expose_gc", "index.js")
	env := os.Environ()
	env = append([]string{"PORT=3001"}, env...)
	cmd.Env = env
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	checkError(err)
	defer func() {
		err = cmd.Wait()
		checkError(err)
	}()

	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = true

	proxy.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		req.Host = "bower.herokuapp.com"
		req.URL.Scheme = "http"
		req.URL.Host = "localhost:3001"
		proxy.ServeHTTP(w, req)
	})

	proxy.OnRequest(UrlHasPrefix("/packages/")).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			var name string
			var url string
			elements := strings.Split(r.URL.Path, "/")
			packageName := elements[len(elements)-1]
			err := pool.QueryRow("getPackage", packageName).Scan(&name, &url)
			switch err {
			case nil:
				result := map[string]string{"name": name, "url": url}
				resultByteArray, _ := json.Marshal(result)
				return r, goproxy.NewResponse(r, "application/json", http.StatusOK, string(resultByteArray))
			case pgx.ErrNoRows:
				return r, goproxy.NewResponse(r, "text/html", http.StatusNotFound, "Package not found")
			default:
				return r, goproxy.NewResponse(r, "text/html", http.StatusInternalServerError, "Internal server error")
			}
		})

	fmt.Println("Starting web server at port", port)

	log.Fatal(http.ListenAndServe(":"+port, proxy))
}
<|endoftext|>


<commit_before>package client

import (
	"context"
	"encoding/json"
	"fmt"

	"github.com/docker/cli/cli/manifest/types"
	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest/manifestlist"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/api/v2"
	distclient "github.com/docker/distribution/registry/client"
	"github.com/docker/docker/registry"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// fetchManifest pulls a manifest from a registry and returns it. An error
// is returned if no manifest is found matching namedRef.
func fetchManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (types.ImageManifest, error) {
	manifest, err := getManifest(ctx, repo, ref)
	if err != nil {
		return types.ImageManifest{}, err
	}

	switch v := manifest.(type) {
	// Removed Schema 1 support
	case *schema2.DeserializedManifest:
		imageManifest, err := pullManifestSchemaV2(ctx, ref, repo, *v)
		if err != nil {
			return types.ImageManifest{}, err
		}
		return imageManifest, nil
	case *manifestlist.DeserializedManifestList:
		return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref)
	}
	return types.ImageManifest{}, errors.Errorf("%s is not a manifest", ref)
}

func fetchList(ctx context.Context, repo distribution.Repository, ref reference.Named) ([]types.ImageManifest, error) {
	manifest, err := getManifest(ctx, repo, ref)
	if err != nil {
		return nil, err
	}

	switch v := manifest.(type) {
	case *manifestlist.DeserializedManifestList:
		imageManifests, err := pullManifestList(ctx, ref, repo, *v)
		if err != nil {
			return nil, err
		}
		return imageManifests, nil
	default:
		return nil, errors.Errorf("unsupported manifest format: %v", v)
	}
}

func getManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (distribution.Manifest, error) {
	manSvc, err := repo.Manifests(ctx)
	if err != nil {
		return nil, err
	}

	dgst, opts, err := getManifestOptionsFromReference(ref)
	if err != nil {
		return nil, errors.Errorf("image manifest for %q does not exist", ref)
	}
	return manSvc.Get(ctx, dgst, opts...)
}

func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst schema2.DeserializedManifest) (types.ImageManifest, error) {
	manifestDesc, err := validateManifestDigest(ref, mfst)
	if err != nil {
		return types.ImageManifest{}, err
	}
	configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo)
	if err != nil {
		return types.ImageManifest{}, err
	}

	if manifestDesc.Platform == nil {
		manifestDesc.Platform = &ocispec.Platform{}
	}

	// Fill in os and architecture fields from config JSON
	if err := json.Unmarshal(configJSON, manifestDesc.Platform); err != nil {
		return types.ImageManifest{}, err
	}

	return types.NewImageManifest(ref, manifestDesc, &mfst), nil
}

func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) {
	blobs := repo.Blobs(ctx)
	configJSON, err := blobs.Get(ctx, dgst)
	if err != nil {
		return nil, err
	}

	verifier := dgst.Verifier()
	if err != nil {
		return nil, err
	}
	if _, err := verifier.Write(configJSON); err != nil {
		return nil, err
	}
	if !verifier.Verified() {
		return nil, errors.Errorf("image config verification failed for digest %s", dgst)
	}
	return configJSON, nil
}

// validateManifestDigest computes the manifest digest, and, if pulling by
// digest, ensures that it matches the requested digest.
func validateManifestDigest(ref reference.Named, mfst distribution.Manifest) (ocispec.Descriptor, error) {
	mediaType, canonical, err := mfst.Payload()
	if err != nil {
		return ocispec.Descriptor{}, err
	}
	desc := ocispec.Descriptor{
		Digest:    digest.FromBytes(canonical),
		Size:      int64(len(canonical)),
		MediaType: mediaType,
	}

	// If pull by digest, then verify the manifest digest.
	if digested, isDigested := ref.(reference.Canonical); isDigested {
		if digested.Digest() != desc.Digest {
			err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
			return ocispec.Descriptor{}, err
		}
	}

	return desc, nil
}

// pullManifestList handles "manifest lists" which point to various
// platform-specific manifests.
func pullManifestList(ctx context.Context, ref reference.Named, repo distribution.Repository, mfstList manifestlist.DeserializedManifestList) ([]types.ImageManifest, error) {
	infos := []types.ImageManifest{}

	if _, err := validateManifestDigest(ref, mfstList); err != nil {
		return nil, err
	}

	for _, manifestDescriptor := range mfstList.Manifests {
		manSvc, err := repo.Manifests(ctx)
		if err != nil {
			return nil, err
		}
		manifest, err := manSvc.Get(ctx, manifestDescriptor.Digest)
		if err != nil {
			return nil, err
		}
		v, ok := manifest.(*schema2.DeserializedManifest)
		if !ok {
			return nil, fmt.Errorf("unsupported manifest format: %v", v)
		}

		manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest)
		if err != nil {
			return nil, err
		}
		imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v)
		if err != nil {
			return nil, err
		}

		// Replace platform from config
		imageManifest.Descriptor.Platform = types.OCIPlatform(&manifestDescriptor.Platform)

		infos = append(infos, imageManifest)
	}
	return infos, nil
}

func continueOnError(err error) bool {
	switch v := err.(type) {
	case errcode.Errors:
		if len(v) == 0 {
			return true
		}
		return continueOnError(v[0])
	case errcode.Error:
		e := err.(errcode.Error)
		switch e.Code {
		case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown:
			return true
		}
		return false
	case *distclient.UnexpectedHTTPResponseError:
		return true
	}
	return false
}

func (c *client) iterateEndpoints(ctx context.Context, namedRef reference.Named, each func(context.Context, distribution.Repository, reference.Named) (bool, error)) error {
	endpoints, err := allEndpoints(namedRef)
	if err != nil {
		return err
	}

	repoInfo, err := registry.ParseRepositoryInfo(namedRef)
	if err != nil {
		return err
	}

	confirmedTLSRegistries := make(map[string]bool)
	for _, endpoint := range endpoints {

		if endpoint.Version == registry.APIVersion1 {
			logrus.Debugf("skipping v1 endpoint %s", endpoint.URL)
			continue
		}

		if endpoint.URL.Scheme != "https" {
			if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS {
				logrus.Debugf("skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL)
				continue
			}
		}

		if c.insecureRegistry {
			endpoint.TLSConfig.InsecureSkipVerify = true
		}
		repoEndpoint := repositoryEndpoint{endpoint: endpoint, info: repoInfo}
		repo, err := c.getRepositoryForReference(ctx, namedRef, repoEndpoint)
		if err != nil {
			logrus.Debugf("error with repo endpoint %s: %s", repoEndpoint, err)
			if _, ok := err.(ErrHTTPProto); ok {
				continue
			}
			return err
		}

		if endpoint.URL.Scheme == "http" && !c.insecureRegistry {
			logrus.Debugf("skipping non-tls registry endpoint: %s", endpoint.URL)
			continue
		}
		done, err := each(ctx, repo, namedRef)
		if err != nil {
			if continueOnError(err) {
				if endpoint.URL.Scheme == "https" {
					confirmedTLSRegistries[endpoint.URL.Host] = true
				}
				logrus.Debugf("continuing on error (%T) %s", err, err)
				continue
			}
			logrus.Debugf("not continuing on error (%T) %s", err, err)
			return err
		}
		if done {
			return nil
		}
	}
	return newNotFoundError(namedRef.String())
}

// allEndpoints returns a list of endpoints ordered by priority (v2, https, v1).
func allEndpoints(namedRef reference.Named) ([]registry.APIEndpoint, error) {
	repoInfo, err := registry.ParseRepositoryInfo(namedRef)
	if err != nil {
		return nil, err
	}
	registryService, err := registry.NewService(registry.ServiceOptions{})
	if err != nil {
		return []registry.APIEndpoint{}, err
	}
	endpoints, err := registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name))
	logrus.Debugf("endpoints for %s: %v", namedRef, endpoints)
	return endpoints, err
}

type notFoundError struct {
	object string
}

func newNotFoundError(ref string) *notFoundError {
	return &notFoundError{object: ref}
}

func (n *notFoundError) Error() string {
	return fmt.Sprintf("no such manifest: %s", n.object)
}

// NotFound interface
func (n *notFoundError) NotFound() {}

// IsNotFound returns true if the error is a not found error
func IsNotFound(err error) bool {
	_, ok := err.(notFound)
	return ok
}

type notFound interface {
	NotFound()
}
<commit_msg>fix insecure manifest inspect with restrictive certs perms<commit_after>package client

import (
	"context"
	"encoding/json"
	"fmt"

	"github.com/docker/cli/cli/manifest/types"
	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest/manifestlist"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/api/errcode"
	"github.com/docker/distribution/registry/api/v2"
	distclient "github.com/docker/distribution/registry/client"
	"github.com/docker/docker/registry"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// fetchManifest pulls a manifest from a registry and returns it. An error
// is returned if no manifest is found matching namedRef.
func fetchManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (types.ImageManifest, error) {
	manifest, err := getManifest(ctx, repo, ref)
	if err != nil {
		return types.ImageManifest{}, err
	}

	switch v := manifest.(type) {
	// Removed Schema 1 support
	case *schema2.DeserializedManifest:
		imageManifest, err := pullManifestSchemaV2(ctx, ref, repo, *v)
		if err != nil {
			return types.ImageManifest{}, err
		}
		return imageManifest, nil
	case *manifestlist.DeserializedManifestList:
		return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref)
	}
	return types.ImageManifest{}, errors.Errorf("%s is not a manifest", ref)
}

func fetchList(ctx context.Context, repo distribution.Repository, ref reference.Named) ([]types.ImageManifest, error) {
	manifest, err := getManifest(ctx, repo, ref)
	if err != nil {
		return nil, err
	}

	switch v := manifest.(type) {
	case *manifestlist.DeserializedManifestList:
		imageManifests, err := pullManifestList(ctx, ref, repo, *v)
		if err != nil {
			return nil, err
		}
		return imageManifests, nil
	default:
		return nil, errors.Errorf("unsupported manifest format: %v", v)
	}
}

func getManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (distribution.Manifest, error) {
	manSvc, err := repo.Manifests(ctx)
	if err != nil {
		return nil, err
	}

	dgst, opts, err := getManifestOptionsFromReference(ref)
	if err != nil {
		return nil, errors.Errorf("image manifest for %q does not exist", ref)
	}
	return manSvc.Get(ctx, dgst, opts...)
}

func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst schema2.DeserializedManifest) (types.ImageManifest, error) {
	manifestDesc, err := validateManifestDigest(ref, mfst)
	if err != nil {
		return types.ImageManifest{}, err
	}
	configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo)
	if err != nil {
		return types.ImageManifest{}, err
	}

	if manifestDesc.Platform == nil {
		manifestDesc.Platform = &ocispec.Platform{}
	}

	// Fill in os and architecture fields from config JSON
	if err := json.Unmarshal(configJSON, manifestDesc.Platform); err != nil {
		return types.ImageManifest{}, err
	}

	return types.NewImageManifest(ref, manifestDesc, &mfst), nil
}

func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) {
	blobs := repo.Blobs(ctx)
	configJSON, err := blobs.Get(ctx, dgst)
	if err != nil {
		return nil, err
	}

	verifier := dgst.Verifier()
	if err != nil {
		return nil, err
	}
	if _, err := verifier.Write(configJSON); err != nil {
		return nil, err
	}
	if !verifier.Verified() {
		return nil, errors.Errorf("image config verification failed for digest %s", dgst)
	}
	return configJSON, nil
}

// validateManifestDigest computes the manifest digest, and, if pulling by
// digest, ensures that it matches the requested digest.
func validateManifestDigest(ref reference.Named, mfst distribution.Manifest) (ocispec.Descriptor, error) {
	mediaType, canonical, err := mfst.Payload()
	if err != nil {
		return ocispec.Descriptor{}, err
	}
	desc := ocispec.Descriptor{
		Digest:    digest.FromBytes(canonical),
		Size:      int64(len(canonical)),
		MediaType: mediaType,
	}

	// If pull by digest, then verify the manifest digest.
	if digested, isDigested := ref.(reference.Canonical); isDigested {
		if digested.Digest() != desc.Digest {
			err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
			return ocispec.Descriptor{}, err
		}
	}

	return desc, nil
}

// pullManifestList handles "manifest lists" which point to various
// platform-specific manifests.
func pullManifestList(ctx context.Context, ref reference.Named, repo distribution.Repository, mfstList manifestlist.DeserializedManifestList) ([]types.ImageManifest, error) {
	infos := []types.ImageManifest{}

	if _, err := validateManifestDigest(ref, mfstList); err != nil {
		return nil, err
	}

	for _, manifestDescriptor := range mfstList.Manifests {
		manSvc, err := repo.Manifests(ctx)
		if err != nil {
			return nil, err
		}
		manifest, err := manSvc.Get(ctx, manifestDescriptor.Digest)
		if err != nil {
			return nil, err
		}
		v, ok := manifest.(*schema2.DeserializedManifest)
		if !ok {
			return nil, fmt.Errorf("unsupported manifest format: %v", v)
		}

		manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest)
		if err != nil {
			return nil, err
		}
		imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v)
		if err != nil {
			return nil, err
		}

		// Replace platform from config
		imageManifest.Descriptor.Platform = types.OCIPlatform(&manifestDescriptor.Platform)

		infos = append(infos, imageManifest)
	}
	return infos, nil
}

func continueOnError(err error) bool {
	switch v := err.(type) {
	case errcode.Errors:
		if len(v) == 0 {
			return true
		}
		return continueOnError(v[0])
	case errcode.Error:
		e := err.(errcode.Error)
		switch e.Code {
		case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown:
			return true
		}
		return false
	case *distclient.UnexpectedHTTPResponseError:
		return true
	}
	return false
}

func (c *client) iterateEndpoints(ctx context.Context, namedRef reference.Named, each func(context.Context, distribution.Repository, reference.Named) (bool, error)) error {
	endpoints, err := allEndpoints(namedRef, c.insecureRegistry)
	if err != nil {
		return err
	}

	repoInfo, err := registry.ParseRepositoryInfo(namedRef)
	if err != nil {
		return err
	}

	confirmedTLSRegistries := make(map[string]bool)
	for _, endpoint := range endpoints {

		if endpoint.Version == registry.APIVersion1 {
			logrus.Debugf("skipping v1 endpoint %s", endpoint.URL)
			continue
		}

		if endpoint.URL.Scheme != "https" {
			if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS {
				logrus.Debugf("skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL)
				continue
			}
		}

		if c.insecureRegistry {
			endpoint.TLSConfig.InsecureSkipVerify = true
		}
		repoEndpoint := repositoryEndpoint{endpoint: endpoint, info: repoInfo}
		repo, err := c.getRepositoryForReference(ctx, namedRef, repoEndpoint)
		if err != nil {
			logrus.Debugf("error with repo endpoint %s: %s", repoEndpoint, err)
			if _, ok := err.(ErrHTTPProto); ok {
				continue
			}
			return err
		}

		if endpoint.URL.Scheme == "http" && !c.insecureRegistry {
			logrus.Debugf("skipping non-tls registry endpoint: %s", endpoint.URL)
			continue
		}
		done, err := each(ctx, repo, namedRef)
		if err != nil {
			if continueOnError(err) {
				if endpoint.URL.Scheme == "https" {
					confirmedTLSRegistries[endpoint.URL.Host] = true
				}
				logrus.Debugf("continuing on error (%T) %s", err, err)
				continue
			}
			logrus.Debugf("not continuing on error (%T) %s", err, err)
			return err
		}
		if done {
			return nil
		}
	}
	return newNotFoundError(namedRef.String())
}

// allEndpoints returns a list of endpoints ordered by priority (v2, https, v1).
func allEndpoints(namedRef reference.Named, insecure bool) ([]registry.APIEndpoint, error) {
	repoInfo, err := registry.ParseRepositoryInfo(namedRef)
	if err != nil {
		return nil, err
	}

	var serviceOpts registry.ServiceOptions
	if insecure {
		logrus.Debugf("allowing insecure registry for: %s", reference.Domain(namedRef))
		serviceOpts.InsecureRegistries = []string{reference.Domain(namedRef)}
	}
	registryService, err := registry.NewService(serviceOpts)
	if err != nil {
		return []registry.APIEndpoint{}, err
	}
	endpoints, err := registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name))
	logrus.Debugf("endpoints for %s: %v", namedRef, endpoints)
	return endpoints, err
}

type notFoundError struct {
	object string
}

func newNotFoundError(ref string) *notFoundError {
	return &notFoundError{object: ref}
}

func (n *notFoundError) Error() string {
	return fmt.Sprintf("no such manifest: %s", n.object)
}

// NotFound interface
func (n *notFoundError) NotFound() {}

// IsNotFound returns true if the error is a not found error
func IsNotFound(err error) bool {
	_, ok := err.(notFound)
	return ok
}

type notFound interface {
	NotFound()
}
<|endoftext|>


<commit_before>package vault

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/vault/helper/forwarding"
	"golang.org/x/net/context"
	"golang.org/x/net/http2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"
)

const (
	clusterListenerAcceptDeadline = 500 * time.Millisecond
	heartbeatInterval             = 30 * time.Second
	requestForwardingALPN         = "req_fw_sb-act_v1"
)

// Starts the listeners and servers necessary to handle forwarded requests
func (c *Core) startForwarding() error {
	c.logger.Trace("core: cluster listener setup function")
	defer c.logger.Trace("core: leaving cluster listener setup function")

	// Clean up in case we have transitioned from a client to a server
	c.requestForwardingConnectionLock.Lock()
	c.clearForwardingClients()
	c.requestForwardingConnectionLock.Unlock()

	// Resolve locally to avoid races
	ha := c.ha != nil

	// Get our TLS config
	tlsConfig, err := c.ClusterTLSConfig()
	if err != nil {
		c.logger.Error("core: failed to get tls configuration when starting forwarding", "error", err)
		return err
	}

	// The server supports all of the possible protos
	tlsConfig.NextProtos = []string{"h2", requestForwardingALPN}

	// Create our RPC server and register the request handler server
	c.clusterParamsLock.Lock()

	if c.rpcServer != nil {
		c.logger.Warn("core: forwarding rpc server already running")
		return nil
	}

	c.rpcServer = grpc.NewServer(
		grpc.KeepaliveParams(keepalive.ServerParameters{
			Time: 2 * heartbeatInterval,
		}),
	)

	if ha && c.clusterHandler != nil {
		RegisterRequestForwardingServer(c.rpcServer, &forwardedRequestRPCServer{
			core:    c,
			handler: c.clusterHandler,
		})
	}
	c.clusterParamsLock.Unlock()

	// Create the HTTP/2 server that will be shared by both RPC and regular
	// duties. Doing it this way instead of listening via the server and gRPC
	// allows us to re-use the same port via ALPN. We can just tell the server
	// to serve a given conn and which handler to use.
	fws := &http2.Server{}

	// Shutdown coordination logic
	var shutdown uint32
	shutdownWg := &sync.WaitGroup{}

	for _, addr := range c.clusterListenerAddrs {
		shutdownWg.Add(1)

		// Force a local resolution to avoid data races
		laddr := addr

		// Start our listening loop
		go func() {
			defer shutdownWg.Done()

			if c.logger.IsInfo() {
				c.logger.Info("core/startClusterListener: starting listener", "listener_address", laddr)
			}

			// Create a TCP listener. We do this separately and specifically
			// with TCP so that we can set deadlines.
			tcpLn, err := net.ListenTCP("tcp", laddr)
			if err != nil {
				c.logger.Error("core/startClusterListener: error starting listener", "error", err)
				return
			}

			// Wrap the listener with TLS
			tlsLn := tls.NewListener(tcpLn, tlsConfig)
			defer tlsLn.Close()

			if c.logger.IsInfo() {
				c.logger.Info("core/startClusterListener: serving cluster requests", "cluster_listen_address", tlsLn.Addr())
			}

			for {
				if atomic.LoadUint32(&shutdown) > 0 {
					return
				}

				// Set the deadline for the accept call. If it passes we'll get
				// an error, causing us to check the condition at the top
				// again.
				tcpLn.SetDeadline(time.Now().Add(clusterListenerAcceptDeadline))

				// Accept the connection
				conn, err := tlsLn.Accept()
				if conn != nil {
					// Always defer although it may be closed ahead of time
					defer conn.Close()
				}
				if err != nil {
					continue
				}

				// Type assert to TLS connection and handshake to populate the
				// connection state
				tlsConn := conn.(*tls.Conn)
				err = tlsConn.Handshake()
				if err != nil {
					if c.logger.IsDebug() {
						c.logger.Debug("core: error handshaking cluster connection", "error", err)
					}
					if conn != nil {
						conn.Close()
					}
					continue
				}

				switch tlsConn.ConnectionState().NegotiatedProtocol {
				case requestForwardingALPN:
					if !ha {
						conn.Close()
						continue
					}

					c.logger.Trace("core: got request forwarding connection")
					c.clusterParamsLock.RLock()
					go fws.ServeConn(conn, &http2.ServeConnOpts{
						Handler: c.rpcServer,
					})
					c.clusterParamsLock.RUnlock()

				default:
					c.logger.Debug("core: unknown negotiated protocol on cluster port")
					conn.Close()
					continue
				}
			}
		}()
	}

	// This is in its own goroutine so that we don't block the main thread, and
	// thus we use atomic and channels to coordinate
	// However, because you can't query the status of a channel, we set a bool
	// here while we have the state lock to know whether to actually send a
	// shutdown (e.g. whether the channel will block). See issue #2083.
	c.clusterListenersRunning = true
	go func() {
		// If we get told to shut down...
		<-c.clusterListenerShutdownCh

		// Stop the RPC server
		c.logger.Info("core: shutting down forwarding rpc listeners")
		c.clusterParamsLock.Lock()
		c.rpcServer.Stop()
		c.rpcServer = nil
		c.clusterParamsLock.Unlock()
		c.logger.Info("core: forwarding rpc listeners stopped")

		// Set the shutdown flag. This will cause the listeners to shut down
		// within the deadline in clusterListenerAcceptDeadline
		atomic.StoreUint32(&shutdown, 1)

		// Wait for them all to shut down
		shutdownWg.Wait()
		c.logger.Info("core: rpc listeners successfully shut down")

		// Tell the main thread that shutdown is done.
		c.clusterListenerShutdownSuccessCh <- struct{}{}
	}()

	return nil
}

// refreshRequestForwardingConnection ensures that the client/transport are
// alive and that the current active address value matches the most
// recently-known address.
func (c *Core) refreshRequestForwardingConnection(clusterAddr string) error {
	c.logger.Trace("core: refreshing forwarding connection")
	defer c.logger.Trace("core: done refreshing forwarding connection")

	c.requestForwardingConnectionLock.Lock()
	defer c.requestForwardingConnectionLock.Unlock()

	// Clean things up first
	c.clearForwardingClients()

	// If we don't have anything to connect to, just return
	if clusterAddr == "" {
		return nil
	}

	clusterURL, err := url.Parse(clusterAddr)
	if err != nil {
		c.logger.Error("core: error parsing cluster address attempting to refresh forwarding connection", "error", err)
		return err
	}

	// Set up grpc forwarding handling
	// It's not really insecure, but we have to dial manually to get the
	// ALPN header right. It's just "insecure" because GRPC isn't managing
	// the TLS state.
	ctx, cancelFunc := context.WithCancel(context.Background())
	c.rpcClientConn, err = grpc.DialContext(ctx, clusterURL.Host,
		grpc.WithDialer(c.getGRPCDialer(requestForwardingALPN, "", nil)),
		grpc.WithInsecure(), // it's not, we handle it in the dialer
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time: 2 * heartbeatInterval,
		}))
	if err != nil {
		cancelFunc()
		c.logger.Error("core: err setting up forwarding rpc client", "error", err)
		return err
	}
	c.rpcClientConnContext = ctx
	c.rpcClientConnCancelFunc = cancelFunc
	c.rpcForwardingClient = &forwardingClient{
		RequestForwardingClient: NewRequestForwardingClient(c.rpcClientConn),
		core:        c,
		echoTicker:  time.NewTicker(heartbeatInterval),
		echoContext: ctx,
	}
	c.rpcForwardingClient.startHeartbeat()

	return nil
}

func (c *Core) clearForwardingClients() {
	c.logger.Trace("core: clearing forwarding clients")
	defer c.logger.Trace("core: done clearing forwarding clients")

	if c.rpcClientConnCancelFunc != nil {
		c.rpcClientConnCancelFunc()
		c.rpcClientConnCancelFunc = nil
	}
	if c.rpcClientConn != nil {
		c.rpcClientConn.Close()
		c.rpcClientConn = nil
	}

	c.rpcClientConnContext = nil
	c.rpcForwardingClient = nil
}

// ForwardRequest forwards a given request to the active node and returns the
// response.
func (c *Core) ForwardRequest(req *http.Request) (int, http.Header, []byte, error) {
	c.requestForwardingConnectionLock.RLock()
	defer c.requestForwardingConnectionLock.RUnlock()

	if c.rpcForwardingClient == nil {
		return 0, nil, nil, ErrCannotForward
	}

	freq, err := forwarding.GenerateForwardedRequest(req)
	if err != nil {
		c.logger.Error("core: error creating forwarding RPC request", "error", err)
		return 0, nil, nil, fmt.Errorf("error creating forwarding RPC request")
	}
	if freq == nil {
		c.logger.Error("core: got nil forwarding RPC request")
		return 0, nil, nil, fmt.Errorf("got nil forwarding RPC request")
	}
	resp, err := c.rpcForwardingClient.ForwardRequest(c.rpcClientConnContext, freq)
	if err != nil {
		c.logger.Error("core: error during forwarded RPC request", "error", err)
		return 0, nil, nil, fmt.Errorf("error during forwarding RPC request")
	}

	var header http.Header
	if resp.HeaderEntries != nil {
		header = make(http.Header)
		for k, v := range resp.HeaderEntries {
			for _, j := range v.Values {
				header.Add(k, j)
			}
		}
	}

	return int(resp.StatusCode), header, resp.Body, nil
}

// getGRPCDialer is used to return a dialer that has the correct TLS
// configuration. Otherwise gRPC tries to be helpful and stomps all over our
// NextProtos.
func (c *Core) getGRPCDialer(alpnProto, serverName string, caCert *x509.Certificate) func(string, time.Duration) (net.Conn, error) {
	return func(addr string, timeout time.Duration) (net.Conn, error) {
		tlsConfig, err := c.ClusterTLSConfig()
		if err != nil {
			c.logger.Error("core: failed to get tls configuration", "error", err)
			return nil, err
		}
		if serverName != "" {
			tlsConfig.ServerName = serverName
		}
		if caCert != nil {
			pool := x509.NewCertPool()
			pool.AddCert(caCert)
			tlsConfig.RootCAs = pool
			tlsConfig.ClientCAs = pool
		}
		c.logger.Trace("core: creating rpc dialer", "host", tlsConfig.ServerName)

		tlsConfig.NextProtos = []string{alpnProto}
		dialer := &net.Dialer{
			Timeout: timeout,
		}
		return tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
	}
}

type forwardedRequestRPCServer struct {
	core    *Core
	handler http.Handler
}

func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *forwarding.Request) (*forwarding.Response, error) {
	//s.core.logger.Trace("forwarding: serving rpc forwarded request")

	// Parse an http.Request out of it
	req, err := forwarding.ParseForwardedRequest(freq)
	if err != nil {
		return nil, err
	}

	// A very dummy response writer that doesn't follow normal semantics, just
	// lets you write a status code (last written wins) and a body. But it
	// meets the interface requirements.
	w := forwarding.NewRPCResponseWriter()

	resp := &forwarding.Response{}

	runRequest := func() {
		defer func() {
			// Logic here comes mostly from the Go source code
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				s.core.logger.Error("forwarding: panic serving request", "path", req.URL.Path, "error", err, "stacktrace", buf)
			}
		}()
		s.handler.ServeHTTP(w, req)
	}
	runRequest()
	resp.StatusCode = uint32(w.StatusCode())
	resp.Body = w.Body().Bytes()

	header := w.Header()
	if header != nil {
		resp.HeaderEntries = make(map[string]*forwarding.HeaderEntry, len(header))
		for k, v := range header {
			resp.HeaderEntries[k] = &forwarding.HeaderEntry{
				Values: v,
			}
		}
	}

	return resp, nil
}

func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) {
	if in.ClusterAddr != "" {
		s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, nil, 0)
	}
	return &EchoReply{
		Message: "pong",
	}, nil
}

type forwardingClient struct {
	RequestForwardingClient

	core *Core

	echoTicker  *time.Ticker
	echoContext context.Context
}

// NOTE: we also take advantage of gRPC's keepalive bits, but as we send data
// with these requests it's useful to keep this as well
func (c *forwardingClient) startHeartbeat() {
	go func() {
		tick := func() {
			c.core.stateLock.RLock()
			clusterAddr := c.core.clusterAddr
			c.core.stateLock.RUnlock()

			ctx, cancel := context.WithTimeout(c.echoContext, 2*time.Second)
			resp, err := c.RequestForwardingClient.Echo(ctx, &EchoRequest{
				Message:     "ping",
				ClusterAddr: clusterAddr,
			})
			cancel()
			if err != nil {
				c.core.logger.Debug("forwarding: error sending echo request to active node", "error", err)
				return
			}
			if resp == nil {
				c.core.logger.Debug("forwarding: empty echo response from active node")
				return
			}
			if resp.Message != "pong" {
				c.core.logger.Debug("forwarding: unexpected echo response from active node", "message", resp.Message)
				return
			}
			c.core.logger.Trace("forwarding: successful heartbeat")
		}

		tick()

		for {
			select {
			case <-c.echoContext.Done():
				c.echoTicker.Stop()
				c.core.logger.Trace("forwarding: stopping heartbeating")
				return
			case <-c.echoTicker.C:
				tick()
			}
		}
	}()
}
<commit_msg>Fix memory leak when a connection would hit the cluster port and go away (#3513)<commit_after>package vault

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/vault/helper/forwarding"
	"golang.org/x/net/context"
	"golang.org/x/net/http2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"
)

const (
	clusterListenerAcceptDeadline = 500 * time.Millisecond
	heartbeatInterval             = 30 * time.Second
	requestForwardingALPN         = "req_fw_sb-act_v1"
)

// Starts the listeners and servers necessary to handle forwarded requests
func (c *Core) startForwarding() error {
	c.logger.Trace("core: cluster listener setup function")
	defer c.logger.Trace("core: leaving cluster listener setup function")

	// Clean up in case we have transitioned from a client to a server
	c.requestForwardingConnectionLock.Lock()
	c.clearForwardingClients()
	c.requestForwardingConnectionLock.Unlock()

	// Resolve locally to avoid races
	ha := c.ha != nil

	// Get our TLS config
	tlsConfig, err := c.ClusterTLSConfig()
	if err != nil {
		c.logger.Error("core: failed to get tls configuration when starting forwarding", "error", err)
		return err
	}

	// The server supports all of the possible protos
	tlsConfig.NextProtos = []string{"h2", requestForwardingALPN}

	// Create our RPC server and register the request handler server
	c.clusterParamsLock.Lock()

	if c.rpcServer != nil {
		c.logger.Warn("core: forwarding rpc server already running")
		return nil
	}

	c.rpcServer = grpc.NewServer(
		grpc.KeepaliveParams(keepalive.ServerParameters{
			Time: 2 * heartbeatInterval,
		}),
	)

	if ha && c.clusterHandler != nil {
		RegisterRequestForwardingServer(c.rpcServer, &forwardedRequestRPCServer{
			core:    c,
			handler: c.clusterHandler,
		})
	}
	c.clusterParamsLock.Unlock()

	// Create the HTTP/2 server that will be shared by both RPC and regular
	// duties. Doing it this way instead of listening via the server and gRPC
	// allows us to re-use the same port via ALPN. We can just tell the server
	// to serve a given conn and which handler to use.
	fws := &http2.Server{}

	// Shutdown coordination logic
	var shutdown uint32
	shutdownWg := &sync.WaitGroup{}

	for _, addr := range c.clusterListenerAddrs {
		shutdownWg.Add(1)

		// Force a local resolution to avoid data races
		laddr := addr

		// Start our listening loop
		go func() {
			defer shutdownWg.Done()

			if c.logger.IsInfo() {
				c.logger.Info("core/startClusterListener: starting listener", "listener_address", laddr)
			}

			// Create a TCP listener. We do this separately and specifically
			// with TCP so that we can set deadlines.
			tcpLn, err := net.ListenTCP("tcp", laddr)
			if err != nil {
				c.logger.Error("core/startClusterListener: error starting listener", "error", err)
				return
			}

			// Wrap the listener with TLS
			tlsLn := tls.NewListener(tcpLn, tlsConfig)
			defer tlsLn.Close()

			if c.logger.IsInfo() {
				c.logger.Info("core/startClusterListener: serving cluster requests", "cluster_listen_address", tlsLn.Addr())
			}

			for {
				if atomic.LoadUint32(&shutdown) > 0 {
					return
				}

				// Set the deadline for the accept call. If it passes we'll get
				// an error, causing us to check the condition at the top
				// again.
				tcpLn.SetDeadline(time.Now().Add(clusterListenerAcceptDeadline))

				// Accept the connection
				conn, err := tlsLn.Accept()
				if err != nil || conn == nil {
					if conn != nil {
						conn.Close()
					}
					continue
				}

				// Type assert to TLS connection and handshake to populate the
				// connection state
				tlsConn := conn.(*tls.Conn)
				err = tlsConn.Handshake()
				if err != nil {
					if c.logger.IsDebug() {
						c.logger.Debug("core: error handshaking cluster connection", "error", err)
					}
					conn.Close()
					continue
				}

				switch tlsConn.ConnectionState().NegotiatedProtocol {
				case requestForwardingALPN:
					if !ha {
						conn.Close()
						continue
					}

					c.logger.Trace("core: got request forwarding connection")
					c.clusterParamsLock.RLock()
					rpcServer := c.rpcServer
					c.clusterParamsLock.RUnlock()
					go func() {
						defer conn.Close()
						fws.ServeConn(conn, &http2.ServeConnOpts{
							Handler: rpcServer,
						})
					}()

				default:
					c.logger.Debug("core: unknown negotiated protocol on cluster port")
					conn.Close()
					continue
				}
			}
		}()
	}

	// This is in its own goroutine so that we don't block the main thread, and
	// thus we use atomic and channels to coordinate
	// However, because you can't query the status of a channel, we set a bool
	// here while we have the state lock to know whether to actually send a
	// shutdown (e.g. whether the channel will block). See issue #2083.
	c.clusterListenersRunning = true
	go func() {
		// If we get told to shut down...
		<-c.clusterListenerShutdownCh

		// Stop the RPC server
		c.logger.Info("core: shutting down forwarding rpc listeners")
		c.clusterParamsLock.Lock()
		c.rpcServer.Stop()
		c.rpcServer = nil
		c.clusterParamsLock.Unlock()
		c.logger.Info("core: forwarding rpc listeners stopped")

		// Set the shutdown flag. This will cause the listeners to shut down
		// within the deadline in clusterListenerAcceptDeadline
		atomic.StoreUint32(&shutdown, 1)

		// Wait for them all to shut down
		shutdownWg.Wait()
		c.logger.Info("core: rpc listeners successfully shut down")

		// Tell the main thread that shutdown is done.
		c.clusterListenerShutdownSuccessCh <- struct{}{}
	}()

	return nil
}

// refreshRequestForwardingConnection ensures that the client/transport are
// alive and that the current active address value matches the most
// recently-known address.
func (c *Core) refreshRequestForwardingConnection(clusterAddr string) error {
	c.logger.Trace("core: refreshing forwarding connection")
	defer c.logger.Trace("core: done refreshing forwarding connection")

	c.requestForwardingConnectionLock.Lock()
	defer c.requestForwardingConnectionLock.Unlock()

	// Clean things up first
	c.clearForwardingClients()

	// If we don't have anything to connect to, just return
	if clusterAddr == "" {
		return nil
	}

	clusterURL, err := url.Parse(clusterAddr)
	if err != nil {
		c.logger.Error("core: error parsing cluster address attempting to refresh forwarding connection", "error", err)
		return err
	}

	// Set up grpc forwarding handling
	// It's not really insecure, but we have to dial manually to get the
	// ALPN header right. It's just "insecure" because GRPC isn't managing
	// the TLS state.
	ctx, cancelFunc := context.WithCancel(context.Background())
	c.rpcClientConn, err = grpc.DialContext(ctx, clusterURL.Host,
		grpc.WithDialer(c.getGRPCDialer(requestForwardingALPN, "", nil)),
		grpc.WithInsecure(), // it's not, we handle it in the dialer
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time: 2 * heartbeatInterval,
		}))
	if err != nil {
		cancelFunc()
		c.logger.Error("core: err setting up forwarding rpc client", "error", err)
		return err
	}
	c.rpcClientConnContext = ctx
	c.rpcClientConnCancelFunc = cancelFunc
	c.rpcForwardingClient = &forwardingClient{
		RequestForwardingClient: NewRequestForwardingClient(c.rpcClientConn),
		core:        c,
		echoTicker:  time.NewTicker(heartbeatInterval),
		echoContext: ctx,
	}
	c.rpcForwardingClient.startHeartbeat()

	return nil
}

func (c *Core) clearForwardingClients() {
	c.logger.Trace("core: clearing forwarding clients")
	defer c.logger.Trace("core: done clearing forwarding clients")

	if c.rpcClientConnCancelFunc != nil {
		c.rpcClientConnCancelFunc()
		c.rpcClientConnCancelFunc = nil
	}
	if c.rpcClientConn != nil {
		c.rpcClientConn.Close()
		c.rpcClientConn = nil
	}

	c.rpcClientConnContext = nil
	c.rpcForwardingClient = nil
}

// ForwardRequest forwards a given request to the active node and returns the
// response.
func (c *Core) ForwardRequest(req *http.Request) (int, http.Header, []byte, error) {
	c.requestForwardingConnectionLock.RLock()
	defer c.requestForwardingConnectionLock.RUnlock()

	if c.rpcForwardingClient == nil {
		return 0, nil, nil, ErrCannotForward
	}

	freq, err := forwarding.GenerateForwardedRequest(req)
	if err != nil {
		c.logger.Error("core: error creating forwarding RPC request", "error", err)
		return 0, nil, nil, fmt.Errorf("error creating forwarding RPC request")
	}
	if freq == nil {
		c.logger.Error("core: got nil forwarding RPC request")
		return 0, nil, nil, fmt.Errorf("got nil forwarding RPC request")
	}
	resp, err := c.rpcForwardingClient.ForwardRequest(c.rpcClientConnContext, freq)
	if err != nil {
		c.logger.Error("core: error during forwarded RPC request", "error", err)
		return 0, nil, nil, fmt.Errorf("error during forwarding RPC request")
	}

	var header http.Header
	if resp.HeaderEntries != nil {
		header = make(http.Header)
		for k, v := range resp.HeaderEntries {
			for _, j := range v.Values {
				header.Add(k, j)
			}
		}
	}

	return int(resp.StatusCode), header, resp.Body, nil
}

// getGRPCDialer is used to return a dialer that has the correct TLS
// configuration. Otherwise gRPC tries to be helpful and stomps all over our
// NextProtos.
func (c *Core) getGRPCDialer(alpnProto, serverName string, caCert *x509.Certificate) func(string, time.Duration) (net.Conn, error) {
	return func(addr string, timeout time.Duration) (net.Conn, error) {
		tlsConfig, err := c.ClusterTLSConfig()
		if err != nil {
			c.logger.Error("core: failed to get tls configuration", "error", err)
			return nil, err
		}
		if serverName != "" {
			tlsConfig.ServerName = serverName
		}
		if caCert != nil {
			pool := x509.NewCertPool()
			pool.AddCert(caCert)
			tlsConfig.RootCAs = pool
			tlsConfig.ClientCAs = pool
		}
		c.logger.Trace("core: creating rpc dialer", "host", tlsConfig.ServerName)

		tlsConfig.NextProtos = []string{alpnProto}
		dialer := &net.Dialer{
			Timeout: timeout,
		}
		return tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
	}
}

type forwardedRequestRPCServer struct {
	core    *Core
	handler http.Handler
}

func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *forwarding.Request) (*forwarding.Response, error) {
	//s.core.logger.Trace("forwarding: serving rpc forwarded request")

	// Parse an http.Request out of it
	req, err := forwarding.ParseForwardedRequest(freq)
	if err != nil {
		return nil, err
	}

	// A very dummy response writer that doesn't follow normal semantics, just
	// lets you write a status code (last written wins) and a body. But it
	// meets the interface requirements.
	w := forwarding.NewRPCResponseWriter()

	resp := &forwarding.Response{}

	runRequest := func() {
		defer func() {
			// Logic here comes mostly from the Go source code
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				s.core.logger.Error("forwarding: panic serving request", "path", req.URL.Path, "error", err, "stacktrace", buf)
			}
		}()
		s.handler.ServeHTTP(w, req)
	}
	runRequest()
	resp.StatusCode = uint32(w.StatusCode())
	resp.Body = w.Body().Bytes()

	header := w.Header()
	if header != nil {
		resp.HeaderEntries = make(map[string]*forwarding.HeaderEntry, len(header))
		for k, v := range header {
			resp.HeaderEntries[k] = &forwarding.HeaderEntry{
				Values: v,
			}
		}
	}

	return resp, nil
}

func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) {
	if in.ClusterAddr != "" {
		s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, nil, 0)
	}
	return &EchoReply{
		Message: "pong",
	}, nil
}

type forwardingClient struct {
	RequestForwardingClient

	core *Core

	echoTicker  *time.Ticker
	echoContext context.Context
}

// NOTE: we also take advantage of gRPC's keepalive bits, but as we send data
// with these requests it's useful to keep this as well
func (c *forwardingClient) startHeartbeat() {
	go func() {
		tick := func() {
			c.core.stateLock.RLock()
			clusterAddr := c.core.clusterAddr
			c.core.stateLock.RUnlock()

			ctx, cancel := context.WithTimeout(c.echoContext, 2*time.Second)
			resp, err := c.RequestForwardingClient.Echo(ctx, &EchoRequest{
				Message:     "ping",
				ClusterAddr: clusterAddr,
			})
			cancel()
			if err != nil {
				c.core.logger.Debug("forwarding: error sending echo request to active node", "error", err)
				return
			}
			if resp == nil {
				c.core.logger.Debug("forwarding: empty echo response from active node")
				return
			}
			if resp.Message != "pong" {
				c.core.logger.Debug("forwarding: unexpected echo response from active node", "message", resp.Message)
				return
			}
			c.core.logger.Trace("forwarding: successful heartbeat")
		}

		tick()

		for {
			select {
			case <-c.echoContext.Done():
				c.echoTicker.Stop()
				c.core.logger.Trace("forwarding: stopping heartbeating")
				return
			case <-c.echoTicker.C:
				tick()
			}
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
  "encoding/json"
  "os"
  "flag"
  "fmt"
  "io"
  "log"
  "net/http"
  "net/http/cookiejar"
  "net/url"
  "strings"
  "path"
  "code.google.com/p/go.net/html"
  "github.com/steveyen/gkvlite"
)

func Split2(str, sep string) (string, string) {
  s := strings.Split(str, sep)
  return s[0], s[1]
}

func fetch(url string) {
  res, err := http.Get(url)
  if err != nil {
    log.Fatal(err)
  }
  if err != nil {
    log.Fatal(err)
  }
  // http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
  fo, err := os.Create(path.Base(url))
  if err != nil {
    panic(err)
  }
  defer func() {
    if err := fo.Close(); err != nil {
      panic(err)
    }
  }()
  buf := make([]byte, 65536)
  for {
    n, err := res.Body.Read(buf)
    if err != nil && err != io.EOF {
      panic(err)
    }
    if n == 0 {
      break
    }

    if _, err := fo.Write(buf[:n]); err != nil {
      panic(err)
    }
  }
  res.Body.Close()
}

func Fuckoff(pres *http.Response) {
}

func FindLinks(body io.Reader) chan string {
  c := make(chan string)

  go func() {
    z := html.NewTokenizer(body)
    for {
      tt := z.Next()
      if tt == html.ErrorToken {
        break
      }
      if tt == html.StartTagToken {
        tn, _ := z.TagName()
        if len(tn) == 1 && tn[0] == 'a' {
          for {
            key, value, more := z.TagAttr()
            // http://stackoverflow.com/questions/14230145/what-is-the-best-way-to-convert-byte-array-to-string
            if string(key) == "href" {
              v := string(value)
              // http://codereview.stackexchange.com/questions/28386/fibonacci-generator-with-golang
              c <- v
            }
            if !more {
              break
            }
          }
        }
      }
    }
    c <- ""
  }()

  return c
}

const (
  _ = iota
  linkfound
  linkcrawled
)

func findBoardLinks(client *http.Client, start string, board_links map[string]int) (board_links_out map[string]int) {
  res, err := client.Get(start)
  if err != nil {
    log.Fatal(err)
  }
  //fmt.Printf("%s\n", start)

  c := FindLinks(res.Body)
  for {
    v := <- c
    if v == "" {
      break
    }
    //fmt.Printf("%v\n", v)
    if strings.Contains(v, start) && strings.Contains(v, "?board=") {
      // index.php?board=18.0;sort=starter -- drop ;.*
      if strings.Contains(v, ";") {
        pre, _ := Split2(v, ";")
        v = pre
      }
      //fmt.Printf("recursing\n")
      _, found := board_links[v]
      if !found {
        board_links[v] = linkfound
      }
    }
  }
  res.Body.Close()
  return board_links
}

func loadBoardLinks(client *http.Client, start string) (flat_links []string) {
  board_links := map[string]int{}
  board_links[start] = linkfound
  for {
    copy := board_links
    any := false
    for key, value := range copy {
      if value == linkcrawled {
        continue
      }
      fmt.Printf("%v\n", key)
      board_links = findBoardLinks(client, key, board_links)
      board_links[key] = linkcrawled
      any = true
    }
    if !any {
      break
    }
  }

  flat_links = []string{}
  for key, _ := range board_links {
    flat_links = append(flat_links, key)
  }
  return flat_links
}

func main() {
  //var start string
  //flag.StringVar(&start, "start", "", "starting url")
  //flag.Parse()

  flag.Parse()
  if len(flag.Args()) < 2 {
    log.Fatal("Usage: mirror start-url credentials")
  }
  start := flag.Args()[0]
  username, password := Split2(flag.Args()[1], ":")

  // http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
  options := cookiejar.Options{}
  cookie_jar, err := cookiejar.New(&options)
  client := &http.Client{
    Jar: cookie_jar,
  }
  pres, err := client.PostForm(start + "index.php?action=login2",
    url.Values{"user": {username}, "passwrd": {password}})
  if err != nil {
    log.Fatal(err)
  }
  Fuckoff(pres)

  f, err := os.OpenFile("smfmirror.gkvlite", os.O_RDWR, 0666)
  if err != nil {
    f, err = os.Create("smfmirror.gkvlite")
  }
  if err != nil {
    panic(err)
  }
  s, err := gkvlite.NewStore(f)
  if err != nil {
    panic(err)
  }
  c := s.GetCollection("smfmirror")
  if c == nil {
    c = s.SetCollection("smfmirror", nil)
  }
  pboard_links, err := c.Get([]byte("board_links"))
  var board_links []string
  if pboard_links != nil {
    err := json.Unmarshal(pboard_links, &board_links)
    if err != nil {
      panic(err)
    }
  } else {
    board_links = loadBoardLinks(client, start)
    b, err := json.Marshal(board_links)
    if err != nil {
      panic(err)
    }
    //c.Set([]byte("hello"), []byte("world"))
    //a, x := c.Get([]byte("hello"))
    //fmt.Printf("%v %v\n", a, x)
    c.Set([]byte("board_links"), b)
    s.Flush()
    s.Close()
    f.Sync()
    f.Close()
  }
  fmt.Printf("%v\n", board_links)
}
<commit_msg>Make sure login worked<commit_after>package main

import (
  "io/ioutil"
  "bytes"
  "encoding/json"
  "os"
  "flag"
  "fmt"
  "io"
  "log"
  "net/http"
  "net/http/cookiejar"
  "net/url"
  "strings"
  "path"
  "code.google.com/p/go.net/html"
  "github.com/steveyen/gkvlite"
)

func Split2(str, sep string) (string, string) {
  s := strings.Split(str, sep)
  return s[0], s[1]
}

func fetch(url string) {
  res, err := http.Get(url)
  if err != nil {
    log.Fatal(err)
  }
  if err != nil {
    log.Fatal(err)
  }
  // http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
  fo, err := os.Create(path.Base(url))
  if err != nil {
    panic(err)
  }
  defer func() {
    if err := fo.Close(); err != nil {
      panic(err)
    }
  }()
  buf := make([]byte, 65536)
  for {
    n, err := res.Body.Read(buf)
    if err != nil && err != io.EOF {
      panic(err)
    }
    if n == 0 {
      break
    }

    if _, err := fo.Write(buf[:n]); err != nil {
      panic(err)
    }
  }
  res.Body.Close()
}

func Fuckoff(pres *http.Response) {
}

func FindLinks(body io.Reader) chan string {
  c := make(chan string)

  go func() {
    z := html.NewTokenizer(body)
    for {
      tt := z.Next()
      if tt == html.ErrorToken {
        break
      }
      if tt == html.StartTagToken {
        tn, _ := z.TagName()
        if len(tn) == 1 && tn[0] == 'a' {
          for {
            key, value, more := z.TagAttr()
            // http://stackoverflow.com/questions/14230145/what-is-the-best-way-to-convert-byte-array-to-string
            if string(key) == "href" {
              v := string(value)
              // http://codereview.stackexchange.com/questions/28386/fibonacci-generator-with-golang
              c <- v
            }
            if !more {
              break
            }
          }
        }
      }
    }
    c <- ""
  }()

  return c
}

const (
  _ = iota
  linkfound
  linkcrawled
)

func findBoardLinks(client *http.Client, start string, board_links map[string]int) (board_links_out map[string]int) {
  res, err := client.Get(start)
  if err != nil {
    log.Fatal(err)
  }
  //fmt.Printf("%s\n", start)

  c := FindLinks(res.Body)
  for {
    v := <- c
    if v == "" {
      break
    }
    //fmt.Printf("%v\n", v)
    if strings.Contains(v, start) && strings.Contains(v, "?board=") {
      // index.php?board=18.0;sort=starter -- drop ;.*
      if strings.Contains(v, ";") {
        pre, _ := Split2(v, ";")
        v = pre
      }
      //fmt.Printf("recursing\n")
      _, found := board_links[v]
      if !found {
        board_links[v] = linkfound
      }
    }
  }
  res.Body.Close()
  return board_links
}

func loadBoardLinks(client *http.Client, start string) (flat_links []string) {
  board_links := map[string]int{}
  board_links[start] = linkfound
  for {
    copy := board_links
    any := false
    for key, value := range copy {
      if value == linkcrawled {
        continue
      }
      fmt.Printf("%v\n", key)
      board_links = findBoardLinks(client, key, board_links)
      board_links[key] = linkcrawled
      any = true
    }
    if !any {
      break
    }
  }

  flat_links = []string{}
  for key, _ := range board_links {
    flat_links = append(flat_links, key)
  }
  return flat_links
}

func main() {
  //var start string
  //flag.StringVar(&start, "start", "", "starting url")
  //flag.Parse()

  flag.Parse()
  if len(flag.Args()) < 2 {
    log.Fatal("Usage: mirror start-url credentials")
  }
  start := flag.Args()[0]
  username, password := Split2(flag.Args()[1], ":")

  // http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
  options := cookiejar.Options{}
  cookie_jar, err := cookiejar.New(&options)
  client := &http.Client{
    Jar: cookie_jar,
  }
  v := url.Values{"user": {username}, "passwrd": {password}}
  req, err := http.NewRequest("POST", start + "index.php?action=login2",
    bytes.NewReader([]byte(v.Encode())))
  // ugh
  req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
  req.Header.Add("Accept-Encoding", "identity")
  pres, err := client.Do(req)
  if err != nil {
    log.Fatal(err)
  }
  contents, err := ioutil.ReadAll(pres.Body)
  if !strings.Contains(string(contents), "action=unread") {
    log.Fatal("Login did not work")
  }

  f, err := os.OpenFile("smfmirror.gkvlite", os.O_RDWR, 0666)
  if err != nil {
    f, err = os.Create("smfmirror.gkvlite")
  }
  if err != nil {
    panic(err)
  }
  s, err := gkvlite.NewStore(f)
  if err != nil {
    panic(err)
  }
  c := s.GetCollection("smfmirror")
  if c == nil {
    c = s.SetCollection("smfmirror", nil)
  }
  pboard_links, err := c.Get([]byte("board_links"))
  var board_links []string
  if pboard_links != nil {
    err := json.Unmarshal(pboard_links, &board_links)
    if err != nil {
      panic(err)
    }
  } else {
    board_links = loadBoardLinks(client, start)
    b, err := json.Marshal(board_links)
    if err != nil {
      panic(err)
    }
    //c.Set([]byte("hello"), []byte("world"))
    //a, x := c.Get([]byte("hello"))
    //fmt.Printf("%v %v\n", a, x)
    c.Set([]byte("board_links"), b)
    s.Flush()
    s.Close()
    f.Sync()
    f.Close()
  }
  fmt.Printf("%v\n", board_links)
}
<|endoftext|>


<commit_before>// +build !luminous,!mimic
//
// Ceph Nautilus is the first release that includes rbd_pool_metadata_get().

package rbd

// #cgo LDFLAGS: -lrbd
// #include <rados/librados.h>
// #include <rbd/librbd.h>
// #include <stdlib.h>
import "C"

import (
	"unsafe"

	"github.com/ceph/go-ceph/internal/retry"
	"github.com/ceph/go-ceph/rados"
)

// GetPoolMetadata returns pool metadata associated with the given key.
//
// Implements:
//  int rbd_pool_metadata_get(rados_ioctx_t io_ctx, const char *key, char *value, size_t *val_len);
func GetPoolMetadata(ioctx *rados.IOContext, key string) (string, error) {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))

	var (
		buf []byte
		err error
	)
	retry.WithSizes(4096, 262144, func(size int) retry.Hint {
		cSize := C.size_t(size)
		buf = make([]byte, cSize)
		ret := C.rbd_pool_metadata_get(cephIoctx(ioctx),
			cKey,
			(*C.char)(unsafe.Pointer(&buf[0])),
			&cSize)
		err = getError(ret)
		return retry.Size(int(cSize)).If(err == errRange)
	})

	if err != nil {
		return "", err
	}
	return C.GoString((*C.char)(unsafe.Pointer(&buf[0]))), nil
}
<commit_msg>rbd: add wrapper for rbd_pool_metadata_set()<commit_after>// +build !luminous,!mimic
//
// Ceph Nautilus is the first release that includes rbd_pool_metadata_get(),
// rbd_pool_metadata_set().

package rbd

// #cgo LDFLAGS: -lrbd
// #include <rados/librados.h>
// #include <rbd/librbd.h>
// #include <stdlib.h>
import "C"

import (
	"unsafe"

	"github.com/ceph/go-ceph/internal/retry"
	"github.com/ceph/go-ceph/rados"
)

// GetPoolMetadata returns pool metadata associated with the given key.
//
// Implements:
//  int rbd_pool_metadata_get(rados_ioctx_t io_ctx, const char *key, char *value, size_t *val_len);
func GetPoolMetadata(ioctx *rados.IOContext, key string) (string, error) {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))

	var (
		buf []byte
		err error
	)
	retry.WithSizes(4096, 262144, func(size int) retry.Hint {
		cSize := C.size_t(size)
		buf = make([]byte, cSize)
		ret := C.rbd_pool_metadata_get(cephIoctx(ioctx),
			cKey,
			(*C.char)(unsafe.Pointer(&buf[0])),
			&cSize)
		err = getError(ret)
		return retry.Size(int(cSize)).If(err == errRange)
	})

	if err != nil {
		return "", err
	}
	return C.GoString((*C.char)(unsafe.Pointer(&buf[0]))), nil
}

// SetPoolMetadata updates the pool metadata string associated with the given key.
//
// Implements:
//  int rbd_pool_metadata_set(rados_ioctx_t io_ctx, const char *key, const char *value);
func SetPoolMetadata(ioctx *rados.IOContext, key, value string) error {
	cKey := C.CString(key)
	defer C.free(unsafe.Pointer(cKey))
	cValue := C.CString(value)
	defer C.free(unsafe.Pointer(cValue))

	ret := C.rbd_pool_metadata_set(cephIoctx(ioctx), cKey, cValue)
	return getError(ret)
}
<|endoftext|>


<commit_before>package rcluster

import (
	"errors"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/gallir/smart-relayer/lib"
	"github.com/mediocregopher/radix.v2/cluster"
	"github.com/mediocregopher/radix.v2/redis"
)

// Server is the thread that listen for clients' connections
type Server struct {
	sync.Mutex
	config     lib.RelayerConfig
	mode       int
	done       chan bool
	ListenAddr string
	RedisAddrs []string
	listener   net.Listener
	cluster    *cluster.Cluster
}

const (
	connectionRetries = 3
	pipelineCommands  = 1000
	requestBufferSize = 1024
	modeSync          = 0
	modeSmart         = 1
)

// errors
var (
	errBadCmd = errors.New("ERR bad command")
)

// New creates a new Redis local server
func New(c lib.RelayerConfig, done chan bool) (*Server, error) {
	srv := &Server{
		done: done,
	}
	srv.ListenAddr = c.Listen
	srv.RedisAddrs = append(srv.RedisAddrs, c.Host())

	srv.Reload(&c)
	if srv.cluster == nil {
		log.Println("no available redis cluster nodes for ", srv.RedisAddrs)
		return nil, errors.New("no available redis cluster nodes")
	}

	return srv, nil
}

func (srv *Server) Reload(c *lib.RelayerConfig) {
	srv.Lock()
	defer srv.Unlock()

	reset := false
	if srv.config.Url != c.Url {
		reset = true
	}
	srv.config = *c // Save a copy
	if c.Mode == "smart" {
		srv.mode = modeSmart
	} else {
		srv.mode = modeSync
	}
	if reset {
		log.Printf("Reload and reset redis cluster server at port %s for target %s", srv.config.Listen, srv.config.Host())
		var err error
		for _, addr := range srv.RedisAddrs {
			if srv.cluster, err = cluster.New(addr); err != nil {
				log.Printf("Error in cluster %s: %s", addr, err)
				srv.cluster = nil
				continue
			}
			return
		}
		log.Println("no available redis cluster nodes for ", srv.RedisAddrs)
	} else {
		log.Printf("Reload redis cluster config at port %s for target %s", srv.config.Listen, srv.config.Host())
	}
}

func (srv *Server) Start() (e error) {
	srv.Lock()
	defer srv.Unlock()

	if srv.cluster == nil {
		return
	}

	connType := srv.config.ListenScheme()
	addr := srv.config.ListenHost()

	// Check that the socket does not exist
	if connType == "unix" {
		if s, err := os.Stat(addr); err == nil {
			if (s.Mode() & os.ModeSocket) > 0 {
				// Remove existing socket
				os.Remove(addr)
			} else {
				log.Println("socket", addr, s.Mode(), os.ModeSocket)
				log.Fatalf("Socket %s exists and it's not a Unix socket", addr)
			}
		}
	}

	srv.listener, e = net.Listen(connType, addr)
	if e != nil {
		log.Println("Error listening to", addr, e)
		return e
	}

	if connType == "unix" {
		// Make sure is accesible for everyone
		os.Chmod(addr, 0777)
	}

	log.Printf("Starting redis cluster server at %s for target %s", addr, srv.config.Host())
	// Serve a client
	go func() {
		defer func() {
			srv.listener.Close()
			srv.done <- true
		}()

		for {
			netConn, e := srv.listener.Accept()
			if e != nil {
				log.Println("Exiting", addr, e)
				return
			}
			go srv.handleConnection(netConn)
		}
	}()

	return nil
}

func (srv *Server) handleConnection(conn net.Conn) {
	defer conn.Close()

	rr := redis.NewRespReader(conn)
	for {
		err := conn.SetReadDeadline(time.Now().Add(1 * time.Second))
		if err != nil {
			log.Printf("error setting read deadline: %s", err)
			return
		}

		r := rr.Read()
		if redis.IsTimeout(r) {
			continue
		} else if r.IsType(redis.IOErr) {
			return
		}

		srv.Cmd(r).WriteTo(conn)
	}

}

func (srv *Server) Cmd(m *redis.Resp) *redis.Resp {
	ms, err := m.Array()
	if err != nil || len(ms) < 1 {
		return redis.NewResp(errBadCmd)
	}

	cmd, err := ms[0].Str()
	if err != nil {
		return redis.NewResp(errBadCmd)
	}

	args := make([]interface{}, 0, len(ms[1:]))
	for _, argm := range ms[1:] {
		arg, err := argm.Str()
		if err != nil {
			return redis.NewResp(errBadCmd)
		}
		args = append(args, arg)
	}

	return srv.cluster.Cmd(cmd, args...)
}

func (srv *Server) Exit() {
	if srv.listener != nil {
		srv.listener.Close()
	}
}
<commit_msg>Added async method<commit_after>package rcluster

import (
	"errors"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"strings"

	"github.com/gallir/smart-relayer/lib"
	"github.com/mediocregopher/radix.v2/cluster"
	"github.com/mediocregopher/radix.v2/redis"
)

// Server is the thread that listen for clients' connections
type Server struct {
	sync.Mutex
	config     lib.RelayerConfig
	mode       int
	done       chan bool
	ListenAddr string
	RedisAddrs []string
	listener   net.Listener
	cluster    *cluster.Cluster
}

const (
	connectionRetries = 3
	pipelineCommands  = 1000
	requestBufferSize = 1024
	modeSync          = 0
	modeSmart         = 1
)

// errors
var (
	errBadCmd = errors.New("ERR bad command")
	commands  map[string][]byte

	protoOK   = []byte("+OK\r\n")
	protoTrue = []byte(":1\r\n")
	protoKO   = []byte("-Error\r\n")
)

func init() {
	// These are the commands that can be sent in "background" when in smart mode
	// The values are the immediate responses to the clients
	commands = map[string][]byte{
		"SET":       protoOK,
		"SETEX":     protoOK,
		"PSETEX":    protoOK,
		"MSET":      protoOK,
		"HMSET":     protoOK,
		"SELECT":    protoOK,
		"HSET":      protoTrue,
		"EXPIRE":    protoTrue,
		"EXPIREAT":  protoTrue,
		"PEXPIRE":   protoTrue,
		"PEXPIREAT": protoTrue,
	}
}

// New creates a new Redis local server
func New(c lib.RelayerConfig, done chan bool) (*Server, error) {
	srv := &Server{
		done: done,
	}
	srv.ListenAddr = c.Listen
	srv.RedisAddrs = append(srv.RedisAddrs, c.Host())

	srv.Reload(&c)
	if srv.cluster == nil {
		log.Println("no available redis cluster nodes for ", srv.RedisAddrs)
		return nil, errors.New("no available redis cluster nodes")
	}

	return srv, nil
}

func (srv *Server) Reload(c *lib.RelayerConfig) {
	srv.Lock()
	defer srv.Unlock()

	reset := false
	if srv.config.Url != c.Url {
		reset = true
	}
	srv.config = *c // Save a copy
	if c.Mode == "smart" {
		srv.mode = modeSmart
	} else {
		srv.mode = modeSync
	}
	if reset {
		log.Printf("Reload and reset redis cluster server at port %s for target %s", srv.config.Listen, srv.config.Host())
		var err error
		for _, addr := range srv.RedisAddrs {
			if srv.cluster, err = cluster.New(addr); err != nil {
				log.Printf("Error in cluster %s: %s", addr, err)
				srv.cluster = nil
				continue
			}
			lib.Debugf("Cluster linked to %s", addr)
			return
		}
		log.Println("no available redis cluster nodes for ", srv.RedisAddrs)
	} else {
		log.Printf("Reload redis cluster config at port %s for target %s", srv.config.Listen, srv.config.Host())
	}
}

func (srv *Server) Start() (e error) {
	srv.Lock()
	defer srv.Unlock()

	if srv.cluster == nil {
		return
	}

	connType := srv.config.ListenScheme()
	addr := srv.config.ListenHost()

	// Check that the socket does not exist
	if connType == "unix" {
		if s, err := os.Stat(addr); err == nil {
			if (s.Mode() & os.ModeSocket) > 0 {
				// Remove existing socket
				os.Remove(addr)
			} else {
				log.Println("socket", addr, s.Mode(), os.ModeSocket)
				log.Fatalf("Socket %s exists and it's not a Unix socket", addr)
			}
		}
	}

	srv.listener, e = net.Listen(connType, addr)
	if e != nil {
		log.Println("Error listening to", addr, e)
		return e
	}

	if connType == "unix" {
		// Make sure is accesible for everyone
		os.Chmod(addr, 0777)
	}

	log.Printf("Starting redis cluster server at %s for target %s", addr, srv.config.Host())
	// Serve a client
	go func() {
		defer func() {
			srv.listener.Close()
			srv.done <- true
		}()

		for {
			netConn, e := srv.listener.Accept()
			if e != nil {
				log.Println("Exiting", addr, e)
				return
			}
			go srv.handleConnection(netConn)
		}
	}()

	return nil
}

func (srv *Server) handleConnection(conn net.Conn) {
	defer conn.Close()

	var asyncCh chan *redis.Resp

	if srv.mode == modeSmart {
		asyncCh = make(chan *redis.Resp, 64)
		defer close(asyncCh)
		go func() {
			for m := range asyncCh {
				srv.Cmd(m, nil)
			}
		}()

	}

	rr := redis.NewRespReader(conn)
	for {
		err := conn.SetReadDeadline(time.Now().Add(1 * time.Second))
		if err != nil {
			log.Printf("error setting read deadline: %s", err)
			return
		}

		r := rr.Read()
		if redis.IsTimeout(r) {
			continue
		} else if r.IsType(redis.IOErr) {
			return
		}

		srv.Cmd(r, asyncCh).WriteTo(conn)
	}

}

func (srv *Server) Cmd(m *redis.Resp, asyncCh chan *redis.Resp) *redis.Resp {
	ms, err := m.Array()
	if err != nil || len(ms) < 1 {
		return redis.NewResp(errBadCmd)
	}

	cmd, err := ms[0].Str()
	if err != nil {
		return redis.NewResp(errBadCmd)
	}

	doAsync := false
	var fastResponse []byte
	if asyncCh != nil {
		fastResponse, doAsync = commands[strings.ToUpper(cmd)]
	}

	if doAsync {
		asyncCh <- m
		return redis.NewResp(fastResponse)
	}

	args := make([]interface{}, 0, len(ms[1:]))
	for _, argm := range ms[1:] {
		arg, err := argm.Str()
		if err != nil {
			return redis.NewResp(errBadCmd)
		}
		args = append(args, arg)
	}

	return srv.cluster.Cmd(cmd, args...)
}

func (srv *Server) Exit() {
	if srv.listener != nil {
		srv.listener.Close()
	}
}
<|endoftext|>


<commit_before>package realpath

import (
  "os"
  "fmt"
  "bytes"
)

func Realpath(filepath string) (string, error) {

  if len(filepath)==0 { return "", os.ErrInvalid }

  sepStr := string(os.PathSeparator)

  if filepath[0] != os.PathSeparator {
    pwd, err := os.Getwd()
    if err!=nil { return "", err }
    filepath = pwd + sepStr + filepath
  }

  path   := []byte(filepath)
  nlinks := 0
  start  := 1
  prev   := 1
  for start < len(path) {
    c   := nextComponent(path, start)
    cur := c[start:]

    switch {

      case len(cur)==0:
        copy(path[start:], path[start+1:])
        path = path[0:len(path)-1]

      case len(cur)==1 && cur[0]=='.':
        if start+2 < len(path) { copy(path[start:], path[start+2:]) }
        path = path[0:len(path)-2]

      case len(cur)==2 && cur[0]=='.' && cur[1] == '.':
        copy(path[prev:], path[start+2:])
        path = path[0:len(path)+prev-(start+2)]
        prev  = 1
        start = 1

      default:

        fi, err := os.Lstat(string(c))
        if err!=nil { return "", err }
        if fi.Mode() & os.ModeSymlink == os.ModeSymlink {

          nlinks++
          if nlinks > 16 { return "", os.ErrInvalid }

          var dst string
          dst, err = os.Readlink(string(c))
          fmt.Printf("SYMLINK -> %s\n", dst)

          rest := string(path[len(c):])
          if dst[0] == os.PathSeparator {
            // Absolute links
            path  = []byte(dst + sepStr + rest)
          } else {
            // Relative links
            path  = []byte(string(path[0:start])+dst+sepStr+rest)
          }
          prev  = 1
          start = 1
        } else {
          // Directories
          prev  = start
          start = len(c) + 1
        }
    }
  }

  for len(path)>1 && path[len(path)-1]==os.PathSeparator { path = path[0:len(path)-1] }
  return string(path), nil

}

func nextComponent(path []byte, start int) []byte {
  v := bytes.IndexByte(path[start:], os.PathSeparator)
  if v < 0 { return path }
  return path[0 : start+v]
}

<commit_msg>Reformat code with go fmt<commit_after>package realpath

import (
	"bytes"
	"fmt"
	"os"
)

func Realpath(filepath string) (string, error) {

	if len(filepath) == 0 {
		return "", os.ErrInvalid
	}

	sepStr := string(os.PathSeparator)

	if filepath[0] != os.PathSeparator {
		pwd, err := os.Getwd()
		if err != nil {
			return "", err
		}
		filepath = pwd + sepStr + filepath
	}

	path := []byte(filepath)
	nlinks := 0
	start := 1
	prev := 1
	for start < len(path) {
		c := nextComponent(path, start)
		cur := c[start:]

		switch {

		case len(cur) == 0:
			copy(path[start:], path[start+1:])
			path = path[0 : len(path)-1]

		case len(cur) == 1 && cur[0] == '.':
			if start+2 < len(path) {
				copy(path[start:], path[start+2:])
			}
			path = path[0 : len(path)-2]

		case len(cur) == 2 && cur[0] == '.' && cur[1] == '.':
			copy(path[prev:], path[start+2:])
			path = path[0 : len(path)+prev-(start+2)]
			prev = 1
			start = 1

		default:

			fi, err := os.Lstat(string(c))
			if err != nil {
				return "", err
			}
			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {

				nlinks++
				if nlinks > 16 {
					return "", os.ErrInvalid
				}

				var dst string
				dst, err = os.Readlink(string(c))
				fmt.Printf("SYMLINK -> %s\n", dst)

				rest := string(path[len(c):])
				if dst[0] == os.PathSeparator {
					// Absolute links
					path = []byte(dst + sepStr + rest)
				} else {
					// Relative links
					path = []byte(string(path[0:start]) + dst + sepStr + rest)
				}
				prev = 1
				start = 1
			} else {
				// Directories
				prev = start
				start = len(c) + 1
			}
		}
	}

	for len(path) > 1 && path[len(path)-1] == os.PathSeparator {
		path = path[0 : len(path)-1]
	}
	return string(path), nil

}

func nextComponent(path []byte, start int) []byte {
	v := bytes.IndexByte(path[start:], os.PathSeparator)
	if v < 0 {
		return path
	}
	return path[0 : start+v]
}
<|endoftext|>


<commit_before>package tun

import (
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"unsafe"
	"log"
)

const (
	cIFF_TUN   = 0x0001
	cIFF_TAP   = 0x0002
	cIFF_NO_PI = 0x1000
)

type ifReq struct {
	Name  [0x10]byte
	Flags uint16
	pad   [0x28 - 0x10 - 2]byte
}

func openDevice(name string) (*os.File, error) {
	return os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
}

func createInterface(file *os.File, name string) (string, error) {
	var req ifReq
	req.Flags = 0
	copy(req.Name[:15], name)
	req.Flags = cIFF_TUN | cIFF_NO_PI
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		file.Fd(),
		uintptr(syscall.TUNSETIFF),
		uintptr(unsafe.Pointer(&req)),
	)
	if err != 0 {
		return "", err
	}
	return strings.Trim(string(req.Name[:]), "\x00"), nil
}

func setupInterface(name string, local_ip string) error {
	ip_net := net.IPNet{net.ParseIP(local_ip), net.CIDRMask(24, 32)}

	log.Printf("ifconfig %s %s", name, ip_net.String())
	err = exec.Command("ifconfig", name, ip_net.String()).Run()
	if err != nil {
		return err
	}

	log.Printf("ifconfig %s mtu 1500 up", name)
	err = exec.Command("ifconfig", name, "mtu", "1500", "up").Run()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>quick fix<commit_after>package tun

import (
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"unsafe"
	"log"
)

const (
	cIFF_TUN   = 0x0001
	cIFF_TAP   = 0x0002
	cIFF_NO_PI = 0x1000
)

type ifReq struct {
	Name  [0x10]byte
	Flags uint16
	pad   [0x28 - 0x10 - 2]byte
}

func openDevice(name string) (*os.File, error) {
	return os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
}

func createInterface(file *os.File, name string) (string, error) {
	var req ifReq
	req.Flags = 0
	copy(req.Name[:15], name)
	req.Flags = cIFF_TUN | cIFF_NO_PI
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		file.Fd(),
		uintptr(syscall.TUNSETIFF),
		uintptr(unsafe.Pointer(&req)),
	)
	if err != 0 {
		return "", err
	}
	return strings.Trim(string(req.Name[:]), "\x00"), nil
}

func setupInterface(name string, local_ip string) error {
	ip_net := net.IPNet{net.ParseIP(local_ip), net.CIDRMask(24, 32)}

	log.Printf("ifconfig %s %s", name, ip_net.String())
	err := exec.Command("ifconfig", name, ip_net.String()).Run()
	if err != nil {
		return err
	}

	log.Printf("ifconfig %s mtu 1500 up", name)
	err = exec.Command("ifconfig", name, "mtu", "1500", "up").Run()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>