<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows

package net

import (
	"os"
	"syscall"
)

// Boolean to int.
func boolint(b bool) int {
	if b {
		return 1
	}
	return 0
}

func ipv4AddrToInterface(ip IP) (*Interface, error) {
	ift, err := Interfaces()
	if err != nil {
		return nil, err
	}
	for _, ifi := range ift {
		ifat, err := ifi.Addrs()
		if err != nil {
			return nil, err
		}
		for _, ifa := range ifat {
			switch v := ifa.(type) {
			case *IPAddr:
				if ip.Equal(v.IP) {
					return &ifi, nil
				}
			case *IPNet:
				if ip.Equal(v.IP) {
					return &ifi, nil
				}
			}
		}
	}
	if ip.Equal(IPv4zero) {
		return nil, nil
	}
	return nil, errNoSuchInterface
}

func interfaceToIPv4Addr(ifi *Interface) (IP, error) {
	if ifi == nil {
		return IPv4zero, nil
	}
	ifat, err := ifi.Addrs()
	if err != nil {
		return nil, err
	}
	for _, ifa := range ifat {
		switch v := ifa.(type) {
		case *IPAddr:
			if v.IP.To4() != nil {
				return v.IP, nil
			}
		case *IPNet:
			if v.IP.To4() != nil {
				return v.IP, nil
			}
		}
	}
	return nil, errNoSuchInterface
}

func setIPv4MreqToInterface(mreq *syscall.IPMreq, ifi *Interface) error {
	if ifi == nil {
		return nil
	}
	ifat, err := ifi.Addrs()
	if err != nil {
		return err
	}
	for _, ifa := range ifat {
		switch v := ifa.(type) {
		case *IPAddr:
			if a := v.IP.To4(); a != nil {
				copy(mreq.Interface[:], a)
				goto done
			}
		case *IPNet:
			if a := v.IP.To4(); a != nil {
				copy(mreq.Interface[:], a)
				goto done
			}
		}
	}
done:
	if bytesEqual(mreq.Multiaddr[:], IPv4zero.To4()) {
		return errNoSuchMulticastInterface
	}
	return nil
}

func setReadBuffer(fd *netFD, bytes int) error {
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes))
}

func setWriteBuffer(fd *netFD, bytes int) error {
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes))
}

func setKeepAlive(fd *netFD, keepalive bool) error {
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, boolint(keepalive)))
}

func setLinger(fd *netFD, sec int) error {
	var l syscall.Linger
	if sec >= 0 {
		l.Onoff = 1
		l.Linger = int32(sec)
	} else {
		l.Onoff = 0
		l.Linger = 0
	}
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptLinger(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_LINGER, &l))
}
<commit_msg>net: make use of SO_LINGER_SEC on darwin<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows

package net

import (
	"os"
	"runtime"
	"syscall"
)

// Boolean to int.
func boolint(b bool) int {
	if b {
		return 1
	}
	return 0
}

func ipv4AddrToInterface(ip IP) (*Interface, error) {
	ift, err := Interfaces()
	if err != nil {
		return nil, err
	}
	for _, ifi := range ift {
		ifat, err := ifi.Addrs()
		if err != nil {
			return nil, err
		}
		for _, ifa := range ifat {
			switch v := ifa.(type) {
			case *IPAddr:
				if ip.Equal(v.IP) {
					return &ifi, nil
				}
			case *IPNet:
				if ip.Equal(v.IP) {
					return &ifi, nil
				}
			}
		}
	}
	if ip.Equal(IPv4zero) {
		return nil, nil
	}
	return nil, errNoSuchInterface
}

func interfaceToIPv4Addr(ifi *Interface) (IP, error) {
	if ifi == nil {
		return IPv4zero, nil
	}
	ifat, err := ifi.Addrs()
	if err != nil {
		return nil, err
	}
	for _, ifa := range ifat {
		switch v := ifa.(type) {
		case *IPAddr:
			if v.IP.To4() != nil {
				return v.IP, nil
			}
		case *IPNet:
			if v.IP.To4() != nil {
				return v.IP, nil
			}
		}
	}
	return nil, errNoSuchInterface
}

func setIPv4MreqToInterface(mreq *syscall.IPMreq, ifi *Interface) error {
	if ifi == nil {
		return nil
	}
	ifat, err := ifi.Addrs()
	if err != nil {
		return err
	}
	for _, ifa := range ifat {
		switch v := ifa.(type) {
		case *IPAddr:
			if a := v.IP.To4(); a != nil {
				copy(mreq.Interface[:], a)
				goto done
			}
		case *IPNet:
			if a := v.IP.To4(); a != nil {
				copy(mreq.Interface[:], a)
				goto done
			}
		}
	}
done:
	if bytesEqual(mreq.Multiaddr[:], IPv4zero.To4()) {
		return errNoSuchMulticastInterface
	}
	return nil
}

func setReadBuffer(fd *netFD, bytes int) error {
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes))
}

func setWriteBuffer(fd *netFD, bytes int) error {
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes))
}

func setKeepAlive(fd *netFD, keepalive bool) error {
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, boolint(keepalive)))
}

func setLinger(fd *netFD, sec int) error {
	var l syscall.Linger
	if sec >= 0 {
		l.Onoff = 1
		l.Linger = int32(sec)
	} else {
		l.Onoff = 0
		l.Linger = 0
	}
	if err := fd.incref(); err != nil {
		return err
	}
	defer fd.decref()
	opt := syscall.SO_LINGER
	if runtime.GOOS == "darwin" {
		opt = syscall.SO_LINGER_SEC
	}
	return os.NewSyscallError("setsockopt", syscall.SetsockoptLinger(fd.sysfd, syscall.SOL_SOCKET, opt, &l))
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Deep equality test via reflection

package reflect

// During deepValueEqual, must keep track of checks that are
// in progress.  The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited are stored in a map indexed by 17 * a1 + a2;
type visit struct {
	a1   uintptr
	a2   uintptr
	typ  Type
	next *visit
}

// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func deepValueEqual(v1, v2 Value, visited map[uintptr]*visit, depth int) (b bool) {
	if !v1.IsValid() || !v2.IsValid() {
		return v1.IsValid() == v2.IsValid()
	}
	if v1.Type() != v2.Type() {
		return false
	}

	// if depth > 10 { panic("deepValueEqual") }	// for debugging

	if v1.CanAddr() && v2.CanAddr() {
		addr1 := v1.UnsafeAddr()
		addr2 := v2.UnsafeAddr()
		if addr1 > addr2 {
			// Canonicalize order to reduce number of entries in visited.
			addr1, addr2 = addr2, addr1
		}

		// Short circuit if references are identical ...
		if addr1 == addr2 {
			return true
		}

		// ... or already seen
		h := 17*addr1 + addr2
		seen := visited[h]
		typ := v1.Type()
		for p := seen; p != nil; p = p.next {
			if p.a1 == addr1 && p.a2 == addr2 && p.typ == typ {
				return true
			}
		}

		// Remember for later.
		visited[h] = &visit{addr1, addr2, typ, seen}
	}

	switch v1.Kind() {
	case Array:
		if v1.Len() != v2.Len() {
			return false
		}
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Slice:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Interface:
		if v1.IsNil() || v2.IsNil() {
			return v1.IsNil() == v2.IsNil()
		}
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Ptr:
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Struct:
		for i, n := 0, v1.NumField(); i < n; i++ {
			if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Map:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		for _, k := range v1.MapKeys() {
			if !deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
				return false
			}
		}
		return true
	case Func:
		if v1.IsNil() && v2.IsNil() {
			return true
		}
		// Can't do better than this:
		return false
	default:
		// Normal equality suffices
		return valueInterface(v1, false) == valueInterface(v2, false)
	}
}

// DeepEqual tests for deep equality. It uses normal == equality where
// possible but will scan elements of arrays, slices, maps, and fields of
// structs. In maps, keys are compared with == but elements use deep
// equality. DeepEqual correctly handles recursive types. Functions are equal
// only if they are both nil.
// An empty slice is not equal to a nil slice.
func DeepEqual(a1, a2 interface{}) bool {
	if a1 == nil || a2 == nil {
		return a1 == a2
	}
	v1 := ValueOf(a1)
	v2 := ValueOf(a2)
	if v1.Type() != v2.Type() {
		return false
	}
	return deepValueEqual(v1, v2, make(map[uintptr]*visit), 0)
}
<commit_msg>reflect: use visit structure for map key in DeepEqual<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Deep equality test via reflection

package reflect

// During deepValueEqual, must keep track of checks that are
// in progress.  The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited comparisons are stored in a map indexed by visit.
type visit struct {
	a1  uintptr
	a2  uintptr
	typ Type
}

// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool {
	if !v1.IsValid() || !v2.IsValid() {
		return v1.IsValid() == v2.IsValid()
	}
	if v1.Type() != v2.Type() {
		return false
	}

	// if depth > 10 { panic("deepValueEqual") }	// for debugging

	if v1.CanAddr() && v2.CanAddr() {
		addr1 := v1.UnsafeAddr()
		addr2 := v2.UnsafeAddr()
		if addr1 > addr2 {
			// Canonicalize order to reduce number of entries in visited.
			addr1, addr2 = addr2, addr1
		}

		// Short circuit if references are identical ...
		if addr1 == addr2 {
			return true
		}

		// ... or already seen
		typ := v1.Type()
		v := visit{addr1, addr2, typ}
		if visited[v] {
			return true
		}

		// Remember for later.
		visited[v] = true
	}

	switch v1.Kind() {
	case Array:
		if v1.Len() != v2.Len() {
			return false
		}
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Slice:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Interface:
		if v1.IsNil() || v2.IsNil() {
			return v1.IsNil() == v2.IsNil()
		}
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Ptr:
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Struct:
		for i, n := 0, v1.NumField(); i < n; i++ {
			if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Map:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		for _, k := range v1.MapKeys() {
			if !deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
				return false
			}
		}
		return true
	case Func:
		if v1.IsNil() && v2.IsNil() {
			return true
		}
		// Can't do better than this:
		return false
	default:
		// Normal equality suffices
		return valueInterface(v1, false) == valueInterface(v2, false)
	}
}

// DeepEqual tests for deep equality. It uses normal == equality where
// possible but will scan elements of arrays, slices, maps, and fields of
// structs. In maps, keys are compared with == but elements use deep
// equality. DeepEqual correctly handles recursive types. Functions are equal
// only if they are both nil.
// An empty slice is not equal to a nil slice.
func DeepEqual(a1, a2 interface{}) bool {
	if a1 == nil || a2 == nil {
		return a1 == a2
	}
	v1 := ValueOf(a1)
	v2 := ValueOf(a2)
	if v1.Type() != v2.Type() {
		return false
	}
	return deepValueEqual(v1, v2, make(map[visit]bool), 0)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
)

func TestChanSendInterface(t *testing.T) {
	type mt struct{}
	m := &mt{}
	c := make(chan interface{}, 1)
	c <- m
	select {
	case c <- m:
	default:
	}
	select {
	case c <- m:
	case c <- &mt{}:
	default:
	}
}

func TestPseudoRandomSend(t *testing.T) {
	n := 100
	c := make(chan int)
	l := make([]int, n)
	var m sync.Mutex
	m.Lock()
	go func() {
		for i := 0; i < n; i++ {
			runtime.Gosched()
			l[i] = <-c
		}
		m.Unlock()
	}()
	for i := 0; i < n; i++ {
		select {
		case c <- 0:
		case c <- 1:
		}
	}
	m.Lock() // wait
	n0 := 0
	n1 := 0
	for _, i := range l {
		n0 += (i + 1) % 2
		n1 += i
		if n0 > n/10 && n1 > n/10 {
			return
		}
	}
	t.Errorf("Want pseudo random, got %d zeros and %d ones", n0, n1)
}

func BenchmarkSelectUncontended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			myc1 := make(chan int, 1)
			myc2 := make(chan int, 1)
			myc1 <- 0
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					select {
					case <-myc1:
						myc2 <- 0
					case <-myc2:
						myc1 <- 0
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSelectContended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	myc1 := make(chan int, procs)
	myc2 := make(chan int, procs)
	for p := 0; p < procs; p++ {
		myc1 <- 0
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					select {
					case <-myc1:
						myc2 <- 0
					case <-myc2:
						myc1 <- 0
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSelectNonblock(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			myc1 := make(chan int)
			myc2 := make(chan int)
			myc3 := make(chan int, 1)
			myc4 := make(chan int, 1)
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					select {
					case <-myc1:
					default:
					}
					select {
					case myc2 <- 0:
					default:
					}
					select {
					case <-myc3:
					default:
					}
					select {
					case myc4 <- 0:
					default:
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkChanUncontended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			myc := make(chan int, CallsPerSched)
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					myc <- 0
				}
				for g := 0; g < CallsPerSched; g++ {
					<-myc
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkChanContended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	myc := make(chan int, procs*CallsPerSched)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					myc <- 0
				}
				for g := 0; g < CallsPerSched; g++ {
					<-myc
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkChanSync(b *testing.B) {
	const CallsPerSched = 1000
	procs := 2
	N := int32(b.N / CallsPerSched / procs * procs)
	c := make(chan bool, procs)
	myc := make(chan int)
	for p := 0; p < procs; p++ {
		go func() {
			for {
				i := atomic.AddInt32(&N, -1)
				if i < 0 {
					break
				}
				for g := 0; g < CallsPerSched; g++ {
					if i%2 == 0 {
						<-myc
						myc <- 0
					} else {
						myc <- 0
						<-myc
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func benchmarkChanProdCons(b *testing.B, chanSize, localWork int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, 2*procs)
	myc := make(chan int, chanSize)
	for p := 0; p < procs; p++ {
		go func() {
			foo := 0
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					for i := 0; i < localWork; i++ {
						foo *= 2
						foo /= 2
					}
					myc <- 1
				}
			}
			myc <- 0
			c <- foo == 42
		}()
		go func() {
			foo := 0
			for {
				v := <-myc
				if v == 0 {
					break
				}
				for i := 0; i < localWork; i++ {
					foo *= 2
					foo /= 2
				}
			}
			c <- foo == 42
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
		<-c
	}
}

func BenchmarkChanProdCons0(b *testing.B) {
	benchmarkChanProdCons(b, 0, 0)
}

func BenchmarkChanProdCons10(b *testing.B) {
	benchmarkChanProdCons(b, 10, 0)
}

func BenchmarkChanProdCons100(b *testing.B) {
	benchmarkChanProdCons(b, 100, 0)
}

func BenchmarkChanProdConsWork0(b *testing.B) {
	benchmarkChanProdCons(b, 0, 100)
}

func BenchmarkChanProdConsWork10(b *testing.B) {
	benchmarkChanProdCons(b, 10, 100)
}

func BenchmarkChanProdConsWork100(b *testing.B) {
	benchmarkChanProdCons(b, 100, 100)
}

func BenchmarkChanCreation(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					myc := make(chan int, 1)
					myc <- 0
					<-myc
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}
<commit_msg>runtime: add test for multiple concurrent channel consumers<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
)

func TestChanSendInterface(t *testing.T) {
	type mt struct{}
	m := &mt{}
	c := make(chan interface{}, 1)
	c <- m
	select {
	case c <- m:
	default:
	}
	select {
	case c <- m:
	case c <- &mt{}:
	default:
	}
}

func TestPseudoRandomSend(t *testing.T) {
	n := 100
	c := make(chan int)
	l := make([]int, n)
	var m sync.Mutex
	m.Lock()
	go func() {
		for i := 0; i < n; i++ {
			runtime.Gosched()
			l[i] = <-c
		}
		m.Unlock()
	}()
	for i := 0; i < n; i++ {
		select {
		case c <- 0:
		case c <- 1:
		}
	}
	m.Lock() // wait
	n0 := 0
	n1 := 0
	for _, i := range l {
		n0 += (i + 1) % 2
		n1 += i
		if n0 > n/10 && n1 > n/10 {
			return
		}
	}
	t.Errorf("Want pseudo random, got %d zeros and %d ones", n0, n1)
}

func TestMultiConsumer(t *testing.T) {
	const nwork = 23
	const niter = 271828

	pn := []int{2, 3, 7, 11, 13, 17, 19, 23, 27, 31}

	q := make(chan int, nwork*3)
	r := make(chan int, nwork*3)

	// workers
	var wg sync.WaitGroup
	for i := 0; i < nwork; i++ {
		wg.Add(1)
		go func(w int) {
			for v := range q {
				// mess with the fifo-ish nature of range
				if pn[w%len(pn)] == v {
					runtime.Gosched()
				}
				r <- v
			}
			wg.Done()
		}(i)
	}

	// feeder & closer
	expect := 0
	go func() {
		for i := 0; i < niter; i++ {
			v := pn[i%len(pn)]
			expect += v
			q <- v
		}
		close(q)  // no more work
		wg.Wait() // workers done
		close(r)  // ... so there can be no more results
	}()

	// consume & check
	n := 0
	s := 0
	for v := range r {
		n++
		s += v
	}
	if n != niter || s != expect {
		t.Errorf("Expected sum %d (got %d) from %d iter (saw %d)",
			expect, s, niter, n)
	}
}

func BenchmarkSelectUncontended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			myc1 := make(chan int, 1)
			myc2 := make(chan int, 1)
			myc1 <- 0
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					select {
					case <-myc1:
						myc2 <- 0
					case <-myc2:
						myc1 <- 0
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSelectContended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	myc1 := make(chan int, procs)
	myc2 := make(chan int, procs)
	for p := 0; p < procs; p++ {
		myc1 <- 0
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					select {
					case <-myc1:
						myc2 <- 0
					case <-myc2:
						myc1 <- 0
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSelectNonblock(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			myc1 := make(chan int)
			myc2 := make(chan int)
			myc3 := make(chan int, 1)
			myc4 := make(chan int, 1)
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					select {
					case <-myc1:
					default:
					}
					select {
					case myc2 <- 0:
					default:
					}
					select {
					case <-myc3:
					default:
					}
					select {
					case myc4 <- 0:
					default:
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkChanUncontended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			myc := make(chan int, CallsPerSched)
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					myc <- 0
				}
				for g := 0; g < CallsPerSched; g++ {
					<-myc
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkChanContended(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	myc := make(chan int, procs*CallsPerSched)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					myc <- 0
				}
				for g := 0; g < CallsPerSched; g++ {
					<-myc
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkChanSync(b *testing.B) {
	const CallsPerSched = 1000
	procs := 2
	N := int32(b.N / CallsPerSched / procs * procs)
	c := make(chan bool, procs)
	myc := make(chan int)
	for p := 0; p < procs; p++ {
		go func() {
			for {
				i := atomic.AddInt32(&N, -1)
				if i < 0 {
					break
				}
				for g := 0; g < CallsPerSched; g++ {
					if i%2 == 0 {
						<-myc
						myc <- 0
					} else {
						myc <- 0
						<-myc
					}
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func benchmarkChanProdCons(b *testing.B, chanSize, localWork int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, 2*procs)
	myc := make(chan int, chanSize)
	for p := 0; p < procs; p++ {
		go func() {
			foo := 0
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					for i := 0; i < localWork; i++ {
						foo *= 2
						foo /= 2
					}
					myc <- 1
				}
			}
			myc <- 0
			c <- foo == 42
		}()
		go func() {
			foo := 0
			for {
				v := <-myc
				if v == 0 {
					break
				}
				for i := 0; i < localWork; i++ {
					foo *= 2
					foo /= 2
				}
			}
			c <- foo == 42
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
		<-c
	}
}

func BenchmarkChanProdCons0(b *testing.B) {
	benchmarkChanProdCons(b, 0, 0)
}

func BenchmarkChanProdCons10(b *testing.B) {
	benchmarkChanProdCons(b, 10, 0)
}

func BenchmarkChanProdCons100(b *testing.B) {
	benchmarkChanProdCons(b, 100, 0)
}

func BenchmarkChanProdConsWork0(b *testing.B) {
	benchmarkChanProdCons(b, 0, 100)
}

func BenchmarkChanProdConsWork10(b *testing.B) {
	benchmarkChanProdCons(b, 10, 100)
}

func BenchmarkChanProdConsWork100(b *testing.B) {
	benchmarkChanProdCons(b, 100, 100)
}

func BenchmarkChanCreation(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				for g := 0; g < CallsPerSched; g++ {
					myc := make(chan int, 1)
					myc <- 0
					<-myc
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"sync/atomic"
	"testing"
)

var stop = make(chan bool, 1)

func perpetuumMobile() {
	select {
	case <-stop:
	default:
		go perpetuumMobile()
	}
}

func TestStopTheWorldDeadlock(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	maxprocs := runtime.GOMAXPROCS(3)
	compl := make(chan bool, 2)
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GC()
		}
		compl <- true
	}()
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GOMAXPROCS(3)
		}
		compl <- true
	}()
	go perpetuumMobile()
	<-compl
	<-compl
	stop <- true
	runtime.GOMAXPROCS(maxprocs)
}

func stackGrowthRecursive(i int) {
	var pad [128]uint64
	if i != 0 && pad[0] == 0 {
		stackGrowthRecursive(i - 1)
	}
}

func benchmarkStackGrowth(b *testing.B, rec int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					stackGrowthRecursive(rec)
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkStackGrowth(b *testing.B) {
	benchmarkStackGrowth(b, 10)
}

func BenchmarkStackGrowthDeep(b *testing.B) {
	benchmarkStackGrowth(b, 1024)
}

func BenchmarkSyscall(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					runtime.Exitsyscall()
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSyscallWork(b *testing.B) {
	const CallsPerSched = 1000
	const LocalWork = 100
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			foo := 42
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					for i := 0; i < LocalWork; i++ {
						foo *= 2
						foo /= 2
					}
					runtime.Exitsyscall()
				}
			}
			c <- foo == 42
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkCreateGoroutines(b *testing.B) {
	benchmarkCreateGoroutines(b, 1)
}

func BenchmarkCreateGoroutinesParallel(b *testing.B) {
	benchmarkCreateGoroutines(b, runtime.GOMAXPROCS(-1))
}

func benchmarkCreateGoroutines(b *testing.B, procs int) {
	c := make(chan bool)
	var f func(n int)
	f = func(n int) {
		if n == 0 {
			c <- true
			return
		}
		go f(n - 1)
	}
	for i := 0; i < procs; i++ {
		go f(b.N / procs)
	}
	for i := 0; i < procs; i++ {
		<-c
	}
}
<commit_msg>runtime: add more tests for LockOSThread() Just test some additional paths through the scheduler.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"sync/atomic"
	"testing"
	"time"
)

var stop = make(chan bool, 1)

func perpetuumMobile() {
	select {
	case <-stop:
	default:
		go perpetuumMobile()
	}
}

func TestStopTheWorldDeadlock(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	maxprocs := runtime.GOMAXPROCS(3)
	compl := make(chan bool, 2)
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GC()
		}
		compl <- true
	}()
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GOMAXPROCS(3)
		}
		compl <- true
	}()
	go perpetuumMobile()
	<-compl
	<-compl
	stop <- true
	runtime.GOMAXPROCS(maxprocs)
}

func TestYieldLocked(t *testing.T) {
	const N = 10
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		for i := 0; i < N; i++ {
			runtime.Gosched()
			time.Sleep(time.Millisecond)
		}
		c <- true
		// runtime.UnlockOSThread() is deliberately omitted
	}()
	<-c
}

func TestBlockLocked(t *testing.T) {
	const N = 10
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		for i := 0; i < N; i++ {
			c <- true
		}
		runtime.UnlockOSThread()
	}()
	for i := 0; i < N; i++ {
		<-c
	}
}

func stackGrowthRecursive(i int) {
	var pad [128]uint64
	if i != 0 && pad[0] == 0 {
		stackGrowthRecursive(i - 1)
	}
}

func benchmarkStackGrowth(b *testing.B, rec int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					stackGrowthRecursive(rec)
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkStackGrowth(b *testing.B) {
	benchmarkStackGrowth(b, 10)
}

func BenchmarkStackGrowthDeep(b *testing.B) {
	benchmarkStackGrowth(b, 1024)
}

func BenchmarkSyscall(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					runtime.Exitsyscall()
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSyscallWork(b *testing.B) {
	const CallsPerSched = 1000
	const LocalWork = 100
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			foo := 42
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					for i := 0; i < LocalWork; i++ {
						foo *= 2
						foo /= 2
					}
					runtime.Exitsyscall()
				}
			}
			c <- foo == 42
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkCreateGoroutines(b *testing.B) {
	benchmarkCreateGoroutines(b, 1)
}

func BenchmarkCreateGoroutinesParallel(b *testing.B) {
	benchmarkCreateGoroutines(b, runtime.GOMAXPROCS(-1))
}

func benchmarkCreateGoroutines(b *testing.B, procs int) {
	c := make(chan bool)
	var f func(n int)
	f = func(n int) {
		if n == 0 {
			c <- true
			return
		}
		go f(n - 1)
	}
	for i := 0; i < procs; i++ {
		go f(b.N / procs)
	}
	for i := 0; i < procs; i++ {
		<-c
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"net/http"
	"sync"
)

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Fetch the URL.
			http.Get(url)
			// Decrement the counter.
			wg.Done()
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Printf("Only once\n")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<commit_msg>sync: improve WaitGroup example by putting the call to Done in a deferred block. This makes hangs in the waiting code less likely if a goroutine exits abnormally.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"net/http"
	"sync"
)

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Decrement the counter when the goroutine completes.
			defer wg.Done()
			// Fetch the URL.
			http.Get(url)
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Println("Only once")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<|endoftext|>


<commit_before>package workload

import (
	"io"
	"math/rand"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
)

func RunWorkload(
	pfsClient pfs.APIClient,
	ppsClient pps.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(pfsClient, ppsClient); err != nil {
			return err
		}
	}
	return nil
}

type worker struct {
	repos     []*pfs.Repo
	finished  []*pfs.Commit
	started   []*pfs.Commit
	files     []*pfs.File
	jobs      []*pps.Job
	pipelines []*pps.Pipeline
	rand      *rand.Rand
}

func newWorker(rand *rand.Rand) *worker {
	return &worker{
		rand: rand,
	}
}

const (
	repo     float64 = .1
	commit           = .3
	file             = .6
	job              = .9
	pipeline         = 1.0
)

const maxStartedCommits = 6

func (w *worker) work(pfsClient pfs.APIClient, ppsClient pps.APIClient) error {
	opt := w.rand.Float64()
	switch {
	case opt < repo:
		repoName := w.name()
		if err := pfsutil.CreateRepo(pfsClient, repoName); err != nil {
			return err
		}
		w.repos = append(w.repos, &pfs.Repo{Name: repoName})
		commitID, err := pfsutil.StartCommit(pfsClient, repoName, "")
		if err != nil {
			return err
		}
		w.started = append(w.started, commitID)
	case opt < commit:
		if len(w.started) >= maxStartedCommits {
			i := w.rand.Intn(len(w.started))
			commit := w.started[i]
			if err := pfsutil.FinishCommit(pfsClient, commit.Repo.Name, commit.Id); err != nil {
				return err
			}
			w.started = append(w.started[:i], w.started[i+1:]...)
			w.finished = append(w.finished, commit)
		} else {
			commit := w.finished[w.rand.Intn(len(w.finished))]
			commitID, err := pfsutil.StartCommit(pfsClient, commit.Repo.Name, commit.Id)
			if err != nil {
				return err
			}
			w.started = append(w.started, commitID)
		}
	case opt < file:
		commit := w.started[w.rand.Intn(len(w.started))]
		if _, err := pfsutil.PutFile(pfsClient, commit.Repo.Name, commit.Id, w.name(), 0, w.reader()); err != nil {
			return err
		}
	case opt < job:
		inputs := make([]*pfs.Commit, w.rand.Intn(5))
		for i := range inputs {
			inputs[i] = w.finished[w.rand.Intn(len(w.finished))]
		}
		var parentJob *pps.Job
		if len(w.finished) > 0 {
			parentJob = w.jobs[w.rand.Intn(len(w.finished))]
		}
		job, err := ppsutil.CreateJob(ppsClient, "", []string{}, "stdin", 1, inputs, parentJob)
		if err != nil {
			return err
		}
		w.jobs = append(w.jobs, job)
	case opt < pipeline:
		inputs := make([]*pfs.Repo, w.rand.Intn(5))
		for i := range inputs {
			inputs[i] = w.repos[w.rand.Intn(len(w.repos))]
		}
		pipelineName := w.name()
		if err := ppsutil.CreatePipeline(ppsClient, pipelineName, "", []string{}, "stdin", 1, inputs, nil); err != nil {
			return err
		}

	}
	return nil
}

const letters = "abcdefghijklmnopqrstuvwxyz"
const lettersAndSpaces = "abcdefghijklmnopqrstuvwxyz      \n\n"

func (w *worker) name() string {
	b := make([]byte, 20)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

type reader struct {
	rand *rand.Rand
}

func (r *reader) Read(p []byte) (int, error) {
	for i := range p {
		p[i] = lettersAndSpaces[r.rand.Intn(len(lettersAndSpaces))]
	}
	return len(p), nil
}

func (w *worker) reader() io.Reader {
	return &reader{w.rand}
}
<commit_msg>Workload now creates plausible jobs & pipelines.<commit_after>package workload

import (
	"fmt"
	"io"
	"math/rand"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
)

func RunWorkload(
	pfsClient pfs.APIClient,
	ppsClient pps.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(pfsClient, ppsClient); err != nil {
			return err
		}
	}
	return nil
}

type worker struct {
	repos     []*pfs.Repo
	finished  []*pfs.Commit
	started   []*pfs.Commit
	files     []*pfs.File
	jobs      []*pps.Job
	pipelines []*pps.Pipeline
	rand      *rand.Rand
}

func newWorker(rand *rand.Rand) *worker {
	return &worker{
		rand: rand,
	}
}

const (
	repo     float64 = .1
	commit           = .3
	file             = .6
	job              = .9
	pipeline         = 1.0
)

const maxStartedCommits = 6

func (w *worker) work(pfsClient pfs.APIClient, ppsClient pps.APIClient) error {
	opt := w.rand.Float64()
	switch {
	case opt < repo:
		repoName := w.name()
		if err := pfsutil.CreateRepo(pfsClient, repoName); err != nil {
			return err
		}
		w.repos = append(w.repos, &pfs.Repo{Name: repoName})
		commitID, err := pfsutil.StartCommit(pfsClient, repoName, "")
		if err != nil {
			return err
		}
		w.started = append(w.started, commitID)
	case opt < commit:
		if len(w.started) >= maxStartedCommits {
			i := w.rand.Intn(len(w.started))
			commit := w.started[i]
			if err := pfsutil.FinishCommit(pfsClient, commit.Repo.Name, commit.Id); err != nil {
				return err
			}
			w.started = append(w.started[:i], w.started[i+1:]...)
			w.finished = append(w.finished, commit)
		} else {
			commit := w.finished[w.rand.Intn(len(w.finished))]
			commitID, err := pfsutil.StartCommit(pfsClient, commit.Repo.Name, commit.Id)
			if err != nil {
				return err
			}
			w.started = append(w.started, commitID)
		}
	case opt < file:
		commit := w.started[w.rand.Intn(len(w.started))]
		if _, err := pfsutil.PutFile(pfsClient, commit.Repo.Name, commit.Id, w.name(), 0, w.reader()); err != nil {
			return err
		}
	case opt < job:
		inputs := [5]string{}
		var inputCommits []*pfs.Commit
		for i := range inputs {
			randI := w.rand.Intn(len(w.finished))
			inputs[i] = w.finished[randI].Repo.Name
			inputCommits = append(inputCommits, w.finished[randI])
		}
		var parentJob *pps.Job
		if len(w.finished) > 0 {
			parentJob = w.jobs[w.rand.Intn(len(w.finished))]
		}
		outFilename := w.name()
		job, err := ppsutil.CreateJob(
			ppsClient,
			"",
			[]string{"sh"},
			w.grepCmd(inputs, outFilename),
			1,
			inputCommits,
			parentJob,
		)
		if err != nil {
			return err
		}
		w.jobs = append(w.jobs, job)
	case opt < pipeline:
		inputs := [5]string{}
		var inputRepos []*pfs.Repo
		for i := range inputs {
			randI := w.rand.Intn(len(w.repos))
			inputs[i] = w.repos[randI].Name
			inputRepos = append(inputRepos, w.repos[randI])
		}
		pipelineName := w.name()
		outFilename := w.name()
		if err := ppsutil.CreatePipeline(
			ppsClient, pipelineName,
			"",
			[]string{"sh"},
			w.grepCmd(inputs, outFilename),
			1,
			inputRepos,
			nil,
		); err != nil {
			return err
		}

	}
	return nil
}

const letters = "abcdefghijklmnopqrstuvwxyz"
const lettersAndSpaces = "abcdefghijklmnopqrstuvwxyz      \n\n"

func (w *worker) name() string {
	b := make([]byte, 20)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

type reader struct {
	rand *rand.Rand
}

func (r *reader) Read(p []byte) (int, error) {
	for i := range p {
		p[i] = lettersAndSpaces[r.rand.Intn(len(lettersAndSpaces))]
	}
	if rand.Intn(10) == 0 {
		return len(p), io.EOF
	}
	return len(p), nil
}

func (w *worker) reader() io.Reader {
	return &reader{w.rand}
}

func (w *worker) grepCmd(inputs [5]string, outFilename string) string {
	pattern := make([]byte, 5)
	_, _ = w.reader().Read(pattern)
	return fmt.Sprintf(
		"grep %s /pfs/{%s,%s,%s,%s,%s}/* >/pfs/out/%s",
		pattern,
		inputs[0],
		inputs[1],
		inputs[2],
		inputs[3],
		inputs[4],
		outFilename,
	)
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows

package log

import (
	"fmt"
	"log"
	"log/syslog"
	"os"
)

var _ Logger = &syslogLogger{}

func NewSyslogLogger(tag string, debug bool) (Logger, error) {
	w, err := syslog.New(syslog.LOG_INFO, tag)
	if err != nil {
		return nil, err
	}
	return &syslogLogger{w: w, debug: debug}, nil
}

type syslogLogger struct {
	w     *syslog.Writer
	debug bool
}

func (l *syslogLogger) Error(o string) {
	l.w.Err(o)
}

func (l *syslogLogger) Errorf(format string, o ...interface{}) {
	l.w.Err(fmt.Sprintf(format, o...))
}

func (l *syslogLogger) Fatal(o string) {
	l.w.Err(fmt.Sprintf(fatalPrefix, o))
	os.Exit(1)
}

func (l *syslogLogger) Fatalf(format string, o ...interface{}) {
	l.Fatal(fmt.Sprintf(format, o...))
}

func (l *syslogLogger) Debug(o string) {
	if l.debug {
		l.w.Debug(o)
	}
}

func (l *syslogLogger) Debugf(format string, o ...interface{}) {
	l.Debug(fmt.Sprintf(format, o...))
}

func (l *syslogLogger) GetStdLogger() *log.Logger {
	return log.New(l.w, "", 0)
}
<commit_msg>log: change priority to local0 instead of kern<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows

package log

import (
	"fmt"
	"log"
	"log/syslog"
	"os"
)

var _ Logger = &syslogLogger{}

func NewSyslogLogger(tag string, debug bool) (Logger, error) {
	priority := syslog.LOG_LOCAL0 | syslog.LOG_INFO
	w, err := syslog.New(priority, tag)
	if err != nil {
		return nil, err
	}
	return &syslogLogger{w: w, debug: debug}, nil
}

type syslogLogger struct {
	w     *syslog.Writer
	debug bool
}

func (l *syslogLogger) Error(o string) {
	l.w.Err(o)
}

func (l *syslogLogger) Errorf(format string, o ...interface{}) {
	l.w.Err(fmt.Sprintf(format, o...))
}

func (l *syslogLogger) Fatal(o string) {
	l.w.Err(fmt.Sprintf(fatalPrefix, o))
	os.Exit(1)
}

func (l *syslogLogger) Fatalf(format string, o ...interface{}) {
	l.Fatal(fmt.Sprintf(format, o...))
}

func (l *syslogLogger) Debug(o string) {
	if l.debug {
		l.w.Debug(o)
	}
}

func (l *syslogLogger) Debugf(format string, o ...interface{}) {
	l.Debug(fmt.Sprintf(format, o...))
}

func (l *syslogLogger) GetStdLogger() *log.Logger {
	return log.New(l.w, "", 0)
}
<|endoftext|>


<commit_before>package analyze

import (
	"math"

	"github.com/phil-mansfield/gotetra/math/mat"
	"github.com/phil-mansfield/gotetra/los"

	"github.com/gonum/matrix/mat64"
)

func pinv(m, t *mat.Matrix) *mat.Matrix {
	// I HATE THIS
	gm := mat64.NewDense(m.Height, m.Width, m.Vals)
	gmt := mat64.NewDense(m.Width, m.Height, t.Vals)

	out1 := mat64.NewDense(m.Height, m.Height,
		make([]float64, m.Height * m.Height))
	out2 := mat64.NewDense(m.Width, m.Height,
		make([]float64, m.Height * m.Width))
	out1.Mul(gm, gmt)
	inv, err := mat64.Inverse(out1)
	if err != nil { panic(err.Error()) }
	out2.Mul(gmt, inv)

	vals := make([]float64, m.Width*m.Height)
	for y := 0; y < m.Width; y++ {
		for x := 0; x < m.Height; x++ {
			vals[y*m.Height + x] = out2.At(y, x)
		}
	}
	return mat.NewMatrix(vals, m.Height, m.Width)
}

func PennaCoeffs(xs, ys, zs []float64, I, J, K int) []float64 {
	N := len(xs)
	// TODO: Pass buffers to the function.
	rs := make([]float64, N)
	cosths := make([]float64, N)
	sinths := make([]float64, N)
	cosphis := make([]float64, N)
	sinphis := make([]float64, N)
	cs := make([]float64, I*J*K)

	// Precompute trig functions.
	for i := range rs {
		rs[i] = math.Sqrt(xs[i]*xs[i] + ys[i]*ys[i] + zs[i]*zs[i])
		cosths[i] = zs[i] / rs[i]
		sinths[i] = math.Sqrt(1 - cosths[i]*cosths[i])
		cosphis[i] = xs[i] / rs[i] / sinths[i]
		sinphis[i] = ys[i] / rs[i] / sinths[i]
	}

	MVals := make([]float64, I*J*K * len(xs))
	M := mat.NewMatrix(MVals, len(rs), I*J*K)

	// Populate matrix.
	for n := 0; n < N; n++ {
		m := 0
		for k := 0; k < K; k++ {
			costh := math.Pow(cosths[n], float64(k))
			for j := 0; j < J; j++ {
				sinphi := math.Pow(sinphis[n], float64(j))
				cosphi := 1.0
				for i := 0; i < I; i++ {
					MVals[m*M.Width + n] =
						math.Pow(sinths[n], float64(i+j)) *
						cosphi * costh * sinphi
					m++
					cosphi *= cosphis[n]
				}
			}
		}
	}

	// Solve.
	mat.VecMult(rs, pinv(M, M.Transpose()), cs)
	return cs
}

func PennaFunc(cs []float64, I, J, K int) func(phi, th float64) float64 {
	return func(phi, th float64) float64 {
		idx, sum := 0, 0.0
		sinPhi, cosPhi := math.Sincos(phi)
		sinTh, cosTh := math.Sincos(th)

		for k := 0; k < K; k++ {
			cosK := math.Pow(cosTh, float64(k))
			for j := 0; j < J; j++ {
				sinJ := math.Pow(sinPhi, float64(j))
				for i := 0; i < I; i++ {
					cosI := math.Pow(cosPhi, float64(i))
					sinIJ := math.Pow(sinTh, float64(i+j))
					sum += cs[idx] * sinIJ * cosK * sinJ * cosI
					idx++
				}
			}
		}
		return sum
	}
}

func PennaVolumeFit(
	xs, ys [][]float64, h los.HaloProfiles, I, J int,
) (cs []float64, shell Shell) {
	n := 0
	for i := range xs { n += len(xs[i]) }
	fXs, fYs, fZs := make([]float64, n), make([]float64, n), make([]float64, n)

	idx := 0
	for i := range xs {
		for j := range xs[i] {
			fXs[idx], fYs[idx], fZs[idx] =
				h.PlaneToVolume(i, xs[i][j], ys[i][j])
			idx++
		}
	}

	cs = PennaCoeffs(fXs, fYs, fZs, I, J, 2)
	return cs, PennaFunc(cs, I, J, 2)
}

func PennaPlaneFit(
	xs, ys [][]float64, hRef *los.HaloProfiles, I, J int,
) (cs []float64, shell ProjectedShell) {
	n := 0
	for i := range xs { n += len(xs[i]) }
	fXs, fYs, fZs := make([]float64, n), make([]float64, n), make([]float64, n)

	idx := 0
	for i := range xs {
		for j := range xs[i] {
			fXs[idx], fYs[idx], fZs[idx] =
				hRef.PlaneToVolume(i, xs[i][j], ys[i][j])
			idx++
		}
	}

	cs = PennaCoeffs(fXs, fYs, fZs, I, J, 2)
	pf := PennaFunc(cs, I, J, 2)
	return cs, func (h *los.HaloProfiles, ring int, phi float64) float64 {
		sin, cos := math.Sincos(phi)
		x, y, z := h.PlaneToVolume(ring, cos, sin)
		pi2 := 2 * math.Pi
		return pf(math.Mod(math.Atan2(y, x) + pi2, pi2), math.Acos(z))
	}
}

func FilterPoints(
	rs []RingBuffer, levels int,
) (pxs, pys [][]float64) {
	pxs, pys = [][]float64{}, [][]float64{}
	for ri := range rs {
		r := &rs[ri]
		validXs := make([]float64, 0, r.N)
		validYs := make([]float64, 0, r.N)

		for i := 0; i < r.N; i++ {
			if r.Oks[i] {
				validXs = append(validXs, r.PlaneXs[i])
				validYs = append(validYs, r.PlaneYs[i])
			}
		}

		validRs, validPhis := []float64{}, []float64{}
		for i := range r.Rs {
			if r.Oks[i] {
				validRs = append(validRs, r.Rs[i])
				validPhis = append(validPhis, r.Phis[i])
			}
		}

		kt := NewKDETree(validRs, validPhis, levels)
		fRs, fThs, _ := kt.FilterNearby(validRs, validPhis, levels, kt.H() / 2)
		fXs, fYs := make([]float64, len(fRs)), make([]float64, len(fRs))
		for i := range fRs {
			sin, cos := math.Sincos(fThs[i])
			fXs[i], fYs[i] = fRs[i] * cos, fRs[i] * sin
		}

		pxs, pys = append(pxs, fXs), append(pys, fYs)
	}
	return pxs, pys
}
<commit_msg>Changed PennaVolumeFit to take a pointer to HaloProfiles instead of HaloProfiles directly.<commit_after>package analyze

import (
	"math"

	"github.com/phil-mansfield/gotetra/math/mat"
	"github.com/phil-mansfield/gotetra/los"

	"github.com/gonum/matrix/mat64"
)

func pinv(m, t *mat.Matrix) *mat.Matrix {
	// I HATE THIS
	gm := mat64.NewDense(m.Height, m.Width, m.Vals)
	gmt := mat64.NewDense(m.Width, m.Height, t.Vals)

	out1 := mat64.NewDense(m.Height, m.Height,
		make([]float64, m.Height * m.Height))
	out2 := mat64.NewDense(m.Width, m.Height,
		make([]float64, m.Height * m.Width))
	out1.Mul(gm, gmt)
	inv, err := mat64.Inverse(out1)
	if err != nil { panic(err.Error()) }
	out2.Mul(gmt, inv)

	vals := make([]float64, m.Width*m.Height)
	for y := 0; y < m.Width; y++ {
		for x := 0; x < m.Height; x++ {
			vals[y*m.Height + x] = out2.At(y, x)
		}
	}
	return mat.NewMatrix(vals, m.Height, m.Width)
}

func PennaCoeffs(xs, ys, zs []float64, I, J, K int) []float64 {
	N := len(xs)
	// TODO: Pass buffers to the function.
	rs := make([]float64, N)
	cosths := make([]float64, N)
	sinths := make([]float64, N)
	cosphis := make([]float64, N)
	sinphis := make([]float64, N)
	cs := make([]float64, I*J*K)

	// Precompute trig functions.
	for i := range rs {
		rs[i] = math.Sqrt(xs[i]*xs[i] + ys[i]*ys[i] + zs[i]*zs[i])
		cosths[i] = zs[i] / rs[i]
		sinths[i] = math.Sqrt(1 - cosths[i]*cosths[i])
		cosphis[i] = xs[i] / rs[i] / sinths[i]
		sinphis[i] = ys[i] / rs[i] / sinths[i]
	}

	MVals := make([]float64, I*J*K * len(xs))
	M := mat.NewMatrix(MVals, len(rs), I*J*K)

	// Populate matrix.
	for n := 0; n < N; n++ {
		m := 0
		for k := 0; k < K; k++ {
			costh := math.Pow(cosths[n], float64(k))
			for j := 0; j < J; j++ {
				sinphi := math.Pow(sinphis[n], float64(j))
				cosphi := 1.0
				for i := 0; i < I; i++ {
					MVals[m*M.Width + n] =
						math.Pow(sinths[n], float64(i+j)) *
						cosphi * costh * sinphi
					m++
					cosphi *= cosphis[n]
				}
			}
		}
	}

	// Solve.
	mat.VecMult(rs, pinv(M, M.Transpose()), cs)
	return cs
}

func PennaFunc(cs []float64, I, J, K int) func(phi, th float64) float64 {
	return func(phi, th float64) float64 {
		idx, sum := 0, 0.0
		sinPhi, cosPhi := math.Sincos(phi)
		sinTh, cosTh := math.Sincos(th)

		for k := 0; k < K; k++ {
			cosK := math.Pow(cosTh, float64(k))
			for j := 0; j < J; j++ {
				sinJ := math.Pow(sinPhi, float64(j))
				for i := 0; i < I; i++ {
					cosI := math.Pow(cosPhi, float64(i))
					sinIJ := math.Pow(sinTh, float64(i+j))
					sum += cs[idx] * sinIJ * cosK * sinJ * cosI
					idx++
				}
			}
		}
		return sum
	}
}

func PennaVolumeFit(
	xs, ys [][]float64, h *los.HaloProfiles, I, J int,
) (cs []float64, shell Shell) {
	n := 0
	for i := range xs { n += len(xs[i]) }
	fXs, fYs, fZs := make([]float64, n), make([]float64, n), make([]float64, n)

	idx := 0
	for i := range xs {
		for j := range xs[i] {
			fXs[idx], fYs[idx], fZs[idx] =
				h.PlaneToVolume(i, xs[i][j], ys[i][j])
			idx++
		}
	}

	cs = PennaCoeffs(fXs, fYs, fZs, I, J, 2)
	return cs, PennaFunc(cs, I, J, 2)
}

func PennaPlaneFit(
	xs, ys [][]float64, hRef *los.HaloProfiles, I, J int,
) (cs []float64, shell ProjectedShell) {
	n := 0
	for i := range xs { n += len(xs[i]) }
	fXs, fYs, fZs := make([]float64, n), make([]float64, n), make([]float64, n)

	idx := 0
	for i := range xs {
		for j := range xs[i] {
			fXs[idx], fYs[idx], fZs[idx] =
				hRef.PlaneToVolume(i, xs[i][j], ys[i][j])
			idx++
		}
	}

	cs = PennaCoeffs(fXs, fYs, fZs, I, J, 2)
	pf := PennaFunc(cs, I, J, 2)
	return cs, func (h *los.HaloProfiles, ring int, phi float64) float64 {
		sin, cos := math.Sincos(phi)
		x, y, z := h.PlaneToVolume(ring, cos, sin)
		pi2 := 2 * math.Pi
		return pf(math.Mod(math.Atan2(y, x) + pi2, pi2), math.Acos(z))
	}
}

func FilterPoints(
	rs []RingBuffer, levels int,
) (pxs, pys [][]float64) {
	pxs, pys = [][]float64{}, [][]float64{}
	for ri := range rs {
		r := &rs[ri]
		validXs := make([]float64, 0, r.N)
		validYs := make([]float64, 0, r.N)

		for i := 0; i < r.N; i++ {
			if r.Oks[i] {
				validXs = append(validXs, r.PlaneXs[i])
				validYs = append(validYs, r.PlaneYs[i])
			}
		}

		validRs, validPhis := []float64{}, []float64{}
		for i := range r.Rs {
			if r.Oks[i] {
				validRs = append(validRs, r.Rs[i])
				validPhis = append(validPhis, r.Phis[i])
			}
		}

		kt := NewKDETree(validRs, validPhis, levels)
		fRs, fThs, _ := kt.FilterNearby(validRs, validPhis, levels, kt.H() / 2)
		fXs, fYs := make([]float64, len(fRs)), make([]float64, len(fRs))
		for i := range fRs {
			sin, cos := math.Sincos(fThs[i])
			fXs[i], fYs[i] = fRs[i] * cos, fRs[i] * sin
		}

		pxs, pys = append(pxs, fXs), append(pys, fYs)
	}
	return pxs, pys
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"log"
	"path"
	"runtime"
	"sort"
	"sync"
)

var timeLocs []string
var tlOnce sync.Once

func listTimeLocations() ([]string, error) {
	zoneinfoZip := path.Join(runtime.GOROOT(), "lib", "time", "zoneinfo.zip")
	z, err := zip.OpenReader(zoneinfoZip)
	if err != nil {
		return nil, err
	}
	defer z.Close()

	locs := []string{}
	for _, f := range z.File {
		if f.Name[len(f.Name)-1] == '/' {
			continue
		}
		locs = append(locs, f.Name)
	}

	sort.Strings(locs)
	return locs, nil
}

func loadTimeLocs() {
	tlOnce.Do(func() {
		var err error
		if timeLocs, err = listTimeLocations(); err != nil {
			log.Fatalf("Could not load time locations: %s", err)
		}
	})
}
<commit_msg>Searching for time locations in other directories.<commit_after>package main

import (
	"archive/zip"
	"log"
	"os"
	"path"
	"runtime"
	"sort"
	"sync"
	"time"
)

var timeLocs []string
var tlOnce sync.Once

func findfiles(p, prefix string, files []string) []string {
	d, err := os.Open(p)
	if err != nil {
		return files
	}
	defer d.Close()

	infos, err := d.Readdir(-1)
	if err != nil {
		return files
	}

	for _, info := range infos {
		if info.Mode().IsRegular() {
			files = append(files, prefix+info.Name())
		} else if info.IsDir() {
			files = findfiles(path.Join(p, info.Name()), info.Name()+"/", files)
		}
	}

	return files
}

func listTimeLocations() ([]string, error) {
	for _, p := range []string{"/usr/share/zoneinfo", "/usr/share/lib/zoneinfo", "/usr/lib/locale/TZ"} {
		files := findfiles(p, "", nil)
		duprem := make(map[string]bool)
		for _, loc := range files {
			if _, err := time.LoadLocation(loc); err == nil {
				duprem[loc] = true
			}
		}
		var locs []string
		for loc := range duprem {
			locs = append(locs, loc)
		}
		if len(locs) > 0 {
			sort.Strings(locs)
			return locs, nil
		}
	}

	zoneinfoZip := path.Join(runtime.GOROOT(), "lib", "time", "zoneinfo.zip")
	z, err := zip.OpenReader(zoneinfoZip)
	if err != nil {
		return nil, err
	}
	defer z.Close()

	locs := []string{}
	for _, f := range z.File {
		if f.Name[len(f.Name)-1] == '/' {
			continue
		}
		locs = append(locs, f.Name)
	}

	sort.Strings(locs)
	return locs, nil
}

func loadTimeLocs() {
	tlOnce.Do(func() {
		var err error
		if timeLocs, err = listTimeLocations(); err != nil {
			log.Fatalf("Could not load time locations: %s", err)
		}
	})
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "os"
//import "net"
import "crypto/tls"

import "crypto/x509"
//import "crypto/rsa"
//import "crypto/dsa"
//import "crypto/ecdsa"

import "encoding/pem"

func main() {
  fmt.Println("google's public key:")

  conn, err := tls.Dial("tcp", "mail.google.com:443", &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()
  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for cert_count, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    fmt.Println("got cert #", cert_count)

    var pem_type string = ""

    switch cert.PublicKeyAlgorithm {
    case x509.RSA:
      fmt.Println("it's a RSA key")
      pem_type = "RSA PUBLIC KEY"

    case x509.ECDSA:
      fmt.Println("it's a ECDSA key")
      pem_type = "ECDSA PUBLIC KEY"

    case x509.DSA:
      fmt.Println("it's a DSA key")
      pem_type = "DSA PUBLIC KEY"

    default:
      fmt.Println("no clue")
    }

    pem_pubkey := &pem.Block{
      Type: pem_type,
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
  fmt.Println("didn't break?")
}
<commit_msg>remove debugging prints<commit_after>package main

//import "fmt"
import "os"
import "crypto/tls"
import "crypto/x509"
import "encoding/pem"

func main() {
  conn, err := tls.Dial("tcp", "mail.google.com:443", &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()
  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    var pem_type string = ""

    switch cert.PublicKeyAlgorithm {
    case x509.RSA:
      pem_type = "RSA PUBLIC KEY"

    case x509.ECDSA:
      pem_type = "ECDSA PUBLIC KEY"

    case x509.DSA:
      pem_type = "DSA PUBLIC KEY"
    }

    pem_pubkey := &pem.Block{
      Type: pem_type,
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type imageStreamCacheEntry struct {
	ss     *shared.SimpleStreams
	expiry time.Time
}

var imageStreamCache = map[string]*imageStreamCacheEntry{}
var imageStreamCacheLock sync.Mutex

// ImageDownload checks if we have that Image Fingerprint else
// downloads the image from a remote server.
func (d *Daemon) ImageDownload(op *operation, server string, protocol string, certificate string, secret string, alias string, forContainer bool, autoUpdate bool) (string, error) {
	var err error
	var ss *shared.SimpleStreams

	if protocol == "" {
		protocol = "lxd"
	}

	fp := alias

	// Expand aliases
	if protocol == "simplestreams" {
		imageStreamCacheLock.Lock()
		entry, _ := imageStreamCache[server]
		if entry == nil || entry.expiry.Before(time.Now()) {
			ss, err = shared.SimpleStreamsClient(server, d.proxy)
			if err != nil {
				imageStreamCacheLock.Unlock()
				return "", err
			}

			entry = &imageStreamCacheEntry{ss: ss, expiry: time.Now().Add(time.Hour)}
			imageStreamCache[server] = entry
		} else {
			shared.LogDebugf("Using SimpleStreams cache entry for %s, expires at %s", server, entry.expiry)
			ss = entry.ss
		}
		imageStreamCacheLock.Unlock()

		target := ss.GetAlias(fp)
		if target != "" {
			fp = target
		}

		image, err := ss.GetImageInfo(fp)
		if err != nil {
			return "", err
		}

		if fp == alias {
			alias = image.Fingerprint
		}
		fp = image.Fingerprint
	} else if protocol == "lxd" {
		target, err := remoteGetImageFingerprint(d, server, certificate, fp)
		if err == nil && target != "" {
			fp = target
		}
	}

	if _, _, err := dbImageGet(d.db, fp, false, false); err == nil {
		shared.Log.Debug("Image already exists in the db", log.Ctx{"image": fp})
		// already have it
		return fp, nil
	}

	shared.Log.Info(
		"Image not in the db, downloading it",
		log.Ctx{"image": fp, "server": server})

	// Now check if we already downloading the image
	d.imagesDownloadingLock.RLock()
	if waitChannel, ok := d.imagesDownloading[fp]; ok {
		// We already download the image
		d.imagesDownloadingLock.RUnlock()

		shared.Log.Info(
			"Already downloading the image, waiting for it to succeed",
			log.Ctx{"image": fp})

		// Wait until the download finishes (channel closes)
		if _, ok := <-waitChannel; ok {
			shared.Log.Warn("Value transmitted over image lock semaphore?")
		}

		if _, _, err := dbImageGet(d.db, fp, false, true); err != nil {
			shared.Log.Error(
				"Previous download didn't succeed",
				log.Ctx{"image": fp})

			return "", fmt.Errorf("Previous download didn't succeed")
		}

		shared.Log.Info(
			"Previous download succeeded",
			log.Ctx{"image": fp})

		return fp, nil
	}

	d.imagesDownloadingLock.RUnlock()

	shared.Log.Info(
		"Downloading the image",
		log.Ctx{"image": fp})

	// Add the download to the queue
	d.imagesDownloadingLock.Lock()
	d.imagesDownloading[fp] = make(chan bool)
	d.imagesDownloadingLock.Unlock()

	// Unlock once this func ends.
	defer func() {
		d.imagesDownloadingLock.Lock()
		if waitChannel, ok := d.imagesDownloading[fp]; ok {
			close(waitChannel)
			delete(d.imagesDownloading, fp)
		}
		d.imagesDownloadingLock.Unlock()
	}()

	exporturl := server

	var info shared.ImageInfo
	info.Fingerprint = fp

	destDir := shared.VarPath("images")
	destName := filepath.Join(destDir, fp)
	if shared.PathExists(destName) {
		d.Storage.ImageDelete(fp)
	}

	progress := func(progressInt int) {
		if op == nil {
			return
		}

		meta := op.metadata
		if meta == nil {
			meta = make(map[string]interface{})
		}

		progress := fmt.Sprintf("%d%%", progressInt)

		if meta["download_progress"] != progress {
			meta["download_progress"] = progress
			op.UpdateMetadata(meta)
		}
	}

	if protocol == "lxd" {
		/* grab the metadata from /1.0/images/%s */
		var url string
		if secret != "" {
			url = fmt.Sprintf(
				"%s/%s/images/%s?secret=%s",
				server, shared.APIVersion, fp, secret)
		} else {
			url = fmt.Sprintf("%s/%s/images/%s", server, shared.APIVersion, fp)
		}

		resp, err := d.httpGetSync(url, certificate)
		if err != nil {
			shared.Log.Error(
				"Failed to download image metadata",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		if err := json.Unmarshal(resp.Metadata, &info); err != nil {
			return "", err
		}

		/* now grab the actual file from /1.0/images/%s/export */
		if secret != "" {
			exporturl = fmt.Sprintf(
				"%s/%s/images/%s/export?secret=%s",
				server, shared.APIVersion, fp, secret)

		} else {
			exporturl = fmt.Sprintf(
				"%s/%s/images/%s/export",
				server, shared.APIVersion, fp)
		}
	} else if protocol == "simplestreams" {
		err := ss.Download(fp, "meta", destName, nil)
		if err != nil {
			return "", err
		}

		err = ss.Download(fp, "root", destName+".rootfs", progress)
		if err != nil {
			return "", err
		}

		info, err := ss.GetImageInfo(fp)
		if err != nil {
			return "", err
		}

		info.Public = false
		info.AutoUpdate = autoUpdate

		_, err = imageBuildFromInfo(d, *info)
		if err != nil {
			return "", err
		}

		if alias != fp {
			id, _, err := dbImageGet(d.db, fp, false, true)
			if err != nil {
				return "", err
			}

			err = dbImageSourceInsert(d.db, id, server, protocol, "", alias)
			if err != nil {
				return "", err
			}
		}

		if forContainer {
			return fp, dbImageLastAccessInit(d.db, fp)
		}

		return fp, nil
	}

	raw, err := d.httpGetFile(exporturl, certificate)
	if err != nil {
		shared.Log.Error(
			"Failed to download image",
			log.Ctx{"image": fp, "err": err})
		return "", err
	}
	info.Size = raw.ContentLength

	ctype, ctypeParams, err := mime.ParseMediaType(raw.Header.Get("Content-Type"))
	if err != nil {
		ctype = "application/octet-stream"
	}

	body := &shared.TransferProgress{Reader: raw.Body, Length: raw.ContentLength, Handler: progress}

	if ctype == "multipart/form-data" {
		// Parse the POST data
		mr := multipart.NewReader(body, ctypeParams["boundary"])

		// Get the metadata tarball
		part, err := mr.NextPart()
		if err != nil {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		if part.FormName() != "metadata" {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return "", fmt.Errorf("Invalid multipart image")
		}

		destName = filepath.Join(destDir, info.Fingerprint)
		f, err := os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		// Get the rootfs tarball
		part, err = mr.NextPart()
		if err != nil {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		if part.FormName() != "rootfs" {
			shared.Log.Error(
				"Invalid multipart image",
				log.Ctx{"image": fp})
			return "", fmt.Errorf("Invalid multipart image")
		}

		destName = filepath.Join(destDir, info.Fingerprint+".rootfs")
		f, err = os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return "", err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return "", err
		}
	} else {
		destName = filepath.Join(destDir, info.Fingerprint)

		f, err := os.Create(destName)
		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		_, err = io.Copy(f, body)
		f.Close()

		if err != nil {
			shared.Log.Error(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return "", err
		}
	}

	if protocol == "direct" {
		imageMeta, err := getImageMetadata(destName)
		if err != nil {
			return "", err
		}

		info.Architecture = imageMeta.Architecture
		info.CreationDate = time.Unix(imageMeta.CreationDate, 0)
		info.ExpiryDate = time.Unix(imageMeta.ExpiryDate, 0)
		info.Properties = imageMeta.Properties
	}

	// By default, make all downloaded images private
	info.Public = false

	if alias != fp && secret == "" {
		info.AutoUpdate = autoUpdate
	}

	_, err = imageBuildFromInfo(d, info)
	if err != nil {
		shared.Log.Error(
			"Failed to create image",
			log.Ctx{"image": fp, "err": err})

		return "", err
	}

	if alias != fp {
		id, _, err := dbImageGet(d.db, fp, false, true)
		if err != nil {
			return "", err
		}

		err = dbImageSourceInsert(d.db, id, server, protocol, "", alias)
		if err != nil {
			return "", err
		}
	}

	shared.Log.Info(
		"Download succeeded",
		log.Ctx{"image": fp})

	if forContainer {
		return fp, dbImageLastAccessInit(d.db, fp)
	}

	return fp, nil
}
<commit_msg>lxd/daemon_images: switch to new logging functions<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type imageStreamCacheEntry struct {
	ss     *shared.SimpleStreams
	expiry time.Time
}

var imageStreamCache = map[string]*imageStreamCacheEntry{}
var imageStreamCacheLock sync.Mutex

// ImageDownload checks if we have that Image Fingerprint else
// downloads the image from a remote server.
func (d *Daemon) ImageDownload(op *operation, server string, protocol string, certificate string, secret string, alias string, forContainer bool, autoUpdate bool) (string, error) {
	var err error
	var ss *shared.SimpleStreams

	if protocol == "" {
		protocol = "lxd"
	}

	fp := alias

	// Expand aliases
	if protocol == "simplestreams" {
		imageStreamCacheLock.Lock()
		entry, _ := imageStreamCache[server]
		if entry == nil || entry.expiry.Before(time.Now()) {
			ss, err = shared.SimpleStreamsClient(server, d.proxy)
			if err != nil {
				imageStreamCacheLock.Unlock()
				return "", err
			}

			entry = &imageStreamCacheEntry{ss: ss, expiry: time.Now().Add(time.Hour)}
			imageStreamCache[server] = entry
		} else {
			shared.LogDebugf("Using SimpleStreams cache entry for %s, expires at %s", server, entry.expiry)
			ss = entry.ss
		}
		imageStreamCacheLock.Unlock()

		target := ss.GetAlias(fp)
		if target != "" {
			fp = target
		}

		image, err := ss.GetImageInfo(fp)
		if err != nil {
			return "", err
		}

		if fp == alias {
			alias = image.Fingerprint
		}
		fp = image.Fingerprint
	} else if protocol == "lxd" {
		target, err := remoteGetImageFingerprint(d, server, certificate, fp)
		if err == nil && target != "" {
			fp = target
		}
	}

	if _, _, err := dbImageGet(d.db, fp, false, false); err == nil {
		shared.LogDebug("Image already exists in the db", log.Ctx{"image": fp})
		// already have it
		return fp, nil
	}

	shared.LogInfo(
		"Image not in the db, downloading it",
		log.Ctx{"image": fp, "server": server})

	// Now check if we already downloading the image
	d.imagesDownloadingLock.RLock()
	if waitChannel, ok := d.imagesDownloading[fp]; ok {
		// We already download the image
		d.imagesDownloadingLock.RUnlock()

		shared.LogInfo(
			"Already downloading the image, waiting for it to succeed",
			log.Ctx{"image": fp})

		// Wait until the download finishes (channel closes)
		if _, ok := <-waitChannel; ok {
			shared.LogWarnf("Value transmitted over image lock semaphore?")
		}

		if _, _, err := dbImageGet(d.db, fp, false, true); err != nil {
			shared.LogError(
				"Previous download didn't succeed",
				log.Ctx{"image": fp})

			return "", fmt.Errorf("Previous download didn't succeed")
		}

		shared.LogInfo(
			"Previous download succeeded",
			log.Ctx{"image": fp})

		return fp, nil
	}

	d.imagesDownloadingLock.RUnlock()

	shared.LogInfo(
		"Downloading the image",
		log.Ctx{"image": fp})

	// Add the download to the queue
	d.imagesDownloadingLock.Lock()
	d.imagesDownloading[fp] = make(chan bool)
	d.imagesDownloadingLock.Unlock()

	// Unlock once this func ends.
	defer func() {
		d.imagesDownloadingLock.Lock()
		if waitChannel, ok := d.imagesDownloading[fp]; ok {
			close(waitChannel)
			delete(d.imagesDownloading, fp)
		}
		d.imagesDownloadingLock.Unlock()
	}()

	exporturl := server

	var info shared.ImageInfo
	info.Fingerprint = fp

	destDir := shared.VarPath("images")
	destName := filepath.Join(destDir, fp)
	if shared.PathExists(destName) {
		d.Storage.ImageDelete(fp)
	}

	progress := func(progressInt int) {
		if op == nil {
			return
		}

		meta := op.metadata
		if meta == nil {
			meta = make(map[string]interface{})
		}

		progress := fmt.Sprintf("%d%%", progressInt)

		if meta["download_progress"] != progress {
			meta["download_progress"] = progress
			op.UpdateMetadata(meta)
		}
	}

	if protocol == "lxd" {
		/* grab the metadata from /1.0/images/%s */
		var url string
		if secret != "" {
			url = fmt.Sprintf(
				"%s/%s/images/%s?secret=%s",
				server, shared.APIVersion, fp, secret)
		} else {
			url = fmt.Sprintf("%s/%s/images/%s", server, shared.APIVersion, fp)
		}

		resp, err := d.httpGetSync(url, certificate)
		if err != nil {
			shared.LogError(
				"Failed to download image metadata",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		if err := json.Unmarshal(resp.Metadata, &info); err != nil {
			return "", err
		}

		/* now grab the actual file from /1.0/images/%s/export */
		if secret != "" {
			exporturl = fmt.Sprintf(
				"%s/%s/images/%s/export?secret=%s",
				server, shared.APIVersion, fp, secret)

		} else {
			exporturl = fmt.Sprintf(
				"%s/%s/images/%s/export",
				server, shared.APIVersion, fp)
		}
	} else if protocol == "simplestreams" {
		err := ss.Download(fp, "meta", destName, nil)
		if err != nil {
			return "", err
		}

		err = ss.Download(fp, "root", destName+".rootfs", progress)
		if err != nil {
			return "", err
		}

		info, err := ss.GetImageInfo(fp)
		if err != nil {
			return "", err
		}

		info.Public = false
		info.AutoUpdate = autoUpdate

		_, err = imageBuildFromInfo(d, *info)
		if err != nil {
			return "", err
		}

		if alias != fp {
			id, _, err := dbImageGet(d.db, fp, false, true)
			if err != nil {
				return "", err
			}

			err = dbImageSourceInsert(d.db, id, server, protocol, "", alias)
			if err != nil {
				return "", err
			}
		}

		if forContainer {
			return fp, dbImageLastAccessInit(d.db, fp)
		}

		return fp, nil
	}

	raw, err := d.httpGetFile(exporturl, certificate)
	if err != nil {
		shared.LogError(
			"Failed to download image",
			log.Ctx{"image": fp, "err": err})
		return "", err
	}
	info.Size = raw.ContentLength

	ctype, ctypeParams, err := mime.ParseMediaType(raw.Header.Get("Content-Type"))
	if err != nil {
		ctype = "application/octet-stream"
	}

	body := &shared.TransferProgress{Reader: raw.Body, Length: raw.ContentLength, Handler: progress}

	if ctype == "multipart/form-data" {
		// Parse the POST data
		mr := multipart.NewReader(body, ctypeParams["boundary"])

		// Get the metadata tarball
		part, err := mr.NextPart()
		if err != nil {
			shared.LogError(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		if part.FormName() != "metadata" {
			shared.LogError(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return "", fmt.Errorf("Invalid multipart image")
		}

		destName = filepath.Join(destDir, info.Fingerprint)
		f, err := os.Create(destName)
		if err != nil {
			shared.LogError(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.LogError(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		// Get the rootfs tarball
		part, err = mr.NextPart()
		if err != nil {
			shared.LogError(
				"Invalid multipart image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		if part.FormName() != "rootfs" {
			shared.LogError(
				"Invalid multipart image",
				log.Ctx{"image": fp})
			return "", fmt.Errorf("Invalid multipart image")
		}

		destName = filepath.Join(destDir, info.Fingerprint+".rootfs")
		f, err = os.Create(destName)
		if err != nil {
			shared.LogError(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return "", err
		}

		_, err = io.Copy(f, part)
		f.Close()

		if err != nil {
			shared.LogError(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return "", err
		}
	} else {
		destName = filepath.Join(destDir, info.Fingerprint)

		f, err := os.Create(destName)
		if err != nil {
			shared.LogError(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})

			return "", err
		}

		_, err = io.Copy(f, body)
		f.Close()

		if err != nil {
			shared.LogError(
				"Failed to save image",
				log.Ctx{"image": fp, "err": err})
			return "", err
		}
	}

	if protocol == "direct" {
		imageMeta, err := getImageMetadata(destName)
		if err != nil {
			return "", err
		}

		info.Architecture = imageMeta.Architecture
		info.CreationDate = time.Unix(imageMeta.CreationDate, 0)
		info.ExpiryDate = time.Unix(imageMeta.ExpiryDate, 0)
		info.Properties = imageMeta.Properties
	}

	// By default, make all downloaded images private
	info.Public = false

	if alias != fp && secret == "" {
		info.AutoUpdate = autoUpdate
	}

	_, err = imageBuildFromInfo(d, info)
	if err != nil {
		shared.LogError(
			"Failed to create image",
			log.Ctx{"image": fp, "err": err})

		return "", err
	}

	if alias != fp {
		id, _, err := dbImageGet(d.db, fp, false, true)
		if err != nil {
			return "", err
		}

		err = dbImageSourceInsert(d.db, id, server, protocol, "", alias)
		if err != nil {
			return "", err
		}
	}

	shared.LogInfo(
		"Download succeeded",
		log.Ctx{"image": fp})

	if forContainer {
		return fp, dbImageLastAccessInit(d.db, fp)
	}

	return fp, nil
}
<|endoftext|>


<commit_before>package audit

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pborman/uuid"
	"github.com/pkg/errors"
	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	"github.com/sirupsen/logrus"
	k8stypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/apiserver/pkg/endpoints/request"
)

const (
	contentTypeJSON = "application/json"
)

const (
	levelNull = iota
	levelMetadata
	levelRequest
	levelRequestResponse
)

var (
	bodyMethods = map[string]bool{
		http.MethodPut:  true,
		http.MethodPost: true,
	}
	sensitiveRequestHeader  = []string{"Cookie", "Authorization"}
	sensitiveResponseHeader = []string{"Cookie", "Set-Cookie"}
)

type auditLog struct {
	log     *log
	writer  *LogWriter
	reqBody []byte
}

type log struct {
	AuditID           k8stypes.UID `json:"auditID,omitempty"`
	RequestURI        string       `json:"requestURI,omitempty"`
	User              *User        `json:"user,omitempty"`
	Method            string       `json:"method,omitempty"`
	RemoteAddr        string       `json:"remoteAddr,omitempty"`
	RequestTimestamp  string       `json:"requestTimestamp,omitempty"`
	ResponseTimestamp string       `json:"responseTimestamp,omitempty"`
	ResponseCode      int          `json:"responseCode,omitempty"`
	RequestHeader     http.Header  `json:"requestHeader,omitempty"`
	ResponseHeader    http.Header  `json:"responseHeader,omitempty"`
	RequestBody       []byte       `json:"requestBody,omitempty"`
	ResponseBody      []byte       `json:"responseBody,omitempty"`
	UserLoginName     string       `json:"userLoginName,omitempty"`
}

var userKey struct{}

type User struct {
	Name  string              `json:"name,omitempty"`
	Group []string            `json:"group,omitempty"`
	Extra map[string][]string `json:"extra,omitempty"`
	// RequestUser is the --as user
	RequestUser string `json:"requestUser,omitempty"`
	// RequestGroups is the --as-group list
	RequestGroups []string `json:"requestGroups,omitempty"`
}

func getUserInfo(req *http.Request) *User {
	user, _ := request.UserFrom(req.Context())
	return &User{
		Name:  user.GetName(),
		Group: user.GetGroups(),
		Extra: user.GetExtra(),
	}
}

func getUserNameForBasicLogin(body []byte) string {
	input := &v32.BasicLogin{}
	err := json.Unmarshal(body, input)
	if err != nil {
		logrus.Debugf("error unmarshalling input, cannot add login info to audit log: %v", err)
		return ""
	}
	return input.Username
}

func FromContext(ctx context.Context) (*User, bool) {
	u, ok := ctx.Value(userKey).(*User)
	return u, ok
}

func newAuditLog(writer *LogWriter, req *http.Request) (*auditLog, error) {
	auditLog := &auditLog{
		writer: writer,
		log: &log{
			AuditID:          k8stypes.UID(uuid.NewRandom().String()),
			RequestURI:       req.RequestURI,
			Method:           req.Method,
			RemoteAddr:       req.RemoteAddr,
			RequestTimestamp: time.Now().Format(time.RFC3339),
		},
	}

	contentType := req.Header.Get("Content-Type")
	loginReq := isLoginRequest(req.RequestURI)
	if writer.Level >= levelRequest || loginReq {
		if bodyMethods[req.Method] && contentType == contentTypeJSON {
			reqBody, err := readBodyWithoutLosingContent(req)
			if err != nil {
				return nil, err
			}
			if loginReq {
				loginName := getUserNameForBasicLogin(reqBody)
				if loginName != "" {
					auditLog.log.UserLoginName = loginName
				}
			}
			if writer.Level >= levelRequest {
				auditLog.reqBody = reqBody
			}
		}
	}
	return auditLog, nil
}

func (a *auditLog) write(userInfo *User, reqHeaders, resHeaders http.Header, resCode int, resBody []byte) error {
	a.log.User = userInfo
	a.log.ResponseTimestamp = time.Now().Format(time.RFC3339)
	a.log.RequestHeader = filterOutHeaders(reqHeaders, sensitiveRequestHeader)
	a.log.ResponseHeader = filterOutHeaders(resHeaders, sensitiveResponseHeader)
	a.log.ResponseCode = resCode
	if a.log.UserLoginName != "" {
		if a.log.User.Extra == nil {
			a.log.User.Extra = make(map[string][]string)
		}
		a.log.User.Extra["username"] = []string{a.log.UserLoginName}
		logrus.Debugf("Added username for login request to audit log %v", a.log.UserLoginName)
	}

	var buffer bytes.Buffer
	alByte, err := json.Marshal(a.log)
	if err != nil {
		return err
	}

	buffer.Write(bytes.TrimSuffix(alByte, []byte("}")))
	if a.writer.Level >= levelRequest && len(a.reqBody) > 0 {
		buffer.WriteString(`,"requestBody":`)
		buffer.Write(bytes.TrimSuffix(a.reqBody, []byte("\n")))
	}
	if a.writer.Level >= levelRequestResponse && resHeaders.Get("Content-Type") == contentTypeJSON && len(resBody) > 0 {
		buffer.WriteString(`,"responseBody":`)
		buffer.Write(bytes.TrimSuffix(resBody, []byte("\n")))
	}
	buffer.WriteString("}")

	var compactBuffer bytes.Buffer
	err = json.Compact(&compactBuffer, buffer.Bytes())
	if err != nil {
		return errors.Wrap(err, "compact audit log json failed")
	}

	compactBuffer.WriteString("\n")
	_, err = a.writer.Output.Write(compactBuffer.Bytes())
	return err
}

func isLoginRequest(uri string) bool {
	return strings.Contains(uri, "?action=login")
}

func readBodyWithoutLosingContent(req *http.Request) ([]byte, error) {
	if !bodyMethods[req.Method] {
		return nil, nil
	}

	bodyBytes, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}
	req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))

	return bodyBytes, nil
}

func filterOutHeaders(headers http.Header, filterKeys []string) map[string][]string {
	newHeader := make(map[string][]string)
	for k, v := range headers {
		if isExist(filterKeys, k) {
			continue
		}
		newHeader[k] = v
	}
	return newHeader
}

func isExist(array []string, key string) bool {
	for _, v := range array {
		if v == key {
			return true
		}
	}
	return false
}
<commit_msg>Check the content-type header having charset as well which is set by Dashboard UI<commit_after>package audit

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pborman/uuid"
	"github.com/pkg/errors"
	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	"github.com/sirupsen/logrus"
	k8stypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/apiserver/pkg/endpoints/request"
)

const (
	contentTypeJSON = "application/json"
)

const (
	levelNull = iota
	levelMetadata
	levelRequest
	levelRequestResponse
)

var (
	bodyMethods = map[string]bool{
		http.MethodPut:  true,
		http.MethodPost: true,
	}
	sensitiveRequestHeader  = []string{"Cookie", "Authorization"}
	sensitiveResponseHeader = []string{"Cookie", "Set-Cookie"}
)

type auditLog struct {
	log     *log
	writer  *LogWriter
	reqBody []byte
}

type log struct {
	AuditID           k8stypes.UID `json:"auditID,omitempty"`
	RequestURI        string       `json:"requestURI,omitempty"`
	User              *User        `json:"user,omitempty"`
	Method            string       `json:"method,omitempty"`
	RemoteAddr        string       `json:"remoteAddr,omitempty"`
	RequestTimestamp  string       `json:"requestTimestamp,omitempty"`
	ResponseTimestamp string       `json:"responseTimestamp,omitempty"`
	ResponseCode      int          `json:"responseCode,omitempty"`
	RequestHeader     http.Header  `json:"requestHeader,omitempty"`
	ResponseHeader    http.Header  `json:"responseHeader,omitempty"`
	RequestBody       []byte       `json:"requestBody,omitempty"`
	ResponseBody      []byte       `json:"responseBody,omitempty"`
	UserLoginName     string       `json:"userLoginName,omitempty"`
}

var userKey struct{}

type User struct {
	Name  string              `json:"name,omitempty"`
	Group []string            `json:"group,omitempty"`
	Extra map[string][]string `json:"extra,omitempty"`
	// RequestUser is the --as user
	RequestUser string `json:"requestUser,omitempty"`
	// RequestGroups is the --as-group list
	RequestGroups []string `json:"requestGroups,omitempty"`
}

func getUserInfo(req *http.Request) *User {
	user, _ := request.UserFrom(req.Context())
	return &User{
		Name:  user.GetName(),
		Group: user.GetGroups(),
		Extra: user.GetExtra(),
	}
}

func getUserNameForBasicLogin(body []byte) string {
	input := &v32.BasicLogin{}
	err := json.Unmarshal(body, input)
	if err != nil {
		logrus.Debugf("error unmarshalling input, cannot add login info to audit log: %v", err)
		return ""
	}
	return input.Username
}

func FromContext(ctx context.Context) (*User, bool) {
	u, ok := ctx.Value(userKey).(*User)
	return u, ok
}

func newAuditLog(writer *LogWriter, req *http.Request) (*auditLog, error) {
	auditLog := &auditLog{
		writer: writer,
		log: &log{
			AuditID:          k8stypes.UID(uuid.NewRandom().String()),
			RequestURI:       req.RequestURI,
			Method:           req.Method,
			RemoteAddr:       req.RemoteAddr,
			RequestTimestamp: time.Now().Format(time.RFC3339),
		},
	}

	contentType := req.Header.Get("Content-Type")
	loginReq := isLoginRequest(req.RequestURI)
	if writer.Level >= levelRequest || loginReq {
		if bodyMethods[req.Method] && strings.HasPrefix(contentType, contentTypeJSON) {
			reqBody, err := readBodyWithoutLosingContent(req)
			if err != nil {
				return nil, err
			}
			if loginReq {
				loginName := getUserNameForBasicLogin(reqBody)
				if loginName != "" {
					auditLog.log.UserLoginName = loginName
				}
			}
			if writer.Level >= levelRequest {
				auditLog.reqBody = reqBody
			}
		}
	}
	return auditLog, nil
}

func (a *auditLog) write(userInfo *User, reqHeaders, resHeaders http.Header, resCode int, resBody []byte) error {
	a.log.User = userInfo
	a.log.ResponseTimestamp = time.Now().Format(time.RFC3339)
	a.log.RequestHeader = filterOutHeaders(reqHeaders, sensitiveRequestHeader)
	a.log.ResponseHeader = filterOutHeaders(resHeaders, sensitiveResponseHeader)
	a.log.ResponseCode = resCode
	if a.log.UserLoginName != "" {
		if a.log.User.Extra == nil {
			a.log.User.Extra = make(map[string][]string)
		}
		a.log.User.Extra["username"] = []string{a.log.UserLoginName}
		logrus.Debugf("Added username for login request to audit log %v", a.log.UserLoginName)
	}

	var buffer bytes.Buffer
	alByte, err := json.Marshal(a.log)
	if err != nil {
		return err
	}

	buffer.Write(bytes.TrimSuffix(alByte, []byte("}")))
	if a.writer.Level >= levelRequest && len(a.reqBody) > 0 {
		buffer.WriteString(`,"requestBody":`)
		buffer.Write(bytes.TrimSuffix(a.reqBody, []byte("\n")))
	}
	if a.writer.Level >= levelRequestResponse && resHeaders.Get("Content-Type") == contentTypeJSON && len(resBody) > 0 {
		buffer.WriteString(`,"responseBody":`)
		buffer.Write(bytes.TrimSuffix(resBody, []byte("\n")))
	}
	buffer.WriteString("}")

	var compactBuffer bytes.Buffer
	err = json.Compact(&compactBuffer, buffer.Bytes())
	if err != nil {
		return errors.Wrap(err, "compact audit log json failed")
	}

	compactBuffer.WriteString("\n")
	_, err = a.writer.Output.Write(compactBuffer.Bytes())
	return err
}

func isLoginRequest(uri string) bool {
	return strings.Contains(uri, "?action=login")
}

func readBodyWithoutLosingContent(req *http.Request) ([]byte, error) {
	if !bodyMethods[req.Method] {
		return nil, nil
	}

	bodyBytes, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}
	req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))

	return bodyBytes, nil
}

func filterOutHeaders(headers http.Header, filterKeys []string) map[string][]string {
	newHeader := make(map[string][]string)
	for k, v := range headers {
		if isExist(filterKeys, k) {
			continue
		}
		newHeader[k] = v
	}
	return newHeader
}

func isExist(array []string, key string) bool {
	for _, v := range array {
		if v == key {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package util

import (
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// SiblingCommand returns a sibling command to the given command
func SiblingCommand(cmd *cobra.Command, name string) string {
	c := cmd.Parent()
	command := []string{}
	for c != nil {
		glog.V(5).Infof("Found parent command: %s", c.Name())
		command = append([]string{c.Name()}, command...)
		c = c.Parent()
	}
	// Replace the root command with what was actually used
	// in the command line
	glog.V(4).Infof("Setting root command to: %s", os.Args[0])
	command[0] = os.Args[0]

	// Append the sibling command
	command = append(command, name)
	glog.V(4).Infof("The sibling command is: %s", strings.Join(command, " "))

	return strings.Join(command, " ")
}
<commit_msg>oc sibling commands (example rsync) fail if there is a space in the path to the oc command<commit_after>package util

import (
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// SiblingCommand returns a sibling command to the given command
func SiblingCommand(cmd *cobra.Command, name string) string {
	c := cmd.Parent()
	command := []string{}
	for c != nil {
		glog.V(5).Infof("Found parent command: %s", c.Name())
		command = append([]string{c.Name()}, command...)
		c = c.Parent()
	}
	// Replace the root command with what was actually used
	// in the command line
	glog.V(4).Infof("Setting root command to: %s", os.Args[0])
	command[0] = "\"" + os.Args[0] + "\""

	// Append the sibling command
	command = append(command, name)
	glog.V(4).Infof("The sibling command is: %s", strings.Join(command, " "))

	return strings.Join(command, " ")
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package csicommon

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/golang/glog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

func ParseEndpoint(ep string) (string, string, error) {
	if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
		s := strings.SplitN(ep, "://", 2)
		if s[1] != "" {
			return s[0], s[1], nil
		}
	}
	return "", "", fmt.Errorf("Invalid endpoint: %v", ep)
}

func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode {
	return &csi.VolumeCapability_AccessMode{Mode: mode}
}

func NewDefaultNodeServer(d *CSIDriver) *DefaultNodeServer {
	return &DefaultNodeServer{
		Driver: d,
	}
}

func NewDefaultIdentityServer(d *CSIDriver) *DefaultIdentityServer {
	return &DefaultIdentityServer{
		Driver: d,
	}
}

func NewDefaultControllerServer(d *CSIDriver) *DefaultControllerServer {
	return &DefaultControllerServer{
		Driver: d,
	}
}

func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability {
	return &csi.ControllerServiceCapability{
		Type: &csi.ControllerServiceCapability_Rpc{
			Rpc: &csi.ControllerServiceCapability_RPC{
				Type: cap,
			},
		},
	}
}

func RunNodePublishServer(endpoint string, d *CSIDriver, ns csi.NodeServer) {
	ids := NewDefaultIdentityServer(d)

	s := NewNonBlockingGRPCServer()
	s.Start(endpoint, ids, nil, ns)
	s.Wait()
}

func RunControllerPublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer) {
	ids := NewDefaultIdentityServer(d)

	s := NewNonBlockingGRPCServer()
	s.Start(endpoint, ids, cs, nil)
	s.Wait()
}

func RunControllerandNodePublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer) {
	ids := NewDefaultIdentityServer(d)

	s := NewNonBlockingGRPCServer()
	s.Start(endpoint, ids, cs, ns)
	s.Wait()
}

func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
	glog.V(3).Infof("GRPC call: %s", info.FullMethod)
	logRedactedRequest(req)
	resp, err := handler(ctx, req)
	if err != nil {
		glog.Errorf("GRPC error: %v", err)
	} else {
		glog.V(5).Infof("GRPC response: %+v", resp)
	}
	return resp, err
}

func logRedactedRequest(req interface{}) {
	re, _ := regexp.Compile("^(\\S{4})(\\S|\\s)*(\\S{4})$")

	r, ok := req.(*csi.NodePublishVolumeRequest)
	if !ok {
		glog.V(5).Infof("GRPC request: %+v", req)
		return
	}

	req1 := *r
	redactedSecrets := make(map[string]string)

	secrets := req1.GetSecrets()
	for k, v := range secrets {
		switch k {
		case "clientid", "clientsecret":
			redactedSecrets[k] = re.ReplaceAllString(v, "$1##### REDACTED #####$3")
		default:
			redactedSecrets[k] = v
		}
	}
	req1.Secrets = redactedSecrets
	glog.V(5).Infof("GRPC request: %+v", req1)
}
<commit_msg>fix lint errors<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package csicommon

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/golang/glog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

func ParseEndpoint(ep string) (string, string, error) {
	if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
		s := strings.SplitN(ep, "://", 2)
		if s[1] != "" {
			return s[0], s[1], nil
		}
	}
	return "", "", fmt.Errorf("Invalid endpoint: %v", ep)
}

func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode {
	return &csi.VolumeCapability_AccessMode{Mode: mode}
}

func NewDefaultNodeServer(d *CSIDriver) *DefaultNodeServer {
	return &DefaultNodeServer{
		Driver: d,
	}
}

func NewDefaultIdentityServer(d *CSIDriver) *DefaultIdentityServer {
	return &DefaultIdentityServer{
		Driver: d,
	}
}

func NewDefaultControllerServer(d *CSIDriver) *DefaultControllerServer {
	return &DefaultControllerServer{
		Driver: d,
	}
}

func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability {
	return &csi.ControllerServiceCapability{
		Type: &csi.ControllerServiceCapability_Rpc{
			Rpc: &csi.ControllerServiceCapability_RPC{
				Type: cap,
			},
		},
	}
}

func RunNodePublishServer(endpoint string, d *CSIDriver, ns csi.NodeServer) {
	ids := NewDefaultIdentityServer(d)

	s := NewNonBlockingGRPCServer()
	s.Start(endpoint, ids, nil, ns)
	s.Wait()
}

func RunControllerPublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer) {
	ids := NewDefaultIdentityServer(d)

	s := NewNonBlockingGRPCServer()
	s.Start(endpoint, ids, cs, nil)
	s.Wait()
}

func RunControllerandNodePublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer) {
	ids := NewDefaultIdentityServer(d)

	s := NewNonBlockingGRPCServer()
	s.Start(endpoint, ids, cs, ns)
	s.Wait()
}

func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
	glog.V(3).Infof("GRPC call: %s", info.FullMethod)
	logRedactedRequest(req)
	resp, err := handler(ctx, req)
	if err != nil {
		glog.Errorf("GRPC error: %v", err)
	} else {
		glog.V(5).Infof("GRPC response: %+v", resp)
	}
	return resp, err
}

func logRedactedRequest(req interface{}) {
	re, _ := regexp.Compile(`^(\\S{4})(\\S|\\s)*(\\S{4})$`)

	r, ok := req.(*csi.NodePublishVolumeRequest)
	if !ok {
		glog.V(5).Infof("GRPC request: %+v", req)
		return
	}

	req1 := *r
	redactedSecrets := make(map[string]string)

	secrets := req1.GetSecrets()
	for k, v := range secrets {
		switch k {
		case "clientid", "clientsecret":
			redactedSecrets[k] = re.ReplaceAllString(v, "$1##### REDACTED #####$3")
		default:
			redactedSecrets[k] = v
		}
	}
	req1.Secrets = redactedSecrets
	glog.V(5).Infof("GRPC request: %+v", req1)
}
<|endoftext|>


<commit_before>package edit

// Implementation of the editor "command" mode.

import (
	"src.elv.sh/pkg/cli/mode"
	"src.elv.sh/pkg/eval"
)

//elvdoc:var command:binding
//
// Key bindings for command mode. The default bindings are a subset of Vi's command mode.
//
// TODO: Document the default bindings. For now note that they are codified in the
// *pkg/edit/default_bindings.go* source file. Specifically the `command:binding` assignment.
//
// @cf edit:command:start

//elvdoc:fn command:start
//
// Enter command mode. This is typically used to emulate the Vi editor's command mode by switching
// to the appropriate key bindings.
//
// @cf edit:command:binding

func initCommandAPI(ed *Editor, ev *eval.Evaler, nb eval.NsBuilder) {
	bindingVar := newBindingVar(emptyBindingsMap)
	bindings := newMapBindings(ed, ev, bindingVar)
	nb.AddNs("command",
		eval.NsBuilder{
			"binding": bindingVar,
		}.AddGoFns("<edit:command>:", map[string]interface{}{
			"start": func() {
				w := mode.NewStub(mode.StubSpec{
					Bindings: bindings,
					Name:     " COMMAND ",
				})
				ed.app.SetAddon(w, false)
			},
		}).Ns())
}
<commit_msg>Update doc for the command mode.<commit_after>package edit

// Implementation of the editor "command" mode.

import (
	"src.elv.sh/pkg/cli/mode"
	"src.elv.sh/pkg/eval"
)

//elvdoc:var command:binding
//
// Key bindings for command mode. This is currently a very small subset of Vi
// command mode bindings.
//
// @cf edit:command:start

//elvdoc:fn command:start
//
// Enter command mode. This mode is intended to emulate Vi's command mode, but
// it is very incomplete right now.
//
// @cf edit:command:binding

func initCommandAPI(ed *Editor, ev *eval.Evaler, nb eval.NsBuilder) {
	bindingVar := newBindingVar(emptyBindingsMap)
	bindings := newMapBindings(ed, ev, bindingVar)
	nb.AddNs("command",
		eval.NsBuilder{
			"binding": bindingVar,
		}.AddGoFns("<edit:command>:", map[string]interface{}{
			"start": func() {
				w := mode.NewStub(mode.StubSpec{
					Bindings: bindings,
					Name:     " COMMAND ",
				})
				ed.app.SetAddon(w, false)
			},
		}).Ns())
}
<|endoftext|>