<commit_before>// 30 august 2014

package pcmio

import (
	"syscall"
	"unsafe"
)

// #cgo CFLAGS: --std=c99
// #cgo LDFLAGS: -lkernel32 -lwinmm
// #include "winapi_windows.h"
import "C"

type writer struct {
	wo		C.HWAVEOUT
	event	C.HANDLE
}

func openDefaultWriter(format Format, rate uint) (*writer, error) {
	var fmt C.WAVEFORMATEX

	w := new(writer)
	lasterr := C.xCreateEvent(&w.event)
	if lasterr != 0 {
		return nil, syscall.Errno(lasterr)
	}

	fmt.wFormatTag = C.WAVE_FORMAT_PCM
	fmt.nChannels = 1
	fmt.nSamplesPerSec = C.DWORD(rate)
	switch format {
	case U8:
		fmt.wBitsPerSample = 8
	}
	fmt.nBlockAlign = fmt.nChannels * fmt.wBitsPerSample
	fmt.nAvgBytesPerSec = fmt.nSamplesPerSec * C.DWORD(fmt.nBlockAlign)
	fmt.cbSize = 0
	err := C.waveOutOpen(&w.wo, C.WAVE_MAPPER, &fmt,
		C.DWORD_PTR(uintptr(unsafe.Pointer(w.event))), 0,
		C.CALLBACK_EVENT | C.WAVE_FORMAT_DIRECT)
	if err != C.MMSYSERR_NOERROR {
		return nil, waveOutError(err)
	}

	return w, nil
}

func (w *writer) Write(p []byte) (n int, err error) {
	var hdr C.WAVEHDR

	hdr.lpData = C.LPSTR(unsafe.Pointer(&p[0]))
	hdr.dwBufferLength = C.DWORD(len(p))
	hdr.dwFlags = 0
	hdr.dwLoops = 0
	lasterr := C.xResetEvent(w.event)
	if lasterr != 0 {
		return 0, syscall.Errno(lasterr)
	}
	xerr := C.waveOutPrepareHeader(w.wo, &hdr, C.UINT(unsafe.Sizeof(hdr)))
	if xerr != C.MMSYSERR_NOERROR {
		return 0, waveOutError(xerr)
	}
	xerr = C.waveOutWrite(w.wo, &hdr, C.UINT(unsafe.Sizeof(hdr)))
	if xerr != C.MMSYSERR_NOERROR {
		return 0, waveOutError(xerr)
	}
	lasterr = C.xWaitForSingleObject(w.event)
	if lasterr != 0 {
		return 0, syscall.Errno(lasterr)
	}
	xerr = C.waveOutUnprepareHeader(w.wo, &hdr, C.UINT(unsafe.Sizeof(hdr)))
	if xerr != C.MMSYSERR_NOERROR {
		return 0, waveOutError(xerr)
	}
	return len(p), nil
}
<commit_msg>Fixed a dumb bug. Thanks to http://stackoverflow.com/questions/25633019/why-does-waveoutopen-on-windows-vista-newer-say-the-default-output-device-can.<commit_after>// 30 august 2014

package pcmio

import (
	"syscall"
	"unsafe"
)

// #cgo CFLAGS: --std=c99
// #cgo LDFLAGS: -lkernel32 -lwinmm
// #include "winapi_windows.h"
import "C"

type writer struct {
	wo		C.HWAVEOUT
	event	C.HANDLE
}

func openDefaultWriter(format Format, rate uint) (*writer, error) {
	var fmt C.WAVEFORMATEX

	w := new(writer)
	lasterr := C.xCreateEvent(&w.event)
	if lasterr != 0 {
		return nil, syscall.Errno(lasterr)
	}

	fmt.wFormatTag = C.WAVE_FORMAT_PCM
	fmt.nChannels = 1
	fmt.nSamplesPerSec = C.DWORD(rate)
	switch format {
	case U8:
		fmt.wBitsPerSample = 8
	}
	fmt.nBlockAlign = (fmt.nChannels * fmt.wBitsPerSample) / 8
	fmt.nAvgBytesPerSec = fmt.nSamplesPerSec * C.DWORD(fmt.nBlockAlign)
	fmt.cbSize = 0
	err := C.waveOutOpen(&w.wo, C.WAVE_MAPPER, &fmt,
		C.DWORD_PTR(uintptr(unsafe.Pointer(w.event))), 0,
		C.CALLBACK_EVENT | C.WAVE_FORMAT_DIRECT)
	if err != C.MMSYSERR_NOERROR {
		return nil, waveOutError(err)
	}

	return w, nil
}

func (w *writer) Write(p []byte) (n int, err error) {
	var hdr C.WAVEHDR

	hdr.lpData = C.LPSTR(unsafe.Pointer(&p[0]))
	hdr.dwBufferLength = C.DWORD(len(p))
	hdr.dwFlags = 0
	hdr.dwLoops = 0
	lasterr := C.xResetEvent(w.event)
	if lasterr != 0 {
		return 0, syscall.Errno(lasterr)
	}
	xerr := C.waveOutPrepareHeader(w.wo, &hdr, C.UINT(unsafe.Sizeof(hdr)))
	if xerr != C.MMSYSERR_NOERROR {
		return 0, waveOutError(xerr)
	}
	xerr = C.waveOutWrite(w.wo, &hdr, C.UINT(unsafe.Sizeof(hdr)))
	if xerr != C.MMSYSERR_NOERROR {
		return 0, waveOutError(xerr)
	}
	lasterr = C.xWaitForSingleObject(w.event)
	if lasterr != 0 {
		return 0, syscall.Errno(lasterr)
	}
	xerr = C.waveOutUnprepareHeader(w.wo, &hdr, C.UINT(unsafe.Sizeof(hdr)))
	if xerr != C.MMSYSERR_NOERROR {
		return 0, waveOutError(xerr)
	}
	return len(p), nil
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/workers/common/response"
)

type Request struct {
}

func Info(u *url.URL, h http.Header, _ interface{}, context *models.Context) (int, http.Header, interface{}, error) {
	token := h.Get("Authorization")

	//user:= Get User with token

	// return response.New(userinfo)
	return response.NewOK(nil)
}
<commit_msg>socialapi: fix build<commit_after>package api

import (
	"net/http"
	"net/url"
	"socialapi/models"
	"socialapi/workers/common/response"
)

type Request struct {
}

func Info(u *url.URL, h http.Header, _ interface{}, context *models.Context) (int, http.Header, interface{}, error) {
	// token := h.Get("Authorization")

	//user:= Get User with token

	// return response.New(userinfo)
	return response.NewOK(nil)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ld

import (
	"cmd/internal/sys"
	"cmd/link/internal/loader"
	"encoding/binary"
	"errors"
	"log"
	"os"
)

// If fallocate is not supported on this platform, return this error. The error
// is ignored where needed, and OutBuf writes to heap memory.
var errNoFallocate = errors.New("operation not supported")

const outbufMode = 0775

// OutBuf is a buffered file writer.
//
// It is simlar to the Writer in cmd/internal/bio with a few small differences.
//
// First, it tracks the output architecture and uses it to provide
// endian helpers.
//
// Second, it provides a very cheap offset counter that doesn't require
// any system calls to read the value.
//
// Third, it also mmaps the output file (if available). The intended usage is:
// - Mmap the output file
// - Write the content
// - possibly apply any edits in the output buffer
// - possibly write more content to the file. These writes take place in a heap
//   backed buffer that will get synced to disk.
// - Munmap the output file
//
// And finally, it provides a mechanism by which you can multithread the
// writing of output files. This mechanism is accomplished by copying a OutBuf,
// and using it in the thread/goroutine.
//
// Parallel OutBuf is intended to be used like:
//
//  func write(out *OutBuf) {
//    var wg sync.WaitGroup
//    for i := 0; i < 10; i++ {
//      wg.Add(1)
//      view, err := out.View(start[i])
//      if err != nil {
//         // handle output
//         continue
//      }
//      go func(out *OutBuf, i int) {
//        // do output
//        wg.Done()
//      }(view, i)
//    }
//    wg.Wait()
//  }
type OutBuf struct {
	arch *sys.Arch
	off  int64

	buf  []byte // backing store of mmap'd output file
	heap []byte // backing store for non-mmapped data

	name   string
	f      *os.File
	encbuf [8]byte // temp buffer used by WriteN methods
	isView bool    // true if created from View()
}

func (out *OutBuf) Open(name string) error {
	if out.f != nil {
		return errors.New("cannot open more than one file")
	}
	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, outbufMode)
	if err != nil {
		return err
	}
	out.off = 0
	out.name = name
	out.f = f
	return nil
}

func NewOutBuf(arch *sys.Arch) *OutBuf {
	return &OutBuf{
		arch: arch,
	}
}

var viewError = errors.New("output not mmapped")

func (out *OutBuf) View(start uint64) (*OutBuf, error) {
	return &OutBuf{
		arch:   out.arch,
		name:   out.name,
		buf:    out.buf,
		heap:   out.heap,
		off:    int64(start),
		isView: true,
	}, nil
}

var viewCloseError = errors.New("cannot Close OutBuf from View")

func (out *OutBuf) Close() error {
	if out.isView {
		return viewCloseError
	}
	if out.isMmapped() {
		out.copyHeap()
		out.munmap()
	}
	if out.f == nil {
		return nil
	}
	if len(out.heap) != 0 {
		if _, err := out.f.Write(out.heap); err != nil {
			return err
		}
	}
	if err := out.f.Close(); err != nil {
		return err
	}
	out.f = nil
	return nil
}

// isMmapped returns true if the OutBuf is mmaped.
func (out *OutBuf) isMmapped() bool {
	return len(out.buf) != 0
}

// copyHeap copies the heap to the mmapped section of memory, returning true if
// a copy takes place.
func (out *OutBuf) copyHeap() bool {
	if !out.isMmapped() { // only valuable for mmapped OutBufs.
		return false
	}
	if out.isView {
		panic("can't copyHeap a view")
	}

	bufLen := len(out.buf)
	heapLen := len(out.heap)
	total := uint64(bufLen + heapLen)
	if heapLen != 0 {
		if err := out.Mmap(total); err != nil { // Mmap will copy out.heap over to out.buf
			panic(err)
		}
	}
	return true
}

// maxOutBufHeapLen limits the growth of the heap area.
const maxOutBufHeapLen = 10 << 20

// writeLoc determines the write location if a buffer is mmaped.
// We maintain two write buffers, an mmapped section, and a heap section for
// writing. When the mmapped section is full, we switch over the heap memory
// for writing.
func (out *OutBuf) writeLoc(lenToWrite int64) (int64, []byte) {
	// See if we have enough space in the mmaped area.
	bufLen := int64(len(out.buf))
	if out.off+lenToWrite <= bufLen {
		return out.off, out.buf
	}

	// Not enough space in the mmaped area, write to heap area instead.
	heapPos := out.off - bufLen
	heapLen := int64(len(out.heap))
	lenNeeded := heapPos + lenToWrite
	if lenNeeded > heapLen { // do we need to grow the heap storage?
		// The heap variables aren't protected by a mutex. For now, just bomb if you
		// try to use OutBuf in parallel. (Note this probably could be fixed.)
		if out.isView {
			panic("cannot write to heap in parallel")
		}
		// See if our heap would grow to be too large, and if so, copy it to the end
		// of the mmapped area.
		if heapLen > maxOutBufHeapLen && out.copyHeap() {
			heapPos, heapLen, lenNeeded = 0, 0, lenToWrite
		}
		out.heap = append(out.heap, make([]byte, lenNeeded-heapLen)...)
	}
	return heapPos, out.heap
}

func (out *OutBuf) SeekSet(p int64) {
	out.off = p
}

func (out *OutBuf) Offset() int64 {
	return out.off
}

// Write writes the contents of v to the buffer.
func (out *OutBuf) Write(v []byte) (int, error) {
	n := len(v)
	pos, buf := out.writeLoc(int64(n))
	copy(buf[pos:], v)
	out.off += int64(n)
	return n, nil
}

func (out *OutBuf) Write8(v uint8) {
	pos, buf := out.writeLoc(1)
	buf[pos] = v
	out.off++
}

// WriteByte is an alias for Write8 to fulfill the io.ByteWriter interface.
func (out *OutBuf) WriteByte(v byte) error {
	out.Write8(v)
	return nil
}

func (out *OutBuf) Write16(v uint16) {
	out.arch.ByteOrder.PutUint16(out.encbuf[:], v)
	out.Write(out.encbuf[:2])
}

func (out *OutBuf) Write32(v uint32) {
	out.arch.ByteOrder.PutUint32(out.encbuf[:], v)
	out.Write(out.encbuf[:4])
}

func (out *OutBuf) Write32b(v uint32) {
	binary.BigEndian.PutUint32(out.encbuf[:], v)
	out.Write(out.encbuf[:4])
}

func (out *OutBuf) Write64(v uint64) {
	out.arch.ByteOrder.PutUint64(out.encbuf[:], v)
	out.Write(out.encbuf[:8])
}

func (out *OutBuf) Write64b(v uint64) {
	binary.BigEndian.PutUint64(out.encbuf[:], v)
	out.Write(out.encbuf[:8])
}

func (out *OutBuf) WriteString(s string) {
	pos, buf := out.writeLoc(int64(len(s)))
	n := copy(buf[pos:], s)
	if n != len(s) {
		log.Fatalf("WriteString truncated. buffer size: %d, offset: %d, len(s)=%d", len(out.buf), out.off, len(s))
	}
	out.off += int64(n)
}

// WriteStringN writes the first n bytes of s.
// If n is larger than len(s) then it is padded with zero bytes.
func (out *OutBuf) WriteStringN(s string, n int) {
	out.WriteStringPad(s, n, zeros[:])
}

// WriteStringPad writes the first n bytes of s.
// If n is larger than len(s) then it is padded with the bytes in pad (repeated as needed).
func (out *OutBuf) WriteStringPad(s string, n int, pad []byte) {
	if len(s) >= n {
		out.WriteString(s[:n])
	} else {
		out.WriteString(s)
		n -= len(s)
		for n > len(pad) {
			out.Write(pad)
			n -= len(pad)

		}
		out.Write(pad[:n])
	}
}

// WriteSym writes the content of a Symbol, and returns the output buffer
// that we just wrote, so we can apply further edit to the symbol content.
// For generator symbols, it also sets the symbol's Data to the output
// buffer.
func (out *OutBuf) WriteSym(ldr *loader.Loader, s loader.Sym) []byte {
	if !ldr.IsGeneratedSym(s) {
		P := ldr.Data(s)
		n := int64(len(P))
		pos, buf := out.writeLoc(n)
		copy(buf[pos:], P)
		out.off += n
		ldr.FreeData(s)
		return buf[pos : pos+n]
	} else {
		n := ldr.SymSize(s)
		pos, buf := out.writeLoc(n)
		out.off += n
		ldr.MakeSymbolUpdater(s).SetData(buf[pos : pos+n])
		return buf[pos : pos+n]
	}
}
<commit_msg>cmd/link: recompute heapPos after copyHeap<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ld

import (
	"cmd/internal/sys"
	"cmd/link/internal/loader"
	"encoding/binary"
	"errors"
	"log"
	"os"
)

// If fallocate is not supported on this platform, return this error. The error
// is ignored where needed, and OutBuf writes to heap memory.
var errNoFallocate = errors.New("operation not supported")

const outbufMode = 0775

// OutBuf is a buffered file writer.
//
// It is simlar to the Writer in cmd/internal/bio with a few small differences.
//
// First, it tracks the output architecture and uses it to provide
// endian helpers.
//
// Second, it provides a very cheap offset counter that doesn't require
// any system calls to read the value.
//
// Third, it also mmaps the output file (if available). The intended usage is:
// - Mmap the output file
// - Write the content
// - possibly apply any edits in the output buffer
// - possibly write more content to the file. These writes take place in a heap
//   backed buffer that will get synced to disk.
// - Munmap the output file
//
// And finally, it provides a mechanism by which you can multithread the
// writing of output files. This mechanism is accomplished by copying a OutBuf,
// and using it in the thread/goroutine.
//
// Parallel OutBuf is intended to be used like:
//
//  func write(out *OutBuf) {
//    var wg sync.WaitGroup
//    for i := 0; i < 10; i++ {
//      wg.Add(1)
//      view, err := out.View(start[i])
//      if err != nil {
//         // handle output
//         continue
//      }
//      go func(out *OutBuf, i int) {
//        // do output
//        wg.Done()
//      }(view, i)
//    }
//    wg.Wait()
//  }
type OutBuf struct {
	arch *sys.Arch
	off  int64

	buf  []byte // backing store of mmap'd output file
	heap []byte // backing store for non-mmapped data

	name   string
	f      *os.File
	encbuf [8]byte // temp buffer used by WriteN methods
	isView bool    // true if created from View()
}

func (out *OutBuf) Open(name string) error {
	if out.f != nil {
		return errors.New("cannot open more than one file")
	}
	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, outbufMode)
	if err != nil {
		return err
	}
	out.off = 0
	out.name = name
	out.f = f
	return nil
}

func NewOutBuf(arch *sys.Arch) *OutBuf {
	return &OutBuf{
		arch: arch,
	}
}

var viewError = errors.New("output not mmapped")

func (out *OutBuf) View(start uint64) (*OutBuf, error) {
	return &OutBuf{
		arch:   out.arch,
		name:   out.name,
		buf:    out.buf,
		heap:   out.heap,
		off:    int64(start),
		isView: true,
	}, nil
}

var viewCloseError = errors.New("cannot Close OutBuf from View")

func (out *OutBuf) Close() error {
	if out.isView {
		return viewCloseError
	}
	if out.isMmapped() {
		out.copyHeap()
		out.munmap()
	}
	if out.f == nil {
		return nil
	}
	if len(out.heap) != 0 {
		if _, err := out.f.Write(out.heap); err != nil {
			return err
		}
	}
	if err := out.f.Close(); err != nil {
		return err
	}
	out.f = nil
	return nil
}

// isMmapped returns true if the OutBuf is mmaped.
func (out *OutBuf) isMmapped() bool {
	return len(out.buf) != 0
}

// copyHeap copies the heap to the mmapped section of memory, returning true if
// a copy takes place.
func (out *OutBuf) copyHeap() bool {
	if !out.isMmapped() { // only valuable for mmapped OutBufs.
		return false
	}
	if out.isView {
		panic("can't copyHeap a view")
	}

	bufLen := len(out.buf)
	heapLen := len(out.heap)
	total := uint64(bufLen + heapLen)
	if heapLen != 0 {
		if err := out.Mmap(total); err != nil { // Mmap will copy out.heap over to out.buf
			panic(err)
		}
	}
	return true
}

// maxOutBufHeapLen limits the growth of the heap area.
const maxOutBufHeapLen = 10 << 20

// writeLoc determines the write location if a buffer is mmaped.
// We maintain two write buffers, an mmapped section, and a heap section for
// writing. When the mmapped section is full, we switch over the heap memory
// for writing.
func (out *OutBuf) writeLoc(lenToWrite int64) (int64, []byte) {
	// See if we have enough space in the mmaped area.
	bufLen := int64(len(out.buf))
	if out.off+lenToWrite <= bufLen {
		return out.off, out.buf
	}

	// Not enough space in the mmaped area, write to heap area instead.
	heapPos := out.off - bufLen
	heapLen := int64(len(out.heap))
	lenNeeded := heapPos + lenToWrite
	if lenNeeded > heapLen { // do we need to grow the heap storage?
		// The heap variables aren't protected by a mutex. For now, just bomb if you
		// try to use OutBuf in parallel. (Note this probably could be fixed.)
		if out.isView {
			panic("cannot write to heap in parallel")
		}
		// See if our heap would grow to be too large, and if so, copy it to the end
		// of the mmapped area.
		if heapLen > maxOutBufHeapLen && out.copyHeap() {
			heapPos -= heapLen
			lenNeeded = heapPos + lenToWrite
			heapLen = 0
		}
		out.heap = append(out.heap, make([]byte, lenNeeded-heapLen)...)
	}
	return heapPos, out.heap
}

func (out *OutBuf) SeekSet(p int64) {
	out.off = p
}

func (out *OutBuf) Offset() int64 {
	return out.off
}

// Write writes the contents of v to the buffer.
func (out *OutBuf) Write(v []byte) (int, error) {
	n := len(v)
	pos, buf := out.writeLoc(int64(n))
	copy(buf[pos:], v)
	out.off += int64(n)
	return n, nil
}

func (out *OutBuf) Write8(v uint8) {
	pos, buf := out.writeLoc(1)
	buf[pos] = v
	out.off++
}

// WriteByte is an alias for Write8 to fulfill the io.ByteWriter interface.
func (out *OutBuf) WriteByte(v byte) error {
	out.Write8(v)
	return nil
}

func (out *OutBuf) Write16(v uint16) {
	out.arch.ByteOrder.PutUint16(out.encbuf[:], v)
	out.Write(out.encbuf[:2])
}

func (out *OutBuf) Write32(v uint32) {
	out.arch.ByteOrder.PutUint32(out.encbuf[:], v)
	out.Write(out.encbuf[:4])
}

func (out *OutBuf) Write32b(v uint32) {
	binary.BigEndian.PutUint32(out.encbuf[:], v)
	out.Write(out.encbuf[:4])
}

func (out *OutBuf) Write64(v uint64) {
	out.arch.ByteOrder.PutUint64(out.encbuf[:], v)
	out.Write(out.encbuf[:8])
}

func (out *OutBuf) Write64b(v uint64) {
	binary.BigEndian.PutUint64(out.encbuf[:], v)
	out.Write(out.encbuf[:8])
}

func (out *OutBuf) WriteString(s string) {
	pos, buf := out.writeLoc(int64(len(s)))
	n := copy(buf[pos:], s)
	if n != len(s) {
		log.Fatalf("WriteString truncated. buffer size: %d, offset: %d, len(s)=%d", len(out.buf), out.off, len(s))
	}
	out.off += int64(n)
}

// WriteStringN writes the first n bytes of s.
// If n is larger than len(s) then it is padded with zero bytes.
func (out *OutBuf) WriteStringN(s string, n int) {
	out.WriteStringPad(s, n, zeros[:])
}

// WriteStringPad writes the first n bytes of s.
// If n is larger than len(s) then it is padded with the bytes in pad (repeated as needed).
func (out *OutBuf) WriteStringPad(s string, n int, pad []byte) {
	if len(s) >= n {
		out.WriteString(s[:n])
	} else {
		out.WriteString(s)
		n -= len(s)
		for n > len(pad) {
			out.Write(pad)
			n -= len(pad)

		}
		out.Write(pad[:n])
	}
}

// WriteSym writes the content of a Symbol, and returns the output buffer
// that we just wrote, so we can apply further edit to the symbol content.
// For generator symbols, it also sets the symbol's Data to the output
// buffer.
func (out *OutBuf) WriteSym(ldr *loader.Loader, s loader.Sym) []byte {
	if !ldr.IsGeneratedSym(s) {
		P := ldr.Data(s)
		n := int64(len(P))
		pos, buf := out.writeLoc(n)
		copy(buf[pos:], P)
		out.off += n
		ldr.FreeData(s)
		return buf[pos : pos+n]
	} else {
		n := ldr.SymSize(s)
		pos, buf := out.writeLoc(n)
		out.off += n
		ldr.MakeSymbolUpdater(s).SetData(buf[pos : pos+n])
		return buf[pos : pos+n]
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kola

import "github.com/coreos/mantle/kola/tests/coretest"

func init() {
	Register(&Test{
		Name:        "coretestsLocal",
		Run:         coretest.LocalTests,
		ClusterSize: 1,
		NativeFuncs: map[string]func() error{
			"CloudConfig":      coretest.TestCloudinitCloudConfig,
			"Script":           coretest.TestCloudinitScript,
			"PortSSH":          coretest.TestPortSsh,
			"DbusPerms":        coretest.TestDbusPerms,
			"Symlink":          coretest.TestSymlinkResolvConf,
			"UpdateEngineKeys": coretest.TestInstalledUpdateEngineRsaKeys,
			"ServicesActive":   coretest.TestServicesActive,
			"ReadOnly":         coretest.TestReadOnlyFs,
			"RandomUUID":       coretest.TestFsRandomUUID,
			"Useradd":          coretest.TestUseradd,
		},
	})
	Register(&Test{
		Name:        "coretestsCluster",
		Run:         coretest.ClusterTests,
		ClusterSize: 3,
		NativeFuncs: map[string]func() error{
			"EtcdUpdateValue":    coretest.TestEtcdUpdateValue,
			"FleetctlRunService": coretest.TestFleetctlRunService,
		},
		CloudConfig: `#cloud-config

coreos:
  etcd2:
    name: $name
    discovery: $discovery
    advertise-client-urls: http://$private_ipv4:2379
    initial-advertise-peer-urls: http://$private_ipv4:2380
    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
    listen-peer-urls: http://$private_ipv4:2380,http://$private_ipv4:7001
  units:
    - name: etcd2.service
      command: start
    - name: fleet.service
      command: start`,
	})

	// tests requiring network connection to internet
	Register(&Test{
		Name:        "coretestsInternetLocal",
		Run:         coretest.InternetTests,
		ClusterSize: 1,
		Platforms:   []string{"gce", "aws"},
		NativeFuncs: map[string]func() error{
			"UpdateEngine": coretest.TestUpdateEngine,
			"DockerPing":   coretest.TestDockerPing,
			"DockerEcho":   coretest.TestDockerEcho,
			"NTPDate":      coretest.TestNTPDate,
		},
	})
}
<commit_msg>kola: bump fleet request timeout to 2.5s<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kola

import "github.com/coreos/mantle/kola/tests/coretest"

func init() {
	Register(&Test{
		Name:        "coretestsLocal",
		Run:         coretest.LocalTests,
		ClusterSize: 1,
		NativeFuncs: map[string]func() error{
			"CloudConfig":      coretest.TestCloudinitCloudConfig,
			"Script":           coretest.TestCloudinitScript,
			"PortSSH":          coretest.TestPortSsh,
			"DbusPerms":        coretest.TestDbusPerms,
			"Symlink":          coretest.TestSymlinkResolvConf,
			"UpdateEngineKeys": coretest.TestInstalledUpdateEngineRsaKeys,
			"ServicesActive":   coretest.TestServicesActive,
			"ReadOnly":         coretest.TestReadOnlyFs,
			"RandomUUID":       coretest.TestFsRandomUUID,
			"Useradd":          coretest.TestUseradd,
		},
	})
	Register(&Test{
		Name:        "coretestsCluster",
		Run:         coretest.ClusterTests,
		ClusterSize: 3,
		NativeFuncs: map[string]func() error{
			"EtcdUpdateValue":    coretest.TestEtcdUpdateValue,
			"FleetctlRunService": coretest.TestFleetctlRunService,
		},
		CloudConfig: `#cloud-config

coreos:
  etcd2:
    name: $name
    discovery: $discovery
    advertise-client-urls: http://$private_ipv4:2379
    initial-advertise-peer-urls: http://$private_ipv4:2380
    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
    listen-peer-urls: http://$private_ipv4:2380,http://$private_ipv4:7001
  fleet:
    etcd-request-timeout: 2.5
  units:
    - name: etcd2.service
      command: start
    - name: fleet.service
      command: start`,
	})

	// tests requiring network connection to internet
	Register(&Test{
		Name:        "coretestsInternetLocal",
		Run:         coretest.InternetTests,
		ClusterSize: 1,
		Platforms:   []string{"gce", "aws"},
		NativeFuncs: map[string]func() error{
			"UpdateEngine": coretest.TestUpdateEngine,
			"DockerPing":   coretest.TestDockerPing,
			"DockerEcho":   coretest.TestDockerEcho,
			"NTPDate":      coretest.TestNTPDate,
		},
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package kv

import (
	"fmt"
	"math/rand"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
)

// setTestRetryOptions sets client retry options for speedier testing.
func setTestRetryOptions() {
	client.TxnRetryOptions = util.RetryOptions{
		Backoff:    1 * time.Millisecond,
		MaxBackoff: 10 * time.Millisecond,
		Constant:   2,
	}
}

// startTestWriter creates a writer which initiates a sequence of
// transactions, each which writes up to 10 times to random keys with
// random values. If not nil, txnChannel is written to every time a
// new transaction starts.
func startTestWriter(db *client.KV, i int64, valBytes int32, wg *sync.WaitGroup, retries *int32,
	txnChannel chan struct{}, done <-chan struct{}, t *testing.T) {
	src := rand.New(rand.NewSource(i))
	for j := 0; ; j++ {
		txnOpts := &client.TransactionOptions{Name: fmt.Sprintf("concurrent test %d:%d", i, j)}
		select {
		case <-done:
			if wg != nil {
				wg.Done()
			}
			return
		default:
			first := true
			err := db.RunTransaction(txnOpts, func(txn *client.KV) error {
				if first && txnChannel != nil {
					txnChannel <- struct{}{}
				} else if !first && retries != nil {
					atomic.AddInt32(retries, 1)
				}
				first = false
				for j := 0; j <= int(src.Int31n(10)); j++ {
					key := util.RandBytes(src, 10)
					val := util.RandBytes(src, int(src.Int31n(valBytes)))
					req := &proto.PutRequest{RequestHeader: proto.RequestHeader{Key: key}, Value: proto.Value{Bytes: val}}
					resp := &proto.PutResponse{}
					if err := txn.Call(proto.Put, req, resp); err != nil {
						log.Infof("experienced an error in routine %d: %s", i, err)
						return err
					}
				}
				return nil
			})
			if err != nil {
				t.Error(err)
			} else {
				time.Sleep(1 * time.Millisecond)
			}
		}
	}
}

// TestRangeSplitsWithConcurrentTxns does 5 consecutive splits while
// 10 concurrent goroutines are each running successive transactions
// composed of a random mix of puts.
func TestRangeSplitsWithConcurrentTxns(t *testing.T) {
	db, _, _, _, _, transport, err := createTestDB()
	if err != nil {
		t.Fatal(err)
	}
	defer transport.Close()
	defer db.Close()

	// This channel shuts the whole apparatus down.
	done := make(chan struct{})
	txnChannel := make(chan struct{}, 1000)

	// Set five split keys, about evenly spaced along the range of random keys.
	splitKeys := []proto.Key{proto.Key("G"), proto.Key("R"), proto.Key("a"), proto.Key("l"), proto.Key("s")}

	// Start up the concurrent goroutines which run transactions.
	const concurrency = 10
	var retries int32
	var wg sync.WaitGroup
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go startTestWriter(db, int64(i), 1<<7, &wg, &retries, txnChannel, done, t)
	}

	// Execute the consecutive splits.
	for _, splitKey := range splitKeys {
		// Allow txns to start before initiating split.
		for i := 0; i < concurrency; i++ {
			<-txnChannel
		}
		log.Infof("starting split at key %q...", splitKey)
		req := &proto.AdminSplitRequest{RequestHeader: proto.RequestHeader{Key: splitKey}, SplitKey: splitKey}
		resp := &proto.AdminSplitResponse{}
		if err := db.Call(proto.AdminSplit, req, resp); err != nil {
			t.Fatal(err)
		}
		log.Infof("split at key %q complete", splitKey)
	}

	close(done)
	wg.Wait()

	if retries != 0 {
		t.Errorf("expected no retries splitting a range with concurrent writes, "+
			"as range splits do not cause conflicts; got %d", retries)
	}
}

// TestRangeSplitsWithWritePressure sets the zone config max bytes for
// a range to 256K and writes data until there are five ranges.
func TestRangeSplitsWithWritePressure(t *testing.T) {
	db, eng, _, _, _, transport, err := createTestDB()
	if err != nil {
		t.Fatal(err)
	}
	defer transport.Close()
	defer db.Close()
	setTestRetryOptions()

	// Rewrite a zone config with low max bytes.
	zoneConfig := &proto.ZoneConfig{
		ReplicaAttrs: []proto.Attributes{
			{},
			{},
			{},
		},
		RangeMinBytes: 1 << 8,
		RangeMaxBytes: 1 << 18,
	}
	if err := db.PutProto(engine.MakeKey(engine.KeyConfigZonePrefix, engine.KeyMin), zoneConfig); err != nil {
		t.Fatal(err)
	}

	// Start test writer write about a 32K/key so there aren't too many writes necessary to split 64K range.
	done := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(1)
	go startTestWriter(db, int64(0), 1<<15, &wg, nil, nil, done, t)

	// Check that we split 5 times in allotted time.
	if err := util.IsTrueWithin(func() bool {
		// Scan the txn records.
		resp := &proto.ScanResponse{}
		if err := db.Call(proto.Scan, proto.ScanArgs(engine.KeyMeta2Prefix, engine.KeyMetaMax, 0), resp); err != nil {
			t.Fatalf("failed to scan meta2 keys: %s", err)
		}
		return len(resp.Rows) >= 5
	}, 3*time.Second); err != nil {
		t.Errorf("failed to split 5 times: %s", err)
	}
	close(done)
	wg.Wait()

	// This write pressure test often causes splits while resolve
	// intents are in flight, causing them to fail with range key
	// mismatch errors. However, LocalSender should retry in these
	// cases. Check here via MVCC scan that there are no dangling write
	// intents. We do this using an IsTrueWithin construct to account
	// for timing of finishing the test writer and a possibly-ongoing
	// asynchronous split.
	if err := util.IsTrueWithin(func() bool {
		if _, err := engine.MVCCScan(eng, engine.KeyLocalMax, engine.KeyMax, 0, proto.MaxTimestamp, nil); err != nil {
			log.Infof("mvcc scan should be clean: %s", err)
			return false
		}
		return true
	}, 500*time.Millisecond); err != nil {
		t.Error("failed to verify no dangling intents within 500ms")
	}
}
<commit_msg>Allow extra time for TestRangeSplitsWithWritePressure<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package kv

import (
	"fmt"
	"math/rand"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
)

// setTestRetryOptions sets client retry options for speedier testing.
func setTestRetryOptions() {
	client.TxnRetryOptions = util.RetryOptions{
		Backoff:    1 * time.Millisecond,
		MaxBackoff: 10 * time.Millisecond,
		Constant:   2,
	}
}

// startTestWriter creates a writer which initiates a sequence of
// transactions, each which writes up to 10 times to random keys with
// random values. If not nil, txnChannel is written to every time a
// new transaction starts.
func startTestWriter(db *client.KV, i int64, valBytes int32, wg *sync.WaitGroup, retries *int32,
	txnChannel chan struct{}, done <-chan struct{}, t *testing.T) {
	src := rand.New(rand.NewSource(i))
	for j := 0; ; j++ {
		txnOpts := &client.TransactionOptions{Name: fmt.Sprintf("concurrent test %d:%d", i, j)}
		select {
		case <-done:
			if wg != nil {
				wg.Done()
			}
			return
		default:
			first := true
			err := db.RunTransaction(txnOpts, func(txn *client.KV) error {
				if first && txnChannel != nil {
					txnChannel <- struct{}{}
				} else if !first && retries != nil {
					atomic.AddInt32(retries, 1)
				}
				first = false
				for j := 0; j <= int(src.Int31n(10)); j++ {
					key := util.RandBytes(src, 10)
					val := util.RandBytes(src, int(src.Int31n(valBytes)))
					req := &proto.PutRequest{RequestHeader: proto.RequestHeader{Key: key}, Value: proto.Value{Bytes: val}}
					resp := &proto.PutResponse{}
					if err := txn.Call(proto.Put, req, resp); err != nil {
						log.Infof("experienced an error in routine %d: %s", i, err)
						return err
					}
				}
				return nil
			})
			if err != nil {
				t.Error(err)
			} else {
				time.Sleep(1 * time.Millisecond)
			}
		}
	}
}

// TestRangeSplitsWithConcurrentTxns does 5 consecutive splits while
// 10 concurrent goroutines are each running successive transactions
// composed of a random mix of puts.
func TestRangeSplitsWithConcurrentTxns(t *testing.T) {
	db, _, _, _, _, transport, err := createTestDB()
	if err != nil {
		t.Fatal(err)
	}
	defer transport.Close()
	defer db.Close()

	// This channel shuts the whole apparatus down.
	done := make(chan struct{})
	txnChannel := make(chan struct{}, 1000)

	// Set five split keys, about evenly spaced along the range of random keys.
	splitKeys := []proto.Key{proto.Key("G"), proto.Key("R"), proto.Key("a"), proto.Key("l"), proto.Key("s")}

	// Start up the concurrent goroutines which run transactions.
	const concurrency = 10
	var retries int32
	var wg sync.WaitGroup
	wg.Add(concurrency)
	for i := 0; i < concurrency; i++ {
		go startTestWriter(db, int64(i), 1<<7, &wg, &retries, txnChannel, done, t)
	}

	// Execute the consecutive splits.
	for _, splitKey := range splitKeys {
		// Allow txns to start before initiating split.
		for i := 0; i < concurrency; i++ {
			<-txnChannel
		}
		log.Infof("starting split at key %q...", splitKey)
		req := &proto.AdminSplitRequest{RequestHeader: proto.RequestHeader{Key: splitKey}, SplitKey: splitKey}
		resp := &proto.AdminSplitResponse{}
		if err := db.Call(proto.AdminSplit, req, resp); err != nil {
			t.Fatal(err)
		}
		log.Infof("split at key %q complete", splitKey)
	}

	close(done)
	wg.Wait()

	if retries != 0 {
		t.Errorf("expected no retries splitting a range with concurrent writes, "+
			"as range splits do not cause conflicts; got %d", retries)
	}
}

// TestRangeSplitsWithWritePressure sets the zone config max bytes for
// a range to 256K and writes data until there are five ranges.
func TestRangeSplitsWithWritePressure(t *testing.T) {
	db, eng, _, _, _, transport, err := createTestDB()
	if err != nil {
		t.Fatal(err)
	}
	defer transport.Close()
	defer db.Close()
	setTestRetryOptions()

	// Rewrite a zone config with low max bytes.
	zoneConfig := &proto.ZoneConfig{
		ReplicaAttrs: []proto.Attributes{
			{},
			{},
			{},
		},
		RangeMinBytes: 1 << 8,
		RangeMaxBytes: 1 << 18,
	}
	if err := db.PutProto(engine.MakeKey(engine.KeyConfigZonePrefix, engine.KeyMin), zoneConfig); err != nil {
		t.Fatal(err)
	}

	// Start test writer write about a 32K/key so there aren't too many writes necessary to split 64K range.
	done := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(1)
	go startTestWriter(db, int64(0), 1<<15, &wg, nil, nil, done, t)

	// Check that we split 5 times in allotted time.
	if err := util.IsTrueWithin(func() bool {
		// Scan the txn records.
		resp := &proto.ScanResponse{}
		if err := db.Call(proto.Scan, proto.ScanArgs(engine.KeyMeta2Prefix, engine.KeyMetaMax, 0), resp); err != nil {
			t.Fatalf("failed to scan meta2 keys: %s", err)
		}
		return len(resp.Rows) >= 5
	}, 6*time.Second); err != nil {
		t.Errorf("failed to split 5 times: %s", err)
	}
	close(done)
	wg.Wait()

	// This write pressure test often causes splits while resolve
	// intents are in flight, causing them to fail with range key
	// mismatch errors. However, LocalSender should retry in these
	// cases. Check here via MVCC scan that there are no dangling write
	// intents. We do this using an IsTrueWithin construct to account
	// for timing of finishing the test writer and a possibly-ongoing
	// asynchronous split.
	if err := util.IsTrueWithin(func() bool {
		if _, err := engine.MVCCScan(eng, engine.KeyLocalMax, engine.KeyMax, 0, proto.MaxTimestamp, nil); err != nil {
			log.Infof("mvcc scan should be clean: %s", err)
			return false
		}
		return true
	}, 500*time.Millisecond); err != nil {
		t.Error("failed to verify no dangling intents within 500ms")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gophergala2016/goad/sqsadaptor"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

func main() {
	address := os.Args[1]
	concurrencycount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	maxRequestCount, err := strconv.Atoi(os.Args[3])
	sqsurl := os.Args[4]
	awsregion := os.Args[5]

	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	client := &http.Client{}
	clientTimeout, _ := time.ParseDuration("1s")
	client.Timeout = clientTimeout
	fmt.Printf("Will spawn %d workers making %d requests to %s\n", concurrencycount, maxRequestCount, address)
	runLoadTest(client, sqsurl, address, maxRequestCount, concurrencycount, awsregion)
}

type Job struct{}

type RequestResult struct {
	Time             int64  `json:"time"`
	Host             string `json:"host"`
	Type             string `json:"type"`
	Status           int    `json:"status"`
	ElapsedFirstByte int64  `json:"elapsed-first-byte"`
	ElapsedLastByte  int64  `json:"elapsed-last-byte"`
	Elapsed          int64  `json:"elapsed"`
	Bytes            int    `json:"bytes"`
	State            string `json:"state"`
}

func runLoadTest(client *http.Client, sqsurl string, url string, totalRequests int, concurrencycount int, awsregion string) {
	sqsAdaptor := sqsadaptor.NewSQSAdaptor(sqsurl)
	jobs := make(chan Job, totalRequests)
	ch := make(chan RequestResult, totalRequests)
	var wg sync.WaitGroup
	var requestsSoFar int
	for i := 0; i < totalRequests; i++ {
		jobs <- Job{}
	}
	close(jobs)
	for i := 0; i < concurrencycount; i++ {
		wg.Add(1)
		go fetch(client, url, totalRequests, jobs, ch, &wg, awsregion)
	}
	fmt.Println("Waiting for results…")

	for requestsSoFar < totalRequests {
		var agg [100]RequestResult
		aggRequestCount := len(agg)
		i := 0

		var timeToFirstTotal int64
		var requestTimeTotal int64
		totBytesRead := 0
		statuses := make(map[string]int)
		var firstRequestTime int64
		var lastRequestTime int64
		var slowest int64
		var fastest int64
		for ; i < aggRequestCount && requestsSoFar < totalRequests; i++ {
			r := <-ch
			agg[i] = r
			requestsSoFar++
			if requestsSoFar%10 == 0 {
				fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(requestsSoFar)/float64(totalRequests))*100.0, requestsSoFar, totalRequests)
			}
			if firstRequestTime == 0 {
				firstRequestTime = r.Time
			}
			if r.ElapsedLastByte > slowest {
				slowest = r.ElapsedLastByte
			}
			if fastest == 0 {
				fastest = r.ElapsedLastByte
			} else {
				if r.ElapsedLastByte < fastest {
					fastest = r.ElapsedLastByte
				}
			}

			lastRequestTime = r.Time
			timeToFirstTotal += r.ElapsedFirstByte
			totBytesRead += r.Bytes
			statusStr := strconv.Itoa(r.Status)
			_, ok := statuses[statusStr]
			if !ok {
				statuses[statusStr] = 1
			} else {
				statuses[statusStr]++
			}
			requestTimeTotal += r.Elapsed
		}
		durationSeconds := lastRequestTime - firstRequestTime
		if i == 0 {
			continue
		}
		if durationSeconds == 0 { // well…
			durationSeconds = 1
		}
		aggData := sqsadaptor.AggData{
			i,
			0, // totalTimedOut
			timeToFirstTotal / int64(i),
			totBytesRead,
			statuses,
			requestTimeTotal / int64(i),
			i / int(durationSeconds),
			slowest,
			fastest,
			awsregion,
		}
		sqsAdaptor.SendResult(aggData)
	}
	fmt.Printf("\nWaiting for workers…")
	wg.Wait()
	fmt.Printf("\nYay🎈  - %d requests completed\n", requestsSoFar)

}

func fetch(client *http.Client, address string, requestcount int, jobs <-chan Job, ch chan RequestResult, wg *sync.WaitGroup, awsregion string) {
	defer wg.Done()
	fmt.Printf("Fetching %s\n", address)
	for _ = range jobs {
		start := time.Now()
		req, err := http.NewRequest("GET", address, nil)
		req.Header.Add("User-Agent", "GOAD/0.1")
		response, err := client.Do(req)
		var status string
		var elapsedFirstByte time.Duration
		var elapsedLastByte time.Duration
		var elapsed time.Duration
		buf := []byte(" ")
		if err != nil {
			status = fmt.Sprintf("client.Do() failed: %s\n", err)
		} else {
			_, err = response.Body.Read(buf)
			if err != nil {
				status = fmt.Sprintf("reading first byte failed: %s\n", err)
			}
			elapsedFirstByte = time.Since(start)
			_, err = ioutil.ReadAll(response.Body)
			elapsedLastByte = time.Since(start)
			if err != nil {
				status = fmt.Sprintf("reading response body failed: %s\n", err)
			} else {
				status = "Success"
			}
			elapsed = time.Since(start)
		}
		result := RequestResult{
			start.Unix(),
			req.URL.Host,
			req.Method,
			response.StatusCode,
			elapsed.Nanoseconds(),
			elapsedFirstByte.Nanoseconds(),
			elapsedLastByte.Nanoseconds(),
			len(buf),
			status,
		}
		ch <- result
	}
}
<commit_msg>Rudimentary timeout detection<commit_after>package main

import (
	"fmt"
	"github.com/gophergala2016/goad/sqsadaptor"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

func main() {
	address := os.Args[1]
	concurrencycount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	maxRequestCount, err := strconv.Atoi(os.Args[3])
	sqsurl := os.Args[4]
	awsregion := os.Args[5]

	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	client := &http.Client{}
	clientTimeout, _ := time.ParseDuration("1s")
	client.Timeout = clientTimeout
	fmt.Printf("Will spawn %d workers making %d requests to %s\n", concurrencycount, maxRequestCount, address)
	runLoadTest(client, sqsurl, address, maxRequestCount, concurrencycount, awsregion)
}

type Job struct{}

type RequestResult struct {
	Time             int64  `json:"time"`
	Host             string `json:"host"`
	Type             string `json:"type"`
	Status           int    `json:"status"`
	ElapsedFirstByte int64  `json:"elapsed-first-byte"`
	ElapsedLastByte  int64  `json:"elapsed-last-byte"`
	Elapsed          int64  `json:"elapsed"`
	Bytes            int    `json:"bytes"`
	Timeout          bool   `json:"timeout"`
	State            string `json:"state"`
}

func runLoadTest(client *http.Client, sqsurl string, url string, totalRequests int, concurrencycount int, awsregion string) {
	sqsAdaptor := sqsadaptor.NewSQSAdaptor(sqsurl)
	jobs := make(chan Job, totalRequests)
	ch := make(chan RequestResult, totalRequests)
	var wg sync.WaitGroup
	var requestsSoFar int
	for i := 0; i < totalRequests; i++ {
		jobs <- Job{}
	}
	close(jobs)
	for i := 0; i < concurrencycount; i++ {
		wg.Add(1)
		go fetch(client, url, totalRequests, jobs, ch, &wg, awsregion)
	}
	fmt.Println("Waiting for results…")

	for requestsSoFar < totalRequests {
		var agg [100]RequestResult
		aggRequestCount := len(agg)
		i := 0

		var timeToFirstTotal int64
		var requestTimeTotal int64
		totBytesRead := 0
		statuses := make(map[string]int)
		var firstRequestTime int64
		var lastRequestTime int64
		var slowest int64
		var fastest int64
		var totalTimedOut int
		for ; i < aggRequestCount && requestsSoFar < totalRequests; i++ {
			r := <-ch
			agg[i] = r
			requestsSoFar++
			if requestsSoFar%10 == 0 {
				fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(requestsSoFar)/float64(totalRequests))*100.0, requestsSoFar, totalRequests)
			}
			if firstRequestTime == 0 {
				firstRequestTime = r.Time
			}
			if r.ElapsedLastByte > slowest {
				slowest = r.ElapsedLastByte
			}
			if fastest == 0 {
				fastest = r.ElapsedLastByte
			} else {
				if r.ElapsedLastByte < fastest {
					fastest = r.ElapsedLastByte
				}
			}

			lastRequestTime = r.Time
			timeToFirstTotal += r.ElapsedFirstByte
			totBytesRead += r.Bytes
			statusStr := strconv.Itoa(r.Status)
			_, ok := statuses[statusStr]
			if !ok {
				statuses[statusStr] = 1
			} else {
				statuses[statusStr]++
			}
			requestTimeTotal += r.Elapsed
			if r.Timeout {
				totalTimedOut++
			}
		}
		durationSeconds := lastRequestTime - firstRequestTime
		if i == 0 {
			continue
		}
		if durationSeconds == 0 { // well…
			durationSeconds = 1
		}
		aggData := sqsadaptor.AggData{
			i,
			totalTimedOut,
			timeToFirstTotal / int64(i),
			totBytesRead,
			statuses,
			requestTimeTotal / int64(i),
			i / int(durationSeconds),
			slowest,
			fastest,
			awsregion,
		}
		sqsAdaptor.SendResult(aggData)
	}
	fmt.Printf("\nWaiting for workers…")
	wg.Wait()
	fmt.Printf("\nYay🎈  - %d requests completed\n", requestsSoFar)

}

func fetch(client *http.Client, address string, requestcount int, jobs <-chan Job, ch chan RequestResult, wg *sync.WaitGroup, awsregion string) {
	defer wg.Done()
	fmt.Printf("Fetching %s\n", address)
	for _ = range jobs {
		start := time.Now()
		req, err := http.NewRequest("GET", address, nil)
		req.Header.Add("User-Agent", "GOAD/0.1")
		response, err := client.Do(req)
		var status string
		var elapsedFirstByte time.Duration
		var elapsedLastByte time.Duration
		var elapsed time.Duration
		var statusCode int
		buf := []byte(" ")
		timedOut := false
		if err != nil {
			status = fmt.Sprintf("ERROR: %s\n", err)
			//fmt.Println(status)
			switch err := err.(type) {
			case *url.Error:
				if err, ok := err.Err.(net.Error); ok && err.Timeout() {
					timedOut = true
				}
			case net.Error:
				if err.Timeout() {
					timedOut = true
				}
			}

			if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
				fmt.Println("Could be from a Transport.CancelRequest")
			}
		} else {
			statusCode = response.StatusCode
			_, err = response.Body.Read(buf)
			if err != nil {
				status = fmt.Sprintf("reading first byte failed: %s\n", err)
			}
			elapsedFirstByte = time.Since(start)
			_, err = ioutil.ReadAll(response.Body)
			elapsedLastByte = time.Since(start)
			if err != nil {
				status = fmt.Sprintf("reading response body failed: %s\n", err)
			} else {
				status = "Success"
			}
			elapsed = time.Since(start)
		}
		result := RequestResult{
			start.Unix(),
			req.URL.Host,
			req.Method,
			statusCode,
			elapsed.Nanoseconds(),
			elapsedFirstByte.Nanoseconds(),
			elapsedLastByte.Nanoseconds(),
			len(buf),
			timedOut,
			status,
		}
		ch <- result
	}
}
<|endoftext|>


<commit_before><commit_msg>[resultdb] Correct artifact and run count mismatch message.<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"math"
	"net"
	"strconv"
	"strings"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	"k8s.io/kubernetes/test/e2e/framework"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	"k8s.io/kubernetes/test/images/agnhost/net/nat"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
)

var kubeProxyE2eImage = imageutils.GetE2EImage(imageutils.Agnhost)

var _ = SIGDescribe("Network", func() {
	const (
		testDaemonHTTPPort     = 11301
		testDaemonTCPPort      = 11302
		deadlineTimeoutSeconds = 10
		postFinTimeoutSeconds  = 30
	)

	fr := framework.NewDefaultFramework("network")

	ginkgo.It("should set TCP CLOSE_WAIT timeout", func() {
		nodes, err := e2enode.GetBoundedReadySchedulableNodes(fr.ClientSet, 2)
		framework.ExpectNoError(err)
		if len(nodes.Items) < 2 {
			framework.Skipf(
				"Test requires >= 2 Ready nodes, but there are only %v nodes",
				len(nodes.Items))
		}

		ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP)

		type NodeInfo struct {
			node   *v1.Node
			name   string
			nodeIP string
		}

		clientNodeInfo := NodeInfo{
			node:   &nodes.Items[0],
			name:   nodes.Items[0].Name,
			nodeIP: ips[0],
		}

		serverNodeInfo := NodeInfo{
			node:   &nodes.Items[1],
			name:   nodes.Items[1].Name,
			nodeIP: ips[1],
		}

		zero := int64(0)

		// Create a pod to check the conntrack entries on the host node
		// It mounts the host /proc/net folder to be able to access
		// the nf_conntrack file with the host conntrack entries
		privileged := true

		hostExecPod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "e2e-net-exec",
				Namespace: fr.Namespace.Name,
				Labels:    map[string]string{"app": "e2e-net-exec"},
			},
			Spec: v1.PodSpec{
				HostNetwork: true,
				NodeName:    clientNodeInfo.name,
				Containers: []v1.Container{
					{
						Name:            "e2e-net-exec",
						Image:           kubeProxyE2eImage,
						ImagePullPolicy: "Always",
						Args:            []string{"pause"},
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "proc-net",
								MountPath: "/rootfs/proc/net",
								ReadOnly:  true,
							},
						},
						SecurityContext: &v1.SecurityContext{
							Privileged: &privileged,
						},
					},
				},
				Volumes: []v1.Volume{
					{
						Name: "proc-net",
						VolumeSource: v1.VolumeSource{
							HostPath: &v1.HostPathVolumeSource{
								Path: "/proc/net",
							},
						},
					},
				},
				TerminationGracePeriodSeconds: &zero,
			},
		}
		fr.PodClient().CreateSync(hostExecPod)
		defer fr.PodClient().DeleteSync(hostExecPod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)

		// Some distributions (Ubuntu 16.04 etc.) don't support the proc file.
		_, err = framework.RunHostCmd(fr.Namespace.Name, "e2e-net-exec",
			"ls /rootfs/proc/net/nf_conntrack")
		if err != nil && strings.Contains(err.Error(), "No such file or directory") {
			framework.Skipf("The node %s does not support /proc/net/nf_conntrack", clientNodeInfo.name)
		}
		framework.ExpectNoError(err)

		// Create the client and server pods
		clientPodSpec := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "e2e-net-client",
				Namespace: fr.Namespace.Name,
				Labels:    map[string]string{"app": "e2e-net-client"},
			},
			Spec: v1.PodSpec{
				NodeName: clientNodeInfo.name,
				Containers: []v1.Container{
					{
						Name:            "e2e-net-client",
						Image:           kubeProxyE2eImage,
						ImagePullPolicy: "Always",
						Args: []string{
							"net", "--serve", fmt.Sprintf(":%d", testDaemonHTTPPort),
						},
					},
				},
				TerminationGracePeriodSeconds: &zero,
			},
		}

		serverPodSpec := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "e2e-net-server",
				Namespace: fr.Namespace.Name,
				Labels:    map[string]string{"app": "e2e-net-server"},
			},
			Spec: v1.PodSpec{
				NodeName: serverNodeInfo.name,
				Containers: []v1.Container{
					{
						Name:            "e2e-net-server",
						Image:           kubeProxyE2eImage,
						ImagePullPolicy: "Always",
						Args: []string{
							"net",
							"--runner", "nat-closewait-server",
							"--options",
							fmt.Sprintf(`{"LocalAddr":":%v", "PostFinTimeoutSeconds":%v}`,
								testDaemonTCPPort,
								postFinTimeoutSeconds),
						},
						Ports: []v1.ContainerPort{
							{
								Name:          "tcp",
								ContainerPort: testDaemonTCPPort,
								HostPort:      testDaemonTCPPort,
							},
						},
					},
				},
				TerminationGracePeriodSeconds: &zero,
			},
		}

		ginkgo.By(fmt.Sprintf(
			"Launching a server daemon on node %v (node ip: %v, image: %v)",
			serverNodeInfo.name,
			serverNodeInfo.nodeIP,
			kubeProxyE2eImage))
		fr.PodClient().CreateSync(serverPodSpec)

		ginkgo.By(fmt.Sprintf(
			"Launching a client daemon on node %v (node ip: %v, image: %v)",
			clientNodeInfo.name,
			clientNodeInfo.nodeIP,
			kubeProxyE2eImage))
		fr.PodClient().CreateSync(clientPodSpec)

		ginkgo.By("Make client connect")

		options := nat.CloseWaitClientOptions{

			RemoteAddr:            net.JoinHostPort(serverNodeInfo.nodeIP, strconv.Itoa(testDaemonTCPPort)),
			TimeoutSeconds:        deadlineTimeoutSeconds,
			PostFinTimeoutSeconds: postFinTimeoutSeconds,
			LeakConnection:        true,
		}

		jsonBytes, err := json.Marshal(options)
		framework.ExpectNoError(err, "could not marshal")

		cmd := fmt.Sprintf(
			`curl -X POST http://localhost:%v/run/nat-closewait-client -d `+
				`'%v' 2>/dev/null`,
			testDaemonHTTPPort,
			string(jsonBytes))
		// Run the closewait command in a subroutine so it keeps waiting during postFinTimeoutSeconds
		// otherwise the pod is deleted and the connection is closed loosing the conntrack entry
		go func() {
			defer ginkgo.GinkgoRecover()
			_, err = framework.RunHostCmd(fr.Namespace.Name, "e2e-net-client", cmd)
			framework.ExpectNoError(err)
		}()

		<-time.After(time.Duration(1) * time.Second)

		ginkgo.By("Checking /proc/net/nf_conntrack for the timeout")
		// These must be synchronized from the default values set in
		// pkg/apis/../defaults.go ConntrackTCPCloseWaitTimeout. The
		// current defaults are hidden in the initialization code.
		const epsilonSeconds = 60
		const expectedTimeoutSeconds = 60 * 60
		// the conntrack file uses the IPv6 expanded format
		ip := fullIPv6(net.ParseIP(serverNodeInfo.nodeIP))
		// Obtain the corresponding conntrack entry on the host checking
		// the nf_conntrack file from the pod e2e-net-exec.
		// It retries in a loop if the entry is not found.
		cmd = fmt.Sprintf("cat /rootfs/proc/net/nf_conntrack "+
			"| grep -m 1 'CLOSE_WAIT.*dst=%v.*dport=%v' ",
			ip, testDaemonTCPPort)
		if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
			result, err := framework.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd)
			// retry if we can't obtain the conntrack entry
			if err != nil {
				framework.Logf("failed to obtain conntrack entry: %v %v", result, err)
				return false, nil
			}
			framework.Logf("conntrack entry for node %v and port %v:  %v", serverNodeInfo.nodeIP, testDaemonTCPPort, result)
			// Timeout in seconds is available as the fifth column of
			// the matched entry in /proc/net/nf_conntrack.
			line := strings.Fields(result)
			if len(line) < 5 {
				return false, fmt.Errorf("conntrack entry does not have a timeout field: %v", line)
			}
			timeoutSeconds, err := strconv.Atoi(line[4])
			if err != nil {
				return false, fmt.Errorf("failed to convert matched timeout %s to integer: %v", line[4], err)
			}
			if math.Abs(float64(timeoutSeconds-expectedTimeoutSeconds)) < epsilonSeconds {
				return true, nil
			}
			return false, fmt.Errorf("wrong TCP CLOSE_WAIT timeout: %v expected: %v", timeoutSeconds, expectedTimeoutSeconds)
		}); err != nil {
			framework.Failf("no conntrack entry for port %d on node %s", testDaemonTCPPort, serverNodeInfo.nodeIP)
		}
	})

	// Regression test for #74839, where:
	// Packets considered INVALID by conntrack are now dropped. In particular, this fixes
	// a problem where spurious retransmits in a long-running TCP connection to a service
	// IP could result in the connection being closed with the error "Connection reset by
	// peer"
	ginkgo.It("should resolve connrection reset issue #74839 [Slow]", func() {
		serverLabel := map[string]string{
			"app": "boom-server",
		}
		clientLabel := map[string]string{
			"app": "client",
		}

		serverPod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "boom-server",
				Labels: serverLabel,
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "boom-server",
						Image: imageutils.GetE2EImage(imageutils.RegressionIssue74839),
						Ports: []v1.ContainerPort{
							{
								ContainerPort: 9000, // Default port exposed by boom-server
							},
						},
					},
				},
				Affinity: &v1.Affinity{
					PodAntiAffinity: &v1.PodAntiAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
							{
								LabelSelector: &metav1.LabelSelector{
									MatchLabels: clientLabel,
								},
								TopologyKey: "kubernetes.io/hostname",
							},
						},
					},
				},
			},
		}
		_, err := fr.ClientSet.CoreV1().Pods(fr.Namespace.Name).Create(serverPod)
		framework.ExpectNoError(err)

		err = e2epod.WaitForPodsRunningReady(fr.ClientSet, fr.Namespace.Name, 1, 0, framework.PodReadyBeforeTimeout, map[string]string{})
		framework.ExpectNoError(err)

		ginkgo.By("Server pod created")

		svc := &v1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name: "boom-server",
			},
			Spec: v1.ServiceSpec{
				Selector: serverLabel,
				Ports: []v1.ServicePort{
					{
						Protocol: v1.ProtocolTCP,
						Port:     9000,
					},
				},
			},
		}
		_, err = fr.ClientSet.CoreV1().Services(fr.Namespace.Name).Create(svc)
		framework.ExpectNoError(err)

		ginkgo.By("Server service created")

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "startup-script",
				Labels: clientLabel,
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "startup-script",
						Image: imageutils.GetE2EImage(imageutils.StartupScript),
						Command: []string{
							"bash", "-c", "while true; do sleep 2; nc boom-server 9000& done",
						},
					},
				},
				Affinity: &v1.Affinity{
					PodAntiAffinity: &v1.PodAntiAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
							{
								LabelSelector: &metav1.LabelSelector{
									MatchLabels: serverLabel,
								},
								TopologyKey: "kubernetes.io/hostname",
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}
		_, err = fr.ClientSet.CoreV1().Pods(fr.Namespace.Name).Create(pod)
		framework.ExpectNoError(err)

		ginkgo.By("Client pod created")

		for i := 0; i < 20; i++ {
			time.Sleep(3 * time.Second)
			resultPod, err := fr.ClientSet.CoreV1().Pods(fr.Namespace.Name).Get(serverPod.Name, metav1.GetOptions{})
			framework.ExpectNoError(err)
			gomega.Expect(resultPod.Status.ContainerStatuses[0].LastTerminationState.Terminated).Should(gomega.BeNil())
		}
	})
})

// fullIPv6 returns a string with the IP representation
// if IPv6 it returns the expanded address format
// credit https://stackoverflow.com/a/52003106/4532704
func fullIPv6(ip net.IP) string {
	if ip.To4() == nil {
		dst := make([]byte, hex.EncodedLen(len(ip)))
		_ = hex.Encode(dst, ip)
		return string(dst[0:4]) + ":" +
			string(dst[4:8]) + ":" +
			string(dst[8:12]) + ":" +
			string(dst[12:16]) + ":" +
			string(dst[16:20]) + ":" +
			string(dst[20:24]) + ":" +
			string(dst[24:28]) + ":" +
			string(dst[28:])
	}
	return ip.String()
}
<commit_msg>Revert "fix flakes on e2e test TCP CLOSE_WAIT timeout"<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	"encoding/json"
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/kubernetes/test/e2e/framework"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
	"k8s.io/kubernetes/test/images/agnhost/net/nat"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
)

var kubeProxyE2eImage = imageutils.GetE2EImage(imageutils.Agnhost)

var _ = SIGDescribe("Network", func() {
	const (
		testDaemonHTTPPort    = 11301
		testDaemonTCPPort     = 11302
		timeoutSeconds        = 10
		postFinTimeoutSeconds = 5
	)

	fr := framework.NewDefaultFramework("network")

	ginkgo.It("should set TCP CLOSE_WAIT timeout", func() {
		nodes, err := e2enode.GetBoundedReadySchedulableNodes(fr.ClientSet, 2)
		framework.ExpectNoError(err)
		if len(nodes.Items) < 2 {
			framework.Skipf(
				"Test requires >= 2 Ready nodes, but there are only %v nodes",
				len(nodes.Items))
		}

		ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP)

		type NodeInfo struct {
			node   *v1.Node
			name   string
			nodeIP string
		}

		clientNodeInfo := NodeInfo{
			node:   &nodes.Items[0],
			name:   nodes.Items[0].Name,
			nodeIP: ips[0],
		}

		serverNodeInfo := NodeInfo{
			node:   &nodes.Items[1],
			name:   nodes.Items[1].Name,
			nodeIP: ips[1],
		}

		zero := int64(0)

		// Some distributions (Ubuntu 16.04 etc.) don't support the proc file.
		_, err = e2essh.IssueSSHCommandWithResult(
			"ls /proc/net/nf_conntrack",
			framework.TestContext.Provider,
			clientNodeInfo.node)
		if err != nil && strings.Contains(err.Error(), "No such file or directory") {
			framework.Skipf("The node %s does not support /proc/net/nf_conntrack", clientNodeInfo.name)
		}
		framework.ExpectNoError(err)

		clientPodSpec := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "e2e-net-client",
				Namespace: fr.Namespace.Name,
				Labels:    map[string]string{"app": "e2e-net-client"},
			},
			Spec: v1.PodSpec{
				NodeName: clientNodeInfo.name,
				Containers: []v1.Container{
					{
						Name:            "e2e-net-client",
						Image:           kubeProxyE2eImage,
						ImagePullPolicy: "Always",
						Args: []string{
							"net", "--serve", fmt.Sprintf("0.0.0.0:%d", testDaemonHTTPPort),
						},
					},
				},
				TerminationGracePeriodSeconds: &zero,
			},
		}

		serverPodSpec := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "e2e-net-server",
				Namespace: fr.Namespace.Name,
				Labels:    map[string]string{"app": "e2e-net-server"},
			},
			Spec: v1.PodSpec{
				NodeName: serverNodeInfo.name,
				Containers: []v1.Container{
					{
						Name:            "e2e-net-server",
						Image:           kubeProxyE2eImage,
						ImagePullPolicy: "Always",
						Args: []string{
							"net",
							"--runner", "nat-closewait-server",
							"--options",
							fmt.Sprintf(`{"LocalAddr":"0.0.0.0:%v", "PostFindTimeoutSeconds":%v}`,
								testDaemonTCPPort,
								postFinTimeoutSeconds),
						},
						Ports: []v1.ContainerPort{
							{
								Name:          "tcp",
								ContainerPort: testDaemonTCPPort,
								HostPort:      testDaemonTCPPort,
							},
						},
					},
				},
				TerminationGracePeriodSeconds: &zero,
			},
		}

		ginkgo.By(fmt.Sprintf(
			"Launching a server daemon on node %v (node ip: %v, image: %v)",
			serverNodeInfo.name,
			serverNodeInfo.nodeIP,
			kubeProxyE2eImage))
		fr.PodClient().CreateSync(serverPodSpec)

		ginkgo.By(fmt.Sprintf(
			"Launching a client daemon on node %v (node ip: %v, image: %v)",
			clientNodeInfo.name,
			clientNodeInfo.nodeIP,
			kubeProxyE2eImage))
		fr.PodClient().CreateSync(clientPodSpec)

		ginkgo.By("Make client connect")

		options := nat.CloseWaitClientOptions{
			RemoteAddr: fmt.Sprintf("%v:%v",
				serverNodeInfo.nodeIP, testDaemonTCPPort),
			TimeoutSeconds:        timeoutSeconds,
			PostFinTimeoutSeconds: 0,
			LeakConnection:        true,
		}

		jsonBytes, err := json.Marshal(options)
		framework.ExpectNoError(err, "could not marshal")

		cmd := fmt.Sprintf(
			`curl -X POST http://localhost:%v/run/nat-closewait-client -d `+
				`'%v' 2>/dev/null`,
			testDaemonHTTPPort,
			string(jsonBytes))
		framework.RunHostCmdOrDie(fr.Namespace.Name, "e2e-net-client", cmd)

		<-time.After(time.Duration(1) * time.Second)

		ginkgo.By("Checking /proc/net/nf_conntrack for the timeout")
		// If test flakes occur here, then this check should be performed
		// in a loop as there may be a race with the client connecting.
		e2essh.IssueSSHCommandWithResult(
			fmt.Sprintf("sudo cat /proc/net/nf_conntrack | grep 'dport=%v'",
				testDaemonTCPPort),
			framework.TestContext.Provider,
			clientNodeInfo.node)

		// Timeout in seconds is available as the fifth column from
		// /proc/net/nf_conntrack.
		result, err := e2essh.IssueSSHCommandWithResult(
			fmt.Sprintf(
				"sudo cat /proc/net/nf_conntrack "+
					"| grep 'CLOSE_WAIT.*dst=%v.*dport=%v' "+
					"| tail -n 1"+
					"| awk '{print $5}' ",
				serverNodeInfo.nodeIP,
				testDaemonTCPPort),
			framework.TestContext.Provider,
			clientNodeInfo.node)
		framework.ExpectNoError(err)

		timeoutSeconds, err := strconv.Atoi(strings.TrimSpace(result.Stdout))
		framework.ExpectNoError(err)

		// These must be synchronized from the default values set in
		// pkg/apis/../defaults.go ConntrackTCPCloseWaitTimeout. The
		// current defaults are hidden in the initialization code.
		const epsilonSeconds = 60
		const expectedTimeoutSeconds = 60 * 60

		framework.Logf("conntrack entry timeout was: %v, expected: %v",
			timeoutSeconds, expectedTimeoutSeconds)

		gomega.Expect(math.Abs(float64(timeoutSeconds - expectedTimeoutSeconds))).Should(
			gomega.BeNumerically("<", (epsilonSeconds)))
	})

	// Regression test for #74839, where:
	// Packets considered INVALID by conntrack are now dropped. In particular, this fixes
	// a problem where spurious retransmits in a long-running TCP connection to a service
	// IP could result in the connection being closed with the error "Connection reset by
	// peer"
	ginkgo.It("should resolve connrection reset issue #74839 [Slow]", func() {
		serverLabel := map[string]string{
			"app": "boom-server",
		}
		clientLabel := map[string]string{
			"app": "client",
		}

		serverPod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "boom-server",
				Labels: serverLabel,
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "boom-server",
						Image: imageutils.GetE2EImage(imageutils.RegressionIssue74839),
						Ports: []v1.ContainerPort{
							{
								ContainerPort: 9000, // Default port exposed by boom-server
							},
						},
					},
				},
				Affinity: &v1.Affinity{
					PodAntiAffinity: &v1.PodAntiAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
							{
								LabelSelector: &metav1.LabelSelector{
									MatchLabels: clientLabel,
								},
								TopologyKey: "kubernetes.io/hostname",
							},
						},
					},
				},
			},
		}
		_, err := fr.ClientSet.CoreV1().Pods(fr.Namespace.Name).Create(serverPod)
		framework.ExpectNoError(err)

		err = e2epod.WaitForPodsRunningReady(fr.ClientSet, fr.Namespace.Name, 1, 0, framework.PodReadyBeforeTimeout, map[string]string{})
		framework.ExpectNoError(err)

		ginkgo.By("Server pod created")

		svc := &v1.Service{
			ObjectMeta: metav1.ObjectMeta{
				Name: "boom-server",
			},
			Spec: v1.ServiceSpec{
				Selector: serverLabel,
				Ports: []v1.ServicePort{
					{
						Protocol: v1.ProtocolTCP,
						Port:     9000,
					},
				},
			},
		}
		_, err = fr.ClientSet.CoreV1().Services(fr.Namespace.Name).Create(svc)
		framework.ExpectNoError(err)

		ginkgo.By("Server service created")

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "startup-script",
				Labels: clientLabel,
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:  "startup-script",
						Image: imageutils.GetE2EImage(imageutils.StartupScript),
						Command: []string{
							"bash", "-c", "while true; do sleep 2; nc boom-server 9000& done",
						},
					},
				},
				Affinity: &v1.Affinity{
					PodAntiAffinity: &v1.PodAntiAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
							{
								LabelSelector: &metav1.LabelSelector{
									MatchLabels: serverLabel,
								},
								TopologyKey: "kubernetes.io/hostname",
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}
		_, err = fr.ClientSet.CoreV1().Pods(fr.Namespace.Name).Create(pod)
		framework.ExpectNoError(err)

		ginkgo.By("Client pod created")

		for i := 0; i < 20; i++ {
			time.Sleep(3 * time.Second)
			resultPod, err := fr.ClientSet.CoreV1().Pods(fr.Namespace.Name).Get(serverPod.Name, metav1.GetOptions{})
			framework.ExpectNoError(err)
			gomega.Expect(resultPod.Status.ContainerStatuses[0].LastTerminationState.Terminated).Should(gomega.BeNil())
		}
	})
})
<|endoftext|>


<commit_before>// Copyright © 2017 Mikael Berthe <mikael@lilotux.net>
//
// Licensed under the MIT license.
// Please see the LICENSE file is this directory.

package printer

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/m0t0k1ch1/gomif"
	"github.com/pkg/errors"

	"github.com/McKael/madon"
)

const themeDirName = "themes"

// ThemePrinter represents a Theme printer
type ThemePrinter struct {
	name        string
	templateDir string
}

// NewPrinterTheme returns a Theme ResourcePrinter
// For ThemePrinter, the options parameter contains the name of the theme
// and the template base directory (themes are assumed to be in the "themes"
// subdirectory).
func NewPrinterTheme(options Options) (*ThemePrinter, error) {
	name, ok := options["name"]
	if !ok || name == "" {
		return nil, fmt.Errorf("no theme name")
	}
	if strings.IndexRune(name, '/') >= 0 {
		// Should we allow that?  (subthemes, etc.)
		return nil, fmt.Errorf("invalid theme name")
	}
	return &ThemePrinter{
		name:        name,
		templateDir: options["template_directory"],
	}, nil
}

// PrintObj sends the object as text to the writer
// If the writer w is nil, standard output will be used.
func (p *ThemePrinter) PrintObj(obj interface{}, w io.Writer, tmpl string) error {
	if w == nil {
		w = os.Stdout
	}

	if p.name == "" {
		return fmt.Errorf("no theme name") // Should not happen
	}

	var objType string

	switch obj.(type) {
	case []madon.Account, madon.Account, *madon.Account:
		objType = "account"
	case []madon.Application, madon.Application, *madon.Application:
		objType = "application"
	case []madon.Attachment, madon.Attachment, *madon.Attachment:
		objType = "attachment"
	case []madon.Card, madon.Card, *madon.Card:
		objType = "card"
	case []madon.Client, madon.Client, *madon.Client:
		objType = "client"
	case []madon.Context, madon.Context, *madon.Context:
		objType = "context"
	case []madon.Instance, madon.Instance, *madon.Instance:
		objType = "instance"
	case []madon.Mention, madon.Mention, *madon.Mention:
		objType = "mention"
	case []madon.Notification, madon.Notification, *madon.Notification:
		objType = "notification"
	case []madon.Relationship, madon.Relationship, *madon.Relationship:
		objType = "relationship"
	case []madon.Report, madon.Report, *madon.Report:
		objType = "report"
	case []madon.Results, madon.Results, *madon.Results:
		objType = "results"
	case []madon.Status, madon.Status, *madon.Status:
		objType = "status"
	case []madon.StreamEvent, madon.StreamEvent, *madon.StreamEvent:
		objType = "streamEvent"
	case []madon.Tag, madon.Tag, *madon.Tag:
		objType = "tag"

	case []*gomif.InstanceStatus, *gomif.InstanceStatus:
		objType = "instancestatus"
	}

	var rp *ResourcePrinter

	if objType != "" {
		// Check template exists
		templatePath := filepath.Join(p.templateDir, themeDirName, p.name, objType) + ".tmpl"
		if _, err := os.Stat(templatePath); err != nil {
			fmt.Fprintf(os.Stderr, "Warning: theme template not found, falling back to plaintext printer\n") // XXX
		} else {
			t, err := ioutil.ReadFile(templatePath)
			if err != nil {
				return errors.Wrap(err, "cannot read template")
			}
			np, err := NewPrinter("template", Options{"template": string(t)})
			if err != nil {
				return errors.Wrap(err, "cannot create template printer")
			}
			rp = &np
		}
	}

	if rp != nil {
		return (*rp).PrintObj(obj, w, "")
	}

	// No resource printer; let's fall back to plain printer
	// XXX Maybe we should just fail?
	plainP, err := NewPrinter("plain", nil)
	if err != nil {
		return errors.Wrap(err, "cannot create plaintext printer")
	}
	return plainP.PrintObj(obj, w, "")
}
<commit_msg>Themes: Template for "instance --stats" is "instancestats.tmpl"<commit_after>// Copyright © 2017 Mikael Berthe <mikael@lilotux.net>
//
// Licensed under the MIT license.
// Please see the LICENSE file is this directory.

package printer

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/m0t0k1ch1/gomif"
	"github.com/pkg/errors"

	"github.com/McKael/madon"
)

const themeDirName = "themes"

// ThemePrinter represents a Theme printer
type ThemePrinter struct {
	name        string
	templateDir string
}

// NewPrinterTheme returns a Theme ResourcePrinter
// For ThemePrinter, the options parameter contains the name of the theme
// and the template base directory (themes are assumed to be in the "themes"
// subdirectory).
func NewPrinterTheme(options Options) (*ThemePrinter, error) {
	name, ok := options["name"]
	if !ok || name == "" {
		return nil, fmt.Errorf("no theme name")
	}
	if strings.IndexRune(name, '/') >= 0 {
		// Should we allow that?  (subthemes, etc.)
		return nil, fmt.Errorf("invalid theme name")
	}
	return &ThemePrinter{
		name:        name,
		templateDir: options["template_directory"],
	}, nil
}

// PrintObj sends the object as text to the writer
// If the writer w is nil, standard output will be used.
func (p *ThemePrinter) PrintObj(obj interface{}, w io.Writer, tmpl string) error {
	if w == nil {
		w = os.Stdout
	}

	if p.name == "" {
		return fmt.Errorf("no theme name") // Should not happen
	}

	var objType string

	switch obj.(type) {
	case []madon.Account, madon.Account, *madon.Account:
		objType = "account"
	case []madon.Application, madon.Application, *madon.Application:
		objType = "application"
	case []madon.Attachment, madon.Attachment, *madon.Attachment:
		objType = "attachment"
	case []madon.Card, madon.Card, *madon.Card:
		objType = "card"
	case []madon.Client, madon.Client, *madon.Client:
		objType = "client"
	case []madon.Context, madon.Context, *madon.Context:
		objType = "context"
	case []madon.Instance, madon.Instance, *madon.Instance:
		objType = "instance"
	case []madon.Mention, madon.Mention, *madon.Mention:
		objType = "mention"
	case []madon.Notification, madon.Notification, *madon.Notification:
		objType = "notification"
	case []madon.Relationship, madon.Relationship, *madon.Relationship:
		objType = "relationship"
	case []madon.Report, madon.Report, *madon.Report:
		objType = "report"
	case []madon.Results, madon.Results, *madon.Results:
		objType = "results"
	case []madon.Status, madon.Status, *madon.Status:
		objType = "status"
	case []madon.StreamEvent, madon.StreamEvent, *madon.StreamEvent:
		objType = "streamEvent"
	case []madon.Tag, madon.Tag, *madon.Tag:
		objType = "tag"

	case []*gomif.InstanceStatus, *gomif.InstanceStatus:
		objType = "instancestats"
	}

	var rp *ResourcePrinter

	if objType != "" {
		// Check template exists
		templatePath := filepath.Join(p.templateDir, themeDirName, p.name, objType) + ".tmpl"
		if _, err := os.Stat(templatePath); err != nil {
			fmt.Fprintf(os.Stderr, "Warning: theme template not found, falling back to plaintext printer\n") // XXX
		} else {
			t, err := ioutil.ReadFile(templatePath)
			if err != nil {
				return errors.Wrap(err, "cannot read template")
			}
			np, err := NewPrinter("template", Options{"template": string(t)})
			if err != nil {
				return errors.Wrap(err, "cannot create template printer")
			}
			rp = &np
		}
	}

	if rp != nil {
		return (*rp).PrintObj(obj, w, "")
	}

	// No resource printer; let's fall back to plain printer
	// XXX Maybe we should just fail?
	plainP, err := NewPrinter("plain", nil)
	if err != nil {
		return errors.Wrap(err, "cannot create plaintext printer")
	}
	return plainP.PrintObj(obj, w, "")
}
<|endoftext|>


<commit_before>package xz

import (
	"errors"
	"io"
)

// padWriter supports the writing of padding aligned to a block size.
// padWriter remembers errors.
type padWriter struct {
	w         io.Writer
	n         int64
	blockSize int
	pad       []byte
	err       error
}

// newPadWriter creates a new pad writer.
func newPadWriter(w io.Writer, blockSize int) *padWriter {
	if blockSize < 1 {
		blockSize = 1
	}
	return &padWriter{
		w:         w,
		blockSize: blockSize,
		pad:       make([]byte, blockSize-1),
	}
}

// Write writes the slide to the underlying writer.
func (w *padWriter) Write(p []byte) (n int, err error) {
	if w.err != nil {
		return 0, w.err
	}
	n, err = w.w.Write(p)
	w.n += int64(n)
	w.err = err
	return n, err
}

// Pad writes the pad to align the stream with the block size.
func (w *padWriter) Pad() (n int, err error) {
	if w.err != nil {
		return 0, w.err
	}
	k := int(w.n % int64(w.blockSize))
	if k == 0 {
		return 0, nil
	}
	n, err = w.Write(w.pad[:w.blockSize-k])
	w.err = err
	return n, err
}

// PadReader supports the reading of pads from a reader.
type padReader struct {
	r         io.Reader
	n         int64
	blockSize int
	p         []byte
	err       error
}

// newPadReader creates a new pad reader.
func newPadReader(r io.Reader, blockSize int) *padReader {
	if blockSize < 1 {
		blockSize = 1
	}
	return &padReader{
		r:         r,
		blockSize: blockSize,
		p:         make([]byte, blockSize-1),
	}
}

// Read reads data from the underlying reader.
func (r *padReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	n, err = r.r.Read(p)
	r.n += int64(n)
	r.err = err
	return n, err
}

// Pad reads the padding from the underlying reader.
func (r *padReader) Pad() (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	k := int(r.n % int64(r.blockSize))
	if k == 0 {
		return 0, nil
	}
	if n, err = io.ReadFull(r, r.p[:r.blockSize-k]); err != nil {
		r.err = err
		return n, err
	}
	for _, c := range r.p[:n] {
		if c != 0 {
			r.err = errors.New("xz: non-zero padding byte")
			return n, r.err
		}
	}
	return n, nil
}
<commit_msg>xz: removed error field of padReader and padWriter<commit_after>package xz

import (
	"errors"
	"io"
)

// padWriter supports the writing of padding aligned to a block size.
// padWriter remembers errors.
type padWriter struct {
	w         io.Writer
	n         int64
	blockSize int
	pad       []byte
}

// newPadWriter creates a new pad writer.
func newPadWriter(w io.Writer, blockSize int) *padWriter {
	if blockSize < 1 {
		blockSize = 1
	}
	return &padWriter{
		w:         w,
		blockSize: blockSize,
		pad:       make([]byte, blockSize-1),
	}
}

// Write writes the slide to the underlying writer.
func (w *padWriter) Write(p []byte) (n int, err error) {
	n, err = w.w.Write(p)
	w.n += int64(n)
	return n, err
}

// Pad writes the pad to align the stream with the block size.
func (w *padWriter) Pad() (n int, err error) {
	k := int(w.n % int64(w.blockSize))
	if k == 0 {
		return 0, nil
	}
	n, err = w.Write(w.pad[:w.blockSize-k])
	return n, err
}

// PadReader supports the reading of pads from a reader.
type padReader struct {
	r         io.Reader
	n         int64
	blockSize int
	p         []byte
}

// newPadReader creates a new pad reader.
func newPadReader(r io.Reader, blockSize int) *padReader {
	if blockSize < 1 {
		blockSize = 1
	}
	return &padReader{
		r:         r,
		blockSize: blockSize,
		p:         make([]byte, blockSize-1),
	}
}

// Read reads data from the underlying reader.
func (r *padReader) Read(p []byte) (n int, err error) {
	n, err = r.r.Read(p)
	r.n += int64(n)
	return n, err
}

// Pad reads the padding from the underlying reader.
func (r *padReader) Pad() (n int, err error) {
	k := int(r.n % int64(r.blockSize))
	if k == 0 {
		return 0, nil
	}
	if n, err = io.ReadFull(r, r.p[:r.blockSize-k]); err != nil {
		return n, err
	}
	for _, c := range r.p[:n] {
		if c != 0 {
			return n, errors.New("xz: non-zero padding byte")
		}
	}
	return n, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	_ "github.com/go-sql-driver/mysql"
	"github.com/go-xorm/xorm"
)

const DriverName = "mysql"
const MasterDataSourceName = "root:passwd@tcp(127.0.0.1:53306)/smoke?charset=utf8"

type GoXormUser struct {
	Id         int `xorm:"not null pk autoincr"`
	Name       string
	Age        int
	SysCreated int
	SysUpdated int
}

var engine *xorm.Engine

func main() {
	engine = newEngin()

	//execute()
	//ormInsert()
	query()
	//ormGet()
	//ormGetCols()
	//ormCount()
	// ormFindRows()
	//ormUpdate()
	//ormOmitUpdate()
	//ormMustColsUpdate()
}

// 连接到数据库
func newEngin() *xorm.Engine {
	engine, err := xorm.NewEngine(DriverName, MasterDataSourceName)
	if err != nil {
		log.Fatal(newEngin, err)
		return nil
	}
	// Debug模式，打印全部的SQL语句，帮助对比，看ORM与SQL执行的对照关系
	engine.ShowSQL(true)
	return engine
}

// 通过query方法查询
func query() {
	sql := "SELECT * FROM go_xorm_user"
	//results, err := engine.Query(sql)
	//results, err := engine.QueryInterface(sql)
	results, err := engine.QueryString(sql)
	if err != nil {
		log.Fatal("query", sql, err)
		return
	}
	total := len(results)
	if total == 0 {
		fmt.Println("没有任何数据", sql)
	} else {
		for i, data := range results {
			fmt.Printf("%d = %v\n", i, data)
		}
	}
}
<commit_msg>Update main.go<commit_after>package main

import (
	"fmt"
	"log"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/go-xorm/xorm"
)

const DriverName = "mysql"
const MasterDataSourceName = "root:passwd@tcp(127.0.0.1:53306)/smoke?charset=utf8"

type GoXormUser struct {
	Id   int `xorm:"not null pk autoincr"`
	Name string
	Age  int
	CAt  int
	MAt  int
}

var engine *xorm.Engine

func main() {
	engine = newEngin()

	execSQL()

	ormDelete()
	ormInsert()
	queryByRawSQL()
	ormGet()
	ormGetFields()
	ormCount()

	ormUpdate()
	ormGet()

	ormOmitUpdate()
	ormGet()

	// 查询多行
	ormFindRows()

	ormMustColsUpdate()
	queryByRawSQL()
}

// 连接到数据库
func newEngin() *xorm.Engine {
	engine, err := xorm.NewEngine(DriverName, MasterDataSourceName)
	if err != nil {
		log.Fatal("newEngin", err)
		return nil
	}
	// Debug模式，打印全部的SQL语句，帮助对比，看ORM与SQL执行的对照关系
	engine.ShowSQL(true)
	return engine
}

// 通过execute方法执行更新
func execSQL() {
	sql := `INSERT INTO go_xorm_user values(NULL, 'da-vinci', 1452, 0, 0)`
	affected, err := engine.Exec(sql)
	if err != nil {
		log.Fatal("execute error", err)
	} else {
		id, _ := affected.LastInsertId()
		rows, _ := affected.RowsAffected()
		fmt.Println("execute id=", id, ", rows=", rows)
	}
}

// 通过query方法查询
func queryByRawSQL() {
	sql := "SELECT * FROM go_xorm_user"
	//results, err := engine.Query(sql)
	//results, err := engine.QueryInterface(sql)
	results, err := engine.QueryString(sql)
	if err != nil {
		log.Fatal("query", sql, err)
		return
	}
	total := len(results)
	if total == 0 {
		fmt.Println("没有任何数据", sql)
	} else {
		for i, item := range results {
			fmt.Printf("%d = %v\n", i, item)
		}
	}
}

// 删除
func ormDelete() {
	user := &GoXormUser{
		Id: 4487,
	}
	affectedCnt, err := engine.Delete(user)
	if err != nil {
		log.Fatal("ormDelete error", err)
	} else {
		fmt.Println("affected row count is:", affectedCnt)
	}
}

// 根据models的结构映射数据
func ormInsert() {
	user := &GoXormUser{
		Id:   4487,
		Name: "74you",
		Age:  34,
	}
	id, err := engine.Insert(user)
	if err != nil {
		log.Fatal("ormInser error", err)
	} else {
		fmt.Println("ormInsert id =", id)
		fmt.Printf("%v\n", *user)
	}
}

// 获取数据
func ormGet() {
	user := &GoXormUser{Id: 4487}
	ok, err := engine.Get(user)
	if ok {
		fmt.Printf("Got User: %v\n", *user)
	} else if err != nil {
		log.Fatal("ormGet error", err)
	} else {
		fmt.Println("ormGet empy id=", user.Id)
	}
}

// 获取制定字段
func ormGetFields() {
	user := &GoXormUser{Id: 4487}
	// ok, err := engine.Cols("name", "age").Get(user)
	ok, err := engine.Cols("age").Get(user)
	if ok {
		fmt.Printf("%v\n", user)
	} else if err != nil {
		log.Fatal("ormGetFields error", err)
	} else {
		fmt.Println("ormGetFields empty id=", user.Id)
	}
}

// 统计
func ormCount() {
	//count, err := engine.Count(&UserInfo{})
	//count, err := engine.Where("name=?", "74you").Count(&UserInfo{})
	count, err := engine.Count(&GoXormUser{Name: "74you"})
	if err != nil {
		log.Fatal("ormCount error", err)
	} else {
		fmt.Printf("count=%v\n", count)
	}
}

// 更新数据
func ormUpdate() {
	// 全部更新
	user := &GoXormUser{MAt: 2020}
	ok, err := engine.Update(user)
	println("update all:", ok, err)
	// by ID
	user2 := &GoXormUser{Name: "魔鬼撒旦"}
	ok, err = engine.ID(4487).Update(user2)
	println("update by id:", ok, err)
}

// 排除某个字段
func ormOmitUpdate() {
	info := &GoXormUser{Id: 4487}
	ok, _ := engine.Get(info)
	if ok {
		if info.MAt > 0 {
			ok, _ := engine.ID(info.Id).Omit("c_at").
				Update(&GoXormUser{CAt: 0, MAt: int(time.Now().Unix())})
			fmt.Printf("ormOmitUpdate, rows=%d, m_at=%d\n", ok, 0)
		} else {
			ok, _ := engine.ID(info.Id).Omit("c_at").
				Update(&GoXormUser{CAt: 1, MAt: int(time.Now().Unix())})
			fmt.Printf("ormOmitUpdate, rows=%d, c_at=%d\n", ok, 0)
		}
	}
}

// 查找多行数据
func ormFindRows() {
	list := make([]GoXormUser, 0)
	err := engine.Cols("name", "id").
		Where("age>?", 3000).
		Limit(10).
		Asc("id", "c_at").Find(&list)
	if err != nil {
		log.Fatal("ormFindRows error", err)
	} else {
		fmt.Printf("%v\n", list)
	}
}

// 字段为空也可以更新（0, 空字符串等）
func ormMustColsUpdate() {
	info := &GoXormUser{Id: 1}
	ok, _ := engine.Get(info)
	if ok {
		if info.CAt > 0 {
			ok, _ := engine.ID(info.Id).
				MustCols("c_at").
				Update(&GoXormUser{CAt: 0, MAt: int(time.Now().Unix())})
			fmt.Printf("ormMustColsUpdate, rows=%d, "+
				"c_at=%d\n", ok, 0)
		} else {
			ok, _ := engine.ID(info.Id).
				MustCols("c_at").
				Update(&GoXormUser{CAt: 1, MAt: int(time.Now().Unix())})
			fmt.Printf("ormMustColsUpdate, rows=%d, "+
				"c_at=%d\n", ok, 0)
		}
	}
}
<|endoftext|>


<commit_before>/* scryptauth is a GO library for secure password handling

It uses hmac(scrypt(user_password, salt), server_key) to protect against
both dictionary attacks and DB leaks.

scryptauth additionally provides encode/decode routines using base64 to create strings
for storing into a DB.

Copyright: Michael Gebetsroither 2012 (michael \x40 mgeb \x2e org)

License: New BSD
*/
package scryptauth

import (
	"code.google.com/p/go.crypto/scrypt"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"crypto/subtle"
	"errors"
)

type ScryptAuth struct {
	HmacKey []byte // HMAC key used to secure scrypt hash
	PwCost  uint   // PwCost parameter used to calculate N parameter of scrypt (1<<PwCost == N)

	// scrypt parameter
	R int
	P int
}

const (
	// Key length and salt length are 32 bytes (256 bits)
	KEYLENGTH = 32
)

// Initialise ScryptAuth struct
func New(pw_cost uint, hmac_key []byte) (*ScryptAuth, error) {
	if pw_cost > 32 {
		return nil, errors.New("scryptauth new() - invalid pw_cost specified")
	}
	if len(hmac_key) != KEYLENGTH {
		return nil, errors.New("scryptauth new() - unsupported hmac_key length")
	}
	return &ScryptAuth{HmacKey: hmac_key, PwCost: pw_cost, R: 8, P: 1}, nil
}

// Create hash_ref suitable for later invocation of Check()
func (s ScryptAuth) Hash(pw_cost uint, user_password, salt []byte) (hash_ref []byte, err error) {
	scrypt_hash, err := scrypt.Key(user_password, salt, 1<<pw_cost, s.R, s.P, KEYLENGTH)
	if err != nil {
		return
	}
	hmac := hmac.New(sha256.New, s.HmacKey)
	if _, err = hmac.Write(scrypt_hash); err != nil {
		return
	}
	hash_ref = hmac.Sum(nil)
	return
}

// Check / Verify user_password against hash_ref/salt
func (s ScryptAuth) Check(pw_cost uint, hash_ref, user_password, salt []byte) (chk bool, err error) {
	result_hash, err := s.Hash(pw_cost, user_password, salt)
	if err != nil {
		return false, err
	}
	if subtle.ConstantTimeCompare(result_hash, hash_ref) != 1 {
		return false, errors.New("Error: Hash verification failed")
	}
	return true, nil
}

// Generate hash_ref and create new salt from crypto.rand
func (s ScryptAuth) Gen(user_password []byte) (hash, salt []byte, err error) {
	salt = make([]byte, KEYLENGTH)
	salt_length, err := rand.Read(salt)
	if salt_length != KEYLENGTH {
		return nil, nil, errors.New("Insufficient random bytes for salt")
	}
	if err != nil {
		return nil, nil, err
	}

	hash, err = s.Hash(s.PwCost, user_password, salt)
	if err != nil {
		return nil, nil, err
	}
	return
}
<commit_msg>put scrypt defaults into global constants<commit_after>/* scryptauth is a GO library for secure password handling

It uses hmac(scrypt(user_password, salt), server_key) to protect against
both dictionary attacks and DB leaks.

scryptauth additionally provides encode/decode routines using base64 to create strings
for storing into a DB.

Copyright: Michael Gebetsroither 2012 (michael \x40 mgeb \x2e org)

License: New BSD
*/
package scryptauth

import (
	"code.google.com/p/go.crypto/scrypt"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"crypto/subtle"
	"errors"
)

type ScryptAuth struct {
	HmacKey []byte // HMAC key used to secure scrypt hash
	PwCost  uint   // PwCost parameter used to calculate N parameter of scrypt (1<<PwCost == N)

	// scrypt parameter
	R int
	P int
}

const (
	// Key length and salt length are 32 bytes (256 bits)
	KEYLENGTH = 32

	// scrypt default parameters
	SCRYPT_CONST_R = 8
	SCRYPT_CONST_P = 1
)

// Initialise ScryptAuth struct
func New(pw_cost uint, hmac_key []byte) (*ScryptAuth, error) {
	if pw_cost > 32 {
		return nil, errors.New("scryptauth new() - invalid pw_cost specified")
	}
	if len(hmac_key) != KEYLENGTH {
		return nil, errors.New("scryptauth new() - unsupported hmac_key length")
	}
	return &ScryptAuth{HmacKey: hmac_key, PwCost: pw_cost, R: SCRYPT_CONST_R, P: SCRYPT_CONST_P}, nil
}

// Create hash_ref suitable for later invocation of Check()
func (s ScryptAuth) Hash(pw_cost uint, user_password, salt []byte) (hash_ref []byte, err error) {
	scrypt_hash, err := scrypt.Key(user_password, salt, 1<<pw_cost, s.R, s.P, KEYLENGTH)
	if err != nil {
		return
	}
	hmac := hmac.New(sha256.New, s.HmacKey)
	if _, err = hmac.Write(scrypt_hash); err != nil {
		return
	}
	hash_ref = hmac.Sum(nil)
	return
}

// Check / Verify user_password against hash_ref/salt
func (s ScryptAuth) Check(pw_cost uint, hash_ref, user_password, salt []byte) (chk bool, err error) {
	result_hash, err := s.Hash(pw_cost, user_password, salt)
	if err != nil {
		return false, err
	}
	if subtle.ConstantTimeCompare(result_hash, hash_ref) != 1 {
		return false, errors.New("Error: Hash verification failed")
	}
	return true, nil
}

// Generate hash_ref and create new salt from crypto.rand
func (s ScryptAuth) Gen(user_password []byte) (hash, salt []byte, err error) {
	salt = make([]byte, KEYLENGTH)
	salt_length, err := rand.Read(salt)
	if salt_length != KEYLENGTH {
		return nil, nil, errors.New("Insufficient random bytes for salt")
	}
	if err != nil {
		return nil, nil, err
	}

	hash, err = s.Hash(s.PwCost, user_password, salt)
	if err != nil {
		return nil, nil, err
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
)

func HttpUse(h http.HandlerFunc, middleware ...func(http.HandlerFunc) http.HandlerFunc) http.HandlerFunc {
	for _, m := range middleware {
		h = m(h)
	}

	return h
}

func AuthMiddleware(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		params := mux.Vars(r)
		app_key := params["app_key"]
		if app_key == "" {
			logger.GetRequestEntry(r).Warn(r, "app_key nil")
			http.Error(w, "app_key nil", 401)
			return
		}

		auth := r.FormValue("auth")
		if auth == "" {
			logger.GetRequestEntry(r).Warn(r, "auth nil")
			http.Error(w, "auth nil", 401)
			return
		}

		/*redis*/
		c := rpool.Get()

		//redis 格式 app_key url http://test.com

		reply, err := redis.Int(c.Do("HEXISTS", app_key, "url"))
		if err != nil || reply == 0 {
			logger.GetRequestEntry(r).Warn(err)
			http.Error(w, "auth process error", 401)
			return
		}

		u, err := redis.String(c.Do("HGET", app_key, "url"))
		if err != nil {
			logger.GetRequestEntry(r).Warn(err)
			http.Error(w, "auth process error", 401)
			return
		}
		c.Close()
		/*redis end*/

		v := url.Values{}

		v.Add("auth", auth)
		req, err := http.NewRequest("POST", u, bytes.NewBufferString(v.Encode()))
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
		req.Header.Add("Content-Length", strconv.Itoa(len(v.Encode())))

		if err != nil {
			logger.GetRequestEntry(r).Warn(err)
			http.Error(w, "auth process error", 401)
			return
		}
		ctx, _ := context.WithTimeout(r.Context(), 30*time.Second)
		err = worker.Execute(ctx, req, func(resp *http.Response, e error) (err error) {
			if e != nil {
				logger.Debug(e)
				return e
			}
			defer resp.Body.Close()

			a := Auth{}
			err = json.NewDecoder(resp.Body).Decode(&a)
			if err != nil {
				return
			}
			if len(a.Channels) == 0 {
				err = errors.New("Channels empty")
				return
			}
			ctx := r.Context()
			ctx = context.WithValue(ctx, "auth", a)
			ctx = context.WithValue(ctx, "app_key", app_key)
			r = r.WithContext(ctx)
			return
		})
		if err != nil {
			logger.GetRequestEntry(r).Warn(err)
			http.Error(w, err.Error(), 401)
			return
		}
		logger.GetRequestEntry(r).Debug("auth finsh")
		h.ServeHTTP(w, r)

	}
}
<commit_msg>fix middleware error response<commit_after>package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
)

func HttpUse(h http.HandlerFunc, middleware ...func(http.HandlerFunc) http.HandlerFunc) http.HandlerFunc {
	for _, m := range middleware {
		h = m(h)
	}

	return h
}

func AuthMiddleware(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		params := mux.Vars(r)
		app_key := params["app_key"]
		if app_key == "" {
			logger.GetRequestEntry(r).Warn(r, "app_key nil")
			http.Error(w, "app_key nil", 401)
			return
		}

		auth := r.FormValue("auth")
		if auth == "" {
			logger.GetRequestEntry(r).Warn("auth nil")
			http.Error(w, "auth nil", 401)
			return
		}

		/*redis*/
		c := rpool.Get()

		//redis 格式 app_key url http://test.com

		reply, err := redis.Int(c.Do("HEXISTS", app_key, "url"))
		if err != nil || reply == 0 {
			logger.GetRequestEntry(r).Warnf("app_key no exists  %s", err)
			http.Error(w, "app_key no exists", 401)
			return
		}

		u, err := redis.String(c.Do("HGET", app_key, "url"))
		if err != nil {
			logger.GetRequestEntry(r).Warn("auth process %s", err)
			http.Error(w, "auth process error", 401)
			return
		}
		c.Close()
		/*redis end*/

		v := url.Values{}

		v.Add("auth", auth)
		req, err := http.NewRequest("POST", u, bytes.NewBufferString(v.Encode()))
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
		req.Header.Add("Content-Length", strconv.Itoa(len(v.Encode())))

		ctx, _ := context.WithTimeout(r.Context(), 30*time.Second)
		err = worker.Execute(ctx, req, func(resp *http.Response, e error) (err error) {
			if e != nil {
				logger.Debug(e)
				return e
			}
			defer resp.Body.Close()

			a := Auth{}
			err = json.NewDecoder(resp.Body).Decode(&a)
			if err != nil {
				return
			}
			if len(a.Channels) == 0 {
				err = errors.New("Channels empty")
				return
			}
			ctx := r.Context()
			ctx = context.WithValue(ctx, "auth", a)
			ctx = context.WithValue(ctx, "app_key", app_key)
			r = r.WithContext(ctx)
			return
		})
		if err != nil {
			logger.GetRequestEntry(r).Warn(err)
			http.Error(w, err.Error(), 401)
			return
		}
		logger.GetRequestEntry(r).Debug("auth finsh")
		h.ServeHTTP(w, r)

	}
}
<|endoftext|>