<commit_before>package main

import (
	"net/http"
	"net/url"
	"strconv"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"strings"
	"time"
	"io/ioutil"
	"fmt"
)

const (
	BTCCHINA_API_URL = "https://api.btcchina.com/"
)

type BTCChina struct {
	APISecret, APIKey string
}

func (b *BTCChina) GetTicker(symbol string) (interface{}) {
	type Ticker struct {
		High string
		Low string
		Buy string
		Sell string
		Last string
		Vol string
		Date int64
		Vwap string
		Prev_close string
		Open string
	}

	type Response struct {
		Ticker Ticker
	}

	resp := Response{}
	req := fmt.Sprintf("%sdata/ticker?market=%s", BTCCHINA_API_URL, symbol)
	err := SendHTTPRequest(req, true, &resp)
	if err != nil {
		fmt.Println(err)
		return nil
	}
	return resp
}

func (b *BTCChina) GetTradesLast24h(symbol string) (bool) {
	req := fmt.Sprintf("%sdata/trades?market=%s", BTCCHINA_API_URL, symbol)
	err := SendHTTPRequest(req, true, nil)
	if err != nil {
		fmt.Println(err)
		return false
	}
	return true
}

func (b *BTCChina) GetTradeHistory(symbol string, limit, sinceTid int64, time time.Time) (bool) {
	req := fmt.Sprintf("%sdata/historydata?market=%s", BTCCHINA_API_URL, symbol)
	v := url.Values{}

	if limit > 0 {
		v.Set("limit", strconv.FormatInt(limit, 10))
	}
	if sinceTid > 0 {
		v.Set("since", strconv.FormatInt(sinceTid, 10))
	}
	if !time.IsZero() {
		v.Set("sincetype", strconv.FormatInt(time.Unix(), 10))
	}

	values := v.Encode()
	if (len(values) > 0) {
		req += "?" + values
	}

	err := SendHTTPRequest(req, true, nil)
	if err != nil {
		fmt.Println(err)
		return false
	}
	return true
}

func (b *BTCChina) GetOrderBook(symbol string, limit int) (bool) {
	req := fmt.Sprintf("%sdata/orderbook?market=%s&limit=%d", BTCCHINA_API_URL, symbol, limit)
	err := SendHTTPRequest(req, true, nil)
	if err != nil {
		fmt.Println(err)
		return false
	}
	return true
}

func (b *BTCChina) GetAccountInfo() {
	err := b.SendAuthenticatedHTTPRequest("getAccountInfo", nil)

	if err != nil {
		fmt.Println(err)
	}
}

func (b *BTCChina) BuyOrder(price, amount float64) {
	params := []string{}

	if (price != 0) {
		params = append(params, strconv.FormatFloat(price, 'f', 8, 64))
	}

	err := b.SendAuthenticatedHTTPRequest("buyOrder2", nil)

	if err != nil {
		fmt.Println(err)
	}
}

func (b *BTCChina) SendAuthenticatedHTTPRequest(method string, params []string) (err error) {
	nonce := strconv.FormatInt(time.Now().UnixNano(), 10)[0:16]

	if (len(params) > 0) {
		params = nil
	}

	encoded := fmt.Sprintf("tonce=%s&accesskey=%s&requestmethod=post&id=%d&method=%s&params=%s", nonce, b.APIKey, 1, method, params)

	fmt.Println(encoded)

	hmac := hmac.New(sha1.New, []byte(b.APISecret))
	hmac.Write([]byte(encoded))
	hash := hex.EncodeToString(hmac.Sum(nil))

	postData := make(map[string]interface{})
	postData["method"] = method
	postData["params"] = []string{}
	postData["id"] = 1

	data, err := json.Marshal(postData)

	fmt.Println(string(data))

	if err != nil {
		return errors.New("Unable to JSON POST data")
	}

	log.Printf("Sending POST request to %s calling method %s with params %s\n", "https://api.btcchina.com/api_trade_v1.php", method, data)
	reqBody := strings.NewReader(string(data))

	b64 := base64.StdEncoding.EncodeToString([]byte(b.APIKey + ":" + hash))

	req, err := http.NewRequest("POST", "https://api.btcchina.com/api_trade_v1.php", reqBody)

	if err != nil {
		return err
	}

	req.Header.Add("Content-type", "application/json-rpc")
	req.Header.Add("Authorization", "Basic " + b64)
	req.Header.Add("Json-Rpc-Tonce", nonce)

	client := &http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		return errors.New("PostRequest: Unable to send request")
	}

	contents, _ := ioutil.ReadAll(resp.Body)
	log.Printf("Recv'd :%s", string(contents))
	resp.Body.Close()
	return nil

}<commit_msg>Use log instead of fmt<commit_after>package main

import (
	"net/http"
	"net/url"
	"strconv"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"strings"
	"time"
	"io/ioutil"
	"fmt"
	"log"
)

const (
	BTCCHINA_API_URL = "https://api.btcchina.com/"
)

type BTCChina struct {
	APISecret, APIKey string
}

func (b *BTCChina) GetTicker(symbol string) (interface{}) {
	type Ticker struct {
		High string
		Low string
		Buy string
		Sell string
		Last string
		Vol string
		Date int64
		Vwap string
		Prev_close string
		Open string
	}

	type Response struct {
		Ticker Ticker
	}

	resp := Response{}
	req := fmt.Sprintf("%sdata/ticker?market=%s", BTCCHINA_API_URL, symbol)
	err := SendHTTPRequest(req, true, &resp)
	if err != nil {
		fmt.Println(err)
		return nil
	}
	return resp
}

func (b *BTCChina) GetTradesLast24h(symbol string) (bool) {
	req := fmt.Sprintf("%sdata/trades?market=%s", BTCCHINA_API_URL, symbol)
	err := SendHTTPRequest(req, true, nil)
	if err != nil {
		fmt.Println(err)
		return false
	}
	return true
}

func (b *BTCChina) GetTradeHistory(symbol string, limit, sinceTid int64, time time.Time) (bool) {
	req := fmt.Sprintf("%sdata/historydata?market=%s", BTCCHINA_API_URL, symbol)
	v := url.Values{}

	if limit > 0 {
		v.Set("limit", strconv.FormatInt(limit, 10))
	}
	if sinceTid > 0 {
		v.Set("since", strconv.FormatInt(sinceTid, 10))
	}
	if !time.IsZero() {
		v.Set("sincetype", strconv.FormatInt(time.Unix(), 10))
	}

	values := v.Encode()
	if (len(values) > 0) {
		req += "?" + values
	}

	err := SendHTTPRequest(req, true, nil)
	if err != nil {
		fmt.Println(err)
		return false
	}
	return true
}

func (b *BTCChina) GetOrderBook(symbol string, limit int) (bool) {
	req := fmt.Sprintf("%sdata/orderbook?market=%s&limit=%d", BTCCHINA_API_URL, symbol, limit)
	err := SendHTTPRequest(req, true, nil)
	if err != nil {
		fmt.Println(err)
		return false
	}
	return true
}

func (b *BTCChina) GetAccountInfo() {
	err := b.SendAuthenticatedHTTPRequest("getAccountInfo", nil)

	if err != nil {
		fmt.Println(err)
	}
}

func (b *BTCChina) BuyOrder(price, amount float64) {
	params := []string{}

	if (price != 0) {
		params = append(params, strconv.FormatFloat(price, 'f', 8, 64))
	}

	err := b.SendAuthenticatedHTTPRequest("buyOrder2", nil)

	if err != nil {
		fmt.Println(err)
	}
}

func (b *BTCChina) SendAuthenticatedHTTPRequest(method string, params []string) (err error) {
	nonce := strconv.FormatInt(time.Now().UnixNano(), 10)[0:16]

	if (len(params) > 0) {
		params = nil
	}

	encoded := fmt.Sprintf("tonce=%s&accesskey=%s&requestmethod=post&id=%d&method=%s&params=%s", nonce, b.APIKey, 1, method, params)

	fmt.Println(encoded)

	hmac := hmac.New(sha1.New, []byte(b.APISecret))
	hmac.Write([]byte(encoded))
	hash := hex.EncodeToString(hmac.Sum(nil))

	postData := make(map[string]interface{})
	postData["method"] = method
	postData["params"] = []string{}
	postData["id"] = 1

	data, err := json.Marshal(postData)

	fmt.Println(string(data))

	if err != nil {
		return errors.New("Unable to JSON POST data")
	}

	log.Printf("Sending POST request to %s calling method %s with params %s\n", "https://api.btcchina.com/api_trade_v1.php", method, data)
	reqBody := strings.NewReader(string(data))

	b64 := base64.StdEncoding.EncodeToString([]byte(b.APIKey + ":" + hash))

	req, err := http.NewRequest("POST", "https://api.btcchina.com/api_trade_v1.php", reqBody)

	if err != nil {
		return err
	}

	req.Header.Add("Content-type", "application/json-rpc")
	req.Header.Add("Authorization", "Basic " + b64)
	req.Header.Add("Json-Rpc-Tonce", nonce)

	client := &http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		return errors.New("PostRequest: Unable to send request")
	}

	contents, _ := ioutil.ReadAll(resp.Body)
	log.Printf("Recv'd :%s", string(contents))
	resp.Body.Close()
	return nil

}<|endoftext|>


<commit_before>package reporter

import (
	"github.com/fw42/go-hpfeeds"
)

func StartReporter(host string, port int, ident, auth string, channels []chan []byte) {
	hp := hpfeeds.NewHpfeeds(host, port, ident, auth)
	hp.Log = true
	hp.Connect()
	for _, channel := range channels {
		hp.Publish("", channel)
	}

}
<commit_msg>Rename StartReporter to Start<commit_after>package reporter

import (
	"github.com/fw42/go-hpfeeds"
)

func Start(host string, port int, ident, auth string, channels []chan []byte) {
	hp := hpfeeds.NewHpfeeds(host, port, ident, auth)
	hp.Log = true
	hp.Connect()
	for _, channel := range channels {
		hp.Publish("", channel)
	}
}
<|endoftext|>


<commit_before>package runtime

import "unsafe"

const (
	_PAGESIZE = 0x1000
)

var (
	_atman_hypercall_page   [2 * _PAGESIZE]byte
	_atman_shared_info_page [2 * _PAGESIZE]byte

	_atman_start_info  = &xenStartInfo{}
	_atman_shared_info *xenSharedInfo
)

//go:nosplit
func getRandomData(r []byte) {
	extendRandom(r, 0)
}

// env

func gogetenv(key string) string { return "" }

var _cgo_setenv unsafe.Pointer   // pointer to C function
var _cgo_unsetenv unsafe.Pointer // pointer to C function

// signals

const _NSIG = 0

func initsig()                 {}
func sigdisable(uint32)        {}
func sigenable(uint32)         {}
func sigignore(uint32)         {}
func raisebadsignal(sig int32) {}

// net

func netpoll(block bool) *g { return nil }
func netpollinited() bool   { return false }

type xenStartInfo struct {
	Magic          [32]byte
	NrPages        uint64
	SharedInfoAddr uintptr // machine address of share info struct
	SIFFlags       uint32
	_              [4]byte
	StoreMfn       uint64 // machine page number of shared page
	StoreEventchn  uint32
	_              [4]byte
	Console        struct {
		Mfn      mfn    // machine page number of console page
		Eventchn uint32 // event channel
		_        [4]byte
	}
	PageTableBase     vaddr // virtual address of page directory
	NrPageTableFrames uint64
	MfnList           uintptr // virtual address of page-frame list
	ModStart          uintptr // virtual address of pre-loaded module
	ModLen            uint64  // size (bytes) of pre-loaded module
	CmdLine           [1024]byte

	// The pfn range here covers both page table and p->m table frames
	FirstP2mPfn uint64 // 1st pfn forming initial P->M table
	NrP2mFrames uint64 // # of pgns forming initial P->M table
}

type xenSharedInfo struct {
	VCPUInfo      [32]vcpuInfo
	EvtchnPending [64]uint64
	EvtchnMask    [64]uint64
	WcVersion     uint32
	WcSec         uint32
	WcNsec        uint32
	_             [4]byte
	Arch          archSharedInfo
}

type archSharedInfo struct {
	MaxPfn                uint64
	PfnToMfnFrameListList uint64
	NmiReason             uint64
	_                     [32]uint64
}

type archVCPUInfo struct {
	CR2 uint64
	_   uint64
}

type vcpuTimeInfo struct {
	Version    uint32
	_          uint32
	TSC        uint64
	SystemNsec uint64
	TSCMul     uint32
	TSCShift   int8
	_          [3]int8
}

type vcpuInfo struct {
	UpcallPending uint8
	UpcallMask    uint8
	_             [6]byte
	PendingSel    uint64
	Arch          archVCPUInfo
	Time          vcpuTimeInfo
}

func atmaninit() {
	println("Atman OS")
	println("     ptr_size: ", ptrSize)
	println("   start_info: ", _atman_start_info)
	println("        magic: ", string(_atman_start_info.Magic[:]))
	println("     nr_pages: ", _atman_start_info.NrPages)
	println("  shared_info: ", _atman_start_info.SharedInfoAddr)
	println("   siff_flags: ", _atman_start_info.SIFFlags)
	println("    store_mfn: ", _atman_start_info.StoreMfn)
	println("    store_evc: ", _atman_start_info.StoreEventchn)
	println("  console_mfn: ", _atman_start_info.Console.Mfn)
	println("  console_evc: ", _atman_start_info.Console.Eventchn)
	println("      pt_base: ", _atman_start_info.PageTableBase)
	println(" nr_pt_frames: ", _atman_start_info.NrPageTableFrames)
	println("     mfn_list: ", _atman_start_info.MfnList)
	println("    mod_start: ", _atman_start_info.ModStart)
	println("      mod_len: ", _atman_start_info.ModLen)
	println("     cmd_line: ", _atman_start_info.CmdLine[:])
	println("    first_pfn: ", _atman_start_info.FirstP2mPfn)
	println("nr_p2m_frames: ", _atman_start_info.NrP2mFrames)

	_atman_console.init()

	println("setting _atman_phys_to_machine_mapping")
	_atman_phys_to_machine_mapping = *(*[8192]uint64)(unsafe.Pointer(
		_atman_start_info.MfnList,
	))

	println("mapping _atman_start_info")
	mapSharedInfo(_atman_start_info.SharedInfoAddr)

	_atman_mm.init()
}

func mapSharedInfo(vaddr uintptr) {
	pageAddr := round(
		uintptr(unsafe.Pointer(&_atman_shared_info_page[0])),
		_PAGESIZE,
	)

	ret := HYPERVISOR_update_va_mapping(
		pageAddr,
		vaddr|7,
		2, // UVMF_INVLPG: flush only one entry
	)

	if ret != 0 {
		println("HYPERVISOR_update_va_mapping returned ", ret)
		panic("HYPERVISOR_update_va_mapping failed")
	}

	_atman_shared_info = (*xenSharedInfo)(unsafe.Pointer(pageAddr))
}

// memory management

var (
	// Map of (pseudo-)physical addresses to machine addresses.
	_atman_phys_to_machine_mapping [8192]uint64
)

// Entry in level 3, 2, or 1 page table.
//
// - 63 if set means No execute (NX)
// - 51-13 the machine frame number
// - 12 available for guest
// - 11 available for guest
// - 10 available for guest
// - 9 available for guest
// - 8 global
// - 7 PAT (PSE is disabled, must use hypercall to make 4MB or 2MB pages)
// - 6 dirty
// - 5 accessed
// - 4 page cached disabled
// - 3 page write through
// - 2 userspace accessible
// - 1 writeable
// - 0 present
type pageTableEntry uintptr

const (
	xenPageTablePresent = 1 << iota
	xenPageTableWritable
	xenPageTableUserspaceAccessible
	xenPageTablePageWriteThrough
	xenPageTablePageCacheDisabled
	xenPageTableAccessed
	xenPageTableDirty
	xenPageTablePAT
	xenPageTableGlobal
	xenPageTableGuest1
	xenPageTableGuest2
	xenPageTableGuest3
	xenPageTableGuest4
	xenPageTableNoExecute = 1 << 63

	xenPageAddrMask  = 1<<52 - 1
	xenPageMask      = 1<<12 - 1
	xenPageFlagShift = 12

	PTE_PAGE_FLAGS       = xenPageTablePresent | xenPageTableWritable | xenPageTableUserspaceAccessible | xenPageTableAccessed
	PTE_PAGE_TABLE_FLAGS = xenPageTablePresent | xenPageTableUserspaceAccessible | xenPageTableAccessed | xenPageTableDirty
	PTE_TEMP             = xenPageTableGuest1
)

func (e pageTableEntry) debug() {
	println(
		"PTE<", unsafe.Pointer(e), ">:",
		" MFN=", e.mfn(),
		"  NX=", e.hasFlag(xenPageTableNoExecute),
		"   G=", e.hasFlag(xenPageTableGlobal),
		" PAT=", e.hasFlag(xenPageTablePAT),
		" DIR=", e.hasFlag(xenPageTableDirty),
		"   A=", e.hasFlag(xenPageTableAccessed),
		" PCD=", e.hasFlag(xenPageTablePageCacheDisabled),
		" PWT=", e.hasFlag(xenPageTablePageWriteThrough),
		"   U=", e.hasFlag(xenPageTableUserspaceAccessible),
		"   W=", e.hasFlag(xenPageTableWritable),
		"   P=", e.hasFlag(xenPageTablePresent),
	)
}

func (e *pageTableEntry) setFlag(f uintptr) {
	*e = pageTableEntry(uintptr(*e) | f)
}

func (e pageTableEntry) hasFlag(f uintptr) bool {
	return uintptr(e)&f == f
}

func (e pageTableEntry) mfn() mfn {
	return mfn((uintptr(e) & (xenPageAddrMask &^ xenPageMask)) >> xenPageFlagShift)
}

func (e pageTableEntry) vaddr() vaddr {
	return vaddr(e.pfn() << xenPageFlagShift)
}

func (e pageTableEntry) pfn() pfn {
	const (
		m2p xenMachineToPhysicalMap = 0xFFFF800000000000
	)

	return m2p.Get(e.mfn())
}

type xenPageTable uintptr

func (t xenPageTable) Get(i int) pageTableEntry {
	return *(*pageTableEntry)(add(unsafe.Pointer(t), uintptr(i)*ptrSize))
}

func (t xenPageTable) vaddr() vaddr {
	return vaddr(t)
}

func newXenPageTable(vaddr vaddr) xenPageTable {
	return xenPageTable(vaddr)
}

type xenMachineToPhysicalMap uintptr

func (m2p xenMachineToPhysicalMap) Get(mfn mfn) pfn {
	offset := uintptr(mfn) * ptrSize

	return pfn(*(*uintptr)(add(unsafe.Pointer(m2p), offset)))
}

type pageTableLevel int

func (l pageTableLevel) shift() uint64 {
	return 12 + uint64(l)*9
}

// mask returns a mask for the pageTableLevel l.
// It's undefined if l is pageTableLevel4.
func (l pageTableLevel) mask() uint64 {
	return (1 << (l + 1).shift()) - 1
}

const (
	pageTableLevel1 pageTableLevel = iota
	pageTableLevel2
	pageTableLevel3
	pageTableLevel4
)

func (a vaddr) pageTableOffset(level pageTableLevel) int {
	return int((a >> level.shift()) & (512 - 1))
}

type pfn uint64

func (n pfn) vaddr() vaddr {
	return vaddr(n << 12)
}

func (n pfn) add(v uint64) pfn {
	return n + pfn(v)
}

func (n pfn) mfn() mfn {
	return mfn(_atman_phys_to_machine_mapping[n])
}

type mfn uintptr

func (m mfn) pfn() pfn {
	const (
		m2p xenMachineToPhysicalMap = 0xFFFF800000000000
	)

	return m2p.Get(m)
}

type vaddr uintptr

func (a vaddr) pfn() pfn {
	return pfn((uint64(a) + _PAGESIZE - 1) >> 12)
}
<commit_msg>Add dynamic pageFrameList<commit_after>package runtime

import "unsafe"

const (
	_PAGESIZE = 0x1000
)

var (
	_atman_hypercall_page   [2 * _PAGESIZE]byte
	_atman_shared_info_page [2 * _PAGESIZE]byte

	_atman_start_info  = &xenStartInfo{}
	_atman_shared_info *xenSharedInfo

	_atman_page_frame_list pageFrameList
)

//go:nosplit
func getRandomData(r []byte) {
	extendRandom(r, 0)
}

// env

func gogetenv(key string) string { return "" }

var _cgo_setenv unsafe.Pointer   // pointer to C function
var _cgo_unsetenv unsafe.Pointer // pointer to C function

// signals

const _NSIG = 0

func initsig()                 {}
func sigdisable(uint32)        {}
func sigenable(uint32)         {}
func sigignore(uint32)         {}
func raisebadsignal(sig int32) {}

// net

func netpoll(block bool) *g { return nil }
func netpollinited() bool   { return false }

type xenStartInfo struct {
	Magic          [32]byte
	NrPages        uint64
	SharedInfoAddr uintptr // machine address of share info struct
	SIFFlags       uint32
	_              [4]byte
	StoreMfn       uint64 // machine page number of shared page
	StoreEventchn  uint32
	_              [4]byte
	Console        struct {
		Mfn      mfn    // machine page number of console page
		Eventchn uint32 // event channel
		_        [4]byte
	}
	PageTableBase     vaddr // virtual address of page directory
	NrPageTableFrames uint64
	PageFrameList     uintptr // virtual address of page-frame list
	ModStart          uintptr // virtual address of pre-loaded module
	ModLen            uint64  // size (bytes) of pre-loaded module
	CmdLine           [1024]byte

	// The pfn range here covers both page table and p->m table frames
	FirstP2mPfn uint64 // 1st pfn forming initial P->M table
	NrP2mFrames uint64 // # of pgns forming initial P->M table
}

type xenSharedInfo struct {
	VCPUInfo      [32]vcpuInfo
	EvtchnPending [64]uint64
	EvtchnMask    [64]uint64
	WcVersion     uint32
	WcSec         uint32
	WcNsec        uint32
	_             [4]byte
	Arch          archSharedInfo
}

type archSharedInfo struct {
	MaxPfn                uint64
	PfnToMfnFrameListList uint64
	NmiReason             uint64
	_                     [32]uint64
}

type archVCPUInfo struct {
	CR2 uint64
	_   uint64
}

type vcpuTimeInfo struct {
	Version    uint32
	_          uint32
	TSC        uint64
	SystemNsec uint64
	TSCMul     uint32
	TSCShift   int8
	_          [3]int8
}

type vcpuInfo struct {
	UpcallPending uint8
	UpcallMask    uint8
	_             [6]byte
	PendingSel    uint64
	Arch          archVCPUInfo
	Time          vcpuTimeInfo
}

func atmaninit() {
	println("Atman OS")
	println("     ptr_size: ", ptrSize)
	println("   start_info: ", _atman_start_info)
	println("        magic: ", string(_atman_start_info.Magic[:]))
	println("     nr_pages: ", _atman_start_info.NrPages)
	println("  shared_info: ", _atman_start_info.SharedInfoAddr)
	println("   siff_flags: ", _atman_start_info.SIFFlags)
	println("    store_mfn: ", _atman_start_info.StoreMfn)
	println("    store_evc: ", _atman_start_info.StoreEventchn)
	println("  console_mfn: ", _atman_start_info.Console.Mfn)
	println("  console_evc: ", _atman_start_info.Console.Eventchn)
	println("      pt_base: ", _atman_start_info.PageTableBase)
	println(" nr_pt_frames: ", _atman_start_info.NrPageTableFrames)
	println("     pfn_list: ", _atman_start_info.PageFrameList)
	println("    mod_start: ", _atman_start_info.ModStart)
	println("      mod_len: ", _atman_start_info.ModLen)
	println("     cmd_line: ", _atman_start_info.CmdLine[:])
	println("    first_pfn: ", _atman_start_info.FirstP2mPfn)
	println("nr_p2m_frames: ", _atman_start_info.NrP2mFrames)

	initSlice(
		unsafe.Pointer(&_atman_page_frame_list),
		unsafe.Pointer(_atman_start_info.PageFrameList),
		int(_atman_start_info.NrPages),
	)

	_atman_console.init()

	println("mapping _atman_start_info")
	mapSharedInfo(_atman_start_info.SharedInfoAddr)

	_atman_mm.init()
}

func mapSharedInfo(vaddr uintptr) {
	pageAddr := round(
		uintptr(unsafe.Pointer(&_atman_shared_info_page[0])),
		_PAGESIZE,
	)

	ret := HYPERVISOR_update_va_mapping(
		pageAddr,
		vaddr|7,
		2, // UVMF_INVLPG: flush only one entry
	)

	if ret != 0 {
		println("HYPERVISOR_update_va_mapping returned ", ret)
		panic("HYPERVISOR_update_va_mapping failed")
	}

	_atman_shared_info = (*xenSharedInfo)(unsafe.Pointer(pageAddr))
}

// initSlice makes the slice s point to array,
// with a length and capacity of len.
func initSlice(s, array unsafe.Pointer, len int) {
	sp := (*slice)(s)
	sp.array = array
	sp.len = len
	sp.cap = len
}

// memory management

// pageFrameList is an array of machine frame numbers
// indexed by page frame numbers.
type pageFrameList []mfn

func (l pageFrameList) Get(n pfn) mfn {
	return l[int(n)]
}

// Entry in level 3, 2, or 1 page table.
//
// - 63 if set means No execute (NX)
// - 51-13 the machine frame number
// - 12 available for guest
// - 11 available for guest
// - 10 available for guest
// - 9 available for guest
// - 8 global
// - 7 PAT (PSE is disabled, must use hypercall to make 4MB or 2MB pages)
// - 6 dirty
// - 5 accessed
// - 4 page cached disabled
// - 3 page write through
// - 2 userspace accessible
// - 1 writeable
// - 0 present
type pageTableEntry uintptr

const (
	xenPageTablePresent = 1 << iota
	xenPageTableWritable
	xenPageTableUserspaceAccessible
	xenPageTablePageWriteThrough
	xenPageTablePageCacheDisabled
	xenPageTableAccessed
	xenPageTableDirty
	xenPageTablePAT
	xenPageTableGlobal
	xenPageTableGuest1
	xenPageTableGuest2
	xenPageTableGuest3
	xenPageTableGuest4
	xenPageTableNoExecute = 1 << 63

	xenPageAddrMask  = 1<<52 - 1
	xenPageMask      = 1<<12 - 1
	xenPageFlagShift = 12

	PTE_PAGE_FLAGS       = xenPageTablePresent | xenPageTableWritable | xenPageTableUserspaceAccessible | xenPageTableAccessed
	PTE_PAGE_TABLE_FLAGS = xenPageTablePresent | xenPageTableUserspaceAccessible | xenPageTableAccessed | xenPageTableDirty
	PTE_TEMP             = xenPageTableGuest1
)

func (e pageTableEntry) debug() {
	println(
		"PTE<", unsafe.Pointer(e), ">:",
		" MFN=", e.mfn(),
		"  NX=", e.hasFlag(xenPageTableNoExecute),
		"   G=", e.hasFlag(xenPageTableGlobal),
		" PAT=", e.hasFlag(xenPageTablePAT),
		" DIR=", e.hasFlag(xenPageTableDirty),
		"   A=", e.hasFlag(xenPageTableAccessed),
		" PCD=", e.hasFlag(xenPageTablePageCacheDisabled),
		" PWT=", e.hasFlag(xenPageTablePageWriteThrough),
		"   U=", e.hasFlag(xenPageTableUserspaceAccessible),
		"   W=", e.hasFlag(xenPageTableWritable),
		"   P=", e.hasFlag(xenPageTablePresent),
	)
}

func (e *pageTableEntry) setFlag(f uintptr) {
	*e = pageTableEntry(uintptr(*e) | f)
}

func (e pageTableEntry) hasFlag(f uintptr) bool {
	return uintptr(e)&f == f
}

func (e pageTableEntry) mfn() mfn {
	return mfn((uintptr(e) & (xenPageAddrMask &^ xenPageMask)) >> xenPageFlagShift)
}

func (e pageTableEntry) vaddr() vaddr {
	return vaddr(e.pfn() << xenPageFlagShift)
}

func (e pageTableEntry) pfn() pfn {
	const (
		m2p xenMachineToPhysicalMap = 0xFFFF800000000000
	)

	return m2p.Get(e.mfn())
}

type xenPageTable uintptr

func (t xenPageTable) Get(i int) pageTableEntry {
	return *(*pageTableEntry)(add(unsafe.Pointer(t), uintptr(i)*ptrSize))
}

func (t xenPageTable) vaddr() vaddr {
	return vaddr(t)
}

func newXenPageTable(vaddr vaddr) xenPageTable {
	return xenPageTable(vaddr)
}

type xenMachineToPhysicalMap uintptr

func (m2p xenMachineToPhysicalMap) Get(mfn mfn) pfn {
	offset := uintptr(mfn) * ptrSize

	return pfn(*(*uintptr)(add(unsafe.Pointer(m2p), offset)))
}

type pageTableLevel int

func (l pageTableLevel) shift() uint64 {
	return 12 + uint64(l)*9
}

// mask returns a mask for the pageTableLevel l.
// It's undefined if l is pageTableLevel4.
func (l pageTableLevel) mask() uint64 {
	return (1 << (l + 1).shift()) - 1
}

const (
	pageTableLevel1 pageTableLevel = iota
	pageTableLevel2
	pageTableLevel3
	pageTableLevel4
)

func (a vaddr) pageTableOffset(level pageTableLevel) int {
	return int((a >> level.shift()) & (512 - 1))
}

type pfn uint64

func (n pfn) vaddr() vaddr {
	return vaddr(n << 12)
}

func (n pfn) add(v uint64) pfn {
	return n + pfn(v)
}

func (n pfn) mfn() mfn {
	return _atman_page_frame_list.Get(n)
}

type mfn uintptr

func (m mfn) pfn() pfn {
	const (
		m2p xenMachineToPhysicalMap = 0xFFFF800000000000
	)

	return m2p.Get(m)
}

type vaddr uintptr

func (a vaddr) pfn() pfn {
	return pfn((uint64(a) + _PAGESIZE - 1) >> 12)
}
<|endoftext|>


<commit_before>package parser

import "github.com/qiniu/logkit/utils"

// ModeUsages 用途说明
var ModeUsages = []utils.KeyValue{
	{TypeJson, "json 格式解析"},
	{TypeNginx, "nginx 日志解析"},
	{TypeGrok, "grok 方式解析"},
	{TypeCSV, "csv 格式日志解析"},
	{TypeRaw, "raw 原始日志按行解析"},
	{TypeLogv1, "qiniulog 七牛日志库解析"},
	{TypeKafkaRest, "kafkarest 日志格式解析"},
	{TypeEmpty, "empty 通过解析清空数据"},
}

var ModeKeyOptions = map[string][]utils.Option{
	TypeJson: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeNginx: {
		{
			KeyName:      NginxConfPath,
			ChooseOnly:   false,
			Default:      "/opt/nginx/conf/nginx.conf",
			DefaultNoUse: true,
			Description:  "nginx配置路径(nginx_log_format_path)",
		},
		{
			KeyName:      NginxLogFormat,
			ChooseOnly:   false,
			Default:      "main",
			DefaultNoUse: true,
			Description:  "nginx日志格式名称(nginx_log_format_name)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      NginxSchema,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "指定nginx字段类型(nginx_schema)",
		},
		{
			KeyName:      NginxFormatRegex,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "直接通过正则表达式解析(nginx_log_format_regex)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeGrok: {
		{
			KeyName:      KeyGrokPatterns,
			ChooseOnly:   false,
			Default:      "%{COMMON_LOG_FORMAT}",
			DefaultNoUse: true,
			Description:  "匹配日志的grok表达式(grok_patterns)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyGrokMode,
			ChooseOnly:    true,
			ChooseOptions: []string{"oneline", ModeMulti},
			Default:       "oneline",
			DefaultNoUse:  false,
			Description:   "grok单行多行模式(grok_mode)",
		},
		{
			KeyName:      KeyGrokCustomPatternFiles,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern 文件路径(grok_custom_pattern_files)",
		},
		{
			KeyName:      KeyGrokCustomPatterns,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern (grok_custom_patterns)",
		},
		{
			KeyName:    KeyTimeZoneOffset,
			ChooseOnly: true,
			Default:    "0",
			ChooseOptions: []string{"0", "-1", "-2", "-3", "-4",
				"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
				"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
			DefaultNoUse: false,
			Description:  "时区偏移量(timezone_offset)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},

	TypeCSV: {
		{
			KeyName:      KeyCSVSchema,
			ChooseOnly:   false,
			Default:      "abc string,xyz long,data1 string,data2 float",
			DefaultNoUse: true,
			Description:  "csv格式的字段类型(csv_schema)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyCSVSplitter,
			ChooseOnly:   false,
			Default:      ",",
			DefaultNoUse: false,
			Description:  "csv分隔符(csv_splitter)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:    KeyTimeZoneOffset,
			ChooseOnly: true,
			Default:    "0",
			ChooseOptions: []string{"0", "-1", "-2", "-3", "-4",
				"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
				"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
			DefaultNoUse: false,
			Description:  "时区偏移量(timezone_offset)",
		},
	},
	TypeRaw: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyTimestamp,
			ChooseOnly:    true,
			ChooseOptions: []string{"true", "false"},
			Default:       "true",
			DefaultNoUse:  false,
			Description:   "数据附带时间戳(timestamp)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeLogv1: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:      KeyQiniulogPrefix,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "日志前缀(qiniulog_prefix)",
		},
		{
			KeyName:      KeyLogHeaders,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "七牛日志格式顺序(qiniulog_log_headers)",
		},
	},
	TypeKafkaRest: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeEmpty: {},
}

// SampleLogs 样例日志，用于前端界面试玩解析器
var SampleLogs = map[string]string{
	TypeNginx: `110.110.101.101 - - [21/Mar/2017:18:14:17 +0800] "GET /files/yyyysx HTTP/1.1" 206 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com llEAAFgmnoIa3q0U "0.040" 0.040 760 "-" "-" - - QCloud
1.11.1.1 - - [25/Mar/2017:18:14:17 +0800] "GET /files HTTP/1.1" 200 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com sfvfv123axs "0.040" 0.040 760 "-" "-" - - QCloud`,
	TypeGrok: `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
123.45.12.1 user-identifier bob [10/Oct/2013:13:55:36 -0700] "GET /hello.gif HTTP/1.0" 200 2326`,
	TypeJson: `{"a":"b","c":1,"d":1.1}`,
	TypeCSV: `a,123,bcd,1.2
xsxs,456,asv,5.12`,
	TypeRaw: `raw log1[05-May-2017 13:44:39]  [pool log] pid 4109
script_filename = /data/html/
[0x00007fec119d1720] curl_exec() /data/html/xyframework/base.go:123
[05-May-2017 13:45:39]  [pool log] pid 4108`,
	TypeLogv1:     `2017/08/11 10:24:17 [WARN][github.com/qiniu/logkit/mgr] runner.go:398: Runner[byds] sender pandora.sender.31 closed`,
	TypeKafkaRest: `[2016-12-05 03:35:20,682] INFO 172.16.16.191 - - [05/Dec/2016:03:35:20 +0000] "POST /topics/VIP_VvBVy0tuMPPspm1A_0000000000 HTTP/1.1" 200 101640  46 (io.confluent.rest-utils.requests)`,
	TypeEmpty:     "empty 通过解析清空数据",
}
<commit_msg>fix sample log<commit_after>package parser

import "github.com/qiniu/logkit/utils"

// ModeUsages 用途说明
var ModeUsages = []utils.KeyValue{
	{TypeJson, "json 格式解析"},
	{TypeNginx, "nginx 日志解析"},
	{TypeGrok, "grok 方式解析"},
	{TypeCSV, "csv 格式日志解析"},
	{TypeRaw, "raw 原始日志按行解析"},
	{TypeLogv1, "qiniulog 七牛日志库解析"},
	{TypeKafkaRest, "kafkarest 日志格式解析"},
	{TypeEmpty, "empty 通过解析清空数据"},
}

var ModeKeyOptions = map[string][]utils.Option{
	TypeJson: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeNginx: {
		{
			KeyName:      NginxConfPath,
			ChooseOnly:   false,
			Default:      "/opt/nginx/conf/nginx.conf",
			DefaultNoUse: true,
			Description:  "nginx配置路径(nginx_log_format_path)",
		},
		{
			KeyName:      NginxLogFormat,
			ChooseOnly:   false,
			Default:      "main",
			DefaultNoUse: true,
			Description:  "nginx日志格式名称(nginx_log_format_name)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      NginxSchema,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "指定nginx字段类型(nginx_schema)",
		},
		{
			KeyName:      NginxFormatRegex,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "直接通过正则表达式解析(nginx_log_format_regex)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeGrok: {
		{
			KeyName:      KeyGrokPatterns,
			ChooseOnly:   false,
			Default:      "%{COMMON_LOG_FORMAT}",
			DefaultNoUse: true,
			Description:  "匹配日志的grok表达式(grok_patterns)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyGrokMode,
			ChooseOnly:    true,
			ChooseOptions: []string{"oneline", ModeMulti},
			Default:       "oneline",
			DefaultNoUse:  false,
			Description:   "grok单行多行模式(grok_mode)",
		},
		{
			KeyName:      KeyGrokCustomPatternFiles,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern 文件路径(grok_custom_pattern_files)",
		},
		{
			KeyName:      KeyGrokCustomPatterns,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern (grok_custom_patterns)",
		},
		{
			KeyName:    KeyTimeZoneOffset,
			ChooseOnly: true,
			Default:    "0",
			ChooseOptions: []string{"0", "-1", "-2", "-3", "-4",
				"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
				"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
			DefaultNoUse: false,
			Description:  "时区偏移量(timezone_offset)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},

	TypeCSV: {
		{
			KeyName:      KeyCSVSchema,
			ChooseOnly:   false,
			Default:      "abc string,xyz long,data1 string,data2 float",
			DefaultNoUse: true,
			Description:  "csv格式的字段类型(csv_schema)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyCSVSplitter,
			ChooseOnly:   false,
			Default:      ",",
			DefaultNoUse: false,
			Description:  "csv分隔符(csv_splitter)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:    KeyTimeZoneOffset,
			ChooseOnly: true,
			Default:    "0",
			ChooseOptions: []string{"0", "-1", "-2", "-3", "-4",
				"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
				"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
			DefaultNoUse: false,
			Description:  "时区偏移量(timezone_offset)",
		},
	},
	TypeRaw: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyTimestamp,
			ChooseOnly:    true,
			ChooseOptions: []string{"true", "false"},
			Default:       "true",
			DefaultNoUse:  false,
			Description:   "数据附带时间戳(timestamp)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeLogv1: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:      KeyQiniulogPrefix,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "日志前缀(qiniulog_prefix)",
		},
		{
			KeyName:      KeyLogHeaders,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "七牛日志格式顺序(qiniulog_log_headers)",
		},
	},
	TypeKafkaRest: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeEmpty: {},
}

// SampleLogs 样例日志，用于前端界面试玩解析器
var SampleLogs = map[string]string{
	TypeNginx: `110.110.101.101 - - [21/Mar/2017:18:14:17 +0800] "GET /files/yyyysx HTTP/1.1" 206 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com llEAAFgmnoIa3q0U "0.040" 0.040 760 "-" "-" - - QCloud
1.11.1.1 - - [25/Mar/2017:18:14:17 +0800] "GET /files HTTP/1.1" 200 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com sfvfv123axs "0.040" 0.040 760 "-" "-" - - QCloud`,
	TypeGrok: `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
123.45.12.1 user-identifier bob [10/Oct/2013:13:55:36 -0700] "GET /hello.gif HTTP/1.0" 200 2326`,
	TypeJson: `{"a":"b","c":1,"d":1.1}`,
	TypeCSV: `a,123,bcd,1.2
xsxs,456,asv,5.12`,
	TypeRaw: `raw log1[05-May-2017 13:44:39]  [pool log] pid 4109
script_filename = /data/html/
[0x00007fec119d1720] curl_exec() /data/html/xyframework/base.go:123
[05-May-2017 13:45:39]  [pool log] pid 4108`,
	TypeLogv1:     `2016/10/20 17:30:21.433423 [GE2owHck-Y4IWJHS][WARN] github.com/qiniu/http/rpcutil.v1/rpc_util.go:203: E18102: The specified repo does not exist under the provided appid ~`,
	TypeKafkaRest: `[2016-12-05 03:35:20,682] INFO 172.16.16.191 - - [05/Dec/2016:03:35:20 +0000] "POST /topics/VIP_VvBVy0tuMPPspm1A_0000000000 HTTP/1.1" 200 101640  46 (io.confluent.rest-utils.requests)`,
	TypeEmpty:     "empty 通过解析清空数据",
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"sync"
)

type httpPkg struct{}

func (httpPkg) Get(url string) {}

var http httpPkg

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Decrement the counter when the goroutine completes.
			defer wg.Done()
			// Fetch the URL.
			http.Get(url)
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Println("Only once")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<commit_msg>sync: use example.com for url in ExampleWaitGroup<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"sync"
)

type httpPkg struct{}

func (httpPkg) Get(url string) {}

var http httpPkg

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.example.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Decrement the counter when the goroutine completes.
			defer wg.Done()
			// Fetch the URL.
			http.Get(url)
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Println("Only once")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"path/filepath"
	//"runtime/debug"
	"tes"
	//"tes/ga4gh"
	"log"
	"tes/scheduler"
	"tes/scheduler/condor"
	_ "tes/scheduler/dumblocal"
	_ "tes/scheduler/local"
	_ "tes/scheduler/openstack"
	"tes/server"
	"tes/server/proto"
)

func main() {
	httpPort := flag.String("port", "8000", "HTTP Port")
	rpcPort := flag.String("rpc", "9090", "TCP+RPC Port")
	storageDirArg := flag.String("storage", "storage", "Storage Dir")
	sharedDirArg := flag.String("shared", "", "Shared File System")
	s3Arg := flag.String("s3", "", "Use S3 object store")
	taskDB := flag.String("db", "ga4gh_tasks.db", "Task DB File")
	configFile := flag.String("config", "", "Config File")

	flag.Parse()

	dir, _ := filepath.Abs(os.Args[0])
	contentDir := filepath.Join(dir, "..", "..", "share")

	config := tes.Config{}
	tes.LoadConfigOrExit(*configFile, &config)

	if *storageDirArg != "" {
		//server meta-data
		storageDir, _ := filepath.Abs(*storageDirArg)
		fs := &ga4gh_task_ref.StorageConfig{Config: map[string]string{}}
		fs.Protocol = "fs"
		fs.Config["basedir"] = storageDir
		config.Storage = append(config.Storage, fs)
	}
	if *s3Arg != "" {
		fs := &ga4gh_task_ref.StorageConfig{Config: map[string]string{}}
		fs.Protocol = "s3"
		fs.Config["endpoint"] = *s3Arg
		config.Storage = append(config.Storage, fs)
	}
	if *sharedDirArg != "" {
		fs := &ga4gh_task_ref.StorageConfig{Config: map[string]string{}}
		fs.Protocol = "file"
		fs.Config["dirs"] = *sharedDirArg
		config.Storage = append(config.Storage, fs)
	}
	log.Printf("Config: %v\n", config)

	//setup GRPC listener
	// TODO if another process has the db open, this will block and it is really
	//      confusing when you don't realize you have the db locked in another
	//      terminal somewhere. Would be good to timeout on startup here.
	taski := tes_server.NewTaskBolt(*taskDB, config.ServerConfig)

	server := tes_server.NewGA4GHServer()
	server.RegisterTaskServer(taski)
	server.RegisterScheduleServer(taski)
	server.Start(*rpcPort)

	// TODO worker will stay alive if the parent process panics
	//go scheduler.StartScheduling(taski, local.NewScheduler(4))
	//go scheduler.StartScheduling(taski, openstack.NewScheduler(4, config.Schedulers.Openstack))
	go scheduler.StartScheduling(taski, condor.NewScheduler(config.Schedulers.Condor))

	tes_server.StartHttpProxy(*rpcPort, *httpPort, contentDir)
}
<commit_msg>scheduler CLI arg<commit_after>package main

import (
	"flag"
	"os"
	"path/filepath"
	//"runtime/debug"
	"tes"
	//"tes/ga4gh"
	"log"
	"tes/scheduler"
	"tes/scheduler/condor"
	"tes/scheduler/dumblocal"
	"tes/scheduler/local"
	"tes/scheduler/openstack"
	"tes/server"
	"tes/server/proto"
)

func main() {
	httpPort := flag.String("port", "8000", "HTTP Port")
	rpcPort := flag.String("rpc", "9090", "TCP+RPC Port")
	storageDirArg := flag.String("storage", "storage", "Storage Dir")
	sharedDirArg := flag.String("shared", "", "Shared File System")
	s3Arg := flag.String("s3", "", "Use S3 object store")
	taskDB := flag.String("db", "ga4gh_tasks.db", "Task DB File")
	schedArg := flag.String("sched", "local", "Scheduler")
	configFile := flag.String("config", "", "Config File")

	flag.Parse()

	dir, _ := filepath.Abs(os.Args[0])
	contentDir := filepath.Join(dir, "..", "..", "share")

	config := tes.Config{}
	tes.LoadConfigOrExit(*configFile, &config)

	if *storageDirArg != "" {
		//server meta-data
		storageDir, _ := filepath.Abs(*storageDirArg)
		fs := &ga4gh_task_ref.StorageConfig{Config: map[string]string{}}
		fs.Protocol = "fs"
		fs.Config["basedir"] = storageDir
		config.Storage = append(config.Storage, fs)
	}
	if *s3Arg != "" {
		fs := &ga4gh_task_ref.StorageConfig{Config: map[string]string{}}
		fs.Protocol = "s3"
		fs.Config["endpoint"] = *s3Arg
		config.Storage = append(config.Storage, fs)
	}
	if *sharedDirArg != "" {
		fs := &ga4gh_task_ref.StorageConfig{Config: map[string]string{}}
		fs.Protocol = "file"
		fs.Config["dirs"] = *sharedDirArg
		config.Storage = append(config.Storage, fs)
	}
	log.Printf("Config: %v\n", config)

	//setup GRPC listener
	// TODO if another process has the db open, this will block and it is really
	//      confusing when you don't realize you have the db locked in another
	//      terminal somewhere. Would be good to timeout on startup here.
	taski := tes_server.NewTaskBolt(*taskDB, config.ServerConfig)

	server := tes_server.NewGA4GHServer()
	server.RegisterTaskServer(taski)
	server.RegisterScheduleServer(taski)
	server.Start(*rpcPort)

	var sched scheduler.Scheduler
	switch *schedArg {
	case "local":
		// TODO worker will stay alive if the parent process panics
		sched = local.NewScheduler(4)
	case "condor":
		sched = condor.NewScheduler(config.Schedulers.Condor)
	case "openstack":
	  sched = openstack.NewScheduler(4, config.Schedulers.Openstack)
	case "dumblocal":
	  sched = dumblocal.NewScheduler(4)
	default:
		log.Printf("Error: unknown scheduler %s", *schedArg)
		return
	}
	go scheduler.StartScheduling(taski, sched)

	tes_server.StartHttpProxy(*rpcPort, *httpPort, contentDir)
}
<|endoftext|>


<commit_before>package testique

import (
    "net"
    "flag"
    "strings"
    "warserver/logger"
    "warserver/PortMgmt"
    "warserver/connection"
)

const (
    RECV_BUF_LEN = 1024
)

var serverPort PortMgmt.PortInfo

var testHandlers = make(map [string]func(msg string) string)

func Main() {
    portString := flag.String("port", ":5269", "Test server port")
    flag.Parse()

    serverPort = PortMgmt.NewPortInfo(*portString)

    setupHandlers();

    // might as well go straight to std out for a testserver
    logger.SetupLoggerHelper("/dev/stdout")

    logger.Infof("Accepting testserver connections on port %s", serverPort)
    handleConnections(serverPort.Port)
}

func setupHandlers() {
    testHandlers["new"] = testNewGame
    testHandlers["view"] = testViewWorld
}

func testNewGame(msg string) string {
    return "new:success"
}

func testViewWorld(msg string) string {
    unit := "{\"unit\":{\"loc\":{\"x\": 1, \"y\": 2}," +
                       "\"name\": \"Tank\"," +
                       "\"nation\": \"0\"," +
                       "\"movement\":{" +
                            "\"speeds\":{\"0\":1}," +
                            "\"name\":\"treads\"" +
                                    "}" +
                     "}" +
            "}"
    return "view:success:{\"world\":{\"terrain\":[[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]]," +
                                    "\"units\": [" + unit + "]}}"
}

}

func handleConnections(port string) {
    ln, err := net.Listen("tcp", port)
    if err != nil {
        logger.Fatalf("Could not open socket for listening: %s", err)
    }
    for {
        conn, err := ln.Accept()
        if err != nil {
            logger.Errorf("Could not accept connection from client: %s", err)
            continue
        }
        sconn := connection.NewSocketConn(conn)
        go serveConnection(sconn)
    }
}

func serveConnection(sock *connection.SocketConn) {
    logger.Info("Serving new test connection")
    for {
        msg, err := sock.Read()
        if err != nil {
            // Should gracefully shutdown someday on EOF or UnexpectedEOF
            logger.Errorf("Error reading from client: %s", err)
            break
        }
        resp := parseMessage(string(msg))
        err = sock.Write([]byte(resp))
        if err != nil {
            logger.Errorf("Error writing to client: %s", err)
            break
        }
    }
}

func parseMessage(msg string) string {
    cmds := strings.SplitN(msg, ":", 2)
    if len(cmds) == 2 {
        if fun, ok := testHandlers[cmds[0]]; ok {
            return fun(cmds[1])
        } else {
            logger.Warnf("Unrecognized command: %s", cmds[0])
            return "unrecognized:"
        }
    }
    return "malformed:"
}<commit_msg>Handle all move requests as successes<commit_after>package testique

import (
    "net"
    "flag"
    "strings"
    "warserver/logger"
    "warserver/PortMgmt"
    "warserver/connection"
)

const (
    RECV_BUF_LEN = 1024
)

var serverPort PortMgmt.PortInfo

var testHandlers = make(map [string]func(msg string) string)

func Main() {
    portString := flag.String("port", ":5269", "Test server port")
    flag.Parse()

    serverPort = PortMgmt.NewPortInfo(*portString)

    setupHandlers();

    // might as well go straight to std out for a testserver
    logger.SetupLoggerHelper("/dev/stdout")

    logger.Infof("Accepting testserver connections on port %s", serverPort)
    handleConnections(serverPort.Port)
}

func setupHandlers() {
    testHandlers["new"] = testNewGame
    testHandlers["view"] = testViewWorld
    testHandlers["move"] = testMoveUnit
}

func testNewGame(msg string) string {
    return "new:success"
}

func testViewWorld(msg string) string {
    unit := "{\"unit\":{\"loc\":{\"x\": 1, \"y\": 2}," +
                       "\"name\": \"Tank\"," +
                       "\"nation\": \"0\"," +
                       "\"movement\":{" +
                            "\"speeds\":{\"0\":1}," +
                            "\"name\":\"treads\"" +
                                    "}" +
                     "}" +
            "}"
    return "view:success:{\"world\":{\"terrain\":[[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]," +
                                                 "[0,0,0,0,0]]," +
                                    "\"units\": [" + unit + "]}}"
}

func testMoveUnit(msg string) string {
    return "move:success"
}

func handleConnections(port string) {
    ln, err := net.Listen("tcp", port)
    if err != nil {
        logger.Fatalf("Could not open socket for listening: %s", err)
    }
    for {
        conn, err := ln.Accept()
        if err != nil {
            logger.Errorf("Could not accept connection from client: %s", err)
            continue
        }
        sconn := connection.NewSocketConn(conn)
        go serveConnection(sconn)
    }
}

func serveConnection(sock *connection.SocketConn) {
    logger.Info("Serving new test connection")
    for {
        msg, err := sock.Read()
        if err != nil {
            // Should gracefully shutdown someday on EOF or UnexpectedEOF
            logger.Errorf("Error reading from client: %s", err)
            break
        }
        resp := parseMessage(string(msg))
        err = sock.Write([]byte(resp))
        if err != nil {
            logger.Errorf("Error writing to client: %s", err)
            break
        }
    }
}

func parseMessage(msg string) string {
    cmds := strings.SplitN(msg, ":", 2)
    if len(cmds) == 2 {
        if fun, ok := testHandlers[cmds[0]]; ok {
            return fun(cmds[1])
        } else {
            logger.Warnf("Unrecognized command: %s", cmds[0])
            return "unrecognized:"
        }
    }
    return "malformed:"
}<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package agent

import (
	"bytes"
	"crypto/rand"
	"errors"
	"net"
	"os"
	"os/exec"
	"strconv"
	"testing"

	"code.google.com/p/go.crypto/ssh"
)

func startAgent(t *testing.T) (client Agent, socket string, cleanup func()) {
	bin, err := exec.LookPath("ssh-agent")
	if err != nil {
		t.Skip("could not find ssh-agent")
	}

	cmd := exec.Command(bin, "-s")
	out, err := cmd.Output()
	if err != nil {
		t.Fatalf("cmd.Output: %v", err)
	}

	/* Output looks like:

		   SSH_AUTH_SOCK=/tmp/ssh-P65gpcqArqvH/agent.15541; export SSH_AUTH_SOCK;
	           SSH_AGENT_PID=15542; export SSH_AGENT_PID;
	           echo Agent pid 15542;
	*/
	fields := bytes.Split(out, []byte(";"))
	line := bytes.SplitN(fields[0], []byte("="), 2)
	line[0] = bytes.TrimLeft(line[0], "\n")
	if string(line[0]) != "SSH_AUTH_SOCK" {
		t.Fatalf("could not find key SSH_AUTH_SOCK in %q", fields[0])
	}
	socket = string(line[1])

	line = bytes.SplitN(fields[2], []byte("="), 2)
	line[0] = bytes.TrimLeft(line[0], "\n")
	if string(line[0]) != "SSH_AGENT_PID" {
		t.Fatalf("could not find key SSH_AGENT_PID in %q", fields[2])
	}
	pidStr := line[1]
	pid, err := strconv.Atoi(string(pidStr))
	if err != nil {
		t.Fatalf("Atoi(%q): %v", pidStr, err)
	}

	conn, err := net.Dial("unix", string(socket))
	if err != nil {
		t.Fatalf("net.Dial: %v", err)
	}

	ac := NewClient(conn)
	return ac, socket, func() {
		proc, _ := os.FindProcess(pid)
		if proc != nil {
			proc.Kill()
		}
		conn.Close()
	}
}

func testAgent(t *testing.T, key interface{}, cert *ssh.Certificate) {
	agent, _, cleanup := startAgent(t)
	defer cleanup()

	testAgentInterface(t, agent, key, cert)
}

func testAgentInterface(t *testing.T, agent Agent, key interface{}, cert *ssh.Certificate) {
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		t.Fatalf("NewSignerFromKey(%T): %v", key, err)
	}
	// The agent should start up empty.
	if keys, err := agent.List(); err != nil {
		t.Fatalf("RequestIdentities: %v", err)
	} else if len(keys) > 0 {
		t.Fatalf("got %d keys, want 0: %v", len(keys), keys)
	}

	// Attempt to insert the key, with certificate if specified.
	var pubKey ssh.PublicKey
	if cert != nil {
		err = agent.Add(key, cert, "comment")
		pubKey = cert
	} else {
		err = agent.Add(key, nil, "comment")
		pubKey = signer.PublicKey()
	}
	if err != nil {
		t.Fatalf("insert(%T): %v", key, err)
	}

	// Did the key get inserted successfully?
	if keys, err := agent.List(); err != nil {
		t.Fatalf("List: %v", err)
	} else if len(keys) != 1 {
		t.Fatalf("got %v, want 1 key", keys)
	} else if keys[0].Comment != "comment" {
		t.Fatalf("key comment: got %v, want %v", keys[0].Comment, "comment")
	} else if !bytes.Equal(keys[0].Blob, pubKey.Marshal()) {
		t.Fatalf("key mismatch")
	}

	// Can the agent make a valid signature?
	data := []byte("hello")
	sig, err := agent.Sign(pubKey, data)
	if err != nil {
		t.Fatalf("Sign(%s): %v", pubKey.Type(), err)
	}

	if err := pubKey.Verify(data, sig); err != nil {
		t.Fatalf("Verify(%s): %v", pubKey.Type(), err)
	}
}

func TestAgent(t *testing.T) {
	for _, keyType := range []string{"rsa", "dsa", "ecdsa"} {
		testAgent(t, testPrivateKeys[keyType], nil)
	}
}

func TestCert(t *testing.T) {
	cert := &ssh.Certificate{
		Key:         testPublicKeys["rsa"],
		ValidBefore: ssh.CertTimeInfinity,
		CertType:    ssh.UserCert,
	}
	cert.SignCert(rand.Reader, testSigners["ecdsa"])

	testAgent(t, testPrivateKeys["rsa"], cert)
}

// netPipe is analogous to net.Pipe, but it uses a real net.Conn, and
// therefore is buffered (net.Pipe deadlocks if both sides start with
// a write.)
func netPipe() (net.Conn, net.Conn, error) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, nil, err
	}
	defer listener.Close()
	c1, err := net.Dial("tcp", listener.Addr().String())
	if err != nil {
		return nil, nil, err
	}

	c2, err := listener.Accept()
	if err != nil {
		c1.Close()
		return nil, nil, err
	}

	return c1, c2, nil
}

func TestAuth(t *testing.T) {
	a, b, err := netPipe()
	if err != nil {
		t.Fatalf("netPipe: %v", err)
	}

	defer a.Close()
	defer b.Close()

	agent, _, cleanup := startAgent(t)
	defer cleanup()

	if err := agent.Add(testPrivateKeys["rsa"], nil, "comment"); err != nil {
		t.Errorf("Add: %v", err)
	}

	serverConf := ssh.ServerConfig{}
	serverConf.AddHostKey(testSigners["rsa"])
	serverConf.PublicKeyCallback = func(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
		if bytes.Equal(key.Marshal(), testPublicKeys["rsa"].Marshal()) {
			return nil, nil
		}

		return nil, errors.New("pubkey rejected")
	}

	go func() {
		conn, _, _, err := ssh.NewServerConn(a, &serverConf)
		if err != nil {
			t.Fatalf("Server: %v", err)
		}
		conn.Close()
	}()

	conf := ssh.ClientConfig{}
	conf.Auth = append(conf.Auth, ssh.PublicKeysCallback(agent.Signers))
	conn, _, _, err := ssh.NewClientConn(b, "", &conf)
	if err != nil {
		t.Fatalf("NewClientConn: %v", err)
	}
	conn.Close()
}

func TestLockClient(t *testing.T) {
	agent, _, cleanup := startAgent(t)
	defer cleanup()
	testLockAgent(agent, t)
}

func testLockAgent(agent Agent, t *testing.T) {
	if err := agent.Add(testPrivateKeys["rsa"], nil, "comment 1"); err != nil {
		t.Errorf("Add: %v", err)
	}
	if err := agent.Add(testPrivateKeys["dsa"], nil, "comment dsa"); err != nil {
		t.Errorf("Add: %v", err)
	}
	if keys, err := agent.List(); err != nil {
		t.Errorf("List: %v", err)
	} else if len(keys) != 2 {
		t.Errorf("Want 2 keys, got %v", keys)
	}

	passphrase := []byte("secret")
	if err := agent.Lock(passphrase); err != nil {
		t.Errorf("Lock: %v", err)
	}

	if keys, err := agent.List(); err != nil {
		t.Errorf("List: %v", err)
	} else if len(keys) != 0 {
		t.Errorf("Want 0 keys, got %v", keys)
	}

	signer, _ := ssh.NewSignerFromKey(testPrivateKeys["rsa"])
	if _, err := agent.Sign(signer.PublicKey(), []byte("hello")); err == nil {
		t.Fatalf("Sign did not fail")
	}

	if err := agent.Remove(signer.PublicKey()); err == nil {
		t.Fatalf("Remove did not fail")
	}

	if err := agent.RemoveAll(); err == nil {
		t.Fatalf("RemoveAll did not fail")
	}

	if err := agent.Unlock(nil); err == nil {
		t.Errorf("Unlock with wrong passphrase succeeded")
	}
	if err := agent.Unlock(passphrase); err != nil {
		t.Errorf("Unlock: %v", err)
	}

	if err := agent.Remove(signer.PublicKey()); err != nil {
		t.Fatalf("Remove: %v", err)
	}

	if keys, err := agent.List(); err != nil {
		t.Errorf("List: %v", err)
	} else if len(keys) != 1 {
		t.Errorf("Want 1 keys, got %v", keys)
	}
}
<commit_msg>go.crypto/ssh: skip tests that start ssh-agent.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package agent

import (
	"bytes"
	"crypto/rand"
	"errors"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"

	"code.google.com/p/go.crypto/ssh"
)

// startAgent executes ssh-agent, and returns a Agent interface to it.
func startAgent(t *testing.T) (client Agent, socket string, cleanup func()) {
	if testing.Short() {
		// ssh-agent is not always available, and the key
		// types supported vary by platform.
		t.Skip("skipping test due to -short")
	}

	bin, err := exec.LookPath("ssh-agent")
	if err != nil {
		t.Skip("could not find ssh-agent")
	}

	cmd := exec.Command(bin, "-s")
	out, err := cmd.Output()
	if err != nil {
		t.Fatalf("cmd.Output: %v", err)
	}

	/* Output looks like:

		   SSH_AUTH_SOCK=/tmp/ssh-P65gpcqArqvH/agent.15541; export SSH_AUTH_SOCK;
	           SSH_AGENT_PID=15542; export SSH_AGENT_PID;
	           echo Agent pid 15542;
	*/
	fields := bytes.Split(out, []byte(";"))
	line := bytes.SplitN(fields[0], []byte("="), 2)
	line[0] = bytes.TrimLeft(line[0], "\n")
	if string(line[0]) != "SSH_AUTH_SOCK" {
		t.Fatalf("could not find key SSH_AUTH_SOCK in %q", fields[0])
	}
	socket = string(line[1])

	line = bytes.SplitN(fields[2], []byte("="), 2)
	line[0] = bytes.TrimLeft(line[0], "\n")
	if string(line[0]) != "SSH_AGENT_PID" {
		t.Fatalf("could not find key SSH_AGENT_PID in %q", fields[2])
	}
	pidStr := line[1]
	pid, err := strconv.Atoi(string(pidStr))
	if err != nil {
		t.Fatalf("Atoi(%q): %v", pidStr, err)
	}

	conn, err := net.Dial("unix", string(socket))
	if err != nil {
		t.Fatalf("net.Dial: %v", err)
	}

	ac := NewClient(conn)
	return ac, socket, func() {
		proc, _ := os.FindProcess(pid)
		if proc != nil {
			proc.Kill()
		}
		conn.Close()
		os.RemoveAll(filepath.Dir(socket))
	}
}

func testAgent(t *testing.T, key interface{}, cert *ssh.Certificate) {
	agent, _, cleanup := startAgent(t)
	defer cleanup()

	testAgentInterface(t, agent, key, cert)
}

func testAgentInterface(t *testing.T, agent Agent, key interface{}, cert *ssh.Certificate) {
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		t.Fatalf("NewSignerFromKey(%T): %v", key, err)
	}
	// The agent should start up empty.
	if keys, err := agent.List(); err != nil {
		t.Fatalf("RequestIdentities: %v", err)
	} else if len(keys) > 0 {
		t.Fatalf("got %d keys, want 0: %v", len(keys), keys)
	}

	// Attempt to insert the key, with certificate if specified.
	var pubKey ssh.PublicKey
	if cert != nil {
		err = agent.Add(key, cert, "comment")
		pubKey = cert
	} else {
		err = agent.Add(key, nil, "comment")
		pubKey = signer.PublicKey()
	}
	if err != nil {
		t.Fatalf("insert(%T): %v", key, err)
	}

	// Did the key get inserted successfully?
	if keys, err := agent.List(); err != nil {
		t.Fatalf("List: %v", err)
	} else if len(keys) != 1 {
		t.Fatalf("got %v, want 1 key", keys)
	} else if keys[0].Comment != "comment" {
		t.Fatalf("key comment: got %v, want %v", keys[0].Comment, "comment")
	} else if !bytes.Equal(keys[0].Blob, pubKey.Marshal()) {
		t.Fatalf("key mismatch")
	}

	// Can the agent make a valid signature?
	data := []byte("hello")
	sig, err := agent.Sign(pubKey, data)
	if err != nil {
		t.Fatalf("Sign(%s): %v", pubKey.Type(), err)
	}

	if err := pubKey.Verify(data, sig); err != nil {
		t.Fatalf("Verify(%s): %v", pubKey.Type(), err)
	}
}

func TestAgent(t *testing.T) {
	for _, keyType := range []string{"rsa", "dsa", "ecdsa"} {
		testAgent(t, testPrivateKeys[keyType], nil)
	}
}

func TestCert(t *testing.T) {
	cert := &ssh.Certificate{
		Key:         testPublicKeys["rsa"],
		ValidBefore: ssh.CertTimeInfinity,
		CertType:    ssh.UserCert,
	}
	cert.SignCert(rand.Reader, testSigners["ecdsa"])

	testAgent(t, testPrivateKeys["rsa"], cert)
}

// netPipe is analogous to net.Pipe, but it uses a real net.Conn, and
// therefore is buffered (net.Pipe deadlocks if both sides start with
// a write.)
func netPipe() (net.Conn, net.Conn, error) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, nil, err
	}
	defer listener.Close()
	c1, err := net.Dial("tcp", listener.Addr().String())
	if err != nil {
		return nil, nil, err
	}

	c2, err := listener.Accept()
	if err != nil {
		c1.Close()
		return nil, nil, err
	}

	return c1, c2, nil
}

func TestAuth(t *testing.T) {
	a, b, err := netPipe()
	if err != nil {
		t.Fatalf("netPipe: %v", err)
	}

	defer a.Close()
	defer b.Close()

	agent, _, cleanup := startAgent(t)
	defer cleanup()

	if err := agent.Add(testPrivateKeys["rsa"], nil, "comment"); err != nil {
		t.Errorf("Add: %v", err)
	}

	serverConf := ssh.ServerConfig{}
	serverConf.AddHostKey(testSigners["rsa"])
	serverConf.PublicKeyCallback = func(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
		if bytes.Equal(key.Marshal(), testPublicKeys["rsa"].Marshal()) {
			return nil, nil
		}

		return nil, errors.New("pubkey rejected")
	}

	go func() {
		conn, _, _, err := ssh.NewServerConn(a, &serverConf)
		if err != nil {
			t.Fatalf("Server: %v", err)
		}
		conn.Close()
	}()

	conf := ssh.ClientConfig{}
	conf.Auth = append(conf.Auth, ssh.PublicKeysCallback(agent.Signers))
	conn, _, _, err := ssh.NewClientConn(b, "", &conf)
	if err != nil {
		t.Fatalf("NewClientConn: %v", err)
	}
	conn.Close()
}

func TestLockClient(t *testing.T) {
	agent, _, cleanup := startAgent(t)
	defer cleanup()
	testLockAgent(agent, t)
}

func testLockAgent(agent Agent, t *testing.T) {
	if err := agent.Add(testPrivateKeys["rsa"], nil, "comment 1"); err != nil {
		t.Errorf("Add: %v", err)
	}
	if err := agent.Add(testPrivateKeys["dsa"], nil, "comment dsa"); err != nil {
		t.Errorf("Add: %v", err)
	}
	if keys, err := agent.List(); err != nil {
		t.Errorf("List: %v", err)
	} else if len(keys) != 2 {
		t.Errorf("Want 2 keys, got %v", keys)
	}

	passphrase := []byte("secret")
	if err := agent.Lock(passphrase); err != nil {
		t.Errorf("Lock: %v", err)
	}

	if keys, err := agent.List(); err != nil {
		t.Errorf("List: %v", err)
	} else if len(keys) != 0 {
		t.Errorf("Want 0 keys, got %v", keys)
	}

	signer, _ := ssh.NewSignerFromKey(testPrivateKeys["rsa"])
	if _, err := agent.Sign(signer.PublicKey(), []byte("hello")); err == nil {
		t.Fatalf("Sign did not fail")
	}

	if err := agent.Remove(signer.PublicKey()); err == nil {
		t.Fatalf("Remove did not fail")
	}

	if err := agent.RemoveAll(); err == nil {
		t.Fatalf("RemoveAll did not fail")
	}

	if err := agent.Unlock(nil); err == nil {
		t.Errorf("Unlock with wrong passphrase succeeded")
	}
	if err := agent.Unlock(passphrase); err != nil {
		t.Errorf("Unlock: %v", err)
	}

	if err := agent.Remove(signer.PublicKey()); err != nil {
		t.Fatalf("Remove: %v", err)
	}

	if keys, err := agent.List(); err != nil {
		t.Errorf("List: %v", err)
	} else if len(keys) != 1 {
		t.Errorf("Want 1 keys, got %v", keys)
	}
}
<|endoftext|>


<commit_before>// Package router provides a simple router to match URL's
// with Controllers/Services/etc. Support for parameters
// and options.
package router

import (
	"errors"
)

// Route struct contains all the information for the route
// including the Controller and the Method to call
type Route struct {
	Pattern string
	Action  string
	Method  string
}

// Router struct stores all the routes configured using
// a simple map of the url to the respective Route struct
// TODO improve mapping
type Router struct {
	Routes map[string]*Route
}

// Router Handler method, receives a url and return the
// associated route
func (router *Router) GetRoute(url string) (*Route, error) {

	// @TODO pattern matching
	route, found := router.Routes[url]
	var err error
	if !found {
		err = errors.New("Route not found:" + url)
	}

	return route, err
}
<commit_msg>add Router Interface<commit_after>// Package router provides a simple router to match URL's
// with Controllers/Services/etc. Support for parameters
// and options.
package router

import (
	"errors"
)

type RouterInterface interface {
	GetRoute(url string) (*Route, error)
}

// Route struct contains all the information for the route
// including the Controller and the Method to call
type Route struct {
	Pattern string
	Action  string
	Method  string
}

// Router struct stores all the routes configured using
// a simple map of the url to the respective Route struct
// TODO improve mapping
type Router struct {
	Routes map[string]*Route
}

// Router Handler method, receives a url and return the
// associated route
func (router *Router) GetRoute(url string) (*Route, error) {

	// @TODO pattern matching
	route, found := router.Routes[url]
	var err error
	if !found {
		err = errors.New("Route not found:" + url)
	}

	return route, err
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies  database.Database
	missingJobIDs linkeddb.LinkedDB
}

func newState(db database.Database) *state {
	return &state{
		runnableJobIDs: linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		jobs:           prefixdb.New(jobsKey, db),
		dependencies:   prefixdb.New(dependenciesKey, db),
		missingJobIDs:  linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}
}

// AddRunnableJob adds [jobID] to the runnable queue
func (ps *state) AddRunnableJob(jobID ids.ID) error {
	return ps.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (ps *state) HasRunnableJob() (bool, error) {
	isEmpty, err := ps.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (ps *state) RemoveRunnableJob() (Job, error) {
	key, err := ps.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := ps.runnableJobIDs.Delete(key); err != nil {
		return nil, err
	}

	jobBytes, err := ps.jobs.Get(key)
	if err != nil {
		return nil, err
	}
	job, err := ps.parser.Parse(jobBytes)
	if err != nil {
		return nil, err
	}
	return job, ps.jobs.Delete(key)
}

// PutJob adds the job to the queue
func (ps *state) PutJob(job Job) error {
	id := job.ID()
	return ps.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (ps *state) HasJob(id ids.ID) (bool, error) {
	return ps.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (ps *state) GetJob(id ids.ID) (Job, error) {
	jobBytes, err := ps.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	return ps.parser.Parse(jobBytes)
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (ps *state) AddDependency(dependency, dependent ids.ID) error {
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	if err := dependentsDB.Put(dependent[:], nil); err != nil {
		return err
	}
	return dependencyDB.Close()
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (ps *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)

	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			_ = dependencyDB.Close()
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			_ = dependencyDB.Close()
			return nil, err
		}
		dependents = append(dependents, dependent)
	}

	if err := dependencyDB.Close(); err != nil {
		return nil, err
	}
	return dependents, iterator.Error()
}

func (ps *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := ps.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}
<commit_msg>added additional caching to the jobs queue<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
)

const (
	cacheSize = 1024
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(db database.Database) *state {
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: cacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}
}

// AddRunnableJob adds [jobID] to the runnable queue
func (ps *state) AddRunnableJob(jobID ids.ID) error {
	return ps.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (ps *state) HasRunnableJob() (bool, error) {
	isEmpty, err := ps.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (ps *state) RemoveRunnableJob() (Job, error) {
	key, err := ps.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := ps.runnableJobIDs.Delete(key); err != nil {
		return nil, err
	}

	jobBytes, err := ps.jobs.Get(key)
	if err != nil {
		return nil, err
	}
	job, err := ps.parser.Parse(jobBytes)
	if err != nil {
		return nil, err
	}
	return job, ps.jobs.Delete(key)
}

// PutJob adds the job to the queue
func (ps *state) PutJob(job Job) error {
	id := job.ID()
	return ps.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (ps *state) HasJob(id ids.ID) (bool, error) {
	return ps.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (ps *state) GetJob(id ids.ID) (Job, error) {
	jobBytes, err := ps.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	return ps.parser.Parse(jobBytes)
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (ps *state) AddDependency(dependency, dependent ids.ID) error {
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (ps *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	var (
		dependentsDB linkeddb.LinkedDB
	)
	if dependentsDBIntf, ok := ps.dependentsCache.Get(dependency); ok {
		dependentsDB = dependentsDBIntf.(linkeddb.LinkedDB)
	} else {
		dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
		dependentsDB = linkeddb.NewDefault(dependencyDB)
		ps.dependentsCache.Put(dependency, dependentsDB)
	}

	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (ps *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := ps.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}
<|endoftext|>


<commit_before>package clock

import (
	"time"
)

type mockTimer struct {
	c       chan time.Time
	release chan bool

	lock   chan struct{}
	clock  Clock
	active bool
	target time.Time
}

var _ Timer = new(mockTimer)

func (m *mockTimer) getTarget() time.Time {
	m.lock <- struct{}{}
	defer func() { <-m.lock }()

	return m.target
}

func (m *mockTimer) setInactive() {
	// If a release was sent in the meantime, that means a new timer
	// was started or that we already stopped manually
	select {
	case m.lock <- struct{}{}:
		defer func() { <-m.lock }()
	case <-m.release:
		return
	}
	m.active = false
}

func (m *mockTimer) wait() {
	select {
	case <-m.clock.After(m.target.Sub(m.clock.Now())):
		m.setInactive()
		m.c <- m.clock.Now()
	case <-m.release:
	}
}

func (m *mockTimer) Chan() <-chan time.Time {
	return m.c
}

func (m *mockTimer) Reset(d time.Duration) bool {
	var wasActive bool
	m.lock <- struct{}{}
	defer func() { <-m.lock }()

	wasActive, m.active = m.active, true
	m.target = m.clock.Now().Add(d)

	if wasActive {
		m.release <- true
	}
	go m.wait()

	return wasActive
}

func (m *mockTimer) Stop() bool {
	var wasActive bool
	m.lock <- struct{}{}
	defer func() { <-m.lock }()

	wasActive, m.active = m.active, false
	if wasActive {
		m.release <- true
	}

	return wasActive
}

// Creates a new Timer using the provided Clock. You should not use this
// directly outside of unit tests; use Clock.NewTimer().
func NewMockTimer(c Clock) Timer {
	return &mockTimer{
		c:       make(chan time.Time, 1),
		release: make(chan bool),
		lock:    make(chan struct{}, 1),
		clock:   c,
	}
}
<commit_msg>refactor: remove unused private method mockTimer.getTarget()<commit_after>package clock

import (
	"time"
)

type mockTimer struct {
	c       chan time.Time
	release chan bool

	lock   chan struct{}
	clock  Clock
	active bool
	target time.Time
}

var _ Timer = new(mockTimer)

func (m *mockTimer) setInactive() {
	// If a release was sent in the meantime, that means a new timer
	// was started or that we already stopped manually
	select {
	case m.lock <- struct{}{}:
		defer func() { <-m.lock }()
	case <-m.release:
		return
	}
	m.active = false
}

func (m *mockTimer) wait() {
	select {
	case <-m.clock.After(m.target.Sub(m.clock.Now())):
		m.setInactive()
		m.c <- m.clock.Now()
	case <-m.release:
	}
}

func (m *mockTimer) Chan() <-chan time.Time {
	return m.c
}

func (m *mockTimer) Reset(d time.Duration) bool {
	var wasActive bool
	m.lock <- struct{}{}
	defer func() { <-m.lock }()

	wasActive, m.active = m.active, true
	m.target = m.clock.Now().Add(d)

	if wasActive {
		m.release <- true
	}
	go m.wait()

	return wasActive
}

func (m *mockTimer) Stop() bool {
	var wasActive bool
	m.lock <- struct{}{}
	defer func() { <-m.lock }()

	wasActive, m.active = m.active, false
	if wasActive {
		m.release <- true
	}

	return wasActive
}

// Creates a new Timer using the provided Clock. You should not use this
// directly outside of unit tests; use Clock.NewTimer().
func NewMockTimer(c Clock) Timer {
	return &mockTimer{
		c:       make(chan time.Time, 1),
		release: make(chan bool),
		lock:    make(chan struct{}, 1),
		clock:   c,
	}
}
<|endoftext|>


<commit_before>// A timer implementation with a fixed Reset behavior
package timer

import "time"

// The Timer type represents a single event. When the Timer expires,
// the current time will be sent on C, unless the Timer was created by AfterFunc.
// A Timer must be created with NewTimer. NewStoppedTimer or AfterFunc.
type Timer struct {
	C <-chan time.Time

	i    int                // heap index
	when int64              // Timer wakes up at when
	f    func(t *time.Time) // Callback function called on timeout.
}

// AfterFunc waits for the duration to elapse and then calls f in its own goroutine.
// It returns a Timer that can be used to cancel the call using its Stop method.
func AfterFunc(d time.Duration, f func()) *Timer {
	t := &Timer{
		f: func(*time.Time) {
			go f()
		},
	}
	addTimer(t, d)
	return t
}

// NewTimer creates a new Timer that will send the current time on its
// channel after at least duration d.
func NewTimer(d time.Duration) *Timer {
	t := NewStoppedTimer()
	addTimer(t, d)
	return t
}

// NewStoppedTimer creates a new stopped Timer.
func NewStoppedTimer() *Timer {
	c := make(chan time.Time, 1)
	t := &Timer{
		C: c,
		f: func(t *time.Time) {
			// Don't block.
			select {
			case c <- *t:
			default:
			}
		},
	}
	return t
}

// Stop prevents the Timer from firing.
// It returns true if the call stops the timer,
// false if the timer has already expired or been stopped.
// Stop does not close the channel, to prevent a read from
// the channel succeeding incorrectly.
func (t *Timer) Stop() (wasActive bool) {
	if t.f == nil {
		panic("timer: Stop called on uninitialized Timer")
	}
	return delTimer(t)
}

// Reset changes the timer to expire after duration d.
// It returns true if the timer had been active,
// false if the timer had expired or been stopped.
// The channel t.C is cleared and calling t.Reset() behaves as creating a
// new Timer.
func (t *Timer) Reset(d time.Duration) bool {
	if t.f == nil {
		panic("timer: Reset called on uninitialized Timer")
	}
	return resetTimer(t, d)
}

// reset is called in a locked context. This function must not block
// and must behave well-defined.
func (t *Timer) reset() {
	// Empty the channel if defined.
	if t.C != nil {
		select {
		case <-t.C:
		default:
		}
	}
}
<commit_msg>Updated package comment.<commit_after>// Package timer is a Go timer implementation with a fixed Reset behavior.
package timer

import "time"

// The Timer type represents a single event. When the Timer expires,
// the current time will be sent on C, unless the Timer was created by AfterFunc.
// A Timer must be created with NewTimer. NewStoppedTimer or AfterFunc.
type Timer struct {
	C <-chan time.Time

	i    int                // heap index
	when int64              // Timer wakes up at when
	f    func(t *time.Time) // Callback function called on timeout.
}

// AfterFunc waits for the duration to elapse and then calls f in its own goroutine.
// It returns a Timer that can be used to cancel the call using its Stop method.
func AfterFunc(d time.Duration, f func()) *Timer {
	t := &Timer{
		f: func(*time.Time) {
			go f()
		},
	}
	addTimer(t, d)
	return t
}

// NewTimer creates a new Timer that will send the current time on its
// channel after at least duration d.
func NewTimer(d time.Duration) *Timer {
	t := NewStoppedTimer()
	addTimer(t, d)
	return t
}

// NewStoppedTimer creates a new stopped Timer.
func NewStoppedTimer() *Timer {
	c := make(chan time.Time, 1)
	t := &Timer{
		C: c,
		f: func(t *time.Time) {
			// Don't block.
			select {
			case c <- *t:
			default:
			}
		},
	}
	return t
}

// Stop prevents the Timer from firing.
// It returns true if the call stops the timer,
// false if the timer has already expired or been stopped.
// Stop does not close the channel, to prevent a read from
// the channel succeeding incorrectly.
func (t *Timer) Stop() (wasActive bool) {
	if t.f == nil {
		panic("timer: Stop called on uninitialized Timer")
	}
	return delTimer(t)
}

// Reset changes the timer to expire after duration d.
// It returns true if the timer had been active,
// false if the timer had expired or been stopped.
// The channel t.C is cleared and calling t.Reset() behaves as creating a
// new Timer.
func (t *Timer) Reset(d time.Duration) bool {
	if t.f == nil {
		panic("timer: Reset called on uninitialized Timer")
	}
	return resetTimer(t, d)
}

// reset is called in a locked context. This function must not block
// and must behave well-defined.
func (t *Timer) reset() {
	// Empty the channel if defined.
	if t.C != nil {
		select {
		case <-t.C:
		default:
		}
	}
}
<|endoftext|>


<commit_before>package cruncy

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/segmentio/ksuid"
	log "github.com/sirupsen/logrus"
)

// TimerData containes run time data
type TimerData struct {
	Title          string
	Uuid           string
	StartTimeRun   time.Time
	StartTimeBatch time.Time
	EndTimeRun     time.Time

	BatchSize  int64
	PrevRows   int64
	Index      int64
	ErrorCount int64
	mu         sync.RWMutex
	muShow     sync.RWMutex
}

// NewTimer creates a new timer struct
func NewTimer(title string) *TimerData {
	timer := &TimerData{}
	timer.Title = title
	timer.Uuid = ksuid.New().String()
	timer.StartTimeRun = time.Now()
	timer.StartTimeBatch = timer.StartTimeRun
	timer.PrevRows = 0
	timer.ErrorCount = 0
	return timer
}

// BatchDuractionSeconds returns durection in seconds
func (timer *TimerData) BatchDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeBatch)
	return int64(duration.Seconds())
}

// TotalDuractionSeconds returns total duration in seconds
func (timer *TimerData) TotalDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeRun)
	return int64(duration.Seconds())
}

// TotalDuration returns duration as a time.Duration
func (timer *TimerData) TotalDuration() time.Duration {
	t1 := time.Now()
	return t1.Sub(timer.StartTimeRun)
}

// ShowTotalDuration outputs duration to log with fields
func (timer *TimerData) ShowTotalDuration() {
	timer.muShow.Lock() // Claim the mutex as a RLock - allowing multiple go routines to log simultaneously
	defer timer.muShow.Unlock()

	duration := timer.TotalDuration()
	ds := timer.TotalDuractionSeconds()
	if ds > 0 {
		msg := fmt.Sprintf("Total duration:, %v rows =%d row time=%d rows/sec ", duration, timer.Index, timer.Index/ds)
		log.WithFields(log.Fields{
			"uuid":       timer.Uuid,
			"title":      timer.Title,
			"total_rows": timer.Index,
			"avg_flow":   timer.Index / ds,
			"State":      "stopped",
		}).Info(msg)
	} else {
		log.WithFields(log.Fields{
			"uuid":       timer.Uuid,
			"title":      timer.Title,
			"total_rows": timer.Index,
			"avg_flow":   timer.Index,
			"State":      "stopped",
		}).Infof("Total duration:, %v rows =%d  SUPER FAST", duration, timer.Index)

	}
}

// ShowBatchTime show averages to now
func (timer *TimerData) ShowBatchTime() {
	timer.muShow.Lock() // Claim the mutex as a RLock - allowing multiple go routines to log simultaneously
	defer timer.muShow.Unlock()

	diff := timer.Index - timer.PrevRows

	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeBatch)
	d2 := timer.TotalDuration()

	ds := int64(d2.Seconds())
	dsBatch := int64(duration.Seconds())

	if ds > 0 && dsBatch > 0 {
		msg := fmt.Sprintf("%d rows avg flow %d/s - batch time %v batch size %d batch_flow %d \n", timer.Index, timer.Index/ds, duration, diff, diff/dsBatch)
		log.WithFields(log.Fields{
			"uuid":       timer.Uuid,
			"title":      timer.Title,
			"index":      timer.Index,
			"total_flow": timer.Index / ds,
			"batch_time": duration,
			"batch_size": diff,
			"batch_flow": diff / dsBatch,
			"State":      "in_batch",
		}).Info(msg)
	} else {
		log.Printf("%d rows - batch time %v \n", timer.Index, duration)
	}
	timer.PrevRows = timer.Index
	timer.StartTimeBatch = time.Now()

}

// Tick increases tick with one
func (timer *TimerData) Tick() {
	timer.mu.Lock() // Claim the mutex as a RLock - allowing multiple go routines to log simultaneously
	defer timer.mu.Unlock()

	atomic.AddInt64(&timer.Index, 1)

	if timer.Index%100000 == 0 {
		timer.ShowBatchTime()
	}
}

// Stop stops the timer
func (timer *TimerData) Stop() time.Time {
	timer.EndTimeRun = time.Now()
	return timer.EndTimeRun
}

// IncError adds one to number of errors
func (timer *TimerData) IncError() int64 {
	timer.mu.Lock() // Claim the mutex as a RLock - allowing multiple go routines to log simultaneously
	defer timer.mu.Unlock()

	timer.ErrorCount++
	return timer.ErrorCount
}
<commit_msg>Adding concurrency guards<commit_after>package cruncy

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/segmentio/ksuid"
	log "github.com/sirupsen/logrus"
)

// AtomicIntCounter uses an int64 internally.
type AtomicIntCounter int64

func (c *AtomicIntCounter) get() int64 {
	return atomic.LoadInt64((*int64)(c))
}

func (c *AtomicIntCounter) inc() int64 {
	return atomic.AddInt64((*int64)(c), 1)
}

// TimerData containes run time data
type TimerData struct {
	Title          string
	Uuid           string
	StartTimeRun   time.Time
	StartTimeBatch time.Time
	EndTimeRun     time.Time

	BatchSize  int64
	PrevRows   int64
	Index      AtomicIntCounter
	ErrorCount AtomicIntCounter
	mu         sync.RWMutex
}

// NewTimer creates a new timer struct
func NewTimer(title string) *TimerData {
	timer := &TimerData{}
	timer.Title = title
	timer.Uuid = ksuid.New().String()
	timer.StartTimeRun = time.Now()
	timer.StartTimeBatch = timer.StartTimeRun
	timer.PrevRows = 0
	timer.ErrorCount = 0
	return timer
}

// BatchDuractionSeconds returns durection in seconds
func (timer *TimerData) BatchDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeBatch)
	return int64(duration.Seconds())
}

// TotalDuractionSeconds returns total duration in seconds
func (timer *TimerData) TotalDuractionSeconds() int64 {
	t1 := time.Now()
	duration := t1.Sub(timer.StartTimeRun)
	return int64(duration.Seconds())
}

// TotalDuration returns duration as a time.Duration
func (timer *TimerData) TotalDuration() time.Duration {
	t1 := time.Now()
	return t1.Sub(timer.StartTimeRun)
}

// ShowTotalDuration outputs duration to log with fields
func (timer *TimerData) ShowTotalDuration() {

	cnt := timer.Index.get()
	timer.mu.RLock()
	uuid := timer.Uuid
	title := timer.Title
	startTime := timer.StartTimeRun
	timer.mu.RUnlock()

	t1 := time.Now()
	duration := t1.Sub(startTime)
	ds := int64(duration.Seconds())
	if ds > 0 {
		msg := fmt.Sprintf("Total duration:, %v rows =%d rate = %d rows/sec ", duration, cnt, cnt/ds)
		log.WithFields(log.Fields{
			"uuid":       uuid,
			"title":      title,
			"total_rows": cnt,
			"avg_flow":   cnt / ds,
			"State":      "stopped",
		}).Info(msg)
	} else {
		log.WithFields(log.Fields{
			"uuid":       uuid,
			"title":      title,
			"total_rows": cnt,
			"avg_flow":   cnt,
			"State":      "stopped",
		}).Infof("Total duration:, %v rows =%d  SUPER FAST", duration, cnt)

	}
}

// ShowBatchTime show averages to now
func (timer *TimerData) ShowBatchTime() {

	cnt := timer.Index.get()
	timer.mu.RLock()
	uuid := timer.Uuid
	title := timer.Title
	prevRows := timer.PrevRows
	startTime := timer.StartTimeBatch
	timer.mu.RUnlock()

	diff := cnt - prevRows

	t1 := time.Now()
	duration := t1.Sub(startTime)
	d2 := timer.TotalDuration()

	ds := int64(d2.Seconds())
	dsBatch := int64(duration.Seconds())

	if ds > 0 && dsBatch > 0 {
		msg := fmt.Sprintf("%d rows avg flow %d/s - batch time %v batch size %d batch_flow %d \n", cnt, cnt/ds, duration, diff, diff/dsBatch)
		log.WithFields(log.Fields{
			"uuid":       uuid,
			"title":      title,
			"index":      cnt,
			"total_flow": cnt / ds,
			"batch_time": duration,
			"batch_size": diff,
			"batch_flow": diff / dsBatch,
			"State":      "in_batch",
		}).Info(msg)
	} else {
		log.Printf("%d rows - batch time %v \n", cnt, duration)
	}

	timer.mu.Lock()
	timer.PrevRows = cnt
	timer.StartTimeBatch = time.Now()
	timer.mu.Unlock()
}

// Tick increases tick with one
func (timer *TimerData) Tick() {
	cnt := timer.Index.inc()

	if cnt%100000 == 0 {
		timer.ShowBatchTime()
	}
}

// Stop stops the timer
func (timer *TimerData) Stop() time.Time {
	timer.mu.Lock()
	timer.EndTimeRun = time.Now()
	timer.mu.Unlock()
	return timer.EndTimeRun
}

// IncError adds one to number of errors
func (timer *TimerData) IncError() int64 {
	timer.ErrorCount.inc()
	return timer.ErrorCount.get()
}
<|endoftext|>