<commit_before>package logx

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var logger *Logger

func init() {
	chaos()

	if out == "file" {
		go poller()
	}
}

func chaos() {

	loadConfig()
	y, m, d := time.Now().Date()
	if logger == nil {
		logger = &Logger{
			look:        uint32(coreDead),
			fileName:    fileName,
			path:        filepath.Join(filePath, fileName),
			timestamp:   y*10000 + int(m)*100 + d*1,
			fileMaxSize: maxSize,
			bucket:      make(chan *bytes.Buffer, bucketLen),
			closeSignal: make(chan string),
			lock:        &sync.RWMutex{},
		}
	}
}

func (l *Logger) loadCurLogFile() error {
	l.link = filepath.Join(l.path, fileName+".log")
	actFileName, ok := isLinkFile(l.link)
	if !ok {
		return errors.New("is not link file")
	}

	l.fileName = filepath.Join(l.path, actFileName)
	f, err := openFile(l.fileName)
	if err != nil {
		return err
	}

	info, err := os.Stat(l.fileName)
	if err != nil {
		return err
	}
	sp := strings.Split(actFileName, ".")
	t, err := time.Parse("2006-01-02", sp[1])
	if err != nil {
		fmt.Errorf("loadCurrentLogFile |err=%v", err)
		return err
	}
	y, m, d := t.Date()
	l.timestamp = y*10000 + int(m)*100 + d*1
	l.file = f
	l.fileActualSize = int(info.Size())
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)

	return nil
}

func (l *Logger) createFile() (err error) {
	if !pathIsExist(l.path) {
		if err = os.MkdirAll(l.path, os.ModePerm); err != nil {
			return
		}
	}

	now := time.Now()

	y, m, d := now.Date()

	l.timestamp = y*10000 + int(m)*100 + d*1
	l.fileName = filepath.Join(
		l.path,
		filepath.Base(os.Args[0])+"."+now.Format("2006-01-02.15.04.05.000")+".log")

	f, err := openFile(l.fileName)
	if err != nil {
		return err
	}

	l.file = f
	l.fileActualSize = 0
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)
	l.link = filepath.Join(l.path, fileName+".log")
	return createLinkFile(l.fileName, l.link)
}

func (l *Logger) sync() {
	if l.lookRunning() {
		l.fileWriter.Flush()
	}
}

const fileMaxDelta = 100

func (l *Logger) rotate(do func()) bool {
	if !l.lookRunning() {
		return false
	}

	y, m, d := time.Now().Date()
	timestamp := y*10000 + int(m)*100 + d*1
	if l.fileActualSize <= l.fileMaxSize-fileMaxDelta || timestamp < l.timestamp {
		return false
	}
	do()

	closeFile(l.file)

	if err := l.createFile(); err != nil {
		return false
	}
	return true
}

func (l *Logger) lookRunning() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreRunning) {
		return true
	}
	return false
}

func (l *Logger) lookDead() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreDead) {
		return true
	}
	return false
}

func (l *Logger) lookBlock() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreBlock) {
		return true
	}
	return false
}

func (l *Logger) signalHandler() {
	var sigChan = make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)

	for {
		select {
		case sig := <-sigChan:
			l.closeSignal <- "close"
			fmt.Println("receive os signal is ", sig)
			l.fileWriter.Flush()
			closeFile(l.file)
			atomic.SwapUint32(&l.look, uint32(coreDead))
			close(l.bucket)
			os.Exit(1)
		}
	}
}

func (l *Logger) release(buf *bytes.Buffer) {
	bufferPoolFree(buf)
}

func caller() string {
	if pc, f, l, ok := runtime.Caller(2); ok {
		funcName := runtime.FuncForPC(pc).Name()
		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	}
	//pc := make([]uintptr, 3, 3)
	//cnt := runtime.Callers(6, pc)
	//
	//for i := 0; i < cnt; i++ {
	//	fu := runtime.FuncForPC(pc[i] - 1)
	//	name := fu.Name()
	//
	//	if !strings.Contains(name, "github.com/kafrax/logx") {
	//		f, l := fu.FileLine(pc[i] - 1)
	//		return path.Base(f) + "|" + path.Base(name) + "|" + strconv.Itoa(l)
	//	}
	//
	//	if pc, f, l, ok := runtime.Caller(8); ok {
	//		funcName := runtime.FuncForPC(pc).Name()
	//		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	//	}
	//}
	return ""
}

func print(buf *bytes.Buffer) {
	switch out {
	case "file":
		logger.bucket <- buf
	case "stdout":
		fmt.Print(buf.String())
	case "kafka":
		//todo send to kafka nsq etc.
	default:
		fmt.Println(buf.String())
	}
}

func Debugf(format string, msg ... interface{}) {
	if levelFlag > _DEBUG {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Infof(format string, msg ... interface{}) {
	if levelFlag > _INFO {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Warnf(format string, msg ... interface{}) {
	if levelFlag > _WARN {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Errorf(format string, msg ... interface{}) {
	if levelFlag > _ERR {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Fatalf(format string, msg ... interface{}) {
	if levelFlag > _DISASTER {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Stackf(format string, msg ... interface{}) {
	s := fmt.Sprintf(format, msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}

func Debug(msg ... interface{}) {
	if levelFlag > _DEBUG {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Info(msg ... interface{}) {
	if levelFlag > _INFO {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Warn(msg ... interface{}) {
	if levelFlag > _WARN {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Error(msg ... interface{}) {
	if levelFlag > _ERR {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Fatal(msg ... interface{}) {
	if levelFlag > _DISASTER {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Stack(msg ... interface{}) {
	s := fmt.Sprintln(msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}
<commit_msg>tps test<commit_after>package logx

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

var logger *Logger

func init() {
	chaos()

	if out == "file" {
		go poller()
	}
}

func chaos() {

	loadConfig()
	y, m, d := time.Now().Date()
	if logger == nil {
		logger = &Logger{
			look:        uint32(coreDead),
			fileName:    fileName,
			path:        filepath.Join(filePath, fileName),
			timestamp:   y*10000 + int(m)*100 + d*1,
			fileMaxSize: maxSize,
			bucket:      make(chan *bytes.Buffer, bucketLen),
			closeSignal: make(chan string),
			lock:        &sync.RWMutex{},
		}
	}
}

func (l *Logger) loadCurLogFile() error {
	l.link = filepath.Join(l.path, fileName+".log")
	actFileName, ok := isLinkFile(l.link)
	if !ok {
		return errors.New("is not link file")
	}

	l.fileName = filepath.Join(l.path, actFileName)
	f, err := openFile(l.fileName)
	if err != nil {
		return err
	}

	info, err := os.Stat(l.fileName)
	if err != nil {
		return err
	}
	sp := strings.Split(actFileName, ".")
	t, err := time.Parse("2006-01-02", sp[1])
	if err != nil {
		fmt.Errorf("loadCurrentLogFile |err=%v", err)
		return err
	}
	y, m, d := t.Date()
	l.timestamp = y*10000 + int(m)*100 + d*1
	l.file = f
	l.fileActualSize = int(info.Size())
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)

	return nil
}

func (l *Logger) createFile() (err error) {
	if !pathIsExist(l.path) {
		if err = os.MkdirAll(l.path, os.ModePerm); err != nil {
			return
		}
	}

	now := time.Now()

	y, m, d := now.Date()

	l.timestamp = y*10000 + int(m)*100 + d*1
	l.fileName = filepath.Join(
		l.path,
		filepath.Base(os.Args[0])+"."+now.Format("2006-01-02.15.04.05.000")+".log")

	f, err := openFile(l.fileName)
	if err != nil {
		return err
	}

	l.file = f
	l.fileActualSize = 0
	l.fileWriter = bufio.NewWriterSize(f, l.fileMaxSize)
	l.link = filepath.Join(l.path, fileName+".log")
	return createLinkFile(l.fileName, l.link)
}

func (l *Logger) sync() {
	if l.lookRunning() {
		l.fileWriter.Flush()
	}
}

const fileMaxDelta = 100

func (l *Logger) rotate(do func()) bool {
	if !l.lookRunning() {
		return false
	}

	y, m, d := time.Now().Date()
	timestamp := y*10000 + int(m)*100 + d*1
	if l.fileActualSize <= l.fileMaxSize-fileMaxDelta || timestamp < l.timestamp {
		return false
	}
	do()

	closeFile(l.file)

	if err := l.createFile(); err != nil {
		return false
	}
	return true
}

func (l *Logger) lookRunning() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreRunning) {
		return true
	}
	return false
}

func (l *Logger) lookDead() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreDead) {
		return true
	}
	return false
}

func (l *Logger) lookBlock() bool {
	if atomic.LoadUint32(&l.look) == uint32(coreBlock) {
		return true
	}
	return false
}

func (l *Logger) signalHandler() {
	var sigChan = make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)

	for {
		select {
		case sig := <-sigChan:
			l.closeSignal <- "close"
			fmt.Println("LOGX receive os signal is ", sig)
			l.fileWriter.Flush()
			closeFile(l.file)
			atomic.SwapUint32(&l.look, uint32(coreDead))
			close(l.bucket)
			os.Exit(1)
		}
	}
}

func (l *Logger) release(buf *bytes.Buffer) {
	bufferPoolFree(buf)
}

func caller() string {
	if pc, f, l, ok := runtime.Caller(2); ok {
		funcName := runtime.FuncForPC(pc).Name()
		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	}
	//pc := make([]uintptr, 3, 3)
	//cnt := runtime.Callers(6, pc)
	//
	//for i := 0; i < cnt; i++ {
	//	fu := runtime.FuncForPC(pc[i] - 1)
	//	name := fu.Name()
	//
	//	if !strings.Contains(name, "github.com/kafrax/logx") {
	//		f, l := fu.FileLine(pc[i] - 1)
	//		return path.Base(f) + "|" + path.Base(name) + "|" + strconv.Itoa(l)
	//	}
	//
	//	if pc, f, l, ok := runtime.Caller(8); ok {
	//		funcName := runtime.FuncForPC(pc).Name()
	//		return path.Base(f) + "|" + path.Base(funcName) + "|" + strconv.Itoa(l)
	//	}
	//}
	return ""
}

func print(buf *bytes.Buffer) {
	switch out {
	case "file":
		logger.bucket <- buf
	case "stdout":
		fmt.Print(buf.String())
	case "kafka":
		//todo send to kafka nsq etc.
	default:
		fmt.Println(buf.String())
	}
}

func Debugf(format string, msg ... interface{}) {
	if levelFlag > _DEBUG {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Infof(format string, msg ... interface{}) {
	if levelFlag > _INFO {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Warnf(format string, msg ... interface{}) {
	if levelFlag > _WARN {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Errorf(format string, msg ... interface{}) {
	if levelFlag > _ERR {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Fatalf(format string, msg ... interface{}) {
	if levelFlag > _DISASTER {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintf(format, msg...) + "\n"))
	print(buf)
}

func Stackf(format string, msg ... interface{}) {
	s := fmt.Sprintf(format, msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}

func Debug(msg ... interface{}) {
	if levelFlag > _DEBUG {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[DEBU][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Info(msg ... interface{}) {
	if levelFlag > _INFO {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[INFO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Warn(msg ... interface{}) {
	if levelFlag > _WARN {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[WARN][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Error(msg ... interface{}) {
	if levelFlag > _ERR {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[ERRO][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Fatal(msg ... interface{}) {
	if levelFlag > _DISASTER {
		return
	}
	buf := bufferPoolGet()
	buf.Write(s2b("[FTAL][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] "))
	buf.Write(s2b(fmt.Sprintln(msg...)))
	print(buf)
}

func Stack(msg ... interface{}) {
	s := fmt.Sprintln(msg...)
	s += "\n"
	buf := make([]byte, 1<<20)
	n := runtime.Stack(buf, true)
	s += string(buf[:n])
	s += "\n"
	fmt.Println("[STAC][" + time.Now().Format("01-02.15.04.05.000") + "]" + "[" + caller() + "] " + s)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

/*
void foo1(void) {}
void foo2(void* p) {}
*/
import "C"
import (
	"fmt"
	"os"
	"runtime"
	"strconv"
	"time"
	"unsafe"
)

func init() {
	register("CgoSignalDeadlock", CgoSignalDeadlock)
	register("CgoTraceback", CgoTraceback)
	register("CgoCheckBytes", CgoCheckBytes)
}

func CgoSignalDeadlock() {
	runtime.GOMAXPROCS(100)
	ping := make(chan bool)
	go func() {
		for i := 0; ; i++ {
			runtime.Gosched()
			select {
			case done := <-ping:
				if done {
					ping <- true
					return
				}
				ping <- true
			default:
			}
			func() {
				defer func() {
					recover()
				}()
				var s *string
				*s = ""
			}()
		}
	}()
	time.Sleep(time.Millisecond)
	for i := 0; i < 64; i++ {
		go func() {
			runtime.LockOSThread()
			select {}
		}()
		go func() {
			runtime.LockOSThread()
			select {}
		}()
		time.Sleep(time.Millisecond)
		ping <- false
		select {
		case <-ping:
		case <-time.After(time.Second):
			fmt.Printf("HANG\n")
			return
		}
	}
	ping <- true
	select {
	case <-ping:
	case <-time.After(time.Second):
		fmt.Printf("HANG\n")
		return
	}
	fmt.Printf("OK\n")
}

func CgoTraceback() {
	C.foo1()
	buf := make([]byte, 1)
	runtime.Stack(buf, true)
	fmt.Printf("OK\n")
}

func CgoCheckBytes() {
	try, _ := strconv.Atoi(os.Getenv("GO_CGOCHECKBYTES_TRY"))
	if try <= 0 {
		try = 1
	}
	b := make([]byte, 1e6*try)
	start := time.Now()
	for i := 0; i < 1e3*try; i++ {
		C.foo2(unsafe.Pointer(&b[0]))
		if time.Since(start) > time.Second {
			break
		}
	}
}
<commit_msg>runtime: get more info for TestCgoSignalDeadlock failures<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

/*
void foo1(void) {}
void foo2(void* p) {}
*/
import "C"
import (
	"fmt"
	"os"
	"runtime"
	"strconv"
	"time"
	"unsafe"
)

func init() {
	register("CgoSignalDeadlock", CgoSignalDeadlock)
	register("CgoTraceback", CgoTraceback)
	register("CgoCheckBytes", CgoCheckBytes)
}

func CgoSignalDeadlock() {
	runtime.GOMAXPROCS(100)
	ping := make(chan bool)
	go func() {
		for i := 0; ; i++ {
			runtime.Gosched()
			select {
			case done := <-ping:
				if done {
					ping <- true
					return
				}
				ping <- true
			default:
			}
			func() {
				defer func() {
					recover()
				}()
				var s *string
				*s = ""
				fmt.Printf("continued after expected panic\n")
			}()
		}
	}()
	time.Sleep(time.Millisecond)
	start := time.Now()
	var times []time.Duration
	for i := 0; i < 64; i++ {
		go func() {
			runtime.LockOSThread()
			select {}
		}()
		go func() {
			runtime.LockOSThread()
			select {}
		}()
		time.Sleep(time.Millisecond)
		ping <- false
		select {
		case <-ping:
			times = append(times, time.Since(start))
		case <-time.After(time.Second):
			fmt.Printf("HANG 1 %v\n", times)
			return
		}
	}
	ping <- true
	select {
	case <-ping:
	case <-time.After(time.Second):
		fmt.Printf("HANG 2 %v\n", times)
		return
	}
	fmt.Printf("OK\n")
}

func CgoTraceback() {
	C.foo1()
	buf := make([]byte, 1)
	runtime.Stack(buf, true)
	fmt.Printf("OK\n")
}

func CgoCheckBytes() {
	try, _ := strconv.Atoi(os.Getenv("GO_CGOCHECKBYTES_TRY"))
	if try <= 0 {
		try = 1
	}
	b := make([]byte, 1e6*try)
	start := time.Now()
	for i := 0; i < 1e3*try; i++ {
		C.foo2(unsafe.Pointer(&b[0]))
		if time.Since(start) > time.Second {
			break
		}
	}
}
<|endoftext|>


<commit_before>package awsSdkGo

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
)

type Sdk struct {
	Ec2 *ec2.EC2
	S3  *s3.S3
	ASG *autoscaling.AutoScaling
}

func NewSdk(region string) (*Sdk, error) {
	sdk := &Sdk{}
	session, err := session.NewSession(&aws.Config{Region: aws.String(region)})
	if err != nil {
		return nil, err
	}
	sdk.Ec2 = ec2.New(session)
	sdk.ASG = autoscaling.New(session)
	sdk.S3 = s3.New(session)
	return sdk, nil
}
<commit_msg>Support MFA and roles when doing auth<commit_after>package awsSdkGo

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"
)

type Sdk struct {
	Ec2 *ec2.EC2
	S3  *s3.S3
	ASG *autoscaling.AutoScaling
}

func NewSdk(region string) (*Sdk, error) {
	sdk := &Sdk{}
	session, err := session.NewSessionWithOptions(session.Options{
		Config: aws.Config{Region: aws.String(region)},
		// Support MFA when authing using assumed roles.
		SharedConfigState:       session.SharedConfigEnable,
		AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
	})
	if err != nil {
		return nil, err
	}
	sdk.Ec2 = ec2.New(session)
	sdk.ASG = autoscaling.New(session)
	sdk.S3 = s3.New(session)
	return sdk, nil
}
<|endoftext|>


<commit_before>package lru

import (
	"container/list"
	"time"
)

// Cache represent a lru cache
type Cache interface {
	Get(k interface{}) interface{}
	Set(k, v interface{}, expires ...time.Time)
	Del(k interface{})
}

type cache struct {
	lru   *list.List
	items map[interface{}]*list.Element
	size  int
}

type item struct {
	k, v    interface{}
	expires time.Time
}

// New create a Cache instance
func New(size int) Cache {
	if size <= 0 {
		panic("lur: must provide a positive num")
	}

	return &cache{
		lru:   list.New(),
		items: make(map[interface{}]*list.Element),
		size:  size,
	}
}

func (c *cache) Get(k interface{}) interface{} {
	if v, ok := c.items[k]; ok {
		item := v.Value.(*item)
		if item.expires.IsZero() || item.expires.After(time.Now()) {
			return item.v
		}
		c.removeItem(v)
	}
	return nil
}

func (c *cache) Set(k, v interface{}, expires ...time.Time) {
	if ele, ok := c.items[k]; ok {
		c.lru.MoveToFront(ele)
		item := ele.Value.(*item)
		item.v = v
		if len(expires) > 0 {
			item.expires = expires[0]
		}
		return
	}

	item := &item{
		k: k,
		v: v,
	}
	if len(expires) > 0 {
		item.expires = expires[0]
	}

	c.items[k] = c.lru.PushFront(item)

	if len(c.items) > c.size {
		item := c.lru.Back()
		c.removeItem(item)
	}
}

func (c *cache) Del(k interface{}) {
	if ele, ok := c.items[k]; ok {
		c.removeItem(ele)
	}
}

func (c *cache) removeItem(ele *list.Element) {
	c.lru.Remove(ele)
	delete(c.items, ele.Value.(*item).k)
}
<commit_msg>format code<commit_after>package lru

import (
	"container/list"
	"time"
)

// Cache represent a lru cache
type Cache interface {
	// Get value form cache with key, return nil if not exists
	Get(key interface{}) interface{}
	// Set value with key, and optional expires time
	Set(key, value interface{}, expires ...time.Time)
	// Del key from cache
	Del(key interface{})
}

type cache struct {
	lru   *list.List
	items map[interface{}]*list.Element
	size  int
}

type item struct {
	k, v    interface{}
	expires time.Time
}

// New create a Cache instance
func New(size int) Cache {
	if size <= 0 {
		panic("lur: must provide a positive num")
	}

	return &cache{
		lru:   list.New(),
		items: make(map[interface{}]*list.Element),
		size:  size,
	}
}

func (c *cache) Get(key interface{}) interface{} {
	if v, ok := c.items[key]; ok {
		item := v.Value.(*item)
		if item.expires.IsZero() || item.expires.After(time.Now()) {
			return item.v
		}
		c.removeItem(v)
	}
	return nil
}

func (c *cache) Set(key, value interface{}, expires ...time.Time) {
	if ele, ok := c.items[key]; ok {
		c.lru.MoveToFront(ele)
		item := ele.Value.(*item)
		item.v = value
		if len(expires) > 0 {
			item.expires = expires[0]
		}
		return
	}

	item := &item{
		k: key,
		v: value,
	}
	if len(expires) > 0 {
		item.expires = expires[0]
	}

	c.items[key] = c.lru.PushFront(item)

	if len(c.items) > c.size {
		item := c.lru.Back()
		c.removeItem(item)
	}
}

func (c *cache) Del(key interface{}) {
	if ele, ok := c.items[key]; ok {
		c.removeItem(ele)
	}
}

func (c *cache) removeItem(ele *list.Element) {
	c.lru.Remove(ele)
	delete(c.items, ele.Value.(*item).k)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"runtime"
	"sync"

	"tlsretriever"

	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func panicIf(err error) {
	if err != nil {
		log.Println(fmt.Sprintf("%s", err))
	}
}

type CertChain struct {
	Domain string   `json:"domain"`
	IP     string   `json:"ip"`
	Certs  []string `json:"certs"`
}

func worker(msg []byte, ch *amqp.Channel) {

	certs, ip, err := tlsretriever.CheckHost(string(msg), "443", true)
	panicIf(err)
	if certs == nil {
		log.Println("no certificate retrieved from", string(msg))
		return
	}

	var chain = CertChain{}

	chain.Domain = string(msg)

	chain.IP = ip

	for _, cert := range certs {

		chain.Certs = append(chain.Certs, base64.StdEncoding.EncodeToString(cert.Raw))

	}

	jsonChain, er := json.MarshalIndent(chain, "", "    ")
	panicIf(er)
	err = ch.Publish(
		"",                   // exchange
		"scan_results_queue", // routing key
		false,                // mandatory
		false,
		amqp.Publishing{
			DeliveryMode: amqp.Persistent,
			ContentType:  "text/plain",
			Body:         []byte(jsonChain),
		})
	panicIf(err)
}

var wg sync.WaitGroup

func main() {

	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"scan_ready_queue", // name
		true,               // durable
		false,              // delete when unused
		false,              // exclusive
		false,              // no-wait
		nil,                // arguments
	)
	failOnError(err, "Failed to declare a queue")

	//In case it has not already been declared before...
	_, err = ch.QueueDeclare(
		"scan_results_queue", // name
		true,                 // durable
		false,                // delete when unused
		false,                // exclusive
		false,                // no-wait
		nil,                  // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = ch.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	)
	failOnError(err, "Failed to set QoS")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	failOnError(err, "Failed to register a consumer")

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores)

	for d := range msgs {
		go worker(d.Body, ch)
	}

	wg.Wait()
}
<commit_msg>changes for confgiration file / propose workaround to file descriptor exhaustion<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"runtime"

	"config"
	"tlsretriever"

	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func panicIf(err error) {
	if err != nil {
		log.Println(fmt.Sprintf("%s", err))
	}
}

type CertChain struct {
	Domain string   `json:"domain"`
	IP     string   `json:"ip"`
	Certs  []string `json:"certs"`
}

func releaseSemaphore() {
	sem <- true
}

func worker(msg []byte, ch *amqp.Channel) {

	log.Println("waiting for channel")
	<-sem
	defer releaseSemaphore()
	log.Println("arrived")

	certs, ip, err := tlsretriever.CheckHost(string(msg), "443", true)
	panicIf(err)
	if certs == nil {
		log.Println("no certificate retrieved from", string(msg))
		return
	}

	var chain = CertChain{}

	chain.Domain = string(msg)

	chain.IP = ip

	for _, cert := range certs {

		chain.Certs = append(chain.Certs, base64.StdEncoding.EncodeToString(cert.Raw))

	}

	jsonChain, er := json.MarshalIndent(chain, "", "    ")
	panicIf(er)
	err = ch.Publish(
		"",                   // exchange
		"scan_results_queue", // routing key
		false,                // mandatory
		false,
		amqp.Publishing{
			DeliveryMode: amqp.Persistent,
			ContentType:  "text/plain",
			Body:         []byte(jsonChain),
		})
	panicIf(err)
}

var sem chan bool

func main() {

	conf := config.ObserverConfig{}

	var er error
	conf, er = config.ConfigLoad("observer.cfg")

	if er != nil {
		panicIf(er)
		conf = config.GetDefaults()
	}

	conn, err := amqp.Dial(conf.General.RabbitMQRelay)
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"scan_ready_queue", // name
		true,               // durable
		false,              // delete when unused
		false,              // exclusive
		false,              // no-wait
		nil,                // arguments
	)
	failOnError(err, "Failed to declare a queue")

	//In case it has not already been declared before...
	_, err = ch.QueueDeclare(
		"scan_results_queue", // name
		true,                 // durable
		false,                // delete when unused
		false,                // exclusive
		false,                // no-wait
		nil,                  // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = ch.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	)
	failOnError(err, "Failed to set QoS")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	failOnError(err, "Failed to register a consumer")

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores)

	maxSimConnections := conf.General.MaxSimConns

	//use channels as semaphores not to exhaust file descriptors
	sem = make(chan bool, maxSimConnections)
	for i := 0; i < maxSimConnections; i++ {
		sem <- true
	}

	for d := range msgs {
		go worker(d.Body, ch)
	}
}
<|endoftext|>


<commit_before>package masterbot

import (
   "fmt"
   "sync"
   "net/http"
   "github.com/luisfurquim/stonelizard"
)

func (svc ServiceT) Stop() stonelizard.Response {
   var botId       string
   var botCfg      BotClientT
   var wg          sync.WaitGroup
   var botInstance int

   Goose.Logf(2,"Stopping slave bots")

   for botId, botCfg = range svc.appcfg.Bot {
      for botInstance, _ = range botCfg.Host {
         wg.Add(len(botCfg.Host)) // wait the stop of each instance of the slavebots
      }
   }
   wg.Add(1) // wait the stop of the masterbot itself

   for botId, botCfg = range svc.appcfg.Bot {
      for botInstance, _ = range botCfg.Host {
         go func(id string, instance int, cfg BotClientT) {
            var err    error
            var url    string
            var resp  *http.Response

            defer wg.Done()

            url   = fmt.Sprintf("https://%s%s/%s/stop", botCfg.Host[instance], botCfg.Listen, id)
            Goose.Logf(2,"Stopping bot %s@%s via %s",id,botCfg.Host[instance],url)
            resp, err = svc.appcfg.HttpsStopClient.Get(url)

            if err != nil {
               Goose.Logf(1,"Error stopping bot %s@%s (%s)",id,botCfg.Host[instance],err)
               return
            }

            if resp.StatusCode != http.StatusNoContent {
               Goose.Logf(1,"Error status stopping bot %s@%s (%s)",id,botCfg.Host[instance],resp.Status)
            }

         }(botId,botInstance,botCfg)
      }
   }

   Goose.Logf(2,"Stopping masterbot")

   go (func () {
      Kairos.Stop()
      wg.Wait()
      svc.onStop()
   })()

   defer wg.Done()

   return stonelizard.Response{
      Status: http.StatusNoContent,
   }
}

<commit_msg>@{typo}Fixed error message<commit_after>package masterbot

import (
   "fmt"
   "sync"
   "net/http"
   "github.com/luisfurquim/stonelizard"
)

func (svc ServiceT) Stop() stonelizard.Response {
   var botId       string
   var botCfg      BotClientT
   var wg          sync.WaitGroup
   var botInstance int

   Goose.Logf(2,"Stopping slave bots")

   for botId, botCfg = range svc.appcfg.Bot {
      for botInstance, _ = range botCfg.Host {
         wg.Add(len(botCfg.Host)) // wait the stop of each instance of the slavebots
      }
   }
   wg.Add(1) // wait the stop of the masterbot itself

   for botId, botCfg = range svc.appcfg.Bot {
      for botInstance, _ = range botCfg.Host {
         go func(id string, instance int, cfg BotClientT) {
            var err    error
            var url    string
            var resp  *http.Response

            defer wg.Done()

            url   = fmt.Sprintf("https://%s%s/%s/stop", botCfg.Host[instance], botCfg.Listen, id)
            Goose.Logf(2,"Stopping bot %s@%s via %s",id,botCfg.Host[instance],url)
            resp, err = svc.appcfg.HttpsStopClient.Get(url)

            if err != nil {
               Goose.Logf(1,"Error stopping bot %s@%s (%s)",id,botCfg.Host[instance],err)
               return
            }

            if resp.StatusCode != http.StatusNoContent {
               Goose.Logf(1,"Error of status code stopping bot %s@%s (%s)",id,botCfg.Host[instance],resp.Status)
            }

         }(botId,botInstance,botCfg)
      }
   }

   Goose.Logf(2,"Stopping masterbot")

   go (func () {
      Kairos.Stop()
      wg.Wait()
      svc.onStop()
   })()

   defer wg.Done()

   return stonelizard.Response{
      Status: http.StatusNoContent,
   }
}

<|endoftext|>


<commit_before>// bulk_load_influx loads an InfluxDB daemon with data from stdin.
//
// The caller is responsible for assuring that the database is empty before
// bulk load.
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/pkg/profile"
	"github.com/valyala/fasthttp"
)

// Program option vars:
var (
	csvDaemonUrls     string
	daemonUrls        []string
	dbName            string
	replicationFactor int
	workers           int
	lineLimit         int64
	batchSize         int
	backoff           time.Duration
	timeLimit         time.Duration
	progressInterval  time.Duration
	doLoad            bool
	doDBCreate        bool
	useGzip           bool
	doAbortOnExist    bool
	memprofile        bool
)

// Global vars
var (
	bufPool         sync.Pool
	batchChan       chan *bytes.Buffer
	inputDone       chan struct{}
	workersGroup    sync.WaitGroup
	backingOffChans []chan bool
	backingOffDones []chan struct{}
)

// Parse args:
func init() {
	flag.StringVar(&csvDaemonUrls, "urls", "http://localhost:8086", "InfluxDB URLs, comma-separated. Will be used in a round-robin fashion.")
	flag.StringVar(&dbName, "db", "benchmark_db", "Database name.")
	flag.IntVar(&replicationFactor, "replication-factor", 2, "Cluster replication factor (only applies to clustered databases).")
	flag.IntVar(&batchSize, "batch-size", 5000, "Batch size (input lines).")
	flag.IntVar(&workers, "workers", 1, "Number of parallel requests to make.")
	flag.Int64Var(&lineLimit, "line-limit", -1, "Number of lines to read from stdin before quitting.")
	flag.DurationVar(&backoff, "backoff", time.Second, "Time to sleep between requests when server indicates backpressure is needed.")
	flag.DurationVar(&timeLimit, "time-limit", -1, "Maximum duration to run (-1 is the default: no limit).")
	flag.DurationVar(&progressInterval, "progress-interval", -1, "Duration between printing progress messages.")
	flag.BoolVar(&useGzip, "gzip", true, "Whether to gzip encode requests (default true).")
	flag.BoolVar(&doLoad, "do-load", true, "Whether to write data. Set this flag to false to check input read speed.")
	flag.BoolVar(&doDBCreate, "do-db-create", true, "Whether to create the database.")
	flag.BoolVar(&doAbortOnExist, "do-abort-on-exist", true, "Whether to abort if the destination database already exists.")
	flag.BoolVar(&memprofile, "memprofile", false, "Whether to write a memprofile (file automatically determined).")

	flag.Parse()

	daemonUrls = strings.Split(csvDaemonUrls, ",")
	if len(daemonUrls) == 0 {
		log.Fatal("missing 'urls' flag")
	}
	fmt.Printf("daemon URLs: %v\n", daemonUrls)
}

func main() {
	if memprofile {
		p := profile.Start(profile.MemProfile)
		defer p.Stop()
	}
	if doLoad && doDBCreate {
		// check that there are no pre-existing databases:
		existingDatabases, err := listDatabases(daemonUrls[0])
		if err != nil {
			log.Fatal(err)
		}

		if len(existingDatabases) > 0 {
			if doAbortOnExist {
				log.Fatalf("There are databases already in the data store. If you know what you are doing, run the command:\ncurl 'http://localhost:8086/query?q=drop%%20database%%20%s'\n", existingDatabases[0])
			} else {
				log.Printf("Info: there are databases already in the data store.")
			}
		}

		if len(existingDatabases) == 0 {
			err = createDb(daemonUrls[0], dbName, replicationFactor)
			if err != nil {
				log.Fatal(err)
			}
			time.Sleep(1000 * time.Millisecond)
		}
	}

	bufPool = sync.Pool{
		New: func() interface{} {
			return bytes.NewBuffer(make([]byte, 0, 4*1024*1024))
		},
	}

	batchChan = make(chan *bytes.Buffer, workers)
	inputDone = make(chan struct{})

	backingOffChans = make([]chan bool, workers)
	backingOffDones = make([]chan struct{}, workers)

	for i := 0; i < workers; i++ {
		daemonUrl := daemonUrls[i%len(daemonUrls)]
		backingOffChans[i] = make(chan bool, 100)
		backingOffDones[i] = make(chan struct{})
		workersGroup.Add(1)
		cfg := HTTPWriterConfig{
			DebugInfo:      fmt.Sprintf("worker #%d, dest url: %s", i, daemonUrl),
			Host:           daemonUrl,
			Database:       dbName,
			BackingOffChan: backingOffChans[i],
			BackingOffDone: backingOffDones[i],
		}
		go processBatches(NewHTTPWriter(cfg), backingOffChans[i], backingOffDones[i])
		go processBackoffMessages(i, backingOffChans[i], backingOffDones[i])
	}

	start := time.Now()
	itemsRead, bytesRead := scan(batchSize)

	<-inputDone
	close(batchChan)

	workersGroup.Wait()

	for i := range backingOffChans {
		close(backingOffChans[i])
		<-backingOffDones[i]
	}

	end := time.Now()
	took := end.Sub(start)
	itemsRate := float64(itemsRead) / float64(took.Seconds())
	bytesRate := float64(bytesRead) / float64(took.Seconds())

	fmt.Printf("loaded %d items in %fsec with %d workers (mean rate %f/sec, %.2fMB/sec from stdin)\n", itemsRead, took.Seconds(), workers, itemsRate, bytesRate / (1<<20))
}

// scan reads one line at a time from stdin.
// When the requested number of lines per batch is met, send a batch over batchChan for the workers to write.
func scan(linesPerBatch int) (int64, int64) {
	buf := bufPool.Get().(*bytes.Buffer)

	var n int
	var itemsRead, bytesRead int64
	newline := []byte("\n")
	start := time.Now()
	var deadline time.Time
	var nextProgress time.Time
	if timeLimit >= 0 {
		deadline = time.Now().Add(timeLimit)
	}
	if progressInterval >= 0 {
		nextProgress = time.Now().Add(progressInterval)
	}

	scanner := bufio.NewScanner(bufio.NewReaderSize(os.Stdin, 4*1024*1024))
outer:
	for scanner.Scan() {
		if itemsRead == lineLimit {
			break
		}

		itemsRead++

		buf.Write(scanner.Bytes())
		buf.Write(newline)

		n++
		if n >= linesPerBatch {
			now := time.Now()
			if timeLimit >= 0 && now.After(deadline) {
				break outer
			}
			if progressInterval >= 0 && now.After(nextProgress) {
				fmt.Printf("[progress] scanned %d total items from stdin in %s\n", itemsRead, time.Now().Sub(start))
				nextProgress = now.Add(progressInterval)
			}
			bytesRead += int64(buf.Len())
			batchChan <- buf
			buf = bufPool.Get().(*bytes.Buffer)
			n = 0
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatalf("Error reading input: %s", err.Error())
	}

	// Finished reading input, make sure last batch goes out.
	if n > 0 {
		batchChan <- buf
	}

	// Closing inputDone signals to the application that we've read everything and can now shut down.
	close(inputDone)

	return itemsRead, bytesRead
}

// processBatches reads byte buffers from batchChan and writes them to the target server, while tracking stats on the write.
func processBatches(w *HTTPWriter, backoffSrc chan bool, backoffDst chan struct{}) {
	for batch := range batchChan {
		// Write the batch: try until backoff is not needed.
		if doLoad {
			var err error
			for {
				if useGzip {
					compressedBatch := bufPool.Get().(*bytes.Buffer)
					fasthttp.WriteGzip(compressedBatch, batch.Bytes())
					_, err = w.WriteLineProtocol(compressedBatch.Bytes(), true)
					// Return the compressed batch buffer to the pool.
					compressedBatch.Reset()
					bufPool.Put(compressedBatch)
				} else {
					_, err = w.WriteLineProtocol(batch.Bytes(), false)
				}

				if err == BackoffError {
					backoffSrc <- true
					time.Sleep(backoff)
				} else {
					backoffSrc <- false
					break
				}
			}
			if err != nil {
				log.Fatalf("Error writing: %s\n", err.Error())
			}
		}

		// Return the batch buffer to the pool.
		batch.Reset()
		bufPool.Put(batch)
	}
	workersGroup.Done()
}

func processBackoffMessages(workerId int, src chan bool, dst chan struct{}) {
	var totalBackoffSecs float64
	var start time.Time
	last := false
	for this := range src {
		if this && !last {
			start = time.Now()
			last = true
		} else if !this && last {
			took := time.Now().Sub(start)
			fmt.Printf("[worker %d] backoff took %.02fsec\n", workerId, took.Seconds())
			totalBackoffSecs += took.Seconds()
			last = false
			start = time.Now()
		}
	}
	fmt.Printf("[worker %d] backoffs took a total of %fsec of runtime\n", workerId, totalBackoffSecs)
	dst <- struct{}{}
}

func createDb(daemon_url, dbname string, replicationFactor int) error {
	u, err := url.Parse(daemon_url)
	if err != nil {
		return err
	}

	// serialize params the right way:
	u.Path = "query"
	v := u.Query()
	v.Set("consistency", "all")
	v.Set("q", fmt.Sprintf("CREATE DATABASE %s WITH REPLICATION %d", dbname, replicationFactor))
	u.RawQuery = v.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// does the body need to be read into the void?

	if resp.StatusCode != 200 {
		return fmt.Errorf("bad db create")
	}
	return nil
}

// listDatabases lists the existing databases in InfluxDB.
func listDatabases(daemonUrl string) ([]string, error) {
	u := fmt.Sprintf("%s/query?q=show%%20databases", daemonUrl)
	resp, err := http.Get(u)
	if err != nil {
		return nil, fmt.Errorf("listDatabases error: %s", err.Error())
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Do ad-hoc parsing to find existing database names:
	// {"results":[{"series":[{"name":"databases","columns":["name"],"values":[["_internal"],["benchmark_db"]]}]}]}%
	type listingType struct {
		Results []struct {
			Series []struct {
				Values [][]string
			}
		}
	}
	var listing listingType
	err = json.Unmarshal(body, &listing)
	if err != nil {
		return nil, err
	}

	ret := []string{}
	for _, nestedName := range listing.Results[0].Series[0].Values {
		name := nestedName[0]
		// the _internal database is skipped:
		if name == "_internal" {
			continue
		}
		ret = append(ret, name)
	}
	return ret, nil
}
<commit_msg>better progress interval<commit_after>// bulk_load_influx loads an InfluxDB daemon with data from stdin.
//
// The caller is responsible for assuring that the database is empty before
// bulk load.
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/pkg/profile"
	"github.com/valyala/fasthttp"
)

// Program option vars:
var (
	csvDaemonUrls     string
	daemonUrls        []string
	dbName            string
	replicationFactor int
	workers           int
	lineLimit         int64
	batchSize         int
	backoff           time.Duration
	timeLimit         time.Duration
	progressInterval  time.Duration
	doLoad            bool
	doDBCreate        bool
	useGzip           bool
	doAbortOnExist    bool
	memprofile        bool
)

// Global vars
var (
	bufPool         sync.Pool
	batchChan       chan *bytes.Buffer
	inputDone       chan struct{}
	workersGroup    sync.WaitGroup
	backingOffChans []chan bool
	backingOffDones []chan struct{}
)

// Parse args:
func init() {
	flag.StringVar(&csvDaemonUrls, "urls", "http://localhost:8086", "InfluxDB URLs, comma-separated. Will be used in a round-robin fashion.")
	flag.StringVar(&dbName, "db", "benchmark_db", "Database name.")
	flag.IntVar(&replicationFactor, "replication-factor", 2, "Cluster replication factor (only applies to clustered databases).")
	flag.IntVar(&batchSize, "batch-size", 5000, "Batch size (input lines).")
	flag.IntVar(&workers, "workers", 1, "Number of parallel requests to make.")
	flag.Int64Var(&lineLimit, "line-limit", -1, "Number of lines to read from stdin before quitting.")
	flag.DurationVar(&backoff, "backoff", time.Second, "Time to sleep between requests when server indicates backpressure is needed.")
	flag.DurationVar(&timeLimit, "time-limit", -1, "Maximum duration to run (-1 is the default: no limit).")
	flag.DurationVar(&progressInterval, "progress-interval", -1, "Duration between printing progress messages.")
	flag.BoolVar(&useGzip, "gzip", true, "Whether to gzip encode requests (default true).")
	flag.BoolVar(&doLoad, "do-load", true, "Whether to write data. Set this flag to false to check input read speed.")
	flag.BoolVar(&doDBCreate, "do-db-create", true, "Whether to create the database.")
	flag.BoolVar(&doAbortOnExist, "do-abort-on-exist", true, "Whether to abort if the destination database already exists.")
	flag.BoolVar(&memprofile, "memprofile", false, "Whether to write a memprofile (file automatically determined).")

	flag.Parse()

	daemonUrls = strings.Split(csvDaemonUrls, ",")
	if len(daemonUrls) == 0 {
		log.Fatal("missing 'urls' flag")
	}
	fmt.Printf("daemon URLs: %v\n", daemonUrls)
}

func main() {
	if memprofile {
		p := profile.Start(profile.MemProfile)
		defer p.Stop()
	}
	if doLoad && doDBCreate {
		// check that there are no pre-existing databases:
		existingDatabases, err := listDatabases(daemonUrls[0])
		if err != nil {
			log.Fatal(err)
		}

		if len(existingDatabases) > 0 {
			if doAbortOnExist {
				log.Fatalf("There are databases already in the data store. If you know what you are doing, run the command:\ncurl 'http://localhost:8086/query?q=drop%%20database%%20%s'\n", existingDatabases[0])
			} else {
				log.Printf("Info: there are databases already in the data store.")
			}
		}

		if len(existingDatabases) == 0 {
			err = createDb(daemonUrls[0], dbName, replicationFactor)
			if err != nil {
				log.Fatal(err)
			}
			time.Sleep(1000 * time.Millisecond)
		}
	}

	bufPool = sync.Pool{
		New: func() interface{} {
			return bytes.NewBuffer(make([]byte, 0, 4*1024*1024))
		},
	}

	batchChan = make(chan *bytes.Buffer, workers)
	inputDone = make(chan struct{})

	backingOffChans = make([]chan bool, workers)
	backingOffDones = make([]chan struct{}, workers)

	for i := 0; i < workers; i++ {
		daemonUrl := daemonUrls[i%len(daemonUrls)]
		backingOffChans[i] = make(chan bool, 100)
		backingOffDones[i] = make(chan struct{})
		workersGroup.Add(1)
		cfg := HTTPWriterConfig{
			DebugInfo:      fmt.Sprintf("worker #%d, dest url: %s", i, daemonUrl),
			Host:           daemonUrl,
			Database:       dbName,
			BackingOffChan: backingOffChans[i],
			BackingOffDone: backingOffDones[i],
		}
		go processBatches(NewHTTPWriter(cfg), backingOffChans[i], backingOffDones[i])
		go processBackoffMessages(i, backingOffChans[i], backingOffDones[i])
	}

	start := time.Now()
	itemsRead, bytesRead := scan(batchSize)

	<-inputDone
	close(batchChan)

	workersGroup.Wait()

	for i := range backingOffChans {
		close(backingOffChans[i])
		<-backingOffDones[i]
	}

	end := time.Now()
	took := end.Sub(start)
	itemsRate := float64(itemsRead) / float64(took.Seconds())
	bytesRate := float64(bytesRead) / float64(took.Seconds())

	fmt.Printf("loaded %d items in %fsec with %d workers (mean rate %f/sec, %.2fMB/sec from stdin)\n", itemsRead, took.Seconds(), workers, itemsRate, bytesRate / (1<<20))
}

// scan reads one line at a time from stdin.
// When the requested number of lines per batch is met, send a batch over batchChan for the workers to write.
func scan(linesPerBatch int) (int64, int64) {
	buf := bufPool.Get().(*bytes.Buffer)

	var n int
	var itemsRead, bytesRead int64
	newline := []byte("\n")
	var deadline time.Time
	if timeLimit >= 0 {
		deadline = time.Now().Add(timeLimit)
	}

	// interval tracking
	intervalStart := time.Now()
	var nextProgress time.Time
	var intervalLinesRead int64
	if progressInterval >= 0 {
		nextProgress = time.Now().Add(progressInterval)
	}

	scanner := bufio.NewScanner(bufio.NewReaderSize(os.Stdin, 4*1024*1024))
outer:
	for scanner.Scan() {
		if itemsRead == lineLimit {
			break
		}

		itemsRead++
		intervalLinesRead++

		buf.Write(scanner.Bytes())
		buf.Write(newline)

		n++
		if n >= linesPerBatch {
			now := time.Now()
			if timeLimit >= 0 && now.After(deadline) {
				break outer
			}

			// interval tracking
			if progressInterval >= 0 && now.After(nextProgress) {
				fmt.Printf("[interval_progress_items] %s, %s, %d\n", intervalStart.Format(time.RFC3339), now.Format(time.RFC3339), intervalLinesRead)
				nextProgress = now.Add(progressInterval)
				intervalLinesRead = 0
				intervalStart = now
			}

			bytesRead += int64(buf.Len())
			batchChan <- buf
			buf = bufPool.Get().(*bytes.Buffer)
			n = 0
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatalf("Error reading input: %s", err.Error())
	}

	// Finished reading input, make sure last batch goes out.
	if n > 0 {
		batchChan <- buf
	}

	// Closing inputDone signals to the application that we've read everything and can now shut down.
	close(inputDone)

	return itemsRead, bytesRead
}

// processBatches reads byte buffers from batchChan and writes them to the target server, while tracking stats on the write.
func processBatches(w *HTTPWriter, backoffSrc chan bool, backoffDst chan struct{}) {
	for batch := range batchChan {
		// Write the batch: try until backoff is not needed.
		if doLoad {
			var err error
			for {
				if useGzip {
					compressedBatch := bufPool.Get().(*bytes.Buffer)
					fasthttp.WriteGzip(compressedBatch, batch.Bytes())
					_, err = w.WriteLineProtocol(compressedBatch.Bytes(), true)
					// Return the compressed batch buffer to the pool.
					compressedBatch.Reset()
					bufPool.Put(compressedBatch)
				} else {
					_, err = w.WriteLineProtocol(batch.Bytes(), false)
				}

				if err == BackoffError {
					backoffSrc <- true
					time.Sleep(backoff)
				} else {
					backoffSrc <- false
					break
				}
			}
			if err != nil {
				log.Fatalf("Error writing: %s\n", err.Error())
			}
		}

		// Return the batch buffer to the pool.
		batch.Reset()
		bufPool.Put(batch)
	}
	workersGroup.Done()
}

func processBackoffMessages(workerId int, src chan bool, dst chan struct{}) {
	var totalBackoffSecs float64
	var start time.Time
	last := false
	for this := range src {
		if this && !last {
			start = time.Now()
			last = true
		} else if !this && last {
			took := time.Now().Sub(start)
			fmt.Printf("[worker %d] backoff took %.02fsec\n", workerId, took.Seconds())
			totalBackoffSecs += took.Seconds()
			last = false
			start = time.Now()
		}
	}
	fmt.Printf("[worker %d] backoffs took a total of %fsec of runtime\n", workerId, totalBackoffSecs)
	dst <- struct{}{}
}

func createDb(daemon_url, dbname string, replicationFactor int) error {
	u, err := url.Parse(daemon_url)
	if err != nil {
		return err
	}

	// serialize params the right way:
	u.Path = "query"
	v := u.Query()
	v.Set("consistency", "all")
	v.Set("q", fmt.Sprintf("CREATE DATABASE %s WITH REPLICATION %d", dbname, replicationFactor))
	u.RawQuery = v.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// does the body need to be read into the void?

	if resp.StatusCode != 200 {
		return fmt.Errorf("bad db create")
	}
	return nil
}

// listDatabases lists the existing databases in InfluxDB.
func listDatabases(daemonUrl string) ([]string, error) {
	u := fmt.Sprintf("%s/query?q=show%%20databases", daemonUrl)
	resp, err := http.Get(u)
	if err != nil {
		return nil, fmt.Errorf("listDatabases error: %s", err.Error())
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Do ad-hoc parsing to find existing database names:
	// {"results":[{"series":[{"name":"databases","columns":["name"],"values":[["_internal"],["benchmark_db"]]}]}]}%
	type listingType struct {
		Results []struct {
			Series []struct {
				Values [][]string
			}
		}
	}
	var listing listingType
	err = json.Unmarshal(body, &listing)
	if err != nil {
		return nil, err
	}

	ret := []string{}
	for _, nestedName := range listing.Results[0].Series[0].Values {
		name := nestedName[0]
		// the _internal database is skipped:
		if name == "_internal" {
			continue
		}
		ret = append(ret, name)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The debugnewvm command creates and destroys a VM-based GCE buildlet
// with lots of logging for debugging. Nothing depends on this.
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"golang.org/x/build/buildenv"
	"golang.org/x/build/buildlet"
	"golang.org/x/build/dashboard"
	"golang.org/x/build/internal/buildgo"
	"golang.org/x/oauth2"
	compute "google.golang.org/api/compute/v1"
)

var (
	hostType      = flag.String("host", "", "host type to create")
	overrideImage = flag.String("override-image", "", "if non-empty, an alternate GCE VM image or container image to use, depending on the host type")
	serial        = flag.Bool("serial", true, "watch serial")
	pauseAfterUp  = flag.Duration("pause-after-up", 0, "pause for this duration before buildlet is destroyed")
	sleepSec      = flag.Int("sleep-test-secs", 0, "number of seconds to sleep when buildlet comes up, to test time source; OpenBSD only for now")

	runBuild = flag.String("run-build", "", "optional builder name to run all.bash or make.bash for")
	makeOnly = flag.Bool("make-only", false, "if a --run-build builder name is given, this controls whether make.bash or all.bash is run")
	buildRev = flag.String("rev", "master", "if --run-build is specified, the git hash or branch name to build")
)

var (
	computeSvc *compute.Service
	env        *buildenv.Environment
)

func main() {
	buildenv.RegisterFlags()
	flag.Parse()

	var bconf *dashboard.BuildConfig
	if *runBuild != "" {
		var ok bool
		bconf, ok = dashboard.Builders[*runBuild]
		if !ok {
			log.Fatalf("unknown builder %q", *runBuild)
		}
		if *hostType == "" {
			*hostType = bconf.HostType
		}
	}

	if *hostType == "" {
		log.Fatalf("missing --host (or --run-build)")
	}
	if *sleepSec != 0 && !strings.Contains(*hostType, "openbsd") {
		log.Fatalf("The --sleep-test-secs is currently only supported for openbsd hosts.")
	}

	hconf, ok := dashboard.Hosts[*hostType]
	if !ok {
		log.Fatalf("unknown host type %q", *hostType)
	}
	if !hconf.IsVM() && !hconf.IsContainer() {
		log.Fatalf("host type %q is type %q; want a VM or container host type", *hostType, hconf.PoolName())
	}
	if img := *overrideImage; img != "" {
		if hconf.IsContainer() {
			hconf.ContainerImage = img
		} else {
			hconf.VMImage = img
		}
	}
	vmImageSummary := fmt.Sprintf("%q", hconf.VMImage)
	if hconf.IsContainer() {
		containerHost := hconf.ContainerVMImage()
		if containerHost == "" {
			containerHost = "default container host"
		}
		vmImageSummary = fmt.Sprintf("%s, running container %q", containerHost, hconf.ContainerImage)
	}

	env = buildenv.FromFlags()
	ctx := context.Background()

	buildenv.CheckUserCredentials()
	creds, err := env.Credentials(ctx)
	if err != nil {
		log.Fatal(err)
	}
	computeSvc, _ = compute.New(oauth2.NewClient(ctx, creds.TokenSource))

	name := fmt.Sprintf("debug-temp-%d", time.Now().Unix())

	log.Printf("Creating %s (with VM image %s)", name, vmImageSummary)
	bc, err := buildlet.StartNewVM(creds, env, name, *hostType, buildlet.VMOpts{
		OnInstanceRequested: func() { log.Printf("instance requested") },
		OnInstanceCreated: func() {
			log.Printf("instance created")
			if *serial {
				go watchSerial(name)
			}
		},
		OnGotInstanceInfo: func() { log.Printf("got instance info") },
		OnBeginBuildletProbe: func(buildletURL string) {
			log.Printf("About to hit %s to see if buildlet is up yet...", buildletURL)
		},
		OnEndBuildletProbe: func(res *http.Response, err error) {
			if err != nil {
				log.Printf("client buildlet probe error: %v", err)
				return
			}
			log.Printf("buildlet probe: %s", res.Status)
		},
	})
	if err != nil {
		log.Fatalf("StartNewVM: %v", err)
	}
	dir, err := bc.WorkDir(ctx)
	log.Printf("WorkDir: %v, %v", dir, err)

	if *sleepSec > 0 {
		bc.Exec(ctx, "sysctl", buildlet.ExecOpts{
			Output:      os.Stdout,
			SystemLevel: true,
			Args:        []string{"kern.timecounter.hardware"},
		})
		bc.Exec(ctx, "bash", buildlet.ExecOpts{
			Output:      os.Stdout,
			SystemLevel: true,
			Args:        []string{"-c", "rdate -p -v time.nist.gov; sleep " + fmt.Sprint(*sleepSec) + "; rdate -p -v time.nist.gov"},
		})
	}

	var buildFailed bool
	if *runBuild != "" {
		// Push GOROOT_BOOTSTRAP, if needed.
		if u := bconf.GoBootstrapURL(env); u != "" {
			log.Printf("Pushing 'go1.4' Go bootstrap dir ...")
			const bootstrapDir = "go1.4" // might be newer; name is the default
			if err := bc.PutTarFromURL(ctx, u, bootstrapDir); err != nil {
				bc.Close()
				log.Fatalf("Putting Go bootstrap: %v", err)
			}
		}

		// Push Go code
		log.Printf("Pushing 'go' dir...")
		goTarGz := "https://go.googlesource.com/go/+archive/" + *buildRev + ".tar.gz"
		if err := bc.PutTarFromURL(ctx, goTarGz, "go"); err != nil {
			bc.Close()
			log.Fatalf("Putting go code: %v", err)
		}

		// Push a synthetic VERSION file to prevent git usage:
		if err := bc.PutTar(ctx, buildgo.VersionTgz(*buildRev), "go"); err != nil {
			bc.Close()
			log.Fatalf("Putting VERSION file: %v", err)
		}

		script := bconf.AllScript()
		if *makeOnly {
			script = bconf.MakeScript()
		}
		t0 := time.Now()
		log.Printf("Running %s ...", script)
		remoteErr, err := bc.Exec(ctx, path.Join("go", script), buildlet.ExecOpts{
			Output:   os.Stdout,
			ExtraEnv: bconf.Env(),
			Debug:    true,
			Args:     bconf.AllScriptArgs(),
		})
		if err != nil {
			log.Fatalf("error trying to run %s: %v", script, err)
		}
		if remoteErr != nil {
			log.Printf("remote failure running %s: %v", script, remoteErr)
			buildFailed = true
		} else {
			log.Printf("ran %s in %v", script, time.Since(t0).Round(time.Second))
		}
	}

	if *pauseAfterUp != 0 {
		log.Printf("Sleeping for %v before shutting down...", *pauseAfterUp)
		time.Sleep(*pauseAfterUp)
	}
	if err := bc.Close(); err != nil {
		log.Fatalf("Close: %v", err)
	}
	log.Printf("done.")
	time.Sleep(2 * time.Second) // wait for serial logging to catch up

	if buildFailed {
		os.Exit(1)
	}
}

// watchSerial streams the named VM's serial port to log.Printf. It's roughly:
//   gcloud compute connect-to-serial-port --zone=xxx $NAME
// but in Go and works. For some reason, gcloud doesn't work as a
// child process and has weird errors.
func watchSerial(name string) {
	start := int64(0)
	indent := strings.Repeat(" ", len("2017/07/25 06:37:14 SERIAL: "))
	for {
		sout, err := computeSvc.Instances.GetSerialPortOutput(env.ProjectName, env.Zone, name).Start(start).Do()
		if err != nil {
			log.Printf("serial output error: %v", err)
			return
		}
		moved := sout.Next != start
		start = sout.Next
		contents := strings.Replace(strings.TrimSpace(sout.Contents), "\r\n", "\r\n"+indent, -1)
		if contents != "" {
			log.Printf("SERIAL: %s", contents)
		}
		if !moved {
			time.Sleep(1 * time.Second)
		}
	}
}
<commit_msg>cmd/debugnewvm: add --zone flag<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The debugnewvm command creates and destroys a VM-based GCE buildlet
// with lots of logging for debugging. Nothing depends on this.
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"golang.org/x/build/buildenv"
	"golang.org/x/build/buildlet"
	"golang.org/x/build/dashboard"
	"golang.org/x/build/internal/buildgo"
	"golang.org/x/oauth2"
	compute "google.golang.org/api/compute/v1"
)

var (
	hostType      = flag.String("host", "", "host type to create")
	zone          = flag.String("zone", "", "if non-empty, force a certain GCP zone")
	overrideImage = flag.String("override-image", "", "if non-empty, an alternate GCE VM image or container image to use, depending on the host type")
	serial        = flag.Bool("serial", true, "watch serial")
	pauseAfterUp  = flag.Duration("pause-after-up", 0, "pause for this duration before buildlet is destroyed")
	sleepSec      = flag.Int("sleep-test-secs", 0, "number of seconds to sleep when buildlet comes up, to test time source; OpenBSD only for now")

	runBuild = flag.String("run-build", "", "optional builder name to run all.bash or make.bash for")
	makeOnly = flag.Bool("make-only", false, "if a --run-build builder name is given, this controls whether make.bash or all.bash is run")
	buildRev = flag.String("rev", "master", "if --run-build is specified, the git hash or branch name to build")
)

var (
	computeSvc *compute.Service
	env        *buildenv.Environment
)

func main() {
	buildenv.RegisterFlags()
	flag.Parse()

	var bconf *dashboard.BuildConfig
	if *runBuild != "" {
		var ok bool
		bconf, ok = dashboard.Builders[*runBuild]
		if !ok {
			log.Fatalf("unknown builder %q", *runBuild)
		}
		if *hostType == "" {
			*hostType = bconf.HostType
		}
	}

	if *hostType == "" {
		log.Fatalf("missing --host (or --run-build)")
	}
	if *sleepSec != 0 && !strings.Contains(*hostType, "openbsd") {
		log.Fatalf("The --sleep-test-secs is currently only supported for openbsd hosts.")
	}

	hconf, ok := dashboard.Hosts[*hostType]
	if !ok {
		log.Fatalf("unknown host type %q", *hostType)
	}
	if !hconf.IsVM() && !hconf.IsContainer() {
		log.Fatalf("host type %q is type %q; want a VM or container host type", *hostType, hconf.PoolName())
	}
	if img := *overrideImage; img != "" {
		if hconf.IsContainer() {
			hconf.ContainerImage = img
		} else {
			hconf.VMImage = img
		}
	}
	vmImageSummary := fmt.Sprintf("%q", hconf.VMImage)
	if hconf.IsContainer() {
		containerHost := hconf.ContainerVMImage()
		if containerHost == "" {
			containerHost = "default container host"
		}
		vmImageSummary = fmt.Sprintf("%s, running container %q", containerHost, hconf.ContainerImage)
	}

	env = buildenv.FromFlags()
	if *zone != "" {
		env.Zone = *zone
	}

	ctx := context.Background()

	buildenv.CheckUserCredentials()
	creds, err := env.Credentials(ctx)
	if err != nil {
		log.Fatal(err)
	}
	computeSvc, _ = compute.New(oauth2.NewClient(ctx, creds.TokenSource))

	name := fmt.Sprintf("debug-temp-%d", time.Now().Unix())

	log.Printf("Creating %s (with VM image %s)", name, vmImageSummary)
	bc, err := buildlet.StartNewVM(creds, env, name, *hostType, buildlet.VMOpts{
		OnInstanceRequested: func() { log.Printf("instance requested") },
		OnInstanceCreated: func() {
			log.Printf("instance created")
			if *serial {
				go watchSerial(name)
			}
		},
		OnGotInstanceInfo: func() { log.Printf("got instance info") },
		OnBeginBuildletProbe: func(buildletURL string) {
			log.Printf("About to hit %s to see if buildlet is up yet...", buildletURL)
		},
		OnEndBuildletProbe: func(res *http.Response, err error) {
			if err != nil {
				log.Printf("client buildlet probe error: %v", err)
				return
			}
			log.Printf("buildlet probe: %s", res.Status)
		},
	})
	if err != nil {
		log.Fatalf("StartNewVM: %v", err)
	}
	dir, err := bc.WorkDir(ctx)
	log.Printf("WorkDir: %v, %v", dir, err)

	if *sleepSec > 0 {
		bc.Exec(ctx, "sysctl", buildlet.ExecOpts{
			Output:      os.Stdout,
			SystemLevel: true,
			Args:        []string{"kern.timecounter.hardware"},
		})
		bc.Exec(ctx, "bash", buildlet.ExecOpts{
			Output:      os.Stdout,
			SystemLevel: true,
			Args:        []string{"-c", "rdate -p -v time.nist.gov; sleep " + fmt.Sprint(*sleepSec) + "; rdate -p -v time.nist.gov"},
		})
	}

	var buildFailed bool
	if *runBuild != "" {
		// Push GOROOT_BOOTSTRAP, if needed.
		if u := bconf.GoBootstrapURL(env); u != "" {
			log.Printf("Pushing 'go1.4' Go bootstrap dir ...")
			const bootstrapDir = "go1.4" // might be newer; name is the default
			if err := bc.PutTarFromURL(ctx, u, bootstrapDir); err != nil {
				bc.Close()
				log.Fatalf("Putting Go bootstrap: %v", err)
			}
		}

		// Push Go code
		log.Printf("Pushing 'go' dir...")
		goTarGz := "https://go.googlesource.com/go/+archive/" + *buildRev + ".tar.gz"
		if err := bc.PutTarFromURL(ctx, goTarGz, "go"); err != nil {
			bc.Close()
			log.Fatalf("Putting go code: %v", err)
		}

		// Push a synthetic VERSION file to prevent git usage:
		if err := bc.PutTar(ctx, buildgo.VersionTgz(*buildRev), "go"); err != nil {
			bc.Close()
			log.Fatalf("Putting VERSION file: %v", err)
		}

		script := bconf.AllScript()
		if *makeOnly {
			script = bconf.MakeScript()
		}
		t0 := time.Now()
		log.Printf("Running %s ...", script)
		remoteErr, err := bc.Exec(ctx, path.Join("go", script), buildlet.ExecOpts{
			Output:   os.Stdout,
			ExtraEnv: bconf.Env(),
			Debug:    true,
			Args:     bconf.AllScriptArgs(),
		})
		if err != nil {
			log.Fatalf("error trying to run %s: %v", script, err)
		}
		if remoteErr != nil {
			log.Printf("remote failure running %s: %v", script, remoteErr)
			buildFailed = true
		} else {
			log.Printf("ran %s in %v", script, time.Since(t0).Round(time.Second))
		}
	}

	if *pauseAfterUp != 0 {
		log.Printf("Sleeping for %v before shutting down...", *pauseAfterUp)
		time.Sleep(*pauseAfterUp)
	}
	if err := bc.Close(); err != nil {
		log.Fatalf("Close: %v", err)
	}
	log.Printf("done.")
	time.Sleep(2 * time.Second) // wait for serial logging to catch up

	if buildFailed {
		os.Exit(1)
	}
}

// watchSerial streams the named VM's serial port to log.Printf. It's roughly:
//   gcloud compute connect-to-serial-port --zone=xxx $NAME
// but in Go and works. For some reason, gcloud doesn't work as a
// child process and has weird errors.
func watchSerial(name string) {
	start := int64(0)
	indent := strings.Repeat(" ", len("2017/07/25 06:37:14 SERIAL: "))
	for {
		sout, err := computeSvc.Instances.GetSerialPortOutput(env.ProjectName, env.Zone, name).Start(start).Do()
		if err != nil {
			log.Printf("serial output error: %v", err)
			return
		}
		moved := sout.Next != start
		start = sout.Next
		contents := strings.Replace(strings.TrimSpace(sout.Contents), "\r\n", "\r\n"+indent, -1)
		if contents != "" {
			log.Printf("SERIAL: %s", contents)
		}
		if !moved {
			time.Sleep(1 * time.Second)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"path/filepath"

	"github.com/kopia/kopia/backup"
	"github.com/kopia/kopia/repo"
	"github.com/kopia/kopia/vault"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	backupsCommand    = app.Command("backups", "List history of file or directory backups.")
	backupsPath       = backupsCommand.Arg("source", "File or directory to show history of.").String()
	maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results.").Default("100").Int()
)

func findBackups(vlt *vault.Vault, path string) ([]string, string, error) {
	var relPath string

	for len(path) > 0 {
		manifest := backup.Manifest{
			Source:   path,
			HostName: getBackupHostName(),
			UserName: getBackupUser(),
		}

		prefix := manifest.SourceID() + "."

		list, err := vlt.List("B" + prefix)
		if err != nil {
			return nil, "", err
		}

		if len(list) > 0 {
			return list, relPath, nil
		}

		if len(relPath) > 0 {
			relPath = filepath.Base(path) + "/" + relPath
		} else {
			relPath = filepath.Base(path)
		}

		log.Printf("No backups of %v@%v:%v", manifest.UserName, manifest.HostName, manifest.Source)

		parent := filepath.Dir(path)
		if parent == path {
			break
		}
		path = parent
	}

	return nil, "", nil
}

func runBackupsCommand(context *kingpin.ParseContext) error {
	var options []repo.RepositoryOption

	if *backupWriteBack > 0 {
		options = append(options, repo.WriteBack(*backupWriteBack))
	}

	if *backupWriteLimit > 0 {
		options = append(options, repo.WriteLimit(*backupWriteLimit*1000000))

	}

	vlt, err := openVault()
	if err != nil {
		return err
	}

	mgr, err := vlt.OpenRepository()
	if err != nil {
		return err
	}
	defer mgr.Close()

	var previous []string
	var relPath string

	if *backupsPath != "" {
		path, err := filepath.Abs(*backupsPath)
		if err != nil {
			return fmt.Errorf("invalid directory: '%s': %s", *backupsPath, err)
		}

		previous, relPath, err = findBackups(vlt, filepath.Clean(path))
		if relPath != "" {
			relPath = "/" + relPath
		}
	} else {
		previous, err = vlt.List("B")
	}

	if err != nil {
		return fmt.Errorf("cannot list backups: %v", err)
	}

	var lastHost string
	var lastUser string
	var lastSource string
	var count int

	for _, n := range previous {
		m, err := loadBackupManifest(vlt, n)
		if err != nil {
			return fmt.Errorf("error loading previous backup: %v", err)
		}

		if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource {
			log.Printf("%v@%v:%v", m.UserName, m.HostName, m.Source)
			lastSource = m.Source
			lastUser = m.UserName
			lastHost = m.HostName
			count = 0
		}

		if count < *maxResultsPerPath {
			log.Printf("  %v%v %v", m.Handle, relPath, m.StartTime.Format("2006-01-02 15:04:05 MST"))
			count++
		}
	}

	return nil
}

func init() {
	backupsCommand.Action(runBackupsCommand)
	backupsCommand.Flag("host", "Override backup hostname.").StringVar(backupHostName)
	backupsCommand.Flag("user", "Override backup user.").StringVar(backupUser)
}
<commit_msg>removed unnecessary copy/pasted code<commit_after>package main

import (
	"fmt"
	"log"
	"path/filepath"

	"github.com/kopia/kopia/backup"
	"github.com/kopia/kopia/vault"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	backupsCommand    = app.Command("backups", "List history of file or directory backups.")
	backupsPath       = backupsCommand.Arg("source", "File or directory to show history of.").String()
	maxResultsPerPath = backupsCommand.Flag("maxresults", "Maximum number of results.").Default("100").Int()
)

func findBackups(vlt *vault.Vault, path string) ([]string, string, error) {
	var relPath string

	for len(path) > 0 {
		manifest := backup.Manifest{
			Source:   path,
			HostName: getBackupHostName(),
			UserName: getBackupUser(),
		}

		prefix := manifest.SourceID() + "."

		list, err := vlt.List("B" + prefix)
		if err != nil {
			return nil, "", err
		}

		if len(list) > 0 {
			return list, relPath, nil
		}

		if len(relPath) > 0 {
			relPath = filepath.Base(path) + "/" + relPath
		} else {
			relPath = filepath.Base(path)
		}

		log.Printf("No backups of %v@%v:%v", manifest.UserName, manifest.HostName, manifest.Source)

		parent := filepath.Dir(path)
		if parent == path {
			break
		}
		path = parent
	}

	return nil, "", nil
}

func runBackupsCommand(context *kingpin.ParseContext) error {
	vlt, err := openVault()
	if err != nil {
		return err
	}

	mgr, err := vlt.OpenRepository()
	if err != nil {
		return err
	}
	defer mgr.Close()

	var previous []string
	var relPath string

	if *backupsPath != "" {
		path, err := filepath.Abs(*backupsPath)
		if err != nil {
			return fmt.Errorf("invalid directory: '%s': %s", *backupsPath, err)
		}

		previous, relPath, err = findBackups(vlt, filepath.Clean(path))
		if relPath != "" {
			relPath = "/" + relPath
		}
	} else {
		previous, err = vlt.List("B")
	}

	if err != nil {
		return fmt.Errorf("cannot list backups: %v", err)
	}

	var lastHost string
	var lastUser string
	var lastSource string
	var count int

	for _, n := range previous {
		m, err := loadBackupManifest(vlt, n)
		if err != nil {
			return fmt.Errorf("error loading previous backup: %v", err)
		}

		if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource {
			log.Printf("%v@%v:%v", m.UserName, m.HostName, m.Source)
			lastSource = m.Source
			lastUser = m.UserName
			lastHost = m.HostName
			count = 0
		}

		if count < *maxResultsPerPath {
			log.Printf("  %v%v %v", m.Handle, relPath, m.StartTime.Format("2006-01-02 15:04:05 MST"))
			count++
		}
	}

	return nil
}

func init() {
	backupsCommand.Action(runBackupsCommand)
	backupsCommand.Flag("host", "Override backup hostname.").StringVar(backupHostName)
	backupsCommand.Flag("user", "Override backup user.").StringVar(backupUser)
}
<|endoftext|>


<commit_before>// svgpyramid draw a pyramid SVG infographic.
package main

import (
	"flag"
	"fmt"
	"math"
	"os"

	"github.com/ajstarks/svgo/float"
)

func choseLineStyle(color, highlightColor string, highlightText int, nums ...int) string {
	for _, num := range nums {
		if num == highlightText {
			return fmt.Sprintf("stroke:%s;stroke-width:3;stroke-linecap:round", highlightColor)
		}
	}
	return fmt.Sprintf("stroke:%s;stroke-width:3;stroke-linecap:round", color)
}

func choseTextStyle(color, highlightColor string, highlightText, num int) string {
	if num == highlightText {
		return fmt.Sprintf("text-anchor:middle;font-size:25px;fill:%s", highlightColor)
	}
	return fmt.Sprintf("text-anchor:middle;font-size:25px;fill:%s", color)
}

func svgPyramid(color, highlightColor string, highlightText int) {
	width := 500.0
	height := 500.0
	xOffset := 50.0
	yOffset := 50.0

	sideLength := width
	halfSideLength := sideLength / 2
	triangleHeight := math.Sqrt(sideLength*sideLength - halfSideLength*halfSideLength)

	canvas := svg.New(os.Stdout)
	canvas.Start(width+xOffset+xOffset, height+yOffset+yOffset)

	h := math.Sqrt(sideLength*sideLength-halfSideLength*halfSideLength) / 6.5
	w := halfSideLength / 6.5

	// left triangle line
	s := choseLineStyle(color, highlightColor, highlightText, 7)
	canvas.Line(6.5*w+xOffset, 0+yOffset, 5.0*w+xOffset, 1.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 5)
	canvas.Line(5.0*w+xOffset, 1.5*h+yOffset, 3.5*w+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 3)
	canvas.Line(3.5*w+xOffset, 3.0*h+yOffset, 2.0*w+xOffset, 4.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 2)
	canvas.Line(2.0*w+xOffset, 4.5*h+yOffset, 1.0*w+xOffset, 5.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 1)
	canvas.Line(1.0*w+xOffset, 5.5*h+yOffset, 0+xOffset, 6.5*h+yOffset, s)

	// right triangle line
	s = choseLineStyle(color, highlightColor, highlightText, 7)
	canvas.Line(6.5*w+xOffset, 0+yOffset, 8.0*w+xOffset, 1.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 6)
	canvas.Line(8.0*w+xOffset, 1.5*h+yOffset, 9.5*w+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 4)
	canvas.Line(9.5*w+xOffset, 3.0*h+yOffset, 11.0*w+xOffset, 4.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 2)
	canvas.Line(11.0*w+xOffset, 4.5*h+yOffset, 12.0*w+xOffset, 5.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 1)
	canvas.Line(12.0*w+xOffset, 5.5*h+yOffset, 13.0*w+xOffset, 6.5*h+yOffset, s)

	// bottom triangle line
	s = choseLineStyle(color, highlightColor, highlightText, 1)
	canvas.Line(0+xOffset, triangleHeight+yOffset, sideLength+xOffset, triangleHeight+yOffset, s)

	// first horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 5, 7)
	canvas.Line(5.0*w+xOffset, 1.5*h+yOffset, 6.5*w+xOffset, 1.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 6, 7)
	canvas.Line(6.5*w+xOffset, 1.5*h+yOffset, 8.0*w+xOffset, 1.5*h+yOffset, s)

	// second horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 3, 5)
	canvas.Line(3.5*w+xOffset, 3.0*h+yOffset, 6.5*w+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 4, 6)
	canvas.Line(6.5*w+xOffset, 3.0*h+yOffset, 9.5*w+xOffset, 3.0*h+yOffset, s)

	// third horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 2, 3)
	canvas.Line(2.0*w+xOffset, 4.5*h+yOffset, 6.5*w+xOffset, 4.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 2, 4)
	canvas.Line(6.5*w+xOffset, 4.5*h+yOffset, 11.0*w+xOffset, 4.5*h+yOffset, s)

	// fourth horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 1, 2)
	canvas.Line(1.0*w+xOffset, 5.5*h+yOffset, 12.0*w+xOffset, 5.5*h+yOffset, s)

	// vertical line
	s = choseLineStyle(color, highlightColor, highlightText, 5, 6)
	canvas.Line(halfSideLength+xOffset, 1.5*h+yOffset, halfSideLength+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 3, 4)
	canvas.Line(halfSideLength+xOffset, 3.0*h+yOffset, halfSideLength+xOffset, 4.5*h+yOffset, s)

	// 1.
	style := choseTextStyle(color, highlightColor, highlightText, 1)
	canvas.Text(halfSideLength+xOffset, 6.125*h+yOffset, "secure devices", style)

	// 2.
	style = choseTextStyle(color, highlightColor, highlightText, 2)
	canvas.Text(halfSideLength+xOffset, 5.125*h+yOffset, "secure software", style)

	// 3.
	style = choseTextStyle(color, highlightColor, highlightText, 3)
	canvas.Text(4.75*w+xOffset, 3.75*h+yOffset, "anon.", style)
	canvas.Text(4.5*w+xOffset, 4.25*h+yOffset, "messaging", style)

	// 4.
	style = choseTextStyle(color, highlightColor, highlightText, 4)
	canvas.Text((6.5+1.75)*w+xOffset, 3.75*h+yOffset, "digital", style)
	canvas.Text((6.5+1.75)*w+xOffset, 4.25*h+yOffset, "cash", style)

	// 5.
	style = choseTextStyle(color, highlightColor, highlightText, 5)
	canvas.Text(5.25*w+xOffset, 2.75*h+yOffset, "nyms", style)

	// 6.
	style = choseTextStyle(color, highlightColor, highlightText, 6)
	canvas.Text(7.75*w+xOffset, 2.75*h+yOffset, "DNMs", style)

	// 7.
	style = choseTextStyle(color, highlightColor, highlightText, 7)
	canvas.Text(halfSideLength+xOffset, 1.25*h+yOffset, "I/F", style)

	canvas.End()
}

func main() {
	color := flag.String("color", "black", "set pyramid color")
	highlightColor := flag.String("highlight-color", "white", "set pyramid highlighting color")
	highlightText := flag.Int("highlight", 0, "highlight text element #")
	flag.Parse()
	svgPyramid(*color, *highlightColor, *highlightText)
}
<commit_msg>goimports -w<commit_after>// svgpyramid draw a pyramid SVG infographic.
package main

import (
	"flag"
	"fmt"
	"math"
	"os"

	svg "github.com/ajstarks/svgo/float"
)

func choseLineStyle(color, highlightColor string, highlightText int, nums ...int) string {
	for _, num := range nums {
		if num == highlightText {
			return fmt.Sprintf("stroke:%s;stroke-width:3;stroke-linecap:round", highlightColor)
		}
	}
	return fmt.Sprintf("stroke:%s;stroke-width:3;stroke-linecap:round", color)
}

func choseTextStyle(color, highlightColor string, highlightText, num int) string {
	if num == highlightText {
		return fmt.Sprintf("text-anchor:middle;font-size:25px;fill:%s", highlightColor)
	}
	return fmt.Sprintf("text-anchor:middle;font-size:25px;fill:%s", color)
}

func svgPyramid(color, highlightColor string, highlightText int) {
	width := 500.0
	height := 500.0
	xOffset := 50.0
	yOffset := 50.0

	sideLength := width
	halfSideLength := sideLength / 2
	triangleHeight := math.Sqrt(sideLength*sideLength - halfSideLength*halfSideLength)

	canvas := svg.New(os.Stdout)
	canvas.Start(width+xOffset+xOffset, height+yOffset+yOffset)

	h := math.Sqrt(sideLength*sideLength-halfSideLength*halfSideLength) / 6.5
	w := halfSideLength / 6.5

	// left triangle line
	s := choseLineStyle(color, highlightColor, highlightText, 7)
	canvas.Line(6.5*w+xOffset, 0+yOffset, 5.0*w+xOffset, 1.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 5)
	canvas.Line(5.0*w+xOffset, 1.5*h+yOffset, 3.5*w+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 3)
	canvas.Line(3.5*w+xOffset, 3.0*h+yOffset, 2.0*w+xOffset, 4.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 2)
	canvas.Line(2.0*w+xOffset, 4.5*h+yOffset, 1.0*w+xOffset, 5.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 1)
	canvas.Line(1.0*w+xOffset, 5.5*h+yOffset, 0+xOffset, 6.5*h+yOffset, s)

	// right triangle line
	s = choseLineStyle(color, highlightColor, highlightText, 7)
	canvas.Line(6.5*w+xOffset, 0+yOffset, 8.0*w+xOffset, 1.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 6)
	canvas.Line(8.0*w+xOffset, 1.5*h+yOffset, 9.5*w+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 4)
	canvas.Line(9.5*w+xOffset, 3.0*h+yOffset, 11.0*w+xOffset, 4.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 2)
	canvas.Line(11.0*w+xOffset, 4.5*h+yOffset, 12.0*w+xOffset, 5.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 1)
	canvas.Line(12.0*w+xOffset, 5.5*h+yOffset, 13.0*w+xOffset, 6.5*h+yOffset, s)

	// bottom triangle line
	s = choseLineStyle(color, highlightColor, highlightText, 1)
	canvas.Line(0+xOffset, triangleHeight+yOffset, sideLength+xOffset, triangleHeight+yOffset, s)

	// first horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 5, 7)
	canvas.Line(5.0*w+xOffset, 1.5*h+yOffset, 6.5*w+xOffset, 1.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 6, 7)
	canvas.Line(6.5*w+xOffset, 1.5*h+yOffset, 8.0*w+xOffset, 1.5*h+yOffset, s)

	// second horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 3, 5)
	canvas.Line(3.5*w+xOffset, 3.0*h+yOffset, 6.5*w+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 4, 6)
	canvas.Line(6.5*w+xOffset, 3.0*h+yOffset, 9.5*w+xOffset, 3.0*h+yOffset, s)

	// third horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 2, 3)
	canvas.Line(2.0*w+xOffset, 4.5*h+yOffset, 6.5*w+xOffset, 4.5*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 2, 4)
	canvas.Line(6.5*w+xOffset, 4.5*h+yOffset, 11.0*w+xOffset, 4.5*h+yOffset, s)

	// fourth horizontal line
	s = choseLineStyle(color, highlightColor, highlightText, 1, 2)
	canvas.Line(1.0*w+xOffset, 5.5*h+yOffset, 12.0*w+xOffset, 5.5*h+yOffset, s)

	// vertical line
	s = choseLineStyle(color, highlightColor, highlightText, 5, 6)
	canvas.Line(halfSideLength+xOffset, 1.5*h+yOffset, halfSideLength+xOffset, 3.0*h+yOffset, s)

	s = choseLineStyle(color, highlightColor, highlightText, 3, 4)
	canvas.Line(halfSideLength+xOffset, 3.0*h+yOffset, halfSideLength+xOffset, 4.5*h+yOffset, s)

	// 1.
	style := choseTextStyle(color, highlightColor, highlightText, 1)
	canvas.Text(halfSideLength+xOffset, 6.125*h+yOffset, "secure devices", style)

	// 2.
	style = choseTextStyle(color, highlightColor, highlightText, 2)
	canvas.Text(halfSideLength+xOffset, 5.125*h+yOffset, "secure software", style)

	// 3.
	style = choseTextStyle(color, highlightColor, highlightText, 3)
	canvas.Text(4.75*w+xOffset, 3.75*h+yOffset, "anon.", style)
	canvas.Text(4.5*w+xOffset, 4.25*h+yOffset, "messaging", style)

	// 4.
	style = choseTextStyle(color, highlightColor, highlightText, 4)
	canvas.Text((6.5+1.75)*w+xOffset, 3.75*h+yOffset, "digital", style)
	canvas.Text((6.5+1.75)*w+xOffset, 4.25*h+yOffset, "cash", style)

	// 5.
	style = choseTextStyle(color, highlightColor, highlightText, 5)
	canvas.Text(5.25*w+xOffset, 2.75*h+yOffset, "nyms", style)

	// 6.
	style = choseTextStyle(color, highlightColor, highlightText, 6)
	canvas.Text(7.75*w+xOffset, 2.75*h+yOffset, "DNMs", style)

	// 7.
	style = choseTextStyle(color, highlightColor, highlightText, 7)
	canvas.Text(halfSideLength+xOffset, 1.25*h+yOffset, "I/F", style)

	canvas.End()
}

func main() {
	color := flag.String("color", "black", "set pyramid color")
	highlightColor := flag.String("highlight-color", "white", "set pyramid highlighting color")
	highlightText := flag.Int("highlight", 0, "highlight text element #")
	flag.Parse()
	svgPyramid(*color, *highlightColor, *highlightText)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && ((*b[0]).IsRed() || (*b[0]).IsThanYesterday())
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var price = b[0].GetPriceList()
	var volume = b[0].GetVolumeList()

	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-45:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-45:]) < 0.25)
}

type check04 struct{}

func (check04) String() string {
	return "(MA3 < MA6) > MA18 and MA3UP(1)"
}

func (check04) Mindata() int {
	return 18
}

func (check04) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); up && days == 1 {
		var (
			ma6       = b[0].MA(6)
			ma18      = b[0].MA(18)
			ma3_last  = len(ma3) - 1
			ma6_last  = len(ma6) - 1
			ma18_last = len(ma18) - 1
		)
		return (ma3[ma3_last] > ma18[ma18_last] && ma6[ma6_last] > ma18[ma18_last]) && ma3[ma3_last] < ma6[ma6_last]
	}
	return false
}

type check05 struct{}

func (check05) String() string {
	return "三日內最大量 K 線收紅 收在 MA18 之上"
}

func (check05) Mindata() int {
	return 18
}

func (check05) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var (
		vols        = b[0].GetVolumeList()
		volsFloat64 = make([]float64, 3)
	)
	for i, v := range vols[len(vols)-3:] {
		volsFloat64[i] = float64(v)
	}
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(volsFloat64)); up && days >= 1 && b[0].IsRed() {
		var (
			ma18      = b[0].MA(18)
			priceList = b[0].GetPriceList()
		)

		if priceList[len(priceList)-1] > ma18[len(ma18)-1] {
			return true
		}
	}
	return false
}

func prepareData(b ...*twse.Data) []bool {
	var result []bool
	var mindata int
	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
	ckList.Add(checkGroup(check04{}))
	ckList.Add(checkGroup(check05{}))
}
<commit_msg>Pretty var.<commit_after>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && ((*b[0]).IsRed() || (*b[0]).IsThanYesterday())
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}

	var (
		price  = b[0].GetPriceList()
		volume = b[0].GetVolumeList()
	)

	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-45:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-45:]) < 0.25)
}

type check04 struct{}

func (check04) String() string {
	return "(MA3 < MA6) > MA18 and MA3UP(1)"
}

func (check04) Mindata() int {
	return 18
}

func (check04) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); up && days == 1 {
		var (
			ma6       = b[0].MA(6)
			ma18      = b[0].MA(18)
			ma3_last  = len(ma3) - 1
			ma6_last  = len(ma6) - 1
			ma18_last = len(ma18) - 1
		)
		return (ma3[ma3_last] > ma18[ma18_last] && ma6[ma6_last] > ma18[ma18_last]) && ma3[ma3_last] < ma6[ma6_last]
	}
	return false
}

type check05 struct{}

func (check05) String() string {
	return "三日內最大量 K 線收紅 收在 MA18 之上"
}

func (check05) Mindata() int {
	return 18
}

func (check05) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var (
		vols        = b[0].GetVolumeList()
		volsFloat64 = make([]float64, 3)
	)
	for i, v := range vols[len(vols)-3:] {
		volsFloat64[i] = float64(v)
	}
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(volsFloat64)); up && days >= 1 && b[0].IsRed() {
		var (
			ma18      = b[0].MA(18)
			priceList = b[0].GetPriceList()
		)

		if priceList[len(priceList)-1] > ma18[len(ma18)-1] {
			return true
		}
	}
	return false
}

func prepareData(b ...*twse.Data) []bool {
	var (
		result  []bool
		mindata int
	)

	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
	ckList.Add(checkGroup(check04{}))
	ckList.Add(checkGroup(check05{}))
}
<|endoftext|>