<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/lowstz/slackhookgo"
)

type alertMessage struct {
	url   string
	text  string
	color string
}

type connection struct {
	protocol string
	host     string
	port     string
	address  string
}

func checkError(err error) {
	if err == nil {
		return
	}
	log.Printf("error: %s", err)
}

func (c *connection) conn() (net.Conn, error) {
	conn, err := net.DialTimeout(c.protocol, c.address, 3*time.Second)
	checkError(err)
	return conn, err
}

func (a *alertMessage) sentSlack() {
	err := slackhookgo.Send(
		a.url,
		slackhookgo.NewSlackMessage(
			"username",
			"backup",
		).AddAttachment(
			slackhookgo.MessageAttachment{
				Color: a.color,
				Text:  a.text,
				Title: "<!channel>",
			},
		),
	)
	checkError(err)
}

func main() {
	var (
		sentUp   = 0
		sentDown = 0
		protocol = flag.String("protocol", "tcp", "protocol tcp/udp")
		host     = flag.String("host", "ya.ru", "destination host")
		port     = flag.String("port", "80", "destination port")
		interval = flag.Uint("interval", 5, "interval check seconds")
		url      = flag.String("url", "", "hook url")
	)

	flag.Parse()

	for {
		c := connection{
			protocol: *protocol,
			address:  fmt.Sprintf("%s:%s", *host, *port),
		}
		conn, err := c.conn()
		if err == nil {
			conn.Close()
			if sentUp == 0 {
				am := alertMessage{
					color: "good",
					text:  "Destination host " + *host + ":" + *port + " reachable",
					url:   *url,
				}
				am.sentSlack()
				sentUp = 1
				sentDown = 0
			}
		} else {
			if sentDown == 0 {
				am := alertMessage{
					color: "danger",
					text:  "Destination host " + *host + ":" + *port + " unreachable",
					url:   *url,
				}
				am.sentSlack()
				sentUp = 0
				sentDown = 1
			}
		}
		time.Sleep(time.Duration(*interval) * time.Second)
	}
}
<commit_msg>upd for{...}<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"time"
	"errors"

	"github.com/lowstz/slackhookgo"
)

type alertMessage struct {
	url   string
	text  string
	color string
}

type connection struct {
	protocol string
	host     string
	port     string
	address  string
}

func checkError(err error) {
	if err == nil {
		return
	}
	log.Printf("error: %s", err)
}

func (c *connection) conn() (net.Conn, error) {
	conn, err := net.DialTimeout(c.protocol, c.address, 3*time.Second)
	checkError(err)
	return conn, err
}

func (a *alertMessage) sentSlack() {
	err := slackhookgo.Send(
		a.url,
		slackhookgo.NewSlackMessage(
			"username",
			"backup",
		).AddAttachment(
			slackhookgo.MessageAttachment{
				Color: a.color,
				Text:  a.text,
				Title: "<!channel>",
			},
		),
	)
	checkError(err)
}

func main() {
	var (
		c 	string
		t	string
		lastSt  error = errors.New("zero step")
		protocol = flag.String("protocol", "tcp", "protocol tcp/udp")
		host     = flag.String("host", "ya.ru", "destination host")
		port     = flag.String("port", "80", "destination port")
		interval = flag.Uint("interval", 5, "interval check seconds")
		url      = flag.String("url", "", "hook url")
	)

	flag.Parse()

	for {
		c := connection{
			protocol: *protocol,
			address:  fmt.Sprintf("%s:%s", *host, *port),
		}
		conn, err := c.conn()
                if err != lastSt {
                        if err == nil {         // normal
                                c = "good"
                                t = " reachable"
                                conn.Close()
                        } else {                // not normal
                                c = "danger"
                                t = " unreachable"
                        }
                        lastSt = err            // key of success
                        am := alertMessage{
                                color:  c,
                                text:   "Destination host " + *host + ":" + *port + t,
                                url:    *url,
                        }
                        am.sentSlack()
                        fmt.Printf("the message is send\n")
                } else {
                fmt.Printf("to do - nothing\n") // to do - nothin
                }
		time.Sleep(time.Duration(*interval) * time.Second)
	}
}
<|endoftext|>


<commit_before>// Package clock is a low consumption, low latency support for frequent updates of large capacity timing manager：
//	1、能够添加一次性、重复性任务，并能在其执行前撤销或频繁更改。
//	2、支持同一时间点，多个任务提醒。
//	3、适用于中等密度，大跨度的单次、多次定时任务。
//	4、支持10万次/秒的定时任务执行、提醒、撤销或添加操作，平均延迟10微秒内
//	5、支持注册任务的函数调用，及事件通知。
// 基本处理逻辑：
//	1、重复性任务，流程是：
//		a、注册重复任务
//		b、时间抵达时，控制器调用注册函数，并发送通知
//		c、如果次数达到限制，则撤销；否则，控制器更新该任务的下次执行时间点
//		d、控制器等待下一个最近需要执行的任务
//	2、一次性任务，可以是服务运行时，当前时间点之后的任意事件，流程是：
//		a、注册一次性任务
//		b、时间抵达时，控制器调用注册函数，并发送通知
//		c、控制器释放该任务
//		d、控制器等待下一个最近需要执行的任务
// 使用方式，参见示例代码。
package clock

import (
	"github.com/HuKeping/rbtree"
	"math"
	"sync"
	"sync/atomic"
	"time"
)

const _UNTOUCHED = time.Duration(math.MaxInt64)

var (
	defaultClock *Clock
	oncedo       sync.Once
)

//Default return singal default clock
func Default() *Clock {
	oncedo.Do(initClock)
	return defaultClock
}
func initClock() {
	defaultClock = NewClock()
}

// Clock is jobs schedule
type Clock struct {
	seq         uint64
	jobQueue    *rbtree.Rbtree //inner memory storage
	count       uint64         //已执行次数，不得大于times
	waitJobsNum uint64         //num of jobs which wait for action
	pauseChan   chan struct{}
	resumeChan  chan struct{}
	exitChan    chan struct{}
}

var singal = struct{}{}

//NewClock Create a task queue controller
func NewClock() *Clock {
	c := &Clock{
		jobQueue:   rbtree.New(),
		pauseChan:  make(chan struct{}, 0),
		resumeChan: make(chan struct{}, 0),
		exitChan:   make(chan struct{}, 0),
	}

	c.start()

	return c
}
func (jl *Clock) start() {
	now := time.Now()
	untouchedJob := jobItem{
		createTime:   now,
		intervalTime: time.Duration(math.MaxInt64),
		fn: func() {
			//this jobItem is untouched.
		},
	}

	_, inserted := jl.addJob(now, untouchedJob.intervalTime, 1, untouchedJob.fn)
	if !inserted {
		panic("[clock] internal error.Reason cannot insert job.")
	}
	//开启守护协程
	go jl.schedule()
	jl.resume()
}

func (jl *Clock) pause() {
	jl.pauseChan <- singal
}
func (jl *Clock) resume() {
	jl.resumeChan <- singal
}
func (jl *Clock) exit() {
	jl.exitChan <- singal
}

func (jl *Clock) immediate() {
	for {
		if item := jl.jobQueue.Min(); item != nil {
			atomic.AddUint64(&jl.count, 1)

			job := item.(*jobItem)
			job.action(false)

			jl.removeJob(job)

		} else {
			break
		}
	}
}

func (jl *Clock) schedule() {
	var (
		timeout time.Duration
		job     *jobItem
		timer   = newSafeTimer(_UNTOUCHED)
	)
	defer timer.Stop()
Pause:
	<-jl.resumeChan
	for {
		job, _ = jl.jobQueue.Min().(*jobItem) //ignore ok-assert
		timeout = job.actionTime.Sub(time.Now())
		timer.SafeReset(timeout)
		select {
		case <-timer.C:
			timer.SCR()

			atomic.AddUint64(&jl.count, 1)

			job.action(true)

			if job.actionMax == 0 || job.actionMax > job.actionCount {
				jl.jobQueue.Delete(job)
				job.actionTime = job.actionTime.Add(job.intervalTime)
				jl.jobQueue.Insert(job)
			} else {
				jl.removeJob(job)
			}
		case <-jl.pauseChan:
			goto Pause
		case <-jl.exitChan:
			goto Exit
		}
	}
Exit:
}

// UpdateJobTimeout update a timed task with time duration after now
//	@job:			job identifier
//	@actionTime:	new job schedule time,must be greater than 0
func (jl *Clock) UpdateJobTimeout(job Job, actionTime time.Duration) (updated bool) {
	if job == nil || actionTime.Nanoseconds() <= 0 || !job.isAvailable() {
		return false
	}
	now := time.Now()

	item, ok := job.(*jobItem)
	if !ok {
		return false
	}
	jl.pause()
	defer jl.resume()

	// update jobitem in job queue
	jl.jobQueue.Delete(item)
	item.actionTime = now.Add(actionTime)
	jl.jobQueue.Insert(item)

	updated = true
	return
}

// AddJobWithInterval insert a timed task with time duration after now
// 	@actionTime:	Duration after now
//	@jobFunc:		Callback function,not nil
//	return
// 	@jobScheduled:	A reference to a task that has been scheduled.
func (jl *Clock) AddJobWithInterval(actionInterval time.Duration, jobFunc func()) (jobScheduled Job, inserted bool) {
	if jobFunc == nil || actionInterval.Nanoseconds() <= 0 {
		return
	}
	now := time.Now()

	jl.pause()
	jobScheduled, inserted = jl.addJob(now, actionInterval, 1, jobFunc)
	jl.resume()

	return
}

// AddJobWithDeadtime insert a timed task with time point after now
//	@actionTime:	Execution start time. must after now
//	@jobFunc:		Callback function,not nil
//	return
// 	@jobScheduled	:	A reference to a task that has been scheduled.
//	@inserted		:	return false ,if actionTime before time.Now or jobFunc is nil
func (jl *Clock) AddJobWithDeadtime(actionTime time.Time, jobFunc func()) (jobScheduled Job, inserted bool) {
	actionInterval := actionTime.Sub(time.Now())
	if jobFunc == nil || actionInterval.Nanoseconds() <= 0 {
		return
	}
	now := time.Now()

	jl.pause()
	jobScheduled, inserted = jl.addJob(now, actionInterval, 1, jobFunc)
	jl.resume()

	return
}

// AddJobRepeat add a repeat task with interval duration
//	@interval:		The interval between two actions of the job
//	@actionMax:		The number of job execution
//	@jobFunc:		Callback function,not nil
//	return
// 	@jobScheduled	:	A reference to a task that has been scheduled.
//	@inserted		:	return false ,if interval is not Positiveor jobFunc is nil
//Note：
// when jobTimes==0,the job will be executed without limitation。If you no longer use, be sure to call the DelJob method to release
func (jl *Clock) AddJobRepeat(interval time.Duration, actionMax uint64, jobFunc func()) (jobScheduled Job, inserted bool) {
	if jobFunc == nil || interval.Nanoseconds() <= 0 {
		return
	}
	now := time.Now()

	jl.pause()
	jobScheduled, inserted = jl.addJob(now, interval, actionMax, jobFunc)
	jl.resume()

	return
}

func (jl *Clock) addJob(createTime time.Time, actionInterval time.Duration, actionMax uint64, jobFunc func()) (job *jobItem, inserted bool) {
	jl.seq++
	jl.waitJobsNum++
	job = &jobItem{
		id:           jl.seq,
		actionMax:    actionMax,
		createTime:   createTime,
		actionTime:   createTime.Add(actionInterval),
		intervalTime: actionInterval,
		msgChan:      make(chan Job, 10),
		fn:           jobFunc,
		clock:        jl,
	}
	jl.jobQueue.Insert(job)
	inserted = true

	return

}

func (jl *Clock) removeJob(item *jobItem) {
	if jl.jobQueue.Delete(item) != nil {
		jl.waitJobsNum--

		//job.Cancel --> rmJob -->removeJob; schedule -->removeJob
		//it is call repeatly when Job.Cancel
		if atomic.CompareAndSwapInt32(&item.cancelFlag, 0, 1) {
			item.innerCancel()
		}
	}
}

func (jl *Clock) rmJob(job *jobItem) {
	jl.pause()
	defer jl.resume()

	jl.removeJob(job)
	return
}

// Count 已经执行的任务数。对于重复任务，会计算多次
func (jl *Clock) Count() uint64 {
	return atomic.LoadUint64(&jl.count)
}

//重置Clock的内部状态
func (jl *Clock) Reset() *Clock {
	jl.exit()
	jl.count = 0

	jl.cleanJobs()
	jl.start()
	return jl
}

func (jl *Clock) cleanJobs() {
	item := jl.jobQueue.Min()
	for item != nil {
		job, ok := item.(*jobItem)
		if ok {
			jl.removeJob(job)
		}
		item = jl.jobQueue.Min()
	}
}

//WaitJobs get how much jobs waiting for call
func (jl *Clock) WaitJobs() uint64 {
	jobs := atomic.LoadUint64(&jl.waitJobsNum) - 1
	return jobs
}

//Stop stop clock , and cancel all waiting jobs
func (jl *Clock) Stop() {
	jl.exit()

	jl.cleanJobs()
}

//StopGracefull stop clock ,and do once every waiting job including Once\Reapeat
//Note:对于任务队列中，即使安排执行多次或者不限次数的，也仅仅执行一次。
func (jl *Clock) StopGraceful() {
	jl.exit()

	jl.immediate()
}
<commit_msg>Update clock.go<commit_after>// Package clock is a low consumption, low latency support for frequent updates of large capacity timing manager：
//	1、能够添加一次性、重复性任务，并能在其执行前撤销或频繁更改。
//	2、支持同一时间点，多个任务提醒。
//	3、适用于中等密度，大跨度的单次、多次定时任务。
//	4、支持10万次/秒的定时任务执行、提醒、撤销或添加操作，平均延迟10微秒内
//	5、支持注册任务的函数调用，及事件通知。
// 基本处理逻辑：
//	1、重复性任务，流程是：
//		a、注册重复任务
//		b、时间抵达时，控制器调用注册函数，并发送通知
//		c、如果次数达到限制，则撤销；否则，控制器更新该任务的下次执行时间点
//		d、控制器等待下一个最近需要执行的任务
//	2、一次性任务，可以是服务运行时，当前时间点之后的任意事件，流程是：
//		a、注册一次性任务
//		b、时间抵达时，控制器调用注册函数，并发送通知
//		c、控制器释放该任务
//		d、控制器等待下一个最近需要执行的任务
// 使用方式，参见示例代码。
package clock

import (
	"github.com/HuKeping/rbtree"
	"math"
	"sync"
	"sync/atomic"
	"time"
)

const _UNTOUCHED = time.Duration(math.MaxInt64)

var (
	defaultClock *Clock
	oncedo       sync.Once
)

//Default return singal default clock
func Default() *Clock {
	oncedo.Do(initClock)
	return defaultClock
}
func initClock() {
	defaultClock = NewClock()
}

// Clock is jobs schedule
type Clock struct {
	seq         uint64
	count       uint64         //已执行次数，不得大于times
	waitJobsNum uint64         //num of jobs which wait for action
	jobQueue    *rbtree.Rbtree //inner memory storage
	pauseChan   chan struct{}
	resumeChan  chan struct{}
	exitChan    chan struct{}
}

var singal = struct{}{}

//NewClock Create a task queue controller
func NewClock() *Clock {
	c := &Clock{
		jobQueue:   rbtree.New(),
		pauseChan:  make(chan struct{}, 0),
		resumeChan: make(chan struct{}, 0),
		exitChan:   make(chan struct{}, 0),
	}

	c.start()

	return c
}
func (jl *Clock) start() {
	now := time.Now()
	untouchedJob := jobItem{
		createTime:   now,
		intervalTime: time.Duration(math.MaxInt64),
		fn: func() {
			//this jobItem is untouched.
		},
	}

	_, inserted := jl.addJob(now, untouchedJob.intervalTime, 1, untouchedJob.fn)
	if !inserted {
		panic("[clock] internal error.Reason cannot insert job.")
	}
	//开启守护协程
	go jl.schedule()
	jl.resume()
}

func (jl *Clock) pause() {
	jl.pauseChan <- singal
}
func (jl *Clock) resume() {
	jl.resumeChan <- singal
}
func (jl *Clock) exit() {
	jl.exitChan <- singal
}

func (jl *Clock) immediate() {
	for {
		if item := jl.jobQueue.Min(); item != nil {
			atomic.AddUint64(&jl.count, 1)

			job := item.(*jobItem)
			job.action(false)

			jl.removeJob(job)

		} else {
			break
		}
	}
}

func (jl *Clock) schedule() {
	var (
		timeout time.Duration
		job     *jobItem
		timer   = newSafeTimer(_UNTOUCHED)
	)
	defer timer.Stop()
Pause:
	<-jl.resumeChan
	for {
		job, _ = jl.jobQueue.Min().(*jobItem) //ignore ok-assert
		timeout = job.actionTime.Sub(time.Now())
		timer.SafeReset(timeout)
		select {
		case <-timer.C:
			timer.SCR()

			atomic.AddUint64(&jl.count, 1)

			job.action(true)

			if job.actionMax == 0 || job.actionMax > job.actionCount {
				jl.jobQueue.Delete(job)
				job.actionTime = job.actionTime.Add(job.intervalTime)
				jl.jobQueue.Insert(job)
			} else {
				jl.removeJob(job)
			}
		case <-jl.pauseChan:
			goto Pause
		case <-jl.exitChan:
			goto Exit
		}
	}
Exit:
}

// UpdateJobTimeout update a timed task with time duration after now
//	@job:			job identifier
//	@actionTime:	new job schedule time,must be greater than 0
func (jl *Clock) UpdateJobTimeout(job Job, actionTime time.Duration) (updated bool) {
	if job == nil || actionTime.Nanoseconds() <= 0 || !job.isAvailable() {
		return false
	}
	now := time.Now()

	item, ok := job.(*jobItem)
	if !ok {
		return false
	}
	jl.pause()
	defer jl.resume()

	// update jobitem in job queue
	jl.jobQueue.Delete(item)
	item.actionTime = now.Add(actionTime)
	jl.jobQueue.Insert(item)

	updated = true
	return
}

// AddJobWithInterval insert a timed task with time duration after now
// 	@actionTime:	Duration after now
//	@jobFunc:		Callback function,not nil
//	return
// 	@jobScheduled:	A reference to a task that has been scheduled.
func (jl *Clock) AddJobWithInterval(actionInterval time.Duration, jobFunc func()) (jobScheduled Job, inserted bool) {
	if jobFunc == nil || actionInterval.Nanoseconds() <= 0 {
		return
	}
	now := time.Now()

	jl.pause()
	jobScheduled, inserted = jl.addJob(now, actionInterval, 1, jobFunc)
	jl.resume()

	return
}

// AddJobWithDeadtime insert a timed task with time point after now
//	@actionTime:	Execution start time. must after now
//	@jobFunc:		Callback function,not nil
//	return
// 	@jobScheduled	:	A reference to a task that has been scheduled.
//	@inserted		:	return false ,if actionTime before time.Now or jobFunc is nil
func (jl *Clock) AddJobWithDeadtime(actionTime time.Time, jobFunc func()) (jobScheduled Job, inserted bool) {
	actionInterval := actionTime.Sub(time.Now())
	if jobFunc == nil || actionInterval.Nanoseconds() <= 0 {
		return
	}
	now := time.Now()

	jl.pause()
	jobScheduled, inserted = jl.addJob(now, actionInterval, 1, jobFunc)
	jl.resume()

	return
}

// AddJobRepeat add a repeat task with interval duration
//	@interval:		The interval between two actions of the job
//	@actionMax:		The number of job execution
//	@jobFunc:		Callback function,not nil
//	return
// 	@jobScheduled	:	A reference to a task that has been scheduled.
//	@inserted		:	return false ,if interval is not Positiveor jobFunc is nil
//Note：
// when jobTimes==0,the job will be executed without limitation。If you no longer use, be sure to call the DelJob method to release
func (jl *Clock) AddJobRepeat(interval time.Duration, actionMax uint64, jobFunc func()) (jobScheduled Job, inserted bool) {
	if jobFunc == nil || interval.Nanoseconds() <= 0 {
		return
	}
	now := time.Now()

	jl.pause()
	jobScheduled, inserted = jl.addJob(now, interval, actionMax, jobFunc)
	jl.resume()

	return
}

func (jl *Clock) addJob(createTime time.Time, actionInterval time.Duration, actionMax uint64, jobFunc func()) (job *jobItem, inserted bool) {
	jl.seq++
	jl.waitJobsNum++
	job = &jobItem{
		id:           jl.seq,
		actionMax:    actionMax,
		createTime:   createTime,
		actionTime:   createTime.Add(actionInterval),
		intervalTime: actionInterval,
		msgChan:      make(chan Job, 10),
		fn:           jobFunc,
		clock:        jl,
	}
	jl.jobQueue.Insert(job)
	inserted = true

	return

}

func (jl *Clock) removeJob(item *jobItem) {
	if jl.jobQueue.Delete(item) != nil {
		jl.waitJobsNum--

		//job.Cancel --> rmJob -->removeJob; schedule -->removeJob
		//it is call repeatly when Job.Cancel
		if atomic.CompareAndSwapInt32(&item.cancelFlag, 0, 1) {
			item.innerCancel()
		}
	}
}

func (jl *Clock) rmJob(job *jobItem) {
	jl.pause()
	defer jl.resume()

	jl.removeJob(job)
	return
}

// Count 已经执行的任务数。对于重复任务，会计算多次
func (jl *Clock) Count() uint64 {
	return atomic.LoadUint64(&jl.count)
}

//重置Clock的内部状态
func (jl *Clock) Reset() *Clock {
	jl.exit()
	jl.count = 0

	jl.cleanJobs()
	jl.start()
	return jl
}

func (jl *Clock) cleanJobs() {
	item := jl.jobQueue.Min()
	for item != nil {
		job, ok := item.(*jobItem)
		if ok {
			jl.removeJob(job)
		}
		item = jl.jobQueue.Min()
	}
}

//WaitJobs get how much jobs waiting for call
func (jl *Clock) WaitJobs() uint64 {
	jobs := atomic.LoadUint64(&jl.waitJobsNum) - 1
	return jobs
}

//Stop stop clock , and cancel all waiting jobs
func (jl *Clock) Stop() {
	jl.exit()

	jl.cleanJobs()
}

//StopGracefull stop clock ,and do once every waiting job including Once\Reapeat
//Note:对于任务队列中，即使安排执行多次或者不限次数的，也仅仅执行一次。
func (jl *Clock) StopGraceful() {
	jl.exit()

	jl.immediate()
}
<|endoftext|>


<commit_before>package krpc

import (
	"fmt"
)

// Msg represents messages that nodes in the network send to each other as specified by the protocol.
// They are also referred to as the KRPC messages.
// There are three types of messages: QUERY, RESPONSE, ERROR
// The message is a dictonary that is then
// "bencoded" (serialization & compression format adopted by the BitTorrent)
// and sent via the UDP connection to peers.
//
// A KRPC message is a single dictionary with two keys common to every message and additional keys depending on the type of message.
// Every message has a key "t" with a string value representing a transaction ID.
// This transaction ID is generated by the querying node and is echoed in the response, so responses
// may be correlated with multiple queries to the same node. The transaction ID should be encoded as a short string of binary numbers, typically 2 characters are enough as they cover 2^16 outstanding queries. The other key contained in every KRPC message is "y" with a single character value describing the type of message. The value of the "y" key is one of "q" for query, "r" for response, or "e" for error.
// 3 message types:  QUERY, RESPONSE, ERROR
type Msg struct {
	Q        string   `bencode:"q,omitempty"` // Query method (one of 4: "ping", "find_node", "get_peers", "announce_peer")
	A        *MsgArgs `bencode:"a,omitempty"` // named arguments sent with a query
	T        string   `bencode:"t"`           // required: transaction ID
	Y        string   `bencode:"y"`           // required: type of the message: q for QUERY, r for RESPONSE, e for ERROR
	R        *Return  `bencode:"r,omitempty"` // RESPONSE type only
	E        *Error   `bencode:"e,omitempty"` // ERROR type only
	IP       NodeAddr `bencode:"ip,omitempty"`
	ReadOnly bool     `bencode:"ro,omitempty"` // BEP 43. Sender does not respond to queries.
}

type MsgArgs struct {
	ID          ID     `bencode:"id"`                     // ID of the querying Node
	InfoHash    ID     `bencode:"info_hash,omitempty"`    // InfoHash of the torrent
	Target      ID     `bencode:"target,omitempty"`       // ID of the node sought
	Token       string `bencode:"token,omitempty"`        // Token received from an earlier get_peers query
	Port        *int   `bencode:"port,omitempty"`         // Sender's torrent port
	ImpliedPort bool   `bencode:"implied_port,omitempty"` // Use senders apparent DHT port
	Want        []Want `bencode:"want,omitempty"`         // Contains strings like "n4" and "n6" from BEP 32.
	NoSeed      int    `bencode:"noseed,omitempty"`       // BEP 33
	Scrape      int    `bencode:"scrape,omitempty"`       // BEP 33
}

type Want string

const (
	WantNodes  Want = "n4"
	WantNodes6 Want = "n6"
)

// BEP 51 (DHT Infohash Indexing)
type Bep51Return struct {
	Interval *int64             `bencode:"interval,omitempty"`
	Num      *int64             `bencode:"num,omitempty"`
	Samples  *CompactInfohashes `bencode:"samples,omitempty"`
}

type Return struct {
	// All returns are supposed to contain an ID, but what if they don't?
	ID     ID                  `bencode:"id"`               // ID of the queried node
	Nodes  CompactIPv4NodeInfo `bencode:"nodes,omitempty"`  // K closest nodes to the requested target
	Nodes6 CompactIPv6NodeInfo `bencode:"nodes6,omitempty"` // K closest nodes to the requested target
	Token  *string             `bencode:"token,omitempty"`  // Token for future announce_peer
	Values []NodeAddr          `bencode:"values,omitempty"` // Torrent peers

	// BEP 33 (scrapes)
	BFsd *ScrapeBloomFilter `bencode:"BFsd,omitempty"`
	BFpe *ScrapeBloomFilter `bencode:"BFpe,omitempty"`

	Bep51Return
}

func (r Return) ForAllNodes(f func(NodeInfo)) {
	for _, n := range r.Nodes {
		f(n)
	}
	for _, n := range r.Nodes6 {
		f(n)
	}
}

var _ fmt.Stringer = Msg{}

func (m Msg) String() string {
	return fmt.Sprintf("%#v", m)
}

// The node ID of the source of this Msg. Returns nil if it isn't present.
// TODO: Can we verify Msgs more aggressively so this is guaranteed to return
// a valid ID for a checked Msg?
func (m Msg) SenderID() *ID {
	switch m.Y {
	case "q":
		if m.A == nil {
			return nil
		}
		return &m.A.ID
	case "r":
		if m.R == nil {
			return nil
		}
		return &m.R.ID
	}
	return nil
}

func (m Msg) Error() *Error {
	if m.Y != "e" {
		return nil
	}
	return m.E
}
<commit_msg>Unimplement Stringer for krpc.Msg<commit_after>package krpc

// Msg represents messages that nodes in the network send to each other as specified by the protocol.
// They are also referred to as the KRPC messages.
// There are three types of messages: QUERY, RESPONSE, ERROR
// The message is a dictonary that is then
// "bencoded" (serialization & compression format adopted by the BitTorrent)
// and sent via the UDP connection to peers.
//
// A KRPC message is a single dictionary with two keys common to every message and additional keys depending on the type of message.
// Every message has a key "t" with a string value representing a transaction ID.
// This transaction ID is generated by the querying node and is echoed in the response, so responses
// may be correlated with multiple queries to the same node. The transaction ID should be encoded as a short string of binary numbers, typically 2 characters are enough as they cover 2^16 outstanding queries. The other key contained in every KRPC message is "y" with a single character value describing the type of message. The value of the "y" key is one of "q" for query, "r" for response, or "e" for error.
// 3 message types:  QUERY, RESPONSE, ERROR
type Msg struct {
	Q        string   `bencode:"q,omitempty"` // Query method (one of 4: "ping", "find_node", "get_peers", "announce_peer")
	A        *MsgArgs `bencode:"a,omitempty"` // named arguments sent with a query
	T        string   `bencode:"t"`           // required: transaction ID
	Y        string   `bencode:"y"`           // required: type of the message: q for QUERY, r for RESPONSE, e for ERROR
	R        *Return  `bencode:"r,omitempty"` // RESPONSE type only
	E        *Error   `bencode:"e,omitempty"` // ERROR type only
	IP       NodeAddr `bencode:"ip,omitempty"`
	ReadOnly bool     `bencode:"ro,omitempty"` // BEP 43. Sender does not respond to queries.
}

type MsgArgs struct {
	ID          ID     `bencode:"id"`                     // ID of the querying Node
	InfoHash    ID     `bencode:"info_hash,omitempty"`    // InfoHash of the torrent
	Target      ID     `bencode:"target,omitempty"`       // ID of the node sought
	Token       string `bencode:"token,omitempty"`        // Token received from an earlier get_peers query
	Port        *int   `bencode:"port,omitempty"`         // Sender's torrent port
	ImpliedPort bool   `bencode:"implied_port,omitempty"` // Use senders apparent DHT port
	Want        []Want `bencode:"want,omitempty"`         // Contains strings like "n4" and "n6" from BEP 32.
	NoSeed      int    `bencode:"noseed,omitempty"`       // BEP 33
	Scrape      int    `bencode:"scrape,omitempty"`       // BEP 33
}

type Want string

const (
	WantNodes  Want = "n4"
	WantNodes6 Want = "n6"
)

// BEP 51 (DHT Infohash Indexing)
type Bep51Return struct {
	Interval *int64             `bencode:"interval,omitempty"`
	Num      *int64             `bencode:"num,omitempty"`
	Samples  *CompactInfohashes `bencode:"samples,omitempty"`
}

type Return struct {
	// All returns are supposed to contain an ID, but what if they don't?
	ID     ID                  `bencode:"id"`               // ID of the queried node
	Nodes  CompactIPv4NodeInfo `bencode:"nodes,omitempty"`  // K closest nodes to the requested target
	Nodes6 CompactIPv6NodeInfo `bencode:"nodes6,omitempty"` // K closest nodes to the requested target
	Token  *string             `bencode:"token,omitempty"`  // Token for future announce_peer
	Values []NodeAddr          `bencode:"values,omitempty"` // Torrent peers

	// BEP 33 (scrapes)
	BFsd *ScrapeBloomFilter `bencode:"BFsd,omitempty"`
	BFpe *ScrapeBloomFilter `bencode:"BFpe,omitempty"`

	Bep51Return
}

func (r Return) ForAllNodes(f func(NodeInfo)) {
	for _, n := range r.Nodes {
		f(n)
	}
	for _, n := range r.Nodes6 {
		f(n)
	}
}

// The node ID of the source of this Msg. Returns nil if it isn't present.
// TODO: Can we verify Msgs more aggressively so this is guaranteed to return
// a valid ID for a checked Msg?
func (m Msg) SenderID() *ID {
	switch m.Y {
	case "q":
		if m.A == nil {
			return nil
		}
		return &m.A.ID
	case "r":
		if m.R == nil {
			return nil
		}
		return &m.R.ID
	}
	return nil
}

func (m Msg) Error() *Error {
	if m.Y != "e" {
		return nil
	}
	return m.E
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	proxier "golang.org/x/net/proxy"
)

func query(word string, isMulti bool) {
	var url string
	var doc *goquery.Document
	isChinese := isChinese(word)

	if isChinese {
		url = "http://dict.youdao.com/w/eng/%s"
	} else {
		url = "http://dict.youdao.com/w/%s"
	}

	//Check proxy
	if proxy != "" {
		client := &http.Client{}
		dialer, err := proxier.SOCKS5("tcp", proxy, nil, proxier.Direct)

		if err != nil {
			color.Red("Can't connect to the proxy: %s", err)
			os.Exit(1)
		}

		httpTransport := &http.Transport{}
		client.Transport = httpTransport
		httpTransport.Dial = dialer.Dial

		resp, err := client.Get(fmt.Sprintf(url, word))

		if err != nil {
			color.Red("Query failed with err: %s", err.Error())
			os.Exit(1)
		}

		doc, _ = goquery.NewDocumentFromResponse(resp)

	} else {
		var err error
		doc, err = goquery.NewDocument(fmt.Sprintf(url, word))
		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}
	}

	if isChinese {
		// Find the result
		fmt.Println()
		doc.Find(".trans-container > ul > p > span.contentTitle").Each(func(i int, s *goquery.Selection) {
			color.Green("    %s", s.Find(".search-js").Text())
		})
	} else {
		// Find the pronounce
		if !isMulti {
			color.Green("\r\n    %s", getPronounce(doc))
		}

		// Find the result
		result := doc.Find("div#phrsListTab > div.trans-container > ul").Text()
		color.Green(result)
	}

	// Show examples
	sentences := getSentences(doc, isChinese)
	if len(sentences) > 0 {
		fmt.Println()
		for _, sentence := range sentences {
			color.Green("    %s", sentence[0])
			color.Magenta("    %s", sentence[1])
		}
		fmt.Println()
	}
}

func getPronounce(doc *goquery.Document) string {
	var pronounce string
	doc.Find("div.baav > span.pronounce").Each(func(i int, s *goquery.Selection) {

		if i == 0 {
			p := fmt.Sprintf("英: %s    ", s.Find("span.phonetic").Text())
			pronounce += p
		}

		if i == 1 {
			p := fmt.Sprintf("美: %s", s.Find("span.phonetic").Text())
			pronounce += p
		}
	})

	return pronounce
}

func getSentences(doc *goquery.Document, isChinese bool) [][]string {
	result := [][]string{}
	doc.Find("#bilingual ul li").Each(func(_ int, s *goquery.Selection) {
		r := []string{}
		s.Children().Each(func(ii int, ss *goquery.Selection) {
			// Ignore source
			if ii == 2 {
				return
			}
			var sentence string
			ss.Children().Each(func(iii int, sss *goquery.Selection) {
				if text := strings.TrimSpace(sss.Text()); text != "" {
					addSpace := (ii == 1 && isChinese) || (ii == 0 && !isChinese)
					if addSpace && iii != 0 && text != "." {
						text = " " + text
					}
					sentence += text
				}
			})
			r = append(r, sentence)
		})
		if len(r) == 2 {
			result = append(result, r)
		}
	})
	return result
}
<commit_msg>add spinner when querying word<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/briandowns/spinner"
	"github.com/fatih/color"
	proxier "golang.org/x/net/proxy"
)

func query(word string, isMulti bool) {
	var url string
	var doc *goquery.Document
	isChinese := isChinese(word)

	if isChinese {
		url = "http://dict.youdao.com/w/eng/%s"
	} else {
		url = "http://dict.youdao.com/w/%s"
	}

	//Init spinner
	s := spinner.New(spinner.CharSets[35], 100*time.Millisecond)
	s.Prefix = "Querying... "
	s.Color("green")
	s.Start()

	//Check proxy
	if proxy != "" {
		client := &http.Client{}
		dialer, err := proxier.SOCKS5("tcp", proxy, nil, proxier.Direct)

		if err != nil {
			color.Red("Can't connect to the proxy: %s", err)
			os.Exit(1)
		}

		httpTransport := &http.Transport{}
		client.Transport = httpTransport
		httpTransport.Dial = dialer.Dial

		resp, err := client.Get(fmt.Sprintf(url, word))

		if err != nil {
			color.Red("Query failed with err: %s", err.Error())
			os.Exit(1)
		}

		doc, _ = goquery.NewDocumentFromResponse(resp)
	} else {
		var err error
		doc, err = goquery.NewDocument(fmt.Sprintf(url, word))
		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}
	}

	s.Stop()

	if isChinese {
		// Find the result
		fmt.Println()
		doc.Find(".trans-container > ul > p > span.contentTitle").Each(func(i int, s *goquery.Selection) {
			color.Green("    %s", s.Find(".search-js").Text())
		})
	} else {
		// Find the pronounce
		if !isMulti {
			color.Green("\r\n    %s", getPronounce(doc))
		}

		// Find the result
		result := doc.Find("div#phrsListTab > div.trans-container > ul").Text()
		color.Green(result)
	}

	// Show examples
	sentences := getSentences(doc, isChinese)
	if len(sentences) > 0 {
		fmt.Println()
		for _, sentence := range sentences {
			color.Green("    %s", sentence[0])
			color.Magenta("    %s", sentence[1])
		}
		fmt.Println()
	}
}

func getPronounce(doc *goquery.Document) string {
	var pronounce string
	doc.Find("div.baav > span.pronounce").Each(func(i int, s *goquery.Selection) {

		if i == 0 {
			p := fmt.Sprintf("英: %s    ", s.Find("span.phonetic").Text())
			pronounce += p
		}

		if i == 1 {
			p := fmt.Sprintf("美: %s", s.Find("span.phonetic").Text())
			pronounce += p
		}
	})

	return pronounce
}

func getSentences(doc *goquery.Document, isChinese bool) [][]string {
	result := [][]string{}
	doc.Find("#bilingual ul li").Each(func(_ int, s *goquery.Selection) {
		r := []string{}
		s.Children().Each(func(ii int, ss *goquery.Selection) {
			// Ignore source
			if ii == 2 {
				return
			}
			var sentence string
			ss.Children().Each(func(iii int, sss *goquery.Selection) {
				if text := strings.TrimSpace(sss.Text()); text != "" {
					addSpace := (ii == 1 && isChinese) || (ii == 0 && !isChinese)
					if addSpace && iii != 0 && text != "." {
						text = " " + text
					}
					sentence += text
				}
			})
			r = append(r, sentence)
		})
		if len(r) == 2 {
			result = append(result, r)
		}
	})
	return result
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cloud is the root of the packages used to access Google Cloud
// Services. See https://godoc.org/cloud.google.com/go for a full list
// of sub-packages.
//
// This package documents how to authorize and authenticate the sub packages.
package cloud // import "cloud.google.com/go"
<commit_msg>cloud: document connection pooling mechanics and client transports<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package cloud is the root of the packages used to access Google Cloud
Services. See https://godoc.org/cloud.google.com/go for a full list
of sub-packages.

Examples in this package show ways to authorize and authenticate the
sub packages.

Connection Pooling

Connection pooling differs in clients based on their transport. Cloud
clients either rely on HTTP or gRPC transports to communicate
with Google Cloud.

Cloud clients that use HTTP rely on the underlying HTTP transport to
cache connections for later re-use. These are cached to the default
http.MaxIdleConns and http.MaxIdleConnsPerHost settings in
http.DefaultTransport.

For gPRC clients, connection pooling is configurable to the user. Users of
cloud client libraries may specify option.WithGRPCConnectionPool(n)
as a client option to NewClient calls. This configures the underlying
gRPC connections to be pooled and addressed in a round robin fashion.
The recommended size is GOMAXPROCS.

Client Transports

The following clients use HTTP: bigquery, compute, storage, translate.

The following clients use gRPC: bigtable, container, dataproc, datastore,
debugger, dlp, errorreporting, firestore, iam, language, logging,
longrunning, monitoring, oslogin, profiler, pubsub, rpcreplay, spanner,
speech, trace, videointelligence, vision.

*/
package cloud // import "cloud.google.com/go"
<|endoftext|>


<commit_before>package typhon

import (
	"io"
	"net/http"
)

func copyChunked(dst io.Writer, src io.Reader) (written int64, err error) {
	flusher, flusherOk := dst.(http.Flusher)
	if !flusherOk {
		return io.Copy(dst, src)
	}

	// This is taken and lightly adapted from the source of io.Copy
	buf := make([]byte, 32*1024)
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if flusherOk {
				flusher.Flush()
			}
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er != nil {
			if er != io.EOF {
				err = er
			}
			break
		}
	}
	return
}
<commit_msg>Remove redundant check<commit_after>package typhon

import (
	"io"
	"net/http"
)

func copyChunked(dst io.Writer, src io.Reader) (written int64, err error) {
	flusher, flusherOk := dst.(http.Flusher)
	if !flusherOk {
		return io.Copy(dst, src)
	}

	// This is taken and lightly adapted from the source of io.Copy
	buf := make([]byte, 32*1024)
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			flusher.Flush()
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er != nil {
			if er != io.EOF {
				err = er
			}
			break
		}
	}
	return
}
<|endoftext|>


<commit_before>package photomgr

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

type CK101 struct {
	//Inherit
	baseCrawler

	//To store current CK101 post result
	BaseDir string
}

func NewCK101() *CK101 {
	c := new(CK101)
	c.baseAddress = "https://www.CK101.cc"
	c.entryAddress = "http://ck101.com/forum-1345-1.html"
	return c
}
func (b *CK101) HasValidURL(url string) bool {
	log.Println("url=", url)
	return true
}

func (p *CK101) GetUrlPhotos(target string) []string {
	var resultSlice []string

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		resultSlice = append(resultSlice, imgUrl)
	})
	return resultSlice
}

func (p *CK101) Crawler(target string, workerNum int) {

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1").Text()
	log.Println("[CK101]:", title, " starting downloading...")
	dir := fmt.Sprintf("%v/%v - %v", p.BaseDir, "CK101", title)
	if exist, _ := exists(dir); exist {
		//fmt.Println("Already download")
		return
	}
	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go p.worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

//Set CK101 board page index, fetch all post and return article count back
func (p *CK101) ParseCK101PageByIndex(page int) int {
	doc, err := goquery.NewDocument(p.entryAddress)
	if err != nil {
		log.Fatal(err)
	}

	urlList := make([]string, 0)
	postList := make([]string, 0)
	starList := make([]int, 0)

	var PageWebSide string
	page = page + 1 //one base
	if page > 1 {
		// Find page result
		PageWebSide = fmt.Sprintf("http://ck101.com/forum-1345-%d.html", page)
	} else {
		PageWebSide = p.entryAddress
	}
	//fmt.Println("Page", PageWebSide)

	doc, err = goquery.NewDocument(PageWebSide)
	if err != nil {
		log.Fatal(err)
	}
	doc.Find(".cl_box").Each(func(i int, s *goquery.Selection) {
		star := ""
		title := ""
		url := ""
		starInt := 0
		s.Find("a").Each(func(i int, tQ *goquery.Selection) {
			title, _ = tQ.Attr("title")
			url, _ = tQ.Attr("href")
		})
		s.Find("em").Each(func(i int, starC *goquery.Selection) {
			star_c, _ := starC.Attr("title")
			fmt.Println("star_c:", star_c)
			if strings.Contains(star_c, "查看") {
				star = strings.Replace(star_c, "查看", "", -1)
				fmt.Println("star:", star)
				star = strings.TrimSpace(star)
				starInt, _ = strconv.Atoi(star)
			}
			//}
		})
		urlList = append(urlList, url)
		starList = append(starList, starInt)
		postList = append(postList, title)
	})

	p.storedPostURLList = urlList
	p.storedStarList = starList
	p.storedPostTitleList = postList

	return len(p.storedPostTitleList)
}
<commit_msg>Fix #5 url only has radix url not full one.<commit_after>package photomgr

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

type CK101 struct {
	//Inherit
	baseCrawler

	//To store current CK101 post result
	BaseDir string
}

func NewCK101() *CK101 {
	c := new(CK101)
	c.baseAddress = "https://ck101.com"
	c.entryAddress = "https://ck101.com/forum-1345-1.html"
	return c
}
func (b *CK101) HasValidURL(url string) bool {
	log.Println("url=", url)
	return true
}

func (p *CK101) GetUrlPhotos(target string) []string {
	var resultSlice []string

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		resultSlice = append(resultSlice, imgUrl)
	})
	return resultSlice
}

func (p *CK101) Crawler(target string, workerNum int) {

	doc, err := goquery.NewDocument(target)
	log.Println("Down load target URL=", target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1").Text()
	log.Println("[CK101]:", title, " starting downloading...")
	dir := fmt.Sprintf("%v/%v - %v", p.BaseDir, "CK101", title)
	if exist, _ := exists(dir); exist {
		//fmt.Println("Already download")
		return
	}
	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go p.worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

//Set CK101 board page index, fetch all post and return article count back
func (p *CK101) ParseCK101PageByIndex(page int) int {
	doc, err := goquery.NewDocument(p.entryAddress)
	if err != nil {
		log.Fatal(err)
	}

	urlList := make([]string, 0)
	postList := make([]string, 0)
	starList := make([]int, 0)

	var PageWebSide string
	page = page + 1 //one base
	if page > 1 {
		// Find page result
		PageWebSide = fmt.Sprintf("http://ck101.com/forum-1345-%d.html", page)
	} else {
		PageWebSide = p.entryAddress
	}
	//fmt.Println("Page", PageWebSide)

	doc, err = goquery.NewDocument(PageWebSide)
	if err != nil {
		log.Fatal(err)
	}
	doc.Find(".cl_box").Each(func(i int, s *goquery.Selection) {
		star := ""
		title := ""
		url := ""
		starInt := 0
		s.Find("a").Each(func(i int, tQ *goquery.Selection) {
			title, _ = tQ.Attr("title")
			goUrl, _ := tQ.Attr("href")
			url = fmt.Sprintf("%s/%s", p.baseAddress, goUrl)
		})
		s.Find("em").Each(func(i int, starC *goquery.Selection) {
			star_c, _ := starC.Attr("title")
			fmt.Println("star_c:", star_c)
			if strings.Contains(star_c, "查看") {
				star = strings.Replace(star_c, "查看", "", -1)
				fmt.Println("star:", star)
				star = strings.TrimSpace(star)
				starInt, _ = strconv.Atoi(star)
			}
			//}
		})
		urlList = append(urlList, url)
		starList = append(starList, starInt)
		postList = append(postList, title)
	})

	p.storedPostURLList = urlList
	p.storedStarList = starList
	p.storedPostTitleList = postList

	return len(p.storedPostTitleList)
}
<|endoftext|>


<commit_before>package scipipe

const (
	// BUFSIZE is the standard buffer size used for channels connecting processes
	BUFSIZE = 16
	// Version is the SciPipe version in string format
	Version = "0.8"
)
<commit_msg>Update version string (need better approach for releasing versions)<commit_after>package scipipe

const (
	// BUFSIZE is the standard buffer size used for channels connecting processes
	BUFSIZE = 16
	// Version is the SciPipe version in string format
	Version = "0.8.1"
)
<|endoftext|>


<commit_before>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"time"
)

func main() {
	const (
		indent          = "\t"
		highlight_start = "\x1b[1;36m"
		highlight_end   = "\x1b[0m"
	)
	fmt.Print(indent, highlight_start, "Just Go", highlight_end, "\n")
	target := time.Date(2015, 11, 30, 0, 0, 0, 0, time.UTC)
	fmt.Print(indent, target.Format(time.UnixDate), "\n")

	var (
		previous time.Time
		days     int
		sign     string
	)
	for {
		now := time.Now()
		now = now.Add(time.Duration(-now.Nanosecond())) // truncate to second
		if now != previous {
			previous = now
			remaining := target.Sub(now)
			if remaining >= 0 {
				sign = "-" // countdown is "T minus..."
			} else {
				sign = "+" // count up is "T plus..."
				remaining = -remaining
			}
			if remaining >= 24*time.Hour {
				days = int(remaining / (24 * time.Hour))
				remaining = remaining % (24 * time.Hour)
			}
			fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
			if days > 0 {
				fmt.Print(days, "d")
			}
			fmt.Print(remaining, "          \r")
		}
		time.Sleep(50 * time.Millisecond)
	}
}
<commit_msg>new target<commit_after>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"time"
)

func main() {
	const (
		indent          = "\t"
		highlight_start = "\x1b[1;36m"
		highlight_end   = "\x1b[0m"
	)
	fmt.Print(indent, highlight_start, "Just Go", highlight_end, "\n")
	target := time.Date(2015, 12, 18, 0, 0, 0, 0, time.Local)
	fmt.Print(indent, target.Format(time.UnixDate), "\n")

	var (
		previous time.Time
		days     int
		sign     string
	)
	for {
		now := time.Now()
		now = now.Add(time.Duration(-now.Nanosecond())) // truncate to second
		if now != previous {
			previous = now
			remaining := target.Sub(now)
			if remaining >= 0 {
				sign = "-" // countdown is "T minus..."
			} else {
				sign = "+" // count up is "T plus..."
				remaining = -remaining
			}
			if remaining >= 24*time.Hour {
				days = int(remaining / (24 * time.Hour))
				remaining = remaining % (24 * time.Hour)
			}
			fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
			if days > 0 {
				fmt.Print(days, "d")
			}
			fmt.Print(remaining, "          \r")
		}
		time.Sleep(50 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>package biasAndVariance

import (
	"fmt"
	"github.com/santiaago/caltechx.go/linear"
	"math"
)

type GFunc func(x float64) float64

type BiasAndVariance struct {
	Interval            linear.Interval   // interval in which the target function is defined.
	TargetFunction      linear.LinearFunc // target function.
	TrainingExampleSize int               // number of training examples of the training set.
	Runs                int               // number of runs that should be used to learn.
	Bias                float64
	Variance            float64
	Slope               float64
	Constant            float64
	ThroughOrigin       bool // defines if hypothesis function goes through origin or not. ie: h(x) = ax or h(x) = ax + b
}

func NewBiasAndVariance() *BiasAndVariance {
	bav := BiasAndVariance{}
	bav.Interval = linear.Interval{-1, 1}
	bav.Runs = 1000
	bav.TargetFunction = func(x ...float64) float64 {
		return math.Sin(math.Pi * x[0])
	}
	bav.TrainingExampleSize = 2
	bav.ThroughOrigin = true
	return &bav
}

func (bav *BiasAndVariance) LearnLine() {
	sumA := float64(0)
	sumB := float64(0)

	type GFunc func(x, a float64) float64
	gs := make([]func(x, a, b float64) float64, bav.Runs)
	as := make([]float64, bav.Runs)
	bs := make([]float64, bav.Runs)

	for i := 0; i < bav.Runs; i++ {
		x1 := bav.Interval.RandFloat()
		y1 := bav.TargetFunction(x1)

		x2 := bav.Interval.RandFloat()
		y2 := bav.TargetFunction(x2)

		x := math.Abs(x2 - x1)
		y := math.Abs(y2 - y1)

		a := y / x
		sumA += a
		as[i] = a

		g := func(ix, ia, ib float64) float64 { return ia*ix + ib }
		gs[i] = g

		if !bav.ThroughOrigin {
			b := y1 - a*x1
			sumB += b
			bs[i] = b
		}

	}

	bav.Slope = sumA / float64(bav.Runs)
	bav.Constant = sumB / float64(bav.Runs)

	gBar := func(x float64) float64 {
		return bav.Slope*x + bav.Constant
	}

	bav.Bias = bav.ComputeBias(gBar)
	bav.Variance = bav.ComputeVariance(gBar, gs, as, bs)
}

func (bav *BiasAndVariance) LearnConstant() {
	sumB := float64(0)

	type GFunc func(x float64, a float64) float64
	gs := make([]func(x, a, b float64) float64, bav.Runs)
	bs := make([]float64, bav.Runs)
	for i := 0; i < bav.Runs; i++ {
		x1 := bav.Interval.RandFloat()
		y1 := bav.TargetFunction(x1)

		x2 := bav.Interval.RandFloat()
		y2 := bav.TargetFunction(x2)

		b := (y1 + y2) / float64(2)

		sumB += b

		g := func(ix, ia, ib float64) float64 { return ib }
		gs[i] = g
		bs[i] = b
	}

	bav.Constant = sumB / float64(bav.Runs)

	gBar := func(x float64) float64 {
		return bav.Constant
	}

	bav.Bias = bav.ComputeBias(gBar)
	bav.Variance = bav.ComputeVariance(gBar, gs, make([]float64, bav.Runs), bs)

}

func (bav *BiasAndVariance) LearnQuadratic() {
	sumA := float64(0)
	sumB := float64(0)

	type GFunc func(x float64, a float64) float64
	gs := make([]func(x, a, b float64) float64, bav.Runs)
	as := make([]float64, bav.Runs)
	bs := make([]float64, bav.Runs)

	for i := 0; i < bav.Runs; i++ {
		x1 := bav.Interval.RandFloat()
		y1 := bav.TargetFunction(x1)

		x2 := bav.Interval.RandFloat()
		y2 := bav.TargetFunction(x2)

		a := (y1 - y2) / (x1*x1 - x2*x2)
		as[i] = a

		if !bav.ThroughOrigin {
			b := y1 - a*x1*x1
			bs[i] = b
			sumB += b
		}
		g := func(ix, ia, ib float64) float64 { return ia*ix*ix + ib }
		gs[i] = g
	}

	// this should definitely not be called slope but First Coefficient or something similar.
	bav.Slope = sumA / float64(bav.Runs)

	if !bav.ThroughOrigin {
		bav.Constant = sumB / float64(bav.Runs)
	}

	gBar := func(x float64) float64 {
		return x*x*bav.Slope + bav.Constant
	}

	bav.Bias = bav.ComputeBias(gBar)
	bav.Variance = bav.ComputeVariance(gBar, gs, as, bs)
}

func (bav *BiasAndVariance) ComputeBias(gBar func(x float64) float64) float64 {
	sumBias := float64(0)
	for i := 0; i < bav.Runs; i++ {
		x := bav.Interval.RandFloat()
		sumBias += math.Pow(gBar(x)-bav.TargetFunction(x), float64(2))
	}
	return sumBias / float64(bav.Runs)
}

func (bav *BiasAndVariance) ComputeVariance(gBar func(x float64) float64, gs []func(x, a, b float64) float64, as []float64, bs []float64) float64 {
	sumVar := float64(0)
	for i := 0; i < bav.Runs; i++ {
		for j := 0; j < bav.Runs; j++ {
			x := bav.Interval.RandFloat()
			sumVar += math.Pow(gs[j](x, as[j], bs[j])-gBar(x), float64(2))
		}
	}
	return sumVar / math.Pow(float64(bav.Runs), float64(2))

}

func (bav *BiasAndVariance) Print() {
	fmt.Printf("bias = %3.2f\n", bav.Bias)
	fmt.Printf("variance = %3.2f\n", bav.Variance)
	fmt.Printf("Eout = %3.2f\n", bav.Bias+bav.Variance)
}
<commit_msg>use ml/linear instead of caltechx.go/linear<commit_after>package biasAndVariance

import (
	"fmt"
	"math"

	"github.com/santiaago/ml/linear"
)

type GFunc func(x float64) float64

type BiasAndVariance struct {
	Interval            linear.Interval // interval in which the target function is defined.
	TargetFunction      linear.Function // target function.
	TrainingExampleSize int             // number of training examples of the training set.
	Runs                int             // number of runs that should be used to learn.
	Bias                float64
	Variance            float64
	Slope               float64
	Constant            float64
	ThroughOrigin       bool // defines if hypothesis function goes through origin or not. ie: h(x) = ax or h(x) = ax + b
}

func NewBiasAndVariance() *BiasAndVariance {
	bav := BiasAndVariance{}
	bav.Interval = linear.NewInterval(float64(-1), float64(1))
	bav.Runs = 1000
	bav.TargetFunction = func(x float64) float64 {
		return math.Sin(math.Pi * x)
	}
	bav.TrainingExampleSize = 2
	bav.ThroughOrigin = true
	return &bav
}

func (bav *BiasAndVariance) LearnLine() {
	sumA := float64(0)
	sumB := float64(0)

	type GFunc func(x, a float64) float64
	gs := make([]func(x, a, b float64) float64, bav.Runs)
	as := make([]float64, bav.Runs)
	bs := make([]float64, bav.Runs)

	for i := 0; i < bav.Runs; i++ {
		x1 := bav.Interval.RandFloat()
		y1 := bav.TargetFunction(x1)

		x2 := bav.Interval.RandFloat()
		y2 := bav.TargetFunction(x2)

		x := math.Abs(x2 - x1)
		y := math.Abs(y2 - y1)

		a := y / x
		sumA += a
		as[i] = a

		g := func(ix, ia, ib float64) float64 { return ia*ix + ib }
		gs[i] = g

		if !bav.ThroughOrigin {
			b := y1 - a*x1
			sumB += b
			bs[i] = b
		}

	}

	bav.Slope = sumA / float64(bav.Runs)
	bav.Constant = sumB / float64(bav.Runs)

	gBar := func(x float64) float64 {
		return bav.Slope*x + bav.Constant
	}

	bav.Bias = bav.ComputeBias(gBar)
	bav.Variance = bav.ComputeVariance(gBar, gs, as, bs)
}

func (bav *BiasAndVariance) LearnConstant() {
	sumB := float64(0)

	type GFunc func(x float64, a float64) float64
	gs := make([]func(x, a, b float64) float64, bav.Runs)
	bs := make([]float64, bav.Runs)
	for i := 0; i < bav.Runs; i++ {
		x1 := bav.Interval.RandFloat()
		y1 := bav.TargetFunction(x1)

		x2 := bav.Interval.RandFloat()
		y2 := bav.TargetFunction(x2)

		b := (y1 + y2) / float64(2)

		sumB += b

		g := func(ix, ia, ib float64) float64 { return ib }
		gs[i] = g
		bs[i] = b
	}

	bav.Constant = sumB / float64(bav.Runs)

	gBar := func(x float64) float64 {
		return bav.Constant
	}

	bav.Bias = bav.ComputeBias(gBar)
	bav.Variance = bav.ComputeVariance(gBar, gs, make([]float64, bav.Runs), bs)

}

func (bav *BiasAndVariance) LearnQuadratic() {
	sumA := float64(0)
	sumB := float64(0)

	type GFunc func(x float64, a float64) float64
	gs := make([]func(x, a, b float64) float64, bav.Runs)
	as := make([]float64, bav.Runs)
	bs := make([]float64, bav.Runs)

	for i := 0; i < bav.Runs; i++ {
		x1 := bav.Interval.RandFloat()
		y1 := bav.TargetFunction(x1)

		x2 := bav.Interval.RandFloat()
		y2 := bav.TargetFunction(x2)

		a := (y1 - y2) / (x1*x1 - x2*x2)
		as[i] = a

		if !bav.ThroughOrigin {
			b := y1 - a*x1*x1
			bs[i] = b
			sumB += b
		}
		g := func(ix, ia, ib float64) float64 { return ia*ix*ix + ib }
		gs[i] = g
	}

	// this should definitely not be called slope but First Coefficient or something similar.
	bav.Slope = sumA / float64(bav.Runs)

	if !bav.ThroughOrigin {
		bav.Constant = sumB / float64(bav.Runs)
	}

	gBar := func(x float64) float64 {
		return x*x*bav.Slope + bav.Constant
	}

	bav.Bias = bav.ComputeBias(gBar)
	bav.Variance = bav.ComputeVariance(gBar, gs, as, bs)
}

func (bav *BiasAndVariance) ComputeBias(gBar func(x float64) float64) float64 {
	sumBias := float64(0)
	for i := 0; i < bav.Runs; i++ {
		x := bav.Interval.RandFloat()
		sumBias += math.Pow(gBar(x)-bav.TargetFunction(x), float64(2))
	}
	return sumBias / float64(bav.Runs)
}

func (bav *BiasAndVariance) ComputeVariance(gBar func(x float64) float64, gs []func(x, a, b float64) float64, as []float64, bs []float64) float64 {
	sumVar := float64(0)
	for i := 0; i < bav.Runs; i++ {
		for j := 0; j < bav.Runs; j++ {
			x := bav.Interval.RandFloat()
			sumVar += math.Pow(gs[j](x, as[j], bs[j])-gBar(x), float64(2))
		}
	}
	return sumVar / math.Pow(float64(bav.Runs), float64(2))

}

func (bav *BiasAndVariance) Print() {
	fmt.Printf("bias = %3.2f\n", bav.Bias)
	fmt.Printf("variance = %3.2f\n", bav.Variance)
	fmt.Printf("Eout = %3.2f\n", bav.Bias+bav.Variance)
}
<|endoftext|>


<commit_before>package comap

import (
	"sync"
	"fmt"
)

// Specify the number of the elements when CoMap is allocated.
var COUNT = 32

// ConcurrentMap is encapsulated into an array.
type CoMap []*ConcurrentMap

// The definition to thread-safe concurrent map.
type ConcurrentMap struct {
	concurrentMap map[string]interface{}
	sync.RWMutex
}

// Get the CoMap[key]'s value.
func (m CoMap) Get(key string) (interface{}, bool) {
	// Get elem
	elem := m.GetShard(key)
	elem.RLock()
	// Get ConcurrentMap from elem.
	val, ok := elem.concurrentMap[key]
	if !ok {
		fmt.Errorf("can not get val correctly when executing elem.concurrentMap[key]")
		// panic("error occurred when executing elem.concurrentMap[key]")
	}
	elem.RUnlock()
	return val, ok
}

// Set the CoMap[key]'s value.
func (m CoMap) Set(key string, value interface{}) {
	// Get map elem.
	elem := m.GetShard(key)
	elem.Lock()
	elem.concurrentMap[key] = value
	elem.Unlock()
}

// Get the corresponding key's map.
func (m CoMap) GetShard(key string) *ConcurrentMap {
	return m[uint(hash(key))%uint(COUNT)]
}

// hashing: bit shifting
func hash(key string) uint32 {
	var hash = uint32(len(key))
	// prime is a prime number to execute hashing bit shifting operation.
	const prime uint32 = 16777619
	for i := 0; i < len(key); i++ {
		hash = (hash << 4) ^ (hash >> 28) ^ uint32(key[i])
	}
	return hash % prime
}

// Create a new CoMap.
func New() CoMap {
	m := make(CoMap, COUNT)
	for i := 0; i < COUNT; i++ {
		m[i] = &ConcurrentMap{concurrentMap: make(map[string]interface{})}
	}
	return m
}
<commit_msg>edit comments<commit_after>package comap

import (
	"sync"
	"fmt"
)

// COUNT specify the number of the elements when CoMap is allocated.
const COUNT = 32

// ConcurrentMap is encapsulated into an array called CoMap.
type CoMap []*ConcurrentMap

// The definition about thread-safe concurrent map.
type ConcurrentMap struct {
	concurrentMap map[string]interface{}
	sync.RWMutex
}

// Get() gets the CoMap[key]'s value.
func (m CoMap) Get(key string) (interface{}, bool) {
	// Get elem
	elem := m.GetShard(key)
	elem.RLock()
	// Get ConcurrentMap from elem.
	val, ok := elem.concurrentMap[key]
	if !ok {
		// error information need to be improved...
		fmt.Errorf("can not get val correctly when executing elem.concurrentMap[key]")
		// panic("error occurred when executing elem.concurrentMap[key]")
	}
	elem.RUnlock()
	return val, ok
}

// Set() sets the CoMap[key]'s value.
func (m CoMap) Set(key string, value interface{}) {
	// Get map elem.
	elem := m.GetShard(key)
	elem.Lock()
	elem.concurrentMap[key] = value
	elem.Unlock()
}

// GetShard() gets the corresponding key's map.
func (m CoMap) GetShard(key string) *ConcurrentMap {
	return m[uint(hash(key))%uint(COUNT)]
}

// hashing: bit shifting
func hash(key string) uint32 {
	var hash = uint32(len(key))
	// prime is a prime number to execute hashing bit shifting operation.
	const prime uint32 = 16777619
	for i := 0; i < len(key); i++ {
		hash = (hash << 4) ^ (hash >> 28) ^ uint32(key[i])
	}
	return hash % prime
}

// New() creates a new CoMap with capacity COUNT.
func New() CoMap {
	m := make(CoMap, COUNT)
	for i := 0; i < COUNT; i++ {
		m[i] = &ConcurrentMap{concurrentMap: make(map[string]interface{})}
	}
	return m
}
<|endoftext|>


<commit_before>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket                 string
	cloudfrontDistribution string
	cloudfrontURLSigner    *sign.URLSigner
	s3                     *s3.S3
	uploader               *s3manager.Uploader
}

func newAmazonClient(bucket string, cloudfrontDistribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	var signer *sign.URLSigner
	cloudfrontDistribution = strings.TrimSpace(cloudfrontDistribution)
	if cloudfrontDistribution != "" {
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err != nil {
			return nil, err
		}

		cloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
		if err != nil {
			return nil, err
		}
		block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
		if block == nil || block.Type != "RSA PRIVATE KEY" {
			return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
		}
		cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		signer = sign.NewURLSigner(string(cloudfrontKeyPairId), cloudfrontPrivateKey)
		lion.Infof("Using cloudfront security credentials - keypair ID (%v) - to sign cloudfront URLs", cloudfrontKeyPairId)
	}
	return &amazonClient{
		bucket:                 bucket,
		cloudfrontDistribution: cloudfrontDistribution,
		cloudfrontURLSigner:    signer,
		s3:                     s3.New(session),
		uploader:               s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	var reader io.ReadCloser
	if c.cloudfrontDistribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.cloudfrontDistribution, name)

		if c.cloudfrontURLSigner != nil {
			signedURL, err := c.cloudfrontURLSigner.Sign(url, time.Now().Add(1*time.Hour))
			if err != nil {
				return nil, err
			}
			url = strings.TrimSpace(signedURL)
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.cloudfrontDistribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<commit_msg>Make sure to print the id as a string<commit_after>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket                 string
	cloudfrontDistribution string
	cloudfrontURLSigner    *sign.URLSigner
	s3                     *s3.S3
	uploader               *s3manager.Uploader
}

func newAmazonClient(bucket string, cloudfrontDistribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	var signer *sign.URLSigner
	cloudfrontDistribution = strings.TrimSpace(cloudfrontDistribution)
	if cloudfrontDistribution != "" {
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err != nil {
			return nil, err
		}

		cloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
		if err != nil {
			return nil, err
		}
		block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
		if block == nil || block.Type != "RSA PRIVATE KEY" {
			return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
		}
		cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			return nil, err
		}
		signer = sign.NewURLSigner(string(cloudfrontKeyPairId), cloudfrontPrivateKey)
		lion.Infof("Using cloudfront security credentials - keypair ID (%v) - to sign cloudfront URLs", string(cloudfrontKeyPairId))
	}
	return &amazonClient{
		bucket:                 bucket,
		cloudfrontDistribution: cloudfrontDistribution,
		cloudfrontURLSigner:    signer,
		s3:                     s3.New(session),
		uploader:               s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	var reader io.ReadCloser
	if c.cloudfrontDistribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.cloudfrontDistribution, name)

		if c.cloudfrontURLSigner != nil {
			signedURL, err := c.cloudfrontURLSigner.Sign(url, time.Now().Add(1*time.Hour))
			if err != nil {
				return nil, err
			}
			url = strings.TrimSpace(signedURL)
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.cloudfrontDistribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<|endoftext|>


<commit_before>package rapid

import (
	"fmt"
	"html/template"
	"net/http"
	"strconv"
)

// Connection - struct for handling http request and write
type Connection struct {
	R *http.Request
	W http.ResponseWriter
}

type routeHandler func(Connection) string

// Route - Create a route for your webserver
func Route(path string, handler routeHandler) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		resp := handler(Connection{r, w})

		if resp != "" {
			fmt.Fprintf(w, resp)
		}
	})
}

func (c *Connection) View(path string) string {
	return c.Render(path, nil)
}

func (c *Connection) Render(path string, object interface{}) string {
	t, err := template.ParseFiles(path)
	if err != nil {
		fmt.Println(err)
	}
	c.W.Header().Set("Content-Type", "text/html; charset=utf-8")
	t.Execute(c.W, object)
	return ""
}

func PublicFolder(path string) {
	http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir(path))))
}

// StartServer - Start webserver on specified port
func StartServer(port int) {
	portString := strconv.Itoa(port)
	http.ListenAndServe(":"+portString, nil)
}
<commit_msg>Very close with url parameters.<commit_after>package rapid

import (
	"fmt"
	"html/template"
	"net/http"
	"strconv"
	"strings"
)

// Connection - struct for handling http request and write
type Connection struct {
	R      *http.Request
	W      http.ResponseWriter
	Params map[string]string
}

type routeHandler func(Connection) string

// Route - Create a route for your webserver
func Route(path string, handler routeHandler) {
	params, path := getParams(path)
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {

		resp := handler(Connection{r, w, params})

		if resp != "" {
			fmt.Fprintf(w, resp)
		}
	})
}

func (c *Connection) View(path string) string {
	return c.Render(path, nil)
}

func (c *Connection) Render(path string, object interface{}) string {
	t, err := template.ParseFiles(path)
	if err != nil {
		fmt.Println(err)
	}
	c.W.Header().Set("Content-Type", "text/html; charset=utf-8")
	t.Execute(c.W, object)
	return ""
}

func PublicFolder(path string) {
	http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir(path))))
}

// StartServer - Start webserver on specified port
func StartServer(port int) {
	portString := strconv.Itoa(port)
	http.ListenAndServe(":"+portString, nil)
}

func getParams(path string, r *http.Request) (map[string]string, string) {

	routePath := strings.Split(path, "/")
	requestPath := strings.Split(r.URL.Path, "/")

	params := map[string]string{}
	for i := len(routePath) - 1; i >= 0; i-- {
		dir := routePath[i]
		if dir[0] == ':' {
			params[dir[1:len(routePath[i])-1]] = requestPath[i]
			routePath = append(routePath[:i], routePath[i+1:]...)
		}
	}

	return params, strings.Join(routePath, "/")
}
<|endoftext|>


<commit_before>package vat

import (
	"encoding/json"
	"errors"
	"net/http"
	"sync"
	"time"
)

// RatePeriod represents a time and the various activate rates at that time.
type RatePeriod struct {
	EffectiveFromStr string `json:"effective_from"`
	EffectiveFrom    time.Time
	Rates            map[string]float32
}

// CountryRates holds the various differing VAT rate periods for a given country
type CountryRates struct {
	Name        string `json:"name"`
	Code        string `json:"code"`
	CountryCode string `json:"country_code"`
	Periods     []RatePeriod
}

var mutex = &sync.Mutex{} // protect countriesRates
var countriesRates []CountryRates

// ErrInvalidCountryCode will be returned when calling GetCountryRates with an invalid country code
var ErrInvalidCountryCode = errors.New("Unknown country code.")

// ErrInvalidRateLevel will be returned when getting wrong rate level
var ErrInvalidRateLevel = errors.New("Unknown rate level")

// GetRateOn returns the effective VAT rate on a given date
func (cr *CountryRates) GetRateOn(t time.Time, level string) (float32, error) {
	var activePeriod RatePeriod

	// find active period for the given time
	for _, p := range cr.Periods {
		if t.After(p.EffectiveFrom) && (activePeriod.EffectiveFrom.IsZero() || p.EffectiveFrom.After(activePeriod.EffectiveFrom)) {
			activePeriod = p
		}
	}

	activeRate, ok := activePeriod.Rates[level]
	if !ok {
		return 0.00, ErrInvalidRateLevel
	}

	return activeRate, nil
}

// GetRate returns the currently active rate
func (cr *CountryRates) GetRate(level string) (float32, error) {
	now := time.Now()
	return cr.GetRateOn(now, level)
}

// GetCountryRates gets the CountryRates struct for a country by its ISO-3166-1-alpha2 country code.
func GetCountryRates(countryCode string) (CountryRates, error) {
	var rate CountryRates
	rates, err := GetRates()

	if err != nil {
		return rate, err
	}

	for _, r := range rates {
		if r.CountryCode == countryCode {
			return r, nil
		}
	}

	return rate, ErrInvalidCountryCode
}

// GetRates returns the in-memory VAT rates
func GetRates() ([]CountryRates, error) {
	var err error

	mutex.Lock()
	if countriesRates == nil {
		countriesRates, err = FetchRates()
	}
	mutex.Unlock()

	return countriesRates, err
}

// FetchRates fetches the latest VAT rates from jsonvat.com and updates the in-memory rates
func FetchRates() ([]CountryRates, error) {

	client := http.Client{
		Timeout: (time.Duration(ServiceTimeout) * time.Second),
	}
	r, err := client.Get("https://jsonvat.com/")
	if err != nil {
		return nil, err
	}

	apiResponse := &struct {
		Details string
		Version string
		Rates   []CountryRates
	}{}

	err = json.NewDecoder(r.Body).Decode(&apiResponse)
	if err != nil {
		return nil, err
	}

	// convert EffectiveFrom to a proper time.Time for each rate period
	for idx1, cr := range apiResponse.Rates {
		for idx2, crp := range cr.Periods {
			apiResponse.Rates[idx1].Periods[idx2].EffectiveFrom, _ = time.Parse("2006-01-01", crp.EffectiveFromStr)
		}
	}

	countriesRates = apiResponse.Rates
	return countriesRates, err
}
<commit_msg>prefix error strings<commit_after>package vat

import (
	"encoding/json"
	"errors"
	"net/http"
	"sync"
	"time"
)

// RatePeriod represents a time and the various activate rates at that time.
type RatePeriod struct {
	EffectiveFromStr string `json:"effective_from"`
	EffectiveFrom    time.Time
	Rates            map[string]float32
}

// CountryRates holds the various differing VAT rate periods for a given country
type CountryRates struct {
	Name        string `json:"name"`
	Code        string `json:"code"`
	CountryCode string `json:"country_code"`
	Periods     []RatePeriod
}

var mutex = &sync.Mutex{} // protect countriesRates
var countriesRates []CountryRates

// ErrInvalidCountryCode will be returned when calling GetCountryRates with an invalid country code
var ErrInvalidCountryCode = errors.New("vat: unknown country code")

// ErrInvalidRateLevel will be returned when getting wrong rate level
var ErrInvalidRateLevel = errors.New("vat: unknown rate level")

// GetRateOn returns the effective VAT rate on a given date
func (cr *CountryRates) GetRateOn(t time.Time, level string) (float32, error) {
	var activePeriod RatePeriod

	// find active period for the given time
	for _, p := range cr.Periods {
		if t.After(p.EffectiveFrom) && (activePeriod.EffectiveFrom.IsZero() || p.EffectiveFrom.After(activePeriod.EffectiveFrom)) {
			activePeriod = p
		}
	}

	activeRate, ok := activePeriod.Rates[level]
	if !ok {
		return 0.00, ErrInvalidRateLevel
	}

	return activeRate, nil
}

// GetRate returns the currently active rate
func (cr *CountryRates) GetRate(level string) (float32, error) {
	now := time.Now()
	return cr.GetRateOn(now, level)
}

// GetCountryRates gets the CountryRates struct for a country by its ISO-3166-1-alpha2 country code.
func GetCountryRates(countryCode string) (CountryRates, error) {
	var rate CountryRates
	rates, err := GetRates()

	if err != nil {
		return rate, err
	}

	for _, r := range rates {
		if r.CountryCode == countryCode {
			return r, nil
		}
	}

	return rate, ErrInvalidCountryCode
}

// GetRates returns the in-memory VAT rates
func GetRates() ([]CountryRates, error) {
	var err error

	mutex.Lock()
	if countriesRates == nil {
		countriesRates, err = FetchRates()
	}
	mutex.Unlock()

	return countriesRates, err
}

// FetchRates fetches the latest VAT rates from jsonvat.com and updates the in-memory rates
func FetchRates() ([]CountryRates, error) {

	client := http.Client{
		Timeout: (time.Duration(ServiceTimeout) * time.Second),
	}
	r, err := client.Get("https://jsonvat.com/")
	if err != nil {
		return nil, err
	}

	apiResponse := &struct {
		Details string
		Version string
		Rates   []CountryRates
	}{}

	err = json.NewDecoder(r.Body).Decode(&apiResponse)
	if err != nil {
		return nil, err
	}

	// convert EffectiveFrom to a proper time.Time for each rate period
	for idx1, cr := range apiResponse.Rates {
		for idx2, crp := range cr.Periods {
			apiResponse.Rates[idx1].Periods[idx2].EffectiveFrom, _ = time.Parse("2006-01-01", crp.EffectiveFromStr)
		}
	}

	countriesRates = apiResponse.Rates
	return countriesRates, err
}
<|endoftext|>


<commit_before>package amf

const (
	AMF0 = 0x00
	AMF3 = 0x03
)

const (
	AMF0_NUMBER_MARKER         = 0x00
	AMF0_BOOLEAN_MARKER        = 0x01
	AMF0_STRING_MARKER         = 0x02
	AMF0_OBJECT_MARKER         = 0x03
	AMF0_MOVIECLIP_MARKER      = 0x04
	AMF0_NULL_MARKER           = 0x05
	AMF0_UNDEFINED_MARKER      = 0x06
	AMF0_REFERENCE_MARKER      = 0x07
	AMF0_ECMA_ARRAY_MARKER     = 0x08
	AMF0_OBJECT_END_MARKER     = 0x09
	AMF0_STRICT_ARRAY_MARKER   = 0x0a
	AMF0_DATE_MARKER           = 0x0b
	AMF0_LONG_STRING_MARKER    = 0x0c
	AMF0_UNSUPPORTED_MARKER    = 0x0d
	AMF0_RECORDSET_MARKER      = 0x0e
	AMF0_XML_DOCUMENT_MARKER   = 0x0f
	AMF0_TYPED_OBJECT_MARKER   = 0x10
	AMF0_ACMPLUS_OBJECT_MARKER = 0x11
)

const (
	AMF0_BOOLEAN_FALSE = 0x00
	AMF0_BOOLEAN_TRUE  = 0x01
	AMF0_STRING_MAX    = 65535
)

const (
	AMF3_UNDEFINED_MARKER = 0x00
	AMF3_NULL_MARKER      = 0x01
	AMF3_FALSE_MARKER     = 0x02
	AMF3_TRUE_MARKER      = 0x03
	AMF3_INTEGER_MARKER   = 0x04
	AMF3_DOUBLE_MARKER    = 0x05
	AMF3_STRING_MARKER    = 0x06
	AMF3_XMLDOC_MARKER    = 0x07
	AMF3_DATE_MARKER      = 0x08
	AMF3_ARRAY_MARKER     = 0x09
	AMF3_OBJECT_MARKER    = 0x0a
	AMF3_XML_MARKER       = 0x0b
	AMF3_BYTEARRAY_MARKER = 0x0c
)

type Version uint8

type StrictArray []interface{}
type EcmaArray map[string]interface{}
type Object map[string]interface{}

type TypedObject struct {
	Type   string
	Object Object
}
<commit_msg>Add method to safely build typed object <commit_after>package amf

const (
	AMF0 = 0x00
	AMF3 = 0x03
)

const (
	AMF0_NUMBER_MARKER         = 0x00
	AMF0_BOOLEAN_MARKER        = 0x01
	AMF0_STRING_MARKER         = 0x02
	AMF0_OBJECT_MARKER         = 0x03
	AMF0_MOVIECLIP_MARKER      = 0x04
	AMF0_NULL_MARKER           = 0x05
	AMF0_UNDEFINED_MARKER      = 0x06
	AMF0_REFERENCE_MARKER      = 0x07
	AMF0_ECMA_ARRAY_MARKER     = 0x08
	AMF0_OBJECT_END_MARKER     = 0x09
	AMF0_STRICT_ARRAY_MARKER   = 0x0a
	AMF0_DATE_MARKER           = 0x0b
	AMF0_LONG_STRING_MARKER    = 0x0c
	AMF0_UNSUPPORTED_MARKER    = 0x0d
	AMF0_RECORDSET_MARKER      = 0x0e
	AMF0_XML_DOCUMENT_MARKER   = 0x0f
	AMF0_TYPED_OBJECT_MARKER   = 0x10
	AMF0_ACMPLUS_OBJECT_MARKER = 0x11
)

const (
	AMF0_BOOLEAN_FALSE = 0x00
	AMF0_BOOLEAN_TRUE  = 0x01
	AMF0_STRING_MAX    = 65535
)

const (
	AMF3_UNDEFINED_MARKER = 0x00
	AMF3_NULL_MARKER      = 0x01
	AMF3_FALSE_MARKER     = 0x02
	AMF3_TRUE_MARKER      = 0x03
	AMF3_INTEGER_MARKER   = 0x04
	AMF3_DOUBLE_MARKER    = 0x05
	AMF3_STRING_MARKER    = 0x06
	AMF3_XMLDOC_MARKER    = 0x07
	AMF3_DATE_MARKER      = 0x08
	AMF3_ARRAY_MARKER     = 0x09
	AMF3_OBJECT_MARKER    = 0x0a
	AMF3_XML_MARKER       = 0x0b
	AMF3_BYTEARRAY_MARKER = 0x0c
)

type Version uint8

type StrictArray []interface{}
type EcmaArray map[string]interface{}
type Object map[string]interface{}

type TypedObject struct {
	Type   string
	Object Object
}

func NewTypedObject() *TypedObject {
	return &TypedObject{
		Type:   "",
		Object: make(Object),
	}
}
<|endoftext|>


<commit_before>package common

import (
	"context"
	"fmt"
	"log"
	"math/rand"

	"github.com/hashicorp/packer/common/net"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

// This step configures the VM to enable the VNC server.
//
// Uses:
//   ui     packer.Ui
//   vmx_path string
//
// Produces:
//   vnc_port int - The port that VNC is configured to listen on.
type StepConfigureVNC struct {
	Enabled            bool
	VNCBindAddress     string
	VNCPortMin         int
	VNCPortMax         int
	VNCDisablePassword bool

	l *net.Listener
}

type VNCAddressFinder interface {
	VNCAddress(context.Context, string, int, int) (string, int, error)

	// UpdateVMX, sets driver specific VNC values to VMX data.
	UpdateVMX(vncAddress, vncPassword string, vncPort int, vmxData map[string]string)
}

func (s *StepConfigureVNC) VNCAddress(ctx context.Context, vncBindAddress string, portMin, portMax int) (string, int, error) {
	var err error
	s.l, err = net.ListenRangeConfig{
		Addr:    s.VNCBindAddress,
		Min:     s.VNCPortMin,
		Max:     s.VNCPortMax,
		Network: "tcp",
	}.Listen(ctx)
	if err != nil {
		return "", 0, err
	}

	s.l.Listener.Close()
	return s.l.Address, s.l.Port, nil
}

func VNCPassword(skipPassword bool) string {
	if skipPassword {
		return ""
	}
	length := int(8)

	charSet := []byte("012345689abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	charSetLength := len(charSet)

	password := make([]byte, length)

	for i := 0; i < length; i++ {
		password[i] = charSet[rand.Intn(charSetLength)]
	}

	return string(password)
}

func (s *StepConfigureVNC) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	if !s.Enabled {
		log.Println("Skipping VNC configuration step...")
		return multistep.ActionContinue
	}

	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)
	vmxPath := state.Get("vmx_path").(string)

	vmxData, err := ReadVMX(vmxPath)
	if err != nil {
		err := fmt.Errorf("Error reading VMX file: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	var vncFinder VNCAddressFinder
	if finder, ok := driver.(VNCAddressFinder); ok {
		vncFinder = finder
	} else {
		vncFinder = s
	}

	log.Printf("Looking for available port between %d and %d", s.VNCPortMin, s.VNCPortMax)
	vncBindAddress, vncPort, err := vncFinder.VNCAddress(ctx, s.VNCBindAddress, s.VNCPortMin, s.VNCPortMax)

	if err != nil {
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	s.l.Listener.Close() // free port, but don't unlock lock file

	vncPassword := VNCPassword(s.VNCDisablePassword)

	log.Printf("Found available VNC port: %v", s.l)

	vncFinder.UpdateVMX(vncBindAddress, vncPassword, vncPort, vmxData)

	if err := WriteVMX(vmxPath, vmxData); err != nil {
		err := fmt.Errorf("Error writing VMX data: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	state.Put("vnc_port", s.l.Port)
	state.Put("vnc_ip", s.l.Address)
	state.Put("vnc_password", vncPassword)

	return multistep.ActionContinue
}

func (*StepConfigureVNC) UpdateVMX(address, password string, port int, data map[string]string) {
	data["remotedisplay.vnc.enabled"] = "TRUE"
	data["remotedisplay.vnc.port"] = fmt.Sprintf("%d", port)
	data["remotedisplay.vnc.ip"] = address
	if len(password) > 0 {
		data["remotedisplay.vnc.password"] = password
	}
}

func (s *StepConfigureVNC) Cleanup(multistep.StateBag) {
	if s.l != nil {
		if err := s.l.Close(); err != nil {
			log.Printf("failed to unlock port lockfile: %v", err)
		}
	}
}
<commit_msg>StepConfigureVNC: close listener only once<commit_after>package common

import (
	"context"
	"fmt"
	"log"
	"math/rand"

	"github.com/hashicorp/packer/common/net"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

// This step configures the VM to enable the VNC server.
//
// Uses:
//   ui     packer.Ui
//   vmx_path string
//
// Produces:
//   vnc_port int - The port that VNC is configured to listen on.
type StepConfigureVNC struct {
	Enabled            bool
	VNCBindAddress     string
	VNCPortMin         int
	VNCPortMax         int
	VNCDisablePassword bool

	l *net.Listener
}

type VNCAddressFinder interface {
	VNCAddress(context.Context, string, int, int) (string, int, error)

	// UpdateVMX, sets driver specific VNC values to VMX data.
	UpdateVMX(vncAddress, vncPassword string, vncPort int, vmxData map[string]string)
}

func (s *StepConfigureVNC) VNCAddress(ctx context.Context, vncBindAddress string, portMin, portMax int) (string, int, error) {
	var err error
	s.l, err = net.ListenRangeConfig{
		Addr:    s.VNCBindAddress,
		Min:     s.VNCPortMin,
		Max:     s.VNCPortMax,
		Network: "tcp",
	}.Listen(ctx)
	if err != nil {
		return "", 0, err
	}

	s.l.Listener.Close() // free port, but don't unlock lock file
	return s.l.Address, s.l.Port, nil
}

func VNCPassword(skipPassword bool) string {
	if skipPassword {
		return ""
	}
	length := int(8)

	charSet := []byte("012345689abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	charSetLength := len(charSet)

	password := make([]byte, length)

	for i := 0; i < length; i++ {
		password[i] = charSet[rand.Intn(charSetLength)]
	}

	return string(password)
}

func (s *StepConfigureVNC) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
	if !s.Enabled {
		log.Println("Skipping VNC configuration step...")
		return multistep.ActionContinue
	}

	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)
	vmxPath := state.Get("vmx_path").(string)

	vmxData, err := ReadVMX(vmxPath)
	if err != nil {
		err := fmt.Errorf("Error reading VMX file: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	var vncFinder VNCAddressFinder
	if finder, ok := driver.(VNCAddressFinder); ok {
		vncFinder = finder
	} else {
		vncFinder = s
	}

	log.Printf("Looking for available port between %d and %d", s.VNCPortMin, s.VNCPortMax)
	vncBindAddress, vncPort, err := vncFinder.VNCAddress(ctx, s.VNCBindAddress, s.VNCPortMin, s.VNCPortMax)

	if err != nil {
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	vncPassword := VNCPassword(s.VNCDisablePassword)

	log.Printf("Found available VNC port: %v", s.l)

	vncFinder.UpdateVMX(vncBindAddress, vncPassword, vncPort, vmxData)

	if err := WriteVMX(vmxPath, vmxData); err != nil {
		err := fmt.Errorf("Error writing VMX data: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	state.Put("vnc_port", s.l.Port)
	state.Put("vnc_ip", s.l.Address)
	state.Put("vnc_password", vncPassword)

	return multistep.ActionContinue
}

func (*StepConfigureVNC) UpdateVMX(address, password string, port int, data map[string]string) {
	data["remotedisplay.vnc.enabled"] = "TRUE"
	data["remotedisplay.vnc.port"] = fmt.Sprintf("%d", port)
	data["remotedisplay.vnc.ip"] = address
	if len(password) > 0 {
		data["remotedisplay.vnc.password"] = password
	}
}

func (s *StepConfigureVNC) Cleanup(multistep.StateBag) {
	if s.l != nil {
		if err := s.l.Close(); err != nil {
			log.Printf("failed to unlock port lockfile: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package ssh

const (
	// This is a constant representing a script to install and uninstall public
	// key in remote hosts.
	DefaultPublicKeyInstallScript = `
#!/bin/bash
#
# This script file installs or uninstalls an RSA public key to/from authoried_keys
# file in a typical linux machine. This script should be registered with vault
# server while creating a role for key type 'dynamic'.
#
# Vault server runs this script on the target machine with the following params:
#
# $1:INSTALL_OPTION: "install" or "uninstall"
#
# $2:PUBLIC_KEY_FILE: File name containing public key to be installed. Vault server
# uses UUID as file name to avoid collisions with public keys generated for requests.
#
# $3:AUTH_KEYS_FILE: Absolute path of the authorized_keys file.
# Currently, vault uses /home/<username>/.ssh/authorized_keys as the path.
#
# [Note: This is a default script and is written to provide convenience.
# If the host platform differs, or if the binaries used in this script are not
# available, write a new script that takes the above parameters and does the
# same task as this script, and register it Vault while role creation using
# 'install_script' parameter.

INSTALL_OPTION=$1
PUBLIC_KEY_FILE=$2
AUTH_KEYS_FILE=$3

# Delete the public key file and the temporary file
function cleanup
{
	echo "$PUBLIC_KEY_FILE" > tempFile
        rm -f "$PUBLIC_KEY_FILE" temp_$PUBLIC_KEY_FILE
}

if [ "$INSTALL_OPTION" != "install" && "$INSTALL_OPTION" != "uninstall" ]; then
	exit 1
fi

# Remove the key from authorized_key file if it is already present.
# This step is common for both installing and uninstalling the key.
grep -vFf "$PUBLIC_KEY_FILE" "$AUTH_KEYS_FILE" > temp_$PUBLIC_KEY_FILE
cat temp_$PUBLIC_KEY_FILE | sudo tee "$AUTH_KEYS_FILE"

if [ "$INSTALL_OPTION" == "install" ]; then
# Append the new public key to authorized_keys file
cat "$PUBLIC_KEY_FILE" | sudo tee --append "$AUTH_KEYS_FILE"
fi

# Delete the auxiliary files
cleanup
`
)
<commit_msg>Vault SSH: Cleanup of aux files in install script<commit_after>package ssh

const (
	// This is a constant representing a script to install and uninstall public
	// key in remote hosts.
	DefaultPublicKeyInstallScript = `
#!/bin/bash
#
# This is a default script which installs or uninstalls an RSA public key to/from
# authoried_keys file in a typical linux machine. Use 'install_script' parameter
# with 'roles/' endpoint to register a custom script (for Dynamic type).
#
# Vault server runs this script on the target machine with the following params:
#
# $1:INSTALL_OPTION: "install" or "uninstall"
#
# $2:PUBLIC_KEY_FILE: File name containing public key to be installed. Vault server
# uses UUID as file name to avoid collisions with public keys generated for requests.
#
# $3:AUTH_KEYS_FILE: Absolute path of the authorized_keys file.
# Currently, vault uses /home/<username>/.ssh/authorized_keys as the path.
#
# [Note: If the platform differs or if the binaries used in this script are not
# available in target machine, provide a custom script.]

set -e

INSTALL_OPTION=$1
PUBLIC_KEY_FILE=$2
AUTH_KEYS_FILE=$3

# Delete the public key file and the temporary file
function cleanup
{
	rm -f "$PUBLIC_KEY_FILE" temp_$PUBLIC_KEY_FILE
}

# This ensures that cleanup is called if any command fails
trap cleanup EXIT

if [ "$INSTALL_OPTION" != "install" && "$INSTALL_OPTION" != "uninstall" ]; then
	exit 1
fi

# Remove the key from authorized_key file if it is already present.
# This step is common for both installing and uninstalling the key.
grep -vFf "$PUBLIC_KEY_FILE" "$AUTH_KEYS_FILE" > temp_$PUBLIC_KEY_FILE
cat temp_$PUBLIC_KEY_FILE | sudo tee "$AUTH_KEYS_FILE"

if [ "$INSTALL_OPTION" == "install" ]; then
# Append the new public key to authorized_keys file
cat "$PUBLIC_KEY_FILE" | sudo tee --append "$AUTH_KEYS_FILE"
fi
`
)
<|endoftext|>