<commit_before>package subdomains

import (
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/engine"
	"github.com/webx-top/echo/engine/fasthttp"
	"github.com/webx-top/echo/engine/standard"
)

func New() *Subdomains {
	s := &Subdomains{
		Hosts:   map[string]*echo.Echo{},
		Alias:   map[string]*Info{},
		Default: ``,
	}
	return s
}

type Info struct {
	Name string
	Host string
	*echo.Echo
}

type Subdomains struct {
	Hosts    map[string]*echo.Echo
	Alias    map[string]*Info
	Default  string //default name
	Protocol string //http/https
}

func (s *Subdomains) Add(name string, e *echo.Echo) *Subdomains {
	r := strings.SplitN(name, `@`, 2) //blog@www.blog.com
	var host string
	if len(r) > 1 {
		name = r[0]
		host = r[1]
	}
	s.Hosts[host] = e
	s.Alias[name] = &Info{Name: name, Host: host, Echo: e}
	return s
}

func (s *Subdomains) Get(args ...string) *Info {
	name := s.Default
	if len(args) > 0 {
		name = args[0]
		if e, ok := s.Alias[name]; ok {
			return e
		}
	}
	return nil
}

func (s *Subdomains) URL(purl string, args ...string) string {
	info := s.Get(args...)
	if info == nil {
		return purl
	}
	if len(s.Protocol) < 1 {
		return `http://` + info.Host + purl
	}
	return s.Protocol + `://` + info.Host + purl
}

func (s *Subdomains) FindByDomain(host string) (*echo.Echo, bool) {
	handler, exists := s.Hosts[host]
	if !exists {
		if p := strings.LastIndexByte(host, ':'); p > -1 {
			handler, exists = s.Hosts[host[0:p]]
		}
		if !exists {
			var info *Info
			info, exists = s.Alias[s.Default]
			if exists {
				handler = info.Echo
			}
		}
	}
	return handler, exists
}

func (s *Subdomains) ServeHTTP(r engine.Request, w engine.Response) {
	domain := r.Host()
	handler, exists := s.FindByDomain(domain)
	if exists && handler != nil {
		handler.ServeHTTP(r, w)
	} else {
		w.NotFound()
	}
}

func (s *Subdomains) Run(args ...interface{}) {
	var eng engine.Engine
	var arg interface{}
	size := len(args)
	if size > 0 {
		arg = args[0]
	}
	if size > 1 {
		if conf, ok := arg.(*engine.Config); ok {
			if v, ok := args[1].(string); ok {
				if v == `fast` {
					eng = fasthttp.NewWithConfig(conf)
				} else {
					eng = standard.NewWithConfig(conf)
				}
			} else {
				eng = fasthttp.NewWithConfig(conf)
			}
		} else {
			addr := `:80`
			if v, ok := arg.(string); ok && len(v) > 0 {
				addr = v
			}
			if v, ok := args[1].(string); ok {
				if v == `fast` {
					eng = fasthttp.New(addr)
				} else {
					eng = standard.New(addr)
				}
			} else {
				eng = fasthttp.New(addr)
			}
		}
	} else {
		switch v := arg.(type) {
		case string:
			eng = fasthttp.New(v)
		case engine.Engine:
			eng = v
		default:
			eng = fasthttp.New(`:80`)
		}
	}
	e := s.Get()
	if e == nil {
		for _, info := range s.Alias {
			e = info
			break
		}
	}
	e.Logger().Info(`Server has been launched.`)
	e.Run(eng, s)
	e.Logger().Info(`Server has been closed.`)
}
<commit_msg>update<commit_after>package subdomains

import (
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/engine"
	"github.com/webx-top/echo/engine/fasthttp"
	"github.com/webx-top/echo/engine/standard"
)

func New() *Subdomains {
	s := &Subdomains{
		Hosts:   map[string]*echo.Echo{},
		Alias:   map[string]*Info{},
		Default: ``,
	}
	return s
}

type Info struct {
	Name string
	Host string
	*echo.Echo
}

type Subdomains struct {
	Hosts    map[string]*echo.Echo
	Alias    map[string]*Info
	Default  string //default name
	Protocol string //http/https
}

func (s *Subdomains) Add(name string, e *echo.Echo) *Subdomains {
	r := strings.SplitN(name, `@`, 2) //blog@www.blog.com
	var host string
	if len(r) > 1 {
		name = r[0]
		host = r[1]
	}
	s.Hosts[host] = e
	s.Alias[name] = &Info{Name: name, Host: host, Echo: e}
	return s
}

func (s *Subdomains) Get(args ...string) *Info {
	name := s.Default
	if len(args) > 0 {
		name = args[0]
	}
	if e, ok := s.Alias[name]; ok {
		return e
	}
	return nil
}

func (s *Subdomains) URL(purl string, args ...string) string {
	info := s.Get(args...)
	if info == nil {
		return purl
	}
	if len(s.Protocol) < 1 {
		return `http://` + info.Host + purl
	}
	return s.Protocol + `://` + info.Host + purl
}

func (s *Subdomains) FindByDomain(host string) (*echo.Echo, bool) {
	handler, exists := s.Hosts[host]
	if !exists {
		if p := strings.LastIndexByte(host, ':'); p > -1 {
			handler, exists = s.Hosts[host[0:p]]
		}
		if !exists {
			var info *Info
			info, exists = s.Alias[s.Default]
			if exists {
				handler = info.Echo
			}
		}
	}
	return handler, exists
}

func (s *Subdomains) ServeHTTP(r engine.Request, w engine.Response) {
	domain := r.Host()
	handler, exists := s.FindByDomain(domain)
	if exists && handler != nil {
		handler.ServeHTTP(r, w)
	} else {
		w.NotFound()
	}
}

func (s *Subdomains) Run(args ...interface{}) {
	var eng engine.Engine
	var arg interface{}
	size := len(args)
	if size > 0 {
		arg = args[0]
	}
	if size > 1 {
		if conf, ok := arg.(*engine.Config); ok {
			if v, ok := args[1].(string); ok {
				if v == `fast` {
					eng = fasthttp.NewWithConfig(conf)
				} else {
					eng = standard.NewWithConfig(conf)
				}
			} else {
				eng = fasthttp.NewWithConfig(conf)
			}
		} else {
			addr := `:80`
			if v, ok := arg.(string); ok && len(v) > 0 {
				addr = v
			}
			if v, ok := args[1].(string); ok {
				if v == `fast` {
					eng = fasthttp.New(addr)
				} else {
					eng = standard.New(addr)
				}
			} else {
				eng = fasthttp.New(addr)
			}
		}
	} else {
		switch v := arg.(type) {
		case string:
			eng = fasthttp.New(v)
		case engine.Engine:
			eng = v
		default:
			eng = fasthttp.New(`:80`)
		}
	}
	e := s.Get()
	if e == nil {
		for _, info := range s.Alias {
			e = info
			break
		}
	}
	e.Logger().Info(`Server has been launched.`)
	e.Run(eng, s)
	e.Logger().Info(`Server has been closed.`)
}
<|endoftext|>


<commit_before>package format

import (
	"io"
	"os"

	log "github.com/Sirupsen/logrus"
)

var osExit = os.Exit

// Formatter defines output printing interface
type Formatter interface {
	PrintItem(item interface{}) error
	PrintList(items interface{}) error
	//PrintList(items [][]string, headers []string) error
	PrintError(context string, err error)
	PrintFatal(context string, err error)
}

var formatter Formatter

// InitializeFormatter creates a singleton Formatter
func InitializeFormatter(ftype string, out io.Writer) {
	if ftype == "json" {
		formatter = NewJSONFormatter(out)
	} else {
		formatter = NewTextFormatter(out)
	}
}

// GetFormatter creates a new JSONFormatter
func GetFormatter() Formatter {
	if formatter != nil {
		return formatter
	}
	log.Warn("Formatter hasn't been initialized. Initializing now to default formatter")
	InitializeFormatter("", os.Stdout)
	return formatter
}
<commit_msg>Added documented description to avoid exiting during tests (issue #76)<commit_after>package format

import (
	"io"
	"os"

	log "github.com/Sirupsen/logrus"
)

// Required workaround for testing os.Exit(1) scenarios in Go with coverage.
// Otherwise, PrintFatal cannot be evaluated due to os.Exit() cannot be captured.
// Implemented in test files (json/text)
var osExit = os.Exit

// Formatter defines output printing interface
type Formatter interface {
	PrintItem(item interface{}) error
	PrintList(items interface{}) error
	//PrintList(items [][]string, headers []string) error
	PrintError(context string, err error)
	PrintFatal(context string, err error)
}

var formatter Formatter

// InitializeFormatter creates a singleton Formatter
func InitializeFormatter(ftype string, out io.Writer) {
	if ftype == "json" {
		formatter = NewJSONFormatter(out)
	} else {
		formatter = NewTextFormatter(out)
	}
}

// GetFormatter creates a new JSONFormatter
func GetFormatter() Formatter {
	if formatter != nil {
		return formatter
	}
	log.Warn("Formatter hasn't been initialized. Initializing now to default formatter")
	InitializeFormatter("", os.Stdout)
	return formatter
}
<|endoftext|>


<commit_before>package supervisor

import (
	"fmt"
	"github.com/pborman/uuid"
	"github.com/starkandwayne/shield/db"
	"github.com/starkandwayne/shield/timespec"
	"github.com/starkandwayne/shield/api"
	"strings"
	"time"
)

type JobRepresentation struct {
	UUID  uuid.UUID
	Tspec string
	Error error
}
type JobFailedError struct {
	FailedJobs []JobRepresentation
}

func (e JobFailedError) Error() string {
	var jobList []string
	for _, j := range e.FailedJobs {
		jobList = append(jobList, j.UUID.String())
	}
	return fmt.Sprintf("the following job(s) failed: %s", strings.Join(jobList, ", "))
}

func (s *Supervisor) GetAllJobs() ([]*Job, error) {
	l := []*Job{}
	result, err := s.Database.Query(`
		SELECT j.uuid, j.paused,
		       t.plugin, t.endpoint,
		       s.plugin, s.endpoint,
		       sc.timespec, r.expiry
		FROM jobs j
			INNER JOIN targets   t    ON  t.uuid = j.target_uuid
			INNER JOIN stores    s    ON  s.uuid = j.store_uuid
			INNER JOIN schedules sc   ON sc.uuid = j.schedule_uuid
			INNER JOIN retention r    ON  r.uuid = j.retention_uuid
	`)
	if err != nil {
		return l, err
	}
	e := JobFailedError{}
	for result.Next() {
		j := &Job{Target: &PluginConfig{}, Store: &PluginConfig{}}
		var id, tspec string
		var expiry int
		//var paused bool
		err = result.Scan(&id, &j.Paused,
			&j.Target.Plugin, &j.Target.Endpoint,
			&j.Store.Plugin, &j.Store.Endpoint,
			&tspec, &expiry)
		j.UUID = uuid.Parse(id)
		if err != nil {
			e.FailedJobs = append(e.FailedJobs, JobRepresentation{j.UUID, tspec, err})
		}
		j.Spec, err = timespec.Parse(tspec)
		if err != nil {
			e.FailedJobs = append(e.FailedJobs, JobRepresentation{j.UUID, tspec, err})
		}
		l = append(l, j)
	}
	if len(e.FailedJobs) == 0 {
		return l, nil
	}
	return l, e
}

type Supervisor struct {
	tick    chan int          /* scheduler will send a message at regular intervals */
	resync  chan int          /* api goroutine will send here when the db changes significantly (i.e. new job, updated target, etc.) */
	workers chan Task         /* workers read from this channel to get tasks */
	updates chan WorkerUpdate /* workers write updates to this channel */

	Database *db.DB

	runq []*Task
	jobq []*Job

	nextWorker uint
}

func NewSupervisor() *Supervisor {
	return &Supervisor{
		tick:    make(chan int),
		resync:  make(chan int),
		workers: make(chan Task),
		updates: make(chan WorkerUpdate),
		runq:    make([]*Task, 0),
		jobq:    make([]*Job, 0),

		Database: &db.DB{},
	}
}

func (s *Supervisor) Resync() error {
	jobq, err := s.GetAllJobs()
	if err != nil {
		return err
	}

	// calculate the initial run of each job
	for _, job := range jobq {
		err := job.Reschedule()
		if err != nil {
			fmt.Printf("error encountered while determining next run of %s (%s): %s\n",
				job.UUID.String(), job.Spec.String(), err)
		} else {
			fmt.Printf("initial run of %s (%s) is at %s\n",
				job.UUID.String(), job.Spec.String(), job.NextRun)
		}
	}

	s.jobq = jobq
	return nil
}

func (s *Supervisor) CheckSchedule() {
	for _, job := range s.jobq {
		if !job.Runnable() {
			continue
		}

		fmt.Printf("scheduling execution of job %s\n", job.UUID.String())
		task := job.Task()
		id, err := s.Database.CreateTask(
			"system", // owner
			"backup",
			"ARGS", // FIXME: need real args
			job.UUID,
		)
		if err != nil {
			fmt.Printf("job -> task conversion / database update failed: %s\n", err)
			continue
		}

		task.UUID = id
		s.runq = append(s.runq, task)

		err = job.Reschedule()
		if err != nil {
			fmt.Printf("error encountered while determining next run of %s (%s): %s\n",
				job.UUID.String(), job.Spec.String(), err)
		} else {
			fmt.Printf("next run of %s (%s) is at %s\n",
				job.UUID.String(), job.Spec.String(), job.NextRun)
		}
	}
}

func (s *Supervisor) Run() error {
	if err := s.Database.Connect(); err != nil {
		return fmt.Errorf("failed to connect to %s database at %s: %s\n",
			s.Database.Driver, s.Database.DSN, err)
	}

	if err := s.Database.CheckCurrentSchema(); err != nil {
		return fmt.Errorf("database failed schema version check: %s\n", err)
	}

	if err := s.Resync(); err != nil {
		return err
	}
	if DEV_MODE_SCHEDULING {
		for _, job := range s.jobq {
			job.NextRun = time.Now()
		}
	}

	for {
		select {
		case <-s.resync:
			if err := s.Resync(); err != nil {
				fmt.Printf("resync error: %s\n", err)
			}

		case <-s.tick:
			s.CheckSchedule()

		case u := <-s.updates:
			if u.Op == STOPPED {
				fmt.Printf("  %s: job stopped at %s\n", u.Task, u.StoppedAt.String())
				if err := s.Database.CompleteTask(u.Task, u.StoppedAt); err != nil {
					fmt.Printf("  %s: !! failed to update database - %s\n", u.Task, err)
				}

			} else if u.Op == OUTPUT {
				fmt.Printf("  %s> %s\n", u.Task, u.Output)
				if err := s.Database.UpdateTaskLog(u.Task, u.Output); err != nil {
					fmt.Printf("  %s: !! failed to update database - %s\n", u.Task, err)
				}

			} else if u.Op == RESTORE_KEY {
				fmt.Printf("  %s: restore key is %s\n", u.Task, u.Output)
				if err := s.Database.CreateTaskArchive(u.Task, u.Output, time.Now()); err != nil {
					fmt.Printf("  %s: !! failed to update database - %s\n", u.Task, err)
				}

			} else {
				fmt.Printf("  %s: !! unrecognized op type\n", u.Task)
			}

		default:
			if len(s.runq) > 0 {
				select {
				case s.workers <- *s.runq[0]:
					s.Database.StartTask(s.runq[0].UUID, time.Now())
					fmt.Printf("sent a task to a worker\n")
					s.runq = s.runq[1:]
				default:
				}
			}
		}
	}

	return nil
}

func (s *Supervisor) SpawnAPI() {
	go api.Run(":8080", s.Database, s.resync)
}

func scheduler(c chan int) {
	for {
		time.Sleep(time.Millisecond * 200)
		c <- 1
	}
}

func (s *Supervisor) SpawnScheduler() {
	go scheduler(s.tick)
}
<commit_msg>Remove unreachable code to appease `go vet`<commit_after>package supervisor

import (
	"fmt"
	"github.com/pborman/uuid"
	"github.com/starkandwayne/shield/db"
	"github.com/starkandwayne/shield/timespec"
	"github.com/starkandwayne/shield/api"
	"strings"
	"time"
)

type JobRepresentation struct {
	UUID  uuid.UUID
	Tspec string
	Error error
}
type JobFailedError struct {
	FailedJobs []JobRepresentation
}

func (e JobFailedError) Error() string {
	var jobList []string
	for _, j := range e.FailedJobs {
		jobList = append(jobList, j.UUID.String())
	}
	return fmt.Sprintf("the following job(s) failed: %s", strings.Join(jobList, ", "))
}

func (s *Supervisor) GetAllJobs() ([]*Job, error) {
	l := []*Job{}
	result, err := s.Database.Query(`
		SELECT j.uuid, j.paused,
		       t.plugin, t.endpoint,
		       s.plugin, s.endpoint,
		       sc.timespec, r.expiry
		FROM jobs j
			INNER JOIN targets   t    ON  t.uuid = j.target_uuid
			INNER JOIN stores    s    ON  s.uuid = j.store_uuid
			INNER JOIN schedules sc   ON sc.uuid = j.schedule_uuid
			INNER JOIN retention r    ON  r.uuid = j.retention_uuid
	`)
	if err != nil {
		return l, err
	}
	e := JobFailedError{}
	for result.Next() {
		j := &Job{Target: &PluginConfig{}, Store: &PluginConfig{}}
		var id, tspec string
		var expiry int
		//var paused bool
		err = result.Scan(&id, &j.Paused,
			&j.Target.Plugin, &j.Target.Endpoint,
			&j.Store.Plugin, &j.Store.Endpoint,
			&tspec, &expiry)
		j.UUID = uuid.Parse(id)
		if err != nil {
			e.FailedJobs = append(e.FailedJobs, JobRepresentation{j.UUID, tspec, err})
		}
		j.Spec, err = timespec.Parse(tspec)
		if err != nil {
			e.FailedJobs = append(e.FailedJobs, JobRepresentation{j.UUID, tspec, err})
		}
		l = append(l, j)
	}
	if len(e.FailedJobs) == 0 {
		return l, nil
	}
	return l, e
}

type Supervisor struct {
	tick    chan int          /* scheduler will send a message at regular intervals */
	resync  chan int          /* api goroutine will send here when the db changes significantly (i.e. new job, updated target, etc.) */
	workers chan Task         /* workers read from this channel to get tasks */
	updates chan WorkerUpdate /* workers write updates to this channel */

	Database *db.DB

	runq []*Task
	jobq []*Job

	nextWorker uint
}

func NewSupervisor() *Supervisor {
	return &Supervisor{
		tick:    make(chan int),
		resync:  make(chan int),
		workers: make(chan Task),
		updates: make(chan WorkerUpdate),
		runq:    make([]*Task, 0),
		jobq:    make([]*Job, 0),

		Database: &db.DB{},
	}
}

func (s *Supervisor) Resync() error {
	jobq, err := s.GetAllJobs()
	if err != nil {
		return err
	}

	// calculate the initial run of each job
	for _, job := range jobq {
		err := job.Reschedule()
		if err != nil {
			fmt.Printf("error encountered while determining next run of %s (%s): %s\n",
				job.UUID.String(), job.Spec.String(), err)
		} else {
			fmt.Printf("initial run of %s (%s) is at %s\n",
				job.UUID.String(), job.Spec.String(), job.NextRun)
		}
	}

	s.jobq = jobq
	return nil
}

func (s *Supervisor) CheckSchedule() {
	for _, job := range s.jobq {
		if !job.Runnable() {
			continue
		}

		fmt.Printf("scheduling execution of job %s\n", job.UUID.String())
		task := job.Task()
		id, err := s.Database.CreateTask(
			"system", // owner
			"backup",
			"ARGS", // FIXME: need real args
			job.UUID,
		)
		if err != nil {
			fmt.Printf("job -> task conversion / database update failed: %s\n", err)
			continue
		}

		task.UUID = id
		s.runq = append(s.runq, task)

		err = job.Reschedule()
		if err != nil {
			fmt.Printf("error encountered while determining next run of %s (%s): %s\n",
				job.UUID.String(), job.Spec.String(), err)
		} else {
			fmt.Printf("next run of %s (%s) is at %s\n",
				job.UUID.String(), job.Spec.String(), job.NextRun)
		}
	}
}

func (s *Supervisor) Run() error {
	if err := s.Database.Connect(); err != nil {
		return fmt.Errorf("failed to connect to %s database at %s: %s\n",
			s.Database.Driver, s.Database.DSN, err)
	}

	if err := s.Database.CheckCurrentSchema(); err != nil {
		return fmt.Errorf("database failed schema version check: %s\n", err)
	}

	if err := s.Resync(); err != nil {
		return err
	}
	if DEV_MODE_SCHEDULING {
		for _, job := range s.jobq {
			job.NextRun = time.Now()
		}
	}

	for {
		select {
		case <-s.resync:
			if err := s.Resync(); err != nil {
				fmt.Printf("resync error: %s\n", err)
			}

		case <-s.tick:
			s.CheckSchedule()

		case u := <-s.updates:
			if u.Op == STOPPED {
				fmt.Printf("  %s: job stopped at %s\n", u.Task, u.StoppedAt.String())
				if err := s.Database.CompleteTask(u.Task, u.StoppedAt); err != nil {
					fmt.Printf("  %s: !! failed to update database - %s\n", u.Task, err)
				}

			} else if u.Op == OUTPUT {
				fmt.Printf("  %s> %s\n", u.Task, u.Output)
				if err := s.Database.UpdateTaskLog(u.Task, u.Output); err != nil {
					fmt.Printf("  %s: !! failed to update database - %s\n", u.Task, err)
				}

			} else if u.Op == RESTORE_KEY {
				fmt.Printf("  %s: restore key is %s\n", u.Task, u.Output)
				if err := s.Database.CreateTaskArchive(u.Task, u.Output, time.Now()); err != nil {
					fmt.Printf("  %s: !! failed to update database - %s\n", u.Task, err)
				}

			} else {
				fmt.Printf("  %s: !! unrecognized op type\n", u.Task)
			}

		default:
			if len(s.runq) > 0 {
				select {
				case s.workers <- *s.runq[0]:
					s.Database.StartTask(s.runq[0].UUID, time.Now())
					fmt.Printf("sent a task to a worker\n")
					s.runq = s.runq[1:]
				default:
				}
			}
		}
	}
}

func (s *Supervisor) SpawnAPI() {
	go api.Run(":8080", s.Database, s.resync)
}

func scheduler(c chan int) {
	for {
		time.Sleep(time.Millisecond * 200)
		c <- 1
	}
}

func (s *Supervisor) SpawnScheduler() {
	go scheduler(s.tick)
}
<|endoftext|>


<commit_before>package ast

type Node struct {
	Parent   *Node
	Children []*Node
	Value    interface{}
	Kind     Kind
}

func NewNode(k Kind, v interface{}, ch ...*Node) *Node {
	n := &Node{
		Kind:  k,
		Value: v,
	}
	for _, c := range ch {
		Insert(n, c)
	}
	return n
}

func (a *Node) Equal(b *Node) bool {
	if a.Kind != b.Kind {
		return false
	}
	if a.Value != b.Value {
		return false
	}
	if len(a.Children) != len(b.Children) {
		return false
	}
	for i, c := range a.Children {
		if !c.Equal(b.Children[i]) {
			return false
		}
	}
	return true
}

func Insert(parent *Node, children ...*Node) {
	parent.Children = append(parent.Children, children...)
	for _, ch := range children {
		ch.Parent = parent
	}
}

type List struct {
	Not   bool
	Chars string
}

type Range struct {
	Not    bool
	Lo, Hi rune
}

type Text struct {
	Text string
}

type Kind int

const (
	KindNothing Kind = iota
	KindPattern
	KindList
	KindRange
	KindText
	KindAny
	KindSuper
	KindSingle
	KindAnyOf
)
<commit_msg>syntax/ast: pretty printing<commit_after>package ast

import (
	"bytes"
	"fmt"
)

type Node struct {
	Parent   *Node
	Children []*Node
	Value    interface{}
	Kind     Kind
}

func NewNode(k Kind, v interface{}, ch ...*Node) *Node {
	n := &Node{
		Kind:  k,
		Value: v,
	}
	for _, c := range ch {
		Insert(n, c)
	}
	return n
}

func (a *Node) Equal(b *Node) bool {
	if a.Kind != b.Kind {
		return false
	}
	if a.Value != b.Value {
		return false
	}
	if len(a.Children) != len(b.Children) {
		return false
	}
	for i, c := range a.Children {
		if !c.Equal(b.Children[i]) {
			return false
		}
	}
	return true
}

func (a *Node) String() string {
	var buf bytes.Buffer
	buf.WriteString(a.Kind.String())
	if a.Value != nil {
		buf.WriteString(" =")
		buf.WriteString(fmt.Sprintf("%v", a.Value))
	}
	if len(a.Children) > 0 {
		buf.WriteString(" [")
		for i, c := range a.Children {
			if i > 0 {
				buf.WriteString(", ")
			}
			buf.WriteString(c.String())
		}
		buf.WriteString("]")
	}
	return buf.String()
}

func Insert(parent *Node, children ...*Node) {
	parent.Children = append(parent.Children, children...)
	for _, ch := range children {
		ch.Parent = parent
	}
}

type List struct {
	Not   bool
	Chars string
}

type Range struct {
	Not    bool
	Lo, Hi rune
}

type Text struct {
	Text string
}

type Kind int

const (
	KindNothing Kind = iota
	KindPattern
	KindList
	KindRange
	KindText
	KindAny
	KindSuper
	KindSingle
	KindAnyOf
)

func (k Kind) String() string {
	switch k {
	case KindNothing:
		return "Nothing"
	case KindPattern:
		return "Pattern"
	case KindList:
		return "List"
	case KindRange:
		return "Range"
	case KindText:
		return "Text"
	case KindAny:
		return "Any"
	case KindSuper:
		return "Super"
	case KindSingle:
		return "Single"
	case KindAnyOf:
		return "AnyOf"
	default:
		return ""
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This package contains helper methods for using object stores.
package objectstore

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/golang/glog"
	"gocloud.dev/blob"
	_ "gocloud.dev/blob/gcsblob"
	"gocloud.dev/blob/s3blob"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

type Config struct {
	Scheme      string
	BucketName  string
	Prefix      string
	QueryString string
}

func OpenBucket(ctx context.Context, k8sClient kubernetes.Interface, namespace string, config *Config) (bucket *blob.Bucket, err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("Failed to open bucket %q: %w", config.BucketName, err)
		}
	}()
	if config.Scheme == "minio://" {
		cred, err := getMinioCredential(ctx, k8sClient, namespace)
		if err != nil {
			return nil, fmt.Errorf("Failed to get minio credential: %w", err)
		}
		sess, err := session.NewSession(&aws.Config{
			Credentials:      credentials.NewStaticCredentials(cred.AccessKey, cred.SecretKey, ""),
			Region:           aws.String("minio"),
			Endpoint:         aws.String(MinioDefaultEndpoint()),
			DisableSSL:       aws.Bool(true),
			S3ForcePathStyle: aws.Bool(true),
		})

		if err != nil {
			return nil, fmt.Errorf("Failed to create session to access minio: %v", err)
		}
		minioBucket, err := s3blob.OpenBucket(ctx, sess, config.BucketName, nil)
		if err != nil {
			return nil, err
		}
		// Directly calling s3blob.OpenBucket does not allow overriding prefix via bucketConfig.BucketURL().
		// Therefore, we need to explicitly configure the prefixed bucket.
		return blob.PrefixedBucket(minioBucket, config.Prefix), nil

	}
	return blob.OpenBucket(ctx, config.bucketURL())
}

func (b *Config) bucketURL() string {
	u := b.Scheme + b.BucketName

	// append prefix=b.prefix to existing queryString
	q := b.QueryString
	if len(b.Prefix) > 0 {
		if len(q) > 0 {
			q = q + "&prefix=" + b.Prefix
		} else {
			q = "?prefix=" + b.Prefix
		}
	}

	u = u + q
	return u
}
func (b *Config) PrefixedBucket() string {
	return b.Scheme + path.Join(b.BucketName, b.Prefix)
}

func (b *Config) KeyFromURI(uri string) (string, error) {
	prefixedBucket := b.PrefixedBucket()
	if !strings.HasPrefix(uri, prefixedBucket) {
		return "", fmt.Errorf("URI %q does not have expected bucket prefix %q", uri, prefixedBucket)
	}

	key := strings.TrimLeft(strings.TrimPrefix(uri, prefixedBucket), "/")
	if len(key) == 0 {
		return "", fmt.Errorf("URI %q has empty key given prefixed bucket %q", uri, prefixedBucket)
	}
	return key, nil
}

func (b *Config) UriFromKey(blobKey string) string {
	return b.Scheme + path.Join(b.BucketName, b.Prefix, blobKey)
}

func UploadBlob(ctx context.Context, bucket *blob.Bucket, localPath, blobPath string) error {
	fileInfo, err := os.Stat(localPath)
	if err != nil {
		return fmt.Errorf("unable to stat local filepath %q: %w", localPath, err)
	}

	if !fileInfo.IsDir() {
		return uploadFile(ctx, bucket, localPath, blobPath)
	}

	// localPath is a directory.
	files, err := ioutil.ReadDir(localPath)
	if err != nil {
		return fmt.Errorf("unable to list local directory %q: %w", localPath, err)
	}

	for _, f := range files {
		if f.IsDir() {
			err = UploadBlob(ctx, bucket, filepath.Join(localPath, f.Name()), blobPath+"/"+f.Name())
			if err != nil {
				return err
			}
		} else {
			blobFilePath := filepath.Join(blobPath, filepath.Base(f.Name()))
			localFilePath := filepath.Join(localPath, f.Name())
			if err := uploadFile(ctx, bucket, localFilePath, blobFilePath); err != nil {
				return err
			}
		}

	}

	return nil
}

func DownloadBlob(ctx context.Context, bucket *blob.Bucket, localDir, blobDir string) error {
	iter := bucket.List(&blob.ListOptions{Prefix: blobDir})
	for {
		obj, err := iter.Next(ctx)
		if err != nil {
			if err == io.EOF {
				break
			}
			return fmt.Errorf("failed to list objects in remote storage %q: %w", blobDir, err)
		}
		if obj.IsDir {
			// TODO: is this branch possible?

			// Object stores list all files with the same prefix,
			// there is no need to recursively list each folder.
			continue
		} else {
			relativePath, err := filepath.Rel(blobDir, obj.Key)
			if err != nil {
				return fmt.Errorf("unexpected object key %q when listing %q: %w", obj.Key, blobDir, err)
			}
			if err := downloadFile(ctx, bucket, obj.Key, filepath.Join(localDir, relativePath)); err != nil {
				return err
			}
		}
	}
	return nil
}

var bucketPattern = regexp.MustCompile(`(^[a-z][a-z0-9]+:///?)([^/?]+)(/[^?]*)?(\?.+)?$`)

func ParseBucketConfig(path string) (*Config, error) {
	ms := bucketPattern.FindStringSubmatch(path)
	if ms == nil || len(ms) != 5 {
		return nil, fmt.Errorf("parse bucket config failed: unrecognized pipeline root format: %q", path)
	}

	// TODO: Verify/add support for file:///.
	if ms[1] != "gs://" && ms[1] != "s3://" && ms[1] != "minio://" {
		return nil, fmt.Errorf("parse bucket config failed: unsupported Cloud bucket: %q", path)
	}

	prefix := strings.TrimPrefix(ms[3], "/")
	if len(prefix) > 0 && !strings.HasSuffix(prefix, "/") {
		prefix = prefix + "/"
	}

	return &Config{
		Scheme:      ms[1],
		BucketName:  ms[2],
		Prefix:      prefix,
		QueryString: ms[4],
	}, nil
}

func ParseBucketConfigForArtifactURI(uri string) (*Config, error) {
	ms := bucketPattern.FindStringSubmatch(uri)
	if ms == nil || len(ms) != 5 {
		return nil, fmt.Errorf("parse bucket config failed: unrecognized uri format: %q", uri)
	}

	// TODO: Verify/add support for file:///.
	if ms[1] != "gs://" && ms[1] != "s3://" && ms[1] != "minio://" {
		return nil, fmt.Errorf("parse bucket config failed: unsupported Cloud bucket: %q", uri)
	}

	return &Config{
		Scheme:     ms[1],
		BucketName: ms[2],
	}, nil
}

// TODO(neuromage): Move these helper functions to a storage package and add tests.
func uploadFile(ctx context.Context, bucket *blob.Bucket, localFilePath, blobFilePath string) error {
	errorF := func(err error) error {
		return fmt.Errorf("uploadFile(): unable to complete copying %q to remote storage %q: %w", localFilePath, blobFilePath, err)
	}

	w, err := bucket.NewWriter(ctx, blobFilePath, nil)
	if err != nil {
		return errorF(fmt.Errorf("unable to open writer for bucket: %w", err))
	}

	r, err := os.Open(localFilePath)
	if err != nil {
		return errorF(fmt.Errorf("unable to open local file %q for reading: %w", localFilePath, err))
	}
	defer r.Close()

	if _, err = io.Copy(w, r); err != nil {
		return errorF(fmt.Errorf("unable to complete copying: %w", err))
	}

	if err = w.Close(); err != nil {
		return errorF(fmt.Errorf("failed to close Writer for bucket: %w", err))
	}

	glog.Infof("uploadFile(localFilePath=%q, blobFilePath=%q)", localFilePath, blobFilePath)
	return nil
}

func downloadFile(ctx context.Context, bucket *blob.Bucket, blobFilePath, localFilePath string) (err error) {
	errorF := func(err error) error {
		return fmt.Errorf("downloadFile(): unable to complete copying %q to local storage %q: %w", blobFilePath, localFilePath, err)
	}

	r, err := bucket.NewReader(ctx, blobFilePath, nil)
	if err != nil {
		return errorF(fmt.Errorf("unable to open reader for bucket: %w", err))
	}
	defer r.Close()

	localDir := filepath.Dir(localFilePath)
	if err := os.MkdirAll(localDir, 0644); err != nil {
		return errorF(fmt.Errorf("failed to create local directory %q: %w", localDir, err))
	}

	w, err := os.Create(localFilePath)
	if err != nil {
		return errorF(fmt.Errorf("unable to open local file %q for writing: %w", localFilePath, err))
	}
	defer func() {
		errClose := w.Close()
		if err == nil && errClose != nil {
			// override named return value "err" when there's a close error
			err = errorF(errClose)
		}
	}()

	if _, err = io.Copy(w, r); err != nil {
		return errorF(fmt.Errorf("unable to complete copying: %w", err))
	}

	return nil
}

// The endpoint uses Kubernetes service DNS name with namespace:
// https://kubernetes.io/docs/concepts/services-networking/service/#dns
const defaultMinioEndpointInMultiUserMode = "minio-service.kubeflow:9000"
const minioArtifactSecretName = "mlpipeline-minio-artifact"

func MinioDefaultEndpoint() string {
	// Discover minio-service in the same namespace by env var.
	// https://kubernetes.io/docs/concepts/services-networking/service/#environment-variables
	minioHost := os.Getenv("MINIO_SERVICE_SERVICE_HOST")
	minioPort := os.Getenv("MINIO_SERVICE_SERVICE_PORT")
	if minioHost != "" && minioPort != "" {
		// If there is a minio-service Kubernetes service in the same namespace,
		// MINIO_SERVICE_SERVICE_HOST and MINIO_SERVICE_SERVICE_PORT env vars should
		// exist by default, so we use it as default.
		return minioHost + ":" + minioPort
	}
	// If the env vars do not exist, we guess that we are running in KFP multi user mode, so default minio service should be `minio-service.kubeflow:9000`.
	glog.Infof("Cannot detect minio-service in the same namespace, default to %s as MinIO endpoint.", defaultMinioEndpointInMultiUserMode)
	return defaultMinioEndpointInMultiUserMode
}

type minioCredential struct {
	AccessKey string
	SecretKey string
}

func getMinioCredential(ctx context.Context, clientSet kubernetes.Interface, namespace string) (cred minioCredential, err error) {
	defer func() {
		if err != nil {
			// wrap error before returning
			err = fmt.Errorf("Failed to get MinIO credential from secret name=%q namespace=%q: %w", minioArtifactSecretName, namespace, err)
		}
	}()
	secret, err := clientSet.CoreV1().Secrets(namespace).Get(
		ctx,
		minioArtifactSecretName,
		metav1.GetOptions{})
	if err != nil {
		return cred, err
	}
	cred.AccessKey = string(secret.Data["accesskey"])
	cred.SecretKey = string(secret.Data["secretkey"])
	if cred.AccessKey == "" {
		return cred, fmt.Errorf("does not have 'accesskey' key")
	}
	if cred.SecretKey == "" {
		return cred, fmt.Errorf("does not have 'secretkey' key")
	}
	return cred, nil
}
<commit_msg>fix(backend): set correct permissions for local directory (#7212)<commit_after>// Copyright 2021 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This package contains helper methods for using object stores.
package objectstore

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/golang/glog"
	"gocloud.dev/blob"
	_ "gocloud.dev/blob/gcsblob"
	"gocloud.dev/blob/s3blob"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

type Config struct {
	Scheme      string
	BucketName  string
	Prefix      string
	QueryString string
}

func OpenBucket(ctx context.Context, k8sClient kubernetes.Interface, namespace string, config *Config) (bucket *blob.Bucket, err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("Failed to open bucket %q: %w", config.BucketName, err)
		}
	}()
	if config.Scheme == "minio://" {
		cred, err := getMinioCredential(ctx, k8sClient, namespace)
		if err != nil {
			return nil, fmt.Errorf("Failed to get minio credential: %w", err)
		}
		sess, err := session.NewSession(&aws.Config{
			Credentials:      credentials.NewStaticCredentials(cred.AccessKey, cred.SecretKey, ""),
			Region:           aws.String("minio"),
			Endpoint:         aws.String(MinioDefaultEndpoint()),
			DisableSSL:       aws.Bool(true),
			S3ForcePathStyle: aws.Bool(true),
		})

		if err != nil {
			return nil, fmt.Errorf("Failed to create session to access minio: %v", err)
		}
		minioBucket, err := s3blob.OpenBucket(ctx, sess, config.BucketName, nil)
		if err != nil {
			return nil, err
		}
		// Directly calling s3blob.OpenBucket does not allow overriding prefix via bucketConfig.BucketURL().
		// Therefore, we need to explicitly configure the prefixed bucket.
		return blob.PrefixedBucket(minioBucket, config.Prefix), nil

	}
	return blob.OpenBucket(ctx, config.bucketURL())
}

func (b *Config) bucketURL() string {
	u := b.Scheme + b.BucketName

	// append prefix=b.prefix to existing queryString
	q := b.QueryString
	if len(b.Prefix) > 0 {
		if len(q) > 0 {
			q = q + "&prefix=" + b.Prefix
		} else {
			q = "?prefix=" + b.Prefix
		}
	}

	u = u + q
	return u
}
func (b *Config) PrefixedBucket() string {
	return b.Scheme + path.Join(b.BucketName, b.Prefix)
}

func (b *Config) KeyFromURI(uri string) (string, error) {
	prefixedBucket := b.PrefixedBucket()
	if !strings.HasPrefix(uri, prefixedBucket) {
		return "", fmt.Errorf("URI %q does not have expected bucket prefix %q", uri, prefixedBucket)
	}

	key := strings.TrimLeft(strings.TrimPrefix(uri, prefixedBucket), "/")
	if len(key) == 0 {
		return "", fmt.Errorf("URI %q has empty key given prefixed bucket %q", uri, prefixedBucket)
	}
	return key, nil
}

func (b *Config) UriFromKey(blobKey string) string {
	return b.Scheme + path.Join(b.BucketName, b.Prefix, blobKey)
}

func UploadBlob(ctx context.Context, bucket *blob.Bucket, localPath, blobPath string) error {
	fileInfo, err := os.Stat(localPath)
	if err != nil {
		return fmt.Errorf("unable to stat local filepath %q: %w", localPath, err)
	}

	if !fileInfo.IsDir() {
		return uploadFile(ctx, bucket, localPath, blobPath)
	}

	// localPath is a directory.
	files, err := ioutil.ReadDir(localPath)
	if err != nil {
		return fmt.Errorf("unable to list local directory %q: %w", localPath, err)
	}

	for _, f := range files {
		if f.IsDir() {
			err = UploadBlob(ctx, bucket, filepath.Join(localPath, f.Name()), blobPath+"/"+f.Name())
			if err != nil {
				return err
			}
		} else {
			blobFilePath := filepath.Join(blobPath, filepath.Base(f.Name()))
			localFilePath := filepath.Join(localPath, f.Name())
			if err := uploadFile(ctx, bucket, localFilePath, blobFilePath); err != nil {
				return err
			}
		}

	}

	return nil
}

func DownloadBlob(ctx context.Context, bucket *blob.Bucket, localDir, blobDir string) error {
	iter := bucket.List(&blob.ListOptions{Prefix: blobDir})
	for {
		obj, err := iter.Next(ctx)
		if err != nil {
			if err == io.EOF {
				break
			}
			return fmt.Errorf("failed to list objects in remote storage %q: %w", blobDir, err)
		}
		if obj.IsDir {
			// TODO: is this branch possible?

			// Object stores list all files with the same prefix,
			// there is no need to recursively list each folder.
			continue
		} else {
			relativePath, err := filepath.Rel(blobDir, obj.Key)
			if err != nil {
				return fmt.Errorf("unexpected object key %q when listing %q: %w", obj.Key, blobDir, err)
			}
			if err := downloadFile(ctx, bucket, obj.Key, filepath.Join(localDir, relativePath)); err != nil {
				return err
			}
		}
	}
	return nil
}

var bucketPattern = regexp.MustCompile(`(^[a-z][a-z0-9]+:///?)([^/?]+)(/[^?]*)?(\?.+)?$`)

func ParseBucketConfig(path string) (*Config, error) {
	ms := bucketPattern.FindStringSubmatch(path)
	if ms == nil || len(ms) != 5 {
		return nil, fmt.Errorf("parse bucket config failed: unrecognized pipeline root format: %q", path)
	}

	// TODO: Verify/add support for file:///.
	if ms[1] != "gs://" && ms[1] != "s3://" && ms[1] != "minio://" {
		return nil, fmt.Errorf("parse bucket config failed: unsupported Cloud bucket: %q", path)
	}

	prefix := strings.TrimPrefix(ms[3], "/")
	if len(prefix) > 0 && !strings.HasSuffix(prefix, "/") {
		prefix = prefix + "/"
	}

	return &Config{
		Scheme:      ms[1],
		BucketName:  ms[2],
		Prefix:      prefix,
		QueryString: ms[4],
	}, nil
}

func ParseBucketConfigForArtifactURI(uri string) (*Config, error) {
	ms := bucketPattern.FindStringSubmatch(uri)
	if ms == nil || len(ms) != 5 {
		return nil, fmt.Errorf("parse bucket config failed: unrecognized uri format: %q", uri)
	}

	// TODO: Verify/add support for file:///.
	if ms[1] != "gs://" && ms[1] != "s3://" && ms[1] != "minio://" {
		return nil, fmt.Errorf("parse bucket config failed: unsupported Cloud bucket: %q", uri)
	}

	return &Config{
		Scheme:     ms[1],
		BucketName: ms[2],
	}, nil
}

// TODO(neuromage): Move these helper functions to a storage package and add tests.
func uploadFile(ctx context.Context, bucket *blob.Bucket, localFilePath, blobFilePath string) error {
	errorF := func(err error) error {
		return fmt.Errorf("uploadFile(): unable to complete copying %q to remote storage %q: %w", localFilePath, blobFilePath, err)
	}

	w, err := bucket.NewWriter(ctx, blobFilePath, nil)
	if err != nil {
		return errorF(fmt.Errorf("unable to open writer for bucket: %w", err))
	}

	r, err := os.Open(localFilePath)
	if err != nil {
		return errorF(fmt.Errorf("unable to open local file %q for reading: %w", localFilePath, err))
	}
	defer r.Close()

	if _, err = io.Copy(w, r); err != nil {
		return errorF(fmt.Errorf("unable to complete copying: %w", err))
	}

	if err = w.Close(); err != nil {
		return errorF(fmt.Errorf("failed to close Writer for bucket: %w", err))
	}

	glog.Infof("uploadFile(localFilePath=%q, blobFilePath=%q)", localFilePath, blobFilePath)
	return nil
}

func downloadFile(ctx context.Context, bucket *blob.Bucket, blobFilePath, localFilePath string) (err error) {
	errorF := func(err error) error {
		return fmt.Errorf("downloadFile(): unable to complete copying %q to local storage %q: %w", blobFilePath, localFilePath, err)
	}

	r, err := bucket.NewReader(ctx, blobFilePath, nil)
	if err != nil {
		return errorF(fmt.Errorf("unable to open reader for bucket: %w", err))
	}
	defer r.Close()

	localDir := filepath.Dir(localFilePath)
	if err := os.MkdirAll(localDir, 0755); err != nil {
		return errorF(fmt.Errorf("failed to create local directory %q: %w", localDir, err))
	}

	w, err := os.Create(localFilePath)
	if err != nil {
		return errorF(fmt.Errorf("unable to open local file %q for writing: %w", localFilePath, err))
	}
	defer func() {
		errClose := w.Close()
		if err == nil && errClose != nil {
			// override named return value "err" when there's a close error
			err = errorF(errClose)
		}
	}()

	if _, err = io.Copy(w, r); err != nil {
		return errorF(fmt.Errorf("unable to complete copying: %w", err))
	}

	return nil
}

// The endpoint uses Kubernetes service DNS name with namespace:
// https://kubernetes.io/docs/concepts/services-networking/service/#dns
const defaultMinioEndpointInMultiUserMode = "minio-service.kubeflow:9000"
const minioArtifactSecretName = "mlpipeline-minio-artifact"

func MinioDefaultEndpoint() string {
	// Discover minio-service in the same namespace by env var.
	// https://kubernetes.io/docs/concepts/services-networking/service/#environment-variables
	minioHost := os.Getenv("MINIO_SERVICE_SERVICE_HOST")
	minioPort := os.Getenv("MINIO_SERVICE_SERVICE_PORT")
	if minioHost != "" && minioPort != "" {
		// If there is a minio-service Kubernetes service in the same namespace,
		// MINIO_SERVICE_SERVICE_HOST and MINIO_SERVICE_SERVICE_PORT env vars should
		// exist by default, so we use it as default.
		return minioHost + ":" + minioPort
	}
	// If the env vars do not exist, we guess that we are running in KFP multi user mode, so default minio service should be `minio-service.kubeflow:9000`.
	glog.Infof("Cannot detect minio-service in the same namespace, default to %s as MinIO endpoint.", defaultMinioEndpointInMultiUserMode)
	return defaultMinioEndpointInMultiUserMode
}

type minioCredential struct {
	AccessKey string
	SecretKey string
}

func getMinioCredential(ctx context.Context, clientSet kubernetes.Interface, namespace string) (cred minioCredential, err error) {
	defer func() {
		if err != nil {
			// wrap error before returning
			err = fmt.Errorf("Failed to get MinIO credential from secret name=%q namespace=%q: %w", minioArtifactSecretName, namespace, err)
		}
	}()
	secret, err := clientSet.CoreV1().Secrets(namespace).Get(
		ctx,
		minioArtifactSecretName,
		metav1.GetOptions{})
	if err != nil {
		return cred, err
	}
	cred.AccessKey = string(secret.Data["accesskey"])
	cred.SecretKey = string(secret.Data["secretkey"])
	if cred.AccessKey == "" {
		return cred, fmt.Errorf("does not have 'accesskey' key")
	}
	if cred.SecretKey == "" {
		return cred, fmt.Errorf("does not have 'secretkey' key")
	}
	return cred, nil
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
	"k8s.io/minikube/pkg/minikube/localpath"
)

func TestDownloadOnly(t *testing.T) {
	for _, r := range []string{"crio", "docker", "containerd"} {
		t.Run(r, func(t *testing.T) {
			// Stores the startup run result for later error messages
			var rrr *RunResult
			var err error

			profile := UniqueProfileName(r)
			ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
			defer Cleanup(t, profile, cancel)

			versions := []string{
				constants.OldestKubernetesVersion,
				constants.DefaultKubernetesVersion,
				constants.NewestKubernetesVersion,
			}

			for _, v := range versions {
				t.Run(v, func(t *testing.T) {
					// Explicitly does not pass StartArgs() to test driver default
					// --force to avoid uid check
					args := append([]string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v), fmt.Sprintf("--container-runtime=%s", r)}, StartArgs()...)

					// Preserve the initial run-result for debugging
					if rrr == nil {
						rrr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
					} else {
						_, err = Run(t, exec.CommandContext(ctx, Target(), args...))
					}

					if err != nil {
						t.Errorf("%s failed: %v", args, err)
					}

					if download.PreloadExists(v, r) {
						// Just make sure the tarball path exists
						if _, err := os.Stat(download.TarballPath(v)); err != nil {
							t.Errorf("preloaded tarball path doesn't exist: %v", err)
						}
						return
					}

					imgs, err := images.Kubeadm("", v)
					if err != nil {
						t.Errorf("kubeadm images: %v %+v", v, err)
					}

					// skip verify for cache images if --driver=none
					if !NoneDriver() {
						for _, img := range imgs {
							img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
							fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
							_, err := os.Stat(fp)
							if err != nil {
								t.Errorf("expected image file exist at %q but got error: %v", fp, err)
							}
						}
					}

					// checking binaries downloaded (kubelet,kubeadm)
					for _, bin := range constants.KubernetesReleaseBinaries {
						fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
						_, err := os.Stat(fp)
						if err != nil {
							t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
						}
					}

					// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
					// as well for the `minikube kubectl` command
					if runtime.GOOS == "linux" {
						return
					}
					binary := "kubectl"
					if runtime.GOOS == "windows" {
						binary = "kubectl.exe"
					}
					fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
					if _, err := os.Stat(fp); err != nil {
						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
					}
				})
			}

			// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
			t.Run("DeleteAll", func(t *testing.T) {
				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
				if err != nil {
					t.Errorf("%s failed: %v", rr.Args, err)
				}
			})
			// Delete should always succeed, even if previously partially or fully deleted.
			t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
				if err != nil {
					t.Errorf("%s failed: %v", rr.Args, err)
				}
			})
		})
	}
}

func TestDownloadOnlyDocker(t *testing.T) {
	if !runningDockerDriver(StartArgs()) {
		t.Skip("this test only runs with the docker driver")
	}

	profile := UniqueProfileName("download-docker")
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer Cleanup(t, profile, cancel)

	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--driver=docker"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("%s failed: %v:\n%s", args, err, rr.Output())
	}

	// Make sure the downloaded image tarball exists
	tarball := download.TarballPath(constants.DefaultKubernetesVersion)
	contents, err := ioutil.ReadFile(tarball)
	if err != nil {
		t.Errorf("reading tarball: %v", err)
	}
	// Make sure it has the correct checksum
	checksum := md5.Sum(contents)
	remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion))
	if err != nil {
		t.Errorf("reading checksum file: %v", err)
	}
	if string(remoteChecksum) != string(checksum[:]) {
		t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:]))
	}
}

func runningDockerDriver(startArgs []string) bool {
	for _, s := range startArgs {
		if s == "--driver=docker" {
			return true
		}
	}
	return false
}
<commit_msg>update integration test<commit_after>// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
	"k8s.io/minikube/pkg/minikube/localpath"
)

func TestDownloadOnly(t *testing.T) {
	for _, r := range []string{"crio", "docker", "containerd"} {
		t.Run(r, func(t *testing.T) {
			// Stores the startup run result for later error messages
			var rrr *RunResult
			var err error

			profile := UniqueProfileName(r)
			ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
			defer Cleanup(t, profile, cancel)

			versions := []string{
				constants.OldestKubernetesVersion,
				constants.DefaultKubernetesVersion,
				constants.NewestKubernetesVersion,
			}

			for _, v := range versions {
				t.Run(v, func(t *testing.T) {
					// Explicitly does not pass StartArgs() to test driver default
					// --force to avoid uid check
					args := append([]string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v), fmt.Sprintf("--container-runtime=%s", r)}, StartArgs()...)

					// Preserve the initial run-result for debugging
					if rrr == nil {
						rrr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
					} else {
						_, err = Run(t, exec.CommandContext(ctx, Target(), args...))
					}

					if err != nil {
						t.Errorf("%s failed: %v", args, err)
					}

					if download.PreloadExists(v, r) {
						// Just make sure the tarball path exists
						if _, err := os.Stat(download.TarballPath(v, r)); err != nil {
							t.Errorf("preloaded tarball path doesn't exist: %v", err)
						}
						return
					}

					imgs, err := images.Kubeadm("", v)
					if err != nil {
						t.Errorf("kubeadm images: %v %+v", v, err)
					}

					// skip verify for cache images if --driver=none
					if !NoneDriver() {
						for _, img := range imgs {
							img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
							fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
							_, err := os.Stat(fp)
							if err != nil {
								t.Errorf("expected image file exist at %q but got error: %v", fp, err)
							}
						}
					}

					// checking binaries downloaded (kubelet,kubeadm)
					for _, bin := range constants.KubernetesReleaseBinaries {
						fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
						_, err := os.Stat(fp)
						if err != nil {
							t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
						}
					}

					// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
					// as well for the `minikube kubectl` command
					if runtime.GOOS == "linux" {
						return
					}
					binary := "kubectl"
					if runtime.GOOS == "windows" {
						binary = "kubectl.exe"
					}
					fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
					if _, err := os.Stat(fp); err != nil {
						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
					}
				})
			}

			// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
			t.Run("DeleteAll", func(t *testing.T) {
				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
				if err != nil {
					t.Errorf("%s failed: %v", rr.Args, err)
				}
			})
			// Delete should always succeed, even if previously partially or fully deleted.
			t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
				if err != nil {
					t.Errorf("%s failed: %v", rr.Args, err)
				}
			})
		})
	}
}

func TestDownloadOnlyDocker(t *testing.T) {
	if !runningDockerDriver(StartArgs()) {
		t.Skip("this test only runs with the docker driver")
	}

	cRuntime := "docker"

	profile := UniqueProfileName("download-docker")
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer Cleanup(t, profile, cancel)

	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--driver=docker"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("%s failed: %v:\n%s", args, err, rr.Output())
	}

	// Make sure the downloaded image tarball exists
	tarball := download.TarballPath(constants.DefaultKubernetesVersion, cRuntime)
	contents, err := ioutil.ReadFile(tarball)
	if err != nil {
		t.Errorf("reading tarball: %v", err)
	}
	// Make sure it has the correct checksum
	checksum := md5.Sum(contents)
	remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime))
	if err != nil {
		t.Errorf("reading checksum file: %v", err)
	}
	if string(remoteChecksum) != string(checksum[:]) {
		t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:]))
	}
}

func runningDockerDriver(startArgs []string) bool {
	for _, s := range startArgs {
		if s == "--driver=docker" {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package dataset

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/xlvector/gogo"
	"github.com/xlvector/hector/core"
	"github.com/xlvector/hector/lr"
)

type Sample struct {
	Board    *gogo.Board
	Info     *gogo.BoardInfo
	K        int
	CurStep  gogo.Point
	NextStep gogo.Point
}

func patternString(label int, pat []int64, h int64) string {
	ret := strconv.Itoa(label)
	for k, v := range pat {
		ret += "\t" + strconv.Itoa(k) + ":" + strconv.FormatInt(v^h, 10)
	}
	return ret
}

func genPatterns(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	ret := []string{}
	for i := len(path) - 1; i >= 0; i-- {
		cur := path[i].Point()
		if !board.Valid(cur) {
			break
		}

		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			label := 0
			if p.X() == cur.X() && p.Y() == cur.Y() {
				label = 1
			} else {
				if rand.Float64() > 0.03 {
					continue
				}
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			ret = append(ret, patternString(label, pat, h))
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	return ret
}

func GenPatternFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genPatterns(gt)
}

func genSamples(gt *gogo.GameTree, stone gogo.Color) []*Sample {
	path := gt.Path2Root()
	ret := []*Sample{}
	board := gogo.NewBoard(gt.SGFSize())
	for i := len(path) - 1; i >= 1; i-- {
		prob := float64(len(path) - i)
		prob = 100.0/100.0 + prob
		if rand.Float64() > prob {
			continue
		}
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		next := path[i-1].Point()
		if !next.Valid() {
			break
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
		sample := &Sample{
			K:        len(path) - i - 1,
			Board:    board.Copy(),
			NextStep: next,
			CurStep:  cur,
		}
		sample.GenComplexFeatures(next.Color())
		ret = append(ret, sample)
	}
	return ret
}

func FeatureString(label int, f map[int64]byte) string {
	ret := fmt.Sprintf("%d", label)
	for k, v := range f {
		ret += fmt.Sprintf("\t%d:%d", k, int(v))
	}
	return ret
}

func genSimpleSamples(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	ret := []string{}
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	last := gogo.InvalidPoint()
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		fs := board.GenSimpleFeatures(last, cur)
		ratio := 15.0 / float64(len(fs))
		for p, f := range fs {
			if p == board.Index(cur) {
				ret = append(ret, FeatureString(1, f))
			} else {
				if rand.Float64() < ratio {
					ret = append(ret, FeatureString(0, f))
				}
			}
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
		last = cur
	}
	return ret
}

func EvaluateLRModel(sgfPath, modelPath string) (int, int) {
	model := &lr.LogisticRegression{
		Model: make(map[int64]float64),
	}
	model.LoadModel(modelPath)

	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evaluateLRModel(gt, model)
}

func loadPatternModel(pat string) []map[int64]float64 {
	f, _ := os.Open(pat)
	reader := bufio.NewReader(f)
	ret := make([]map[int64]float64, gogo.PATTERN_SIZE)
	for i := 0; i < gogo.PATTERN_SIZE; i++ {
		ret[i] = make(map[int64]float64)
	}
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}
		tks := strings.Split(line, "\t")
		k, _ := strconv.Atoi(tks[0])
		p, _ := strconv.ParseInt(tks[1], 10, 64)
		v, _ := strconv.ParseFloat(tks[2], 64)
		ret[k][p] = v
	}
	return ret
}

func EvaluatePattern(sgfPath, modelPath string) (int, int) {
	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evalPattern(gt, modelPath)
}

func evalPattern(gt *gogo.GameTree, pat string) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	patModel := loadPatternModel(pat)
	hit := 0
	total := 0
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		st := make([]gogo.IntFloatPairList, gogo.PATTERN_SIZE)
		for k := 0; k < gogo.PATTERN_SIZE; k++ {
			st[k] = make(gogo.IntFloatPairList, 0, 20)
		}
		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
				hh := pat[j] ^ h
				if v, ok := patModel[j][hh]; ok {
					st[j] = append(st[j], gogo.IntFloatPair{k, v})
					break
				}
			}
		}
		for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
			if len(st[j]) == 0 {
				continue
			}
			sort.Sort(sort.Reverse(st[j]))
			if st[j][0].First == board.Index(cur) {
				hit += 1
			}
			fmt.Println(j, cur.String(), board.W()[st[j][0].First].String(), st[j][0].Second, st[j][0:10])
			break
		}
		total += 1
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func evaluateLRModel(gt *gogo.GameTree, model *lr.LogisticRegression) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	last := gogo.InvalidPoint()
	hit := 0
	total := 0
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		fs := board.GenSimpleFeatures(last, cur)
		best := -1
		maxProb := 0.0
		for p, f := range fs {
			s := core.NewSample()
			for k, v := range f {
				s.AddFeature(core.Feature{k, float64(v)})
			}
			prob := model.Predict(s)
			if maxProb < prob {
				best = p
			}
		}
		if best == board.Index(cur) {
			hit += 1
		}
		total += 1
		fmt.Println(cur.String(), board.W()[best].String())
		board.Put(cur.X(), cur.Y(), cur.Color())
		last = cur
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func GenSimpleSamplesFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSimpleSamples(gt)
}

func GenSamplesFromSGF(buf string, stone gogo.Color) []*Sample {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSamples(gt, stone)
}

func (s *Sample) feature(i, t int) int {
	return t*1000 + i
}

func (s *Sample) GenComplexFeatures(stone gogo.Color) *Sample {
	s.Info = s.Board.CollectBoardInfo(gogo.InvalidPoint())
	s.Info.GenComplexFeatures(stone)
	return s
}

func (s *Sample) FeatureString() []string {
	ret := []string{}
	for p, pf := range s.Info.PointFetures {
		if pf.P.Color() != gogo.GRAY {
			continue
		}
		if p == s.Board.Index(s.NextStep) {
			ret = append(ret, gogo.FeatureString(1, pf.Fc))
		} else {
			if rand.Float64() < 0.05 {
				ret = append(ret, gogo.FeatureString(0, pf.Fc))
			}
		}
	}
	return ret
}
<commit_msg>simple feature<commit_after>package dataset

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/xlvector/gogo"
	"github.com/xlvector/hector/core"
	"github.com/xlvector/hector/lr"
)

type Sample struct {
	Board    *gogo.Board
	Info     *gogo.BoardInfo
	K        int
	CurStep  gogo.Point
	NextStep gogo.Point
}

func patternString(label int, pat []int64, h int64) string {
	ret := strconv.Itoa(label)
	for k, v := range pat {
		ret += "\t" + strconv.Itoa(k) + ":" + strconv.FormatInt(v^h, 10)
	}
	return ret
}

func genPatterns(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	ret := []string{}
	for i := len(path) - 1; i >= 0; i-- {
		cur := path[i].Point()
		if !board.Valid(cur) {
			break
		}

		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			label := 0
			if p.X() == cur.X() && p.Y() == cur.Y() {
				label = 1
			} else {
				if rand.Float64() > 0.03 {
					continue
				}
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			ret = append(ret, patternString(label, pat, h))
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	return ret
}

func GenPatternFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genPatterns(gt)
}

func genSamples(gt *gogo.GameTree, stone gogo.Color) []*Sample {
	path := gt.Path2Root()
	ret := []*Sample{}
	board := gogo.NewBoard(gt.SGFSize())
	for i := len(path) - 1; i >= 1; i-- {
		prob := float64(len(path) - i)
		prob = 100.0/100.0 + prob
		if rand.Float64() > prob {
			continue
		}
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		next := path[i-1].Point()
		if !next.Valid() {
			break
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
		sample := &Sample{
			K:        len(path) - i - 1,
			Board:    board.Copy(),
			NextStep: next,
			CurStep:  cur,
		}
		sample.GenComplexFeatures(next.Color())
		ret = append(ret, sample)
	}
	return ret
}

func FeatureString(label int, f map[int64]byte) string {
	ret := fmt.Sprintf("%d", label)
	for k, v := range f {
		ret += fmt.Sprintf("\t%d:%d", k, int(v))
	}
	return ret
}

func genSimpleSamples(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	ret := []string{}
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	last := gogo.InvalidPoint()
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		fs := board.GenSimpleFeatures(last, cur)
		ratio := 15.0 / float64(len(fs))
		for p, f := range fs {
			if p == board.Index(cur) {
				ret = append(ret, FeatureString(1, f))
			} else {
				if rand.Float64() < ratio {
					ret = append(ret, FeatureString(0, f))
				}
			}
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
		last = cur
	}
	return ret
}

func EvaluateLRModel(sgfPath, modelPath string) (int, int) {
	model := &lr.LogisticRegression{
		Model: make(map[int64]float64),
	}
	model.LoadModel(modelPath)

	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evaluateLRModel(gt, model)
}

func loadPatternModel(pat string) []map[int64]float64 {
	f, _ := os.Open(pat)
	reader := bufio.NewReader(f)
	ret := make([]map[int64]float64, gogo.PATTERN_SIZE)
	for i := 0; i < gogo.PATTERN_SIZE; i++ {
		ret[i] = make(map[int64]float64)
	}
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}
		tks := strings.Split(line, "\t")
		k, _ := strconv.Atoi(tks[0])
		p, _ := strconv.ParseInt(tks[1], 10, 64)
		v, _ := strconv.ParseFloat(tks[2], 64)
		ret[k][p] = v
	}
	return ret
}

func EvaluatePattern(sgfPath, modelPath string) (int, int) {
	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evalPattern(gt, modelPath)
}

func evalPattern(gt *gogo.GameTree, pat string) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	patModel := loadPatternModel(pat)
	hit := 0
	total := 0
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		st := make([]gogo.IntFloatPairList, gogo.PATTERN_SIZE)
		for k := 0; k < gogo.PATTERN_SIZE; k++ {
			st[k] = make(gogo.IntFloatPairList, 0, 20)
		}
		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
				hh := pat[j] ^ h
				if v, ok := patModel[j][hh]; ok {
					st[j] = append(st[j], gogo.IntFloatPair{k, v})
					break
				}
			}
		}
		for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
			if len(st[j]) == 0 {
				continue
			}
			sort.Sort(sort.Reverse(st[j]))
			if st[j][0].First == board.Index(cur) {
				hit += 1
			}
			fmt.Println(j, cur.String(), board.W()[st[j][0].First].String(), st[j][0].Second, st[j][0:10])
			break
		}
		total += 1
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func evaluateLRModel(gt *gogo.GameTree, model *lr.LogisticRegression) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	last := gogo.InvalidPoint()
	hit := 0
	total := 0
	for i := len(path) - 2; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		fs := board.GenSimpleFeatures(last, cur)
		best := -1
		maxProb := 0.0
		for p, f := range fs {
			s := core.NewSample()
			for k, v := range f {
				s.AddFeature(core.Feature{k, float64(v)})
			}
			prob := model.Predict(s)
			if maxProb < prob {
				best = p
			}
		}
		if best == board.Index(cur) {
			hit += 1
		}
		total += 1
		fmt.Println(cur.String(), board.W()[best].String())
		board.Put(cur.X(), cur.Y(), cur.Color())
		last = cur
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func GenSimpleSamplesFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSimpleSamples(gt)
}

func GenSamplesFromSGF(buf string, stone gogo.Color) []*Sample {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSamples(gt, stone)
}

func (s *Sample) feature(i, t int) int {
	return t*1000 + i
}

func (s *Sample) GenComplexFeatures(stone gogo.Color) *Sample {
	s.Info = s.Board.CollectBoardInfo(gogo.InvalidPoint())
	s.Info.GenComplexFeatures(stone)
	return s
}

func (s *Sample) FeatureString() []string {
	ret := []string{}
	for p, pf := range s.Info.PointFetures {
		if pf.P.Color() != gogo.GRAY {
			continue
		}
		if p == s.Board.Index(s.NextStep) {
			ret = append(ret, gogo.FeatureString(1, pf.Fc))
		} else {
			if rand.Float64() < 0.05 {
				ret = append(ret, gogo.FeatureString(0, pf.Fc))
			}
		}
	}
	return ret
}
<|endoftext|>


<commit_before>package dataset

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/xlvector/gogo"
	"github.com/xlvector/hector/core"
	"github.com/xlvector/hector/lr"
)

type Sample struct {
	Board    *gogo.Board
	Info     *gogo.BoardInfo
	K        int
	CurStep  gogo.Point
	NextStep gogo.Point
}

func patternString(label int, pat []int64, h int64) string {
	ret := strconv.Itoa(label)
	for k, v := range pat {
		ret += "\t" + strconv.Itoa(k) + ":" + strconv.FormatInt(v^h, 10)
	}
	return ret
}

func genPatterns(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	ret := []string{}
	for i := len(path) - 1; i >= 0; i-- {
		cur := path[i].Point()
		if !board.Valid(cur) {
			break
		}

		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			label := 0
			if p.X() == cur.X() && p.Y() == cur.Y() {
				label = 1
			} else {
				if rand.Float64() > 0.03 {
					continue
				}
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			ret = append(ret, patternString(label, pat, h))
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	return ret
}

func GenPatternFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genPatterns(gt)
}

func genSamples(gt *gogo.GameTree, stone gogo.Color) []*Sample {
	path := gt.Path2Root()
	ret := []*Sample{}
	board := gogo.NewBoard(gt.SGFSize())
	for i := len(path) - 1; i >= 1; i-- {
		prob := float64(len(path) - i)
		prob = 100.0/100.0 + prob
		if rand.Float64() > prob {
			continue
		}
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		next := path[i-1].Point()
		if !next.Valid() {
			break
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
		sample := &Sample{
			K:        len(path) - i - 1,
			Board:    board.Copy(),
			NextStep: next,
			CurStep:  cur,
		}
		sample.GenComplexFeatures(next.Color())
		ret = append(ret, sample)
	}
	return ret
}

func SimpleFeatureString(label, index int, f []int64) string {
	ret := fmt.Sprintf("%d", label)
	for _, v := range f {
		ret += fmt.Sprintf("\t%d:1", v*1000+int64(index))
	}
	return ret
}

func genSimpleSamples(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	ret := []string{}
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	lastPattern := []int64{}

	for i := len(path) - 2; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		fs := board.GenSimpleFeatures(lastPattern, cur)
		for p, pat := range fs {
			if p == board.Index(cur) {
				ret = append(ret, SimpleFeatureString(1, p, pat))
			} else {
				ret = append(ret, SimpleFeatureString(0, p, pat))
			}
		}
		lastPattern = board.PointSimpleFeature(cur)
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	return ret
}

func EvaluateLRModel(sgfPath, modelPath string) (int, int) {
	model := &lr.LogisticRegression{
		Model: make(map[int64]float64),
	}
	model.LoadModel(modelPath)

	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evaluateLRModel(gt, model)
}

func loadPatternModel(pat string) []map[int64]float64 {
	f, _ := os.Open(pat)
	reader := bufio.NewReader(f)
	ret := make([]map[int64]float64, gogo.PATTERN_SIZE)
	for i := 0; i < gogo.PATTERN_SIZE; i++ {
		ret[i] = make(map[int64]float64)
	}
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}
		tks := strings.Split(line, "\t")
		k, _ := strconv.Atoi(tks[0])
		p, _ := strconv.ParseInt(tks[1], 10, 64)
		v, _ := strconv.ParseFloat(tks[2], 64)
		ret[k][p] = v
	}
	return ret
}

func EvaluatePattern(sgfPath, modelPath string) (int, int) {
	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evalPattern(gt, modelPath)
}

func evalPattern(gt *gogo.GameTree, pat string) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	patModel := loadPatternModel(pat)
	hit := 0
	total := 0
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		st := make([]gogo.IntFloatPairList, gogo.PATTERN_SIZE)
		for k := 0; k < gogo.PATTERN_SIZE; k++ {
			st[k] = make(gogo.IntFloatPairList, 0, 20)
		}
		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
				hh := pat[j] ^ h
				if v, ok := patModel[j][hh]; ok {
					st[j] = append(st[j], gogo.IntFloatPair{k, v})
					break
				}
			}
		}
		for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
			if len(st[j]) == 0 {
				continue
			}
			sort.Sort(sort.Reverse(st[j]))
			if st[j][0].First == board.Index(cur) {
				hit += 1
			}
			fmt.Println(j, cur.String(), board.W()[st[j][0].First].String(), st[j][0].Second, st[j][0:10])
			break
		}
		total += 1
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func evaluateLRModel(gt *gogo.GameTree, model *lr.LogisticRegression) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	lastPattern := []int64{}
	hit := 0
	total := 0
	for i := len(path) - 2; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		rank := make(gogo.IntFloatPairList, 0, 50)
		for j, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			s := core.NewSample()
			pat := board.PointSimpleFeature(p)
			pat = append(pat, lastPattern...)
			for _, v := range pat {
				s.AddFeature(core.Feature{v*1000 + int64(j), 1.0})
			}
			prob := model.Predict(s)
			rank = append(rank, gogo.IntFloatPair{j, prob})
		}
		sort.Sort(sort.Reverse(rank))
		for k := 0; k < 5 && k < len(rank); k++ {
			if rank[k].First == board.Index(cur) {
				hit += 1
				break
			}
		}
		total += 1
		fmt.Println(cur.String(), board.W()[rank[0].First].String(), rank[0].Second)
		lastPattern = board.PointSimpleFeature(cur)
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func GenSimpleSamplesFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSimpleSamples(gt)
}

func GenSamplesFromSGF(buf string, stone gogo.Color) []*Sample {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSamples(gt, stone)
}

func (s *Sample) feature(i, t int) int {
	return t*1000 + i
}

func (s *Sample) GenComplexFeatures(stone gogo.Color) *Sample {
	s.Info = s.Board.CollectBoardInfo(gogo.InvalidPoint())
	s.Info.GenComplexFeatures(stone)
	return s
}

func (s *Sample) FeatureString() []string {
	ret := []string{}
	for p, pf := range s.Info.PointFetures {
		if pf.P.Color() != gogo.GRAY {
			continue
		}
		if p == s.Board.Index(s.NextStep) {
			ret = append(ret, gogo.FeatureString(1, pf.Fc))
		} else {
			if rand.Float64() < 0.05 {
				ret = append(ret, gogo.FeatureString(0, pf.Fc))
			}
		}
	}
	return ret
}
<commit_msg>print<commit_after>package dataset

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/xlvector/gogo"
	"github.com/xlvector/hector/core"
	"github.com/xlvector/hector/lr"
)

type Sample struct {
	Board    *gogo.Board
	Info     *gogo.BoardInfo
	K        int
	CurStep  gogo.Point
	NextStep gogo.Point
}

func patternString(label int, pat []int64, h int64) string {
	ret := strconv.Itoa(label)
	for k, v := range pat {
		ret += "\t" + strconv.Itoa(k) + ":" + strconv.FormatInt(v^h, 10)
	}
	return ret
}

func genPatterns(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	ret := []string{}
	for i := len(path) - 1; i >= 0; i-- {
		cur := path[i].Point()
		if !board.Valid(cur) {
			break
		}

		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			label := 0
			if p.X() == cur.X() && p.Y() == cur.Y() {
				label = 1
			} else {
				if rand.Float64() > 0.03 {
					continue
				}
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			ret = append(ret, patternString(label, pat, h))
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	return ret
}

func GenPatternFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genPatterns(gt)
}

func genSamples(gt *gogo.GameTree, stone gogo.Color) []*Sample {
	path := gt.Path2Root()
	ret := []*Sample{}
	board := gogo.NewBoard(gt.SGFSize())
	for i := len(path) - 1; i >= 1; i-- {
		prob := float64(len(path) - i)
		prob = 100.0/100.0 + prob
		if rand.Float64() > prob {
			continue
		}
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		next := path[i-1].Point()
		if !next.Valid() {
			break
		}
		board.Put(cur.X(), cur.Y(), cur.Color())
		sample := &Sample{
			K:        len(path) - i - 1,
			Board:    board.Copy(),
			NextStep: next,
			CurStep:  cur,
		}
		sample.GenComplexFeatures(next.Color())
		ret = append(ret, sample)
	}
	return ret
}

func SimpleFeatureString(label, index int, f []int64) string {
	ret := fmt.Sprintf("%d", label)
	for _, v := range f {
		ret += fmt.Sprintf("\t%d:1", v*1000+int64(index))
	}
	return ret
}

func genSimpleSamples(gt *gogo.GameTree) []string {
	path := gt.Path2Root()
	ret := []string{}
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	lastPattern := []int64{}

	for i := len(path) - 2; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		fs := board.GenSimpleFeatures(lastPattern, cur)
		for p, pat := range fs {
			if p == board.Index(cur) {
				ret = append(ret, SimpleFeatureString(1, p, pat))
			} else {
				ret = append(ret, SimpleFeatureString(0, p, pat))
			}
		}
		lastPattern = board.PointSimpleFeature(cur)
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	return ret
}

func EvaluateLRModel(sgfPath, modelPath string) (int, int) {
	model := &lr.LogisticRegression{
		Model: make(map[int64]float64),
	}
	model.LoadModel(modelPath)

	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evaluateLRModel(gt, model)
}

func loadPatternModel(pat string) []map[int64]float64 {
	f, _ := os.Open(pat)
	reader := bufio.NewReader(f)
	ret := make([]map[int64]float64, gogo.PATTERN_SIZE)
	for i := 0; i < gogo.PATTERN_SIZE; i++ {
		ret[i] = make(map[int64]float64)
	}
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}
		tks := strings.Split(line, "\t")
		k, _ := strconv.Atoi(tks[0])
		p, _ := strconv.ParseInt(tks[1], 10, 64)
		v, _ := strconv.ParseFloat(tks[2], 64)
		ret[k][p] = v
	}
	return ret
}

func EvaluatePattern(sgfPath, modelPath string) (int, int) {
	gt := &gogo.GameTree{}
	buf, _ := ioutil.ReadFile(sgfPath)
	gt.ParseSGF(string(buf))
	return evalPattern(gt, modelPath)
}

func evalPattern(gt *gogo.GameTree, pat string) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	patModel := loadPatternModel(pat)
	hit := 0
	total := 0
	for i := len(path) - 1; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		st := make([]gogo.IntFloatPairList, gogo.PATTERN_SIZE)
		for k := 0; k < gogo.PATTERN_SIZE; k++ {
			st[k] = make(gogo.IntFloatPairList, 0, 20)
		}
		for k, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			pat := board.GetPatternHash(k)
			h := board.FeatureHash(gogo.MakePoint(p.X(), p.Y(), cur.Color()))
			for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
				hh := pat[j] ^ h
				if v, ok := patModel[j][hh]; ok {
					st[j] = append(st[j], gogo.IntFloatPair{k, v})
					break
				}
			}
		}
		for j := gogo.PATTERN_SIZE - 1; j >= 0; j-- {
			if len(st[j]) == 0 {
				continue
			}
			sort.Sort(sort.Reverse(st[j]))
			if st[j][0].First == board.Index(cur) {
				hit += 1
			}
			fmt.Println(j, cur.String(), board.W()[st[j][0].First].String(), st[j][0].Second, st[j][0:10])
			break
		}
		total += 1
		board.Put(cur.X(), cur.Y(), cur.Color())
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func evaluateLRModel(gt *gogo.GameTree, model *lr.LogisticRegression) (int, int) {
	path := gt.Path2Root()
	board := gogo.NewBoard(gt.SGFSize())
	pdm := gogo.NewPointDistanceMap(board, gogo.PATTERN_SIZE)
	board.SetPointDistanceMap(pdm)
	lastPattern := []int64{}
	hit := 0
	total := 0
	for i := len(path) - 2; i >= 1; i-- {
		cur := path[i].Point()
		if !cur.Valid() {
			break
		}
		rank := make(gogo.IntFloatPairList, 0, 50)
		for j, p := range board.W() {
			if p.Color() != gogo.GRAY {
				continue
			}
			s := core.NewSample()
			pat := board.PointSimpleFeature(p)
			pat = append(pat, lastPattern...)
			for _, v := range pat {
				s.AddFeature(core.Feature{v*1000 + int64(j), 1.0})
			}
			prob := model.Predict(s)
			rank = append(rank, gogo.IntFloatPair{j, prob})
		}
		sort.Sort(sort.Reverse(rank))
		for k := 0; k < 5 && k < len(rank); k++ {
			if rank[k].First == board.Index(cur) {
				hit += 1
				break
			}
		}
		total += 1
		fmt.Println(cur.String(), board.W()[rank[0].First].String(), rank[0].Second)
		lastPattern = board.PointSimpleFeature(cur)
		board.Put(cur.X(), cur.Y(), cur.Color())
		if i%20 == 0 {
			fmt.Println(board.String(cur))
		}
	}
	fmt.Println(hit, total, float64(hit)/float64(total))
	return hit, total
}

func GenSimpleSamplesFromSGF(buf string) []string {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSimpleSamples(gt)
}

func GenSamplesFromSGF(buf string, stone gogo.Color) []*Sample {
	gt := &gogo.GameTree{}
	gt.ParseSGF(buf)
	return genSamples(gt, stone)
}

func (s *Sample) feature(i, t int) int {
	return t*1000 + i
}

func (s *Sample) GenComplexFeatures(stone gogo.Color) *Sample {
	s.Info = s.Board.CollectBoardInfo(gogo.InvalidPoint())
	s.Info.GenComplexFeatures(stone)
	return s
}

func (s *Sample) FeatureString() []string {
	ret := []string{}
	for p, pf := range s.Info.PointFetures {
		if pf.P.Color() != gogo.GRAY {
			continue
		}
		if p == s.Board.Index(s.NextStep) {
			ret = append(ret, gogo.FeatureString(1, pf.Fc))
		} else {
			if rand.Float64() < 0.05 {
				ret = append(ret, gogo.FeatureString(0, pf.Fc))
			}
		}
	}
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

func main() {
	color.Yellow("Updating anime thumbnails")

	defer arn.Node.Close()
	defer color.Green("Finished.")

	// Parse flags
	var animeID string
	flag.StringVar(&animeID, "id", "", "ID of the anime that you want to refresh")
	flag.Parse()

	// Refresh 1 anime in case ID was specified
	if animeID != "" {
		anime, _ := arn.GetAnime(animeID)

		if anime != nil {
			sync(anime)
		}

		return
	}

	// Otherwise refresh all anime
	for anime := range arn.StreamAnime() {
		sync(anime)
	}
}

// sync refreshes the image of the given anime.
func sync(anime *arn.Anime) {
	base := path.Join(arn.Root, "/images/anime/original/", anime.ID)

	if _, err := os.Stat(base + ".png"); err == nil {
		update(anime, base+".png")
		return
	}

	if _, err := os.Stat(base + ".jpg"); err == nil {
		update(anime, base+".jpg")
		return
	}

	if _, err := os.Stat(base + ".jpeg"); err == nil {
		update(anime, base+".jpg")
		return
	}

	if _, err := os.Stat(base + ".gif"); err == nil {
		update(anime, base+".gif")
		return
	}

	if _, err := os.Stat(base + ".webp"); err == nil {
		update(anime, base+".webp")
		return
	}
}

// update expects a file to load as image for the anime and updates it.
func update(anime *arn.Anime, filePath string) {
	fmt.Println(anime.ID, anime)

	data, err := ioutil.ReadFile(filePath)

	if err != nil {
		return
	}

	anime.SetImageBytes(data)
	anime.Save()
}
<commit_msg>Improved thumbnail refresh<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

func main() {
	color.Yellow("Updating anime thumbnails")

	defer arn.Node.Close()
	defer color.Green("Finished.")

	// Parse flags
	var animeID string
	flag.StringVar(&animeID, "id", "", "ID of the anime that you want to refresh")
	flag.Parse()

	// Refresh 1 anime in case ID was specified
	if animeID != "" {
		anime, _ := arn.GetAnime(animeID)

		if anime != nil {
			sync(anime)
		}

		return
	}

	// Otherwise refresh all anime
	for anime := range arn.StreamAnime() {
		sync(anime)
	}
}

// sync refreshes the image of the given anime.
func sync(anime *arn.Anime) {
	base := path.Join(arn.Root, "/images/anime/original/", anime.ID)

	if _, err := os.Stat(base + ".png"); err == nil {
		update(anime, base+".png")
		return
	}

	if _, err := os.Stat(base + ".jpg"); err == nil {
		update(anime, base+".jpg")
		return
	}

	if _, err := os.Stat(base + ".jpeg"); err == nil {
		update(anime, base+".jpg")
		return
	}

	if _, err := os.Stat(base + ".gif"); err == nil {
		update(anime, base+".gif")
		return
	}

	if _, err := os.Stat(base + ".webp"); err == nil {
		update(anime, base+".webp")
		return
	}
}

// update expects a file to load as image for the anime and updates it.
func update(anime *arn.Anime, filePath string) {
	fmt.Println(anime.ID, anime)

	data, err := ioutil.ReadFile(filePath)

	if err != nil {
		color.Red(err.Error())
		return
	}

	err = anime.SetImageBytes(data)

	if err != nil {
		color.Red(err.Error())
		return
	}

	anime.Save()
}
<|endoftext|>


<commit_before>package template

var filesTemplate = `package {{.Pkg}}
{{$Compression := .Compression}}

import (
  "bytes"
  {{if not .Spread}}{{if $Compression.Compress}}{{if not $Compression.Keep}}"compress/gzip"{{end}}{{end}}{{end}}
  "io"
  "log"
  "net/http"
  "os"
  "time"

  "golang.org/x/net/webdav"
  "golang.org/x/net/context"
)

var (
  // CTX is a context for webdav vfs
  {{exported "CTX"}} = context.Background()

  // FS is a virtual memory file system
  {{exported "FS"}} = webdav.NewMemFS()

  // Handler is used to server files through a http handler
  {{exportedTitle "Handler"}} *webdav.Handler

  // HTTP is the http file system
  {{exportedTitle "HTTP"}} http.FileSystem = new({{exported "HTTPFS"}})
)

// HTTPFS implements http.FileSystem
type {{exported "HTTPFS"}} struct {}

{{if (and (not .Spread) (not .Debug))}}
{{range .Files}}
// {{exportedTitle "File"}}{{buildSafeVarName .Path}} is a file
var {{exportedTitle "File"}}{{buildSafeVarName .Path}} = {{.Data}}
{{end}}
{{end}}

func init() {
  var err error

  if {{exported "CTX"}}.Err() != nil {
		log.Fatal({{exported "CTX"}}.Err())
	}

{{range $index, $dir := .DirList}}
  {{if ne $dir "./"}}
  err = {{exported "FS"}}.Mkdir({{exported "CTX"}}, "{{$dir}}", 0777)
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
{{end}}

{{if (and (not .Spread) (not .Debug))}}
  var f webdav.File
  {{if $Compression.Compress}}
  {{if not $Compression.Keep}}
  var rb *bytes.Reader
  var r *gzip.Reader
  {{end}}
  {{end}}

  {{range .Files}}
  {{if $Compression.Compress}}
  {{if not $Compression.Keep}}
  rb = bytes.NewReader({{exportedTitle "File"}}{{buildSafeVarName .Path}})
  r, err = gzip.NewReader(rb)
  if err != nil {
    log.Fatal(err)
  }

  err = r.Close()
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
  {{end}}

  f, err = {{exported "FS"}}.OpenFile({{exported "CTX"}}, "{{.Path}}", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777)
  if err != nil {
    log.Fatal(err)
  }

  {{if $Compression.Compress}}
  {{if not $Compression.Keep}}
  _, err = io.Copy(f, r)
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
  {{else}}
  _, err = f.Write({{exportedTitle "File"}}{{buildSafeVarName .Path}})
  if err != nil {
    log.Fatal(err)
  }
  {{end}}

  err = f.Close()
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
{{end}}

  {{exportedTitle "Handler"}} = &webdav.Handler{
    FileSystem: FS,
    LockSystem: webdav.NewMemLS(),
  }
}

// Open a file
func (hfs *{{exported "HTTPFS"}}) Open(path string) (http.File, error) {
  f, err := {{if .Debug}}os{{else}}{{exported "FS"}}{{end}}.OpenFile({{if not .Debug}}{{exported "CTX"}}, {{end}}path, os.O_RDONLY, 0644)
  if err != nil {
    return nil, err
  }

  return f, nil
}

// ReadFile is adapTed from ioutil
func {{exportedTitle "ReadFile"}}(path string) ([]byte, error) {
  f, err := {{if .Debug}}os{{else}}{{exported "FS"}}{{end}}.OpenFile({{if not .Debug}}{{exported "CTX"}}, {{end}}path, os.O_RDONLY, 0644)
  if err != nil {
    return nil, err
  }

  buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead))

  // If the buffer overflows, we will get bytes.ErrTooLarge.
  // Return that as an error. Any other panic remains.
  defer func() {
    e := recover()
    if e == nil {
      return
    }
    if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge {
      err = panicErr
    } else {
      panic(e)
    }
  }()
  _, err = buf.ReadFrom(f)
  return buf.Bytes(), err
}

// WriteFile is adapTed from ioutil
func {{exportedTitle "WriteFile"}}(filename string, data []byte, perm os.FileMode) error {
  f, err := {{exported "FS"}}.OpenFile({{exported "CTX"}}, filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
  if err != nil {
    return err
  }
  n, err := f.Write(data)
  if err == nil && n < len(data) {
    err = io.ErrShortWrite
  }
  if err1 := f.Close(); err == nil {
    err = err1
  }
  return err
}

// FileNames is a list of files included in this filebox
var {{exportedTitle "FileNames"}} = []string {
  {{range .Files}}"{{.Path}}",
  {{end}}
}

`
<commit_msg>fix: remove time import<commit_after>package template

var filesTemplate = `package {{.Pkg}}
{{$Compression := .Compression}}

import (
  "bytes"
  {{if not .Spread}}{{if $Compression.Compress}}{{if not $Compression.Keep}}"compress/gzip"{{end}}{{end}}{{end}}
  "io"
  "log"
  "net/http"
  "os"

  "golang.org/x/net/webdav"
  "golang.org/x/net/context"
)

var (
  // CTX is a context for webdav vfs
  {{exported "CTX"}} = context.Background()

  // FS is a virtual memory file system
  {{exported "FS"}} = webdav.NewMemFS()

  // Handler is used to server files through a http handler
  {{exportedTitle "Handler"}} *webdav.Handler

  // HTTP is the http file system
  {{exportedTitle "HTTP"}} http.FileSystem = new({{exported "HTTPFS"}})
)

// HTTPFS implements http.FileSystem
type {{exported "HTTPFS"}} struct {}

{{if (and (not .Spread) (not .Debug))}}
{{range .Files}}
// {{exportedTitle "File"}}{{buildSafeVarName .Path}} is a file
var {{exportedTitle "File"}}{{buildSafeVarName .Path}} = {{.Data}}
{{end}}
{{end}}

func init() {
  var err error

  if {{exported "CTX"}}.Err() != nil {
		log.Fatal({{exported "CTX"}}.Err())
	}

{{range $index, $dir := .DirList}}
  {{if ne $dir "./"}}
  err = {{exported "FS"}}.Mkdir({{exported "CTX"}}, "{{$dir}}", 0777)
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
{{end}}

{{if (and (not .Spread) (not .Debug))}}
  var f webdav.File
  {{if $Compression.Compress}}
  {{if not $Compression.Keep}}
  var rb *bytes.Reader
  var r *gzip.Reader
  {{end}}
  {{end}}

  {{range .Files}}
  {{if $Compression.Compress}}
  {{if not $Compression.Keep}}
  rb = bytes.NewReader({{exportedTitle "File"}}{{buildSafeVarName .Path}})
  r, err = gzip.NewReader(rb)
  if err != nil {
    log.Fatal(err)
  }

  err = r.Close()
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
  {{end}}

  f, err = {{exported "FS"}}.OpenFile({{exported "CTX"}}, "{{.Path}}", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777)
  if err != nil {
    log.Fatal(err)
  }

  {{if $Compression.Compress}}
  {{if not $Compression.Keep}}
  _, err = io.Copy(f, r)
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
  {{else}}
  _, err = f.Write({{exportedTitle "File"}}{{buildSafeVarName .Path}})
  if err != nil {
    log.Fatal(err)
  }
  {{end}}

  err = f.Close()
  if err != nil {
    log.Fatal(err)
  }
  {{end}}
{{end}}

  {{exportedTitle "Handler"}} = &webdav.Handler{
    FileSystem: FS,
    LockSystem: webdav.NewMemLS(),
  }
}

// Open a file
func (hfs *{{exported "HTTPFS"}}) Open(path string) (http.File, error) {
  f, err := {{if .Debug}}os{{else}}{{exported "FS"}}{{end}}.OpenFile({{if not .Debug}}{{exported "CTX"}}, {{end}}path, os.O_RDONLY, 0644)
  if err != nil {
    return nil, err
  }

  return f, nil
}

// ReadFile is adapTed from ioutil
func {{exportedTitle "ReadFile"}}(path string) ([]byte, error) {
  f, err := {{if .Debug}}os{{else}}{{exported "FS"}}{{end}}.OpenFile({{if not .Debug}}{{exported "CTX"}}, {{end}}path, os.O_RDONLY, 0644)
  if err != nil {
    return nil, err
  }

  buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead))

  // If the buffer overflows, we will get bytes.ErrTooLarge.
  // Return that as an error. Any other panic remains.
  defer func() {
    e := recover()
    if e == nil {
      return
    }
    if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge {
      err = panicErr
    } else {
      panic(e)
    }
  }()
  _, err = buf.ReadFrom(f)
  return buf.Bytes(), err
}

// WriteFile is adapTed from ioutil
func {{exportedTitle "WriteFile"}}(filename string, data []byte, perm os.FileMode) error {
  f, err := {{exported "FS"}}.OpenFile({{exported "CTX"}}, filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
  if err != nil {
    return err
  }
  n, err := f.Write(data)
  if err == nil && n < len(data) {
    err = io.ErrShortWrite
  }
  if err1 := f.Close(); err == nil {
    err = err1
  }
  return err
}

// FileNames is a list of files included in this filebox
var {{exportedTitle "FileNames"}} = []string {
  {{range .Files}}"{{.Path}}",
  {{end}}
}

`
<|endoftext|>