<commit_before>package topology

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func ReplicatedWrite(masterNode string, s *storage.Store,
	volumeId storage.VolumeId, needle *storage.Needle,
	r *http.Request) (size uint32, errorStatus string) {

	//check JWT
	jwt := security.GetJwt(r)

	ret, err := s.Write(volumeId, needle)
	needToReplicate := !s.HasVolume(volumeId)
	if err != nil {
		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
	} else if ret > 0 {
		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
	} else {
		errorStatus = "Failed to write to local disk"
	}
	if !needToReplicate && ret > 0 {
		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {

			if err = distributedOperation(masterNode, s, volumeId, func(location operation.Location) error {
				u := url.URL{
					Scheme: "http",
					Host:   location.Url,
					Path:   r.URL.Path,
				}
				q := url.Values{
					"type": {"replicate"},
				}
				if needle.LastModified > 0 {
					q.Set("ts", strconv.FormatUint(needle.LastModified, 10))
				}
				if needle.IsChunkedManifest() {
					q.Set("cm", "true")
				}
				u.RawQuery = q.Encode()

				pairMap := make(map[string]string)
				if needle.HasPairs() {
					err := json.Unmarshal(needle.Pairs, &pairMap)
					if err != nil {
						glog.V(0).Infoln("Unmarshal pairs error:", err)
					}
				}

				_, err := operation.Upload(u.String(),
					string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
					pairMap, jwt)
				return err
			}); err != nil {
				ret = 0
				errorStatus = fmt.Sprintf("Failed to write to replicas for volume %d: %v", volumeId, err)
			}
		}
	}
	size = ret
	return
}

func ReplicatedDelete(masterNode string, store *storage.Store,
	volumeId storage.VolumeId, n *storage.Needle,
	r *http.Request) (uint32, error) {

	//check JWT
	jwt := security.GetJwt(r)

	ret, err := store.Delete(volumeId, n)
	if err != nil {
		glog.V(0).Infoln("delete error:", err)
		return ret, err
	}

	needToReplicate := !store.HasVolume(volumeId)
	if !needToReplicate && ret > 0 {
		needToReplicate = store.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {
			if err = distributedOperation(masterNode, store, volumeId, func(location operation.Location) error {
				return util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", jwt)
			}); err != nil {
				ret = 0
			}
		}
	}
	return ret, err
}

type DistributedOperationResult map[string]error

func (dr DistributedOperationResult) Error() error {
	var errs []string
	for k, v := range dr {
		if v != nil {
			errs = append(errs, fmt.Sprintf("[%s]: %v", k, v))
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return errors.New(strings.Join(errs, "\n"))
}

type RemoteResult struct {
	Host  string
	Error error
}

func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) error) error {
	if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil {
		length := 0
		selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port))
		results := make(chan RemoteResult)
		for _, location := range lookupResult.Locations {
			if location.Url != selfUrl {
				length++
				go func(location operation.Location, results chan RemoteResult) {
					results <- RemoteResult{location.Url, op(location)}
				}(location, results)
			}
		}
		ret := DistributedOperationResult(make(map[string]error))
		for i := 0; i < length; i++ {
			result := <-results
			ret[result.Host] = result.Error
		}
		if volume := store.GetVolume(volumeId); volume != nil {
			if length+1 < volume.ReplicaPlacement.GetCopyCount() {
				return fmt.Errorf("replicating opetations [%d] is less than volume's replication copy count [%d]", length+1, volume.ReplicaPlacement.GetCopyCount())
			}
		}
		return ret.Error()
	} else {
		glog.V(0).Infoln()
		return fmt.Errorf("Failed to lookup for %d: %v", volumeId, lookupErr)
	}
	return nil
}
<commit_msg>fix bug: replication, metadata not replicated<commit_after>package topology

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func ReplicatedWrite(masterNode string, s *storage.Store,
	volumeId storage.VolumeId, needle *storage.Needle,
	r *http.Request) (size uint32, errorStatus string) {

	//check JWT
	jwt := security.GetJwt(r)

	ret, err := s.Write(volumeId, needle)
	needToReplicate := !s.HasVolume(volumeId)
	if err != nil {
		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
	} else if ret > 0 {
		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
	} else {
		errorStatus = "Failed to write to local disk"
	}
	if !needToReplicate && ret > 0 {
		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {

			if err = distributedOperation(masterNode, s, volumeId, func(location operation.Location) error {
				u := url.URL{
					Scheme: "http",
					Host:   location.Url,
					Path:   r.URL.Path,
				}
				q := url.Values{
					"type": {"replicate"},
				}
				if needle.LastModified > 0 {
					q.Set("ts", strconv.FormatUint(needle.LastModified, 10))
				}
				if needle.IsChunkedManifest() {
					q.Set("cm", "true")
				}
				u.RawQuery = q.Encode()

				pairMap := make(map[string]string)
				if needle.HasPairs() {
					tmpMap := make(map[string]string)
					err := json.Unmarshal(needle.Pairs, &tmpMap)
					if err != nil {
						glog.V(0).Infoln("Unmarshal pairs error:", err)
					}
					for k, v := range tmpMap {
						pairMap[storage.PairNamePrefix+k] = v
					}
				}

				_, err := operation.Upload(u.String(),
					string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
					pairMap, jwt)
				return err
			}); err != nil {
				ret = 0
				errorStatus = fmt.Sprintf("Failed to write to replicas for volume %d: %v", volumeId, err)
			}
		}
	}
	size = ret
	return
}

func ReplicatedDelete(masterNode string, store *storage.Store,
	volumeId storage.VolumeId, n *storage.Needle,
	r *http.Request) (uint32, error) {

	//check JWT
	jwt := security.GetJwt(r)

	ret, err := store.Delete(volumeId, n)
	if err != nil {
		glog.V(0).Infoln("delete error:", err)
		return ret, err
	}

	needToReplicate := !store.HasVolume(volumeId)
	if !needToReplicate && ret > 0 {
		needToReplicate = store.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {
			if err = distributedOperation(masterNode, store, volumeId, func(location operation.Location) error {
				return util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", jwt)
			}); err != nil {
				ret = 0
			}
		}
	}
	return ret, err
}

type DistributedOperationResult map[string]error

func (dr DistributedOperationResult) Error() error {
	var errs []string
	for k, v := range dr {
		if v != nil {
			errs = append(errs, fmt.Sprintf("[%s]: %v", k, v))
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return errors.New(strings.Join(errs, "\n"))
}

type RemoteResult struct {
	Host  string
	Error error
}

func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) error) error {
	if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil {
		length := 0
		selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port))
		results := make(chan RemoteResult)
		for _, location := range lookupResult.Locations {
			if location.Url != selfUrl {
				length++
				go func(location operation.Location, results chan RemoteResult) {
					results <- RemoteResult{location.Url, op(location)}
				}(location, results)
			}
		}
		ret := DistributedOperationResult(make(map[string]error))
		for i := 0; i < length; i++ {
			result := <-results
			ret[result.Host] = result.Error
		}
		if volume := store.GetVolume(volumeId); volume != nil {
			if length+1 < volume.ReplicaPlacement.GetCopyCount() {
				return fmt.Errorf("replicating opetations [%d] is less than volume's replication copy count [%d]", length+1, volume.ReplicaPlacement.GetCopyCount())
			}
		}
		return ret.Error()
	} else {
		glog.V(0).Infoln()
		return fmt.Errorf("Failed to lookup for %d: %v", volumeId, lookupErr)
	}
	return nil
}
<|endoftext|>


<commit_before>package agents

import (
	"encoding/json"
	"fmt"
	"reflect"
	"time"

	log "github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/shirou/gopsutil/cpu"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/shirou/gopsutil/host"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/shirou/gopsutil/process"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/streadway/amqp"
)

// ProcessStats provides an interface for collecting statistics about the pid
type ProcessStats interface {
	Sample() error // Take a statistical sample and emit it on AMQP
}

// ProcessStatCollector is a container for internal state
type ProcessStatCollector struct {
	connection *amqp.Connection
	channel    *amqp.Channel
	routingKey string
	exchange   string
	job        *JobControl
	ticker     <-chan time.Time
	hostInfo   *host.HostInfoStat
}

// ProcessStatSample contains a single sample of the underlying process system usage.
// See gopsutil for defintions of these structures and support for particular OSes.
type ProcessStatSample struct {
	Host       host.HostInfoStat       // See gopsutil for description
	Memory     process.MemoryInfoStat  // See gopsutil for description
	CPUTimes   cpu.CPUTimesStat        // See gopsutil for description
	IOCounters process.IOCountersStat  // See gopsutil for description
	OpenFiles  []process.OpenFilesStat // See gopsutil for description
	NumThreads int32                   // Number of threads in use by the child processes (if supported)
	Pid        int32                   // Process ID for the group
	ChildPids  []int32                 // Children process IDs
	CPUPercent float64                 // Sum of parent and children CPU percent usage
}

// NewProcessStats establishes a new AMQP channel and configures sampling period
func NewProcessStats(amqp *amqp.Connection, routingKey string,
	exchange string, job *JobControl, interval time.Duration) (ProcessStats, error) {

	var err error
	psc := &ProcessStatCollector{
		amqp,
		nil,
		routingKey,
		exchange,
		job,
		nil,
		nil,
	}

	psc.channel, err = psc.connection.Channel()
	if err != nil {
		return nil, fmt.Errorf("Unable to open a channel on AMQP connection: %s\n", err)
	}

	if err = psc.channel.ExchangeDeclare(
		exchange, // name
		"topic",  // type
		true,     // durable
		false,    // auto-deleted
		false,    // internal
		false,    // nowait
		nil,      // arguments
	); err != nil {
		return nil, fmt.Errorf("Unable to declare the exchange: %s", err)
	}

	psc.ticker = time.NewTicker(interval).C

	go func(psc *ProcessStatCollector) {
		for {
			select {
			case _ = <-psc.ticker:
				psc.Sample()
			}
		}
	}(psc)

	// Collect basic host info for conditional sampling of some data
	// as a workaround for certain platforms.
	psc.hostInfo, err = host.HostInfo()

	return psc, nil
}

// Sample collects process statistcs and emits them.
func (ps *ProcessStatCollector) Sample() error {
	var err error
	job := *ps.job
	proc := job.Process()

	stat := ProcessStatSample{}

	stat.Pid = proc.Pid

	hostinfo, err := host.HostInfo()
	var hostnameRoutingKey string
	if err != nil {
		log.Warnf("Error encountered collecting host stats: %s", err)
		hostnameRoutingKey = ""
	} else {
		stat.Host = *hostinfo
		hostnameRoutingKey = fmt.Sprintf(".%s", stat.Host.Hostname)
	}

	// Collect for parent
	stat.aggregateStatForProc(proc)

	children, err := proc.Children()
	// Collect on children (if any)
	if err == nil {
		for _, cproc := range children {
			stat.ChildPids = append(stat.ChildPids, cproc.Pid)
			stat.aggregateStatForProc(cproc)
		}
	}

	log.Debugf("Sample: %#v\n", stat)

	var body []byte
	body, err = json.Marshal(stat)

	routingKey := fmt.Sprintf("%s%s", ps.routingKey, hostnameRoutingKey)
	err = ps.channel.Publish(
		ps.exchange, // publish to an exchange
		routingKey,  // routing to queues
		false,       // mandatory
		false,       // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/javascript",
			ContentEncoding: "",
			Body:            body,
			DeliveryMode:    amqp.Transient, // non-persistent
			Priority:        0,              // 0-9
		},
	)
	if err != nil {
		log.Warnf("Error publishing statistics sample %s", err)
	}

	return err
}

func (p *ProcessStatSample) aggregateStatForProc(proc *process.Process) {
	// Split individual stat calls into functions for panic recovery
	// without losing the entire statistic.

	p.collectMemInfo(proc)

	p.collectCPUTimes(proc)

	p.collectIOCounters(proc)

	p.collectOpenFiles(proc)

	// NOTE: This will end up counting separate pids as "threads"
	p.collectNumThreads(proc)

	p.collectCPUPercent(proc)
}

func sum(src *reflect.Value, dest *reflect.Value) {
	for i := 0; i < src.NumField(); i++ {
		switch src.Field(i).Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			var total uint64
			total += src.Field(i).Uint()
			total += dest.Field(i).Uint()
			dest.Field(i).SetUint(total)
		case reflect.Float32, reflect.Float64:
			var total float64
			total += src.Field(i).Float()
			total += dest.Field(i).Float()
			dest.Field(i).SetFloat(total)
		}
	}
}

func (p *ProcessStatSample) collectMemInfo(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on memory stats collection. Maybe unsupported on this platform.")
		}
	}()
	meminfo, err := proc.MemoryInfo()
	if err != nil {
		log.Warnf("Error encountered collecting memory stats: %s", err)
	} else {
		src := reflect.ValueOf(meminfo).Elem()
		dest := reflect.ValueOf(&p.Memory).Elem()
		sum(&src, &dest)
	}
}

func (p *ProcessStatSample) collectCPUTimes(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on CPU times collection. Maybe unsupported on this platform.")
		}
	}()
	cputimes, err := proc.CPUTimes()
	if err != nil {
		log.Warnf("Error encountered collecting CPU stats: %s", err)
	} else {
		src := reflect.ValueOf(cputimes).Elem()
		dest := reflect.ValueOf(&p.CPUTimes).Elem()
		sum(&src, &dest)
	}
}

func (p *ProcessStatSample) collectIOCounters(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on IO counters collection. Maybe unsupported on this platform.")
		}
	}()
	iocnt, err := proc.IOCounters()
	if err != nil {
		log.Warnf("Error encountered collecting I/O stats: %s", err)
	} else {
		src := reflect.ValueOf(iocnt).Elem()
		dest := reflect.ValueOf(&p.IOCounters).Elem()
		sum(&src, &dest)
	}
}

func (p *ProcessStatSample) collectOpenFiles(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on Open Files collection. Maybe unsupported on this platform.")
		}
	}()
	openFiles, err := proc.OpenFiles()
	if err != nil {
		log.Warnf("Error encountered collecting open files stats: %s", err)
	} else {
		p.OpenFiles = append(p.OpenFiles, openFiles...)
	}
}

func (p *ProcessStatSample) collectNumThreads(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on Number of Threads collection. Maybe unsupported on this platform.")
		}
	}()
	numThreads, err := proc.NumThreads()
	if err != nil {
		log.Warnf("Error encountered collecting thread count stats: %s", err)
	} else {
		p.NumThreads += numThreads
	}
}

func (p *ProcessStatSample) collectCPUPercent(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on CPU Percent collection. Maybe unsupported on this platform.")
		}
	}()
	// Use 0 interval to get difference since the last call
	cpuPercent, err := proc.CPUPercent(0 * time.Second)
	if err != nil {
		log.Warnf("Error encountered collecting CPU percent: %s", err)
	} else {
		p.CPUPercent += cpuPercent
	}
}
<commit_msg>Remove partial work on workarounds logic.<commit_after>package agents

import (
	"encoding/json"
	"fmt"
	"reflect"
	"time"

	log "github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/shirou/gopsutil/cpu"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/shirou/gopsutil/host"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/shirou/gopsutil/process"
	"github.com/nickjones/proc_box/Godeps/_workspace/src/github.com/streadway/amqp"
)

// ProcessStats provides an interface for collecting statistics about the pid
type ProcessStats interface {
	Sample() error // Take a statistical sample and emit it on AMQP
}

// ProcessStatCollector is a container for internal state
type ProcessStatCollector struct {
	connection *amqp.Connection
	channel    *amqp.Channel
	routingKey string
	exchange   string
	job        *JobControl
	ticker     <-chan time.Time
}

// ProcessStatSample contains a single sample of the underlying process system usage.
// See gopsutil for defintions of these structures and support for particular OSes.
type ProcessStatSample struct {
	Host       host.HostInfoStat       // See gopsutil for description
	Memory     process.MemoryInfoStat  // See gopsutil for description
	CPUTimes   cpu.CPUTimesStat        // See gopsutil for description
	IOCounters process.IOCountersStat  // See gopsutil for description
	OpenFiles  []process.OpenFilesStat // See gopsutil for description
	NumThreads int32                   // Number of threads in use by the child processes (if supported)
	Pid        int32                   // Process ID for the group
	ChildPids  []int32                 // Children process IDs
	CPUPercent float64                 // Sum of parent and children CPU percent usage
}

// NewProcessStats establishes a new AMQP channel and configures sampling period
func NewProcessStats(amqp *amqp.Connection, routingKey string,
	exchange string, job *JobControl, interval time.Duration) (ProcessStats, error) {

	var err error
	psc := &ProcessStatCollector{
		amqp,
		nil,
		routingKey,
		exchange,
		job,
		nil,
	}

	psc.channel, err = psc.connection.Channel()
	if err != nil {
		return nil, fmt.Errorf("Unable to open a channel on AMQP connection: %s\n", err)
	}

	if err = psc.channel.ExchangeDeclare(
		exchange, // name
		"topic",  // type
		true,     // durable
		false,    // auto-deleted
		false,    // internal
		false,    // nowait
		nil,      // arguments
	); err != nil {
		return nil, fmt.Errorf("Unable to declare the exchange: %s", err)
	}

	psc.ticker = time.NewTicker(interval).C

	go func(psc *ProcessStatCollector) {
		for {
			select {
			case _ = <-psc.ticker:
				psc.Sample()
			}
		}
	}(psc)

	return psc, nil
}

// Sample collects process statistcs and emits them.
func (ps *ProcessStatCollector) Sample() error {
	var err error
	job := *ps.job
	proc := job.Process()

	stat := ProcessStatSample{}

	stat.Pid = proc.Pid

	hostinfo, err := host.HostInfo()
	var hostnameRoutingKey string
	if err != nil {
		log.Warnf("Error encountered collecting host stats: %s", err)
		hostnameRoutingKey = ""
	} else {
		stat.Host = *hostinfo
		hostnameRoutingKey = fmt.Sprintf(".%s", stat.Host.Hostname)
	}

	// Collect for parent
	stat.aggregateStatForProc(proc)

	children, err := proc.Children()
	// Collect on children (if any)
	if err == nil {
		for _, cproc := range children {
			stat.ChildPids = append(stat.ChildPids, cproc.Pid)
			stat.aggregateStatForProc(cproc)
		}
	}

	log.Debugf("Sample: %#v\n", stat)

	var body []byte
	body, err = json.Marshal(stat)

	routingKey := fmt.Sprintf("%s%s", ps.routingKey, hostnameRoutingKey)
	err = ps.channel.Publish(
		ps.exchange, // publish to an exchange
		routingKey,  // routing to queues
		false,       // mandatory
		false,       // immediate
		amqp.Publishing{
			Headers:         amqp.Table{},
			ContentType:     "text/javascript",
			ContentEncoding: "",
			Body:            body,
			DeliveryMode:    amqp.Transient, // non-persistent
			Priority:        0,              // 0-9
		},
	)
	if err != nil {
		log.Warnf("Error publishing statistics sample %s", err)
	}

	return err
}

func (p *ProcessStatSample) aggregateStatForProc(proc *process.Process) {
	// Split individual stat calls into functions for panic recovery
	// without losing the entire statistic.

	p.collectMemInfo(proc)

	p.collectCPUTimes(proc)

	p.collectIOCounters(proc)

	p.collectOpenFiles(proc)

	// NOTE: This will end up counting separate pids as "threads"
	p.collectNumThreads(proc)

	p.collectCPUPercent(proc)
}

func sum(src *reflect.Value, dest *reflect.Value) {
	for i := 0; i < src.NumField(); i++ {
		switch src.Field(i).Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			var total uint64
			total += src.Field(i).Uint()
			total += dest.Field(i).Uint()
			dest.Field(i).SetUint(total)
		case reflect.Float32, reflect.Float64:
			var total float64
			total += src.Field(i).Float()
			total += dest.Field(i).Float()
			dest.Field(i).SetFloat(total)
		}
	}
}

func (p *ProcessStatSample) collectMemInfo(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on memory stats collection. Maybe unsupported on this platform.")
		}
	}()
	meminfo, err := proc.MemoryInfo()
	if err != nil {
		log.Warnf("Error encountered collecting memory stats: %s", err)
	} else {
		src := reflect.ValueOf(meminfo).Elem()
		dest := reflect.ValueOf(&p.Memory).Elem()
		sum(&src, &dest)
	}
}

func (p *ProcessStatSample) collectCPUTimes(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on CPU times collection. Maybe unsupported on this platform.")
		}
	}()
	cputimes, err := proc.CPUTimes()
	if err != nil {
		log.Warnf("Error encountered collecting CPU stats: %s", err)
	} else {
		src := reflect.ValueOf(cputimes).Elem()
		dest := reflect.ValueOf(&p.CPUTimes).Elem()
		sum(&src, &dest)
	}
}

func (p *ProcessStatSample) collectIOCounters(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on IO counters collection. Maybe unsupported on this platform.")
		}
	}()
	iocnt, err := proc.IOCounters()
	if err != nil {
		log.Warnf("Error encountered collecting I/O stats: %s", err)
	} else {
		src := reflect.ValueOf(iocnt).Elem()
		dest := reflect.ValueOf(&p.IOCounters).Elem()
		sum(&src, &dest)
	}
}

func (p *ProcessStatSample) collectOpenFiles(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on Open Files collection. Maybe unsupported on this platform.")
		}
	}()
	openFiles, err := proc.OpenFiles()
	if err != nil {
		log.Warnf("Error encountered collecting open files stats: %s", err)
	} else {
		p.OpenFiles = append(p.OpenFiles, openFiles...)
	}
}

func (p *ProcessStatSample) collectNumThreads(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on Number of Threads collection. Maybe unsupported on this platform.")
		}
	}()
	numThreads, err := proc.NumThreads()
	if err != nil {
		log.Warnf("Error encountered collecting thread count stats: %s", err)
	} else {
		p.NumThreads += numThreads
	}
}

func (p *ProcessStatSample) collectCPUPercent(proc *process.Process) {
	defer func() {
		if e := recover(); e != nil {
			log.Warnf("Recovered from panic on CPU Percent collection. Maybe unsupported on this platform.")
		}
	}()
	// Use 0 interval to get difference since the last call
	cpuPercent, err := proc.CPUPercent(0 * time.Second)
	if err != nil {
		log.Warnf("Error encountered collecting CPU percent: %s", err)
	} else {
		p.CPUPercent += cpuPercent
	}
}
<|endoftext|>


<commit_before>package clusterconf

import (
	"encoding/json"
	"errors"
	"net"
	"net/url"
	"path"
	"strconv"

	"github.com/cerana/cerana/acomm"
)

const heartbeatPrefix = "heartbeats"

// DatasetHeartbeatArgs are arguments for updating a dataset node heartbeat.
type DatasetHeartbeatArgs struct {
	ID    string `json:"id"`
	IP    net.IP `json:"ip"`
	InUse bool   `json:"inUse"`
}

// DatasetHeartbeat is dataset heartbeat information.
type DatasetHeartbeat struct {
	IP    net.IP `json:"ip"`
	InUse bool   `json:"inUse"`
}

// DatasetHeartbeatList is the result of a ListDatasetHeartbeats.
type DatasetHeartbeatList struct {
	Heartbeats map[string]map[string]DatasetHeartbeat
}

// DatasetHeartbeat registers a new node heartbeat that is using the dataset.
func (c *ClusterConf) DatasetHeartbeat(req *acomm.Request) (interface{}, *url.URL, error) {
	var args DatasetHeartbeatArgs
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.ID == "" {
		return nil, nil, errors.New("missing arg: id")
	}
	if args.IP == nil {
		return nil, nil, errors.New("missing arg: ip")
	}

	key := path.Join(heartbeatPrefix, datasetsPrefix, args.ID, args.IP.String())
	return nil, nil, c.kvEphemeral(key, args.InUse, c.config.DatasetTTL())
}

// ListDatasetHeartbeats returns a list of all active dataset heartbeats.
func (c *ClusterConf) ListDatasetHeartbeats(req *acomm.Request) (interface{}, *url.URL, error) {
	base := path.Join(heartbeatPrefix, datasetsPrefix)
	values, err := c.kvGetAll(base)
	if err != nil {
		return nil, nil, err
	}
	heartbeats := make(map[string]map[string]DatasetHeartbeat)
	for key, value := range values {
		if key == base {
			continue
		}
		// key: {base}/{id}/{ip}
		id := path.Base(path.Dir(key))
		ip := net.ParseIP(path.Base(key))
		var inUse bool
		if err := json.Unmarshal(value.Data, &inUse); err != nil {
			return nil, nil, err
		}
		if _, ok := heartbeats[id]; !ok {
			heartbeats[id] = make(map[string]DatasetHeartbeat)
		}
		heartbeats[id][ip.String()] = DatasetHeartbeat{IP: ip, InUse: inUse}
	}

	return DatasetHeartbeatList{heartbeats}, nil, nil
}

// BundleHeartbeatArgs are argumenst for updating a bundle heartbeat.
type BundleHeartbeatArgs struct {
	ID           uint64           `json:"id"`
	Serial       string           `json:"serial"`
	IP           net.IP           `json:"ip"`
	HealthErrors map[string]error `json:"healthErrors"`
}

// BundleHeartbeat is bundle heartbeat information.
type BundleHeartbeat struct {
	IP           net.IP           `json:"ip"`
	HealthErrors map[string]error `json:"healthErrors"`
}

// MarshalJSON marshals BundleHeartbeat into a JSON map, converting error
// values to strings.
func (b BundleHeartbeat) MarshalJSON() ([]byte, error) {
	type Alias BundleHeartbeat
	errors := make(map[string]string)
	for key, err := range b.HealthErrors {
		errors[key] = err.Error()
	}
	return json.Marshal(&struct {
		HealthErrors map[string]string `json:"healthErrors"`
		Alias
	}{
		HealthErrors: errors,
		Alias:        (Alias)(b),
	})
}

// UnmarshalJSON unmarshals JSON into a BundleHeartbeat, converting string
// values to errors.
func (b *BundleHeartbeat) UnmarshalJSON(data []byte) error {
	type Alias BundleHeartbeat
	aux := &struct {
		HealthErrors map[string]string `json:"healthErrors"`
		*Alias
	}{
		Alias: (*Alias)(b),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}
	b.HealthErrors = make(map[string]error)
	for key, errS := range aux.HealthErrors {
		b.HealthErrors[key] = errors.New(errS)
	}
	return nil
}

// BundleHeartbeats are a set of bundle heartbeats for a node.
type BundleHeartbeats map[string]BundleHeartbeat

// BundleHeartbeatList is the result of a ListBundleHeartbeats.
type BundleHeartbeatList struct {
	Heartbeats map[uint64]BundleHeartbeats `json:"heartbeats"`
}

// MarshalJSON marshals BundleHeartbeatList into a JSON map, converting uint
// keys to strings.
// TODO: Needed until go 1.7 is released
func (b BundleHeartbeatList) MarshalJSON() ([]byte, error) {
	type Alias BundleHeartbeatList
	hbs := make(map[string]BundleHeartbeats)
	for id, value := range b.Heartbeats {
		hbs[strconv.FormatUint(id, 10)] = value
	}
	return json.Marshal(&struct {
		Heartbeats map[string]BundleHeartbeats `json:"heartbeats"`
		Alias
	}{
		Heartbeats: hbs,
		Alias:      (Alias)(b),
	})
}

// UnmarshalJSON unmarshals JSON into a BundleHeartbeatList, converting string
// keys to uints.
// TODO: Needed until go 1.7 is released
func (b *BundleHeartbeatList) UnmarshalJSON(data []byte) error {
	type Alias BundleHeartbeatList
	aux := &struct {
		Heartbeats map[string]BundleHeartbeats `json:"heartbeats"`
		*Alias
	}{
		Alias: (*Alias)(b),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}
	b.Heartbeats = make(map[uint64]BundleHeartbeats)
	for idS, value := range aux.Heartbeats {
		id, err := strconv.ParseUint(idS, 10, 64)
		if err != nil {
			return err
		}
		b.Heartbeats[id] = value
	}
	return nil
}

// BundleHeartbeat registers a new node heartbeat that is using the dataset.
func (c *ClusterConf) BundleHeartbeat(req *acomm.Request) (interface{}, *url.URL, error) {
	var args BundleHeartbeatArgs
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.ID == 0 {
		return nil, nil, errors.New("missing arg: id")
	}
	if args.Serial == "" {
		return nil, nil, errors.New("missing arg: serial")
	}
	if args.IP == nil {
		return nil, nil, errors.New("missing arg: ip")
	}

	heartbeat := BundleHeartbeat{
		IP:           args.IP,
		HealthErrors: args.HealthErrors,
	}

	key := path.Join(heartbeatPrefix, bundlesPrefix, strconv.FormatUint(args.ID, 10), args.Serial)
	return nil, nil, c.kvEphemeral(key, heartbeat, c.config.BundleTTL())
}

// ListBundleHeartbeats returns a list of all active bundle heartbeats.
func (c *ClusterConf) ListBundleHeartbeats(req *acomm.Request) (interface{}, *url.URL, error) {
	base := path.Join(heartbeatPrefix, bundlesPrefix)
	values, err := c.kvGetAll(base)
	if err != nil {
		return nil, nil, err
	}
	heartbeats := make(map[uint64]BundleHeartbeats)
	for key, value := range values {
		if key == base {
			continue
		}
		// key: {base}/{id}/{serial}
		id, err := strconv.ParseUint(path.Base(path.Dir(key)), 10, 64)
		if err != nil {
			return nil, nil, err
		}
		serial := path.Base(key)
		var hb BundleHeartbeat
		if err := json.Unmarshal(value.Data, &hb); err != nil {
			return nil, nil, err
		}
		if _, ok := heartbeats[id]; !ok {
			heartbeats[id] = make(BundleHeartbeats)
		}
		heartbeats[id][serial] = hb
	}

	return BundleHeartbeatList{heartbeats}, nil, nil
}
<commit_msg>providers/clusterconf: make heartbeatPrefix match other consts<commit_after>package clusterconf

import (
	"encoding/json"
	"errors"
	"net"
	"net/url"
	"path"
	"strconv"

	"github.com/cerana/cerana/acomm"
)

const heartbeatPrefix string = "heartbeats"

// DatasetHeartbeatArgs are arguments for updating a dataset node heartbeat.
type DatasetHeartbeatArgs struct {
	ID    string `json:"id"`
	IP    net.IP `json:"ip"`
	InUse bool   `json:"inUse"`
}

// DatasetHeartbeat is dataset heartbeat information.
type DatasetHeartbeat struct {
	IP    net.IP `json:"ip"`
	InUse bool   `json:"inUse"`
}

// DatasetHeartbeatList is the result of a ListDatasetHeartbeats.
type DatasetHeartbeatList struct {
	Heartbeats map[string]map[string]DatasetHeartbeat
}

// DatasetHeartbeat registers a new node heartbeat that is using the dataset.
func (c *ClusterConf) DatasetHeartbeat(req *acomm.Request) (interface{}, *url.URL, error) {
	var args DatasetHeartbeatArgs
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.ID == "" {
		return nil, nil, errors.New("missing arg: id")
	}
	if args.IP == nil {
		return nil, nil, errors.New("missing arg: ip")
	}

	key := path.Join(heartbeatPrefix, datasetsPrefix, args.ID, args.IP.String())
	return nil, nil, c.kvEphemeral(key, args.InUse, c.config.DatasetTTL())
}

// ListDatasetHeartbeats returns a list of all active dataset heartbeats.
func (c *ClusterConf) ListDatasetHeartbeats(req *acomm.Request) (interface{}, *url.URL, error) {
	base := path.Join(heartbeatPrefix, datasetsPrefix)
	values, err := c.kvGetAll(base)
	if err != nil {
		return nil, nil, err
	}
	heartbeats := make(map[string]map[string]DatasetHeartbeat)
	for key, value := range values {
		if key == base {
			continue
		}
		// key: {base}/{id}/{ip}
		id := path.Base(path.Dir(key))
		ip := net.ParseIP(path.Base(key))
		var inUse bool
		if err := json.Unmarshal(value.Data, &inUse); err != nil {
			return nil, nil, err
		}
		if _, ok := heartbeats[id]; !ok {
			heartbeats[id] = make(map[string]DatasetHeartbeat)
		}
		heartbeats[id][ip.String()] = DatasetHeartbeat{IP: ip, InUse: inUse}
	}

	return DatasetHeartbeatList{heartbeats}, nil, nil
}

// BundleHeartbeatArgs are argumenst for updating a bundle heartbeat.
type BundleHeartbeatArgs struct {
	ID           uint64           `json:"id"`
	Serial       string           `json:"serial"`
	IP           net.IP           `json:"ip"`
	HealthErrors map[string]error `json:"healthErrors"`
}

// BundleHeartbeat is bundle heartbeat information.
type BundleHeartbeat struct {
	IP           net.IP           `json:"ip"`
	HealthErrors map[string]error `json:"healthErrors"`
}

// MarshalJSON marshals BundleHeartbeat into a JSON map, converting error
// values to strings.
func (b BundleHeartbeat) MarshalJSON() ([]byte, error) {
	type Alias BundleHeartbeat
	errors := make(map[string]string)
	for key, err := range b.HealthErrors {
		errors[key] = err.Error()
	}
	return json.Marshal(&struct {
		HealthErrors map[string]string `json:"healthErrors"`
		Alias
	}{
		HealthErrors: errors,
		Alias:        (Alias)(b),
	})
}

// UnmarshalJSON unmarshals JSON into a BundleHeartbeat, converting string
// values to errors.
func (b *BundleHeartbeat) UnmarshalJSON(data []byte) error {
	type Alias BundleHeartbeat
	aux := &struct {
		HealthErrors map[string]string `json:"healthErrors"`
		*Alias
	}{
		Alias: (*Alias)(b),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}
	b.HealthErrors = make(map[string]error)
	for key, errS := range aux.HealthErrors {
		b.HealthErrors[key] = errors.New(errS)
	}
	return nil
}

// BundleHeartbeats are a set of bundle heartbeats for a node.
type BundleHeartbeats map[string]BundleHeartbeat

// BundleHeartbeatList is the result of a ListBundleHeartbeats.
type BundleHeartbeatList struct {
	Heartbeats map[uint64]BundleHeartbeats `json:"heartbeats"`
}

// MarshalJSON marshals BundleHeartbeatList into a JSON map, converting uint
// keys to strings.
// TODO: Needed until go 1.7 is released
func (b BundleHeartbeatList) MarshalJSON() ([]byte, error) {
	type Alias BundleHeartbeatList
	hbs := make(map[string]BundleHeartbeats)
	for id, value := range b.Heartbeats {
		hbs[strconv.FormatUint(id, 10)] = value
	}
	return json.Marshal(&struct {
		Heartbeats map[string]BundleHeartbeats `json:"heartbeats"`
		Alias
	}{
		Heartbeats: hbs,
		Alias:      (Alias)(b),
	})
}

// UnmarshalJSON unmarshals JSON into a BundleHeartbeatList, converting string
// keys to uints.
// TODO: Needed until go 1.7 is released
func (b *BundleHeartbeatList) UnmarshalJSON(data []byte) error {
	type Alias BundleHeartbeatList
	aux := &struct {
		Heartbeats map[string]BundleHeartbeats `json:"heartbeats"`
		*Alias
	}{
		Alias: (*Alias)(b),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}
	b.Heartbeats = make(map[uint64]BundleHeartbeats)
	for idS, value := range aux.Heartbeats {
		id, err := strconv.ParseUint(idS, 10, 64)
		if err != nil {
			return err
		}
		b.Heartbeats[id] = value
	}
	return nil
}

// BundleHeartbeat registers a new node heartbeat that is using the dataset.
func (c *ClusterConf) BundleHeartbeat(req *acomm.Request) (interface{}, *url.URL, error) {
	var args BundleHeartbeatArgs
	if err := req.UnmarshalArgs(&args); err != nil {
		return nil, nil, err
	}
	if args.ID == 0 {
		return nil, nil, errors.New("missing arg: id")
	}
	if args.Serial == "" {
		return nil, nil, errors.New("missing arg: serial")
	}
	if args.IP == nil {
		return nil, nil, errors.New("missing arg: ip")
	}

	heartbeat := BundleHeartbeat{
		IP:           args.IP,
		HealthErrors: args.HealthErrors,
	}

	key := path.Join(heartbeatPrefix, bundlesPrefix, strconv.FormatUint(args.ID, 10), args.Serial)
	return nil, nil, c.kvEphemeral(key, heartbeat, c.config.BundleTTL())
}

// ListBundleHeartbeats returns a list of all active bundle heartbeats.
func (c *ClusterConf) ListBundleHeartbeats(req *acomm.Request) (interface{}, *url.URL, error) {
	base := path.Join(heartbeatPrefix, bundlesPrefix)
	values, err := c.kvGetAll(base)
	if err != nil {
		return nil, nil, err
	}
	heartbeats := make(map[uint64]BundleHeartbeats)
	for key, value := range values {
		if key == base {
			continue
		}
		// key: {base}/{id}/{serial}
		id, err := strconv.ParseUint(path.Base(path.Dir(key)), 10, 64)
		if err != nil {
			return nil, nil, err
		}
		serial := path.Base(key)
		var hb BundleHeartbeat
		if err := json.Unmarshal(value.Data, &hb); err != nil {
			return nil, nil, err
		}
		if _, ok := heartbeats[id]; !ok {
			heartbeats[id] = make(BundleHeartbeats)
		}
		heartbeats[id][serial] = hb
	}

	return BundleHeartbeatList{heartbeats}, nil, nil
}
<|endoftext|>


<commit_before>package oracle

import (
	"context"
	"encoding/json"
	"strings"
	"time"

	"github.com/oracle/oci-go-sdk/v32/dns"

	"github.com/oracle/oci-go-sdk/v32/common"
	"github.com/oracle/oci-go-sdk/v32/example/helpers"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
	"github.com/StackExchange/dnscontrol/v3/pkg/txtutil"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

var features = providers.DocumentationNotes{
	providers.DocCreateDomains:       providers.Can(),
	providers.DocDualHost:            providers.Can(),
	providers.DocOfficiallySupported: providers.Cannot(),

	providers.CanGetZones: providers.Can(),
	providers.CanUseAlias: providers.Can(),
	providers.CanUseCAA:   providers.Can(),
	providers.CanUseDS:    providers.Cannot(), // should be supported, but getting 500s in tests
	providers.CanUseNAPTR: providers.Can(),
	providers.CanUsePTR:   providers.Can(),
	providers.CanUseSRV:   providers.Can(),
	providers.CanUseSSHFP: providers.Can(),
	providers.CanUseTLSA:  providers.Can(),
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   New,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("ORACLE", fns, features)
}

type oracleProvider struct {
	client      dns.DnsClient
	compartment string
}

// New creates a new provider for Oracle Cloud DNS
func New(settings map[string]string, _ json.RawMessage) (providers.DNSServiceProvider, error) {
	client, err := dns.NewDnsClientWithConfigurationProvider(common.NewRawConfigurationProvider(
		settings["tenancy_ocid"],
		settings["user_ocid"],
		settings["region"],
		settings["fingerprint"],
		settings["private_key"],
		nil,
	))
	if err != nil {
		return nil, err
	}

	return &oracleProvider{
		client:      client,
		compartment: settings["compartment"],
	}, nil
}

// ListZones lists the zones on this account.
func (o *oracleProvider) ListZones() ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	listResp, err := o.client.ListZones(ctx, dns.ListZonesRequest{
		CompartmentId: &o.compartment,
	})
	if err != nil {
		return nil, err
	}

	zones := make([]string, len(listResp.Items))
	for i, zone := range listResp.Items {
		zones[i] = *zone.Name
	}
	return zones, nil
}

// EnsureDomainExists creates the domain if it does not exist.
func (o *oracleProvider) EnsureDomainExists(domain string) error {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	getResp, err := o.client.GetZone(ctx, dns.GetZoneRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	})
	if err == nil {
		return nil
	}
	if err != nil && getResp.RawResponse.StatusCode != 404 {
		return err
	}

	_, err = o.client.CreateZone(ctx, dns.CreateZoneRequest{
		CreateZoneDetails: dns.CreateZoneDetails{
			CompartmentId: &o.compartment,
			Name:          &domain,
			ZoneType:      dns.CreateZoneDetailsZoneTypePrimary,
		},
	})
	if err != nil {
		return err
	}

	// poll until the zone is ready
	pollUntilAvailable := func(r common.OCIOperationResponse) bool {
		if converted, ok := r.Response.(dns.GetZoneResponse); ok {
			return converted.LifecycleState != dns.ZoneLifecycleStateActive
		}
		return true
	}
	_, err = o.client.GetZone(ctx, dns.GetZoneRequest{
		ZoneNameOrId:    &domain,
		CompartmentId:   &o.compartment,
		RequestMetadata: helpers.GetRequestMetadataWithCustomizedRetryPolicy(pollUntilAvailable),
	})

	return err
}

func (o *oracleProvider) GetNameservers(domain string) ([]*models.Nameserver, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	getResp, err := o.client.GetZone(ctx, dns.GetZoneRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	})
	if err != nil {
		return nil, err
	}

	nss := make([]string, len(getResp.Zone.Nameservers))
	for i, ns := range getResp.Zone.Nameservers {
		nss[i] = *ns.Hostname
	}

	return models.ToNameserversStripTD(nss)
}

func (o *oracleProvider) GetZoneRecords(domain string) (models.Records, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	records := models.Records{}

	request := dns.GetZoneRecordsRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	}

	for {
		getResp, err := o.client.GetZoneRecords(ctx, request)
		if err != nil {
			return nil, err
		}

		for _, record := range getResp.Items {
			// Hide SOAs
			if *record.Rtype == "SOA" {
				continue
			}

			rc := &models.RecordConfig{
				Type:     *record.Rtype,
				TTL:      uint32(*record.Ttl),
				Original: record,
			}
			rc.SetLabelFromFQDN(*record.Domain, domain)

			switch rc.Type {
			case "ALIAS":
				err = rc.SetTarget(*record.Rdata)
			default:
				err = rc.PopulateFromString(*record.Rtype, *record.Rdata, domain)
			}

			if err != nil {
				return nil, err
			}

			records = append(records, rc)
		}

		if getResp.OpcNextPage == nil {
			break
		}

		request.Page = getResp.OpcNextPage
	}

	return records, nil
}

func (o *oracleProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc, err := dc.Copy()
	if err != nil {
		return nil, err
	}

	err = dc.Punycode()
	if err != nil {
		return nil, err
	}
	domain := dc.Name

	existingRecords, err := o.GetZoneRecords(domain)
	if err != nil {
		return nil, err
	}

	//  Normalize
	models.PostProcessRecords(existingRecords)
	txtutil.SplitSingleLongTxt(dc.Records) // Autosplit long TXT records

	filteredNewRecords := models.Records{}

	// Ensure we don't emit changes for attempted modification of built-in apex NSs
	for _, rec := range dc.Records {
		if rec.Type != "NS" {
			filteredNewRecords = append(filteredNewRecords, rec)
			continue
		}

		recNS := rec.GetTargetField()
		if rec.GetLabel() == "@" && strings.HasSuffix(recNS, "dns.oraclecloud.com.") {
			printer.Warnf("Oracle Cloud does not allow changes to built-in apex NS records. Ignoring change to %s...\n", recNS)
			continue
		}

		if rec.TTL != 86400 {
			printer.Warnf("Oracle Cloud forces TTL=86400 for NS records. Ignoring configured TTL of %d for %s\n", rec.TTL, recNS)
			rec.TTL = 86400
		}
		filteredNewRecords = append(filteredNewRecords, rec)
	}

	differ := diff.New(dc)
	_, create, dels, modify, err := differ.IncrementalDiff(existingRecords)
	if err != nil {
		return nil, err
	}

	/*
		Oracle's API doesn't have a way to update an existing record.
		You can either update an existing RRSet, Domain (FQDN), or Zone in which you have to supply
		the entire desired state, or you can patch specifying ADD/REMOVE actions.
		Oracle's API is also increadibly slow, so updating individual RRSets is unbearably slow
		for any size zone.
	*/

	corrections := []*models.Correction{}

	if len(create) > 0 {
		createRecords := models.Records{}
		desc := ""
		for _, d := range create {
			createRecords = append(createRecords, d.Desired)
			desc += d.String() + "\n"
		}
		desc = desc[:len(desc)-1]

		corrections = append(corrections, &models.Correction{
			Msg: desc,
			F: func() error {
				return o.patch(createRecords, nil, domain)
			},
		})
	}

	if len(dels) > 0 {
		deleteRecords := models.Records{}
		desc := ""
		for _, d := range dels {
			deleteRecords = append(deleteRecords, d.Existing)
			desc += d.String() + "\n"
		}
		desc = desc[:len(desc)-1]

		corrections = append(corrections, &models.Correction{
			Msg: desc,
			F: func() error {
				return o.patch(nil, deleteRecords, domain)
			},
		})
	}

	if len(modify) > 0 {
		createRecords := models.Records{}
		deleteRecords := models.Records{}
		desc := ""
		for _, d := range modify {
			createRecords = append(createRecords, d.Desired)
			deleteRecords = append(deleteRecords, d.Existing)
			desc += d.String() + "\n"
		}
		desc = desc[:len(desc)-1]

		corrections = append(corrections, &models.Correction{
			Msg: desc,
			F: func() error {
				return o.patch(createRecords, deleteRecords, domain)
			},
		})
	}

	return corrections, nil
}

func (o *oracleProvider) patch(createRecords, deleteRecords models.Records, domain string) error {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	patchReq := dns.PatchZoneRecordsRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	}

	ops := make([]dns.RecordOperation, 0, len(createRecords)+len(deleteRecords))

	for _, rec := range deleteRecords {
		ops = append(ops, convertToRecordOperation(rec, dns.RecordOperationOperationRemove))
	}
	for _, rec := range createRecords {
		ops = append(ops, convertToRecordOperation(rec, dns.RecordOperationOperationAdd))
	}

	for batchStart := 0; batchStart < len(ops); batchStart += 100 {
		batchEnd := batchStart + 100
		if batchEnd > len(ops) {
			batchEnd = len(ops)
		}
		patchReq.Items = ops[batchStart:batchEnd]
		_, err := o.client.PatchZoneRecords(ctx, patchReq)
		if err != nil {
			return err
		}
	}

	return nil
}

func convertToRecordOperation(rec *models.RecordConfig, op dns.RecordOperationOperationEnum) dns.RecordOperation {
	fqdn := rec.GetLabelFQDN()
	rtype := rec.Type
	rdata := rec.GetTargetCombined()
	ttl := int(rec.TTL)
	return dns.RecordOperation{
		Domain:    &fqdn,
		Rtype:     &rtype,
		Rdata:     &rdata,
		Ttl:       &ttl,
		Operation: op,
	}
}
<commit_msg>oracle: fix nameserver trailing dot issue, and resolve staticcheck warnings (#1341)<commit_after>package oracle

import (
	"context"
	"encoding/json"
	"strings"
	"time"

	"github.com/oracle/oci-go-sdk/v32/dns"

	"github.com/oracle/oci-go-sdk/v32/common"
	"github.com/oracle/oci-go-sdk/v32/example/helpers"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/pkg/printer"
	"github.com/StackExchange/dnscontrol/v3/pkg/txtutil"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

var features = providers.DocumentationNotes{
	providers.DocCreateDomains:       providers.Can(),
	providers.DocDualHost:            providers.Can(),
	providers.DocOfficiallySupported: providers.Cannot(),

	providers.CanGetZones: providers.Can(),
	providers.CanUseAlias: providers.Can(),
	providers.CanUseCAA:   providers.Can(),
	providers.CanUseDS:    providers.Cannot(), // should be supported, but getting 500s in tests
	providers.CanUseNAPTR: providers.Can(),
	providers.CanUsePTR:   providers.Can(),
	providers.CanUseSRV:   providers.Can(),
	providers.CanUseSSHFP: providers.Can(),
	providers.CanUseTLSA:  providers.Can(),
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   New,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("ORACLE", fns, features)
}

type oracleProvider struct {
	client      dns.DnsClient
	compartment string
}

// New creates a new provider for Oracle Cloud DNS
func New(settings map[string]string, _ json.RawMessage) (providers.DNSServiceProvider, error) {
	client, err := dns.NewDnsClientWithConfigurationProvider(common.NewRawConfigurationProvider(
		settings["tenancy_ocid"],
		settings["user_ocid"],
		settings["region"],
		settings["fingerprint"],
		settings["private_key"],
		nil,
	))
	if err != nil {
		return nil, err
	}

	return &oracleProvider{
		client:      client,
		compartment: settings["compartment"],
	}, nil
}

// ListZones lists the zones on this account.
func (o *oracleProvider) ListZones() ([]string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	listResp, err := o.client.ListZones(ctx, dns.ListZonesRequest{
		CompartmentId: &o.compartment,
	})
	if err != nil {
		return nil, err
	}

	zones := make([]string, len(listResp.Items))
	for i, zone := range listResp.Items {
		zones[i] = *zone.Name
	}
	return zones, nil
}

// EnsureDomainExists creates the domain if it does not exist.
func (o *oracleProvider) EnsureDomainExists(domain string) error {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	getResp, err := o.client.GetZone(ctx, dns.GetZoneRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	})
	if err == nil {
		return nil
	}
	if err != nil && getResp.RawResponse.StatusCode != 404 {
		return err
	}

	_, err = o.client.CreateZone(ctx, dns.CreateZoneRequest{
		CreateZoneDetails: dns.CreateZoneDetails{
			CompartmentId: &o.compartment,
			Name:          &domain,
			ZoneType:      dns.CreateZoneDetailsZoneTypePrimary,
		},
	})
	if err != nil {
		return err
	}

	// poll until the zone is ready
	pollUntilAvailable := func(r common.OCIOperationResponse) bool {
		if converted, ok := r.Response.(dns.GetZoneResponse); ok {
			return converted.LifecycleState != dns.ZoneLifecycleStateActive
		}
		return true
	}
	_, err = o.client.GetZone(ctx, dns.GetZoneRequest{
		ZoneNameOrId:    &domain,
		CompartmentId:   &o.compartment,
		RequestMetadata: helpers.GetRequestMetadataWithCustomizedRetryPolicy(pollUntilAvailable),
	})

	return err
}

func (o *oracleProvider) GetNameservers(domain string) ([]*models.Nameserver, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	getResp, err := o.client.GetZone(ctx, dns.GetZoneRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	})
	if err != nil {
		return nil, err
	}

	nss := make([]string, len(getResp.Zone.Nameservers))
	for i, ns := range getResp.Zone.Nameservers {
		nss[i] = *ns.Hostname
	}

	return models.ToNameservers(nss)
}

func (o *oracleProvider) GetZoneRecords(domain string) (models.Records, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	records := models.Records{}

	request := dns.GetZoneRecordsRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	}

	for {
		getResp, err := o.client.GetZoneRecords(ctx, request)
		if err != nil {
			return nil, err
		}

		for _, record := range getResp.Items {
			// Hide SOAs
			if *record.Rtype == "SOA" {
				continue
			}

			rc := &models.RecordConfig{
				Type:     *record.Rtype,
				TTL:      uint32(*record.Ttl),
				Original: record,
			}
			rc.SetLabelFromFQDN(*record.Domain, domain)

			switch rc.Type {
			case "ALIAS":
				err = rc.SetTarget(*record.Rdata)
			default:
				err = rc.PopulateFromString(*record.Rtype, *record.Rdata, domain)
			}

			if err != nil {
				return nil, err
			}

			records = append(records, rc)
		}

		if getResp.OpcNextPage == nil {
			break
		}

		request.Page = getResp.OpcNextPage
	}

	return records, nil
}

func (o *oracleProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc, err := dc.Copy()
	if err != nil {
		return nil, err
	}

	err = dc.Punycode()
	if err != nil {
		return nil, err
	}
	domain := dc.Name

	existingRecords, err := o.GetZoneRecords(domain)
	if err != nil {
		return nil, err
	}

	//  Normalize
	models.PostProcessRecords(existingRecords)
	txtutil.SplitSingleLongTxt(dc.Records) // Autosplit long TXT records

	// Ensure we don't emit changes for attempted modification of built-in apex NSs
	for _, rec := range dc.Records {
		if rec.Type != "NS" {
			continue
		}

		recNS := rec.GetTargetField()
		if rec.GetLabel() == "@" && strings.HasSuffix(recNS, "dns.oraclecloud.com.") {
			printer.Warnf("Oracle Cloud does not allow changes to built-in apex NS records. Ignoring change to %s...\n", recNS)
			continue
		}

		if rec.TTL != 86400 {
			printer.Warnf("Oracle Cloud forces TTL=86400 for NS records. Ignoring configured TTL of %d for %s\n", rec.TTL, recNS)
			rec.TTL = 86400
		}
	}

	differ := diff.New(dc)
	_, create, dels, modify, err := differ.IncrementalDiff(existingRecords)
	if err != nil {
		return nil, err
	}

	/*
		Oracle's API doesn't have a way to update an existing record.
		You can either update an existing RRSet, Domain (FQDN), or Zone in which you have to supply
		the entire desired state, or you can patch specifying ADD/REMOVE actions.
		Oracle's API is also increadibly slow, so updating individual RRSets is unbearably slow
		for any size zone.
	*/

	corrections := []*models.Correction{}

	if len(create) > 0 {
		createRecords := models.Records{}
		desc := ""
		for _, d := range create {
			createRecords = append(createRecords, d.Desired)
			desc += d.String() + "\n"
		}
		desc = desc[:len(desc)-1]

		corrections = append(corrections, &models.Correction{
			Msg: desc,
			F: func() error {
				return o.patch(createRecords, nil, domain)
			},
		})
	}

	if len(dels) > 0 {
		deleteRecords := models.Records{}
		desc := ""
		for _, d := range dels {
			deleteRecords = append(deleteRecords, d.Existing)
			desc += d.String() + "\n"
		}
		desc = desc[:len(desc)-1]

		corrections = append(corrections, &models.Correction{
			Msg: desc,
			F: func() error {
				return o.patch(nil, deleteRecords, domain)
			},
		})
	}

	if len(modify) > 0 {
		createRecords := models.Records{}
		deleteRecords := models.Records{}
		desc := ""
		for _, d := range modify {
			createRecords = append(createRecords, d.Desired)
			deleteRecords = append(deleteRecords, d.Existing)
			desc += d.String() + "\n"
		}
		desc = desc[:len(desc)-1]

		corrections = append(corrections, &models.Correction{
			Msg: desc,
			F: func() error {
				return o.patch(createRecords, deleteRecords, domain)
			},
		})
	}

	return corrections, nil
}

func (o *oracleProvider) patch(createRecords, deleteRecords models.Records, domain string) error {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()

	patchReq := dns.PatchZoneRecordsRequest{
		ZoneNameOrId:  &domain,
		CompartmentId: &o.compartment,
	}

	ops := make([]dns.RecordOperation, 0, len(createRecords)+len(deleteRecords))

	for _, rec := range deleteRecords {
		ops = append(ops, convertToRecordOperation(rec, dns.RecordOperationOperationRemove))
	}
	for _, rec := range createRecords {
		ops = append(ops, convertToRecordOperation(rec, dns.RecordOperationOperationAdd))
	}

	for batchStart := 0; batchStart < len(ops); batchStart += 100 {
		batchEnd := batchStart + 100
		if batchEnd > len(ops) {
			batchEnd = len(ops)
		}
		patchReq.Items = ops[batchStart:batchEnd]
		_, err := o.client.PatchZoneRecords(ctx, patchReq)
		if err != nil {
			return err
		}
	}

	return nil
}

func convertToRecordOperation(rec *models.RecordConfig, op dns.RecordOperationOperationEnum) dns.RecordOperation {
	fqdn := rec.GetLabelFQDN()
	rtype := rec.Type
	rdata := rec.GetTargetCombined()
	ttl := int(rec.TTL)
	return dns.RecordOperation{
		Domain:    &fqdn,
		Rtype:     &rtype,
		Rdata:     &rdata,
		Ttl:       &ttl,
		Operation: op,
	}
}
<|endoftext|>


<commit_before>package ai

import (
	"bytes"
	"log"
	"math/rand"
	"time"

	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

const (
	maxEval      int64 = 1 << 30
	minEval            = -maxEval
	WinThreshold       = 1 << 29

	tableSize uint64 = (1 << 20)

	maxStack = 10
)

type EvaluationFunc func(m *MinimaxAI, p *tak.Position) int64

type MinimaxAI struct {
	cfg  MinimaxConfig
	rand *rand.Rand

	st Stats
	c  bitboard.Constants

	heatMap []uint64

	evaluate EvaluationFunc

	table []tableEntry
	stack [maxStack]struct {
		p     *tak.Position
		moves [500]tak.Move
	}
}

type tableEntry struct {
	hash  uint64
	depth int
	value int64
	bound boundType
	m     tak.Move
	p     *tak.Position
}

type boundType byte

const (
	lowerBound = iota
	exactBound = iota
	upperBound = iota
)

type Stats struct {
	Depth     int
	Generated uint64
	Evaluated uint64
	Terminal  uint64
	Visited   uint64

	CutNodes  uint64
	Cut0      uint64
	Cut1      uint64
	CutSearch uint64

	AllNodes uint64

	TTHits uint64
}

type MinimaxConfig struct {
	Size  int
	Depth int
	Debug int
	Seed  int64

	NoSort  bool
	NoTable bool

	Evaluate EvaluationFunc
}

func NewMinimax(cfg MinimaxConfig) *MinimaxAI {
	m := &MinimaxAI{cfg: cfg}
	m.precompute()
	m.evaluate = cfg.Evaluate
	if m.evaluate == nil {
		m.evaluate = DefaultEvaluate
	}
	m.heatMap = make([]uint64, m.cfg.Size*m.cfg.Size)
	m.table = make([]tableEntry, tableSize)
	for i := range m.stack {
		m.stack[i].p = tak.Alloc(m.cfg.Size)
	}
	return m
}

func (m *MinimaxAI) ttGet(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	te := &m.table[h%tableSize]
	if te.hash != h {
		return nil
	}
	return te
}

func (m *MinimaxAI) ttPut(h uint64) *tableEntry {
	return &m.table[h%tableSize]
}

func (m *MinimaxAI) precompute() {
	s := uint(m.cfg.Size)
	m.c = bitboard.Precompute(s)
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(p *tak.Position, limit time.Duration) tak.Move {
	ms, _, _ := m.Analyze(p, limit)
	return ms[0]
}

func (m *MinimaxAI) Analyze(p *tak.Position, limit time.Duration) ([]tak.Move, int64, Stats) {
	if m.cfg.Size != p.Size() {
		panic("Analyze: wrong size")
	}
	for i, v := range m.heatMap {
		m.heatMap[i] = v / 2
	}

	var seed = m.cfg.Seed
	if seed == 0 {
		seed = time.Now().Unix()
	}
	m.rand = rand.New(rand.NewSource(seed))
	if m.cfg.Debug > 0 {
		log.Printf("seed=%d", seed)
	}

	var ms []tak.Move
	var v int64
	top := time.Now()
	var prevEval uint64
	var branchSum uint64
	base := 0
	te := m.ttGet(p.Hash())
	if te != nil && te.bound == exactBound {
		base = te.depth
		ms = []tak.Move{te.m}
	}

	for i := 1; i+base <= m.cfg.Depth; i++ {
		m.st = Stats{Depth: i + base}
		start := time.Now()
		ms, v = m.minimax(p, 0, i+base, ms, minEval-1, maxEval+1)
		timeUsed := time.Now().Sub(top)
		timeMove := time.Now().Sub(start)
		if m.cfg.Debug > 0 {
			log.Printf("[minimax] deepen: depth=%d val=%d pv=%s time=%s total=%s evaluated=%d tt=%d branch=%d",
				base+i, v, formatpv(ms),
				timeMove,
				timeUsed,
				m.st.Evaluated,
				m.st.TTHits,
				m.st.Evaluated/(prevEval+1),
			)
		}
		if m.cfg.Debug > 1 {
			log.Printf("[minimax]  stats: visited=%d evaluated=%d terminal=%d cut=%d cut0=%d(%2.2f) cut1=%d(%2.2f) m/cut=%2.2f m/ms=%f all=%d",
				m.st.Visited,
				m.st.Evaluated,
				m.st.Terminal,
				m.st.CutNodes,
				m.st.Cut0,
				float64(m.st.Cut0)/float64(m.st.CutNodes+1),
				m.st.Cut1,
				float64(m.st.Cut0+m.st.Cut1)/float64(m.st.CutNodes+1),
				float64(m.st.CutSearch)/float64(m.st.CutNodes-m.st.Cut0-m.st.Cut1+1),
				float64(m.st.Visited+m.st.Evaluated)/float64(timeMove.Seconds()*1000),
				m.st.AllNodes)
		}
		if i > 1 {
			branchSum += m.st.Evaluated / (prevEval + 1)
		}
		prevEval = m.st.Evaluated
		if v > WinThreshold || v < -WinThreshold {
			break
		}
		if i+base != m.cfg.Depth && limit != 0 {
			var branch uint64
			if i > 2 {
				branch = branchSum / uint64(i-1)
			} else {
				// conservative estimate if we haven't
				// run enough plies to have one
				// yet. This can matter if the table
				// returns a deep move
				branch = 20
			}
			estimate := timeUsed + time.Now().Sub(start)*time.Duration(branch)
			if estimate > limit {
				if m.cfg.Debug > 0 {
					log.Printf("[minimax] time cutoff: depth=%d used=%s estimate=%s",
						i, timeUsed, estimate)
				}
				break
			}
		}
	}
	return ms, v, m.st
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	ply, depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		ai.st.Evaluated++
		if over {
			ai.st.Terminal++
		}
		return nil, ai.evaluate(ai, p)
	}

	ai.st.Visited++

	te := ai.ttGet(p.Hash())
	if te != nil {
		teSuffices := false
		if te.depth >= depth {
			if te.bound == exactBound ||
				(te.value < α && te.bound == upperBound) ||
				(te.value > β && te.bound == lowerBound) {
				teSuffices = true
			}
		}

		if te.bound == exactBound &&
			(te.value > WinThreshold || te.value < -WinThreshold) {
			teSuffices = true
		}
		if teSuffices {
			_, e := p.Move(&te.m)
			if e == nil {
				ai.st.TTHits++
				return []tak.Move{te.m}, te.value
			}
			te = nil
		}
	}
	mg := moveGenerator{
		ai:    ai,
		ply:   ply,
		depth: depth,
		p:     p,
		te:    te,
		pv:    pv,
	}

	best := make([]tak.Move, 0, depth)
	best = append(best, pv...)
	improved := false
	var i int
	for m, child := mg.Next(); child != nil; m, child = mg.Next() {
		i++
		var ms []tak.Move
		var newpv []tak.Move
		var v int64
		if len(best) != 0 {
			newpv = best[1:]
		}
		if i > 1 {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -α-1, -α)
			if -v > α && -v < β {
				ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
			}
		} else {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
		}
		v = -v
		if ai.cfg.Debug > 2 && ply == 0 {
			log.Printf("[minimax] search: depth=%d ply=%d m=%s pv=%s window=(%d,%d) ms=%s v=%d evaluated=%d",
				depth, ply, ptn.FormatMove(&m), formatpv(newpv), α, β, formatpv(ms), v, ai.st.Evaluated)
		}

		if len(best) == 0 {
			best = append(best[:0], m)
			best = append(best, ms...)
		}
		if v > α {
			improved = true
			best = append(best[:0], m)
			best = append(best, ms...)
			α = v
			if α >= β {
				ai.st.CutNodes++
				switch i {
				case 1:
					ai.st.Cut0++
				case 2:
					ai.st.Cut1++
				default:
					ai.st.CutSearch += uint64(i + 1)
				}
				ai.heatMap[m.X+m.Y*ai.cfg.Size] += (1 << uint(depth))
				if ai.cfg.Debug > 3 && i > 20 && depth >= 3 {
					var tm tak.Move
					td := 0
					if te != nil {
						tm = te.m
						td = te.depth
					}
					log.Printf("[minimax] late cutoff depth=%d m=%d pv=%s te=%d:%s killer=%s pos=%q",
						depth, i, formatpv(pv), td, ptn.FormatMove(&tm), ptn.FormatMove(&m), ptn.FormatTPS(p),
					)
				}
				break
			}
		}
	}

	te = ai.ttPut(p.Hash())
	te.hash = p.Hash()
	te.depth = depth
	te.m = best[0]
	te.value = α
	if !improved {
		te.bound = upperBound
		ai.st.AllNodes++
	} else if α >= β {
		te.bound = lowerBound
	} else {
		te.bound = exactBound
	}

	return best, α
}
<commit_msg>move the stored pv into the preallocated buffer<commit_after>package ai

import (
	"bytes"
	"log"
	"math/rand"
	"time"

	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

const (
	maxEval      int64 = 1 << 30
	minEval            = -maxEval
	WinThreshold       = 1 << 29

	tableSize uint64 = (1 << 20)

	maxStack = 10
)

type EvaluationFunc func(m *MinimaxAI, p *tak.Position) int64

type MinimaxAI struct {
	cfg  MinimaxConfig
	rand *rand.Rand

	st Stats
	c  bitboard.Constants

	heatMap []uint64

	evaluate EvaluationFunc

	table []tableEntry
	stack [maxStack]struct {
		p     *tak.Position
		moves [500]tak.Move
		pv    [10]tak.Move
	}
}

type tableEntry struct {
	hash  uint64
	depth int
	value int64
	bound boundType
	m     tak.Move
	p     *tak.Position
}

type boundType byte

const (
	lowerBound = iota
	exactBound = iota
	upperBound = iota
)

type Stats struct {
	Depth     int
	Generated uint64
	Evaluated uint64
	Terminal  uint64
	Visited   uint64

	CutNodes  uint64
	Cut0      uint64
	Cut1      uint64
	CutSearch uint64

	AllNodes uint64

	TTHits uint64
}

type MinimaxConfig struct {
	Size  int
	Depth int
	Debug int
	Seed  int64

	NoSort  bool
	NoTable bool

	Evaluate EvaluationFunc
}

func NewMinimax(cfg MinimaxConfig) *MinimaxAI {
	m := &MinimaxAI{cfg: cfg}
	m.precompute()
	m.evaluate = cfg.Evaluate
	if m.evaluate == nil {
		m.evaluate = DefaultEvaluate
	}
	m.heatMap = make([]uint64, m.cfg.Size*m.cfg.Size)
	m.table = make([]tableEntry, tableSize)
	for i := range m.stack {
		m.stack[i].p = tak.Alloc(m.cfg.Size)
	}
	return m
}

func (m *MinimaxAI) ttGet(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	te := &m.table[h%tableSize]
	if te.hash != h {
		return nil
	}
	return te
}

func (m *MinimaxAI) ttPut(h uint64) *tableEntry {
	return &m.table[h%tableSize]
}

func (m *MinimaxAI) precompute() {
	s := uint(m.cfg.Size)
	m.c = bitboard.Precompute(s)
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(p *tak.Position, limit time.Duration) tak.Move {
	ms, _, _ := m.Analyze(p, limit)
	return ms[0]
}

func (m *MinimaxAI) Analyze(p *tak.Position, limit time.Duration) ([]tak.Move, int64, Stats) {
	if m.cfg.Size != p.Size() {
		panic("Analyze: wrong size")
	}
	for i, v := range m.heatMap {
		m.heatMap[i] = v / 2
	}

	var seed = m.cfg.Seed
	if seed == 0 {
		seed = time.Now().Unix()
	}
	m.rand = rand.New(rand.NewSource(seed))
	if m.cfg.Debug > 0 {
		log.Printf("seed=%d", seed)
	}

	var ms []tak.Move
	var v int64
	top := time.Now()
	var prevEval uint64
	var branchSum uint64
	base := 0
	te := m.ttGet(p.Hash())
	if te != nil && te.bound == exactBound {
		base = te.depth
		ms = []tak.Move{te.m}
	}

	for i := 1; i+base <= m.cfg.Depth; i++ {
		m.st = Stats{Depth: i + base}
		start := time.Now()
		ms, v = m.minimax(p, 0, i+base, ms, minEval-1, maxEval+1)
		timeUsed := time.Now().Sub(top)
		timeMove := time.Now().Sub(start)
		if m.cfg.Debug > 0 {
			log.Printf("[minimax] deepen: depth=%d val=%d pv=%s time=%s total=%s evaluated=%d tt=%d branch=%d",
				base+i, v, formatpv(ms),
				timeMove,
				timeUsed,
				m.st.Evaluated,
				m.st.TTHits,
				m.st.Evaluated/(prevEval+1),
			)
		}
		if m.cfg.Debug > 1 {
			log.Printf("[minimax]  stats: visited=%d evaluated=%d terminal=%d cut=%d cut0=%d(%2.2f) cut1=%d(%2.2f) m/cut=%2.2f m/ms=%f all=%d",
				m.st.Visited,
				m.st.Evaluated,
				m.st.Terminal,
				m.st.CutNodes,
				m.st.Cut0,
				float64(m.st.Cut0)/float64(m.st.CutNodes+1),
				m.st.Cut1,
				float64(m.st.Cut0+m.st.Cut1)/float64(m.st.CutNodes+1),
				float64(m.st.CutSearch)/float64(m.st.CutNodes-m.st.Cut0-m.st.Cut1+1),
				float64(m.st.Visited+m.st.Evaluated)/float64(timeMove.Seconds()*1000),
				m.st.AllNodes)
		}
		if i > 1 {
			branchSum += m.st.Evaluated / (prevEval + 1)
		}
		prevEval = m.st.Evaluated
		if v > WinThreshold || v < -WinThreshold {
			break
		}
		if i+base != m.cfg.Depth && limit != 0 {
			var branch uint64
			if i > 2 {
				branch = branchSum / uint64(i-1)
			} else {
				// conservative estimate if we haven't
				// run enough plies to have one
				// yet. This can matter if the table
				// returns a deep move
				branch = 20
			}
			estimate := timeUsed + time.Now().Sub(start)*time.Duration(branch)
			if estimate > limit {
				if m.cfg.Debug > 0 {
					log.Printf("[minimax] time cutoff: depth=%d used=%s estimate=%s",
						i, timeUsed, estimate)
				}
				break
			}
		}
	}
	r := make([]tak.Move, len(ms))
	copy(r, ms)
	return r, v, m.st
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	ply, depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		ai.st.Evaluated++
		if over {
			ai.st.Terminal++
		}
		return nil, ai.evaluate(ai, p)
	}

	ai.st.Visited++

	te := ai.ttGet(p.Hash())
	if te != nil {
		teSuffices := false
		if te.depth >= depth {
			if te.bound == exactBound ||
				(te.value < α && te.bound == upperBound) ||
				(te.value > β && te.bound == lowerBound) {
				teSuffices = true
			}
		}

		if te.bound == exactBound &&
			(te.value > WinThreshold || te.value < -WinThreshold) {
			teSuffices = true
		}
		if teSuffices {
			_, e := p.Move(&te.m)
			if e == nil {
				ai.st.TTHits++
				return []tak.Move{te.m}, te.value
			}
			te = nil
		}
	}
	mg := moveGenerator{
		ai:    ai,
		ply:   ply,
		depth: depth,
		p:     p,
		te:    te,
		pv:    pv,
	}

	best := ai.stack[ply].pv[:0]
	best = append(best, pv...)
	improved := false
	var i int
	for m, child := mg.Next(); child != nil; m, child = mg.Next() {
		i++
		var ms []tak.Move
		var newpv []tak.Move
		var v int64
		if len(best) != 0 {
			newpv = best[1:]
		}
		if i > 1 {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -α-1, -α)
			if -v > α && -v < β {
				ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
			}
		} else {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
		}
		v = -v
		if ai.cfg.Debug > 2 && ply == 0 {
			log.Printf("[minimax] search: depth=%d ply=%d m=%s pv=%s window=(%d,%d) ms=%s v=%d evaluated=%d",
				depth, ply, ptn.FormatMove(&m), formatpv(newpv), α, β, formatpv(ms), v, ai.st.Evaluated)
		}

		if len(best) == 0 {
			best = append(best[:0], m)
			best = append(best, ms...)
		}
		if v > α {
			improved = true
			best = append(best[:0], m)
			best = append(best, ms...)
			α = v
			if α >= β {
				ai.st.CutNodes++
				switch i {
				case 1:
					ai.st.Cut0++
				case 2:
					ai.st.Cut1++
				default:
					ai.st.CutSearch += uint64(i + 1)
				}
				ai.heatMap[m.X+m.Y*ai.cfg.Size] += (1 << uint(depth))
				if ai.cfg.Debug > 3 && i > 20 && depth >= 3 {
					var tm tak.Move
					td := 0
					if te != nil {
						tm = te.m
						td = te.depth
					}
					log.Printf("[minimax] late cutoff depth=%d m=%d pv=%s te=%d:%s killer=%s pos=%q",
						depth, i, formatpv(pv), td, ptn.FormatMove(&tm), ptn.FormatMove(&m), ptn.FormatTPS(p),
					)
				}
				break
			}
		}
	}

	te = ai.ttPut(p.Hash())
	te.hash = p.Hash()
	te.depth = depth
	te.m = best[0]
	te.value = α
	if !improved {
		te.bound = upperBound
		ai.st.AllNodes++
	} else if α >= β {
		te.bound = lowerBound
	} else {
		te.bound = exactBound
	}

	return best, α
}
<|endoftext|>


<commit_before>package ai

import (
	"bytes"
	"log"
	"math/rand"
	"sync/atomic"
	"time"

	"golang.org/x/net/context"

	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

const (
	MaxEval      int64 = 1 << 30
	MinEval            = -MaxEval
	WinThreshold       = 1 << 29

	tableSize uint64 = (1 << 20)

	maxDepth = 15
)

type EvaluationFunc func(m *MinimaxAI, p *tak.Position) int64

type MinimaxAI struct {
	cfg  MinimaxConfig
	rand *rand.Rand

	st Stats
	c  bitboard.Constants

	history  map[uint64]int
	response map[uint64]tak.Move

	evaluate EvaluationFunc

	table []tableEntry
	stack [maxDepth]struct {
		p     *tak.Position
		mg    moveGenerator
		moves [500]tak.Move
		pv    [maxDepth]tak.Move
		m     tak.Move
	}

	cancel *int32
}

type tableEntry struct {
	hash  uint64
	depth int
	value int64
	bound boundType
	m     tak.Move
	p     *tak.Position
}

type boundType byte

const (
	lowerBound = iota
	exactBound = iota
	upperBound = iota
)

type Stats struct {
	Depth     int
	Generated uint64
	Evaluated uint64
	Scout     uint64
	Terminal  uint64
	Visited   uint64

	CutNodes   uint64
	NullSearch uint64
	NullCut    uint64
	Cut0       uint64
	Cut1       uint64
	CutSearch  uint64

	ReSearch uint64

	AllNodes uint64

	TTHits uint64
}

type MinimaxConfig struct {
	Size  int
	Depth int
	Debug int
	Seed  int64

	NoSort     bool
	NoTable    bool
	NoNullMove bool

	Evaluate EvaluationFunc
}

func NewMinimax(cfg MinimaxConfig) *MinimaxAI {
	m := &MinimaxAI{cfg: cfg}
	if m.cfg.Depth == 0 {
		m.cfg.Depth = maxDepth
	}
	m.precompute()
	m.evaluate = cfg.Evaluate
	if m.evaluate == nil {
		m.evaluate = MakeEvaluator(cfg.Size, nil)
	}
	m.history = make(map[uint64]int, m.cfg.Size*m.cfg.Size*m.cfg.Size)
	m.response = make(map[uint64]tak.Move, m.cfg.Size*m.cfg.Size*m.cfg.Size)
	if !cfg.NoTable {
		m.table = make([]tableEntry, tableSize)
	}
	for i := range m.stack {
		m.stack[i].p = tak.Alloc(m.cfg.Size)
	}
	return m
}

func (m *MinimaxAI) ttGet(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	te := &m.table[h%tableSize]
	if te.hash != h {
		return nil
	}
	return te
}

func (m *MinimaxAI) ttPut(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	if atomic.LoadInt32(m.cancel) != 0 {
		return nil
	}
	return &m.table[h%tableSize]
}

func (m *MinimaxAI) precompute() {
	s := uint(m.cfg.Size)
	m.c = bitboard.Precompute(s)
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(ctx context.Context, p *tak.Position) tak.Move {
	ms, _, _ := m.Analyze(ctx, p)
	return ms[0]
}

func (m *MinimaxAI) Analyze(ctx context.Context, p *tak.Position) ([]tak.Move, int64, Stats) {
	if m.cfg.Size != p.Size() {
		panic("Analyze: wrong size")
	}
	for i, v := range m.history {
		m.history[i] = v / 2
	}
	var cancel int32
	m.cancel = &cancel
	go func() {
		<-ctx.Done()
		atomic.StoreInt32(&cancel, 1)
	}()

	var seed = m.cfg.Seed
	if seed == 0 {
		seed = time.Now().Unix()
	}
	m.rand = rand.New(rand.NewSource(seed))
	if m.cfg.Debug > 0 {
		log.Printf("start search ply=%d color=%s seed=%d",
			p.MoveNumber(), p.ToMove(), seed)
	}
	deadline, limited := ctx.Deadline()

	var next []tak.Move
	ms := make([]tak.Move, 0, maxDepth)
	var v int64
	top := time.Now()
	var prevEval uint64
	var branchSum uint64
	base := 0
	te := m.ttGet(p.Hash())
	if te != nil && te.bound == exactBound {
		base = te.depth
		ms = append(ms[:0], te.m)
	}

	for i := 1; i+base <= m.cfg.Depth; i++ {
		m.st = Stats{Depth: i + base}
		start := time.Now()
		next, v = m.minimax(p, 0, i+base, ms, MinEval-1, MaxEval+1)
		if next == nil || atomic.LoadInt32(m.cancel) != 0 {
			break
		}
		ms = append(ms[:0], next...)
		timeUsed := time.Now().Sub(top)
		timeMove := time.Now().Sub(start)
		if m.cfg.Debug > 0 {
			log.Printf("[minimax] deepen: depth=%d val=%d pv=%s time=%s total=%s evaluated=%d tt=%d branch=%d",
				base+i, v, formatpv(ms),
				timeMove,
				timeUsed,
				m.st.Evaluated,
				m.st.TTHits,
				m.st.Evaluated/(prevEval+1),
			)
		}
		if m.cfg.Debug > 1 {
			log.Printf("[minimax]  stats: visited=%d scout=%d evaluated=%d null=%d/%d cut=%d cut0=%d(%2.2f) cut1=%d(%2.2f) m/cut=%2.2f m/ms=%f all=%d research=%d",
				m.st.Visited,
				m.st.Scout,
				m.st.Evaluated,
				m.st.NullCut,
				m.st.NullSearch,
				m.st.CutNodes,
				m.st.Cut0,
				float64(m.st.Cut0)/float64(m.st.CutNodes+1),
				m.st.Cut1,
				float64(m.st.Cut0+m.st.Cut1)/float64(m.st.CutNodes+1),
				float64(m.st.CutSearch)/float64(m.st.CutNodes-m.st.Cut0-m.st.Cut1+1),
				float64(m.st.Visited+m.st.Evaluated)/float64(timeMove.Seconds()*1000),
				m.st.AllNodes,
				m.st.ReSearch)
		}
		if i > 1 {
			branchSum += m.st.Evaluated / (prevEval + 1)
		}
		prevEval = m.st.Evaluated
		if v > WinThreshold || v < -WinThreshold {
			break
		}
		if limited && i+base != m.cfg.Depth {
			var branch uint64
			if i > 2 {
				// conservatively multiply by 2 to
				// account for the bimodal branching
				// factor
				branch = 2 * branchSum / uint64(i-1)
			} else {
				// conservative estimate if we haven't
				// run enough plies to have one
				// yet. This can matter if the table
				// returns a deep move
				branch = 20
			}
			estimate := time.Now().Add(time.Now().Sub(start) * time.Duration(branch))
			if estimate.After(deadline) {
				if m.cfg.Debug > 0 {
					log.Printf("[minimax] time cutoff: depth=%d used=%s estimate=%s",
						i, timeUsed, estimate.Sub(top))
				}
				break
			}
		}
	}
	return ms, v, m.st
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	ply, depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		ai.st.Evaluated++
		if over {
			ai.st.Terminal++
		}
		return nil, ai.evaluate(ai, p)
	}

	ai.st.Visited++
	if β == α+1 {
		ai.st.Scout++
	}

	te := ai.ttGet(p.Hash())
	if te != nil {
		teSuffices := false
		if te.depth >= depth {
			if te.bound == exactBound ||
				(te.value < α && te.bound == upperBound) ||
				(te.value > β && te.bound == lowerBound) {
				teSuffices = true
			}
		}

		if te.bound == exactBound &&
			(te.value > WinThreshold || te.value < -WinThreshold) {
			teSuffices = true
		}
		if teSuffices {
			_, e := p.MovePreallocated(&te.m, ai.stack[ply].p)
			if e == nil {
				ai.st.TTHits++
				ai.stack[ply].pv[0] = te.m
				return ai.stack[ply].pv[:1], te.value
			}
			te = nil
		}
	}

	if β == α+1 && ai.nullMoveOK(ply, depth, p) {
		ai.stack[ply].m = tak.Move{Type: tak.Pass}
		child, e := p.MovePreallocated(&ai.stack[ply].m, ai.stack[ply].p)
		if e == nil {
			ai.st.NullSearch++
			_, v := ai.minimax(child, ply+1, depth-3, nil, -α-1, -α)
			v = -v
			if v >= β {
				ai.st.NullCut++
				return nil, v
			}
		}
	}

	// As of 1.6.2, Go's escape analysis can't tell that a
	// stack-allocated object here doesn't escape. So we force it
	// into our manual stack.
	mg := &ai.stack[ply].mg
	*mg = moveGenerator{
		ai:    ai,
		ply:   ply,
		depth: depth,
		p:     p,
		te:    te,
		pv:    pv,
	}

	best := ai.stack[ply].pv[:0]
	best = append(best, pv...)
	improved := false
	var i int
	for m, child := mg.Next(); child != nil; m, child = mg.Next() {
		i++
		var ms []tak.Move
		var newpv []tak.Move
		var v int64
		if len(best) != 0 {
			newpv = best[1:]
		}
		ai.stack[ply].m = m
		if i > 1 {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -α-1, -α)
			if -v > α && -v < β {
				ai.st.ReSearch++
				ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
			}
		} else {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
		}
		v = -v

		if len(best) == 0 {
			best = append(best[:0], m)
			best = append(best, ms...)
		}
		if v > α {
			improved = true
			best = append(best[:0], m)
			best = append(best, ms...)
			α = v
			if α >= β {
				ai.st.CutNodes++
				switch i {
				case 1:
					ai.st.Cut0++
				case 2:
					ai.st.Cut1++
				default:
					ai.st.CutSearch += uint64(i + 1)
				}
				ai.history[m.Hash()] += (1 << uint(depth))
				if ply > 0 {
					ai.response[ai.stack[ply-1].m.Hash()] = m
				}
				if ai.cfg.Debug > 3 && i > 20 && depth >= 3 {
					var tm tak.Move
					td := 0
					if te != nil {
						tm = te.m
						td = te.depth
					}
					log.Printf("[minimax] late cutoff depth=%d m=%d pv=%s te=%d:%s killer=%s pos=%q",
						depth, i, formatpv(pv), td, ptn.FormatMove(&tm), ptn.FormatMove(&m), ptn.FormatTPS(p),
					)
				}
				break
			}
		}
		if atomic.LoadInt32(ai.cancel) != 0 {
			return nil, 0
		}
	}

	if te = ai.ttPut(p.Hash()); te != nil {
		te.hash = p.Hash()
		te.depth = depth
		te.m = best[0]
		te.value = α
		if !improved {
			te.bound = upperBound
			ai.st.AllNodes++
		} else if α >= β {
			te.bound = lowerBound
		} else {
			te.bound = exactBound
		}
	}

	return best, α
}

func (ai *MinimaxAI) nullMoveOK(ply, depth int, p *tak.Position) bool {
	if ai.cfg.NoNullMove {
		return false
	}
	if ply == 0 || depth < 3 {
		return false
	}
	if ai.stack[ply-1].m.Type == tak.Pass {
		return false
	}
	if p.WhiteStones() < 3 || p.BlackStones() < 3 {
		return false
	}
	if bitboard.Popcount(p.White|p.Black)+3 >= len(p.Stacks) {
		return false
	}
	return true
}
<commit_msg>probe two hash slots<commit_after>package ai

import (
	"bytes"
	"log"
	"math/rand"
	"sync/atomic"
	"time"

	"golang.org/x/net/context"

	"github.com/nelhage/taktician/bitboard"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

const (
	MaxEval      int64 = 1 << 30
	MinEval            = -MaxEval
	WinThreshold       = 1 << 29

	tableSize uint64 = (1 << 20)

	maxDepth = 15
)

type EvaluationFunc func(m *MinimaxAI, p *tak.Position) int64

type MinimaxAI struct {
	cfg  MinimaxConfig
	rand *rand.Rand

	st Stats
	c  bitboard.Constants

	history  map[uint64]int
	response map[uint64]tak.Move

	evaluate EvaluationFunc

	table []tableEntry
	stack [maxDepth]struct {
		p     *tak.Position
		mg    moveGenerator
		moves [500]tak.Move
		pv    [maxDepth]tak.Move
		m     tak.Move
	}

	cancel *int32
}

type tableEntry struct {
	hash  uint64
	depth int
	value int64
	bound boundType
	m     tak.Move
	p     *tak.Position
}

type boundType byte

const (
	lowerBound = iota
	exactBound = iota
	upperBound = iota
)

type Stats struct {
	Depth     int
	Generated uint64
	Evaluated uint64
	Scout     uint64
	Terminal  uint64
	Visited   uint64

	CutNodes   uint64
	NullSearch uint64
	NullCut    uint64
	Cut0       uint64
	Cut1       uint64
	CutSearch  uint64

	ReSearch uint64

	AllNodes uint64

	TTHits     uint64
	TTShortcut uint64
}

type MinimaxConfig struct {
	Size  int
	Depth int
	Debug int
	Seed  int64

	NoSort     bool
	NoTable    bool
	NoNullMove bool

	Evaluate EvaluationFunc
}

func NewMinimax(cfg MinimaxConfig) *MinimaxAI {
	m := &MinimaxAI{cfg: cfg}
	if m.cfg.Depth == 0 {
		m.cfg.Depth = maxDepth
	}
	m.precompute()
	m.evaluate = cfg.Evaluate
	if m.evaluate == nil {
		m.evaluate = MakeEvaluator(cfg.Size, nil)
	}
	m.history = make(map[uint64]int, m.cfg.Size*m.cfg.Size*m.cfg.Size)
	m.response = make(map[uint64]tak.Move, m.cfg.Size*m.cfg.Size*m.cfg.Size)
	if !cfg.NoTable {
		m.table = make([]tableEntry, tableSize)
	}
	for i := range m.stack {
		m.stack[i].p = tak.Alloc(m.cfg.Size)
	}
	return m
}

const hashMul = 0x61C8864680B583EB

func (m *MinimaxAI) ttGet(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	i1 := h % tableSize
	i2 := (h * hashMul) % tableSize
	te := &m.table[i1]
	if te.hash == h {
		return te
	}
	te = &m.table[i2]
	if te.hash == h {
		return te
	}
	return nil
}

func (m *MinimaxAI) ttPut(h uint64) *tableEntry {
	if m.cfg.NoTable {
		return nil
	}
	if atomic.LoadInt32(m.cancel) != 0 {
		return nil
	}
	i1 := h % tableSize
	i2 := (h * hashMul) % tableSize
	if m.table[i1].hash != 0 {
		m.table[i2] = m.table[i1]
	}
	return &m.table[i1]
}

func (m *MinimaxAI) precompute() {
	s := uint(m.cfg.Size)
	m.c = bitboard.Precompute(s)
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(ctx context.Context, p *tak.Position) tak.Move {
	ms, _, _ := m.Analyze(ctx, p)
	return ms[0]
}

func (m *MinimaxAI) Analyze(ctx context.Context, p *tak.Position) ([]tak.Move, int64, Stats) {
	if m.cfg.Size != p.Size() {
		panic("Analyze: wrong size")
	}
	for i, v := range m.history {
		m.history[i] = v / 2
	}
	var cancel int32
	m.cancel = &cancel
	go func() {
		<-ctx.Done()
		atomic.StoreInt32(&cancel, 1)
	}()

	var seed = m.cfg.Seed
	if seed == 0 {
		seed = time.Now().Unix()
	}
	m.rand = rand.New(rand.NewSource(seed))
	if m.cfg.Debug > 0 {
		log.Printf("start search ply=%d color=%s seed=%d",
			p.MoveNumber(), p.ToMove(), seed)
	}
	deadline, limited := ctx.Deadline()

	var next []tak.Move
	ms := make([]tak.Move, 0, maxDepth)
	var v int64
	top := time.Now()
	var prevEval uint64
	var branchSum uint64
	base := 0
	te := m.ttGet(p.Hash())
	if te != nil && te.bound == exactBound {
		base = te.depth
		ms = append(ms[:0], te.m)
	}

	for i := 1; i+base <= m.cfg.Depth; i++ {
		m.st = Stats{Depth: i + base}
		start := time.Now()
		next, v = m.minimax(p, 0, i+base, ms, MinEval-1, MaxEval+1)
		if next == nil || atomic.LoadInt32(m.cancel) != 0 {
			break
		}
		ms = append(ms[:0], next...)
		timeUsed := time.Now().Sub(top)
		timeMove := time.Now().Sub(start)
		if m.cfg.Debug > 0 {
			log.Printf("[minimax] deepen: depth=%d val=%d pv=%s time=%s total=%s evaluated=%d tt=%d/%d branch=%d",
				base+i, v, formatpv(ms),
				timeMove,
				timeUsed,
				m.st.Evaluated,
				m.st.TTShortcut,
				m.st.TTHits,
				m.st.Evaluated/(prevEval+1),
			)
		}
		if m.cfg.Debug > 1 {
			log.Printf("[minimax]  stats: visited=%d scout=%d evaluated=%d null=%d/%d cut=%d cut0=%d(%2.2f) cut1=%d(%2.2f) m/cut=%2.2f m/ms=%f all=%d research=%d",
				m.st.Visited,
				m.st.Scout,
				m.st.Evaluated,
				m.st.NullCut,
				m.st.NullSearch,
				m.st.CutNodes,
				m.st.Cut0,
				float64(m.st.Cut0)/float64(m.st.CutNodes+1),
				m.st.Cut1,
				float64(m.st.Cut0+m.st.Cut1)/float64(m.st.CutNodes+1),
				float64(m.st.CutSearch)/float64(m.st.CutNodes-m.st.Cut0-m.st.Cut1+1),
				float64(m.st.Visited+m.st.Evaluated)/float64(timeMove.Seconds()*1000),
				m.st.AllNodes,
				m.st.ReSearch)
		}
		if i > 1 {
			branchSum += m.st.Evaluated / (prevEval + 1)
		}
		prevEval = m.st.Evaluated
		if v > WinThreshold || v < -WinThreshold {
			break
		}
		if limited && i+base != m.cfg.Depth {
			var branch uint64
			if i > 2 {
				// conservatively multiply by 2 to
				// account for the bimodal branching
				// factor
				branch = 2 * branchSum / uint64(i-1)
			} else {
				// conservative estimate if we haven't
				// run enough plies to have one
				// yet. This can matter if the table
				// returns a deep move
				branch = 20
			}
			estimate := time.Now().Add(time.Now().Sub(start) * time.Duration(branch))
			if estimate.After(deadline) {
				if m.cfg.Debug > 0 {
					log.Printf("[minimax] time cutoff: depth=%d used=%s estimate=%s",
						i, timeUsed, estimate.Sub(top))
				}
				break
			}
		}
	}
	return ms, v, m.st
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	ply, depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		ai.st.Evaluated++
		if over {
			ai.st.Terminal++
		}
		return nil, ai.evaluate(ai, p)
	}

	ai.st.Visited++
	if β == α+1 {
		ai.st.Scout++
	}

	te := ai.ttGet(p.Hash())
	if te != nil {
		ai.st.TTHits++
		teSuffices := false
		if te.depth >= depth {
			if te.bound == exactBound ||
				(te.value < α && te.bound == upperBound) ||
				(te.value > β && te.bound == lowerBound) {
				teSuffices = true
			}
		}

		if te.bound == exactBound &&
			(te.value > WinThreshold || te.value < -WinThreshold) {
			teSuffices = true
		}
		if teSuffices {
			_, e := p.MovePreallocated(&te.m, ai.stack[ply].p)
			if e == nil {
				ai.st.TTShortcut++
				ai.stack[ply].pv[0] = te.m
				return ai.stack[ply].pv[:1], te.value
			}
			te = nil
		}
	}

	if β == α+1 && ai.nullMoveOK(ply, depth, p) {
		ai.stack[ply].m = tak.Move{Type: tak.Pass}
		child, e := p.MovePreallocated(&ai.stack[ply].m, ai.stack[ply].p)
		if e == nil {
			ai.st.NullSearch++
			_, v := ai.minimax(child, ply+1, depth-3, nil, -α-1, -α)
			v = -v
			if v >= β {
				ai.st.NullCut++
				return nil, v
			}
		}
	}

	// As of 1.6.2, Go's escape analysis can't tell that a
	// stack-allocated object here doesn't escape. So we force it
	// into our manual stack.
	mg := &ai.stack[ply].mg
	*mg = moveGenerator{
		ai:    ai,
		ply:   ply,
		depth: depth,
		p:     p,
		te:    te,
		pv:    pv,
	}

	best := ai.stack[ply].pv[:0]
	best = append(best, pv...)
	improved := false
	var i int
	for m, child := mg.Next(); child != nil; m, child = mg.Next() {
		i++
		var ms []tak.Move
		var newpv []tak.Move
		var v int64
		if len(best) != 0 {
			newpv = best[1:]
		}
		ai.stack[ply].m = m
		if i > 1 {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -α-1, -α)
			if -v > α && -v < β {
				ai.st.ReSearch++
				ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
			}
		} else {
			ms, v = ai.minimax(child, ply+1, depth-1, newpv, -β, -α)
		}
		v = -v

		if len(best) == 0 {
			best = append(best[:0], m)
			best = append(best, ms...)
		}
		if v > α {
			improved = true
			best = append(best[:0], m)
			best = append(best, ms...)
			α = v
			if α >= β {
				ai.st.CutNodes++
				switch i {
				case 1:
					ai.st.Cut0++
				case 2:
					ai.st.Cut1++
				default:
					ai.st.CutSearch += uint64(i + 1)
				}
				ai.history[m.Hash()] += (1 << uint(depth))
				if ply > 0 {
					ai.response[ai.stack[ply-1].m.Hash()] = m
				}
				if ai.cfg.Debug > 3 && i > 20 && depth >= 3 {
					var tm tak.Move
					td := 0
					if te != nil {
						tm = te.m
						td = te.depth
					}
					log.Printf("[minimax] late cutoff depth=%d m=%d pv=%s te=%d:%s killer=%s pos=%q",
						depth, i, formatpv(pv), td, ptn.FormatMove(&tm), ptn.FormatMove(&m), ptn.FormatTPS(p),
					)
				}
				break
			}
		}
		if atomic.LoadInt32(ai.cancel) != 0 {
			return nil, 0
		}
	}

	if te = ai.ttPut(p.Hash()); te != nil {
		te.hash = p.Hash()
		te.depth = depth
		te.m = best[0]
		te.value = α
		if !improved {
			te.bound = upperBound
			ai.st.AllNodes++
		} else if α >= β {
			te.bound = lowerBound
		} else {
			te.bound = exactBound
		}
	}

	return best, α
}

func (ai *MinimaxAI) nullMoveOK(ply, depth int, p *tak.Position) bool {
	if ai.cfg.NoNullMove {
		return false
	}
	if ply == 0 || depth < 3 {
		return false
	}
	if ai.stack[ply-1].m.Type == tak.Pass {
		return false
	}
	if p.WhiteStones() < 3 || p.BlackStones() < 3 {
		return false
	}
	if bitboard.Popcount(p.White|p.Black)+3 >= len(p.Stacks) {
		return false
	}
	return true
}
<|endoftext|>