<commit_before>/* Go wrapper around Docker Compose, useful for integration testing.

	var composeYML =`
	test_mockserver:
	  container_name: ms
	  image: jamesdbloom/mockserver
	  ports:
	    - "10000:1080"
	    - "1090"
	test_postgres:
	  container_name: pg
	  image: postgres
	  ports:
	    - "5432"

	compose, err := compose.Start(composeYML, true, true)
	if err != nil {
		panic(err)
	}
	defer compose.Kill()
*/
package compose

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"log"
)

// Main type exported by the package, used to interact with a running Docker Compose configuration.
type Compose struct {
	FileName   string
	Containers map[string]*Container
}

var (
	logger = log.New(os.Stdout, "go-compose: ", log.LstdFlags)
	replaceEnvRegexp = regexp.MustCompile("\\$\\{[^\\}]+\\}")
	composeUpRegexp  = regexp.MustCompile("(?m:^docker start <- \\(u'(.*)'\\)$)")
)

// Starts a Docker Compose configuration.
// If forcePull is true, it attempts do pull newer versions of the images.
// If rmFirst is true, it attempts to kill and delete containers before starting new ones.
func Start(dockerComposeYML string, forcePull, rmFirst bool) (*Compose, error) {
	logger.Println("initializing...")
	dockerComposeYML = replaceEnv(dockerComposeYML)

	fName, err := writeTmp(dockerComposeYML)
	if err != nil {
		return nil, err
	}

	ids, err := startCompose(fName, forcePull, rmFirst)
	if err != nil {
		return nil, err
	}

	containers := make(map[string]*Container)

	for _, id := range ids {
		container, err := Inspect(id)
		if err != nil {
			return nil, err
		}
		if !container.State.Running {
			return nil, fmt.Errorf("compose: container '%v' is not running", container.Name)
		}
		containers[container.Name[1:]] = container
	}

	return &Compose{FileName: fName, Containers: containers}, nil
}

// Like Start, but panics on error.
func MustStart(dockerComposeYML string, forcePull, killFirst bool) *Compose {
	compose, err := Start(dockerComposeYML, forcePull, killFirst)
	if err != nil {
		panic(err)
	}
	return compose
}

// Kills any running containers for the current configuration.
func (c *Compose) Kill() error {
	logger.Println("killing containers...")
	if _, _, err := runCmd("docker-compose", "-f", c.FileName, "kill"); err == nil {
		logger.Println("containers killed")
		return nil
	} else {
		return fmt.Errorf("compose: error killing containers: %v", err)
	}
}

// Like Kill, but panics on error.
func (c *Compose) MustKill() {
	if err := c.Kill(); err != nil {
		panic(err)
	}
}

func replaceEnv(dockerComposeYML string) string {
	return replaceEnvRegexp.ReplaceAllStringFunc(dockerComposeYML, replaceEnvFunc)
}

func replaceEnvFunc(s string) string {
	return os.Getenv(strings.TrimSpace(s[2 : len(s)-1]))
}

func startCompose(fName string, forcePull, rmFirst bool) ([]string, error) {
	if forcePull {
		logger.Println("pulling images...")
		if _, _, err := runCmd("docker-compose", "-f", fName, "pull"); err != nil {
			return nil, fmt.Errorf("compose: error pulling images: %v", err)
		}
	}

	if rmFirst {
		logger.Println("removing stale containers...")
		_, _, err := runCmd("docker-compose", "-f", fName, "rm", "--force")
		if err != nil {
			return nil, fmt.Errorf("compose: error killing stale containers: %v", err)
		}
	}

	logger.Println("starting containers...")
	_, stderr, err := runCmd("docker-compose", "--verbose", "-f", fName, "up", "-d")
	if err != nil {
		return nil, fmt.Errorf("compose: error starting containers: %v", err)
	}
	logger.Println("containers started")

	matches := composeUpRegexp.FindAllStringSubmatch(stderr, -1)
	ids := make([]string, 0, len(matches))
	for _, match := range matches {
		ids = append(ids, match[1])
	}

	return ids, nil
}
<commit_msg>Improve documentation.<commit_after>/*
Go wrapper around Docker Compose, useful for integration testing.

	var composeYML =`
	test_mockserver:
	  container_name: ms
	  image: jamesdbloom/mockserver
	  ports:
	    - "10000:1080"
	    - "1090"
	test_postgres:
	  container_name: pg
	  image: postgres
	  ports:
	    - "5432"

	compose, err := compose.Start(composeYML, true, true)
	if err != nil {
		panic(err)
	}
	defer compose.Kill()
*/
package compose

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"log"
)

// Main type exported by the package, used to interact with a running Docker Compose configuration.
type Compose struct {
	FileName   string
	Containers map[string]*Container
}

var (
	logger = log.New(os.Stdout, "go-compose: ", log.LstdFlags)
	replaceEnvRegexp = regexp.MustCompile("\\$\\{[^\\}]+\\}")
	composeUpRegexp  = regexp.MustCompile("(?m:^docker start <- \\(u'(.*)'\\)$)")
)

// Starts a Docker Compose configuration.
// If forcePull is true, it attempts do pull newer versions of the images.
// If rmFirst is true, it attempts to kill and delete containers before starting new ones.
func Start(dockerComposeYML string, forcePull, rmFirst bool) (*Compose, error) {
	logger.Println("initializing...")
	dockerComposeYML = replaceEnv(dockerComposeYML)

	fName, err := writeTmp(dockerComposeYML)
	if err != nil {
		return nil, err
	}

	ids, err := startCompose(fName, forcePull, rmFirst)
	if err != nil {
		return nil, err
	}

	containers := make(map[string]*Container)

	for _, id := range ids {
		container, err := Inspect(id)
		if err != nil {
			return nil, err
		}
		if !container.State.Running {
			return nil, fmt.Errorf("compose: container '%v' is not running", container.Name)
		}
		containers[container.Name[1:]] = container
	}

	return &Compose{FileName: fName, Containers: containers}, nil
}

// Like Start, but panics on error.
func MustStart(dockerComposeYML string, forcePull, killFirst bool) *Compose {
	compose, err := Start(dockerComposeYML, forcePull, killFirst)
	if err != nil {
		panic(err)
	}
	return compose
}

// Kills any running containers for the current configuration.
func (c *Compose) Kill() error {
	logger.Println("killing containers...")
	if _, _, err := runCmd("docker-compose", "-f", c.FileName, "kill"); err == nil {
		logger.Println("containers killed")
		return nil
	} else {
		return fmt.Errorf("compose: error killing containers: %v", err)
	}
}

// Like Kill, but panics on error.
func (c *Compose) MustKill() {
	if err := c.Kill(); err != nil {
		panic(err)
	}
}

func replaceEnv(dockerComposeYML string) string {
	return replaceEnvRegexp.ReplaceAllStringFunc(dockerComposeYML, replaceEnvFunc)
}

func replaceEnvFunc(s string) string {
	return os.Getenv(strings.TrimSpace(s[2 : len(s)-1]))
}

func startCompose(fName string, forcePull, rmFirst bool) ([]string, error) {
	if forcePull {
		logger.Println("pulling images...")
		if _, _, err := runCmd("docker-compose", "-f", fName, "pull"); err != nil {
			return nil, fmt.Errorf("compose: error pulling images: %v", err)
		}
	}

	if rmFirst {
		logger.Println("removing stale containers...")
		_, _, err := runCmd("docker-compose", "-f", fName, "rm", "--force")
		if err != nil {
			return nil, fmt.Errorf("compose: error killing stale containers: %v", err)
		}
	}

	logger.Println("starting containers...")
	_, stderr, err := runCmd("docker-compose", "--verbose", "-f", fName, "up", "-d")
	if err != nil {
		return nil, fmt.Errorf("compose: error starting containers: %v", err)
	}
	logger.Println("containers started")

	matches := composeUpRegexp.FindAllStringSubmatch(stderr, -1)
	ids := make([]string, 0, len(matches))
	for _, match := range matches {
		ids = append(ids, match[1])
	}

	return ids, nil
}
<|endoftext|>


<commit_before>package cfitsio_test

import (
	"bytes"
	"strings"
	"testing"

	cfitsio "github.com/sbinet/go-cfitsio"
)

func TestOpenFile(t *testing.T) {
	const fname = "testdata/file001.fits"
	f, err := cfitsio.OpenFile(fname, cfitsio.ReadOnly)
	if err != nil {
		t.Fatalf("could not open FITS file [%s]: %v", fname, err)
	}
	defer f.Close()

	fmode, err := f.Mode()
	if err != nil {
		t.Fatalf("error mode: %v", err)
	}
	if fmode != cfitsio.ReadOnly {
		t.Fatalf("expected file-mode [%v]. got [%v]", cfitsio.ReadOnly, fmode)
	}

	name, err := f.Name()
	if err != nil {
		t.Fatalf("error name: %v", err)
	}
	if name != fname {
		t.Fatalf("expected file-name [%v]. got [%v]", fname, name)
	}

	furl, err := f.UrlType()
	if err != nil {
		t.Fatalf("error url: %v", err)
	}
	if furl != "file://" {
		t.Fatalf("expected file-url [%v]. got [%v]", "file://", furl)
	}

}

func TestHdu(t *testing.T) {
	const fname = "testdata/file001.fits"
	f, err := cfitsio.OpenFile(fname, cfitsio.ReadOnly)
	if err != nil {
		t.Fatalf("could not open FITS file [%s]: %v", fname, err)
	}
	defer f.Close()

	nhdus, err := f.NumHdus()
	if err != nil {
		t.Fatalf("error hdu: %v", err)
	}

	if nhdus != 2 {
		t.Fatalf("expected #hdus [%v]. got [%v]", 2, nhdus)
	}

	ihdu := f.HduNum()
	if ihdu != 1 {
		t.Fatalf("expected hdu number [%v]. got [%v]", 1, ihdu)
	}

	hdutype, err := f.HduType()
	if err != nil {
		t.Fatalf("error hdu-type: %v", err)
	}
	if hdutype != cfitsio.IMAGE_HDU {
		t.Fatalf("expected hdu type [%v]. got [%v]", cfitsio.IMAGE_HDU, hdutype)
	}

	buf := bytes.NewBuffer(nil)
	err = f.WriteHdu(buf)
	if err != nil {
		t.Fatalf("error write-hdu: %v", err)
	}
	hdu_tmp := strings.Trim(string(buf.Bytes()), " ")
	const hdu_ref = `SIMPLE  =                    T          / STANDARD FITS FORMAT (REV OCT 1981)   BITPIX  =                    8          / CHARACTER INFORMATION                 NAXIS   =                    0          / NO IMAGE DATA ARRAY PRESENT           EXTEND  =                    T          / THERE IS AN EXTENSION                 ORIGIN  = 'ESO     '                    / EUROPEAN SOUTHERN OBSERVATORY         OBJECT  = 'SNG - CAT.      '            / THE IDENTIFIER                        DATE    = '27/ 5/84'                    / DATE THIS TAPE WRITTEN DD/MM/YY       END`
	if hdu_ref != hdu_tmp {
		t.Fatalf("expected hdu-write:\n%v\ngot:\n%v", hdu_ref, hdu_tmp)
	}
}

// EOF
<commit_msg>test: test MovAbsHdu, MovRelHdu<commit_after>package cfitsio_test

import (
	"bytes"
	"strings"
	"testing"

	cfitsio "github.com/sbinet/go-cfitsio"
)

func TestOpenFile(t *testing.T) {
	const fname = "testdata/file001.fits"
	f, err := cfitsio.OpenFile(fname, cfitsio.ReadOnly)
	if err != nil {
		t.Fatalf("could not open FITS file [%s]: %v", fname, err)
	}
	defer f.Close()

	fmode, err := f.Mode()
	if err != nil {
		t.Fatalf("error mode: %v", err)
	}
	if fmode != cfitsio.ReadOnly {
		t.Fatalf("expected file-mode [%v]. got [%v]", cfitsio.ReadOnly, fmode)
	}

	name, err := f.Name()
	if err != nil {
		t.Fatalf("error name: %v", err)
	}
	if name != fname {
		t.Fatalf("expected file-name [%v]. got [%v]", fname, name)
	}

	furl, err := f.UrlType()
	if err != nil {
		t.Fatalf("error url: %v", err)
	}
	if furl != "file://" {
		t.Fatalf("expected file-url [%v]. got [%v]", "file://", furl)
	}

}

func TestHdu(t *testing.T) {
	const fname = "testdata/file001.fits"
	f, err := cfitsio.OpenFile(fname, cfitsio.ReadOnly)
	if err != nil {
		t.Fatalf("could not open FITS file [%s]: %v", fname, err)
	}
	defer f.Close()

	nhdus, err := f.NumHdus()
	if err != nil {
		t.Fatalf("error hdu: %v", err)
	}

	if nhdus != 2 {
		t.Fatalf("expected #hdus [%v]. got [%v]", 2, nhdus)
	}

	ihdu := f.HduNum()
	if ihdu != 1 {
		t.Fatalf("expected hdu number [%v]. got [%v]", 1, ihdu)
	}

	hdutype, err := f.HduType()
	if err != nil {
		t.Fatalf("error hdu-type: %v", err)
	}
	if hdutype != cfitsio.IMAGE_HDU {
		t.Fatalf("expected hdu type [%v]. got [%v]", cfitsio.IMAGE_HDU, hdutype)
	}

	buf := bytes.NewBuffer(nil)
	err = f.WriteHdu(buf)
	if err != nil {
		t.Fatalf("error write-hdu: %v", err)
	}
	hdu_tmp := strings.Trim(string(buf.Bytes()), " ")
	const hdu_ref = `SIMPLE  =                    T          / STANDARD FITS FORMAT (REV OCT 1981)   BITPIX  =                    8          / CHARACTER INFORMATION                 NAXIS   =                    0          / NO IMAGE DATA ARRAY PRESENT           EXTEND  =                    T          / THERE IS AN EXTENSION                 ORIGIN  = 'ESO     '                    / EUROPEAN SOUTHERN OBSERVATORY         OBJECT  = 'SNG - CAT.      '            / THE IDENTIFIER                        DATE    = '27/ 5/84'                    / DATE THIS TAPE WRITTEN DD/MM/YY       END`
	if hdu_ref != hdu_tmp {
		t.Fatalf("expected hdu-write:\n%v\ngot:\n%v", hdu_ref, hdu_tmp)
	}

	// move to next header
	_, err = f.MovRelHdu(1)
	if err != nil {
		t.Fatalf("error next-hdu: %v", err)
	}

	ihdu = f.HduNum()
	if ihdu != 2 {
		t.Fatalf("expected hdu number [%v]. got [%v]", 2, ihdu)
	}

	hdutype, err = f.HduType()
	if err != nil {
		t.Fatalf("error hdu-type: %v", err)
	}
	if hdutype != cfitsio.ASCII_TBL {
		t.Fatalf("expected hdu type [%v]. got [%v]", cfitsio.ASCII_TBL, hdutype)
	}

	// move to first header
	_, err = f.MovAbsHdu(1)
	if err != nil {
		t.Fatalf("error abs-hdu: %v", err)
	}

	ihdu = f.HduNum()
	if ihdu != 1 {
		t.Fatalf("expected hdu number [%v]. got [%v]", 1, ihdu)
	}

	hdutype, err = f.HduType()
	if err != nil {
		t.Fatalf("error hdu-type: %v", err)
	}
	if hdutype != cfitsio.IMAGE_HDU {
		t.Fatalf("expected hdu type [%v]. got [%v]", cfitsio.IMAGE_HDU, hdutype)
	}

	// move to second header
	_, err = f.MovAbsHdu(2)
	if err != nil {
		t.Fatalf("error abs-hdu: %v", err)
	}

	ihdu = f.HduNum()
	if ihdu != 2 {
		t.Fatalf("expected hdu number [%v]. got [%v]", 2, ihdu)
	}

	hdutype, err = f.HduType()
	if err != nil {
		t.Fatalf("error hdu-type: %v", err)
	}
	if hdutype != cfitsio.ASCII_TBL {
		t.Fatalf("expected hdu type [%v]. got [%v]", cfitsio.ASCII_TBL, hdutype)
	}

	// move to non-existent header
	_, err = f.MovAbsHdu(3)
	if err == nil {
		t.Fatalf("expected error")
	}
	if err != cfitsio.END_OF_FILE {
		t.Fatalf("expected error [%v]. got [%v]", cfitsio.END_OF_FILE, err)
	}

	// move to non-existent header
	_, err = f.MovAbsHdu(0)
	if err == nil {
		t.Fatalf("expected error")
	}
	if err != cfitsio.BAD_HDU_NUM {
		t.Fatalf("expected error [%v]. got [%v]", cfitsio.BAD_HDU_NUM, err)
	}
}

// EOF
<|endoftext|>


<commit_before>// package fronted provides a client and server for domain-fronted proxying
// using enproxy proxies.
package fronted

import (
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"encoding/asn1"
	"github.com/getlantern/connpool"
	"github.com/getlantern/enproxy"
	"github.com/getlantern/golog"
	"github.com/getlantern/proxy"
	"github.com/getlantern/tlsdialer"
)

const (
	CONNECT = "CONNECT" // HTTP CONNECT method
)

var (
	log = golog.LoggerFor("fronted")

	// Cutoff for logging warnings about a dial having taken a long time.
	longDialLimit = 10 * time.Second

	// idleTimeout needs to be small enough that we stop using connections
	// before the upstream server/CDN closes them itself.
	// TODO: make this configurable.
	idleTimeout = 10 * time.Second
)

// Dialer is a domain-fronted proxy.Dialer.
type Dialer interface {
	proxy.Dialer

	// HttpClientUsing creates a simple domain-fronted HTTP client using the
	// specified Masquerade.
	HttpClientUsing(masquerade *Masquerade) *http.Client

	// NewDirectDomainFronter creates an HttpClient that domain-fronts but instead
	// of using enproxy proxies routes to the destination server directly from the
	// CDN. This is useful for web properties registered on the CDN itself, for
	// example geo.getiantem.org.
	//
	// Note - the connection is already encrypted by domain-fronting, so this
	// client should only be used to make HTTP requests. Using it for HTTPS
	// requests will result in an error.
	NewDirectDomainFronter() *http.Client
}

// Config captures the configuration of a domain-fronted dialer.
type Config struct {
	// Host: the host (e.g. getiantem.org)
	Host string

	// Port: the port (e.g. 443)
	Port int

	// Masquerades: the Masquerades to use when domain-fronting. These will be
	// verified when the Dialer starts.
	Masquerades []*Masquerade

	// MaxMasquerades: the maximum number of masquerades to verify. If 0,
	// the masquerades are uncapped.
	MaxMasquerades int

	// PoolSize: if greater than 0, outbound connections will be pooled in an
	// eagerly loading connection pool. This can reduce latency when using
	// enproxy.
	PoolSize int

	// InsecureSkipVerify: if true, server's certificate is not verified.
	InsecureSkipVerify bool

	// RootCAs: optional CertPool specifying the root CAs to use for verifying
	// servers
	RootCAs *x509.CertPool

	// BufferRequests: if true, requests to the proxy will be buffered and sent
	// with identity encoding.  If false, they'll be streamed with chunked
	// encoding.
	BufferRequests bool

	// DialTimeoutMillis: how long to wait on dialing server before timing out
	// (defaults to 20 seconds)
	DialTimeoutMillis int

	// RedialAttempts: number of times to try redialing. The total number of
	// dial attempts will be 1 + RedialAttempts.
	RedialAttempts int

	// Weight: relative weight versus other servers (for round-robin)
	Weight int

	// QOS: relative quality of service offered. Should be >= 0, with higher
	// values indicating higher QOS.
	QOS int

	// OnDial: optional callback that gets invoked whenever we dial the server.
	// The Conn and error returned from this callback will be used in lieu of
	// the originals.
	OnDial func(conn net.Conn, err error) (net.Conn, error)

	// OnDialStats is an optional callback that will get called on every dial to
	// the server to report stats on what was dialed and how long each step
	// took.
	OnDialStats func(success bool, domain, addr string, resolutionTime, connectTime, handshakeTime time.Duration)
}

// dialer implements the proxy.Dialer interface by dialing domain-fronted
// servers.
type dialer struct {
	Config
	masquerades     *verifiedMasqueradeSet
	connPool        connpool.Pool
	enproxyConfig   *enproxy.Config
	tlsConfigs      map[string]*tls.Config
	tlsConfigsMutex sync.Mutex
}

// NewDialer creates a new Dialer for the given Config.
// WARNING - depending on configuration, this Dialer may contain a connection
// pool and/or a set of Masquerades that will leak resources.  Make sure to call
// Close() to clean these up when the Dialer is no longer in use.
func NewDialer(cfg Config) Dialer {
	d := &dialer{
		Config:     cfg,
		tlsConfigs: make(map[string]*tls.Config),
	}
	if d.Masquerades != nil {
		if d.MaxMasquerades == 0 {
			d.MaxMasquerades = len(d.Masquerades)
		}
		d.masquerades = d.verifiedMasquerades()
	}
	if cfg.PoolSize > 0 {
		d.connPool = connpool.New(connpool.Config{
			Size:         cfg.PoolSize,
			ClaimTimeout: idleTimeout,
			Dial:         d.dialServer,
		})
	}
	d.enproxyConfig = d.enproxyConfigWith(func(addr string) (net.Conn, error) {
		var conn net.Conn
		var err error
		if d.connPool != nil {
			conn, err = d.connPool.Get()
		} else {
			conn, err = d.dialServer()
		}
		if d.OnDial != nil {
			conn, err = d.OnDial(conn, err)
		}
		return conn, err
	})
	return d
}

// Dial dials upstream using domain-fronting.
func (d *dialer) Dial(network, addr string) (net.Conn, error) {
	if !strings.Contains(network, "tcp") {
		return nil, fmt.Errorf("Protocol %s is not supported, only tcp is supported", network)
	}

	return enproxy.Dial(addr, d.enproxyConfig)
}

// Close closes the dialer, in particular closing the underlying connection
// pool.
func (d *dialer) Close() error {
	if d.connPool != nil {
		// We close the connPool on a goroutine so as not to wait for Close to finish
		go d.connPool.Close()
	}
	if d.masquerades != nil {
		go d.masquerades.stop()
	}
	return nil
}

func (d *dialer) HttpClientUsing(masquerade *Masquerade) *http.Client {
	enproxyConfig := d.enproxyConfigWith(func(addr string) (net.Conn, error) {
		return d.dialServerWith(masquerade)
	})

	return &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				return enproxy.Dial(addr, enproxyConfig)
			},
		},
	}
}

func (d *dialer) NewDirectDomainFronter() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				return d.dialServer()
			},
		},
	}
}

func (d *dialer) enproxyConfigWith(dialProxy func(addr string) (net.Conn, error)) *enproxy.Config {
	return &enproxy.Config{
		DialProxy: dialProxy,
		NewRequest: func(upstreamHost string, method string, body io.Reader) (req *http.Request, err error) {
			if upstreamHost == "" {
				// No specific host requested, use configured one
				upstreamHost = d.Host
			}
			return http.NewRequest(method, "http://"+upstreamHost+"/", body)
		},
		BufferRequests: d.BufferRequests,
		IdleTimeout:    idleTimeout, // TODO: make this configurable
	}
}

func (d *dialer) dialServer() (net.Conn, error) {
	var masquerade *Masquerade
	if d.masquerades != nil {
		masquerade = d.masquerades.nextVerified()
	}
	return d.dialServerWith(masquerade)
}

func (d *dialer) dialServerWith(masquerade *Masquerade) (net.Conn, error) {
	dialTimeout := time.Duration(d.DialTimeoutMillis) * time.Millisecond
	if dialTimeout == 0 {
		dialTimeout = 20 * time.Second
	}

	// Note - we need to suppress the sending of the ServerName in the client
	// handshake to make host-spoofing work with Fastly.  If the client Hello
	// includes a server name, Fastly checks to make sure that this matches the
	// Host header in the HTTP request and if they don't match, it returns
	// a 400 Bad Request error.
	sendServerNameExtension := false

	cwt, err := tlsdialer.DialForTimings(
		&net.Dialer{
			Timeout: dialTimeout,
		},
		"tcp",
		d.addressForServer(masquerade),
		sendServerNameExtension,
		d.tlsConfig(masquerade))

	if d.OnDialStats != nil {
		domain := ""
		if masquerade != nil {
			domain = masquerade.Domain
		}

		resultAddr := ""
		if err == nil {
			resultAddr = cwt.Conn.RemoteAddr().String()
		}

		d.OnDialStats(err == nil, domain, resultAddr, cwt.ResolutionTime, cwt.ConnectTime, cwt.HandshakeTime)
	}

	if err != nil && masquerade != nil {
		err = fmt.Errorf("Unable to dial masquerade %s: %s", masquerade.Domain, err)
	}
	return cwt.Conn, err
}

// Get the address to dial for reaching the server
func (d *dialer) addressForServer(masquerade *Masquerade) string {
	return fmt.Sprintf("%s:%d", d.serverHost(masquerade), d.Port)
}

func (d *dialer) serverHost(masquerade *Masquerade) string {
	serverHost := d.Host
	if masquerade != nil {
		if masquerade.IpAddress != "" {
			serverHost = masquerade.IpAddress
		} else if masquerade.Domain != "" {
			serverHost = masquerade.Domain
		}
	}
	return serverHost
}

// tlsInfo is a temporary function that could help catching a bug. See this
// related PR: https://github.com/getlantern/lantern/issues/2398
func (d *dialer) tlsInfo(masquerade *Masquerade) string {
	var data []string

	serverName := d.Host

	if masquerade != nil {
		serverName = masquerade.Domain
	}

	var certpool []string
	subjects := d.RootCAs.Subjects()

	var dest pkix.RDNSequence

	for i, _ := range subjects {
		_, err := asn1.Unmarshal(subjects[i], &dest)
		if err != nil {
			certpool = append(certpool, fmt.Sprintf("Error[%d]: %q", i, err.Error()))
		} else {
			certpool = append(certpool, fmt.Sprintf("RDNSequence[%d]: %q", i, dest))
		}
	}

	data = append(data, fmt.Sprintf("Insecure Skip Verify: %v", d.InsecureSkipVerify))
	data = append(data, fmt.Sprintf("Host: %v", d.Host))
	data = append(data, fmt.Sprintf("Masquerade Domain: %v", masquerade.Domain))
	data = append(data, fmt.Sprintf("Server Name: %v", serverName))
	data = append(data, fmt.Sprintf("x509 cert pool subjects: %#s", strings.Join(certpool, " | ")))

	return strings.Join(data, ", ")
}

// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *dialer) tlsConfig(masquerade *Masquerade) *tls.Config {
	d.tlsConfigsMutex.Lock()
	defer d.tlsConfigsMutex.Unlock()

	serverName := d.Host
	if masquerade != nil {
		serverName = masquerade.Domain
	}
	tlsConfig := d.tlsConfigs[serverName]
	if tlsConfig == nil {
		tlsConfig = &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(1000),
			InsecureSkipVerify: d.InsecureSkipVerify,
			ServerName:         serverName,
			RootCAs:            d.RootCAs,
		}
		d.tlsConfigs[serverName] = tlsConfig
	}

	return tlsConfig
}
<commit_msg>make the result of NewDirectDomainFronter() accept https URLs<commit_after>// package fronted provides a client and server for domain-fronted proxying
// using enproxy proxies.
package fronted

import (
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	"encoding/asn1"
	"github.com/getlantern/connpool"
	"github.com/getlantern/enproxy"
	"github.com/getlantern/golog"
	"github.com/getlantern/proxy"
	"github.com/getlantern/tlsdialer"
)

const (
	CONNECT = "CONNECT" // HTTP CONNECT method
)

var (
	log = golog.LoggerFor("fronted")

	// Cutoff for logging warnings about a dial having taken a long time.
	longDialLimit = 10 * time.Second

	// idleTimeout needs to be small enough that we stop using connections
	// before the upstream server/CDN closes them itself.
	// TODO: make this configurable.
	idleTimeout = 10 * time.Second
)

// Dialer is a domain-fronted proxy.Dialer.
type Dialer interface {
	proxy.Dialer

	// HttpClientUsing creates a simple domain-fronted HTTP client using the
	// specified Masquerade.
	HttpClientUsing(masquerade *Masquerade) *http.Client

	// NewDirectDomainFronter creates an HttpClient that domain-fronts but instead
	// of using enproxy proxies routes to the destination server directly from the
	// CDN. This is useful for web properties registered on the CDN itself, for
	// example geo.getiantem.org.
	NewDirectDomainFronter() *http.Client
}

// Config captures the configuration of a domain-fronted dialer.
type Config struct {
	// Host: the host (e.g. getiantem.org)
	Host string

	// Port: the port (e.g. 443)
	Port int

	// Masquerades: the Masquerades to use when domain-fronting. These will be
	// verified when the Dialer starts.
	Masquerades []*Masquerade

	// MaxMasquerades: the maximum number of masquerades to verify. If 0,
	// the masquerades are uncapped.
	MaxMasquerades int

	// PoolSize: if greater than 0, outbound connections will be pooled in an
	// eagerly loading connection pool. This can reduce latency when using
	// enproxy.
	PoolSize int

	// InsecureSkipVerify: if true, server's certificate is not verified.
	InsecureSkipVerify bool

	// RootCAs: optional CertPool specifying the root CAs to use for verifying
	// servers
	RootCAs *x509.CertPool

	// BufferRequests: if true, requests to the proxy will be buffered and sent
	// with identity encoding.  If false, they'll be streamed with chunked
	// encoding.
	BufferRequests bool

	// DialTimeoutMillis: how long to wait on dialing server before timing out
	// (defaults to 20 seconds)
	DialTimeoutMillis int

	// RedialAttempts: number of times to try redialing. The total number of
	// dial attempts will be 1 + RedialAttempts.
	RedialAttempts int

	// Weight: relative weight versus other servers (for round-robin)
	Weight int

	// QOS: relative quality of service offered. Should be >= 0, with higher
	// values indicating higher QOS.
	QOS int

	// OnDial: optional callback that gets invoked whenever we dial the server.
	// The Conn and error returned from this callback will be used in lieu of
	// the originals.
	OnDial func(conn net.Conn, err error) (net.Conn, error)

	// OnDialStats is an optional callback that will get called on every dial to
	// the server to report stats on what was dialed and how long each step
	// took.
	OnDialStats func(success bool, domain, addr string, resolutionTime, connectTime, handshakeTime time.Duration)
}

// dialer implements the proxy.Dialer interface by dialing domain-fronted
// servers.
type dialer struct {
	Config
	masquerades     *verifiedMasqueradeSet
	connPool        connpool.Pool
	enproxyConfig   *enproxy.Config
	tlsConfigs      map[string]*tls.Config
	tlsConfigsMutex sync.Mutex
}

// NewDialer creates a new Dialer for the given Config.
// WARNING - depending on configuration, this Dialer may contain a connection
// pool and/or a set of Masquerades that will leak resources.  Make sure to call
// Close() to clean these up when the Dialer is no longer in use.
func NewDialer(cfg Config) Dialer {
	d := &dialer{
		Config:     cfg,
		tlsConfigs: make(map[string]*tls.Config),
	}
	if d.Masquerades != nil {
		if d.MaxMasquerades == 0 {
			d.MaxMasquerades = len(d.Masquerades)
		}
		d.masquerades = d.verifiedMasquerades()
	}
	if cfg.PoolSize > 0 {
		d.connPool = connpool.New(connpool.Config{
			Size:         cfg.PoolSize,
			ClaimTimeout: idleTimeout,
			Dial:         d.dialServer,
		})
	}
	d.enproxyConfig = d.enproxyConfigWith(func(addr string) (net.Conn, error) {
		var conn net.Conn
		var err error
		if d.connPool != nil {
			conn, err = d.connPool.Get()
		} else {
			conn, err = d.dialServer()
		}
		if d.OnDial != nil {
			conn, err = d.OnDial(conn, err)
		}
		return conn, err
	})
	return d
}

// Dial dials upstream using domain-fronting.
func (d *dialer) Dial(network, addr string) (net.Conn, error) {
	if !strings.Contains(network, "tcp") {
		return nil, fmt.Errorf("Protocol %s is not supported, only tcp is supported", network)
	}

	return enproxy.Dial(addr, d.enproxyConfig)
}

// Close closes the dialer, in particular closing the underlying connection
// pool.
func (d *dialer) Close() error {
	if d.connPool != nil {
		// We close the connPool on a goroutine so as not to wait for Close to finish
		go d.connPool.Close()
	}
	if d.masquerades != nil {
		go d.masquerades.stop()
	}
	return nil
}

func (d *dialer) HttpClientUsing(masquerade *Masquerade) *http.Client {
	enproxyConfig := d.enproxyConfigWith(func(addr string) (net.Conn, error) {
		return d.dialServerWith(masquerade)
	})

	return &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				return enproxy.Dial(addr, enproxyConfig)
			},
		},
	}
}

type DirectDomainTransport struct {
	orig *http.Transport
}

func (ddf *DirectDomainTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// The connection is already encrypted by domain fronting.  We need to rewrite URLs starting
	// with "https://" to "http://", lest we get an error for doubling up on TLS.

	// The RoundTrip interface requires that we not modify the memory in the request, so we just
	// create a new one. Note this currently doesn't support request bodies.
	normalized := replacePrefix(req.URL.String(), "https://", "http://")
	norm, err := http.NewRequest(req.Method, normalized, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to construct request for url '%s' with error '%v'", normalized, err)
	}
	return ddf.orig.RoundTrip(norm)
}

func replacePrefix(s string, old string, new string) string {
	if strings.HasPrefix(s, old) {
		return new + strings.TrimPrefix(s, old)
	} else {
		return s
	}
}

func (ddf *DirectDomainTransport) CancelRequest(req *http.Request) {
	ddf.orig.CancelRequest(req)
}

func (ddf *DirectDomainTransport) CloseIdleConnections() {
	ddf.orig.CloseIdleConnections()
}

// Creates a new http.Client that does direct domain fronting.
func (d *dialer) NewDirectDomainFronter() *http.Client {
	return &http.Client{
		Transport: &DirectDomainTransport{
			orig: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					log.Debugf("Dialing server with direct domain fronter")
					return d.dialServer()
				},
			},
		},
	}
}

func (d *dialer) enproxyConfigWith(dialProxy func(addr string) (net.Conn, error)) *enproxy.Config {
	return &enproxy.Config{
		DialProxy: dialProxy,
		NewRequest: func(upstreamHost string, method string, body io.Reader) (req *http.Request, err error) {
			if upstreamHost == "" {
				// No specific host requested, use configured one
				upstreamHost = d.Host
			}
			return http.NewRequest(method, "http://"+upstreamHost+"/", body)
		},
		BufferRequests: d.BufferRequests,
		IdleTimeout:    idleTimeout, // TODO: make this configurable
	}
}

func (d *dialer) dialServer() (net.Conn, error) {
	var masquerade *Masquerade
	if d.masquerades != nil {
		masquerade = d.masquerades.nextVerified()
	}
	return d.dialServerWith(masquerade)
}

func (d *dialer) dialServerWith(masquerade *Masquerade) (net.Conn, error) {
	dialTimeout := time.Duration(d.DialTimeoutMillis) * time.Millisecond
	if dialTimeout == 0 {
		dialTimeout = 20 * time.Second
	}

	// Note - we need to suppress the sending of the ServerName in the client
	// handshake to make host-spoofing work with Fastly.  If the client Hello
	// includes a server name, Fastly checks to make sure that this matches the
	// Host header in the HTTP request and if they don't match, it returns
	// a 400 Bad Request error.
	sendServerNameExtension := false

	cwt, err := tlsdialer.DialForTimings(
		&net.Dialer{
			Timeout: dialTimeout,
		},
		"tcp",
		d.addressForServer(masquerade),
		sendServerNameExtension,
		d.tlsConfig(masquerade))

	if d.OnDialStats != nil {
		domain := ""
		if masquerade != nil {
			domain = masquerade.Domain
		}

		resultAddr := ""
		if err == nil {
			resultAddr = cwt.Conn.RemoteAddr().String()
		}

		d.OnDialStats(err == nil, domain, resultAddr, cwt.ResolutionTime, cwt.ConnectTime, cwt.HandshakeTime)
	}

	if err != nil && masquerade != nil {
		err = fmt.Errorf("Unable to dial masquerade %s: %s", masquerade.Domain, err)
	}
	return cwt.Conn, err
}

// Get the address to dial for reaching the server
func (d *dialer) addressForServer(masquerade *Masquerade) string {
	return fmt.Sprintf("%s:%d", d.serverHost(masquerade), d.Port)
}

func (d *dialer) serverHost(masquerade *Masquerade) string {
	serverHost := d.Host
	if masquerade != nil {
		if masquerade.IpAddress != "" {
			serverHost = masquerade.IpAddress
		} else if masquerade.Domain != "" {
			serverHost = masquerade.Domain
		}
	}
	return serverHost
}

// tlsInfo is a temporary function that could help catching a bug. See this
// related PR: https://github.com/getlantern/lantern/issues/2398
func (d *dialer) tlsInfo(masquerade *Masquerade) string {
	var data []string

	serverName := d.Host

	if masquerade != nil {
		serverName = masquerade.Domain
	}

	var certpool []string
	subjects := d.RootCAs.Subjects()

	var dest pkix.RDNSequence

	for i, _ := range subjects {
		_, err := asn1.Unmarshal(subjects[i], &dest)
		if err != nil {
			certpool = append(certpool, fmt.Sprintf("Error[%d]: %q", i, err.Error()))
		} else {
			certpool = append(certpool, fmt.Sprintf("RDNSequence[%d]: %q", i, dest))
		}
	}

	data = append(data, fmt.Sprintf("Insecure Skip Verify: %v", d.InsecureSkipVerify))
	data = append(data, fmt.Sprintf("Host: %v", d.Host))
	data = append(data, fmt.Sprintf("Masquerade Domain: %v", masquerade.Domain))
	data = append(data, fmt.Sprintf("Server Name: %v", serverName))
	data = append(data, fmt.Sprintf("x509 cert pool subjects: %#s", strings.Join(certpool, " | ")))

	return strings.Join(data, ", ")
}

// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *dialer) tlsConfig(masquerade *Masquerade) *tls.Config {
	d.tlsConfigsMutex.Lock()
	defer d.tlsConfigsMutex.Unlock()

	serverName := d.Host
	if masquerade != nil {
		serverName = masquerade.Domain
	}
	tlsConfig := d.tlsConfigs[serverName]
	if tlsConfig == nil {
		tlsConfig = &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(1000),
			InsecureSkipVerify: d.InsecureSkipVerify,
			ServerName:         serverName,
			RootCAs:            d.RootCAs,
		}
		d.tlsConfigs[serverName] = tlsConfig
	}

	return tlsConfig
}
<|endoftext|>


<commit_before>package streamer

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/pkg/profile"
)

type benchFn func(b *testing.B, strm Mux)

func cbormuxLooper(b *testing.B, fn benchFn) {
	os.Remove("./logfile")
	for i := 0; i < b.N; i++ {
		strm := CborFileMux("./logfile")
		fn(b, strm)
		strm.(*CborMux).file.Close() // TODO inability to do this in api is mistake
		os.Remove("./logfile")
	}
}

// Writing : just bench the fsyncs of scribbling things down.

func BenchmarkCbormuxWriting(b *testing.B) {
	cbormuxLooper(b, _BenchmarkWriting)
}
func _BenchmarkWriting(b *testing.B, strm Mux) {
	a1 := strm.Appender(1)
	for i := 0; i < 16; i++ { // unsurprisingly, if you wiggle this const, it's linear
		a1.Write([]byte("asdf"))
	}
	a1.Close()
}

// WriteReadRead : all writes are done, then all reads are done (twice).

func BenchmarkCbormuxWriteReadRead(b *testing.B) {
	cbormuxLooper(b, _BenchmarkWriteReadRead)
}
func _BenchmarkWriteReadRead(b *testing.B, strm Mux) {
	a1 := strm.Appender(1)
	for i := 0; i < 16; i++ {
		a1.Write([]byte("asdf"))
	}
	a1.Close()
	ioutil.ReadAll(strm.Reader(1))
	ioutil.ReadAll(strm.Reader(1))
}

// PlayingTagInterleaved : writes (once) and reads (again twice) are interleaved -- but paired; reads never block.

func BenchmarkCbormuxPlayingTagInterleaved(b *testing.B) {
	cbormuxLooper(b, _BenchmarkPlayingTagInterleaved)
}
func _BenchmarkPlayingTagInterleaved(b *testing.B, strm Mux) {
	a1 := strm.Appender(1)
	r1 := strm.Reader(1)
	for i := 0; i < 16; i++ {
		a1.Write([]byte("asdf"))
		r1.Read(make([]byte, 4))
	}
	a1.Close()
}

// PlayingTagShuffled : what happens when writes come in from a goroutine, and we have reads in the bench routine?
// Reads are aligned, fwiw.

func BenchmarkCbormuxPlayingTagShuffled(b *testing.B) {
	cbormuxLooper(b, _BenchmarkPlayingTagShuffled)
}
func _BenchmarkPlayingTagShuffled(b *testing.B, strm Mux) {
	go func() {
		a1 := strm.Appender(1)
		for i := 0; i < 16; i++ {
			a1.Write([]byte("asdf"))
		}
		a1.Close()
	}()
	r1 := strm.Reader(1)
	for i := 0; i < 16; i++ {
		r1.Read(make([]byte, 4))
	}
	// block until eof (to avoid filedescriptor close race -- could also waitgroup the writer routine)
	r1.Read(make([]byte, 0))
}

// PlayingTagBlocking : what happens when writes come in from a *slower* goroutine, and we have reads in the bench routine?
// Reads are aligned, fwiw.
// The pause between writes is smallish: 10ms; but enough to check we're not going *totally* nuts polling for changes.

func BenchmarkCbormuxPlayingTagBlocking(b *testing.B) {
	defer profile.Start().Stop()
	cbormuxLooper(b, _BenchmarkPlayingTagBlocking)
}
func _BenchmarkPlayingTagBlocking(b *testing.B, strm Mux) {
	go func() {
		a1 := strm.Appender(1)
		for i := 0; i < 16; i++ {
			time.Sleep(10 * time.Millisecond)
			a1.Write([]byte("asdf"))
		}
		a1.Close()
	}()
	r1 := strm.Reader(1)
	for i := 0; i < 16; i++ {
		r1.Read(make([]byte, 4))
	}
	// block until eof (to avoid filedescriptor close race -- could also waitgroup the writer routine)
	r1.Read(make([]byte, 0))
}
<commit_msg>Sancheck: leaking goroutines?  No.<commit_after>package streamer

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"testing"
	"time"

	"github.com/pkg/profile"
)

type benchFn func(b *testing.B, strm Mux)

func cbormuxLooper(b *testing.B, fn benchFn) {
	os.Remove("./logfile")
	for i := 0; i < b.N; i++ {
		strm := CborFileMux("./logfile")
		fn(b, strm)
		strm.(*CborMux).file.Close() // TODO inability to do this in api is mistake
		os.Remove("./logfile")
	}
}

// Writing : just bench the fsyncs of scribbling things down.

func BenchmarkCbormuxWriting(b *testing.B) {
	cbormuxLooper(b, _BenchmarkWriting)
}
func _BenchmarkWriting(b *testing.B, strm Mux) {
	a1 := strm.Appender(1)
	for i := 0; i < 16; i++ { // unsurprisingly, if you wiggle this const, it's linear
		a1.Write([]byte("asdf"))
	}
	a1.Close()
}

// WriteReadRead : all writes are done, then all reads are done (twice).

func BenchmarkCbormuxWriteReadRead(b *testing.B) {
	cbormuxLooper(b, _BenchmarkWriteReadRead)
}
func _BenchmarkWriteReadRead(b *testing.B, strm Mux) {
	a1 := strm.Appender(1)
	for i := 0; i < 16; i++ {
		a1.Write([]byte("asdf"))
	}
	a1.Close()
	ioutil.ReadAll(strm.Reader(1))
	ioutil.ReadAll(strm.Reader(1))
}

// PlayingTagInterleaved : writes (once) and reads (again twice) are interleaved -- but paired; reads never block.

func BenchmarkCbormuxPlayingTagInterleaved(b *testing.B) {
	cbormuxLooper(b, _BenchmarkPlayingTagInterleaved)
}
func _BenchmarkPlayingTagInterleaved(b *testing.B, strm Mux) {
	a1 := strm.Appender(1)
	r1 := strm.Reader(1)
	for i := 0; i < 16; i++ {
		a1.Write([]byte("asdf"))
		r1.Read(make([]byte, 4))
	}
	a1.Close()
}

// PlayingTagShuffled : what happens when writes come in from a goroutine, and we have reads in the bench routine?
// Reads are aligned, fwiw.

func BenchmarkCbormuxPlayingTagShuffled(b *testing.B) {
	cbormuxLooper(b, _BenchmarkPlayingTagShuffled)
}
func _BenchmarkPlayingTagShuffled(b *testing.B, strm Mux) {
	go func() {
		a1 := strm.Appender(1)
		for i := 0; i < 16; i++ {
			a1.Write([]byte("asdf"))
		}
		a1.Close()
	}()
	r1 := strm.Reader(1)
	for i := 0; i < 16; i++ {
		r1.Read(make([]byte, 4))
	}
	// block until eof (to avoid filedescriptor close race -- could also waitgroup the writer routine)
	r1.Read(make([]byte, 0))
}

// PlayingTagBlocking : what happens when writes come in from a *slower* goroutine, and we have reads in the bench routine?
// Reads are aligned, fwiw.
// The pause between writes is smallish: 10ms; but enough to check we're not going *totally* nuts polling for changes.

func BenchmarkCbormuxPlayingTagBlocking(b *testing.B) {
	defer profile.Start().Stop()
	fmt.Printf("goroutines at start: %d\n", runtime.NumGoroutine())
	cbormuxLooper(b, _BenchmarkPlayingTagBlocking)
	fmt.Printf("goroutines at end:   %d\n", runtime.NumGoroutine())
}
func _BenchmarkPlayingTagBlocking(b *testing.B, strm Mux) {
	go func() {
		a1 := strm.Appender(1)
		for i := 0; i < 16; i++ {
			time.Sleep(10 * time.Millisecond)
			a1.Write([]byte("asdf"))
		}
		a1.Close()
	}()
	r1 := strm.Reader(1)
	for i := 0; i < 16; i++ {
		r1.Read(make([]byte, 4))
	}
	// block until eof (to avoid filedescriptor close race -- could also waitgroup the writer routine)
	r1.Read(make([]byte, 0))
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import ()

type Score struct {
	midgame  int
	endgame  int
}

type Evaluator struct {
        stage     int
        midgame   int
        endgame   int
        position  *Position
}

func (p *Position) Evaluate() int {
        evaluator := &Evaluator{ 0, 0, 0, p }

        evaluator.analyzeMaterial()
        evaluator.analyzeCoordination()
        evaluator.analyzePawnStructure()
        evaluator.analyzeRooks()
        evaluator.analyzeKingShield()
        // evaluator.analyzeKingSafety()

        return p.score(evaluator.midgame, evaluator.endgame)
}

func (e *Evaluator) analyzeMaterial() {
        color, opposite := e.position.color, e.position.color^1

        for _,piece := range []int{ WhitePawn, WhiteKnight, WhiteBishop, WhiteRook, WhiteQueen } {
                count := e.position.count[Piece(piece|color)] - e.position.count[Piece(piece|opposite)]
                midgame, endgame := Piece(piece).value()
                e.midgame += midgame * count
                e.endgame += endgame * count
        }
}

func (e *Evaluator) analyzeCoordination() {
        var moves, attacks [2]int
        var bonus [2]Score

        underAttack := [2]Bitmask{e.position.attacks(White), e.position.attacks(Black)}
        for square, piece := range e.position.pieces {
                if piece == 0 {
                        continue
                }
                color := piece.color()

                // Mobility: how many moves are available to squares not attacked by
                // the opponent?
                moves[color] += (e.position.targets(square) & underAttack[color^1]).count()

                // Agressivness: how many opponent's pieces are being attacked?
                attacks[color] += (e.position.targets(square) & e.position.board[color^1]).count()

                // Calculate bonus or penalty for a piece being at the given square.
                midgame, endgame := piece.bonus(flip[color][square])
                bonus[color].midgame += midgame
                bonus[color].endgame += endgame
        }

        color, opposite := e.position.color, e.position.color^1
        e.midgame += bonus[color].midgame - bonus[opposite].midgame
        e.endgame += bonus[color].endgame - bonus[opposite].endgame

        mobility := moves[color] - moves[opposite]
        e.midgame += mobility * movesAvailable.midgame
        e.endgame += mobility * movesAvailable.endgame

        aggression := attacks[color] - attacks[opposite]
        e.midgame += aggression * attackForce.midgame
        e.endgame += aggression * attackForce.endgame

        if bishops := e.position.count[Bishop(color)]; bishops >= 2 {
                e.midgame += bishopPair.midgame
                e.endgame += bishopPair.endgame
        }
        if bishops := e.position.count[Bishop(opposite)]; bishops >= 2 {
                e.midgame -= bishopPair.midgame
                e.endgame -= bishopPair.endgame
        }
}

func (e *Evaluator) analyzePawnStructure() {
        var bonus, penalty [2]Score
        pawn := [2]Piece{ WhitePawn, BlackPawn }

        for color := White; color <= Black; color++ {
                var doubled [8]int // Number of doubled pawns in each column.

                pawns := e.position.outposts[pawn[color]]
                for pawns != 0 {
                        square := pawns.pop()
                        column := Col(square)
                        //
                        // count doubled pawns in the column as they carry a penalty.
                        //
                        doubled[column] = (maskFile[column] & e.position.outposts[pawn[color]]).count()
                        //
                        // The pawn is passed if a) there are no enemy pawns in the
                        // same and adjacent columns; and b) there is no same color
                        // pawns in front of us.
                        //
                        if maskPassed[color][square] & e.position.outposts[pawn[color^1]] == 0 &&
                           maskInFront[color][square] & e.position.outposts[pawn[color]] == 0 {
                                   bonus[color].midgame += bonusPassedPawn[0][flip[color][square]]
                                   bonus[color].endgame += bonusPassedPawn[1][flip[color][square]]
                        }
                        //
                        // Check if the pawn is isolated, i.e. has no pawns of the
                        // same color on either sides.
                        //
                        if maskIsolated[column] & e.position.outposts[pawn[color]] == 0 {
                                penalty[color].midgame += penaltyIsolatedPawn[0][column]
                                penalty[color].endgame += penaltyIsolatedPawn[1][column]
                        }
                }
                //
                // Penalties for doubled pawns.
                //
                for i := 0;  i < len(doubled); i++ {
                        if doubled[i] > 0 {
                                penalty[color].midgame += (doubled[i] - 1) * penaltyDoubledPawn[0][i]
                                penalty[color].endgame += (doubled[i] - 1) * penaltyDoubledPawn[1][i]
                        }
                }
        }

        color, opposite := e.position.color, e.position.color^1
        e.midgame += bonus[color].midgame   - bonus[opposite].midgame
        e.endgame += bonus[color].endgame   - bonus[opposite].endgame
        e.midgame += penalty[color].midgame - penalty[opposite].midgame
        e.endgame += penalty[color].endgame - penalty[opposite].endgame
}

func (e *Evaluator) analyzeRooks() {
        var bonus [2]Score
        seventh := [2]Bitmask{ 0x00FF000000000000, 0x000000000000FF00 }

        for color := White; color <= Black; color++ {
                rook := Rook(color)
                if e.position.outposts[rook] == 0 {
                        continue
                }
                //
                // Bonus if rooks are on 7th rank.
                //
                if count := (e.position.outposts[rook] & seventh[color]).count(); count > 0 {
                        bonus[color].midgame += count * rookOn7th.midgame
                        bonus[color].endgame += count * rookOn7th.endgame
                }
                //
                // Bonuses if rooks are on open or semi-open files.
                //
                rooks := e.position.outposts[rook]
                for rooks != 0 {
                        square := rooks.pop()
                        column := Col(square)
                        if e.position.outposts[Pawn(color)] & maskFile[column] == 0 {
                                if e.position.outposts[Pawn(color^1)] & maskFile[column] == 0 {
                                        bonus[color].midgame += rookOnOpen.midgame
                                        bonus[color].endgame += rookOnOpen.endgame
                                } else {
                                        bonus[color].midgame += rookOnSemiOpen.midgame
                                        bonus[color].endgame += rookOnSemiOpen.endgame
                                }
                        }
                }
        }
        color, opposite := e.position.color, e.position.color^1
        e.midgame += bonus[color].midgame - bonus[opposite].midgame
        e.endgame += bonus[color].endgame - bonus[opposite].endgame
}

func (e *Evaluator) analyzeKingShield() {
        var penalty [2]int

        for color := White; color <= Black; color++ {
                king, pawn := King(color), Pawn(color)
                //
                // Pass if a) the king is missing, b) the king is on the initial square
                // or c) the opposite side doesn't have a queen with one major piece.
                //
                if e.position.outposts[king] == 0 || e.position.pieces[homeKing[color]] == king || !e.strongEnough(color^1) {
                        continue
                }
                //
                // Calculate relative square for the king so we could treat black king
                // as white. Don't bother with the shield if the king is too far.
                //
                square := flip[color^1][e.position.outposts[king].first()]
                if square > H3 {
                        continue
                }
                row, col := Coordinate(square)
                from, to := Max(0, col - 1), Min(7, col + 1)
                //
                // For each of the shield columns find the closest same color pawn. The
                // penalty is carried if the pawn is missing or is too far from the king
                // (more than one row apart).
                //
                for column := from; column <= to; column++ {
                        if shield := (e.position.outposts[pawn] & maskFile[column]); shield != 0 {
                                closest := flip[color^1][shield.first()] // Make it relative.
                                if distance := Abs(Row(closest) - row); distance > 1 {
                                        penalty[color] += distance * -shieldDistance.midgame
                                }
                        } else {
                                penalty[color] += -shieldMissing.midgame
                        }
                }
                Log("penalty[%s] => %d\n", C(color), penalty[color])
        }
        color, opposite := e.position.color, e.position.color^1
        e.midgame += penalty[color] - penalty[opposite]
        // No endgame bonus or penalty.
}

func (e *Evaluator) strongEnough(color int) bool {
        return e.position.count[Queen(color)] > 0 &&
               (e.position.count[Rook(color)] > 0 || e.position.count[Bishop(color)] > 0 || e.position.count[Knight(color)] > 0)
}
<commit_msg>Fixed mobility evaluation<commit_after>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import ()

type Score struct {
	midgame  int
	endgame  int
}

type Evaluator struct {
        stage     int
        midgame   int
        endgame   int
        position  *Position
}

func (p *Position) Evaluate() int {
        evaluator := &Evaluator{ 0, 0, 0, p }
        evaluator.analyzeMaterial()
        evaluator.analyzeCoordination()
        evaluator.analyzePawnStructure()
        evaluator.analyzeRooks()
        evaluator.analyzeKingShield()
        // evaluator.analyzeKingSafety()

        return p.score(evaluator.midgame, evaluator.endgame)
}

func (e *Evaluator) analyzeMaterial() {
        color, opposite := e.position.color, e.position.color^1

        for _,piece := range []int{ WhitePawn, WhiteKnight, WhiteBishop, WhiteRook, WhiteQueen } {
                count := e.position.count[Piece(piece|color)] - e.position.count[Piece(piece|opposite)]
                midgame, endgame := Piece(piece).value()
                e.midgame += midgame * count
                e.endgame += endgame * count
        }
}

func (e *Evaluator) analyzeCoordination() {
        var moves, attacks [2]int
        var bonus [2]Score

        notAttacked := [2]Bitmask{^e.position.attacks(White), ^e.position.attacks(Black)}
        for square, piece := range e.position.pieces {
                if piece == 0 {
                        continue
                }
                color := piece.color()

                // Mobility: how many moves are available to squares not attacked by
                // the opponent?
                moves[color] += (e.position.targets(square) & notAttacked[color^1]).count()

                // Agressivness: how many opponent's pieces are being attacked?
                attacks[color] += (e.position.targets(square) & e.position.board[color^1]).count()

                // Calculate bonus or penalty for a piece being at the given square.
                midgame, endgame := piece.bonus(flip[color][square])
                bonus[color].midgame += midgame
                bonus[color].endgame += endgame
        }

        color, opposite := e.position.color, e.position.color^1
        e.midgame += bonus[color].midgame - bonus[opposite].midgame
        e.endgame += bonus[color].endgame - bonus[opposite].endgame

        mobility := moves[color] - moves[opposite]
        e.midgame += mobility * movesAvailable.midgame
        e.endgame += mobility * movesAvailable.endgame

        aggression := attacks[color] - attacks[opposite]
        e.midgame += aggression * attackForce.midgame
        e.endgame += aggression * attackForce.endgame

        if bishops := e.position.count[Bishop(color)]; bishops >= 2 {
                e.midgame += bishopPair.midgame
                e.endgame += bishopPair.endgame
        }
        if bishops := e.position.count[Bishop(opposite)]; bishops >= 2 {
                e.midgame -= bishopPair.midgame
                e.endgame -= bishopPair.endgame
        }
}

func (e *Evaluator) analyzePawnStructure() {
        var bonus, penalty [2]Score
        pawn := [2]Piece{ WhitePawn, BlackPawn }

        for color := White; color <= Black; color++ {
                var doubled [8]int // Number of doubled pawns in each column.

                pawns := e.position.outposts[pawn[color]]
                for pawns != 0 {
                        square := pawns.pop()
                        column := Col(square)
                        //
                        // count doubled pawns in the column as they carry a penalty.
                        //
                        doubled[column] = (maskFile[column] & e.position.outposts[pawn[color]]).count()
                        //
                        // The pawn is passed if a) there are no enemy pawns in the
                        // same and adjacent columns; and b) there is no same color
                        // pawns in front of us.
                        //
                        if maskPassed[color][square] & e.position.outposts[pawn[color^1]] == 0 &&
                           maskInFront[color][square] & e.position.outposts[pawn[color]] == 0 {
                                   bonus[color].midgame += bonusPassedPawn[0][flip[color][square]]
                                   bonus[color].endgame += bonusPassedPawn[1][flip[color][square]]
                        }
                        //
                        // Check if the pawn is isolated, i.e. has no pawns of the
                        // same color on either sides.
                        //
                        if maskIsolated[column] & e.position.outposts[pawn[color]] == 0 {
                                penalty[color].midgame += penaltyIsolatedPawn[0][column]
                                penalty[color].endgame += penaltyIsolatedPawn[1][column]
                        }
                }
                //
                // Penalties for doubled pawns.
                //
                for i := 0;  i < len(doubled); i++ {
                        if doubled[i] > 0 {
                                penalty[color].midgame += (doubled[i] - 1) * penaltyDoubledPawn[0][i]
                                penalty[color].endgame += (doubled[i] - 1) * penaltyDoubledPawn[1][i]
                        }
                }
        }

        color, opposite := e.position.color, e.position.color^1
        e.midgame += bonus[color].midgame   - bonus[opposite].midgame
        e.endgame += bonus[color].endgame   - bonus[opposite].endgame
        e.midgame += penalty[color].midgame - penalty[opposite].midgame
        e.endgame += penalty[color].endgame - penalty[opposite].endgame
}

func (e *Evaluator) analyzeRooks() {
        var bonus [2]Score
        seventh := [2]Bitmask{ 0x00FF000000000000, 0x000000000000FF00 }

        for color := White; color <= Black; color++ {
                rook := Rook(color)
                if e.position.outposts[rook] == 0 {
                        continue
                }
                //
                // Bonus if rooks are on 7th rank.
                //
                if count := (e.position.outposts[rook] & seventh[color]).count(); count > 0 {
                        bonus[color].midgame += count * rookOn7th.midgame
                        bonus[color].endgame += count * rookOn7th.endgame
                }
                //
                // Bonuses if rooks are on open or semi-open files.
                //
                rooks := e.position.outposts[rook]
                for rooks != 0 {
                        square := rooks.pop()
                        column := Col(square)
                        if e.position.outposts[Pawn(color)] & maskFile[column] == 0 {
                                if e.position.outposts[Pawn(color^1)] & maskFile[column] == 0 {
                                        bonus[color].midgame += rookOnOpen.midgame
                                        bonus[color].endgame += rookOnOpen.endgame
                                } else {
                                        bonus[color].midgame += rookOnSemiOpen.midgame
                                        bonus[color].endgame += rookOnSemiOpen.endgame
                                }
                        }
                }
        }
        color, opposite := e.position.color, e.position.color^1
        e.midgame += bonus[color].midgame - bonus[opposite].midgame
        e.endgame += bonus[color].endgame - bonus[opposite].endgame
}

func (e *Evaluator) analyzeKingShield() {
        var penalty [2]int

        for color := White; color <= Black; color++ {
                king, pawn := King(color), Pawn(color)
                //
                // Pass if a) the king is missing, b) the king is on the initial square
                // or c) the opposite side doesn't have a queen with one major piece.
                //
                if e.position.outposts[king] == 0 || e.position.pieces[homeKing[color]] == king || !e.strongEnough(color^1) {
                        continue
                }
                //
                // Calculate relative square for the king so we could treat black king
                // as white. Don't bother with the shield if the king is too far.
                //
                square := flip[color^1][e.position.outposts[king].first()]
                if square > H3 {
                        continue
                }
                row, col := Coordinate(square)
                from, to := Max(0, col - 1), Min(7, col + 1)
                //
                // For each of the shield columns find the closest same color pawn. The
                // penalty is carried if the pawn is missing or is too far from the king
                // (more than one row apart).
                //
                for column := from; column <= to; column++ {
                        if shield := (e.position.outposts[pawn] & maskFile[column]); shield != 0 {
                                closest := flip[color^1][shield.first()] // Make it relative.
                                if distance := Abs(Row(closest) - row); distance > 1 {
                                        penalty[color] += distance * -shieldDistance.midgame
                                }
                        } else {
                                penalty[color] += -shieldMissing.midgame
                        }
                }
                Log("penalty[%s] => %d\n", C(color), penalty[color])
        }
        color, opposite := e.position.color, e.position.color^1
        e.midgame += penalty[color] - penalty[opposite]
        // No endgame bonus or penalty.
}

func (e *Evaluator) strongEnough(color int) bool {
        return e.position.count[Queen(color)] > 0 &&
               (e.position.count[Rook(color)] > 0 || e.position.count[Bishop(color)] > 0 || e.position.count[Knight(color)] > 0)
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"go/ast"
	"net/url"
	"reflect"
	"sort"
	"strings"

	"github.com/lxc/lxd/lxd/db/generate/lex"
	"github.com/lxc/lxd/shared"
	"github.com/pkg/errors"
)

// Packages returns the the AST packages in which to search for structs.
//
// By default it includes the lxd/db and shared/api packages.
func Packages() (map[string]*ast.Package, error) {
	packages := map[string]*ast.Package{}

	for _, name := range defaultPackages {
		pkg, err := lex.Parse(name)
		if err != nil {
			return nil, errors.Wrapf(err, "Parse %q", name)
		}
		parts := strings.Split(name, "/")
		packages[parts[len(parts)-1]] = pkg
	}

	return packages, nil
}

var defaultPackages = []string{
	"github.com/lxc/lxd/shared/api",
	"github.com/lxc/lxd/lxd/db",
}

// FiltersFromStmt parses all filtering statement defined for the given entity. It
// returns all supported combinations of filters, sorted by number of criteria.
func FiltersFromStmt(pkg *ast.Package, kind string, entity string) [][]string {
	objects := pkg.Scope.Objects
	filters := [][]string{}

	prefix := fmt.Sprintf("%s%sBy", lex.Minuscule(lex.Camel(entity)), lex.Camel(kind))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		filters = append(filters, strings.Split(rest, "And"))
	}

	return sortFilters(filters)
}

// RefFiltersFromStmt parses all filtering statement defined for the given entity reference.
func RefFiltersFromStmt(pkg *ast.Package, entity string, ref string) [][]string {
	objects := pkg.Scope.Objects
	filters := [][]string{}

	prefix := fmt.Sprintf("%s%sRefBy", lex.Minuscule(lex.Camel(entity)), lex.Capital(ref))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		filters = append(filters, strings.Split(rest, "And"))
	}

	return sortFilters(filters)
}

func sortFilters(filters [][]string) [][]string {
	sort.Slice(filters, func(i, j int) bool {
		n1 := len(filters[i])
		n2 := len(filters[j])
		if n1 != n2 {
			return n1 > n2
		}
		f1 := sortFilter(filters[i])
		f2 := sortFilter(filters[j])
		for k := range f1 {
			if f1[k] == f2[k] {
				continue
			}
			return f1[k] > f2[k]
		}
		panic("duplicate filter")
	})
	return filters
}

func sortFilter(filter []string) []string {
	f := make([]string, len(filter))
	copy(f, filter)
	sort.Sort(sort.Reverse(sort.StringSlice(f)))
	return f
}

// Parse the structure declaration with the given name found in the given Go package.
// Any 'Entity' struct should also have an 'EntityFilter' struct defined in the same file.
func Parse(pkg *ast.Package, name string, kind string) (*Mapping, error) {
	// The main entity struct.
	str := findStruct(pkg.Scope, name)
	if str == nil {
		return nil, fmt.Errorf("No declaration found for %q", name)
	}

	fields, err := parseStruct(str, kind)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to parse %q", name)
	}

	m := &Mapping{
		Package: pkg.Name,
		Name:    name,
		Fields:  fields,
	}

	// The 'EntityFilter' struct. This is used for filtering on specific fields of the entity.
	filterName := name + "Filter"
	filterStr := findStruct(pkg.Scope, filterName)
	if filterStr == nil {
		return nil, fmt.Errorf("No declaration found for %q", filterName)
	}

	filters, err := parseStruct(filterStr, kind)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to parse %q", name)
	}

	for i, filter := range filters {
		// Any field in EntityFilter must be present in the original struct.
		field := m.FieldByName(filter.Name)
		if field == nil {
			return nil, fmt.Errorf("Filter field %q is not in struct %q", filter.Name, name)
		}

		// Assign the config tags from the main entity struct to the Filter struct.
		filters[i].Config = field.Config

		// A Filter field and its indirect references must all be in the Filter struct.
		if field.IsIndirect() {
			indirectField := lex.Camel(field.Config.Get("via"))
			for i, f := range filters {
				if f.Name == indirectField {
					break
				}
				if i == len(filters)-1 {
					return nil, fmt.Errorf("Field %q requires field %q in struct %q", field.Name, indirectField, name+"Filter")
				}
			}
		}
	}

	m.Filters = filters

	return m, nil
}

// Find the StructType node for the structure with the given name
func findStruct(scope *ast.Scope, name string) *ast.StructType {
	obj := scope.Lookup(name)
	if obj == nil {
		return nil
	}

	typ, ok := obj.Decl.(*ast.TypeSpec)
	if !ok {
		return nil
	}

	str, ok := typ.Type.(*ast.StructType)
	if !ok {
		return nil
	}

	return str
}

// Extract field information from the given structure.
func parseStruct(str *ast.StructType, kind string) ([]*Field, error) {
	fields := make([]*Field, 0)

	for _, f := range str.Fields.List {
		if len(f.Names) == 0 {
			// Check if this is a parent struct.
			ident, ok := f.Type.(*ast.Ident)
			if !ok {
				continue
			}

			typ, ok := ident.Obj.Decl.(*ast.TypeSpec)
			if !ok {
				continue
			}

			parentStr, ok := typ.Type.(*ast.StructType)
			if !ok {
				continue
			}

			parentFields, err := parseStruct(parentStr, kind)
			if err != nil {
				return nil, errors.Wrapf(err, "Failed to parse parent struct")
			}
			fields = append(fields, parentFields...)

			continue
		}

		if len(f.Names) != 1 {
			return nil, fmt.Errorf("Expected a single field name, got %q", f.Names)
		}

		field, err := parseField(f, kind)
		if err != nil {
			return nil, err
		}

		// Don't add field if it has been ignored.
		if field != nil {
			fields = append(fields, field)
		}
	}

	return fields, nil
}

func parseField(f *ast.Field, kind string) (*Field, error) {
	name := f.Names[0]

	if !name.IsExported() {
		//return nil, fmt.Errorf("Unexported field name %q", name.Name)
	}

	// Ignore fields that are marked with a tag of `db:"ingore"`
	if f.Tag != nil {
		tag := f.Tag.Value
		tagValue := reflect.StructTag(tag[1 : len(tag)-1]).Get("db")
		if tagValue == "ignore" {
			return nil, nil
		}
	}

	typeName := parseType(f.Type)
	if typeName == "" {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	typeObj := Type{
		Name: typeName,
	}

	if IsColumnType(typeName) {
		typeObj.Code = TypeColumn
	} else if strings.HasPrefix(typeName, "[]") {
		typeObj.Code = TypeSlice
	} else if strings.HasPrefix(typeName, "map[") {
		typeObj.Code = TypeMap
	} else {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	var config url.Values
	if f.Tag != nil {
		tag := f.Tag.Value
		var err error
		config, err = url.ParseQuery(reflect.StructTag(tag[1 : len(tag)-1]).Get("db"))
		if err != nil {
			return nil, errors.Wrap(err, "Parse 'db' structure tag")
		}
	}

	// Ignore fields that are marked with `db:"omit"`.
	if omit := config.Get("omit"); omit != "" {
		omitFields := strings.Split(omit, ",")
		kind = strings.Replace(lex.Snake(kind), "_", "-", -1)
		if shared.StringInSlice(kind, omitFields) {
			return nil, nil
		} else if kind == "exists" && shared.StringInSlice("id", omitFields) {
			// Exists checks ID, so if we are omitting the field from ID, also omit it from Exists.
			return nil, nil
		}
	}

	field := Field{
		Name:   name.Name,
		Type:   typeObj,
		Config: config,
	}

	return &field, nil
}

func parseType(x ast.Expr) string {
	switch t := x.(type) {
	case *ast.StarExpr:
		return parseType(t.X)
	case *ast.SelectorExpr:
		return parseType(t.X) + "." + t.Sel.String()
	case *ast.Ident:
		s := t.String()
		if s == "byte" {
			return "uint8"
		}
		return s
	case *ast.ArrayType:
		return "[" + parseType(t.Len) + "]" + parseType(t.Elt)
	case *ast.MapType:
		return "map[" + parseType(t.Key) + "]" + parseType(t.Value)
	case *ast.BasicLit:
		return t.Value
	case nil:
		return ""
	default:
		return ""
	}
}
<commit_msg>lxd/db/generate/db/parse: return active and ignored filters from (Ref)FiltersFromStmt<commit_after>package db

import (
	"fmt"
	"go/ast"
	"net/url"
	"reflect"
	"sort"
	"strings"

	"github.com/lxc/lxd/lxd/db/generate/lex"
	"github.com/lxc/lxd/shared"
	"github.com/pkg/errors"
)

// Packages returns the the AST packages in which to search for structs.
//
// By default it includes the lxd/db and shared/api packages.
func Packages() (map[string]*ast.Package, error) {
	packages := map[string]*ast.Package{}

	for _, name := range defaultPackages {
		pkg, err := lex.Parse(name)
		if err != nil {
			return nil, errors.Wrapf(err, "Parse %q", name)
		}
		parts := strings.Split(name, "/")
		packages[parts[len(parts)-1]] = pkg
	}

	return packages, nil
}

var defaultPackages = []string{
	"github.com/lxc/lxd/shared/api",
	"github.com/lxc/lxd/lxd/db",
}

// FiltersFromStmt parses all filtering statement defined for the given entity. It
// returns all supported combinations of filters, sorted by number of criteria, and
// the corresponding set of unused filters from the Filter struct.
func FiltersFromStmt(pkg *ast.Package, kind string, entity string, filters []*Field) ([][]string, [][]string) {
	objects := pkg.Scope.Objects
	stmtFilters := [][]string{}

	prefix := fmt.Sprintf("%s%sBy", lex.Minuscule(lex.Camel(entity)), lex.Camel(kind))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		stmtFilters = append(stmtFilters, strings.Split(rest, "And"))
	}

	stmtFilters = sortFilters(stmtFilters)
	ignoredFilters := [][]string{}

	for _, filterGroup := range stmtFilters {
		ignoredFilterGroup := []string{}
		for _, filter := range filters {
			if !shared.StringInSlice(filter.Name, filterGroup) {
				ignoredFilterGroup = append(ignoredFilterGroup, filter.Name)
			}
		}
		ignoredFilters = append(ignoredFilters, ignoredFilterGroup)
	}

	return stmtFilters, ignoredFilters
}

// RefFiltersFromStmt parses all filtering statement defined for the given entity reference.
func RefFiltersFromStmt(pkg *ast.Package, entity string, ref string, filters []*Field) ([][]string, [][]string) {
	objects := pkg.Scope.Objects
	stmtFilters := [][]string{}

	prefix := fmt.Sprintf("%s%sRefBy", lex.Minuscule(lex.Camel(entity)), lex.Capital(ref))

	for name := range objects {
		if !strings.HasPrefix(name, prefix) {
			continue
		}
		rest := name[len(prefix):]
		stmtFilters = append(stmtFilters, strings.Split(rest, "And"))
	}

	stmtFilters = sortFilters(stmtFilters)
	ignoredFilters := [][]string{}

	for _, filterGroup := range stmtFilters {
		ignoredFilterGroup := []string{}
		for _, filter := range filters {
			if !shared.StringInSlice(filter.Name, filterGroup) {
				ignoredFilterGroup = append(ignoredFilterGroup, filter.Name)
			}
		}
		ignoredFilters = append(ignoredFilters, ignoredFilterGroup)
	}

	return stmtFilters, ignoredFilters
}

func sortFilters(filters [][]string) [][]string {
	sort.Slice(filters, func(i, j int) bool {
		n1 := len(filters[i])
		n2 := len(filters[j])
		if n1 != n2 {
			return n1 > n2
		}
		f1 := sortFilter(filters[i])
		f2 := sortFilter(filters[j])
		for k := range f1 {
			if f1[k] == f2[k] {
				continue
			}
			return f1[k] > f2[k]
		}
		panic("duplicate filter")
	})
	return filters
}

func sortFilter(filter []string) []string {
	f := make([]string, len(filter))
	copy(f, filter)
	sort.Sort(sort.Reverse(sort.StringSlice(f)))
	return f
}

// Parse the structure declaration with the given name found in the given Go package.
// Any 'Entity' struct should also have an 'EntityFilter' struct defined in the same file.
func Parse(pkg *ast.Package, name string, kind string) (*Mapping, error) {
	// The main entity struct.
	str := findStruct(pkg.Scope, name)
	if str == nil {
		return nil, fmt.Errorf("No declaration found for %q", name)
	}

	fields, err := parseStruct(str, kind)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to parse %q", name)
	}

	m := &Mapping{
		Package: pkg.Name,
		Name:    name,
		Fields:  fields,
	}

	// The 'EntityFilter' struct. This is used for filtering on specific fields of the entity.
	filterName := name + "Filter"
	filterStr := findStruct(pkg.Scope, filterName)
	if filterStr == nil {
		return nil, fmt.Errorf("No declaration found for %q", filterName)
	}

	filters, err := parseStruct(filterStr, kind)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to parse %q", name)
	}

	for i, filter := range filters {
		// Any field in EntityFilter must be present in the original struct.
		field := m.FieldByName(filter.Name)
		if field == nil {
			return nil, fmt.Errorf("Filter field %q is not in struct %q", filter.Name, name)
		}

		// Assign the config tags from the main entity struct to the Filter struct.
		filters[i].Config = field.Config

		// A Filter field and its indirect references must all be in the Filter struct.
		if field.IsIndirect() {
			indirectField := lex.Camel(field.Config.Get("via"))
			for i, f := range filters {
				if f.Name == indirectField {
					break
				}
				if i == len(filters)-1 {
					return nil, fmt.Errorf("Field %q requires field %q in struct %q", field.Name, indirectField, name+"Filter")
				}
			}
		}
	}

	m.Filters = filters

	return m, nil
}

// Find the StructType node for the structure with the given name
func findStruct(scope *ast.Scope, name string) *ast.StructType {
	obj := scope.Lookup(name)
	if obj == nil {
		return nil
	}

	typ, ok := obj.Decl.(*ast.TypeSpec)
	if !ok {
		return nil
	}

	str, ok := typ.Type.(*ast.StructType)
	if !ok {
		return nil
	}

	return str
}

// Extract field information from the given structure.
func parseStruct(str *ast.StructType, kind string) ([]*Field, error) {
	fields := make([]*Field, 0)

	for _, f := range str.Fields.List {
		if len(f.Names) == 0 {
			// Check if this is a parent struct.
			ident, ok := f.Type.(*ast.Ident)
			if !ok {
				continue
			}

			typ, ok := ident.Obj.Decl.(*ast.TypeSpec)
			if !ok {
				continue
			}

			parentStr, ok := typ.Type.(*ast.StructType)
			if !ok {
				continue
			}

			parentFields, err := parseStruct(parentStr, kind)
			if err != nil {
				return nil, errors.Wrapf(err, "Failed to parse parent struct")
			}
			fields = append(fields, parentFields...)

			continue
		}

		if len(f.Names) != 1 {
			return nil, fmt.Errorf("Expected a single field name, got %q", f.Names)
		}

		field, err := parseField(f, kind)
		if err != nil {
			return nil, err
		}

		// Don't add field if it has been ignored.
		if field != nil {
			fields = append(fields, field)
		}
	}

	return fields, nil
}

func parseField(f *ast.Field, kind string) (*Field, error) {
	name := f.Names[0]

	if !name.IsExported() {
		//return nil, fmt.Errorf("Unexported field name %q", name.Name)
	}

	// Ignore fields that are marked with a tag of `db:"ingore"`
	if f.Tag != nil {
		tag := f.Tag.Value
		tagValue := reflect.StructTag(tag[1 : len(tag)-1]).Get("db")
		if tagValue == "ignore" {
			return nil, nil
		}
	}

	typeName := parseType(f.Type)
	if typeName == "" {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	typeObj := Type{
		Name: typeName,
	}

	if IsColumnType(typeName) {
		typeObj.Code = TypeColumn
	} else if strings.HasPrefix(typeName, "[]") {
		typeObj.Code = TypeSlice
	} else if strings.HasPrefix(typeName, "map[") {
		typeObj.Code = TypeMap
	} else {
		return nil, fmt.Errorf("Unsupported type for field %q", name.Name)
	}

	var config url.Values
	if f.Tag != nil {
		tag := f.Tag.Value
		var err error
		config, err = url.ParseQuery(reflect.StructTag(tag[1 : len(tag)-1]).Get("db"))
		if err != nil {
			return nil, errors.Wrap(err, "Parse 'db' structure tag")
		}
	}

	// Ignore fields that are marked with `db:"omit"`.
	if omit := config.Get("omit"); omit != "" {
		omitFields := strings.Split(omit, ",")
		kind = strings.Replace(lex.Snake(kind), "_", "-", -1)
		if shared.StringInSlice(kind, omitFields) {
			return nil, nil
		} else if kind == "exists" && shared.StringInSlice("id", omitFields) {
			// Exists checks ID, so if we are omitting the field from ID, also omit it from Exists.
			return nil, nil
		}
	}

	field := Field{
		Name:   name.Name,
		Type:   typeObj,
		Config: config,
	}

	return &field, nil
}

func parseType(x ast.Expr) string {
	switch t := x.(type) {
	case *ast.StarExpr:
		return parseType(t.X)
	case *ast.SelectorExpr:
		return parseType(t.X) + "." + t.Sel.String()
	case *ast.Ident:
		s := t.String()
		if s == "byte" {
			return "uint8"
		}
		return s
	case *ast.ArrayType:
		return "[" + parseType(t.Len) + "]" + parseType(t.Elt)
	case *ast.MapType:
		return "map[" + parseType(t.Key) + "]" + parseType(t.Value)
	case *ast.BasicLit:
		return t.Value
	case nil:
		return ""
	default:
		return ""
	}
}
<|endoftext|>


<commit_before>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/pkg/errors"
)

// Magic Transit GRE Tunnel Error messages.
const (
	errMagicTransitGRETunnelNotModified = "When trying to modify GRE tunnel, API returned modified: false"
	errMagicTransitGRETunnelNotDeleted  = "When trying to delete GRE tunnel, API returned deleted: false"
)

// MagicTransitGRETunnel contains information about a GRE tunnel.
type MagicTransitGRETunnel struct {
	ID                    string                           `json:"id,omitempty"`
	CreatedOn             *time.Time                       `json:"created_on,omitempty"`
	ModifiedOn            *time.Time                       `json:"modified_on,omitempty"`
	Name                  string                           `json:"name"`
	CustomerGREEndpoint   string                           `json:"customer_gre_endpoint"`
	CloudflareGREEndpoint string                           `json:"cloudflare_gre_endpoint"`
	InterfaceAddress      string                           `json:"interface_address"`
	Description           string                           `json:"description,omitempty"`
	TTL                   uint8                            `json:"ttl,omitempty"`
	MTU                   uint16                           `json:"mtu,omitempty"`
	HealthCheck           MagicTransitGRETunnelHealthcheck `json:"health_check"`
}

// MagicTransitGRETunnelHealthcheck contains information about a GRE tunnel health check.
type MagicTransitGRETunnelHealthcheck struct {
	Enabled bool   `json:"enabled,omitempty"`
	Target  string `json:"target,omitempty"`
	Type    string `json:"type,omitempty"`
}

// ListMagicTransitGRETunnelsResponse contains a response including GRE tunnels.
type ListMagicTransitGRETunnelsResponse struct {
	Response
	Result struct {
		GRETunnels []MagicTransitGRETunnel `json:"gre_tunnels"`
	} `json:"result"`
}

// GetMagicTransitGRETunnelResponse contains a response including zero or one GRE tunnels.
type GetMagicTransitGRETunnelResponse struct {
	Response
	Result struct {
		GRETunnel MagicTransitGRETunnel `json:"gre_tunnel"`
	} `json:"result"`
}

// CreateMagicTransitGRETunnelsRequest is an array of GRE tunnels to create.
type CreateMagicTransitGRETunnelsRequest struct {
	GRETunnels []MagicTransitGRETunnel `json:"gre_tunnels"`
}

// UpdateMagicTransitGRETunnelResponse contains a response after updating a GRE Tunnel.
type UpdateMagicTransitGRETunnelResponse struct {
	Response
	Result struct {
		Modified          bool                  `json:"modified"`
		ModifiedGRETunnel MagicTransitGRETunnel `json:"modified_gre_tunnel"`
	} `json:"result"`
}

// DeleteMagicTransitGRETunnelResponse contains a response after deleting a GRE Tunnel.
type DeleteMagicTransitGRETunnelResponse struct {
	Response
	Result struct {
		Deleted          bool                  `json:"deleted"`
		DeletedGRETunnel MagicTransitGRETunnel `json:"deleted_gre_tunnel"`
	} `json:"result"`
}

// ListMagicTransitGRETunnels lists all GRE tunnels for a given account
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-list-gre-tunnels
func (api *API) ListMagicTransitGRETunnels(ctx context.Context, accountID string) ([]MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels", accountID)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return []MagicTransitGRETunnel{}, err
	}

	result := ListMagicTransitGRETunnelsResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return []MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return result.Result.GRETunnels, nil
}

// GetMagicTransitGRETunnel returns zero or one GRE tunnel
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-gre-tunnel-details
func (api *API) GetMagicTransitGRETunnel(ctx context.Context, accountID string, id string) (MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels/%s", accountID, id)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return MagicTransitGRETunnel{}, err
	}

	result := GetMagicTransitGRETunnelResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return result.Result.GRETunnel, nil
}

// CreateMagicTransitGRETunnels creates one or more GRE tunnels
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-create-gre-tunnels
func (api *API) CreateMagicTransitGRETunnels(ctx context.Context, accountID string, tunnels []MagicTransitGRETunnel) ([]MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels", accountID)
	res, err := api.makeRequestContext(ctx, http.MethodPost, uri, CreateMagicTransitGRETunnelsRequest{
		GRETunnels: tunnels,
	})

	if err != nil {
		return []MagicTransitGRETunnel{}, err
	}

	result := ListMagicTransitGRETunnelsResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return []MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return result.Result.GRETunnels, nil
}

// UpdateMagicTransitGRETunnel updates a GRE tunnel
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-update-gre-tunnel
func (api *API) UpdateMagicTransitGRETunnel(ctx context.Context, accountID string, id string, tunnel MagicTransitGRETunnel) (MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels/%s", accountID, id)
	res, err := api.makeRequestContext(ctx, http.MethodPut, uri, tunnel)

	if err != nil {
		return MagicTransitGRETunnel{}, err
	}

	result := UpdateMagicTransitGRETunnelResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	if !result.Result.Modified {
		return MagicTransitGRETunnel{}, errors.New(errMagicTransitGRETunnelNotModified)
	}

	return result.Result.ModifiedGRETunnel, nil
}

// DeleteMagicTransitGRETunnel deletes a GRE tunnel
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-delete-gre-tunnel
func (api *API) DeleteMagicTransitGRETunnel(ctx context.Context, accountID string, id string) (MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels/%s", accountID, id)
	res, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil)

	if err != nil {
		return MagicTransitGRETunnel{}, err
	}

	result := DeleteMagicTransitGRETunnelResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	if !result.Result.Deleted {
		return MagicTransitGRETunnel{}, errors.New(errMagicTransitGRETunnelNotDeleted)
	}

	return result.Result.DeletedGRETunnel, nil
}
<commit_msg>Make healthcheck a pointer to avoid empty json objects<commit_after>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/pkg/errors"
)

// Magic Transit GRE Tunnel Error messages.
const (
	errMagicTransitGRETunnelNotModified = "When trying to modify GRE tunnel, API returned modified: false"
	errMagicTransitGRETunnelNotDeleted  = "When trying to delete GRE tunnel, API returned deleted: false"
)

// MagicTransitGRETunnel contains information about a GRE tunnel.
type MagicTransitGRETunnel struct {
	ID                    string                            `json:"id,omitempty"`
	CreatedOn             *time.Time                        `json:"created_on,omitempty"`
	ModifiedOn            *time.Time                        `json:"modified_on,omitempty"`
	Name                  string                            `json:"name"`
	CustomerGREEndpoint   string                            `json:"customer_gre_endpoint"`
	CloudflareGREEndpoint string                            `json:"cloudflare_gre_endpoint"`
	InterfaceAddress      string                            `json:"interface_address"`
	Description           string                            `json:"description,omitempty"`
	TTL                   uint8                             `json:"ttl,omitempty"`
	MTU                   uint16                            `json:"mtu,omitempty"`
	HealthCheck           *MagicTransitGRETunnelHealthcheck `json:"health_check,omitempty"`
}

// MagicTransitGRETunnelHealthcheck contains information about a GRE tunnel health check.
type MagicTransitGRETunnelHealthcheck struct {
	Enabled bool   `json:"enabled"`
	Target  string `json:"target,omitempty"`
	Type    string `json:"type,omitempty"`
}

// ListMagicTransitGRETunnelsResponse contains a response including GRE tunnels.
type ListMagicTransitGRETunnelsResponse struct {
	Response
	Result struct {
		GRETunnels []MagicTransitGRETunnel `json:"gre_tunnels"`
	} `json:"result"`
}

// GetMagicTransitGRETunnelResponse contains a response including zero or one GRE tunnels.
type GetMagicTransitGRETunnelResponse struct {
	Response
	Result struct {
		GRETunnel MagicTransitGRETunnel `json:"gre_tunnel"`
	} `json:"result"`
}

// CreateMagicTransitGRETunnelsRequest is an array of GRE tunnels to create.
type CreateMagicTransitGRETunnelsRequest struct {
	GRETunnels []MagicTransitGRETunnel `json:"gre_tunnels"`
}

// UpdateMagicTransitGRETunnelResponse contains a response after updating a GRE Tunnel.
type UpdateMagicTransitGRETunnelResponse struct {
	Response
	Result struct {
		Modified          bool                  `json:"modified"`
		ModifiedGRETunnel MagicTransitGRETunnel `json:"modified_gre_tunnel"`
	} `json:"result"`
}

// DeleteMagicTransitGRETunnelResponse contains a response after deleting a GRE Tunnel.
type DeleteMagicTransitGRETunnelResponse struct {
	Response
	Result struct {
		Deleted          bool                  `json:"deleted"`
		DeletedGRETunnel MagicTransitGRETunnel `json:"deleted_gre_tunnel"`
	} `json:"result"`
}

// ListMagicTransitGRETunnels lists all GRE tunnels for a given account
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-list-gre-tunnels
func (api *API) ListMagicTransitGRETunnels(ctx context.Context, accountID string) ([]MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels", accountID)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return []MagicTransitGRETunnel{}, err
	}

	result := ListMagicTransitGRETunnelsResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return []MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return result.Result.GRETunnels, nil
}

// GetMagicTransitGRETunnel returns zero or one GRE tunnel
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-gre-tunnel-details
func (api *API) GetMagicTransitGRETunnel(ctx context.Context, accountID string, id string) (MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels/%s", accountID, id)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return MagicTransitGRETunnel{}, err
	}

	result := GetMagicTransitGRETunnelResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return result.Result.GRETunnel, nil
}

// CreateMagicTransitGRETunnels creates one or more GRE tunnels
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-create-gre-tunnels
func (api *API) CreateMagicTransitGRETunnels(ctx context.Context, accountID string, tunnels []MagicTransitGRETunnel) ([]MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels", accountID)
	res, err := api.makeRequestContext(ctx, http.MethodPost, uri, CreateMagicTransitGRETunnelsRequest{
		GRETunnels: tunnels,
	})

	if err != nil {
		return []MagicTransitGRETunnel{}, err
	}

	result := ListMagicTransitGRETunnelsResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return []MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	return result.Result.GRETunnels, nil
}

// UpdateMagicTransitGRETunnel updates a GRE tunnel
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-update-gre-tunnel
func (api *API) UpdateMagicTransitGRETunnel(ctx context.Context, accountID string, id string, tunnel MagicTransitGRETunnel) (MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels/%s", accountID, id)
	res, err := api.makeRequestContext(ctx, http.MethodPut, uri, tunnel)

	if err != nil {
		return MagicTransitGRETunnel{}, err
	}

	result := UpdateMagicTransitGRETunnelResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	if !result.Result.Modified {
		return MagicTransitGRETunnel{}, errors.New(errMagicTransitGRETunnelNotModified)
	}

	return result.Result.ModifiedGRETunnel, nil
}

// DeleteMagicTransitGRETunnel deletes a GRE tunnel
//
// API reference: https://api.cloudflare.com/#magic-gre-tunnels-delete-gre-tunnel
func (api *API) DeleteMagicTransitGRETunnel(ctx context.Context, accountID string, id string) (MagicTransitGRETunnel, error) {
	uri := fmt.Sprintf("/accounts/%s/magic/gre_tunnels/%s", accountID, id)
	res, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil)

	if err != nil {
		return MagicTransitGRETunnel{}, err
	}

	result := DeleteMagicTransitGRETunnelResponse{}
	if err := json.Unmarshal(res, &result); err != nil {
		return MagicTransitGRETunnel{}, errors.Wrap(err, errUnmarshalError)
	}

	if !result.Result.Deleted {
		return MagicTransitGRETunnel{}, errors.New(errMagicTransitGRETunnelNotDeleted)
	}

	return result.Result.DeletedGRETunnel, nil
}
<|endoftext|>