<commit_before>package main

import (
	"encoding/json"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/armon/go-metrics"
	"github.com/newrelic-forks/memberlist"
	"github.com/newrelic/sidecar/catalog"
	"github.com/newrelic/sidecar/service"
	"sync"
)

const (
	MAX_PENDING_LENGTH = 100 // Number of messages we can replace into the pending queue
)

type servicesDelegate struct {
	state             *catalog.ServicesState
	pendingBroadcasts [][]byte
	notifications     chan []byte
	inProcess         bool
	Metadata          NodeMetadata
	sync.Mutex
}

type NodeMetadata struct {
	ClusterName string
	State       string
}

func NewServicesDelegate(state *catalog.ServicesState) *servicesDelegate {
	delegate := servicesDelegate{
		state:             state,
		pendingBroadcasts: make([][]byte, 0),
		notifications:     make(chan []byte, 25),
		inProcess:         false,
		Metadata:          NodeMetadata{ClusterName: "default"},
	}

	return &delegate
}

func (d *servicesDelegate) NodeMeta(limit int) []byte {
	log.Printf("NodeMeta(): %d", limit)
	data, err := json.Marshal(d.Metadata)
	if err != nil {
		log.Error("Error encoding Node metadata!")
		data = []byte("{}")
	}
	return data
}

func (d *servicesDelegate) NotifyMsg(message []byte) {
	defer metrics.MeasureSince([]string{"delegate", "NotifyMsg"}, time.Now())

	if len(message) < 1 {
		log.Debug("NotifyMsg(): empty")
		return
	}

	log.Printf("NotifyMsg(): %s", string(message))

	// TODO don't just send container structs, send message structs
	d.notifications <- message

	// Lazily kick off goroutine
	d.Lock()
	defer d.Unlock()
	if !d.inProcess {
		go func() {
			for message := range d.notifications {
				entry := service.Decode(message)
				if entry == nil {
					log.Printf("NotifyMsg(): error decoding!")
					continue
				}
				d.state.AddServiceEntry(*entry)
			}
		}()
		d.inProcess = true
	}
}

func (d *servicesDelegate) GetBroadcasts(overhead, limit int) [][]byte {
	defer metrics.MeasureSince([]string{"delegate", "GetBroadcasts"}, time.Now())
	metrics.SetGauge([]string{"delegate", "pendingBroadcasts"}, float32(len(d.pendingBroadcasts)))

	log.Debugf("GetBroadcasts(): %d %d", overhead, limit)

	broadcast := make([][]byte, 0, 1)

	select {
	case broadcast = <-d.state.Broadcasts:
	default:
		if len(d.pendingBroadcasts) < 1 {
			return nil
		}
	}

	// Prefer newest messages (TODO what about tombstones?)
	broadcast = append(broadcast, d.pendingBroadcasts...)
	d.pendingBroadcasts = make([][]byte, 0, 1)

	broadcast, leftover := packPacket(broadcast, limit, overhead)
	if len(leftover) > 0 {
		// We don't want to store old messages forever, or starve ourselves to death
		if len(leftover) > MAX_PENDING_LENGTH {
			d.pendingBroadcasts = leftover[:MAX_PENDING_LENGTH]
		} else {
			d.pendingBroadcasts = leftover
		}
	}

	if broadcast == nil || len(broadcast) < 1 {
		log.Debug("Note: Not enough space to fit any messages or message was nil")
		return nil
	}

	log.Printf("Sending broadcast %d msgs %d 1st length",
		len(broadcast), len(broadcast[0]),
	)
	if len(leftover) > 0 {
		log.Warnf("Leaving %d messages unsent", len(leftover))
	}

	return broadcast
}

func (d *servicesDelegate) LocalState(join bool) []byte {
	log.Printf("LocalState(): %b", join)
	return d.state.Encode()
}

func (d *servicesDelegate) MergeRemoteState(buf []byte, join bool) {
	defer metrics.MeasureSince([]string{"delegate", "MergeRemoteState"}, time.Now())

	log.Printf("MergeRemoteState(): %s %b", string(buf), join)

	otherState, err := catalog.Decode(buf)
	if err != nil {
		log.Errorf("Failed to MergeRemoteState(): %s", err.Error())
		return
	}

	log.Printf("Merging state: %s", otherState.Format(nil))

	d.state.Merge(otherState)
}

func (d *servicesDelegate) NotifyJoin(node *memberlist.Node) {
	log.Debugf("NotifyJoin(): %s %s", node.Name, string(node.Meta))
}

func (d *servicesDelegate) NotifyLeave(node *memberlist.Node) {
	log.Debugf("NotifyLeave(): %s", node.Name)
	go d.state.ExpireServer(node.Name)
}

func (d *servicesDelegate) NotifyUpdate(node *memberlist.Node) {
	log.Debugf("NotifyUpdate(): %s", node.Name)
}

func packPacket(broadcasts [][]byte, limit int, overhead int) (packet [][]byte, leftover [][]byte) {
	total := 0
	leftover = make([][]byte, 0) // So we don't return unallocated buffer
	for _, message := range broadcasts {
		if total+len(message)+overhead < limit {
			packet = append(packet, message)
			total += len(message) + overhead
		} else {
			leftover = append(leftover, message)
		}
	}

	return packet, leftover
}
<commit_msg>Run some goimports.<commit_after>package main

import (
	"encoding/json"
	"time"

	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/armon/go-metrics"
	"github.com/newrelic-forks/memberlist"
	"github.com/newrelic/sidecar/catalog"
	"github.com/newrelic/sidecar/service"
)

const (
	MAX_PENDING_LENGTH = 100 // Number of messages we can replace into the pending queue
)

type servicesDelegate struct {
	state             *catalog.ServicesState
	pendingBroadcasts [][]byte
	notifications     chan []byte
	inProcess         bool
	Metadata          NodeMetadata
	sync.Mutex
}

type NodeMetadata struct {
	ClusterName string
	State       string
}

func NewServicesDelegate(state *catalog.ServicesState) *servicesDelegate {
	delegate := servicesDelegate{
		state:             state,
		pendingBroadcasts: make([][]byte, 0),
		notifications:     make(chan []byte, 25),
		inProcess:         false,
		Metadata:          NodeMetadata{ClusterName: "default"},
	}

	return &delegate
}

func (d *servicesDelegate) NodeMeta(limit int) []byte {
	log.Printf("NodeMeta(): %d", limit)
	data, err := json.Marshal(d.Metadata)
	if err != nil {
		log.Error("Error encoding Node metadata!")
		data = []byte("{}")
	}
	return data
}

func (d *servicesDelegate) NotifyMsg(message []byte) {
	defer metrics.MeasureSince([]string{"delegate", "NotifyMsg"}, time.Now())

	if len(message) < 1 {
		log.Debug("NotifyMsg(): empty")
		return
	}

	log.Printf("NotifyMsg(): %s", string(message))

	// TODO don't just send container structs, send message structs
	d.notifications <- message

	// Lazily kick off goroutine
	d.Lock()
	defer d.Unlock()
	if !d.inProcess {
		go func() {
			for message := range d.notifications {
				entry := service.Decode(message)
				if entry == nil {
					log.Printf("NotifyMsg(): error decoding!")
					continue
				}
				d.state.AddServiceEntry(*entry)
			}
		}()
		d.inProcess = true
	}
}

func (d *servicesDelegate) GetBroadcasts(overhead, limit int) [][]byte {
	defer metrics.MeasureSince([]string{"delegate", "GetBroadcasts"}, time.Now())
	metrics.SetGauge([]string{"delegate", "pendingBroadcasts"}, float32(len(d.pendingBroadcasts)))

	log.Debugf("GetBroadcasts(): %d %d", overhead, limit)

	broadcast := make([][]byte, 0, 1)

	select {
	case broadcast = <-d.state.Broadcasts:
	default:
		if len(d.pendingBroadcasts) < 1 {
			return nil
		}
	}

	// Prefer newest messages (TODO what about tombstones?)
	broadcast = append(broadcast, d.pendingBroadcasts...)
	d.pendingBroadcasts = make([][]byte, 0, 1)

	broadcast, leftover := packPacket(broadcast, limit, overhead)
	if len(leftover) > 0 {
		// We don't want to store old messages forever, or starve ourselves to death
		if len(leftover) > MAX_PENDING_LENGTH {
			d.pendingBroadcasts = leftover[:MAX_PENDING_LENGTH]
		} else {
			d.pendingBroadcasts = leftover
		}
	}

	if broadcast == nil || len(broadcast) < 1 {
		log.Debug("Note: Not enough space to fit any messages or message was nil")
		return nil
	}

	log.Printf("Sending broadcast %d msgs %d 1st length",
		len(broadcast), len(broadcast[0]),
	)
	if len(leftover) > 0 {
		log.Warnf("Leaving %d messages unsent", len(leftover))
	}

	return broadcast
}

func (d *servicesDelegate) LocalState(join bool) []byte {
	log.Printf("LocalState(): %b", join)
	return d.state.Encode()
}

func (d *servicesDelegate) MergeRemoteState(buf []byte, join bool) {
	defer metrics.MeasureSince([]string{"delegate", "MergeRemoteState"}, time.Now())

	log.Printf("MergeRemoteState(): %s %b", string(buf), join)

	otherState, err := catalog.Decode(buf)
	if err != nil {
		log.Errorf("Failed to MergeRemoteState(): %s", err.Error())
		return
	}

	log.Printf("Merging state: %s", otherState.Format(nil))

	d.state.Merge(otherState)
}

func (d *servicesDelegate) NotifyJoin(node *memberlist.Node) {
	log.Debugf("NotifyJoin(): %s %s", node.Name, string(node.Meta))
}

func (d *servicesDelegate) NotifyLeave(node *memberlist.Node) {
	log.Debugf("NotifyLeave(): %s", node.Name)
	go d.state.ExpireServer(node.Name)
}

func (d *servicesDelegate) NotifyUpdate(node *memberlist.Node) {
	log.Debugf("NotifyUpdate(): %s", node.Name)
}

func packPacket(broadcasts [][]byte, limit int, overhead int) (packet [][]byte, leftover [][]byte) {
	total := 0
	leftover = make([][]byte, 0) // So we don't return unallocated buffer
	for _, message := range broadcasts {
		if total+len(message)+overhead < limit {
			packet = append(packet, message)
			total += len(message) + overhead
		} else {
			leftover = append(leftover, message)
		}
	}

	return packet, leftover
}
<|endoftext|>


<commit_before>package settings

import "github.com/filebrowser/filebrowser/v2/rules"

// AuthMethod describes an authentication method.
type AuthMethod string

// Settings contain the main settings of the application.
type Settings struct {
	Key        []byte              `json:"key"`
	BaseURL    string              `json:"baseURL"`
	Log        string              `json:"log"`
	Server     Server              `json:"server"`
	Signup     bool                `json:"signup"`
	Defaults   UserDefaults        `json:"defaults"`
	AuthMethod AuthMethod          `json:"authMethod"`
	Branding   Branding            `json:"branding"`
	Commands   map[string][]string `json:"commands"`
	Shell      []string            `json:"shell"`
	Rules      []rules.Rule        `json:"rules"` // TODO: use this add to cli
}

// Server settings.
type Server struct {
	Port    int    `json:"port"`
	Address string `json:"address"`
	TLSCert string `json:"tlsCert"`
	TLSKey  string `json:"tlsKey"`
}

// GetRules implements rules.Provider.
func (s *Settings) GetRules() []rules.Rule {
	return s.Rules
}
<commit_msg>🧼: remove comment [ci skip]<commit_after>package settings

import "github.com/filebrowser/filebrowser/v2/rules"

// AuthMethod describes an authentication method.
type AuthMethod string

// Settings contain the main settings of the application.
type Settings struct {
	Key        []byte              `json:"key"`
	BaseURL    string              `json:"baseURL"`
	Log        string              `json:"log"`
	Server     Server              `json:"server"`
	Signup     bool                `json:"signup"`
	Defaults   UserDefaults        `json:"defaults"`
	AuthMethod AuthMethod          `json:"authMethod"`
	Branding   Branding            `json:"branding"`
	Commands   map[string][]string `json:"commands"`
	Shell      []string            `json:"shell"`
	Rules      []rules.Rule        `json:"rules"`
}

// Server settings.
type Server struct {
	Port    int    `json:"port"`
	Address string `json:"address"`
	TLSCert string `json:"tlsCert"`
	TLSKey  string `json:"tlsKey"`
}

// GetRules implements rules.Provider.
func (s *Settings) GetRules() []rules.Rule {
	return s.Rules
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"camli/blobref"
	"camli/client"
	"flag"
	"fmt"
	"log"
	"os"
)

// Things that can be uploaded.  (at most one of these)
var flagLoop = flag.Bool("loop", false, "sync in a loop once done; requires --removesrc")
var flagVerbose = flag.Bool("verbose", false, "be verbose")

var flagSrc = flag.String("src", "", "Source blobserver prefix (generally a mirrored queue partition)")
var flagSrcPass = flag.String("srcpassword", "", "Source password")
var flagDest = flag.String("dest", "", "Destination blobserver, or 'stdout' to just enumerate the --src blobs to stdout")
var flagDestPass = flag.String("destpassword", "", "Destination password")

var flagRemoveSource = flag.Bool("removesrc", false,
	"remove each blob from the source after syncing to the destination; for queue processing")

func usage(err string) {
	if err != "" {
		fmt.Fprintf(os.Stderr, "Error: %s\n\nUsage:\n", err)
	}
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Parse()

	if *flagSrc == "" {
		usage("No --src specified.")
	}
	if *flagDest == "" {
		usage("No --dest specified.")
	}
	if *flagLoop && !*flagRemoveSource {
		usage("Can't use --loop without --removesrc")
	}

	sc := client.New(*flagSrc, *flagSrcPass)
	dc := client.New(*flagDest, *flagDestPass)

	var logger *log.Logger = nil
	if *flagVerbose {
		logger = log.New(os.Stderr, "", 0)
	}
	sc.SetLogger(logger)
	dc.SetLogger(logger)

	passNum := 0
	for {
		passNum++
		if err := doPass(sc, dc, passNum); err != nil {
			log.Fatalf("sync failed: %v", err)
		}
		if !*flagLoop {
			break
		}
	}
}

func doPass(sc, dc *client.Client, passNum int) (retErr os.Error) {
	srcBlobs := make(chan *blobref.SizedBlobRef, 100)
	destBlobs := make(chan *blobref.SizedBlobRef, 100)
	srcErr := make(chan os.Error)
	destErr := make(chan os.Error)
	errorCount := 0

	go func() {
		srcErr <- sc.EnumerateBlobs(srcBlobs)
	}()
	checkSourceError := func() {
		if err := <-srcErr; err != nil {
			retErr = os.NewError(fmt.Sprintf("Enumerate error from source: %v", err))
		}
	}

	if *flagDest == "stdout" {
		for sb := range srcBlobs {
			fmt.Printf("%s %d\n", sb.BlobRef, sb.Size)
		}
		checkSourceError()
		return
	}

	go func() {
		destErr <- dc.EnumerateBlobs(destBlobs)
	}()
	checkDestError := func() {
		if err := <-destErr; err != nil {
			retErr = os.NewError(fmt.Sprintf("Enumerate error from destination: %v", err))
		}
	}

	bytesCopied := int64(0)
	blobsCopied := 0

	destNotHaveBlobs := make(chan *blobref.SizedBlobRef, 100)
	go client.ListMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs)
	for sb := range destNotHaveBlobs {
		fmt.Printf("Destination needs blob: %s\n", sb)

		blobReader, size, err := sc.Fetch(sb.BlobRef)
		if err != nil {
			errorCount++
			log.Printf("Error fetching %s: %v", sb.BlobRef, err)
			continue
		}
		if size != sb.Size {
			errorCount++
			log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
				sb.Size, sb.BlobRef, size)
			continue
		}
		uh := &client.UploadHandle{BlobRef: sb.BlobRef, Size: size, Contents: blobReader}
		pr, err := dc.Upload(uh)
		if err != nil {
			errorCount++
			log.Printf("Upload of %s to destination blobserver failed: %v", sb.BlobRef, err)
			continue
		}
		if !pr.Skipped {
			blobsCopied++
			bytesCopied += pr.Size
		}
		if *flagRemoveSource {
			if err = sc.RemoveBlob(sb.BlobRef); err != nil {
				errorCount++
				log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err)
			}
		}
	}

	// TODO: return blobsCopied & bytesCopied
	checkSourceError()
	checkDestError()
	if retErr == nil && errorCount > 0 {
		retErr = os.NewError(fmt.Sprintf("%d errors during sync", errorCount))
	}
	return
}
<commit_msg>camsync: use FetchStreaming, not Fetch<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"camli/blobref"
	"camli/client"
	"flag"
	"fmt"
	"log"
	"os"
)

// Things that can be uploaded.  (at most one of these)
var flagLoop = flag.Bool("loop", false, "sync in a loop once done; requires --removesrc")
var flagVerbose = flag.Bool("verbose", false, "be verbose")

var flagSrc = flag.String("src", "", "Source blobserver prefix (generally a mirrored queue partition)")
var flagSrcPass = flag.String("srcpassword", "", "Source password")
var flagDest = flag.String("dest", "", "Destination blobserver, or 'stdout' to just enumerate the --src blobs to stdout")
var flagDestPass = flag.String("destpassword", "", "Destination password")

var flagRemoveSource = flag.Bool("removesrc", false,
	"remove each blob from the source after syncing to the destination; for queue processing")

func usage(err string) {
	if err != "" {
		fmt.Fprintf(os.Stderr, "Error: %s\n\nUsage:\n", err)
	}
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Parse()

	if *flagSrc == "" {
		usage("No --src specified.")
	}
	if *flagDest == "" {
		usage("No --dest specified.")
	}
	if *flagLoop && !*flagRemoveSource {
		usage("Can't use --loop without --removesrc")
	}

	sc := client.New(*flagSrc, *flagSrcPass)
	dc := client.New(*flagDest, *flagDestPass)

	var logger *log.Logger = nil
	if *flagVerbose {
		logger = log.New(os.Stderr, "", 0)
	}
	sc.SetLogger(logger)
	dc.SetLogger(logger)

	passNum := 0
	for {
		passNum++
		if err := doPass(sc, dc, passNum); err != nil {
			log.Fatalf("sync failed: %v", err)
		}
		if !*flagLoop {
			break
		}
	}
}

func doPass(sc, dc *client.Client, passNum int) (retErr os.Error) {
	srcBlobs := make(chan *blobref.SizedBlobRef, 100)
	destBlobs := make(chan *blobref.SizedBlobRef, 100)
	srcErr := make(chan os.Error)
	destErr := make(chan os.Error)
	errorCount := 0

	go func() {
		srcErr <- sc.EnumerateBlobs(srcBlobs)
	}()
	checkSourceError := func() {
		if err := <-srcErr; err != nil {
			retErr = os.NewError(fmt.Sprintf("Enumerate error from source: %v", err))
		}
	}

	if *flagDest == "stdout" {
		for sb := range srcBlobs {
			fmt.Printf("%s %d\n", sb.BlobRef, sb.Size)
		}
		checkSourceError()
		return
	}

	go func() {
		destErr <- dc.EnumerateBlobs(destBlobs)
	}()
	checkDestError := func() {
		if err := <-destErr; err != nil {
			retErr = os.NewError(fmt.Sprintf("Enumerate error from destination: %v", err))
		}
	}

	bytesCopied := int64(0)
	blobsCopied := 0

	destNotHaveBlobs := make(chan *blobref.SizedBlobRef, 100)
	go client.ListMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs)
	for sb := range destNotHaveBlobs {
		fmt.Printf("Destination needs blob: %s\n", sb)

		blobReader, size, err := sc.FetchStreaming(sb.BlobRef)
		if err != nil {
			errorCount++
			log.Printf("Error fetching %s: %v", sb.BlobRef, err)
			continue
		}
		if size != sb.Size {
			errorCount++
			log.Printf("Source blobserver's enumerate size of %d for blob %s doesn't match its Get size of %d",
				sb.Size, sb.BlobRef, size)
			continue
		}
		uh := &client.UploadHandle{BlobRef: sb.BlobRef, Size: size, Contents: blobReader}
		pr, err := dc.Upload(uh)
		if err != nil {
			errorCount++
			log.Printf("Upload of %s to destination blobserver failed: %v", sb.BlobRef, err)
			continue
		}
		if !pr.Skipped {
			blobsCopied++
			bytesCopied += pr.Size
		}
		if *flagRemoveSource {
			if err = sc.RemoveBlob(sb.BlobRef); err != nil {
				errorCount++
				log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err)
			}
		}
	}

	// TODO: return blobsCopied & bytesCopied
	checkSourceError()
	checkDestError()
	if retErr == nil && errorCount > 0 {
		retErr = os.NewError(fmt.Sprintf("%d errors during sync", errorCount))
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"

	"github.com/FreeFeed/clio-restore/internal/dbutil"
	"github.com/davidmz/mustbe"
	"github.com/rwcarlsen/goexif/exif"
	"github.com/satori/go.uuid"
)

type imageSizes map[string]imageSizesEntry

type imageSizesEntry struct {
	Width   int    `json:"w"`
	Height  int    `json:"h"`
	DirName string `json:"-"`
	URL     string `json:"url"`
	Body    []byte `json:"-"`
}

var defaultImageSizes = map[string]imageSizesEntry{
	"o":  {0, 0, "attachments", "", nil},
	"t":  {525, 175, "attachments/thumbnails", "", nil},
	"t2": {1050, 350, "attachments/thumbnails2", "", nil},
}

var supportedFormats = map[string]struct {
	MIMEType string
	Ext      string
	GMFormat string
}{
	"jpeg": {"image/jpeg", "jpg", "jpeg"},
	"png":  {"image/png", "png", "png"},
	"gif":  {"image/gif", "gif", "gif"},
}

func (i imageSizes) setName(baseURL, uid, ext string) {
	for t, e := range i {
		e.URL = baseURL + "/" + e.DirName + "/" + uid + "." + ext
		i[t] = e
	}
}

// Create image attachment from the first suitable from provided URLs
func (a *App) createImageAttachment(URLs ...string) (uid string, ok bool) {
	for _, u := range URLs {
		uid, ok = a.processSingleImage(u)
		if ok {
			break
		}
	}
	return
}

func (a *App) processSingleImage(URL string) (uid string, ok bool) {
	if ffMediaURLRe.MatchString(URL) {
		// Local image
		id := ffMediaURLRe.FindStringSubmatch(URL)[1]
		if lf, exists := a.ImageFiles[id]; exists {
			r := mustbe.OKVal(lf.Open()).(io.ReadCloser)
			body := mustbe.OKVal(ioutil.ReadAll(r)).([]byte)
			r.Close()
			uid, ok = a.makeAttachment(filepath.Base(lf.Name), body)
		} else {
			errorLog.Printf("Local image not found: %s", URL)
		}
		return
	}

	// Trying to Load remote image
	infoLog.Println("Loading image:", URL)
	resp, err := http.Get(URL)
	if err != nil {
		errorLog.Println("Cannot fetch URL", URL)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK { // redirects?
		errorLog.Printf("Error fetching URL: %s (%s)", resp.Status, URL)
		return
	}

	ct := resp.Header.Get("Content-Type")
	if !(ct == "image/jpeg" || ct == "image/png" || ct == "image/gif") {
		errorLog.Printf("Unsupported content type: %s (%s)", resp.Header.Get("Content-Type"), URL)
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		errorLog.Printf("Cannot read URL data: %v (%s)", err, URL)
		return
	}

	uid, ok = a.makeAttachment("", body)

	return
}

func (a *App) makeAttachment(name string, body []byte) (uid string, ok bool) {
	// do not trust content-type
	cfg, fmtString, err := image.DecodeConfig(bytes.NewReader(body))
	if err != nil {
		errorLog.Printf("Cannot decode image: %v", err)
		return
	}

	format, ok := supportedFormats[fmtString]
	if !ok {
		errorLog.Printf("Unsupported image format: %s", format)
		return
	}

	if format.MIMEType == "image/jpeg" {
		if orient := getEXIFOrientation(bytes.NewReader(body)); orient != 0 && orient != 1 {
			cmd := exec.Command(a.GM,
				"convert",
				"-", // stdin
				"-profile", a.SRGB,
				"-auto-orient",
				"-quality", "95",
				"jpeg:-", // stdout
			)
			cmd.Stdin = bytes.NewReader(body)
			newBody := new(bytes.Buffer)
			cmd.Stdout = newBody
			if err := cmd.Run(); err != nil {
				errorLog.Printf("Cannot auto-orient image: %s", err)
			} else {
				body = newBody.Bytes()
			}
		}
	}

	// Store original image size
	iSizes := make(imageSizes)

	szEntry := defaultImageSizes["o"]
	szEntry.Width = cfg.Width
	szEntry.Height = cfg.Height
	szEntry.Body = body
	iSizes["o"] = szEntry

	for szID, szEntry := range defaultImageSizes {
		if szID == "o" || cfg.Width <= szEntry.Width && cfg.Height <= szEntry.Height {
			continue
		}
		szEntry.Width, szEntry.Height = fitInto(cfg.Width, cfg.Height, szEntry.Width, szEntry.Height)

		// Do resize
		var cmd *exec.Cmd
		if format.MIMEType != "image/gif" {
			cmd = exec.Command(a.GM,
				"convert",
				"-", // stdin
				"-resize", fmt.Sprintf("%dx%d!", szEntry.Width, szEntry.Height),
				"-profile", a.SRGB,
				"-auto-orient",
				"-quality", "95",
				format.GMFormat+":-", // stdout
			)
		} else {
			cmd = exec.Command(a.GifSicle,
				"--resize", fmt.Sprintf("%dx%d", szEntry.Width, szEntry.Height),
				"-O3",
			)

		}
		cmd.Stdin = bytes.NewReader(body)
		newBodyBuf := new(bytes.Buffer)
		cmd.Stdout = newBodyBuf
		if err := cmd.Run(); err != nil {
			errorLog.Printf("Cannot resize image: %s", err)
			continue
		}
		if newBodyBuf.Len() == 0 {
			errorLog.Printf("Cannot resize image: empty result")
			continue
		}
		szEntry.Body = newBodyBuf.Bytes()
		iSizes[szID] = szEntry
	}

	uid, ok = uuid.NewV4().String(), true

	iSizes.setName(a.AttURL, uid, format.Ext)

	// Upload all versions
	for _, entry := range iSizes {
		a.storeAttachment(entry.Body, entry.DirName+"/"+uid+"."+format.Ext, name, format.MIMEType)
	}

	// Write to DB without post_id, user_id, created_at and updated_at
	dbutil.MustInsert(a.Tx, "attachments", dbutil.H{
		"uid":            uid,
		"file_name":      name,
		"file_size":      len(body),
		"mime_type":      format.MIMEType,
		"media_type":     "image",
		"file_extension": format.Ext,
		"no_thumbnail":   len(iSizes) == 1,
		"image_sizes":    dbutil.JSONVal(iSizes),
	})

	return
}

func getEXIFOrientation(r io.Reader) (orient int) {
	x, err := exif.Decode(r)
	if err != nil {
		return
	}
	oTag, err := x.Get(exif.Orientation)
	if err != nil {
		return
	}
	orient, _ = oTag.Int(0)
	return
}

func fitInto(w, h, fitW, fitH int) (newW, newH int) {
	if w*fitH > h*fitW {
		newW, newH = fitW, h*fitW/w
	} else {
		newW, newH = w*fitH/h, fitH
	}
	if newW == 0 {
		newW = 1
	}
	if newH == 0 {
		newH = 1
	}
	return
}
<commit_msg>Analyze only the type/subtype part of Content-Type<commit_after>package main

import (
	"bytes"
	"fmt"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/FreeFeed/clio-restore/internal/dbutil"
	"github.com/davidmz/mustbe"
	"github.com/rwcarlsen/goexif/exif"
	"github.com/satori/go.uuid"
)

type imageSizes map[string]imageSizesEntry

type imageSizesEntry struct {
	Width   int    `json:"w"`
	Height  int    `json:"h"`
	DirName string `json:"-"`
	URL     string `json:"url"`
	Body    []byte `json:"-"`
}

var defaultImageSizes = map[string]imageSizesEntry{
	"o":  {0, 0, "attachments", "", nil},
	"t":  {525, 175, "attachments/thumbnails", "", nil},
	"t2": {1050, 350, "attachments/thumbnails2", "", nil},
}

var supportedFormats = map[string]struct {
	MIMEType string
	Ext      string
	GMFormat string
}{
	"jpeg": {"image/jpeg", "jpg", "jpeg"},
	"png":  {"image/png", "png", "png"},
	"gif":  {"image/gif", "gif", "gif"},
}

func (i imageSizes) setName(baseURL, uid, ext string) {
	for t, e := range i {
		e.URL = baseURL + "/" + e.DirName + "/" + uid + "." + ext
		i[t] = e
	}
}

// Create image attachment from the first suitable from provided URLs
func (a *App) createImageAttachment(URLs ...string) (uid string, ok bool) {
	for _, u := range URLs {
		uid, ok = a.processSingleImage(u)
		if ok {
			break
		}
	}
	return
}

func (a *App) processSingleImage(URL string) (uid string, ok bool) {
	if ffMediaURLRe.MatchString(URL) {
		// Local image
		id := ffMediaURLRe.FindStringSubmatch(URL)[1]
		if lf, exists := a.ImageFiles[id]; exists {
			r := mustbe.OKVal(lf.Open()).(io.ReadCloser)
			body := mustbe.OKVal(ioutil.ReadAll(r)).([]byte)
			r.Close()
			uid, ok = a.makeAttachment(filepath.Base(lf.Name), body)
		} else {
			errorLog.Printf("Local image not found: %s", URL)
		}
		return
	}

	// Trying to Load remote image
	infoLog.Println("Loading image:", URL)
	resp, err := http.Get(URL)
	if err != nil {
		errorLog.Println("Cannot fetch URL", URL)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK { // redirects?
		errorLog.Printf("Error fetching URL: %s (%s)", resp.Status, URL)
		return
	}

	ct := strings.Split(strings.ToLower(resp.Header.Get("Content-Type")), ";")[0]
	if !(ct == "image/jpeg" || ct == "image/png" || ct == "image/gif") {
		errorLog.Printf("Unsupported content type: %s (%s)", resp.Header.Get("Content-Type"), URL)
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		errorLog.Printf("Cannot read URL data: %v (%s)", err, URL)
		return
	}

	uid, ok = a.makeAttachment("", body)

	return
}

func (a *App) makeAttachment(name string, body []byte) (uid string, ok bool) {
	// do not trust content-type
	cfg, fmtString, err := image.DecodeConfig(bytes.NewReader(body))
	if err != nil {
		errorLog.Printf("Cannot decode image: %v", err)
		return
	}

	format, ok := supportedFormats[fmtString]
	if !ok {
		errorLog.Printf("Unsupported image format: %s", format)
		return
	}

	if format.MIMEType == "image/jpeg" {
		if orient := getEXIFOrientation(bytes.NewReader(body)); orient != 0 && orient != 1 {
			cmd := exec.Command(a.GM,
				"convert",
				"-", // stdin
				"-profile", a.SRGB,
				"-auto-orient",
				"-quality", "95",
				"jpeg:-", // stdout
			)
			cmd.Stdin = bytes.NewReader(body)
			newBody := new(bytes.Buffer)
			cmd.Stdout = newBody
			if err := cmd.Run(); err != nil {
				errorLog.Printf("Cannot auto-orient image: %s", err)
			} else {
				body = newBody.Bytes()
			}
		}
	}

	// Store original image size
	iSizes := make(imageSizes)

	szEntry := defaultImageSizes["o"]
	szEntry.Width = cfg.Width
	szEntry.Height = cfg.Height
	szEntry.Body = body
	iSizes["o"] = szEntry

	for szID, szEntry := range defaultImageSizes {
		if szID == "o" || cfg.Width <= szEntry.Width && cfg.Height <= szEntry.Height {
			continue
		}
		szEntry.Width, szEntry.Height = fitInto(cfg.Width, cfg.Height, szEntry.Width, szEntry.Height)

		// Do resize
		var cmd *exec.Cmd
		if format.MIMEType != "image/gif" {
			cmd = exec.Command(a.GM,
				"convert",
				"-", // stdin
				"-resize", fmt.Sprintf("%dx%d!", szEntry.Width, szEntry.Height),
				"-profile", a.SRGB,
				"-auto-orient",
				"-quality", "95",
				format.GMFormat+":-", // stdout
			)
		} else {
			cmd = exec.Command(a.GifSicle,
				"--resize", fmt.Sprintf("%dx%d", szEntry.Width, szEntry.Height),
				"-O3",
			)

		}
		cmd.Stdin = bytes.NewReader(body)
		newBodyBuf := new(bytes.Buffer)
		cmd.Stdout = newBodyBuf
		if err := cmd.Run(); err != nil {
			errorLog.Printf("Cannot resize image: %s", err)
			continue
		}
		if newBodyBuf.Len() == 0 {
			errorLog.Printf("Cannot resize image: empty result")
			continue
		}
		szEntry.Body = newBodyBuf.Bytes()
		iSizes[szID] = szEntry
	}

	uid, ok = uuid.NewV4().String(), true

	iSizes.setName(a.AttURL, uid, format.Ext)

	// Upload all versions
	for _, entry := range iSizes {
		a.storeAttachment(entry.Body, entry.DirName+"/"+uid+"."+format.Ext, name, format.MIMEType)
	}

	// Write to DB without post_id, user_id, created_at and updated_at
	dbutil.MustInsert(a.Tx, "attachments", dbutil.H{
		"uid":            uid,
		"file_name":      name,
		"file_size":      len(body),
		"mime_type":      format.MIMEType,
		"media_type":     "image",
		"file_extension": format.Ext,
		"no_thumbnail":   len(iSizes) == 1,
		"image_sizes":    dbutil.JSONVal(iSizes),
	})

	return
}

func getEXIFOrientation(r io.Reader) (orient int) {
	x, err := exif.Decode(r)
	if err != nil {
		return
	}
	oTag, err := x.Get(exif.Orientation)
	if err != nil {
		return
	}
	orient, _ = oTag.Int(0)
	return
}

func fitInto(w, h, fitW, fitH int) (newW, newH int) {
	if w*fitH > h*fitW {
		newW, newH = fitW, h*fitW/w
	} else {
		newW, newH = w*fitH/h, fitH
	}
	if newW == 0 {
		newW = 1
	}
	if newH == 0 {
		newH = 1
	}
	return
}
<|endoftext|>


<commit_before>package aks

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/appscode/go/crypto/rand"
	api "github.com/pharmer/pharmer/apis/v1alpha1"
	. "github.com/pharmer/pharmer/cloud"
	core "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
)

func (cm *ClusterManager) GetDefaultNodeSpec(cluster *api.Cluster, sku string) (api.NodeSpec, error) {
	if sku == "" {
		sku = "Standard_D2_v2"
	}
	return api.NodeSpec{
		SKU: sku,
		//	DiskType:      "",
		//	DiskSize:      100,
	}, nil
}

func (cm *ClusterManager) SetDefaults(cluster *api.Cluster) error {
	n := namer{cluster: cluster}

	// Init object meta
	cluster.ObjectMeta.UID = uuid.NewUUID()
	cluster.ObjectMeta.CreationTimestamp = metav1.Time{Time: time.Now()}
	cluster.ObjectMeta.Generation = time.Now().UnixNano()
	api.AssignTypeKind(cluster)

	// Init spec
	cluster.Spec.Cloud.Region = cluster.Spec.Cloud.Zone
	cluster.Spec.Cloud.SSHKeyName = n.GenSSHKeyExternalID()
	cluster.Spec.Networking.NonMasqueradeCIDR = "10.0.0.0/8"
	cluster.Spec.API.BindPort = kubeadmapi.DefaultAPIBindPort
	cluster.Spec.AuthorizationModes = strings.Split(kubeadmapi.DefaultAuthorizationModes, ",")
	cluster.Spec.APIServerCertSANs = NameGenerator(cm.ctx).ExtraNames(cluster.Name)
	cluster.Spec.APIServerExtraArgs = map[string]string{
		// ref: https://github.com/kubernetes/kubernetes/blob/d595003e0dc1b94455d1367e96e15ff67fc920fa/cmd/kube-apiserver/app/options/options.go#L99
		"kubelet-preferred-address-types": strings.Join([]string{
			string(core.NodeInternalDNS),
			string(core.NodeInternalIP),
			string(core.NodeExternalDNS),
			string(core.NodeExternalIP),
		}, ","),
		"cloud-config": "/etc/kubernetes/ccm/cloud-config",
	}
	if cluster.IsMinorVersion("1.9") {
		cluster.Spec.APIServerExtraArgs["admission-control"] = api.DefaultV19AdmissionControl
	}
	cluster.Spec.ControllerManagerExtraArgs = map[string]string{
		"cloud-config": "/etc/kubernetes/ccm/cloud-config",
	}
	cluster.Spec.Cloud.CCMCredentialName = cluster.Spec.CredentialName
	cluster.Spec.Cloud.Azure = &api.AzureSpec{
		ResourceGroup:      n.ResourceGroupName(),
		SubnetName:         n.SubnetName(),
		SecurityGroupName:  n.NetworkSecurityGroupName(),
		VnetName:           n.VirtualNetworkName(),
		RouteTableName:     n.RouteTableName(),
		StorageAccountName: n.GenStorageAccountName(),
		SubnetCIDR:         "10.240.0.0/16",
		RootPassword:       rand.GeneratePassword(),
	}

	// Init status
	cluster.Status = api.ClusterStatus{
		Phase: api.ClusterPending,
	}

	return nil
}

func (cm *ClusterManager) IsValid(cluster *api.Cluster) (bool, error) {
	return false, ErrNotImplemented
}

func (cm *ClusterManager) GetSSHConfig(cluster *api.Cluster, node *core.Node) (*api.SSHConfig, error) {
	cfg := &api.SSHConfig{
		PrivateKey: SSHKey(cm.ctx).PrivateKey,
		User:       "ubuntu",
		HostPort:   int32(22),
	}
	for _, addr := range node.Status.Addresses {
		if addr.Type == core.NodeExternalIP {
			cfg.HostIP = addr.Address
		}
	}
	if net.ParseIP(cfg.HostIP) == nil {
		return nil, fmt.Errorf("failed to detect external Ip for node %s of cluster %s", node.Name, cluster.Name)
	}
	return cfg, nil
}
<commit_msg>Use pkg/errors package<commit_after>package aks

import (
	"fmt"
	"net"
	"strings"
	"time"

	"github.com/appscode/go/crypto/rand"
	api "github.com/pharmer/pharmer/apis/v1alpha1"
	. "github.com/pharmer/pharmer/cloud"
	core "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
)

func (cm *ClusterManager) GetDefaultNodeSpec(cluster *api.Cluster, sku string) (api.NodeSpec, error) {
	if sku == "" {
		sku = "Standard_D2_v2"
	}
	return api.NodeSpec{
		SKU: sku,
		//	DiskType:      "",
		//	DiskSize:      100,
	}, nil
}

func (cm *ClusterManager) SetDefaults(cluster *api.Cluster) error {
	n := namer{cluster: cluster}

	// Init object meta
	cluster.ObjectMeta.UID = uuid.NewUUID()
	cluster.ObjectMeta.CreationTimestamp = metav1.Time{Time: time.Now()}
	cluster.ObjectMeta.Generation = time.Now().UnixNano()
	api.AssignTypeKind(cluster)

	// Init spec
	cluster.Spec.Cloud.Region = cluster.Spec.Cloud.Zone
	cluster.Spec.Cloud.SSHKeyName = n.GenSSHKeyExternalID()
	cluster.Spec.Networking.NonMasqueradeCIDR = "10.0.0.0/8"
	cluster.Spec.API.BindPort = kubeadmapi.DefaultAPIBindPort
	cluster.Spec.AuthorizationModes = strings.Split(kubeadmapi.DefaultAuthorizationModes, ",")
	cluster.Spec.APIServerCertSANs = NameGenerator(cm.ctx).ExtraNames(cluster.Name)
	cluster.Spec.APIServerExtraArgs = map[string]string{
		// ref: https://github.com/kubernetes/kubernetes/blob/d595003e0dc1b94455d1367e96e15ff67fc920fa/cmd/kube-apiserver/app/options/options.go#L99
		"kubelet-preferred-address-types": strings.Join([]string{
			string(core.NodeInternalDNS),
			string(core.NodeInternalIP),
			string(core.NodeExternalDNS),
			string(core.NodeExternalIP),
		}, ","),
		"cloud-config": "/etc/kubernetes/ccm/cloud-config",
	}
	if cluster.IsMinorVersion("1.9") {
		cluster.Spec.APIServerExtraArgs["admission-control"] = api.DefaultV19AdmissionControl
	}
	cluster.Spec.ControllerManagerExtraArgs = map[string]string{
		"cloud-config": "/etc/kubernetes/ccm/cloud-config",
	}
	cluster.Spec.Cloud.CCMCredentialName = cluster.Spec.CredentialName
	cluster.Spec.Cloud.Azure = &api.AzureSpec{
		ResourceGroup:      n.ResourceGroupName(),
		SubnetName:         n.SubnetName(),
		SecurityGroupName:  n.NetworkSecurityGroupName(),
		VnetName:           n.VirtualNetworkName(),
		RouteTableName:     n.RouteTableName(),
		StorageAccountName: n.GenStorageAccountName(),
		SubnetCIDR:         "10.240.0.0/16",
		RootPassword:       rand.GeneratePassword(),
	}

	// Init status
	cluster.Status = api.ClusterStatus{
		Phase: api.ClusterPending,
	}

	return nil
}

func (cm *ClusterManager) IsValid(cluster *api.Cluster) (bool, error) {
	return false, ErrNotImplemented
}

func (cm *ClusterManager) GetSSHConfig(cluster *api.Cluster, node *core.Node) (*api.SSHConfig, error) {
	cfg := &api.SSHConfig{
		PrivateKey: SSHKey(cm.ctx).PrivateKey,
		User:       "ubuntu",
		HostPort:   int32(22),
	}
	for _, addr := range node.Status.Addresses {
		if addr.Type == core.NodeExternalIP {
			cfg.HostIP = addr.Address
		}
	}
	if net.ParseIP(cfg.HostIP) == nil {
		return nil, errors.Errorf("failed to detect external Ip for node %s of cluster %s", node.Name, cluster.Name)
	}
	return cfg, nil
}
<|endoftext|>


<commit_before>package ignition

const DefaultStorageClass = `apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: default
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
  labels:
    kubernetes.io/cluster-service: "true"
provisioner: kubernetes.io/azure-disk
parameters:
  kind: Managed
  storageaccounttype: Premium_LRS
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-premium
  annotations:
  labels:
    kubernetes.io/cluster-service: "true"
provisioner: kubernetes.io/azure-disk
parameters:
  kind: Managed
  storageaccounttype: Premium_LRS
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-standard
  annotations:
  labels:
    kubernetes.io/cluster-service: "true"
provisioner: kubernetes.io/azure-disk
parameters:
  kind: Managed
  storageaccounttype: Standard_LRS
`
<commit_msg>Add azure file storage classes (#517)<commit_after>package ignition

const DefaultStorageClass = `apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: default
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
  labels:
    kubernetes.io/cluster-service: "true"
provisioner: kubernetes.io/azure-disk
parameters:
  kind: Managed
  storageaccounttype: Premium_LRS
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-premium
  annotations:
  labels:
    kubernetes.io/cluster-service: "true"
provisioner: kubernetes.io/azure-disk
parameters:
  kind: Managed
  storageaccounttype: Premium_LRS
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-standard
  annotations:
  labels:
    kubernetes.io/cluster-service: "true"
provisioner: kubernetes.io/azure-disk
parameters:
  kind: Managed
  storageaccounttype: Standard_LRS
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:azure-files-cloud-provider
rules:
- apiGroups: ['']
  resources: ['secrets']
  verbs:     ['get','create']
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:azure-files-cloud-provider
roleRef:
  kind: ClusterRole
  apiGroup: rbac.authorization.k8s.io
  name: system:azure-files-cloud-provider
subjects:
- kind: ServiceAccount
  name: persistent-volume-binder
  namespace: kube-system
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: af-premium
provisioner: kubernetes.io/azure-file
mountOptions:
  - dir_mode=0755
  - file_mode=0755
  - uid=1000
  - gid=1000
parameters:
  skuName: Premium_LRS
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: Immediate
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: af-standard
provisioner: kubernetes.io/azure-file
mountOptions:
  - dir_mode=0755
  - file_mode=0755
  - uid=1000
  - gid=1000
parameters:
  skuName: Standard_LRS
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: Immediate
`
<|endoftext|>


<commit_before>
package endpoint

import (
	"bytes"
	"net/http"
	"encoding/json"
	"io/ioutil"
	"fmt"
	"sync"
	"net/http/httputil"
)

const (
	automl_api = "https://comprehension-247816.appspot.com/feedback/ml"
	grammar_check_api = "https://us-central1-comprehension-247816.cloudfunctions.net/topic-grammar-API"
	plagiarism_api = "https://comprehension-247816.appspot.com/feedback/plagiarism"
	regex_rules_api = "https://comprehension-247816.appspot.com/feedback/rules/first_pass"
	spell_check_local = "https://us-central1-comprehension-247816.cloudfunctions.net/spell-check-cloud-function"
	spell_check_bing = "https://us-central1-comprehension-247816.cloudfunctions.net/bing-API-spell-check"

	feedback_history_url = "https://comprehension-247816.appspot.com/feedback/history"

)

var wg sync.WaitGroup

func Endpoint(responseWriter http.ResponseWriter, request *http.Request) {
	// need this for javascript cors requests
	// https://cloud.google.com/functions/docs/writing/http#functions_http_cors-go
	if request.Method == http.MethodOptions {
		responseWriter.Header().Set("Access-Control-Allow-Origin", "*")
		responseWriter.Header().Set("Access-Control-Allow-Methods", "POST")
		responseWriter.Header().Set("Access-Control-Allow-Headers", "Content-Type")
		responseWriter.Header().Set("Access-Control-Max-Age", "3600")
		responseWriter.WriteHeader(http.StatusNoContent)
		return
	}

	requestDump, err := httputil.DumpRequest(request, true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(string(requestDump))

	request_body, err := ioutil.ReadAll(request.Body)
	if err != nil {
		//TODO make this response in the same format maybe?
		http.Error(responseWriter, err.Error(), http.StatusInternalServerError)
		return
	}

	// Note, arrays can't be constants in Go, so this has to stay in the method
	urls := [...]string{
		plagiarism_api,
		automl_api,
		regex_rules_api,
		grammar_check_api,
		spell_check_bing,
	}

	results := map[int]APIResponse{}

	c := make(chan InternalAPIResponse)

	for priority, url := range urls {
		go getAPIResponse(url, priority, request_body, c)
	}

	var returnable_result APIResponse

	for response := range c {
		results[response.Priority] = response.APIResponse
		return_index, returnable := processResults(results, len(urls))

		if returnable {
			returnable_result = results[return_index]
			break
		}
	}

	// TODO make this a purely async task instead of coroutine that waits to finish
	wg.Add(1)
	go recordFeedback(request_body, returnable_result)

	responseWriter.Header().Set("Access-Control-Allow-Origin", "*")
	responseWriter.Header().Set("Content-Type", "application/json")
	json.NewEncoder(responseWriter).Encode(returnable_result)

	wg.Wait()
}
// returns a typle of results index and that should be returned.
func processResults(results map[int]APIResponse, length int) (int, bool) {
	for i := 0; i < len(results); i++ {
		result, has_key := results[i]
		if !has_key {
			return 0, false
		}

		if !result.Optimal {
			return i, true
		}
	}

	all_correct := len(results) >= length

	return 1, all_correct
}

func getAPIResponse(url string, priority int, json_params [] byte, c chan InternalAPIResponse) {
	response_json, err := http.Post(url, "application/json", bytes.NewReader(json_params))

	if err != nil {
		c <- InternalAPIResponse{Priority: priority, APIResponse: APIResponse{Feedback: "There was an error hitting the API", Optimal: false}}
		return
	}

	var result APIResponse

	if err := json.NewDecoder(response_json.Body).Decode(&result); err != nil {
		// TODO might want to think about what this should be.
		c <- InternalAPIResponse{Priority: priority, APIResponse: APIResponse{Feedback: "There was an JSON error", Feedback_type: err.Error(), Labels: url, Optimal: false}}
		return
	}

	c <- InternalAPIResponse{Priority: priority, APIResponse: result}
}

func recordFeedback(incoming_params [] byte, feedback APIResponse) {
	var request_object APIRequest

	// TODO convert the 'feedback' bytes and combine with incoming_params bytes
	// instead of transforming from bytes to object, combining, and then converting back to bytes
	if err := json.NewDecoder(bytes.NewReader(incoming_params)).Decode(&request_object); err != nil {
		return
	}

	history := HistoryAPIRequest{
		Entry: request_object.Entry,
		Prompt_id: request_object.Prompt_id,
		Session_id: request_object.Session_id,
		Attempt: request_object.Attempt,
		Feedback: feedback,
	}

	history_json, _ := json.Marshal(history)

	// TODO For now, just swallow any errors from this, but we'd want to report errors.
	http.Post(feedback_history_url, "application/json",  bytes.NewBuffer(history_json))
	wg.Done() // mark task as done in WaitGroup
}

type APIRequest struct {
	Entry string `json:"entry"`
	Prompt_id int `json:"prompt_id"`
	Prompt_text string `json:"prompt_text"`
	Session_id string `json:"session_id"`
	Attempt int `json:"attempt"`
}

type APIResponse struct {
	Feedback string `json:"feedback"`
	Feedback_type string `json:"feedback_type"`
	Optimal bool `json:"optimal"`
	Response_id string `json:"response_id"`
	Highlight []Highlight `json:"highlight"`
	Labels string `json:"labels,omitempty"`
}

type Highlight struct {
	Type string `json:"type"`
	Id int `json:"id,omitempty"`
	Text string `json:"text"`
	Category string `json:"category"`
	Character int `json:"character,omitempty"`
}

type InternalAPIResponse struct {
	Priority int
	APIResponse APIResponse
}

type HistoryAPIRequest struct {
	Entry string `json:"entry"`
	Prompt_id int `json:"prompt_id"`
	Session_id string `json:"session_id"`
	Attempt int `json:"attempt"`
	Feedback APIResponse `json:"feedback"`
}
<commit_msg>take out unneeded<commit_after>
package endpoint

import (
	"bytes"
	"net/http"
	"encoding/json"
	"io/ioutil"
	"fmt"
	"sync"
	"net/http/httputil"
)

const (
	automl_api = "https://comprehension-247816.appspot.com/feedback/ml"
	grammar_check_api = "https://us-central1-comprehension-247816.cloudfunctions.net/topic-grammar-API"
	plagiarism_api = "https://comprehension-247816.appspot.com/feedback/plagiarism"
	regex_rules_api = "https://comprehension-247816.appspot.com/feedback/rules/first_pass"
	spell_check_local = "https://us-central1-comprehension-247816.cloudfunctions.net/spell-check-cloud-function"
	spell_check_bing = "https://us-central1-comprehension-247816.cloudfunctions.net/bing-API-spell-check"

	feedback_history_url = "https://comprehension-247816.appspot.com/feedback/history"

)

var wg sync.WaitGroup

func Endpoint(responseWriter http.ResponseWriter, request *http.Request) {
	// need this for javascript cors requests
	// https://cloud.google.com/functions/docs/writing/http#functions_http_cors-go
	if request.Method == http.MethodOptions {
		responseWriter.Header().Set("Access-Control-Allow-Origin", "*")
		responseWriter.Header().Set("Access-Control-Allow-Methods", "POST")
		responseWriter.Header().Set("Access-Control-Allow-Headers", "Content-Type")
		responseWriter.Header().Set("Access-Control-Max-Age", "3600")
		responseWriter.WriteHeader(http.StatusNoContent)
		return
	}

	requestDump, err := httputil.DumpRequest(request, true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(string(requestDump))

	request_body, err := ioutil.ReadAll(request.Body)
	if err != nil {
		//TODO make this response in the same format maybe?
		http.Error(responseWriter, err.Error(), http.StatusInternalServerError)
		return
	}

	// Note, arrays can't be constants in Go, so this has to stay in the method
	urls := [...]string{
		plagiarism_api,
		automl_api,
		regex_rules_api,
		grammar_check_api,
		spell_check_bing,
	}

	results := map[int]APIResponse{}

	c := make(chan InternalAPIResponse)

	for priority, url := range urls {
		go getAPIResponse(url, priority, request_body, c)
	}

	var returnable_result APIResponse

	for response := range c {
		results[response.Priority] = response.APIResponse
		return_index, returnable := processResults(results, len(urls))

		if returnable {
			returnable_result = results[return_index]
			break
		}
	}

	// TODO make this a purely async task instead of coroutine that waits to finish
	wg.Add(1)
	go recordFeedback(request_body, returnable_result)

	responseWriter.Header().Set("Access-Control-Allow-Origin", "*")
	responseWriter.Header().Set("Content-Type", "application/json")
	json.NewEncoder(responseWriter).Encode(returnable_result)

	wg.Wait()
}
// returns a typle of results index and that should be returned.
func processResults(results map[int]APIResponse, length int) (int, bool) {
	for i := 0; i < len(results); i++ {
		result, has_key := results[i]
		if !has_key {
			return 0, false
		}

		if !result.Optimal {
			return i, true
		}
	}

	all_correct := len(results) >= length

	return 1, all_correct
}

func getAPIResponse(url string, priority int, json_params [] byte, c chan InternalAPIResponse) {
	response_json, err := http.Post(url, "application/json", bytes.NewReader(json_params))

	if err != nil {
		c <- InternalAPIResponse{Priority: priority, APIResponse: APIResponse{Feedback: "There was an error hitting the API", Optimal: false}}
		return
	}

	var result APIResponse

	if err := json.NewDecoder(response_json.Body).Decode(&result); err != nil {
		// TODO might want to think about what this should be.
		c <- InternalAPIResponse{Priority: priority, APIResponse: APIResponse{Feedback: "There was an JSON error", Feedback_type: err.Error(), Labels: url, Optimal: false}}
		return
	}

	c <- InternalAPIResponse{Priority: priority, APIResponse: result}
}

func recordFeedback(incoming_params [] byte, feedback APIResponse) {
	var request_object APIRequest

	// TODO convert the 'feedback' bytes and combine with incoming_params bytes
	// instead of transforming from bytes to object, combining, and then converting back to bytes
	if err := json.NewDecoder(bytes.NewReader(incoming_params)).Decode(&request_object); err != nil {
		return
	}

	history := HistoryAPIRequest{
		Entry: request_object.Entry,
		Prompt_id: request_object.Prompt_id,
		Session_id: request_object.Session_id,
		Attempt: request_object.Attempt,
		Feedback: feedback,
	}

	history_json, _ := json.Marshal(history)

	// TODO For now, just swallow any errors from this, but we'd want to report errors.
	http.Post(feedback_history_url, "application/json",  bytes.NewBuffer(history_json))
	wg.Done() // mark task as done in WaitGroup
}

type APIRequest struct {
	Entry string `json:"entry"`
	Prompt_id int `json:"prompt_id"`
	Session_id string `json:"session_id"`
	Attempt int `json:"attempt"`
}

type APIResponse struct {
	Feedback string `json:"feedback"`
	Feedback_type string `json:"feedback_type"`
	Optimal bool `json:"optimal"`
	Response_id string `json:"response_id"`
	Highlight []Highlight `json:"highlight"`
	Labels string `json:"labels,omitempty"`
}

type Highlight struct {
	Type string `json:"type"`
	Id int `json:"id,omitempty"`
	Text string `json:"text"`
	Category string `json:"category"`
	Character int `json:"character,omitempty"`
}

type InternalAPIResponse struct {
	Priority int
	APIResponse APIResponse
}

type HistoryAPIRequest struct {
	Entry string `json:"entry"`
	Prompt_id int `json:"prompt_id"`
	Session_id string `json:"session_id"`
	Attempt int `json:"attempt"`
	Feedback APIResponse `json:"feedback"`
}
<|endoftext|>


<commit_before>package hostedtsdb

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"log"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core/ctypes"

	"github.com/raintank/raintank-metric/schema"
)

const (
	name                 = "rt-hostedtsdb"
	version              = 1
	pluginType           = plugin.PublisherPluginType
	maxMetricsPerPayload = 3000
)

type HostedtsdbPublisher struct {
}

func NewHostedtsdbPublisher() *HostedtsdbPublisher {
	return &HostedtsdbPublisher{}
}

type Queue struct {
	sync.Mutex
	RemoteUrl *url.URL
	Token     string
	Metrics   []*schema.MetricData
}

func (q *Queue) Add(metrics []*schema.MetricData) {
	q.Lock()
	q.Metrics = append(q.Metrics, metrics...)
	flush := false
	if len(q.Metrics) > maxMetricsPerPayload {
		flush = true
	}
	q.Unlock()
	if flush {
		q.Flush()
	}
}

func (q *Queue) Flush() {
	q.Lock()
	metrics := make([]*schema.MetricData, len(q.Metrics))
	copy(metrics, q.Metrics)
	q.Metrics = q.Metrics[:0]
	q.Unlock()
	// Write the metrics to our HTTP server.
	log.Printf("writing %d metrics to API", len(metrics))
}

type WriteQueue struct {
	sync.Mutex
	Queues    map[string]*Queue
	QueueFull chan string
}

func (w *WriteQueue) Run() {
	ticker := time.NewTicker(time.Second)
	for {
		select {
		case <-ticker.C:
			w.FlushAll()
		case key := <-w.QueueFull:
			w.Lock()
			if q, ok := w.Queues[key]; ok {
				go q.Flush()
			}
			w.Unlock()
		}
	}
}

func (w *WriteQueue) FlushAll() {
	w.Lock()
	for _, q := range w.Queues {
		go q.Flush()
	}
	w.Unlock()
}

func (w *WriteQueue) Add(metrics []*schema.MetricData, url *url.URL, token string) {
	qKey := fmt.Sprintf("%s:%s", token, url.String())
	var q *Queue
	var ok bool
	w.Lock()
	if q, ok = w.Queues[qKey]; !ok {
		q = &Queue{
			RemoteUrl: url,
			Token:     token,
			Metrics:   make([]*schema.MetricData, 0),
		}
		w.Queues[qKey] = q
	}
	w.Unlock()
	q.Add(metrics)
}

func NewWriteQueue() *WriteQueue {
	return &WriteQueue{
		Queues:    make(map[string]*Queue),
		QueueFull: make(chan string, 10),
	}
}

type EventQueue struct {
}

func (e *EventQueue) Add(m *plugin.PluginMetricType) error {
	return nil
}

func NewEventQueue() *EventQueue {
	return &EventQueue{}
}

var writeQueue *WriteQueue

var eventQueue *EventQueue

func init() {
	writeQueue = NewWriteQueue()
	go writeQueue.Run()
	eventQueue = NewEventQueue()
	//go eventQueue.Run()
}

func (f *HostedtsdbPublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
	log.Println("Publishing started")
	var metrics []plugin.PluginMetricType

	switch contentType {
	case plugin.SnapGOBContentType:
		dec := gob.NewDecoder(bytes.NewBuffer(content))
		if err := dec.Decode(&metrics); err != nil {
			log.Printf("Error decoding: error=%v content=%v", err, content)
			return err
		}
	default:
		log.Printf("Error unknown content type '%v'", contentType)
		return errors.New(fmt.Sprintf("Unknown content type '%s'", contentType))
	}

	log.Printf("publishing %v metrics to %v", len(metrics), config)
	remoteUrl, err := url.Parse(config["url"].(ctypes.ConfigValueStr).Value)
	if err != nil {
		return err
	}
	token := config["token"].(ctypes.ConfigValueStr).Value
	interval := config["interval"].(ctypes.ConfigValueInt).Value

	metricsArray := make([]*schema.MetricData, len(metrics))
	for i, m := range metrics {
		tags := make([]string, 0)
		targetType := "gauge"
		unit := ""
		for k, v := range m.Tags() {
			switch k {
			case "targetType":
				targetType = v
			case "unit":
				unit = v
			default:
				tags = append(tags, fmt.Sprintf("%s:%s", k, v))
			}
		}
		var value float64
		rawData := m.Data()
		switch rawData.(type) {
		case int:
			value = float64(rawData.(int))
		case int8:
			value = float64(rawData.(int8))
		case int16:
			value = float64(rawData.(int16))
		case int32:
			value = float64(rawData.(int32))
		case int64:
			value = float64(rawData.(int64))
		case uint8:
			value = float64(rawData.(uint8))
		case uint16:
			value = float64(rawData.(uint16))
		case uint32:
			value = float64(rawData.(uint32))
		case uint64:
			value = float64(rawData.(uint64))
		case float32:
			value = float64(rawData.(float32))
		case float64:
			value = rawData.(float64)
		case string:
			//payload is an event.
			return eventQueue.Add(&m)
		default:
			return errors.New("unknown data type")
		}

		metricsArray[i] = &schema.MetricData{
			OrgId:      1,
			Name:       strings.Join(m.Namespace(), "."),
			Interval:   interval,
			Value:      value,
			Time:       m.Timestamp().Unix(),
			TargetType: targetType,
			Unit:       unit,
			Tags:       tags,
		}
	}
	writeQueue.Add(metricsArray, remoteUrl, token)

	return nil
}

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		name,
		version,
		pluginType,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Exclusive(true),
	)
}

func (f *HostedtsdbPublisher) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("url", true)
	rule2, _ := cpolicy.NewStringRule("token", true)
	rule3, _ := cpolicy.NewIntegerRule("interval", true)
	rule4, _ := cpolicy.NewIntegerRule("orgId", true)

	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule2)
	p.Add(rule3)
	p.Add(rule4)
	c.Add([]string{""}, p)
	return c, nil
}

func handleErr(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>refactor publisher now that url and token are stored globally.<commit_after>package hostedtsdb

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core/ctypes"

	"github.com/raintank/raintank-metric/msg"
	"github.com/raintank/raintank-metric/schema"
)

const (
	name                 = "rt-hostedtsdb"
	version              = 1
	pluginType           = plugin.PublisherPluginType
	maxMetricsPerPayload = 3000
)

var (
	RemoteUrl *url.URL
	Token     string
)

type HostedtsdbPublisher struct {
}

func NewHostedtsdbPublisher() *HostedtsdbPublisher {
	return &HostedtsdbPublisher{}
}

type WriteQueue struct {
	sync.Mutex
	Metrics   []*schema.MetricData
	QueueFull chan struct{}
}

func (q *WriteQueue) Add(metrics []*schema.MetricData) {
	q.Lock()
	q.Metrics = append(q.Metrics, metrics...)
	if len(q.Metrics) > maxMetricsPerPayload {
		q.QueueFull <- struct{}{}
	}
	q.Unlock()
}

func (q *WriteQueue) Flush() {
	q.Lock()
	if len(q.Metrics) == 0 {
		q.Unlock()
		return
	}
	metrics := make([]*schema.MetricData, len(q.Metrics))
	copy(metrics, q.Metrics)
	q.Metrics = q.Metrics[:0]
	q.Unlock()
	// Write the metrics to our HTTP server.
	log.Printf("writing %d metrics to API", len(metrics))
	id := time.Now().UnixNano()
	body, err := msg.CreateMsg(metrics, id, msg.FormatMetricDataArrayMsgp)
	if err != nil {
		log.Printf("Error: unable to convert metrics to MetricDataArrayMsgp. %s", err)
		return
	}
	sent := false
	for !sent {
		if err = PostData(RemoteUrl, Token, body); err != nil {
			log.Printf("Error: %s", err)
			time.Sleep(time.Second)
		} else {
			sent = true
		}
	}
}

func (q *WriteQueue) Run() {
	ticker := time.NewTicker(time.Second)
	for {
		select {
		case <-ticker.C:
			q.Flush()
		case <-q.QueueFull:
			q.Flush()
		}
	}
}

func NewWriteQueue() *WriteQueue {
	return &WriteQueue{
		Metrics:   make([]*schema.MetricData, 0),
		QueueFull: make(chan struct{}),
	}
}

type EventQueue struct {
}

func (e *EventQueue) Add(m *plugin.PluginMetricType) error {
	return nil
}

func NewEventQueue() *EventQueue {
	return &EventQueue{}
}

var writeQueue *WriteQueue

var eventQueue *EventQueue

func init() {
	writeQueue = NewWriteQueue()
	go writeQueue.Run()
	eventQueue = NewEventQueue()
	//go eventQueue.Run()
}

func (f *HostedtsdbPublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
	log.Println("Publishing started")
	var metrics []plugin.PluginMetricType

	switch contentType {
	case plugin.SnapGOBContentType:
		dec := gob.NewDecoder(bytes.NewBuffer(content))
		if err := dec.Decode(&metrics); err != nil {
			log.Printf("Error decoding: error=%v content=%v", err, content)
			return err
		}
	default:
		log.Printf("Error unknown content type '%v'", contentType)
		return errors.New(fmt.Sprintf("Unknown content type '%s'", contentType))
	}

	log.Printf("publishing %v metrics to %v", len(metrics), config)

	// set the RemoteURL and Token when the first metrics is recieved.
	var err error
	if RemoteUrl == nil {
		RemoteUrl, err = url.Parse(config["url"].(ctypes.ConfigValueStr).Value)
		if err != nil {
			return err
		}
	}
	if Token == "" {
		Token = config["token"].(ctypes.ConfigValueStr).Value
	}
	//-----------------

	interval := config["interval"].(ctypes.ConfigValueInt).Value
	orgId := config["orgId"].(ctypes.ConfigValueInt).Value

	metricsArray := make([]*schema.MetricData, len(metrics))
	for i, m := range metrics {
		tags := make([]string, 0)
		targetType := "gauge"
		unit := ""
		for k, v := range m.Tags() {
			switch k {
			case "targetType":
				targetType = v
			case "unit":
				unit = v
			default:
				tags = append(tags, fmt.Sprintf("%s:%s", k, v))
			}
		}
		var value float64
		rawData := m.Data()
		switch rawData.(type) {
		case int:
			value = float64(rawData.(int))
		case int8:
			value = float64(rawData.(int8))
		case int16:
			value = float64(rawData.(int16))
		case int32:
			value = float64(rawData.(int32))
		case int64:
			value = float64(rawData.(int64))
		case uint8:
			value = float64(rawData.(uint8))
		case uint16:
			value = float64(rawData.(uint16))
		case uint32:
			value = float64(rawData.(uint32))
		case uint64:
			value = float64(rawData.(uint64))
		case float32:
			value = float64(rawData.(float32))
		case float64:
			value = rawData.(float64)
		case string:
			//payload is an event.
			return eventQueue.Add(&m)
		default:
			return errors.New("unknown data type")
		}

		metricsArray[i] = &schema.MetricData{
			OrgId:      orgId,
			Name:       strings.Join(m.Namespace(), "."),
			Interval:   interval,
			Value:      value,
			Time:       m.Timestamp().Unix(),
			TargetType: targetType,
			Unit:       unit,
			Tags:       tags,
		}
	}
	writeQueue.Add(metricsArray)

	return nil
}

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		name,
		version,
		pluginType,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Exclusive(true),
	)
}

func (f *HostedtsdbPublisher) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("url", true)
	rule2, _ := cpolicy.NewStringRule("token", true)
	rule3, _ := cpolicy.NewIntegerRule("interval", true)
	rule4, _ := cpolicy.NewIntegerRule("orgId", true)

	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule2)
	p.Add(rule3)
	p.Add(rule4)
	c.Add([]string{""}, p)
	return c, nil
}

func handleErr(e error) {
	if e != nil {
		panic(e)
	}
}

func PostData(remoteUrl *url.URL, token string, body []byte) error {
	req, err := http.NewRequest("POST", remoteUrl.String(), bytes.NewBuffer(body))
	req.Header.Set("Content-Type", "rt-metric-binary")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	respBody, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("Posting data failed. %d - %s", resp.StatusCode, string(respBody))
	}
	return nil
}
<|endoftext|>


<commit_before>// cmd_measurements.go

package main

import (
	"fmt"
	"github.com/urfave/cli"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"github.com/keltia/ripe-atlas"
)

// init injects our measurement-related commands
func init() {
	cliCommands = append(cliCommands, cli.Command{
		Name: "measurements",
		Aliases: []string{
			"measures",
			"m",
		},
		Usage:       "measurements-related keywords",
		Description: "All the commands for measurements",
		Subcommands: []cli.Command{
			{
				Name:        "list",
				Aliases:     []string{"ls"},
				Usage:       "lists all measurements",
				Description: "displays all measurements",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:        "country, c",
						Usage:       "filter on country",
						Destination: &fCountry,
					},
					cli.StringFlag{
						Name:        "asn",
						Usage:       "filter on asn",
						Value:       "",
						Destination: &fAsn,
					},
					cli.BoolFlag{
						Name:        "A, all",
						Usage:       "all measurements even inactive ones",
						Destination: &fAllMeasurements,
					},
					cli.BoolFlag{
						Name:        "is-anchor",
						Usage:       "select anchor measurements",
						Destination: &fWantAnchor,
					},
					cli.BoolTFlag{
						Name:        "mine",
						Usage:       "limit to my own measurements",
						Destination: &fWantMine,
					},
					cli.StringFlag{
						Name:        "t, type",
						Usage:       "filter on measurement type",
						Value:       "",
						Destination: &fMeasureType,
					},
				},
				Action: measurementsList,
			},
			{
				Name:        "info",
				Usage:       "info for one measurement",
				Description: "gives info for one measurement",
				Action:      measurementInfo,
			},
			{
				Name:        "results",
				Aliases:     []string{"r", "res"},
				Usage:       "results for one measurement",
				Description: "returns results for one measurement",
				Action:      measurementResults,
			},
			{
				Name:        "delete",
				Aliases:     []string{"rm", "del", "destroy"},
				Usage:       "info for one measurement",
				Description: "stops one measurement (or all)",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:        "A",
						Usage:       "select all measurements",
						Destination: &fAllMeasurements,
					},
				},
				Action: measurementDelete,
			},
		},
	})
}

// displayMeasurement returns a string with one measurement
func displayMeasurement(m *atlas.Measurement, verbose bool) (res string) {
	if verbose {
		res = fmt.Sprintf("%v\n", m)
	} else {
		res = fmt.Sprintf("ID: %d type: %s description: %s Target: %s\n", m.ID, m.Type, m.Description, m.Target)
	}
	return
}

// displayAllMeasurements returns a string with all measurements
func displayAllMeasurements(ml *[]atlas.Measurement, verbose bool) (res string) {
	res = ""
	for _, m := range *ml {
		res += displayMeasurement(&m, verbose)
	}
	return
}

// displayResult returns a string with <obj>.Result
func displayResult(body []byte, verbose bool) (res string) {
	return string(body)
}

// measurementsList returns a list of measurements according to parameters
func measurementsList(c *cli.Context) error {
	opts := make(map[string]string)

	if fCountry != "" {
		opts["country_code"] = fCountry
	}

	if fAsn != "" {
		opts["asn"] = fAsn
	}

	if fMeasureType != "" {
		opts["type"] = fMeasureType
	}

	if fWantMine {
		opts["mine"] = "true"
	}

	// Check global parameters
	opts = checkGlobalFlags(opts)

	if fVerbose {
		displayOptions(opts)
	}

	q, err := client.GetMeasurements(opts)
	if err != nil {
		log.Printf("GetMeasurements err: %v - q:%v", err, q)
		os.Exit(1)
	}
	log.Printf("Got %d measurements with %v\n", len(q), opts)
	fmt.Print(displayAllMeasurements(&q, fVerbose))

	return nil
}

// measurementInfo is for one measurement only
func measurementInfo(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	p, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}
	fmt.Print(displayMeasurement(p, fVerbose))

	return nil
}

// measurementResults returns the result part of the measurement
func measurementResults(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	m, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}

	// m.Result is an URI pointing to results, fetch it
	if m.Result == "" {
		fmt.Println("Empty result")
	}

	resp, err := http.Get(m.Result)
	if err != nil {
		err = fmt.Errorf("bad net/http answer for %s: %v", m.Result, err)
		return err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("error reading body for %s: %v", m.Result, err)
		return err
	}

	fmt.Print(displayResult(body, fVerbose))
	return nil
}

func measurementCreate(c *cli.Context) error {
	return nil
}

func measurementDelete(c *cli.Context) (err error) {

	if fAllMeasurements {
		opts := make(map[string]string)

		// Check global parameters
		opts = checkGlobalFlags(opts)

		if fVerbose {
			displayOptions(opts)
		}

		list, err := client.GetMeasurements(opts)
		if err != nil {
			err = fmt.Errorf("Delete all failed: %v", err)
		} else {
			for _, m := range list {
				err = client.DeleteMeasurement(m.ID)
				if err != nil {
					err = fmt.Errorf("Error: can not delete measurement %d", m.ID)
				}
			}
			fmt.Println("All measurements stopped.")
		}
	} else {
		args := c.Args()
		if args[0] == "" {
			log.Fatal("Error: you must specify a measurement ID!")
		}

		id, _ := strconv.Atoi(args[0])
		err = client.DeleteMeasurement(id)
	}

	return
}
<commit_msg>"WantMine" is global.<commit_after>// cmd_measurements.go

package main

import (
	"fmt"
	"github.com/urfave/cli"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"github.com/keltia/ripe-atlas"
)

// init injects our measurement-related commands
func init() {
	cliCommands = append(cliCommands, cli.Command{
		Name: "measurements",
		Aliases: []string{
			"measures",
			"m",
		},
		Usage:       "measurements-related keywords",
		Description: "All the commands for measurements",
		Subcommands: []cli.Command{
			{
				Name:        "list",
				Aliases:     []string{"ls"},
				Usage:       "lists all measurements",
				Description: "displays all measurements",
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:        "country, c",
						Usage:       "filter on country",
						Destination: &fCountry,
					},
					cli.StringFlag{
						Name:        "asn",
						Usage:       "filter on asn",
						Value:       "",
						Destination: &fAsn,
					},
					cli.BoolFlag{
						Name:        "A, all",
						Usage:       "all measurements even inactive ones",
						Destination: &fAllMeasurements,
					},
					cli.BoolFlag{
						Name:        "is-anchor",
						Usage:       "select anchor measurements",
						Destination: &fWantAnchor,
					},
					cli.StringFlag{
						Name:        "t, type",
						Usage:       "filter on measurement type",
						Value:       "",
						Destination: &fMeasureType,
					},
				},
				Action: measurementsList,
			},
			{
				Name:        "info",
				Usage:       "info for one measurement",
				Description: "gives info for one measurement",
				Action:      measurementInfo,
			},
			{
				Name:        "results",
				Aliases:     []string{"r", "res"},
				Usage:       "results for one measurement",
				Description: "returns results for one measurement",
				Action:      measurementResults,
			},
			{
				Name:        "delete",
				Aliases:     []string{"rm", "del", "destroy"},
				Usage:       "info for one measurement",
				Description: "stops one measurement (or all)",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:        "A",
						Usage:       "select all measurements",
						Destination: &fAllMeasurements,
					},
				},
				Action: measurementDelete,
			},
		},
	})
}

// displayMeasurement returns a string with one measurement
func displayMeasurement(m *atlas.Measurement, verbose bool) (res string) {
	if verbose {
		res = fmt.Sprintf("%v\n", m)
	} else {
		res = fmt.Sprintf("ID: %d type: %s description: %s Target: %s\n", m.ID, m.Type, m.Description, m.Target)
	}
	return
}

// displayAllMeasurements returns a string with all measurements
func displayAllMeasurements(ml *[]atlas.Measurement, verbose bool) (res string) {
	res = ""
	for _, m := range *ml {
		res += displayMeasurement(&m, verbose)
	}
	return
}

// displayResult returns a string with <obj>.Result
func displayResult(body []byte, verbose bool) (res string) {
	return string(body)
}

// measurementsList returns a list of measurements according to parameters
func measurementsList(c *cli.Context) error {
	opts := make(map[string]string)

	if fCountry != "" {
		opts["country_code"] = fCountry
	}

	if fAsn != "" {
		opts["asn"] = fAsn
	}

	if fMeasureType != "" {
		opts["type"] = fMeasureType
	}

	// Check global parameters
	opts = checkGlobalFlags(opts)

	if fVerbose {
		displayOptions(opts)
	}

	q, err := client.GetMeasurements(opts)
	if err != nil {
		log.Printf("GetMeasurements err: %v - q:%v", err, q)
		os.Exit(1)
	}
	log.Printf("Got %d measurements with %v\n", len(q), opts)
	fmt.Print(displayAllMeasurements(&q, fVerbose))

	return nil
}

// measurementInfo is for one measurement only
func measurementInfo(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	p, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}
	fmt.Print(displayMeasurement(p, fVerbose))

	return nil
}

// measurementResults returns the result part of the measurement
func measurementResults(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a measurement ID!")
	}

	id, _ := strconv.Atoi(args[0])

	m, err := client.GetMeasurement(id)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}

	// m.Result is an URI pointing to results, fetch it
	if m.Result == "" {
		fmt.Println("Empty result")
	}

	resp, err := http.Get(m.Result)
	if err != nil {
		err = fmt.Errorf("bad net/http answer for %s: %v", m.Result, err)
		return err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("error reading body for %s: %v", m.Result, err)
		return err
	}

	fmt.Print(displayResult(body, fVerbose))
	return nil
}

func measurementCreate(c *cli.Context) error {
	return nil
}

func measurementDelete(c *cli.Context) (err error) {

	if fAllMeasurements {
		opts := make(map[string]string)

		// Check global parameters
		opts = checkGlobalFlags(opts)

		if fVerbose {
			displayOptions(opts)
		}

		list, err := client.GetMeasurements(opts)
		if err != nil {
			err = fmt.Errorf("Delete all failed: %v", err)
		} else {
			for _, m := range list {
				err = client.DeleteMeasurement(m.ID)
				if err != nil {
					err = fmt.Errorf("Error: can not delete measurement %d", m.ID)
				}
			}
			fmt.Println("All measurements stopped.")
		}
	} else {
		args := c.Args()
		if args[0] == "" {
			log.Fatal("Error: you must specify a measurement ID!")
		}

		id, _ := strconv.Atoi(args[0])
		err = client.DeleteMeasurement(id)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"testing"

	"golang.org/x/tools/benchmark/parse"
)

func TestSelectBest(t *testing.T) {
	have := parse.Set{
		"Benchmark1": []*parse.Benchmark{
			{
				Name: "Benchmark1",
				N:    10, NsPerOp: 100, Measured: parse.NsPerOp,
				Ord: 0,
			},
			{
				Name: "Benchmark1",
				N:    10, NsPerOp: 50, Measured: parse.NsPerOp,
				Ord: 3,
			},
		},
		"Benchmark2": []*parse.Benchmark{
			{
				Name: "Benchmark2",
				N:    10, NsPerOp: 60, Measured: parse.NsPerOp,
				Ord: 1,
			},
			{
				Name: "Benchmark2",
				N:    10, NsPerOp: 500, Measured: parse.NsPerOp,
				Ord: 2,
			},
		},
	}

	want := parse.Set{
		"Benchmark1": []*parse.Benchmark{
			{
				Name: "Benchmark1",
				N:    10, NsPerOp: 50, Measured: parse.NsPerOp,
				Ord: 0,
			},
		},
		"Benchmark2": []*parse.Benchmark{
			{
				Name: "Benchmark2",
				N:    10, NsPerOp: 60, Measured: parse.NsPerOp,
				Ord: 1,
			},
		},
	}

	selectBest(have)
	if !reflect.DeepEqual(want, have) {
		t.Errorf("filtered bench set incorrectly, want %v have %v", want, have)
	}
}
<commit_msg>cmd/benchcmp: add unit test for formatNs function<commit_after>package main

import (
	"reflect"
	"testing"

	"golang.org/x/tools/benchmark/parse"
)

func TestSelectBest(t *testing.T) {
	have := parse.Set{
		"Benchmark1": []*parse.Benchmark{
			{
				Name: "Benchmark1",
				N:    10, NsPerOp: 100, Measured: parse.NsPerOp,
				Ord: 0,
			},
			{
				Name: "Benchmark1",
				N:    10, NsPerOp: 50, Measured: parse.NsPerOp,
				Ord: 3,
			},
		},
		"Benchmark2": []*parse.Benchmark{
			{
				Name: "Benchmark2",
				N:    10, NsPerOp: 60, Measured: parse.NsPerOp,
				Ord: 1,
			},
			{
				Name: "Benchmark2",
				N:    10, NsPerOp: 500, Measured: parse.NsPerOp,
				Ord: 2,
			},
		},
	}

	want := parse.Set{
		"Benchmark1": []*parse.Benchmark{
			{
				Name: "Benchmark1",
				N:    10, NsPerOp: 50, Measured: parse.NsPerOp,
				Ord: 0,
			},
		},
		"Benchmark2": []*parse.Benchmark{
			{
				Name: "Benchmark2",
				N:    10, NsPerOp: 60, Measured: parse.NsPerOp,
				Ord: 1,
			},
		},
	}

	selectBest(have)
	if !reflect.DeepEqual(want, have) {
		t.Errorf("filtered bench set incorrectly, want %v have %v", want, have)
	}
}

func TestFormatNs(t *testing.T) {
	tests := []struct {
		input    float64
		expected string
	}{
		{input: 0, expected: "0.00"},
		{input: 0.2, expected: "0.20"},
		{input: 2, expected: "2.00"},
		{input: 2.2, expected: "2.20"},
		{input: 4, expected: "4.00"},
		{input: 16, expected: "16.0"},
		{input: 16.08, expected: "16.1"},
		{input: 128, expected: "128"},
		{input: 256.2, expected: "256"},
	}

	for _, tt := range tests {
		actual := formatNs(tt.input)
		if actual != tt.expected {
			t.Fatalf("%f. got %q, want %q", tt.input, actual, tt.expected)
		}
	}
}
<|endoftext|>


<commit_before>package openings

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/logs"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/symmetry"
	"github.com/nelhage/taktician/tak"
)

type Command struct {
	logDir    string
	size      int
	minRating int
	minCount  int
	maxDepth  int
}

func (*Command) Name() string     { return "openings" }
func (*Command) Synopsis() string { return "Analyze openings from the playtak DB" }
func (*Command) Usage() string {
	return `openings [flags]
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.StringVar(&c.logDir, "games", "games", "game log directory")
	flags.IntVar(&c.size, "size", 5, "what size to analyze")
	flags.IntVar(&c.minRating, "rating", 1600, "minimum rating to consider")
	flags.IntVar(&c.minCount, "count", 100, "render games with >= [this many] moves")
	flags.IntVar(&c.maxDepth, "depth", 8, "track tree to this many plies")
}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	repo, e := logs.Open(flag.Arg(0))
	if e != nil {
		log.Fatalf("parse %s: %v", flag.Arg(0), e)
	}
	defer repo.Close()
	sql := repo.DB()

	rows, e := sql.Query(
		`
SELECT day, id
FROM games g, ratings r1, ratings r2
WHERE r1.name = g.player1
 AND r2.name = g.player2
 AND NOT r1.bot AND NOT r2.bot
 AND r1.rating >= ?
 AND r2.rating >= ?
 AND g.size = ?
`, c.minRating, c.minRating, c.size)
	if e != nil {
		log.Fatal("select: ", e)
	}
	defer rows.Close()

	tree := &tree{}

	for rows.Next() {
		var day string
		var id int
		e = rows.Scan(&day, &id)
		if e != nil {
			panic(e)
		}

		ptnPath := path.Join(c.logDir, day, fmt.Sprintf("%d.ptn", id))
		g, e := ptn.ParseFile(ptnPath)
		if e != nil {
			log.Printf("parse %s: %v", ptnPath, e)
			continue
		}
		p, e := g.PositionAtMove(0, tak.NoColor)
		if e != nil {
			log.Printf("parse %s: %v", ptnPath, e)
			continue
		}

		var ms []tak.Move
		for _, o := range g.Ops {
			if m, ok := o.(*ptn.Move); ok {
				ms = append(ms, m.Move)
				if len(ms) >= 10 {
					break
				}
			}
		}
		ms, e = symmetry.Canonical(p.Size(), ms)
		if e != nil {
			log.Printf("%s: %v", ptnPath, e)
			continue
		}

		if len(ms) > c.maxDepth {
			ms = ms[:c.maxDepth]
		}

		result := ptn.Result{Result: g.FindTag("Result")}

		insertTree(tree, ms, result.Winner())
	}

	bs, _ := json.Marshal(tree)
	ioutil.WriteFile("gametree.json", bs, 0644)
	f, e := os.Create("gametree.dot")
	defer f.Close()
	c.writeTree(f, tree)
	fmt.Printf("Common openings:\n")
	c.printLines(tree)

	line := c.minmax(tree)
	fmt.Printf("Best line:\n")
	for i, t := range line {
		dots := ""
		if i%2 == 1 {
			dots = ".. "
		}
		fmt.Printf("%d. %s%s %d-%d %0.0f%%\n",
			i/2+1, dots, t.Move, t.White, t.Black,
			100*float64(t.White)/float64(t.Count))
	}

	return subcommands.ExitSuccess
}

type tree struct {
	id int

	Move     string  `json:",omitempty"`
	Children []*tree `json:",omitempty"`
	Count    int
	White    int
	Black    int
}

var nextID = 1

func insertTree(t *tree, ms []tak.Move, winner tak.Color) {
	t.Count++
	switch winner {
	case tak.White:
		t.White++
	case tak.Black:
		t.Black++
	}
	if len(ms) == 0 {
		return
	}
	var child *tree
	m := ptn.FormatMove(ms[0])
	for _, ch := range t.Children {
		if ch.Move == m {
			child = ch
			break
		}
	}
	if child == nil {
		child = &tree{Move: m, id: nextID}
		nextID++
		t.Children = append(t.Children, child)
	}
	insertTree(child, ms[1:], winner)
}

func (c *Command) writeTree(f io.Writer, t *tree) {
	fmt.Fprintf(f, "digraph G {\n")
	c.writeTreeNode(0, f, t)
	fmt.Fprintf(f, "}\n")
}

func (c *Command) writeTreeNode(ply int, f io.Writer, t *tree) {
	var mno string
	move := ply/2 + 1
	if ply%2 == 0 {
		mno = fmt.Sprintf("%d. ", move)
	} else {
		mno = fmt.Sprintf("%d. .. ", move)
	}

	fmt.Fprintf(f, `  n%d [shape=box, label="%s %d-%d/%0.0f%%"]`,
		t.id, t.Move, t.White, t.Black, 100*float64(t.White)/float64(t.Count))
	fmt.Fprintln(f)
	for _, ch := range t.Children {
		if ch.Count < c.minCount {
			continue
		}
		fmt.Fprintf(f, `  n%d -> n%d [label="%s%s %d/%0.0f%%"]`,
			t.id, ch.id, mno, ch.Move,
			ch.Count, 100*float64(ch.Count)/float64(t.Count))
		fmt.Fprintln(f)
		c.writeTreeNode(ply+1, f, ch)
	}
}

func (c *Command) printLines(t *tree) {
	c.walkLines([]*tree{}, t)
}

func (c *Command) walkLines(line []*tree, t *tree) {
	found := false
	for _, ch := range t.Children {
		if ch.Count >= c.minCount && float64(ch.Count) >= 0.05*float64(t.Count) {
			c.walkLines(append(line, t), ch)
			found = true
		}
	}
	if !found {
		for _, m := range line {
			if m.Move == "" {
				continue
			}
			fmt.Printf("%s ", m.Move)
		}
		fmt.Printf("%s\n", t.Move)
	}
}

func (c *Command) minmax(t *tree) []*tree {
	var line []*tree
	who := tak.White
	for t != nil {
		var best *tree
		var max float64 = -1
		for _, ch := range t.Children {
			if ch.Count < c.minCount {
				continue
			}
			var wins int
			if who == tak.White {
				wins = ch.White
			} else {
				wins = ch.Black
			}
			score := float64(wins) / float64(ch.Count)
			if score > max {
				max = score
				best = ch
			}
		}
		if best != nil {
			line = append(line, best)
		}
		t = best
		who = who.Flip()
	}
	return line
}
<commit_msg>port opening analysis to playtak db<commit_after>package openings

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/logs"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/symmetry"
	"github.com/nelhage/taktician/tak"
)

type Command struct {
	size      int
	minRating int
	minCount  int
	maxDepth  int
}

func (*Command) Name() string     { return "openings" }
func (*Command) Synopsis() string { return "Analyze openings from the playtak DB" }
func (*Command) Usage() string {
	return `openings [flags] GAMES.db`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.size, "size", 5, "what size to analyze")
	flags.IntVar(&c.minRating, "rating", 1600, "minimum rating to consider")
	flags.IntVar(&c.minCount, "count", 100, "render games with >= [this many] moves")
	flags.IntVar(&c.maxDepth, "depth", 8, "track tree to this many plies")
}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	repo, e := logs.Open(flag.Arg(0))
	if e != nil {
		log.Fatalf("open %s: %v", flag.Arg(0), e)
	}
	defer repo.Close()
	sql := repo.DB()

	rows, e := sql.Query(
		`
SELECT g.id, p.ptn
FROM games g, ratings r1, ratings r2, ptns p
WHERE r1.name = g.player_white
 AND r2.name = g.player_black
 AND NOT r1.bot AND NOT r2.bot
 AND r1.rating >= ?
 AND r2.rating >= ?
 AND g.size = ?
 AND p.id = g.id
 AND p.id IS NOT NULL
`, c.minRating, c.minRating, c.size)
	if e != nil {
		log.Fatal("select: ", e)
	}
	defer rows.Close()

	tree := &tree{}

	for rows.Next() {
		var id int
		var notation string
		e = rows.Scan(&id, &notation)
		if e != nil {
			panic(e)
		}

		g, e := ptn.ParsePTN(strings.NewReader(notation))
		if e != nil {
			log.Printf("parse %d: %v", id, e)
			continue
		}

		p, e := g.PositionAtMove(0, tak.NoColor)
		if e != nil {
			log.Printf("parse %d: %v", id, e)
			continue
		}

		var ms []tak.Move
		for _, o := range g.Ops {
			if m, ok := o.(*ptn.Move); ok {
				ms = append(ms, m.Move)
				if len(ms) >= 10 {
					break
				}
			}
		}
		ms, e = symmetry.Canonical(p.Size(), ms)
		if e != nil {
			log.Printf("%d: %v", id, e)
			continue
		}

		if len(ms) > c.maxDepth {
			ms = ms[:c.maxDepth]
		}

		result := ptn.Result{Result: g.FindTag("Result")}

		insertTree(tree, ms, result.Winner())
	}

	bs, _ := json.Marshal(tree)
	ioutil.WriteFile("gametree.json", bs, 0644)
	f, e := os.Create("gametree.dot")
	defer f.Close()
	c.writeTree(f, tree)
	fmt.Printf("Common openings:\n")
	c.printLines(tree)

	line := c.minmax(tree)
	fmt.Printf("Best line:\n")
	for i, t := range line {
		dots := ""
		if i%2 == 1 {
			dots = ".. "
		}
		fmt.Printf("%d. %s%s %d-%d %0.0f%%\n",
			i/2+1, dots, t.Move, t.White, t.Black,
			100*float64(t.White)/float64(t.Count))
	}

	return subcommands.ExitSuccess
}

type tree struct {
	id int

	Move     string  `json:",omitempty"`
	Children []*tree `json:",omitempty"`
	Count    int
	White    int
	Black    int
}

var nextID = 1

func insertTree(t *tree, ms []tak.Move, winner tak.Color) {
	t.Count++
	switch winner {
	case tak.White:
		t.White++
	case tak.Black:
		t.Black++
	}
	if len(ms) == 0 {
		return
	}
	var child *tree
	m := ptn.FormatMove(ms[0])
	for _, ch := range t.Children {
		if ch.Move == m {
			child = ch
			break
		}
	}
	if child == nil {
		child = &tree{Move: m, id: nextID}
		nextID++
		t.Children = append(t.Children, child)
	}
	insertTree(child, ms[1:], winner)
}

func (c *Command) writeTree(f io.Writer, t *tree) {
	fmt.Fprintf(f, "digraph G {\n")
	c.writeTreeNode(0, f, t)
	fmt.Fprintf(f, "}\n")
}

func (c *Command) writeTreeNode(ply int, f io.Writer, t *tree) {
	var mno string
	move := ply/2 + 1
	if ply%2 == 0 {
		mno = fmt.Sprintf("%d. ", move)
	} else {
		mno = fmt.Sprintf("%d. .. ", move)
	}

	fmt.Fprintf(f, `  n%d [shape=box, label="%s %d-%d/%0.0f%%"]`,
		t.id, t.Move, t.White, t.Black, 100*float64(t.White)/float64(t.Count))
	fmt.Fprintln(f)
	for _, ch := range t.Children {
		if ch.Count < c.minCount {
			continue
		}
		fmt.Fprintf(f, `  n%d -> n%d [label="%s%s %d/%0.0f%%"]`,
			t.id, ch.id, mno, ch.Move,
			ch.Count, 100*float64(ch.Count)/float64(t.Count))
		fmt.Fprintln(f)
		c.writeTreeNode(ply+1, f, ch)
	}
}

func (c *Command) printLines(t *tree) {
	c.walkLines([]*tree{}, t)
}

func (c *Command) walkLines(line []*tree, t *tree) {
	found := false
	for _, ch := range t.Children {
		if ch.Count >= c.minCount && float64(ch.Count) >= 0.05*float64(t.Count) {
			c.walkLines(append(line, t), ch)
			found = true
		}
	}
	if !found {
		for _, m := range line {
			if m.Move == "" {
				continue
			}
			fmt.Printf("%s ", m.Move)
		}
		fmt.Printf("%s\n", t.Move)
	}
}

func (c *Command) minmax(t *tree) []*tree {
	var line []*tree
	who := tak.White
	for t != nil {
		var best *tree
		var max float64 = -1
		for _, ch := range t.Children {
			if ch.Count < c.minCount {
				continue
			}
			var wins int
			if who == tak.White {
				wins = ch.White
			} else {
				wins = ch.Black
			}
			score := float64(wins) / float64(ch.Count)
			if score > max {
				max = score
				best = ch
			}
		}
		if best != nil {
			line = append(line, best)
		}
		t = best
		who = who.Flip()
	}
	return line
}
<|endoftext|>