<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"sync"

	"code.google.com/p/go.net/websocket"
)

const socketPresent = true

func HandleSocket(path string) {
	http.Handle(path, websocket.Handler(socketHandler))
}

const msgLimit = 1000 // max number of messages to send per session

var uniq = make(chan int) // a source of numbers for naming temporary files

func init() {
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()
}

// Message is the wire format for the websocket connection to the browser.
// It is used for both sending output messages and receiving commands, as
// distinguished by the Kind field.
type Message struct {
	Id   string // client-provided unique id for the process
	Kind string // in: "run", "kill" out: "stdout", "stderr", "end"
	Body string
}

// socketHandler handles the websocket connection for a given present session.
// It constructs a new Client and handles transcoding Messages to and from JSON
// format, sending and receiving those messages on the Client's in and out
// channels.
func socketHandler(conn *websocket.Conn) {
	in, out := make(chan *Message), make(chan *Message)
	c := &Client{
		proc: make(map[string]*Process),
		in:   in,
		out:  out,
	}
	go c.loop()

	errc := make(chan error, 1)

	// Decode messages from client and send to the in channel.
	go func() {
		dec := json.NewDecoder(conn)
		for {
			m := new(Message)
			if err := dec.Decode(m); err != nil {
				errc <- err
				close(in)
				return
			}
			in <- m
		}
	}()

	// Receive messages from the out channel and encode to the client.
	go func() {
		enc := json.NewEncoder(conn)
		counts := make(map[string]int)
		for m := range out {
			cnt := counts[m.Id]
			switch {
			case m.Kind == "end" || cnt < msgLimit:
				if err := enc.Encode(m); err != nil {
					errc <- err
					return
				}
				if m.Kind == "end" {
					delete(counts, m.Id)
				}
			case cnt == msgLimit:
				// Process produced too much output. Kill it.
				c.kill(m.Id)
			}
			counts[m.Id]++
		}
	}()

	// Wait for one of the send or receive goroutines to exit.
	if err := <-errc; err != nil && err != io.EOF {
		log.Println(err)
	}

	// Kill any running processes associated with this Client.
	c.Lock()
	for _, p := range c.proc {
		p.kill()
	}
	c.Unlock()
}

// Client represents a connected present client.
// It manages any processes being compiled and run for the client.
type Client struct {
	sync.Mutex // guards proc
	proc       map[string]*Process
	in         <-chan *Message
	out        chan<- *Message
}

// loop handles incoming messages from the client.
func (c *Client) loop() {
	for m := range c.in {
		switch m.Kind {
		case "run":
			c.kill(m.Id)
			go c.run(m.Id, m.Body)
		case "kill":
			c.kill(m.Id)
		}
	}
}

// kill shuts down a running process.
func (c *Client) kill(id string) {
	c.Lock()
	defer c.Unlock()
	if p := c.proc[id]; p != nil {
		p.kill()
	}
}

// run compiles and runs the given program, associating it with the given id.
func (c *Client) run(id, body string) {
	p := NewProcess(id, c.out)
	c.Lock()
	c.proc[id] = p
	c.Unlock()
	err := p.run(body)
	m := &Message{Id: id, Kind: "end"}
	if err != nil {
		m.Body = err.Error()
	}
	c.Lock()
	delete(c.proc, id)
	c.Unlock()
	c.out <- m
}

// Process represents a running process.
type Process struct {
	id             string
	stdout, stderr io.Writer

	sync.Mutex // guards cmd
	cmd        *exec.Cmd
	done       chan struct{} // closed when run complete
}

func NewProcess(id string, out chan<- *Message) *Process {
	return &Process{
		id:     id,
		stdout: newPiper(id, "stdout", out),
		stderr: newPiper(id, "stderr", out),
		done:   make(chan struct{}),
	}
}

// run compiles and runs the given go program.
func (p *Process) run(body string) error {
	defer close(p.done)

	// x is the base name for .go and executable files
	x := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := x + ".go"
	bin := x
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// write body to x.go
	defer os.Remove(src)
	if err := ioutil.WriteFile(src, []byte(body), 0666); err != nil {
		return err
	}

	// build x.go, creating x
	dir, file := filepath.Split(src)
	err := p.exec(dir, "go", "build", "-o", bin, file)
	defer os.Remove(bin)
	if err != nil {
		return err
	}

	// run x
	return p.exec("", bin)
}

// exec runs the specified command in the given directory, writing all standard
// output and standard error to the Process' stdout and stderr fields. It
// stores the running command in the cmd field, and returns when the command
// exits.
func (p *Process) exec(dir string, args ...string) error {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = p.stdout
	cmd.Stderr = p.stderr

	if err := cmd.Start(); err != nil {
		return err
	}

	p.Lock()
	p.cmd = cmd
	p.Unlock()

	err := cmd.Wait()

	p.Lock()
	p.cmd = nil
	p.Unlock()

	return err
}

// kill stops the process if it is running and waits for it to exit.
func (p *Process) kill() {
	p.Lock()
	if p.cmd != nil {
		p.cmd.Process.Kill()
	}
	p.Unlock()
	<-p.done
}

// newPiper returns a writer that converts all writes to Message sends on the
// given channel with the specified id and kind.
func newPiper(id, kind string, out chan<- *Message) io.Writer {
	return &piper{id, kind, out}
}

type piper struct {
	id, kind string
	out      chan<- *Message
}

func (p *piper) Write(b []byte) (n int, err error) {
	p.out <- &Message{
		Id:   p.id,
		Kind: p.kind,
		Body: string(b),
	}
	return len(b), nil
}

var tmpdir string

func init() {
	// find real path to temporary directory
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>go.talks/present: rename piper to messageWriter<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"sync"

	"code.google.com/p/go.net/websocket"
)

const socketPresent = true

func HandleSocket(path string) {
	http.Handle(path, websocket.Handler(socketHandler))
}

const msgLimit = 1000 // max number of messages to send per session

var uniq = make(chan int) // a source of numbers for naming temporary files

func init() {
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()
}

// Message is the wire format for the websocket connection to the browser.
// It is used for both sending output messages and receiving commands, as
// distinguished by the Kind field.
type Message struct {
	Id   string // client-provided unique id for the process
	Kind string // in: "run", "kill" out: "stdout", "stderr", "end"
	Body string
}

// socketHandler handles the websocket connection for a given present session.
// It constructs a new Client and handles transcoding Messages to and from JSON
// format, sending and receiving those messages on the Client's in and out
// channels.
func socketHandler(conn *websocket.Conn) {
	in, out := make(chan *Message), make(chan *Message)
	c := &Client{
		proc: make(map[string]*Process),
		in:   in,
		out:  out,
	}
	go c.loop()

	errc := make(chan error, 1)

	// Decode messages from client and send to the in channel.
	go func() {
		dec := json.NewDecoder(conn)
		for {
			m := new(Message)
			if err := dec.Decode(m); err != nil {
				errc <- err
				close(in)
				return
			}
			in <- m
		}
	}()

	// Receive messages from the out channel and encode to the client.
	go func() {
		enc := json.NewEncoder(conn)
		counts := make(map[string]int)
		for m := range out {
			cnt := counts[m.Id]
			switch {
			case m.Kind == "end" || cnt < msgLimit:
				if err := enc.Encode(m); err != nil {
					errc <- err
					return
				}
				if m.Kind == "end" {
					delete(counts, m.Id)
				}
			case cnt == msgLimit:
				// Process produced too much output. Kill it.
				c.kill(m.Id)
			}
			counts[m.Id]++
		}
	}()

	// Wait for one of the send or receive goroutines to exit.
	if err := <-errc; err != nil && err != io.EOF {
		log.Println(err)
	}

	// Kill any running processes associated with this Client.
	c.Lock()
	for _, p := range c.proc {
		p.kill()
	}
	c.Unlock()
}

// Client represents a connected present client.
// It manages any processes being compiled and run for the client.
type Client struct {
	sync.Mutex // guards proc
	proc       map[string]*Process
	in         <-chan *Message
	out        chan<- *Message
}

// loop handles incoming messages from the client.
func (c *Client) loop() {
	for m := range c.in {
		switch m.Kind {
		case "run":
			c.kill(m.Id)
			go c.run(m.Id, m.Body)
		case "kill":
			c.kill(m.Id)
		}
	}
}

// kill shuts down a running process.
func (c *Client) kill(id string) {
	c.Lock()
	defer c.Unlock()
	if p := c.proc[id]; p != nil {
		p.kill()
	}
}

// run compiles and runs the given program, associating it with the given id.
func (c *Client) run(id, body string) {
	p := NewProcess(id, c.out)
	c.Lock()
	c.proc[id] = p
	c.Unlock()
	err := p.run(body)
	m := &Message{Id: id, Kind: "end"}
	if err != nil {
		m.Body = err.Error()
	}
	c.Lock()
	delete(c.proc, id)
	c.Unlock()
	c.out <- m
}

// Process represents a running process.
type Process struct {
	id             string
	stdout, stderr io.Writer

	sync.Mutex // guards cmd
	cmd        *exec.Cmd
	done       chan struct{} // closed when run complete
}

func NewProcess(id string, out chan<- *Message) *Process {
	return &Process{
		id:     id,
		stdout: &messageWriter{id, "stdout", out},
		stderr: &messageWriter{id, "stdout", out},
		done:   make(chan struct{}),
	}
}

// run compiles and runs the given go program.
func (p *Process) run(body string) error {
	defer close(p.done)

	// x is the base name for .go and executable files
	x := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := x + ".go"
	bin := x
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// write body to x.go
	defer os.Remove(src)
	if err := ioutil.WriteFile(src, []byte(body), 0666); err != nil {
		return err
	}

	// build x.go, creating x
	dir, file := filepath.Split(src)
	err := p.exec(dir, "go", "build", "-o", bin, file)
	defer os.Remove(bin)
	if err != nil {
		return err
	}

	// run x
	return p.exec("", bin)
}

// exec runs the specified command in the given directory, writing all standard
// output and standard error to the Process' stdout and stderr fields. It
// stores the running command in the cmd field, and returns when the command
// exits.
func (p *Process) exec(dir string, args ...string) error {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = p.stdout
	cmd.Stderr = p.stderr

	if err := cmd.Start(); err != nil {
		return err
	}

	p.Lock()
	p.cmd = cmd
	p.Unlock()

	err := cmd.Wait()

	p.Lock()
	p.cmd = nil
	p.Unlock()

	return err
}

// kill stops the process if it is running and waits for it to exit.
func (p *Process) kill() {
	p.Lock()
	if p.cmd != nil {
		p.cmd.Process.Kill()
	}
	p.Unlock()
	<-p.done
}

// messageWriter is an io.Writer that converts all writes to Message sends on
// the out channel with the specified id and kind.
type messageWriter struct {
	id, kind string
	out      chan<- *Message
}

func (w *messageWriter) Write(b []byte) (n int, err error) {
	w.out <- &Message{Id: w.id, Kind: w.kind, Body: string(b)}
	return len(b), nil
}

var tmpdir string

func init() {
	// find real path to temporary directory
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package metadata

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/jacksontj/dataman/src/datamantype"
)

func NewCollection(name string) *Collection {
	return &Collection{
		Name:    name,
		Indexes: make(map[string]*CollectionIndex),
	}
}

type Collection struct {
	ID   int64  `json:"_id,omitempty"`
	Name string `json:"name"`

	// NOTE: we reserve the "_" namespace for fields for our own data (created, etc.)
	// All the columns in this table
	Fields map[string]*CollectionField `json:"fields"`

	// map of name -> index
	Indexes map[string]*CollectionIndex `json:"indexes,omitempty"`
	// Link directly to primary index (for convenience)
	PrimaryIndex *CollectionIndex `json:"-"`

	ProvisionState ProvisionState `json:"provision_state"`
}

func (c *Collection) UnmarshalJSON(data []byte) error {
	type Alias Collection
	aux := &struct {
		*Alias
	}{
		Alias: (*Alias)(c),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for _, index := range c.Indexes {
		if index.Primary {
			if c.PrimaryIndex == nil {
				c.PrimaryIndex = index
			} else {
				return fmt.Errorf("Collections can only have one primary index")
			}
		}
	}
	if c.PrimaryIndex == nil {
		return fmt.Errorf("Collection %s missing primary index", c.Name)
	}

	return nil
}

func (c *Collection) GetFieldByName(name string) *CollectionField {
	return c.GetField(strings.Split(name, "."))
}

func (c *Collection) GetField(nameParts []string) *CollectionField {
	field := c.Fields[nameParts[0]]

	for _, part := range nameParts[1:] {
		field = field.SubFields[part]
	}

	return field
}

func (c *Collection) Equal(o *Collection) bool {
	if c.Name != o.Name {
		return false
	}

	return true
}

func (c *Collection) ListIndexes() []string {
	indexes := make([]string, 0, len(c.Indexes))
	for name, _ := range c.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

func (c *Collection) ValidateRecordInsert(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	// TODO: We need to check that we where given no more than the Fields we know about
	for fieldName, field := range c.Fields {
		switch field.FieldType.DatamanType {
		case datamantype.Serial:
			// TODO: check the serial type
		default:
			if v, ok := record[fieldName]; ok {
				record[fieldName], result.Fields[fieldName] = field.Normalize(v)
			} else {
				if field.NotNull && field.Default == nil {
					result.Fields[fieldName] = &ValidationResult{
						Error: fmt.Sprintf("Missing required field %s %v", fieldName, field.Default),
					}
				}
				// TODO: include an empty result? Not sure if an empty one is any good (also-- check for subfields?)
			}
		}
	}
	return result
}

// TODO: underlying datasources should know how to do this-- us doing it shouldn't
// be necessary
// For updates we want to ensure that all fields we where given are (1) valid and (2) are ones we know about
func (c *Collection) ValidateRecordUpdate(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			record[fieldName], result.Fields[fieldName] = field.Normalize(v)
		}
	}
	return result
}
<commit_msg>Only update the value to the "normalized" value if validation was error-free<commit_after>package metadata

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/jacksontj/dataman/src/datamantype"
)

func NewCollection(name string) *Collection {
	return &Collection{
		Name:    name,
		Indexes: make(map[string]*CollectionIndex),
	}
}

type Collection struct {
	ID   int64  `json:"_id,omitempty"`
	Name string `json:"name"`

	// NOTE: we reserve the "_" namespace for fields for our own data (created, etc.)
	// All the columns in this table
	Fields map[string]*CollectionField `json:"fields"`

	// map of name -> index
	Indexes map[string]*CollectionIndex `json:"indexes,omitempty"`
	// Link directly to primary index (for convenience)
	PrimaryIndex *CollectionIndex `json:"-"`

	ProvisionState ProvisionState `json:"provision_state"`
}

func (c *Collection) UnmarshalJSON(data []byte) error {
	type Alias Collection
	aux := &struct {
		*Alias
	}{
		Alias: (*Alias)(c),
	}
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for _, index := range c.Indexes {
		if index.Primary {
			if c.PrimaryIndex == nil {
				c.PrimaryIndex = index
			} else {
				return fmt.Errorf("Collections can only have one primary index")
			}
		}
	}
	if c.PrimaryIndex == nil {
		return fmt.Errorf("Collection %s missing primary index", c.Name)
	}

	return nil
}

func (c *Collection) GetFieldByName(name string) *CollectionField {
	return c.GetField(strings.Split(name, "."))
}

func (c *Collection) GetField(nameParts []string) *CollectionField {
	field := c.Fields[nameParts[0]]

	for _, part := range nameParts[1:] {
		field = field.SubFields[part]
	}

	return field
}

func (c *Collection) Equal(o *Collection) bool {
	if c.Name != o.Name {
		return false
	}

	return true
}

func (c *Collection) ListIndexes() []string {
	indexes := make([]string, 0, len(c.Indexes))
	for name, _ := range c.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

func (c *Collection) ValidateRecordInsert(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	// TODO: We need to check that we where given no more than the Fields we know about
	for fieldName, field := range c.Fields {
		switch field.FieldType.DatamanType {
		case datamantype.Serial:
			// TODO: check the serial type
		default:
			if v, ok := record[fieldName]; ok {
				normalizedValue, normalizationResult := field.Normalize(v)
				result.Fields[fieldName] = normalizationResult
				if normalizationResult.IsValid() {
					record[fieldName] = normalizedValue
				}
			} else {
				if field.NotNull && field.Default == nil {
					result.Fields[fieldName] = &ValidationResult{
						Error: fmt.Sprintf("Missing required field %s %v", fieldName, field.Default),
					}
				}
				// TODO: include an empty result? Not sure if an empty one is any good (also-- check for subfields?)
			}
		}
	}
	return result
}

// TODO: underlying datasources should know how to do this-- us doing it shouldn't
// be necessary
// For updates we want to ensure that all fields we where given are (1) valid and (2) are ones we know about
func (c *Collection) ValidateRecordUpdate(record map[string]interface{}) *ValidationResult {
	result := &ValidationResult{Fields: make(map[string]*ValidationResult)}
	for fieldName, field := range c.Fields {
		if v, ok := record[fieldName]; ok {
			normalizedValue, normalizationResult := field.Normalize(v)
			result.Fields[fieldName] = normalizationResult
			if normalizationResult.IsValid() {
				record[fieldName] = normalizedValue
			}
		}
	}
	return result
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"path"
	"strings"

	car "github.com/mohae/carchivum"
	"github.com/mohae/contour"
)

func Extract(src string) (string, error) {
	fmt.Printf("\nExtract %q", src)
	// check the extension: use zip for .zip and tar for everything else
	ext := strings.ToLower(path.Ext(src))
	if ext == ".zip" {
		z := car.NewZip(src)
		z.OutDir = contour.GetString("output_dir")
		err := z.Extract()

	} else {
		t := car.NewTar(src)
		t.OutDir = contour.GetString("output_dir")
		err := t.Extract()
	}

}
<commit_msg>extract should return something/early return<commit_after>package app

import (
	"fmt"
	"path"
	"strings"

	car "github.com/mohae/carchivum"
	"github.com/mohae/contour"
)

func Extract(src string) (string, error) {
	fmt.Printf("\nExtract %q", src)
	// check the extension: use zip for .zip and tar for everything else
	ext := strings.ToLower(path.Ext(src))
	if ext == ".zip" {
		z := car.NewZip(src)
		z.OutDir = contour.GetString("output_dir")
		err := z.Extract()
		return "", err
	}
	t := car.NewTar(src)
	t.OutDir = contour.GetString("output_dir")
	err := t.Extract()
	return "", err
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command agent forwards requests from an inverting proxy to a backend server.
//
// To build, run:
//
//    $ make
//
// And to use, run:
//
//    $ $(GOPATH)/bin/proxy-forwarding-agent -proxy <proxy-url> -backend <backend-ID>

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/golang/groupcache/lru"
	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"

	"github.com/google/inverting-proxy/agent/banner"
	"github.com/google/inverting-proxy/agent/sessions"
	"github.com/google/inverting-proxy/agent/utils"
	"github.com/google/inverting-proxy/agent/websockets"
)

const (
	requestCacheLimit = 1000
	emailScope        = "email"
)

var (
	proxy                = flag.String("proxy", "", "URL (including scheme) of the inverting proxy")
	proxyTimeout         = flag.Duration("proxy-timeout", 60*time.Second, "Client timeout when sending requests to the inverting proxy")
	host                 = flag.String("host", "localhost:8080", "Hostname (including port) of the backend server")
	backendID            = flag.String("backend", "", "Unique ID for this backend.")
	debug                = flag.Bool("debug", false, "Whether or not to print debug log messages")
	disableSSLForTest    = flag.Bool("disable-ssl-for-test", false, "Disable requirements for SSL when running tests locally")
	favIconURL           = flag.String("favicon-url", "", "URL of the favicon")
	forwardUserID        = flag.Bool("forward-user-id", false, "Whether or not to include the ID (email address) of the end user in requests to the backend")
	injectBanner         = flag.String("inject-banner", "", "HTML snippet to inject in served webpages")
	bannerHeight         = flag.String("banner-height", "40px", "Height of the injected banner. This is ignored if no banner is set.")
	shimWebsockets       = flag.Bool("shim-websockets", false, "Whether or not to replace websockets with a shim")
	shimPath             = flag.String("shim-path", "", "Path under which to handle websocket shim requests")
	healthCheckPath      = flag.String("health-check-path", "/", "Path on backend host to issue health checks against.  Defaults to the root.")
	healthCheckFreq      = flag.Int("health-check-interval-seconds", 0, "Wait time in seconds between health checks.  Set to zero to disable health checks.  Checks disabled by default.")
	healthCheckUnhealthy = flag.Int("health-check-unhealthy-threshold", 2, "A so-far healthy backend will be marked unhealthy after this many consecutive failures. The minimum value is 1.")
	disableGCEVM         = flag.Bool("disable-gce-vm-header", false, "Disable the agent from adding a GCE VM header.")

	sessionCookieName       = flag.String("session-cookie-name", "", "Name of the session cookie; an empty value disables agent-based session tracking")
	sessionCookieTimeout    = flag.Duration("session-cookie-timeout", 12*time.Hour, "Expiration flag for the session cookie")
	sessionCookieCacheLimit = flag.Int("session-cookie-cache-limit", 1000, "Upper bound on the number of concurrent sessions that can be tracked by the agent")

	sessionLRU *sessions.Cache
)

func hostProxy(ctx context.Context, host, shimPath string, injectShimCode bool) (http.Handler, error) {
	hostProxy := httputil.NewSingleHostReverseProxy(&url.URL{
		Scheme: "http",
		Host:   host,
	})
	hostProxy.FlushInterval = 100 * time.Millisecond
	var h http.Handler = hostProxy
	if shimPath != "" {
		var err error
		h, err = websockets.Proxy(ctx, hostProxy, host, shimPath, injectShimCode)
		if err != nil {
			return nil, err
		}
	}
	h = sessionLRU.SessionHandler(h)
	if *injectBanner == "" {
		return h, nil
	}
	return banner.Proxy(ctx, h, *injectBanner, *bannerHeight, *favIconURL)
}

// forwardRequest forwards the given request from the proxy to
// the backend server and reports the response back to the proxy.
func forwardRequest(client *http.Client, hostProxy http.Handler, request *utils.ForwardedRequest) error {
	httpRequest := request.Contents
	if *forwardUserID {
		httpRequest.Header.Add(utils.HeaderUserID, request.User)
	}
	responseForwarder, err := utils.NewResponseForwarder(client, *proxy, request.BackendID, request.RequestID)
	if err != nil {
		return fmt.Errorf("failed to create the response forwarder: %v", err)
	}
	hostProxy.ServeHTTP(responseForwarder, httpRequest)
	if *debug {
		log.Printf("Backend latency for request %s: %s\n", request.RequestID, time.Since(request.StartTime).String())
	}
	if err := responseForwarder.Close(); err != nil {
		return fmt.Errorf("failed to close the response forwarder: %v", err)
	}
	return nil
}

// healthCheck issues a health check against the backend server
// and returns the result.
func healthCheck() error {
	resp, err := http.Get("http://" + *host + *healthCheckPath)
	if err != nil {
		log.Printf("Health Check request failed: %s", err.Error())
		return err
	}
	if resp.StatusCode != 200 {
		log.Printf("Health Check request had non-200 status code: %d", resp.StatusCode)
		return fmt.Errorf("Bad Health Check Response Code: %s", resp.Status)
	}
	return nil
}

// processOneRequest reads a single request from the proxy and forwards it to the backend server.
func processOneRequest(client *http.Client, hostProxy http.Handler, backendID string, requestID string) {
	requestForwarder := func(client *http.Client, request *utils.ForwardedRequest) error {
		if err := forwardRequest(client, hostProxy, request); err != nil {
			log.Printf("Failure forwarding a request: [%s] %q\n", requestID, err.Error())
			return fmt.Errorf("failed to forward the request %q: %v", requestID, err)
		}
		return nil
	}
	if err := utils.ReadRequest(client, *proxy, backendID, requestID, requestForwarder); err != nil {
		log.Printf("Failed to forward a request: [%s] %q\n", requestID, err.Error())
	}
}

// pollForNewRequests repeatedly reaches out to the proxy server to ask if any pending are available, and then
// processes any newly-seen ones.
func pollForNewRequests(client *http.Client, hostProxy http.Handler, backendID string) {
	previouslySeenRequests := lru.New(requestCacheLimit)
	var retryCount uint
	for {
		if requests, err := utils.ListPendingRequests(client, *proxy, backendID); err != nil {
			log.Printf("Failed to read pending requests: %q\n", err.Error())
			time.Sleep(utils.ExponentialBackoffDuration(retryCount))
			retryCount++
		} else {
			retryCount = 0
			for _, requestID := range requests {
				if _, ok := previouslySeenRequests.Get(requestID); !ok {
					previouslySeenRequests.Add(requestID, requestID)
					go processOneRequest(client, hostProxy, backendID, requestID)
				}
			}
		}
	}
}

func getGoogleClient(ctx context.Context) (*http.Client, error) {
	sdkConfig, err := google.NewSDKConfig("")
	if err == nil {
		return sdkConfig.Client(ctx), nil
	}

	client, err := google.DefaultClient(ctx, compute.CloudPlatformScope, emailScope)
	if err != nil {
		return nil, err
	}
	client.Transport = utils.RoundTripperWithVMIdentity(ctx, client.Transport, *proxy, *disableGCEVM)
	client.Jar, err = cookiejar.New(&cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	})
	if err != nil {
		return nil, err
	}
	return client, nil
}

// waitForHealthy runs health checks against the backend and returns
// the first time it sees a healthy check.
func waitForHealthy() {
	if *healthCheckFreq <= 0 {
		return
	}
	if healthCheck() == nil {
		return
	}
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	for _ = range ticker.C {
		if healthCheck() == nil {
			ticker.Stop()
			return
		}
	}
}

// runHealthChecks runs health checks against the backend and shuts down
// the proxy if the backend is unhealthy.
func runHealthChecks() {
	if *healthCheckFreq <= 0 {
		return
	}
	if *healthCheckUnhealthy < 1 {
		*healthCheckUnhealthy = 1
	}
	// Always start in the unhealthy state, but only require a single positive
	// health check to become healthy for the first time, and do the first check
	// immediately.
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	badHealthChecks := 0
	for _ = range ticker.C {
		if healthCheck() != nil {
			badHealthChecks++
		} else {
			badHealthChecks = 0
		}
		if badHealthChecks >= *healthCheckUnhealthy {
			ticker.Stop()
			log.Fatal("Too many unhealthy checks")
		}
	}
}

// runAdapter sets up the HTTP client for the agent to use (including OAuth credentials),
// and then does the actual work of forwarding requests and responses.
func runAdapter() error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	client, err := getGoogleClient(ctx)
	if err != nil {
		return err
	}
	client.Timeout = *proxyTimeout

	hostProxy, err := hostProxy(ctx, *host, *shimPath, *shimWebsockets)
	if err != nil {
		return err
	}
	pollForNewRequests(client, hostProxy, *backendID)
	return nil
}

func main() {
	flag.Parse()

	if *proxy == "" {
		log.Fatal("You must specify the address of the proxy")
	}
	if *backendID == "" {
		log.Fatal("You must specify a backend ID")
	}
	if !strings.HasPrefix(*healthCheckPath, "/") {
		*healthCheckPath = "/" + *healthCheckPath
	}
	if *sessionCookieName != "" {
		sessionLRU = sessions.NewCache(*sessionCookieName, *sessionCookieTimeout, *sessionCookieCacheLimit, *disableSSLForTest)
	}

	waitForHealthy()
	go runHealthChecks()

	if err := runAdapter(); err != nil {
		log.Fatal(err.Error())
	}
}
<commit_msg>Close health check response body<commit_after>/*
Copyright 2016 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Command agent forwards requests from an inverting proxy to a backend server.
//
// To build, run:
//
//    $ make
//
// And to use, run:
//
//    $ $(GOPATH)/bin/proxy-forwarding-agent -proxy <proxy-url> -backend <backend-ID>

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"

	"github.com/golang/groupcache/lru"
	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"

	"github.com/google/inverting-proxy/agent/banner"
	"github.com/google/inverting-proxy/agent/sessions"
	"github.com/google/inverting-proxy/agent/utils"
	"github.com/google/inverting-proxy/agent/websockets"
)

const (
	requestCacheLimit = 1000
	emailScope        = "email"
)

var (
	proxy                = flag.String("proxy", "", "URL (including scheme) of the inverting proxy")
	proxyTimeout         = flag.Duration("proxy-timeout", 60*time.Second, "Client timeout when sending requests to the inverting proxy")
	host                 = flag.String("host", "localhost:8080", "Hostname (including port) of the backend server")
	backendID            = flag.String("backend", "", "Unique ID for this backend.")
	debug                = flag.Bool("debug", false, "Whether or not to print debug log messages")
	disableSSLForTest    = flag.Bool("disable-ssl-for-test", false, "Disable requirements for SSL when running tests locally")
	favIconURL           = flag.String("favicon-url", "", "URL of the favicon")
	forwardUserID        = flag.Bool("forward-user-id", false, "Whether or not to include the ID (email address) of the end user in requests to the backend")
	injectBanner         = flag.String("inject-banner", "", "HTML snippet to inject in served webpages")
	bannerHeight         = flag.String("banner-height", "40px", "Height of the injected banner. This is ignored if no banner is set.")
	shimWebsockets       = flag.Bool("shim-websockets", false, "Whether or not to replace websockets with a shim")
	shimPath             = flag.String("shim-path", "", "Path under which to handle websocket shim requests")
	healthCheckPath      = flag.String("health-check-path", "/", "Path on backend host to issue health checks against.  Defaults to the root.")
	healthCheckFreq      = flag.Int("health-check-interval-seconds", 0, "Wait time in seconds between health checks.  Set to zero to disable health checks.  Checks disabled by default.")
	healthCheckUnhealthy = flag.Int("health-check-unhealthy-threshold", 2, "A so-far healthy backend will be marked unhealthy after this many consecutive failures. The minimum value is 1.")
	disableGCEVM         = flag.Bool("disable-gce-vm-header", false, "Disable the agent from adding a GCE VM header.")

	sessionCookieName       = flag.String("session-cookie-name", "", "Name of the session cookie; an empty value disables agent-based session tracking")
	sessionCookieTimeout    = flag.Duration("session-cookie-timeout", 12*time.Hour, "Expiration flag for the session cookie")
	sessionCookieCacheLimit = flag.Int("session-cookie-cache-limit", 1000, "Upper bound on the number of concurrent sessions that can be tracked by the agent")

	sessionLRU *sessions.Cache
)

func hostProxy(ctx context.Context, host, shimPath string, injectShimCode bool) (http.Handler, error) {
	hostProxy := httputil.NewSingleHostReverseProxy(&url.URL{
		Scheme: "http",
		Host:   host,
	})
	hostProxy.FlushInterval = 100 * time.Millisecond
	var h http.Handler = hostProxy
	if shimPath != "" {
		var err error
		h, err = websockets.Proxy(ctx, hostProxy, host, shimPath, injectShimCode)
		if err != nil {
			return nil, err
		}
	}
	h = sessionLRU.SessionHandler(h)
	if *injectBanner == "" {
		return h, nil
	}
	return banner.Proxy(ctx, h, *injectBanner, *bannerHeight, *favIconURL)
}

// forwardRequest forwards the given request from the proxy to
// the backend server and reports the response back to the proxy.
func forwardRequest(client *http.Client, hostProxy http.Handler, request *utils.ForwardedRequest) error {
	httpRequest := request.Contents
	if *forwardUserID {
		httpRequest.Header.Add(utils.HeaderUserID, request.User)
	}
	responseForwarder, err := utils.NewResponseForwarder(client, *proxy, request.BackendID, request.RequestID)
	if err != nil {
		return fmt.Errorf("failed to create the response forwarder: %v", err)
	}
	hostProxy.ServeHTTP(responseForwarder, httpRequest)
	if *debug {
		log.Printf("Backend latency for request %s: %s\n", request.RequestID, time.Since(request.StartTime).String())
	}
	if err := responseForwarder.Close(); err != nil {
		return fmt.Errorf("failed to close the response forwarder: %v", err)
	}
	return nil
}

// healthCheck issues a health check against the backend server
// and returns the result.
func healthCheck() error {
	resp, err := http.Get("http://" + *host + *healthCheckPath)
	if err != nil {
		log.Printf("Health Check request failed: %s", err.Error())
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		log.Printf("Health Check request had non-200 status code: %d", resp.StatusCode)
		return fmt.Errorf("Bad Health Check Response Code: %s", resp.Status)
	}
	return nil
}

// processOneRequest reads a single request from the proxy and forwards it to the backend server.
func processOneRequest(client *http.Client, hostProxy http.Handler, backendID string, requestID string) {
	requestForwarder := func(client *http.Client, request *utils.ForwardedRequest) error {
		if err := forwardRequest(client, hostProxy, request); err != nil {
			log.Printf("Failure forwarding a request: [%s] %q\n", requestID, err.Error())
			return fmt.Errorf("failed to forward the request %q: %v", requestID, err)
		}
		return nil
	}
	if err := utils.ReadRequest(client, *proxy, backendID, requestID, requestForwarder); err != nil {
		log.Printf("Failed to forward a request: [%s] %q\n", requestID, err.Error())
	}
}

// pollForNewRequests repeatedly reaches out to the proxy server to ask if any pending are available, and then
// processes any newly-seen ones.
func pollForNewRequests(client *http.Client, hostProxy http.Handler, backendID string) {
	previouslySeenRequests := lru.New(requestCacheLimit)
	var retryCount uint
	for {
		if requests, err := utils.ListPendingRequests(client, *proxy, backendID); err != nil {
			log.Printf("Failed to read pending requests: %q\n", err.Error())
			time.Sleep(utils.ExponentialBackoffDuration(retryCount))
			retryCount++
		} else {
			retryCount = 0
			for _, requestID := range requests {
				if _, ok := previouslySeenRequests.Get(requestID); !ok {
					previouslySeenRequests.Add(requestID, requestID)
					go processOneRequest(client, hostProxy, backendID, requestID)
				}
			}
		}
	}
}

func getGoogleClient(ctx context.Context) (*http.Client, error) {
	sdkConfig, err := google.NewSDKConfig("")
	if err == nil {
		return sdkConfig.Client(ctx), nil
	}

	client, err := google.DefaultClient(ctx, compute.CloudPlatformScope, emailScope)
	if err != nil {
		return nil, err
	}
	client.Transport = utils.RoundTripperWithVMIdentity(ctx, client.Transport, *proxy, *disableGCEVM)
	client.Jar, err = cookiejar.New(&cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	})
	if err != nil {
		return nil, err
	}
	return client, nil
}

// waitForHealthy runs health checks against the backend and returns
// the first time it sees a healthy check.
func waitForHealthy() {
	if *healthCheckFreq <= 0 {
		return
	}
	if healthCheck() == nil {
		return
	}
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	for _ = range ticker.C {
		if healthCheck() == nil {
			ticker.Stop()
			return
		}
	}
}

// runHealthChecks runs health checks against the backend and shuts down
// the proxy if the backend is unhealthy.
func runHealthChecks() {
	if *healthCheckFreq <= 0 {
		return
	}
	if *healthCheckUnhealthy < 1 {
		*healthCheckUnhealthy = 1
	}
	// Always start in the unhealthy state, but only require a single positive
	// health check to become healthy for the first time, and do the first check
	// immediately.
	ticker := time.NewTicker(time.Duration(*healthCheckFreq) * time.Second)
	badHealthChecks := 0
	for _ = range ticker.C {
		if healthCheck() != nil {
			badHealthChecks++
		} else {
			badHealthChecks = 0
		}
		if badHealthChecks >= *healthCheckUnhealthy {
			ticker.Stop()
			log.Fatal("Too many unhealthy checks")
		}
	}
}

// runAdapter sets up the HTTP client for the agent to use (including OAuth credentials),
// and then does the actual work of forwarding requests and responses.
func runAdapter() error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	client, err := getGoogleClient(ctx)
	if err != nil {
		return err
	}
	client.Timeout = *proxyTimeout

	hostProxy, err := hostProxy(ctx, *host, *shimPath, *shimWebsockets)
	if err != nil {
		return err
	}
	pollForNewRequests(client, hostProxy, *backendID)
	return nil
}

func main() {
	flag.Parse()

	if *proxy == "" {
		log.Fatal("You must specify the address of the proxy")
	}
	if *backendID == "" {
		log.Fatal("You must specify a backend ID")
	}
	if !strings.HasPrefix(*healthCheckPath, "/") {
		*healthCheckPath = "/" + *healthCheckPath
	}
	if *sessionCookieName != "" {
		sessionLRU = sessions.NewCache(*sessionCookieName, *sessionCookieTimeout, *sessionCookieCacheLimit, *disableSSLForTest)
	}

	waitForHealthy()
	go runHealthChecks()

	if err := runAdapter(); err != nil {
		log.Fatal(err.Error())
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"unicode/utf8"

	"github.com/mattn/go-runewidth"
	"github.com/nsf/termbox-go"
)

// type Agent present the togo terminate's all attributes.
type Agent struct {
	// Now, it contain togo's prompt strings.
	Chars string

	// alert string, it is "Press Ctr+Q to quit.".
	AlertString string

	// foreground color.
	Fg termbox.Attribute

	// background color.
	Bg termbox.Attribute

	// version
	Version string
}

// return a default Agent.
func New() *Agent {
	return &Agent{
		AlertString: "Press what you want. Press Ctr+C to quit.",
		Fg:          termbox.ColorWhite,
		Bg:          termbox.ColorBlack,
		Version:     "0.0.1",
	}
}

// given a string, this func print them to the terminal,
// return the total width of the string.
func (agent *Agent) PringLine(str string, x int, y int) int {
	// strWidth is the total width of the string
	strWidth := 0

	width, _ := termbox.Size()

	for len(str) > 0 {
		c, w := utf8.DecodeRuneInString(str)
		if c == utf8.RuneError {
			c = '?'
			w = 1
		}
		str = str[w:]

		termbox.SetCell(x, y, c, agent.Fg, agent.Bg)

		x += runewidth.RuneWidth(c)
		strWidth += runewidth.RuneWidth(c)
	}

	blankPosition := strWidth

	for blankPosition < width {
		termbox.SetCell(blankPosition, y, ' ', agent.Fg, agent.Bg)
		blankPosition++
	}

	return strWidth
}

func (agent *Agent) DeletePromp() {
	_, size := utf8.DecodeLastRuneInString(agent.Chars)
	agent.Chars = agent.Chars[:len(agent.Chars)-size]
	agent.DrawPromp("")
}

// draw the promp line.
func (agent *Agent) DrawPromp(str string) {
	agent.Chars = agent.Chars + string(str)

	prompStr := "TOGO> " + agent.Chars

	x := 0
	y := 0

	width := agent.PringLine(prompStr, x, y)
	termbox.SetCursor(width, 0)
	termbox.Flush()
}

// the public api of the Agent package.
func (agent *Agent) Run() {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}

	defer termbox.Close()
	termbox.Clear(agent.Fg, agent.Bg)
	agent.DrawPromp("")

	agent.PringLine(agent.AlertString, 0, 2)
	agent.PringLine("Latest Version: "+agent.Version, 0, 3)
	termbox.Flush()

loop:

	for {
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			switch ev.Key {

			// quit the program.
			case termbox.KeyCtrlC:
				break loop

			// puts " " to the screen, when we press space.
			case termbox.KeySpace:
				str := " "
				agent.DrawPromp(str)

			// support back space key
			case termbox.KeyBackspace, termbox.KeyBackspace2:
				agent.DeletePromp()

			// convert rune to string, then draw the promp.
			default:
				str := string(ev.Ch)
				agent.DrawPromp(str)
			}
		}
	}
}
<commit_msg>Bump to 0.0.2<commit_after>package agent

import (
	"unicode/utf8"

	"github.com/mattn/go-runewidth"
	"github.com/nsf/termbox-go"
)

// type Agent present the togo terminate's all attributes.
type Agent struct {
	// Now, it contain togo's prompt strings.
	Chars string

	// alert string, it is "Press Ctr+Q to quit.".
	AlertString string

	// foreground color.
	Fg termbox.Attribute

	// background color.
	Bg termbox.Attribute

	// version
	Version string
}

// return a default Agent.
func New() *Agent {
	return &Agent{
		AlertString: "Press what you want. Press Ctr+C to quit.",
		Fg:          termbox.ColorWhite,
		Bg:          termbox.ColorBlack,
		Version:     "0.0.2",
	}
}

// given a string, this func print them to the terminal,
// return the total width of the string.
func (agent *Agent) PringLine(str string, x int, y int) int {
	// strWidth is the total width of the string
	strWidth := 0

	width, _ := termbox.Size()

	for len(str) > 0 {
		c, w := utf8.DecodeRuneInString(str)
		if c == utf8.RuneError {
			c = '?'
			w = 1
		}
		str = str[w:]

		termbox.SetCell(x, y, c, agent.Fg, agent.Bg)

		x += runewidth.RuneWidth(c)
		strWidth += runewidth.RuneWidth(c)
	}

	blankPosition := strWidth

	for blankPosition < width {
		termbox.SetCell(blankPosition, y, ' ', agent.Fg, agent.Bg)
		blankPosition++
	}

	return strWidth
}

func (agent *Agent) DeletePromp() {
	_, size := utf8.DecodeLastRuneInString(agent.Chars)
	agent.Chars = agent.Chars[:len(agent.Chars)-size]
	agent.DrawPromp("")
}

// draw the promp line.
func (agent *Agent) DrawPromp(str string) {
	agent.Chars = agent.Chars + string(str)

	prompStr := "TOGO> " + agent.Chars

	x := 0
	y := 0

	width := agent.PringLine(prompStr, x, y)
	termbox.SetCursor(width, 0)
	termbox.Flush()
}

// the public api of the Agent package.
func (agent *Agent) Run() {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}

	defer termbox.Close()
	termbox.Clear(agent.Fg, agent.Bg)
	agent.DrawPromp("")

	agent.PringLine(agent.AlertString, 0, 2)
	agent.PringLine("Latest Version: "+agent.Version, 0, 3)
	termbox.Flush()

loop:

	for {
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			switch ev.Key {

			// quit the program.
			case termbox.KeyCtrlC:
				break loop

			// puts " " to the screen, when we press space.
			case termbox.KeySpace:
				str := " "
				agent.DrawPromp(str)

			// support back space key
			case termbox.KeyBackspace, termbox.KeyBackspace2:
				agent.DeletePromp()

			// convert rune to string, then draw the promp.
			default:
				str := string(ev.Ch)
				agent.DrawPromp(str)
			}
		}
	}
}
<|endoftext|>


<commit_before>package elastic

import ()

const (
	// Aggs abreviateed constant name for the Aggregation query.
	Aggs = "aggs"
	// Aggregations constant name for the Aggregation query.
	Aggregations = "aggregations"
	// Terms constant name of terms Bucket
	Terms = "terms"
	// Histogram constant name of the Histogram bucket
	Histogram = "histogram"
	// DateHistogram constant name of the Date Histogram bucket.
	DateHistogram = "date_histogram"
	// Global constant name of the global bucket which is used to by pass aggregation scope.
	Global = "global"
	// FilterBucket constant name of filter bucket which is used to filter aggregation results.
	FilterBucket = "filter"
)

// Constant name of Elasticsearch metrics
const (
	// Count constant name of 'count' metric.
	Count = "count"
	// Sum constant name of 'sum' metric.
	Sum = "sum"
	// Avg constant name of 'avg' metric.
	Avg = "avg"
	// Min constant name of 'min' metric.
	Min = "min"
	// Max constant name of 'max' metric.
	Max = "max"
	// ExtendedStats constant name of a metric that will return a variety of statistics (e.g. stats.avg, stats.count, stats.std_deviation).
	ExtendedStats = "extended_stats"
	// Cardiality constant name of the 'cardinality' approximation metric.
	Cardiality = "cardinality"
	// Percentiles constant name of the 'percentiles' approximation metric.
	Percentiles = "percentiles"
	// PercentileRank constant name of an approximation metric that tells to which percentile the given value belongs.
	PercentileRanks = "percentile_ranks"
)

const (
	// Field name of parameter that defines the document's field that will be used to create buckets using its unique values.
	Field = "field"
	// Interval name of parameter that define a histogram interval, i.e. the value that Elasticsearch will use to create new buckets.
	Interval = "interval"
	// Size name of parameter that defines how many terms we want to generate. Example of values, for histograms: 10, for date histograms: "month", "quarter".
	Size = "size"
	// Format name of parameter in date histogram, used to define the  dates format for bucket keys.
	Format = "format"
	// MinDocCount name of parameter in date histogram, used to force empty buckets to be returned.
	MinDocCount = "min_doc_count"
	// ExtendedBound name of parameter in date histogram. It is used to extend the boudaries of bucket from the boudaries of actual data. This, it forces all bucket betwen the min and max bound to be returned.
	ExtendedBound = "extended_bound"
	// Order name of an object that defines how the create buckets should be generatedas well as the the ordering mode (e.g. asc). Example of values: _count (sort by document count), _term (sort alphabetically by string value), _key (sort by bucket key, works only for histogram & date_histogram).
	Order = "order"
	// PrecisionThreshold configure the precision of the HyperLogLog algorithm used by the 'cardinality' metric.
	PrecisionThreshold = "precision_threshold"
	// Percents a parameter of the 'percentiles' metric. It's used to define an array of the percentiles that should be calculated instead of the default one (i.e. 5, 25, 50, 75, 95, 99).
	Percents = "percents"
	// Values a parameter of the 'percentile_ranks' metric. It is used to define the values that Elasticsearch should find their percentile.
	Values = "values"
	// Compression a parameter of the 'percentiles' metric (default value is 100). It is used to control the memory footprint (an thus the accuracy) by limiting the number of nodes involved in the calculation.
	Compression = "compression"
)

// Aggregations a structure representing an aggregation request
type Aggregation struct {
	client *Elasticsearch
	parser Parser
	url    string
	params map[string]string
	query  Dict
}

// Aggs creates an aggregation request
func (client *Elasticsearch) Aggs(index, doc string) *Aggregation {
	url := client.request(index, doc, -1, SEARCH)
	return &Aggregation{
		client: client,
		parser: &AggregationResultParser{},
		url:    url,
		params: make(map[string]string),
		query:  make(Dict),
	}
}

// urlString constructs the url of this Search API call
func (agg *Aggregation) urlString() string {
	return urlString(agg.url, agg.params)
}

// String returns a string representation of this Search API call
func (agg *Aggregation) String() string {
	body := ""
	if len(agg.query) > 0 {
		body = String(agg.query)
	}
	return body
}

// Get submits request mappings between the json fields and how Elasticsearch store them
// GET /:index/:type/_search
func (agg *Aggregation) Get() {
	// construct the url
	url := agg.urlString()
	// construct the body
	query := agg.String()

	agg.client.Execute("GET", url, query, agg.parser)
}

// SetMetric sets the search type with the given value (e.g. count)
func (agg *Aggregation) SetMetric(name string) *Aggregation {
	agg.params[SearchType] = name
	return agg
}

// Bucket a structure that defines how Elasticsearch should create Bucket for aggregations.
type Bucket struct {
	name  string
	query Dict
}

// Metric a structure that defines a bucket metric.
type Metric struct {
	name  string
	query Dict
}

// NewBucket creates a new Bucket definition
func NewBucket(name string) *Bucket {
	return &Bucket{
		name:  name,
		query: make(Dict),
	}
}

func (bucket *Bucket) AddTerm(name string, value interface{}) *Bucket {
	bucket.AddMetric(Terms, name, value)
	return bucket
}

func (bucket *Bucket) AddMetric(metric, name string, value interface{}) *Bucket {
	if bucket.query[metric] == nil {
		bucket.query[metric] = make(Dict)
	}
	bucket.query[metric].(Dict)[name] = value
	return bucket
}

func (bucket *Bucket) AddDict(name string, value Dict) *Bucket {
	bucket.query[name] = value
	return bucket
}

// SetOrder set the ordering for this bucket.
// name is the name of ordering, e.g. _count, _term, _key, name of metric
// value defines the sens of ordering, e.g. asc
func (bucket *Bucket) SetOrder(metric, name, value string) *Bucket {
	bucket.AddMetric(metric, Order, Dict{name: value})
	return bucket
}

// AddBucket adds a nested bucket to this bucket
func (bucket *Bucket) AddBucket(b *Bucket) *Bucket {
	if bucket.query[Aggs] == nil {
		bucket.query[Aggs] = make(Dict)
	}
	bucket.query[Aggs].(Dict)[b.name] = b.query
	return bucket
}

// Add adds a bucket definition to this aggregation request
func (agg *Aggregation) Add(bucket *Bucket) *Aggregation {
	if agg.query[Aggs] == nil {
		agg.query[Aggs] = make(Dict)
	}
	aggs := agg.query[Aggs].(Dict)
	aggs[bucket.name] = bucket.query
	return agg
}

// AddQuery defines a scope query for this aggregation request
func (agg *Aggregation) AddQuery(q Query) *Aggregation {
	if agg.query["query"] == nil {
		agg.query["query"] = make(Dict)
	}
	agg.query["query"].(Dict)[q.Name()] = q.KV()
	return agg
}

func (agg *Aggregation) AddPostFilter(q Query) *Aggregation {
	if agg.query[PostFilter] == nil {
		agg.query[PostFilter] = make(Dict)
	}
	agg.query[PostFilter].(Dict)[q.Name()] = q.KV()
	return agg
}
<commit_msg>constant for SigTerms aggregation<commit_after>package elastic

import ()

const (
	// Aggs abreviateed constant name for the Aggregation query.
	Aggs = "aggs"
	// Aggregations constant name for the Aggregation query.
	Aggregations = "aggregations"
	// Terms constant name of terms Bucket
	Terms = "terms"
	// Histogram constant name of the Histogram bucket
	Histogram = "histogram"
	// DateHistogram constant name of the Date Histogram bucket.
	DateHistogram = "date_histogram"
	// Global constant name of the global bucket which is used to by pass aggregation scope.
	Global = "global"
	// FilterBucket constant name of filter bucket which is used to filter aggregation results.
	FilterBucket = "filter"
)

// Constant name of Elasticsearch metrics
const (
	// Count constant name of 'count' metric.
	Count = "count"
	// Sum constant name of 'sum' metric.
	Sum = "sum"
	// Avg constant name of 'avg' metric.
	Avg = "avg"
	// Min constant name of 'min' metric.
	Min = "min"
	// Max constant name of 'max' metric.
	Max = "max"
	// ExtendedStats constant name of a metric that will return a variety of statistics (e.g. stats.avg, stats.count, stats.std_deviation).
	ExtendedStats = "extended_stats"
	// Cardiality constant name of the 'cardinality' approximation metric.
	Cardiality = "cardinality"
	// Percentiles constant name of the 'percentiles' approximation metric.
	Percentiles = "percentiles"
	// PercentileRank constant name of an approximation metric that tells to which percentile the given value belongs.
	PercentileRanks = "percentile_ranks"
	// SignificantTerms constant name of the statistical anomalie aggregation. By default, it will use the entire index as the background group while the foreground will be aggregation query scope.
	SignificantTerms = "significant_terms"
)

const (
	// Field name of parameter that defines the document's field that will be used to create buckets using its unique values.
	Field = "field"
	// Interval name of parameter that define a histogram interval, i.e. the value that Elasticsearch will use to create new buckets.
	Interval = "interval"
	// Size name of parameter that defines how many terms we want to generate. Example of values, for histograms: 10, for date histograms: "month", "quarter".
	Size = "size"
	// Format name of parameter in date histogram, used to define the  dates format for bucket keys.
	Format = "format"
	// MinDocCount name of parameter in date histogram, used to force empty buckets to be returned.
	MinDocCount = "min_doc_count"
	// ExtendedBound name of parameter in date histogram. It is used to extend the boudaries of bucket from the boudaries of actual data. This, it forces all bucket betwen the min and max bound to be returned.
	ExtendedBound = "extended_bound"
	// Order name of an object that defines how the create buckets should be generatedas well as the the ordering mode (e.g. asc). Example of values: _count (sort by document count), _term (sort alphabetically by string value), _key (sort by bucket key, works only for histogram & date_histogram).
	Order = "order"
	// PrecisionThreshold configure the precision of the HyperLogLog algorithm used by the 'cardinality' metric.
	PrecisionThreshold = "precision_threshold"
	// Percents a parameter of the 'percentiles' metric. It's used to define an array of the percentiles that should be calculated instead of the default one (i.e. 5, 25, 50, 75, 95, 99).
	Percents = "percents"
	// Values a parameter of the 'percentile_ranks' metric. It is used to define the values that Elasticsearch should find their percentile.
	Values = "values"
	// Compression a parameter of the 'percentiles' metric (default value is 100). It is used to control the memory footprint (an thus the accuracy) by limiting the number of nodes involved in the calculation.
	Compression = "compression"
)

// Aggregations a structure representing an aggregation request
type Aggregation struct {
	client *Elasticsearch
	parser Parser
	url    string
	params map[string]string
	query  Dict
}

// Aggs creates an aggregation request
func (client *Elasticsearch) Aggs(index, doc string) *Aggregation {
	url := client.request(index, doc, -1, SEARCH)
	return &Aggregation{
		client: client,
		parser: &AggregationResultParser{},
		url:    url,
		params: make(map[string]string),
		query:  make(Dict),
	}
}

// urlString constructs the url of this Search API call
func (agg *Aggregation) urlString() string {
	return urlString(agg.url, agg.params)
}

// String returns a string representation of this Search API call
func (agg *Aggregation) String() string {
	body := ""
	if len(agg.query) > 0 {
		body = String(agg.query)
	}
	return body
}

// Get submits request mappings between the json fields and how Elasticsearch store them
// GET /:index/:type/_search
func (agg *Aggregation) Get() {
	// construct the url
	url := agg.urlString()
	// construct the body
	query := agg.String()

	agg.client.Execute("GET", url, query, agg.parser)
}

// SetMetric sets the search type with the given value (e.g. count)
func (agg *Aggregation) SetMetric(name string) *Aggregation {
	agg.params[SearchType] = name
	return agg
}

// Bucket a structure that defines how Elasticsearch should create Bucket for aggregations.
type Bucket struct {
	name  string
	query Dict
}

// Metric a structure that defines a bucket metric.
type Metric struct {
	name  string
	query Dict
}

// NewBucket creates a new Bucket definition
func NewBucket(name string) *Bucket {
	return &Bucket{
		name:  name,
		query: make(Dict),
	}
}

func (bucket *Bucket) AddTerm(name string, value interface{}) *Bucket {
	bucket.AddMetric(Terms, name, value)
	return bucket
}

func (bucket *Bucket) AddMetric(metric, name string, value interface{}) *Bucket {
	if bucket.query[metric] == nil {
		bucket.query[metric] = make(Dict)
	}
	bucket.query[metric].(Dict)[name] = value
	return bucket
}

func (bucket *Bucket) AddDict(name string, value Dict) *Bucket {
	bucket.query[name] = value
	return bucket
}

// SetOrder set the ordering for this bucket.
// name is the name of ordering, e.g. _count, _term, _key, name of metric
// value defines the sens of ordering, e.g. asc
func (bucket *Bucket) SetOrder(metric, name, value string) *Bucket {
	bucket.AddMetric(metric, Order, Dict{name: value})
	return bucket
}

// AddBucket adds a nested bucket to this bucket
func (bucket *Bucket) AddBucket(b *Bucket) *Bucket {
	if bucket.query[Aggs] == nil {
		bucket.query[Aggs] = make(Dict)
	}
	bucket.query[Aggs].(Dict)[b.name] = b.query
	return bucket
}

// Add adds a bucket definition to this aggregation request
func (agg *Aggregation) Add(bucket *Bucket) *Aggregation {
	if agg.query[Aggs] == nil {
		agg.query[Aggs] = make(Dict)
	}
	aggs := agg.query[Aggs].(Dict)
	aggs[bucket.name] = bucket.query
	return agg
}

// AddQuery defines a scope query for this aggregation request
func (agg *Aggregation) AddQuery(q Query) *Aggregation {
	if agg.query["query"] == nil {
		agg.query["query"] = make(Dict)
	}
	agg.query["query"].(Dict)[q.Name()] = q.KV()
	return agg
}

func (agg *Aggregation) AddPostFilter(q Query) *Aggregation {
	if agg.query[PostFilter] == nil {
		agg.query[PostFilter] = make(Dict)
	}
	agg.query[PostFilter].(Dict)[q.Name()] = q.KV()
	return agg
}
<|endoftext|>


<commit_before>package service

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"runtime"
	"time"

	"math"
	"os"
	"strconv"
	"sync"

	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

func (app *goActiveLearningApp) UpdateOrCreateExample(e *model.Example) error {
	return app.repo.UpdateOrCreateExample(e)
}

func (app *goActiveLearningApp) UpdateScore(e *model.Example) error {
	return app.repo.UpdateScore(e)
}

func (app *goActiveLearningApp) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	return app.repo.InsertExampleFromScanner(scanner)
}

func (app *goActiveLearningApp) InsertExamplesFromReader(reader io.Reader) error {
	return app.repo.InsertExamplesFromReader(reader)
}

func (app *goActiveLearningApp) SearchExamples() (model.Examples, error) {
	return app.repo.SearchExamples()
}

func (app *goActiveLearningApp) SearchRecentExamples(from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamples(from, limit)
}

func (app *goActiveLearningApp) SearchRecentExamplesByHost(host string, from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamplesByHost(host, from, limit)
}

func (app *goActiveLearningApp) SearchExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByLabel(label, limit)
}

func (app *goActiveLearningApp) SearchLabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchLabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveExamples(limit)
}

func (app *goActiveLearningApp) SearchNegativeExamples(limit int) (model.Examples, error) {
	return app.repo.SearchNegativeExamples(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchUnlabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveScoredExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveScoredExamples(limit)
}

func (app *goActiveLearningApp) FindExampleByUlr(url string) (*model.Example, error) {
	return app.repo.FindExampleByUlr(url)
}

func (app *goActiveLearningApp) FindExampleById(id int) (*model.Example, error) {
	return app.repo.FindExampleById(id)
}

func (app *goActiveLearningApp) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	return app.repo.SearchExamplesByUlrs(urls)
}

func (app *goActiveLearningApp) SearchExamplesByIds(ids []int) (model.Examples, error) {
	return app.repo.SearchExamplesByIds(ids)
}

func (app *goActiveLearningApp) SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByKeywords(keywords, aggregator, limit)
}

func (app *goActiveLearningApp) DeleteAllExamples() error {
	return app.repo.DeleteAllExamples()
}

func (app *goActiveLearningApp) CountPositiveExamples() (int, error) {
	return app.repo.CountPositiveExamples()
}

func (app *goActiveLearningApp) CountNegativeExamples() (int, error) {
	return app.repo.CountNegativeExamples()
}

func (app *goActiveLearningApp) CountUnlabeledExamples() (int, error) {
	return app.repo.CountUnlabeledExamples()
}

func (app *goActiveLearningApp) UpdateFeatureVector(e *model.Example) error {
	return app.repo.UpdateFeatureVector(e)
}

func (app *goActiveLearningApp) UpdateHatenaBookmark(e *model.Example) error {
	return app.repo.UpdateHatenaBookmark(e)
}

func (app *goActiveLearningApp) UpdateOrCreateReferringTweets(e *model.Example) error {
	return app.repo.UpdateOrCreateReferringTweets(e)
}

func (app *goActiveLearningApp) UpdateTweetLabel(exampleId int, idStr string, label model.LabelType) error {
	return app.repo.UpdateTweetLabel(exampleId, idStr, label)
}

func (app *goActiveLearningApp) SearchReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchPositiveReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchPositiveReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchNegativeReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchNegativeReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchUnlabeledReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchRecentReferringTweetsWithHighScore(from time.Time, scoreThreshold float64, limit int) (model.ReferringTweets, error) {
	return app.repo.SearchRecentReferringTweetsWithHighScore(from, scoreThreshold, limit)
}

func hatenaBookmarkByExampleId(hatenaBookmarks []*model.HatenaBookmark) map[int]*model.HatenaBookmark {
	result := make(map[int]*model.HatenaBookmark)
	for _, hb := range hatenaBookmarks {
		result[hb.ExampleId] = hb
	}
	return result
}

func (app *goActiveLearningApp) AttachMetadataIncludingFeatureVector(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	// make sure that example id must be filled
	for _, e := range examples {
		if e.Id == 0 {
			tmp, err := app.FindExampleByUlr(e.Url)
			if err != nil {
				return err
			}
			e.Id = tmp.Id
		}
	}

	fvList, err := app.repo.SearchFeatureVector(examples)
	if err != nil {
		return err
	}

	for _, e := range examples {
		if fv, ok := fvList[e.Id]; ok {
			e.Fv = fv
		}
	}

	return app.AttachMetadata(examples, bookmarkLimit, tweetLimit)
}

func (app *goActiveLearningApp) AttachMetadata(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples, bookmarkLimit)
	if err != nil {
		return err
	}
	hbByid := hatenaBookmarkByExampleId(hatenaBookmarks)
	for _, e := range examples {
		if b, ok := hbByid[e.Id]; ok {
			e.HatenaBookmark = b
		} else {
			e.HatenaBookmark = &model.HatenaBookmark{Bookmarks: []*model.Bookmark{}}
		}
	}

	referringTweetsById, err := app.repo.SearchReferringTweetsList(examples, tweetLimit)
	if err != nil {
		return err
	}
	for _, e := range examples {
		if t, ok := referringTweetsById[e.Id]; ok {
			e.ReferringTweets = &t
		} else {
			e.ReferringTweets = &model.ReferringTweets{}
		}
	}
	return nil
}

func (app *goActiveLearningApp) UpdateRecommendation(listName string, examples model.Examples) error {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return err
	}

	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	rec := model.Recommendation{RecommendationListType: listType, ExampleIds: exampleIds}
	return app.repo.UpdateRecommendation(rec)
}

func (app *goActiveLearningApp) GetRecommendation(listName string) (model.Examples, error) {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return nil, err
	}
	rec, err := app.repo.FindRecommendation(listType)
	return app.repo.SearchExamplesByIds(rec.ExampleIds)
}

func (app *goActiveLearningApp) ExistMetadata(e model.Example) bool {
	tmp, err := app.FindExampleByUlr(e.Url)
	if err != nil {
		return false
	}

	if tmp.StatusCode == http.StatusOK {
		return true
	}
	return false
}

func fetchMetaData(e *model.Example) error {
	article, err := fetcher.GetArticle(e.Url)
	if err != nil {
		return err
	}

	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.OgType = article.OgType
	e.OgImage = article.OgImage
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Favicon = article.Favicon

	now := time.Now()
	if article.PublishDate != nil && now.After(*article.PublishDate) {
		e.CreatedAt = *article.PublishDate
		e.UpdatedAt = *article.PublishDate
	}

	fv := util.RemoveDuplicate(example.ExtractFeatures(*e))
	if len(fv) > 100000 {
		return fmt.Errorf("too large features (N = %d) for %s", len(fv), e.FinalUrl)
	}
	e.Fv = fv

	return nil
}

func (app *goActiveLearningApp) Fetch(examples model.Examples) {
	batchSize := 100
	examplesList := make([]model.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		examplesWithMetaData := model.Examples{}
		examplesWithEmptyMetaData := model.Examples{}
		for _, e := range l {
			if !app.ExistMetadata(*e) {
				examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
			} else {
				examplesWithMetaData = append(examplesWithMetaData, e)
			}
		}
		// ToDo: 本当に必要か考える
		app.AttachMetadataIncludingFeatureVector(examplesWithMetaData, 0, 0)

		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *model.Example, idx int) {
				defer wg.Done()
				cnt, err := app.repo.GetErrorCount(e)
				if err != nil {
					log.Println(err.Error())
				}
				if cnt < 5 {
					fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
					if err := fetchMetaData(e); err != nil {
						app.repo.IncErrorCount(e)
						log.Println(err.Error())
					}
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}
<commit_msg>n+1を潰す<commit_after>package service

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"runtime"
	"time"

	"math"
	"os"
	"strconv"
	"sync"

	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/fetcher"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util"
)

func (app *goActiveLearningApp) UpdateOrCreateExample(e *model.Example) error {
	return app.repo.UpdateOrCreateExample(e)
}

func (app *goActiveLearningApp) UpdateScore(e *model.Example) error {
	return app.repo.UpdateScore(e)
}

func (app *goActiveLearningApp) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	return app.repo.InsertExampleFromScanner(scanner)
}

func (app *goActiveLearningApp) InsertExamplesFromReader(reader io.Reader) error {
	return app.repo.InsertExamplesFromReader(reader)
}

func (app *goActiveLearningApp) SearchExamples() (model.Examples, error) {
	return app.repo.SearchExamples()
}

func (app *goActiveLearningApp) SearchRecentExamples(from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamples(from, limit)
}

func (app *goActiveLearningApp) SearchRecentExamplesByHost(host string, from time.Time, limit int) (model.Examples, error) {
	return app.repo.SearchRecentExamplesByHost(host, from, limit)
}

func (app *goActiveLearningApp) SearchExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByLabel(label, limit)
}

func (app *goActiveLearningApp) SearchLabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchLabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveExamples(limit)
}

func (app *goActiveLearningApp) SearchNegativeExamples(limit int) (model.Examples, error) {
	return app.repo.SearchNegativeExamples(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledExamples(limit int) (model.Examples, error) {
	return app.repo.SearchUnlabeledExamples(limit)
}

func (app *goActiveLearningApp) SearchPositiveScoredExamples(limit int) (model.Examples, error) {
	return app.repo.SearchPositiveScoredExamples(limit)
}

func (app *goActiveLearningApp) FindExampleByUlr(url string) (*model.Example, error) {
	return app.repo.FindExampleByUlr(url)
}

func (app *goActiveLearningApp) FindExampleById(id int) (*model.Example, error) {
	return app.repo.FindExampleById(id)
}

func (app *goActiveLearningApp) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	return app.repo.SearchExamplesByUlrs(urls)
}

func (app *goActiveLearningApp) SearchExamplesByIds(ids []int) (model.Examples, error) {
	return app.repo.SearchExamplesByIds(ids)
}

func (app *goActiveLearningApp) SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error) {
	return app.repo.SearchExamplesByKeywords(keywords, aggregator, limit)
}

func (app *goActiveLearningApp) DeleteAllExamples() error {
	return app.repo.DeleteAllExamples()
}

func (app *goActiveLearningApp) CountPositiveExamples() (int, error) {
	return app.repo.CountPositiveExamples()
}

func (app *goActiveLearningApp) CountNegativeExamples() (int, error) {
	return app.repo.CountNegativeExamples()
}

func (app *goActiveLearningApp) CountUnlabeledExamples() (int, error) {
	return app.repo.CountUnlabeledExamples()
}

func (app *goActiveLearningApp) UpdateFeatureVector(e *model.Example) error {
	return app.repo.UpdateFeatureVector(e)
}

func (app *goActiveLearningApp) UpdateHatenaBookmark(e *model.Example) error {
	return app.repo.UpdateHatenaBookmark(e)
}

func (app *goActiveLearningApp) UpdateOrCreateReferringTweets(e *model.Example) error {
	return app.repo.UpdateOrCreateReferringTweets(e)
}

func (app *goActiveLearningApp) UpdateTweetLabel(exampleId int, idStr string, label model.LabelType) error {
	return app.repo.UpdateTweetLabel(exampleId, idStr, label)
}

func (app *goActiveLearningApp) SearchReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchPositiveReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchPositiveReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchNegativeReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchNegativeReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchUnlabeledReferringTweets(limit int) (model.ReferringTweets, error) {
	return app.repo.SearchUnlabeledReferringTweets(limit)
}

func (app *goActiveLearningApp) SearchRecentReferringTweetsWithHighScore(from time.Time, scoreThreshold float64, limit int) (model.ReferringTweets, error) {
	return app.repo.SearchRecentReferringTweetsWithHighScore(from, scoreThreshold, limit)
}

func hatenaBookmarkByExampleId(hatenaBookmarks []*model.HatenaBookmark) map[int]*model.HatenaBookmark {
	result := make(map[int]*model.HatenaBookmark)
	for _, hb := range hatenaBookmarks {
		result[hb.ExampleId] = hb
	}
	return result
}

func (app *goActiveLearningApp) AttachMetadataIncludingFeatureVector(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	// make sure that example id must be filled
	for _, e := range examples {
		if e.Id == 0 {
			tmp, err := app.FindExampleByUlr(e.Url)
			if err != nil {
				return err
			}
			e.Id = tmp.Id
		}
	}

	fvList, err := app.repo.SearchFeatureVector(examples)
	if err != nil {
		return err
	}

	for _, e := range examples {
		if fv, ok := fvList[e.Id]; ok {
			e.Fv = fv
		}
	}

	return app.AttachMetadata(examples, bookmarkLimit, tweetLimit)
}

func (app *goActiveLearningApp) AttachMetadata(examples model.Examples, bookmarkLimit int, tweetLimit int) error {
	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples, bookmarkLimit)
	if err != nil {
		return err
	}
	hbByid := hatenaBookmarkByExampleId(hatenaBookmarks)
	for _, e := range examples {
		if b, ok := hbByid[e.Id]; ok {
			e.HatenaBookmark = b
		} else {
			e.HatenaBookmark = &model.HatenaBookmark{Bookmarks: []*model.Bookmark{}}
		}
	}

	referringTweetsById, err := app.repo.SearchReferringTweetsList(examples, tweetLimit)
	if err != nil {
		return err
	}
	for _, e := range examples {
		if t, ok := referringTweetsById[e.Id]; ok {
			e.ReferringTweets = &t
		} else {
			e.ReferringTweets = &model.ReferringTweets{}
		}
	}
	return nil
}

func (app *goActiveLearningApp) UpdateRecommendation(listName string, examples model.Examples) error {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return err
	}

	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	rec := model.Recommendation{RecommendationListType: listType, ExampleIds: exampleIds}
	return app.repo.UpdateRecommendation(rec)
}

func (app *goActiveLearningApp) GetRecommendation(listName string) (model.Examples, error) {
	listType, err := model.GetRecommendationListType(listName)
	if err != nil {
		return nil, err
	}
	rec, err := app.repo.FindRecommendation(listType)
	return app.repo.SearchExamplesByIds(rec.ExampleIds)
}

func (app *goActiveLearningApp) ExistMetadata(e model.Example) bool {
	tmp, err := app.FindExampleByUlr(e.Url)
	if err != nil {
		return false
	}

	if tmp.StatusCode == http.StatusOK {
		return true
	}
	return false
}

func (app *goActiveLearningApp) splitExamplesByStatusOK(examples model.Examples) (model.Examples, model.Examples, error) {
	urls := make([]string, 0)
	exampleByurl := make(map[string]*model.Example)
	for _, e := range examples {
		exampleByurl[e.Url] = e
		urls = append(urls, e.Url)
	}
	tmpExamples, err := app.SearchExamplesByUlrs(urls)
	if err != nil {
		return nil, nil, err
	}

	examplesWithMetaData := model.Examples{}
	examplesWithEmptyMetaData := model.Examples{}
	for _, e := range tmpExamples {
		if e.StatusCode == http.StatusOK {
			examplesWithMetaData = append(examplesWithMetaData, exampleByurl[e.Url])
			delete(exampleByurl, e.Url)
		} else {
			examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, exampleByurl[e.Url])
			delete(exampleByurl, e.Url)
		}
	}
	for _, e := range exampleByurl {
		examplesWithEmptyMetaData = append(examplesWithEmptyMetaData, e)
	}
	return examplesWithMetaData, examplesWithEmptyMetaData, nil
}

func fetchMetaData(e *model.Example) error {
	article, err := fetcher.GetArticle(e.Url)
	if err != nil {
		return err
	}

	e.Title = article.Title
	e.FinalUrl = article.Url
	e.Description = article.Description
	e.OgDescription = article.OgDescription
	e.OgType = article.OgType
	e.OgImage = article.OgImage
	e.Body = article.Body
	e.StatusCode = article.StatusCode
	e.Favicon = article.Favicon

	now := time.Now()
	if article.PublishDate != nil && now.After(*article.PublishDate) {
		e.CreatedAt = *article.PublishDate
		e.UpdatedAt = *article.PublishDate
	}

	fv := util.RemoveDuplicate(example.ExtractFeatures(*e))
	if len(fv) > 100000 {
		return fmt.Errorf("too large features (N = %d) for %s", len(fv), e.FinalUrl)
	}
	e.Fv = fv

	return nil
}

func (app *goActiveLearningApp) Fetch(examples model.Examples) {
	batchSize := 100
	examplesList := make([]model.Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		examplesWithMetaData, examplesWithEmptyMetaData, err := app.splitExamplesByStatusOK(l)
		if err != nil {
			log.Println(err.Error())
		}
		// ToDo: 本当に必要か考える
		app.AttachMetadataIncludingFeatureVector(examplesWithMetaData, 0, 0)

		wg := &sync.WaitGroup{}
		cpus := runtime.NumCPU()
		runtime.GOMAXPROCS(cpus)
		sem := make(chan struct{}, batchSize)
		for idx, e := range examplesWithEmptyMetaData {
			wg.Add(1)
			sem <- struct{}{}
			go func(e *model.Example, idx int) {
				defer wg.Done()
				cnt, err := app.repo.GetErrorCount(e)
				if err != nil {
					log.Println(err.Error())
				}
				if cnt < 5 {
					fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
					if err := fetchMetaData(e); err != nil {
						app.repo.IncErrorCount(e)
						log.Println(err.Error())
					}
				}
				<-sem
			}(e, idx)
		}
		wg.Wait()
	}
}
<|endoftext|>


<commit_before>package devd
<commit_msg>Add test for the Route.Watch/3 method.<commit_after>package devd

import (
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/cortesi/termlog"
)

func addTempFile(wg *sync.WaitGroup, t *testing.T, tmpFolder string, fname string, content string) {
	if err := ioutil.WriteFile(tmpFolder+"/"+fname, []byte(content), 0644); err != nil {
		t.Error(err)
	}
	wg.Add(1)
}

// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
	c := make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return false // completed normally
	case <-time.After(timeout):
		return true // timed out
	}
}

func TestRouteWatch(t *testing.T) {
	logger := termlog.NewLog()
	logger.Quiet()

	tmpFolder, err := ioutil.TempDir("", "")
	if err != nil {
		t.Error(err)
	}
	defer os.RemoveAll(tmpFolder)

  // Ensure that using . for the path works:
	os.Chdir(tmpFolder)
	routes := make(RouteCollection)
	routes.Add(".", nil)

	changedFiles := make(map[string]int)

	ch := make(chan []string, 1)
	var wg sync.WaitGroup
	go func() {
		for {
			data, more := <-ch
			if more {
				t.Log("received notification for changed file(s):", strings.Join(data, ", "))
				for i := range data {
					changedFiles[data[i]] = 1
					wg.Done()
				}
			} else {
				t.Log("No more changes are expected")
				return
			}
		}
	}()
	for i := range routes {
		err := routes[i].Watch(ch, nil, logger)
		if err != nil {
			t.Error(err)
		}
	}
	//////////////////////////////////
	t.Log("Temp folder:", tmpFolder)
	addTempFile(&wg, t, tmpFolder, "a.txt", "foo\n")
	addTempFile(&wg, t, tmpFolder, "c.txt", "bar\n")
	addTempFile(&wg, t, tmpFolder, "another.file.txt", "bar\n")
	//////////////////////////////////
	waitTimeout(&wg, 700*time.Millisecond)
	close(ch)
	if len(changedFiles) != 3 {
		t.Error("The watch should have been notified about 3 changed files")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/aqatl/mal/anilist"
	"github.com/fatih/color"
	"github.com/skratchdot/open-golang/open"
	"github.com/urfave/cli"
	"net/url"
	"sort"
	"strconv"
)

func AniListApp(app *cli.App) *cli.App {
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "max",
			Usage: "visible entries threshold",
		},
		cli.BoolFlag{
			Name:  "all, a",
			Usage: "display all entries; same as --max -1",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "mal",
			Aliases:   []string{"s"},
			Usage:     "Switches app mode to MyAnimeList",
			UsageText: "mal mal",
			Action:    switchToMal,
		},
		cli.Command{
			Name:     "eps",
			Aliases:  []string{"episodes"},
			Category: "Update",
			Usage: "Set the watched episodes value. " +
				"If n not specified, the number will be increased by one",
			UsageText: "mal eps <n>",
			Action:    alSetEntryEpisodes,
		},
		cli.Command{
			Name:      "status",
			Category:  "Update",
			Usage:     "Set your status for selected entry",
			UsageText: "mal status [watching|planning|completed|dropped|paused|repeating]",
			Action:    alSetEntryStatus,
		},
		cli.Command{
			Name:      "fuzzy-select",
			Aliases:   []string{"fsel"},
			Category:  "Config",
			Usage:     "Interactive fuzzy search through your list",
			UsageText: "mal fsel [search string (optional)]",
			Action:    alFuzzySelectEntry,
		},
		cli.Command{
			Name:     "nyaa",
			Aliases:  []string{"n"},
			Category: "Action",
			Usage:    "Open interactive torrent search",
			Action:   alNyaaCui,
		},
		cli.Command{
			Name:      "nyaa-web",
			Aliases:   []string{"nw"},
			Category:  "Action",
			Usage:     "Open torrent search in browser",
			UsageText: "mal nyaa-web",
			Action:    alNyaaWebsite,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "alt",
					Usage: "choose an alternative title",
				},
			},
		},
		cli.Command{
			Name:      "web",
			Aliases:   []string{"website", "open", "url"},
			Category:  "Action",
			Usage:     "Open url associated with selected entry or change url if provided",
			UsageText: "mal web <url>",
			Action:    alOpenWebsite,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "clear",
					Usage: "Clear url for current entry",
				},
			},
			Subcommands: []cli.Command{
				cli.Command{
					Name:      "get-all",
					Usage:     "Print all set urls",
					UsageText: "mal web get-all",
					Action:    alPrintWebsites,
				},
			},
		},
	}

	app.Action = cli.ActionFunc(aniListDefaultAction)

	return app
}

func aniListDefaultAction(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return err
	}
	cfg := LoadConfig()
	list := alGetList(al, cfg.ALStatus)

	sort.Slice(list, func(i, j int) bool {
		return list[i].UpdatedAt > list[j].UpdatedAt
	})

	var visibleEntries int
	if visibleEntries = ctx.Int("max"); visibleEntries == 0 {
		//`Max` flag not specified, get value from config
		visibleEntries = cfg.MaxVisibleEntries
	}
	if visibleEntries > len(list) || visibleEntries < 0 || ctx.Bool("all") {
		visibleEntries = len(list)
	}

	fmt.Printf("No%64s%8s%6s\n", "Title", "Eps", "Score")
	fmt.Println("================================================================================")
	pattern := "%2d%64.64s%8s%6d\n"
	var entry *anilist.MediaListEntry
	for i := visibleEntries - 1; i >= 0; i-- {
		entry = &list[i]
		if entry.ListId == cfg.ALSelectedID {
			color.HiYellow(pattern, i+1, entry.Title.UserPreferred,
				fmt.Sprintf("%d/%d", entry.Progress, entry.Episodes),
				entry.Score)
		} else {
			fmt.Printf(pattern, i+1, entry.Title.UserPreferred,
				fmt.Sprintf("%d/%d", entry.Progress, entry.Episodes),
				entry.Score)
		}
	}

	return nil
}

func alGetList(al *AniList, status anilist.MediaListStatus) List {
	if status == anilist.All {
		return al.List
	} else {
		list := make(List, 0)
		for i := range al.List {
			if al.List[i].Status == status {
				list = append(list, al.List[i])
			}
		}
		return list
	}
}

func loadAniListFull() (al *AniList, entry *anilist.MediaListEntry, cfg *Config, err error) {
	al, err = loadAniList()
	if err != nil {
		return
	}
	cfg = LoadConfig()
	if cfg.ALSelectedID == 0 {
		fmt.Println("No entry selected")
	}
	entry = al.GetMediaListById(cfg.ALSelectedID)
	if entry == nil {
		err = fmt.Errorf("no entry found")
	}
	return
}

func switchToMal(ctx *cli.Context) error {
	appCfg := AppConfig{}
	LoadJsonFile(AppConfigFile, &appCfg)
	appCfg.Mode = MalMode
	if err := SaveJsonFile(AppConfigFile, &appCfg); err != nil {
		return err
	}
	fmt.Println("App mode switched to MyAnimeList")
	return nil
}

func alSetEntryEpisodes(ctx *cli.Context) error {
	al, entry, cfg, err := loadAniListFull()
	if err != nil {
		return err
	}
	epsBefore := entry.Progress

	if arg := ctx.Args().First(); arg != "" {
		n, err := strconv.Atoi(arg)
		if err != nil {
			return fmt.Errorf("n must be a non-negative integer")
		}
		if n < 0 {
			return fmt.Errorf("n can't be lower than 0")
		}
		entry.Progress = n
	} else {
		entry.Progress++
	}

	alStatusAutoUpdate(cfg, entry)

	if err = anilist.SaveMediaListEntry(entry, al.Token); err != nil {
		return err
	}
	if err = saveAniListAnimeLists(al); err != nil {
		return err
	}

	fmt.Println("Updated successfully")
	alPrintEntryDetailsAfterUpdatedEpisodes(entry, epsBefore)
	return nil
}

func alStatusAutoUpdate(cfg *Config, entry *anilist.MediaListEntry) {
	if cfg.StatusAutoUpdateMode == Off || entry.Episodes == 0 {
		return
	}

	if (cfg.StatusAutoUpdateMode == Normal && entry.Progress >= entry.Episodes) ||
		(cfg.StatusAutoUpdateMode == AfterThreshold && entry.Progress > entry.Episodes) {
		entry.Status = anilist.Completed
		entry.Progress = entry.Episodes
		return
	}

	if entry.Status == anilist.Completed && entry.Progress < entry.Episodes {
		entry.Status = anilist.Current
		return
	}
}

func alSetEntryStatus(ctx *cli.Context) error {
	al, entry, _, err := loadAniListFull()
	if err != nil {
		return err
	}

	status := anilist.ParseStatus(ctx.Args().First())
	if status == anilist.All {
		return fmt.Errorf("invalid status; possible values: " +
			"watching|planning|completed|dropped|paused|repeating")
	}

	entry.Status = status

	if err = anilist.SaveMediaListEntry(entry, al.Token); err != nil {
		return err
	}
	if err = saveAniListAnimeLists(al); err != nil {
		return err
	}

	fmt.Println("Updated successfully")
	alPrintEntryDetails(entry)
	return nil
}

func alNyaaWebsite(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return err
	}
	cfg := LoadConfig()

	entry := al.GetMediaListById(cfg.ALSelectedID)
	if entry == nil {
		return fmt.Errorf("no entry selected")
	}

	var searchTerm string
	if ctx.Bool("alt") {
		alts := make([]string, 0, 3+len(entry.Synonyms))
		if t := entry.Title.English; t != "" {
			alts = append(alts, t)
		}
		if t := entry.Title.Native; t != "" {
			alts = append(alts, t)
		}
		if t := entry.Title.Romaji; t != "" {
			alts = append(alts, t)
		}
		alts = append(alts, entry.Synonyms...)
		fmt.Printf("Select desired title\n\n")
		if searchTerm = chooseStrFromSlice(alts); searchTerm == "" {
			return fmt.Errorf("no alternative titles")
		}
	} else {
		searchTerm = entry.Title.Romaji
	}

	address := "https://nyaa.si/?f=0&c=1_2&q=" + url.QueryEscape(searchTerm)
	if path := cfg.BrowserPath; path == "" {
		open.Start(address)
	} else {
		open.StartWith(address, path)
	}

	fmt.Println("Searched for:")
	alPrintEntryDetails(entry)
	return nil
}

func alOpenWebsite(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return nil
	}

	cfg := LoadConfig()

	entry := al.GetMediaListById(cfg.ALSelectedID)
	if entry == nil {
		return fmt.Errorf("no entry selected")
	}

	if newUrl := ctx.Args().First(); newUrl != "" {
		cfg.Websites[entry.IdMal] = newUrl
		cfg.Save()

		fmt.Print("Entry: ")
		color.HiYellow("%v", entry.Title)
		fmt.Print("URL: ")
		color.HiRed("%v", cfg.Websites[entry.IdMal])

		return nil
	}

	if ctx.Bool("clear") {
		delete(cfg.Websites, entry.IdMal)
		cfg.Save()

		fmt.Println("Entry cleared")
		return nil
	}

	if entryUrl, ok := cfg.Websites[entry.IdMal]; ok {
		if path := cfg.BrowserPath; path == "" {
			open.Start(entryUrl)
		} else {
			open.StartWith(entryUrl, path)
		}

		fmt.Println("Opened website for:")
		alPrintEntryDetails(entry)
		fmt.Fprintf(color.Output, "URL: %v\n", color.CyanString("%v", entryUrl))
	} else {
		fmt.Println("Nothing to open")
	}

	return nil
}

func alPrintWebsites(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return err
	}

	cfg := LoadConfig()

	for k, v := range cfg.Websites {
		entryUrl := fmt.Sprintf("\033[3%d;%dm%s\033[0m ", 3, 1, v)

		var title string
		if entry := al.GetMediaListByMalId(k); entry != nil {
			title = entry.Title.UserPreferred
		}

		fmt.Fprintf(color.Output, "%6d (%s): %s\n", k, title, entryUrl)
	}

	return nil
}
<commit_msg>Updating AniList entry score<commit_after>package main

import (
	"fmt"
	"github.com/aqatl/mal/anilist"
	"github.com/fatih/color"
	"github.com/skratchdot/open-golang/open"
	"github.com/urfave/cli"
	"net/url"
	"sort"
	"strconv"
)

func AniListApp(app *cli.App) *cli.App {
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "max",
			Usage: "visible entries threshold",
		},
		cli.BoolFlag{
			Name:  "all, a",
			Usage: "display all entries; same as --max -1",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "mal",
			Aliases:   []string{"s"},
			Usage:     "Switches app mode to MyAnimeList",
			UsageText: "mal mal",
			Action:    switchToMal,
		},
		cli.Command{
			Name:     "eps",
			Aliases:  []string{"episodes"},
			Category: "Update",
			Usage: "Set the watched episodes value. " +
				"If n not specified, the number will be increased by one",
			UsageText: "mal eps <n>",
			Action:    alSetEntryEpisodes,
		},
		cli.Command{
			Name:      "status",
			Category:  "Update",
			Usage:     "Set your status for selected entry",
			UsageText: "mal status [watching|planning|completed|dropped|paused|repeating]",
			Action:    alSetEntryStatus,
		},
		cli.Command{
			Name:      "score",
			Category:  "Update",
			Usage:     "Set your rating for selected entry",
			UsageText: "mal score <0-10>",
			Action:    alSetEntryScore,
		},
		cli.Command{
			Name:      "fuzzy-select",
			Aliases:   []string{"fsel"},
			Category:  "Config",
			Usage:     "Interactive fuzzy search through your list",
			UsageText: "mal fsel [search string (optional)]",
			Action:    alFuzzySelectEntry,
		},
		cli.Command{
			Name:     "nyaa",
			Aliases:  []string{"n"},
			Category: "Action",
			Usage:    "Open interactive torrent search",
			Action:   alNyaaCui,
		},
		cli.Command{
			Name:      "nyaa-web",
			Aliases:   []string{"nw"},
			Category:  "Action",
			Usage:     "Open torrent search in browser",
			UsageText: "mal nyaa-web",
			Action:    alNyaaWebsite,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "alt",
					Usage: "choose an alternative title",
				},
			},
		},
		cli.Command{
			Name:      "web",
			Aliases:   []string{"website", "open", "url"},
			Category:  "Action",
			Usage:     "Open url associated with selected entry or change url if provided",
			UsageText: "mal web <url>",
			Action:    alOpenWebsite,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "clear",
					Usage: "Clear url for current entry",
				},
			},
			Subcommands: []cli.Command{
				cli.Command{
					Name:      "get-all",
					Usage:     "Print all set urls",
					UsageText: "mal web get-all",
					Action:    alPrintWebsites,
				},
			},
		},
	}

	app.Action = cli.ActionFunc(aniListDefaultAction)

	return app
}

func aniListDefaultAction(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return err
	}
	cfg := LoadConfig()
	list := alGetList(al, cfg.ALStatus)

	sort.Slice(list, func(i, j int) bool {
		return list[i].UpdatedAt > list[j].UpdatedAt
	})

	var visibleEntries int
	if visibleEntries = ctx.Int("max"); visibleEntries == 0 {
		//`Max` flag not specified, get value from config
		visibleEntries = cfg.MaxVisibleEntries
	}
	if visibleEntries > len(list) || visibleEntries < 0 || ctx.Bool("all") {
		visibleEntries = len(list)
	}

	fmt.Printf("No%64s%8s%6s\n", "Title", "Eps", "Score")
	fmt.Println("================================================================================")
	pattern := "%2d%64.64s%8s%6d\n"
	var entry *anilist.MediaListEntry
	for i := visibleEntries - 1; i >= 0; i-- {
		entry = &list[i]
		if entry.ListId == cfg.ALSelectedID {
			color.HiYellow(pattern, i+1, entry.Title.UserPreferred,
				fmt.Sprintf("%d/%d", entry.Progress, entry.Episodes),
				entry.Score)
		} else {
			fmt.Printf(pattern, i+1, entry.Title.UserPreferred,
				fmt.Sprintf("%d/%d", entry.Progress, entry.Episodes),
				entry.Score)
		}
	}

	return nil
}

func alGetList(al *AniList, status anilist.MediaListStatus) List {
	if status == anilist.All {
		return al.List
	} else {
		list := make(List, 0)
		for i := range al.List {
			if al.List[i].Status == status {
				list = append(list, al.List[i])
			}
		}
		return list
	}
}

func loadAniListFull() (al *AniList, entry *anilist.MediaListEntry, cfg *Config, err error) {
	al, err = loadAniList()
	if err != nil {
		return
	}
	cfg = LoadConfig()
	if cfg.ALSelectedID == 0 {
		fmt.Println("No entry selected")
	}
	entry = al.GetMediaListById(cfg.ALSelectedID)
	if entry == nil {
		err = fmt.Errorf("no entry found")
	}
	return
}

func switchToMal(ctx *cli.Context) error {
	appCfg := AppConfig{}
	LoadJsonFile(AppConfigFile, &appCfg)
	appCfg.Mode = MalMode
	if err := SaveJsonFile(AppConfigFile, &appCfg); err != nil {
		return err
	}
	fmt.Println("App mode switched to MyAnimeList")
	return nil
}

func alSetEntryEpisodes(ctx *cli.Context) error {
	al, entry, cfg, err := loadAniListFull()
	if err != nil {
		return err
	}
	epsBefore := entry.Progress

	if arg := ctx.Args().First(); arg != "" {
		n, err := strconv.Atoi(arg)
		if err != nil {
			return fmt.Errorf("n must be a non-negative integer")
		}
		if n < 0 {
			return fmt.Errorf("n can't be lower than 0")
		}
		entry.Progress = n
	} else {
		entry.Progress++
	}

	alStatusAutoUpdate(cfg, entry)

	if err = anilist.SaveMediaListEntry(entry, al.Token); err != nil {
		return err
	}
	if err = saveAniListAnimeLists(al); err != nil {
		return err
	}

	fmt.Println("Updated successfully")
	alPrintEntryDetailsAfterUpdatedEpisodes(entry, epsBefore)
	return nil
}

func alStatusAutoUpdate(cfg *Config, entry *anilist.MediaListEntry) {
	if cfg.StatusAutoUpdateMode == Off || entry.Episodes == 0 {
		return
	}

	if (cfg.StatusAutoUpdateMode == Normal && entry.Progress >= entry.Episodes) ||
		(cfg.StatusAutoUpdateMode == AfterThreshold && entry.Progress > entry.Episodes) {
		entry.Status = anilist.Completed
		entry.Progress = entry.Episodes
		return
	}

	if entry.Status == anilist.Completed && entry.Progress < entry.Episodes {
		entry.Status = anilist.Current
		return
	}
}

func alSetEntryStatus(ctx *cli.Context) error {
	al, entry, _, err := loadAniListFull()
	if err != nil {
		return err
	}

	status := anilist.ParseStatus(ctx.Args().First())
	if status == anilist.All {
		return fmt.Errorf("invalid status; possible values: " +
			"watching|planning|completed|dropped|paused|repeating")
	}

	entry.Status = status

	if err = anilist.SaveMediaListEntry(entry, al.Token); err != nil {
		return err
	}
	if err = saveAniListAnimeLists(al); err != nil {
		return err
	}

	fmt.Println("Updated successfully")
	alPrintEntryDetails(entry)
	return nil
}

func alSetEntryScore(ctx *cli.Context) error {
	al, entry, _, err := loadAniListFull()
	if err != nil {
		return err
	}

	score, err := strconv.Atoi(ctx.Args().First())
	if err != nil || score < 0 || score > 10 {
		return fmt.Errorf("invalid score; valid range: <0;10>")
	}

	entry.Score = score

	if err = anilist.SaveMediaListEntry(entry, al.Token); err != nil {
		return err
	}
	if err = saveAniListAnimeLists(al); err != nil {
		return err
	}

	fmt.Println("Updated successfully")
	alPrintEntryDetails(entry)
	return nil
}

func alNyaaWebsite(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return err
	}
	cfg := LoadConfig()

	entry := al.GetMediaListById(cfg.ALSelectedID)
	if entry == nil {
		return fmt.Errorf("no entry selected")
	}

	var searchTerm string
	if ctx.Bool("alt") {
		alts := make([]string, 0, 3+len(entry.Synonyms))
		if t := entry.Title.English; t != "" {
			alts = append(alts, t)
		}
		if t := entry.Title.Native; t != "" {
			alts = append(alts, t)
		}
		if t := entry.Title.Romaji; t != "" {
			alts = append(alts, t)
		}
		alts = append(alts, entry.Synonyms...)
		fmt.Printf("Select desired title\n\n")
		if searchTerm = chooseStrFromSlice(alts); searchTerm == "" {
			return fmt.Errorf("no alternative titles")
		}
	} else {
		searchTerm = entry.Title.Romaji
	}

	address := "https://nyaa.si/?f=0&c=1_2&q=" + url.QueryEscape(searchTerm)
	if path := cfg.BrowserPath; path == "" {
		open.Start(address)
	} else {
		open.StartWith(address, path)
	}

	fmt.Println("Searched for:")
	alPrintEntryDetails(entry)
	return nil
}

func alOpenWebsite(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return nil
	}

	cfg := LoadConfig()

	entry := al.GetMediaListById(cfg.ALSelectedID)
	if entry == nil {
		return fmt.Errorf("no entry selected")
	}

	if newUrl := ctx.Args().First(); newUrl != "" {
		cfg.Websites[entry.IdMal] = newUrl
		cfg.Save()

		fmt.Print("Entry: ")
		color.HiYellow("%v", entry.Title)
		fmt.Print("URL: ")
		color.HiRed("%v", cfg.Websites[entry.IdMal])

		return nil
	}

	if ctx.Bool("clear") {
		delete(cfg.Websites, entry.IdMal)
		cfg.Save()

		fmt.Println("Entry cleared")
		return nil
	}

	if entryUrl, ok := cfg.Websites[entry.IdMal]; ok {
		if path := cfg.BrowserPath; path == "" {
			open.Start(entryUrl)
		} else {
			open.StartWith(entryUrl, path)
		}

		fmt.Println("Opened website for:")
		alPrintEntryDetails(entry)
		fmt.Fprintf(color.Output, "URL: %v\n", color.CyanString("%v", entryUrl))
	} else {
		fmt.Println("Nothing to open")
	}

	return nil
}

func alPrintWebsites(ctx *cli.Context) error {
	al, err := loadAniList()
	if err != nil {
		return err
	}

	cfg := LoadConfig()

	for k, v := range cfg.Websites {
		entryUrl := fmt.Sprintf("\033[3%d;%dm%s\033[0m ", 3, 1, v)

		var title string
		if entry := al.GetMediaListByMalId(k); entry != nil {
			title = entry.Title.UserPreferred
		}

		fmt.Fprintf(color.Output, "%6d (%s): %s\n", k, title, entryUrl)
	}

	return nil
}
<|endoftext|>