<commit_before>package client

import (
	"github.com/fluffle/goirc/logging"
	"runtime"
	"strings"
	"sync"
)

// Handlers are triggered on incoming Lines from the server, with the handler
// "name" being equivalent to Line.Cmd. Read the RFCs for details on what
// replies could come from the server. They'll generally be things like
// "PRIVMSG", "JOIN", etc. but all the numeric replies are left as ascii
// strings of digits like "332" (mainly because I really didn't feel like
// putting massive constant tables in).
//
// Foreground handlers have a guarantee of protocol consistency: all the
// handlers for one event will have finished before the handlers for the
// next start processing. They are run in parallel but block the event
// loop, so care should be taken to ensure these handlers are quick :-)
//
// Background handlers are run in parallel and do not block the event loop.
// This is useful for things that may need to do significant work.
type Handler interface {
	Handle(*Conn, *Line)
}

// Removers allow for a handler that has been previously added to the client
// to be removed.
type Remover interface {
	Remove()
}

// HandlerFunc allows a bare function with this signature to implement the
// Handler interface. It is used by Conn.HandleFunc.
type HandlerFunc func(*Conn, *Line)

func (hf HandlerFunc) Handle(conn *Conn, line *Line) {
	hf(conn, line)
}

// Handlers are organised using a map of linked-lists, with each map
// key representing an IRC verb or numeric, and the linked list values
// being handlers that are executed in parallel when a Line from the
// server with that verb or numeric arrives.
type hSet struct {
	set map[string]*hList
	sync.RWMutex
}

type hList struct {
	start, end *hNode
}

// Storing the forward and backward links in the node allows O(1) removal.
// This probably isn't strictly necessary but I think it's kinda nice.
type hNode struct {
	next, prev *hNode
	set        *hSet
	event      string
	handler    Handler
}

// A hNode implements both Handler (with configurable panic recovery)...
func (hn *hNode) Handle(conn *Conn, line *Line) {
	defer conn.cfg.Recover(conn, line)
	hn.handler.Handle(conn, line)
}

// ... and Remover.
func (hn *hNode) Remove() {
	hn.set.remove(hn)
}

func handlerSet() *hSet {
	return &hSet{set: make(map[string]*hList)}
}

// When a new Handler is added for an event, it is wrapped in a hNode and
// returned as a Remover so the caller can remove it at a later time.
func (hs *hSet) add(ev string, h Handler) Remover {
	hs.Lock()
	defer hs.Unlock()
	ev = strings.ToLower(ev)
	l, ok := hs.set[ev]
	if !ok {
		l = &hList{}
	}
	hn := &hNode{
		set:     hs,
		event:   ev,
		handler: h,
	}
	if !ok {
		l.start = hn
	} else {
		hn.prev = l.end
		l.end.next = hn
	}
	l.end = hn
	hs.set[ev] = l
	return hn
}

func (hs *hSet) remove(hn *hNode) {
	hs.Lock()
	defer hs.Unlock()
	l, ok := hs.set[hn.event]
	if !ok {
		logging.Error("Removing node for unknown event '%s'", hn.event)
		return
	}
	if hn.next == nil {
		l.end = hn.prev
	} else {
		hn.next.prev = hn.prev
	}
	if hn.prev == nil {
		l.start = hn.next
	} else {
		hn.prev.next = hn.next
	}
	hn.next = nil
	hn.prev = nil
	hn.set = nil
	if l.start == nil || l.end == nil {
		delete(hs.set, hn.event)
	}
}

func (hs *hSet) dispatch(conn *Conn, line *Line) {
	hs.RLock()
	defer hs.RUnlock()
	ev := strings.ToLower(line.Cmd)
	list, ok := hs.set[ev]
	if !ok {
		return
	}
	wg := &sync.WaitGroup{}
	for hn := list.start; hn != nil; hn = hn.next {
		wg.Add(1)
		go func(hn *hNode) {
			hn.Handle(conn, line.Copy())
			wg.Done()
		}(hn)
	}
	wg.Wait()
}

// Handle adds the provided handler to the foreground set for the named event.
// It will return a Remover that allows that handler to be removed again.
func (conn *Conn) Handle(name string, h Handler) Remover {
	return conn.fgHandlers.add(name, h)
}

// HandleBG adds the provided handler to the background set for the named
// event. It may go away in the future.
// It will return a Remover that allows that handler to be removed again.
func (conn *Conn) HandleBG(name string, h Handler) Remover {
	return conn.bgHandlers.add(name, h)
}

func (conn *Conn) handle(name string, h Handler) Remover {
	return conn.intHandlers.add(name, h)
}

// HandleFunc adds the provided function as a handler in the foreground set
// for the named event.
// It will return a Remover that allows that handler to be removed again.
func (conn *Conn) HandleFunc(name string, hf HandlerFunc) Remover {
	return conn.Handle(name, hf)
}

func (conn *Conn) dispatch(line *Line) {
	// We run the internal handlers first, including all state tracking ones.
	// This ensures that user-supplied handlers that use the tracker have a
	// consistent view of the connection state in handlers that mutate it.
	conn.intHandlers.dispatch(conn, line)
	go conn.bgHandlers.dispatch(conn, line)
	conn.fgHandlers.dispatch(conn, line)
}

// LogPanic is used as the default panic catcher for the client. If, like me,
// you are not good with computer, and you'd prefer your bot not to vanish into
// the ether whenever you make unfortunate programming mistakes, you may find
// this useful: it will recover panics from handler code and log the errors.
func (conn *Conn) LogPanic(line *Line) {
	if err := recover(); err != nil {
		_, f, l, _ := runtime.Caller(2)
		logging.Error("%s:%d: panic: %v", f, l, err)
	}
}
<commit_msg>Avoid holding hSet lock while executing handlers. Fixes #102.<commit_after>package client

import (
	"runtime"
	"strings"
	"sync"

	"github.com/fluffle/goirc/logging"
)

// Handlers are triggered on incoming Lines from the server, with the handler
// "name" being equivalent to Line.Cmd. Read the RFCs for details on what
// replies could come from the server. They'll generally be things like
// "PRIVMSG", "JOIN", etc. but all the numeric replies are left as ascii
// strings of digits like "332" (mainly because I really didn't feel like
// putting massive constant tables in).
//
// Foreground handlers have a guarantee of protocol consistency: all the
// handlers for one event will have finished before the handlers for the
// next start processing. They are run in parallel but block the event
// loop, so care should be taken to ensure these handlers are quick :-)
//
// Background handlers are run in parallel and do not block the event loop.
// This is useful for things that may need to do significant work.
type Handler interface {
	Handle(*Conn, *Line)
}

// Removers allow for a handler that has been previously added to the client
// to be removed.
type Remover interface {
	Remove()
}

// HandlerFunc allows a bare function with this signature to implement the
// Handler interface. It is used by Conn.HandleFunc.
type HandlerFunc func(*Conn, *Line)

func (hf HandlerFunc) Handle(conn *Conn, line *Line) {
	hf(conn, line)
}

// Handlers are organised using a map of linked-lists, with each map
// key representing an IRC verb or numeric, and the linked list values
// being handlers that are executed in parallel when a Line from the
// server with that verb or numeric arrives.
type hSet struct {
	set map[string]*hList
	sync.RWMutex
}

type hList struct {
	start, end *hNode
}

// Storing the forward and backward links in the node allows O(1) removal.
// This probably isn't strictly necessary but I think it's kinda nice.
type hNode struct {
	next, prev *hNode
	set        *hSet
	event      string
	handler    Handler
}

// A hNode implements both Handler (with configurable panic recovery)...
func (hn *hNode) Handle(conn *Conn, line *Line) {
	defer conn.cfg.Recover(conn, line)
	hn.handler.Handle(conn, line)
}

// ... and Remover.
func (hn *hNode) Remove() {
	hn.set.remove(hn)
}

func handlerSet() *hSet {
	return &hSet{set: make(map[string]*hList)}
}

// When a new Handler is added for an event, it is wrapped in a hNode and
// returned as a Remover so the caller can remove it at a later time.
func (hs *hSet) add(ev string, h Handler) Remover {
	hs.Lock()
	defer hs.Unlock()
	ev = strings.ToLower(ev)
	l, ok := hs.set[ev]
	if !ok {
		l = &hList{}
	}
	hn := &hNode{
		set:     hs,
		event:   ev,
		handler: h,
	}
	if !ok {
		l.start = hn
	} else {
		hn.prev = l.end
		l.end.next = hn
	}
	l.end = hn
	hs.set[ev] = l
	return hn
}

func (hs *hSet) remove(hn *hNode) {
	hs.Lock()
	defer hs.Unlock()
	l, ok := hs.set[hn.event]
	if !ok {
		logging.Error("Removing node for unknown event '%s'", hn.event)
		return
	}
	if hn.next == nil {
		l.end = hn.prev
	} else {
		hn.next.prev = hn.prev
	}
	if hn.prev == nil {
		l.start = hn.next
	} else {
		hn.prev.next = hn.next
	}
	hn.next = nil
	hn.prev = nil
	hn.set = nil
	if l.start == nil || l.end == nil {
		delete(hs.set, hn.event)
	}
}

func (hs *hSet) getHandlers(ev string) []*hNode {
	hs.RLock()
	defer hs.RUnlock()
	list, ok := hs.set[ev]
	if !ok {
		return nil
	}
	// Copy current list of handlers to a temporary slice under the lock.
	handlers := make([]*hNode, 0)
	for hn := list.start; hn != nil; hn = hn.next {
		handlers = append(handlers, hn)
	}
	return handlers
}

func (hs *hSet) dispatch(conn *Conn, line *Line) {
	ev := strings.ToLower(line.Cmd)
	wg := &sync.WaitGroup{}
	for _, hn := range hs.getHandlers(ev) {
		wg.Add(1)
		go func(hn *hNode) {
			hn.Handle(conn, line.Copy())
			wg.Done()
		}(hn)
	}
	wg.Wait()
}

// Handle adds the provided handler to the foreground set for the named event.
// It will return a Remover that allows that handler to be removed again.
func (conn *Conn) Handle(name string, h Handler) Remover {
	return conn.fgHandlers.add(name, h)
}

// HandleBG adds the provided handler to the background set for the named
// event. It may go away in the future.
// It will return a Remover that allows that handler to be removed again.
func (conn *Conn) HandleBG(name string, h Handler) Remover {
	return conn.bgHandlers.add(name, h)
}

func (conn *Conn) handle(name string, h Handler) Remover {
	return conn.intHandlers.add(name, h)
}

// HandleFunc adds the provided function as a handler in the foreground set
// for the named event.
// It will return a Remover that allows that handler to be removed again.
func (conn *Conn) HandleFunc(name string, hf HandlerFunc) Remover {
	return conn.Handle(name, hf)
}

func (conn *Conn) dispatch(line *Line) {
	// We run the internal handlers first, including all state tracking ones.
	// This ensures that user-supplied handlers that use the tracker have a
	// consistent view of the connection state in handlers that mutate it.
	conn.intHandlers.dispatch(conn, line)
	go conn.bgHandlers.dispatch(conn, line)
	conn.fgHandlers.dispatch(conn, line)
}

// LogPanic is used as the default panic catcher for the client. If, like me,
// you are not good with computer, and you'd prefer your bot not to vanish into
// the ether whenever you make unfortunate programming mistakes, you may find
// this useful: it will recover panics from handler code and log the errors.
func (conn *Conn) LogPanic(line *Line) {
	if err := recover(); err != nil {
		_, f, l, _ := runtime.Caller(2)
		logging.Error("%s:%d: panic: %v", f, l, err)
	}
}
<|endoftext|>


<commit_before>package collection

import "fmt"
import "log"
import "github.com/gosuri/uitable"

import "github.com/nlewo/contrail-introspect-cli/utils"

// Several representations of resources
type Shower interface {
	Long()
	Short()
	Xml()
}

func (e Element) Xml() {
	fmt.Printf("%s", e.Node)
}
func (elts Elements) Xml() {
	for _, e := range elts {
		e.Xml()
	}
}
func (c Collection) Xml() {
	fmt.Printf("%s", c.rootNode)
}

func (e Element) Short() {
	s, _ := e.Node.Search(e.desc.ShortDetailXpath)
	if len(s) != 1 {
		log.Fatal("Xpath '" + e.desc.ShortDetailXpath + "' is not valid (verify ShortDetailXpath)")
	}
	fmt.Printf("%s\n", s[0])
}
func (col Collection) Short() {
	Elements(col.elements).Short()
}
func (elts Elements) Short() {
	for _, e := range elts {
		e.Short()
	}
}
func (e Element) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	e.desc.LongDetail.LongFormat(table, FORMAT_TEXT, e)
	fmt.Println(table)
}
func (col Collection) Long() {
	Elements(col.elements).Long()
}
func (elts Elements) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	for i, e := range elts {
		format := FORMAT_TABLE
		if i == 0 {
			format = FORMAT_TABLE_HEADER
		}
		e.desc.LongDetail.LongFormat(table, format, e)
	}
	fmt.Println(table)
}

// This is used to show the long version of an Element.
type LongFormatter interface {
	LongFormat(t *uitable.Table, f Format, e Element)
}

type LongFormatFn (func(*uitable.Table, Element))
type LongFormatXpaths []string

type Format uint8

const (
	FORMAT_TEXT         Format = 1
	FORMAT_TABLE_HEADER Format = 2
	FORMAT_TABLE        Format = 3
)

func (fn LongFormatFn) LongFormat(table *uitable.Table, format Format, e Element) {
	fn(table, e)
}

func (xpaths LongFormatXpaths) LongFormat(table *uitable.Table, format Format, e Element) {
	if format == FORMAT_TABLE_HEADER || format == FORMAT_TABLE {
		longFormatTable(table, format, e, xpaths)
	} else {
		for _, xpath := range xpaths {
			s, _ := e.Node.Search(xpath + "/text()")
			if len(s) == 1 {
				table.AddRow(utils.Pretty(s))
			}
		}
	}
}

func longFormatTable(table *uitable.Table, format Format, e Element, xpaths LongFormatXpaths) {
	if format == FORMAT_TABLE_HEADER {
		tmp := make([]interface{}, len(xpaths))
		for i, v := range xpaths {
			tmp[i] = v
		}
		table.AddRow(tmp...)
	}

	tmp := make([]interface{}, len(xpaths))
	for i, xpath := range xpaths {
		s, _ := e.Node.Search(xpath + "/text()")
		if len(s) == 1 {
			tmp[i] = utils.Pretty(s)
		}
	}
	table.AddRow(tmp...)
}
<commit_msg>Allow empty ShortDetailXpath attribute<commit_after>package collection

import "fmt"
import "log"
import "github.com/gosuri/uitable"

import "github.com/nlewo/contrail-introspect-cli/utils"

// Several representations of resources
type Shower interface {
	Long()
	Short()
	Xml()
}

func (e Element) Xml() {
	fmt.Printf("%s", e.Node)
}
func (elts Elements) Xml() {
	for _, e := range elts {
		e.Xml()
	}
}
func (c Collection) Xml() {
	fmt.Printf("%s", c.rootNode)
}

func (e Element) Short() {
	if e.desc.ShortDetailXpath == "" {
		log.Fatal("ShortDetailXpath is not defined. Use XML output instead (-x option)")
		return
	}
	s, _ := e.Node.Search(e.desc.ShortDetailXpath)
	if len(s) != 1 {
		log.Fatal("Xpath '" + e.desc.ShortDetailXpath + "' is not valid (check your ShortDetailXpath definition)")
	}
	fmt.Printf("%s\n", s[0])
}
func (col Collection) Short() {
	Elements(col.elements).Short()
}
func (elts Elements) Short() {
	for _, e := range elts {
		e.Short()
	}
}
func (e Element) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	e.desc.LongDetail.LongFormat(table, FORMAT_TEXT, e)
	fmt.Println(table)
}
func (col Collection) Long() {
	Elements(col.elements).Long()
}
func (elts Elements) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	for i, e := range elts {
		format := FORMAT_TABLE
		if i == 0 {
			format = FORMAT_TABLE_HEADER
		}
		e.desc.LongDetail.LongFormat(table, format, e)
	}
	fmt.Println(table)
}

// This is used to show the long version of an Element.
type LongFormatter interface {
	LongFormat(t *uitable.Table, f Format, e Element)
}

type LongFormatFn (func(*uitable.Table, Element))
type LongFormatXpaths []string

type Format uint8

const (
	FORMAT_TEXT         Format = 1
	FORMAT_TABLE_HEADER Format = 2
	FORMAT_TABLE        Format = 3
)

func (fn LongFormatFn) LongFormat(table *uitable.Table, format Format, e Element) {
	fn(table, e)
}

func (xpaths LongFormatXpaths) LongFormat(table *uitable.Table, format Format, e Element) {
	if format == FORMAT_TABLE_HEADER || format == FORMAT_TABLE {
		longFormatTable(table, format, e, xpaths)
	} else {
		for _, xpath := range xpaths {
			s, _ := e.Node.Search(xpath + "/text()")
			if len(s) == 1 {
				table.AddRow(utils.Pretty(s))
			}
		}
	}
}

func longFormatTable(table *uitable.Table, format Format, e Element, xpaths LongFormatXpaths) {
	if format == FORMAT_TABLE_HEADER {
		tmp := make([]interface{}, len(xpaths))
		for i, v := range xpaths {
			tmp[i] = v
		}
		table.AddRow(tmp...)
	}

	tmp := make([]interface{}, len(xpaths))
	for i, xpath := range xpaths {
		s, _ := e.Node.Search(xpath + "/text()")
		if len(s) == 1 {
			tmp[i] = utils.Pretty(s)
		}
	}
	table.AddRow(tmp...)
}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"github.com/timeredbull/tsuru/collector"
)

func main() {
	var xpto collector.Collector
	c := time.Tick(1 * time.Minute)
	for now := range c {
		data, _ := xpto.Collect()
		output := xpto.Parse(data)
		xpto.Update(output)
	}
}
<commit_msg>fixed some sintax errors on main<commit_after>package main

import (
	"time"
	"github.com/timeredbull/tsuru/collector"
)

func main() {
	var xpto collector.Collector
	c := time.Tick(1 * time.Minute)
	for _ = range c {
		data, _ := xpto.Collect()
		output := xpto.Parse(data)
		xpto.Update(output)
	}
}
<|endoftext|>


<commit_before>package comborpc

import (
	"io"
	"log"
	"mime"
	"net/http"

	"github.com/flynn/rpcplus"
	"github.com/flynn/rpcplus/jsonrpc"
)

type Server struct {
	s *rpcplus.Server
}

func New(s *rpcplus.Server) *Server {
	return &Server{s}
}

func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if req.Method != "CONNECT" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte("405 must CONNECT\n"))
		return
	}

	var serve func(conn io.ReadWriteCloser)
	accept, _, _ := mime.ParseMediaType(req.Header.Get("Accept"))
	switch accept {
	case "application/vnd.flynn.rpc-hijack+json":
		serve = func(conn io.ReadWriteCloser) {
			codec := jsonrpc.NewServerCodec(conn)
			server.s.ServeCodec(codec)
		}
	default:
		serve = server.s.ServeConn
	}

	conn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Print("rpc hijacking error:", req.RemoteAddr, ": ", err.Error())
		return
	}
	conn.Write([]byte("HTTP/1.0 200 Connected to Go RPC\n\n"))
	serve(conn)
}

func (server *Server) HandleHTTP(path string) {
	http.Handle(path, server)
}

func HandleHTTP() {
	New(rpcplus.DefaultServer).HandleHTTP(rpcplus.DefaultRPCPath)
}

func Register(rcvr interface{}) error {
	return rpcplus.Register(rcvr)
}
<commit_msg>comborpc: Add RegisterName<commit_after>package comborpc

import (
	"io"
	"log"
	"mime"
	"net/http"

	"github.com/flynn/rpcplus"
	"github.com/flynn/rpcplus/jsonrpc"
)

type Server struct {
	s *rpcplus.Server
}

func New(s *rpcplus.Server) *Server {
	return &Server{s}
}

func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if req.Method != "CONNECT" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte("405 must CONNECT\n"))
		return
	}

	var serve func(conn io.ReadWriteCloser)
	accept, _, _ := mime.ParseMediaType(req.Header.Get("Accept"))
	switch accept {
	case "application/vnd.flynn.rpc-hijack+json":
		serve = func(conn io.ReadWriteCloser) {
			codec := jsonrpc.NewServerCodec(conn)
			server.s.ServeCodec(codec)
		}
	default:
		serve = server.s.ServeConn
	}

	conn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Print("rpc hijacking error:", req.RemoteAddr, ": ", err.Error())
		return
	}
	conn.Write([]byte("HTTP/1.0 200 Connected to Go RPC\n\n"))
	serve(conn)
}

func (server *Server) HandleHTTP(path string) {
	http.Handle(path, server)
}

func HandleHTTP() {
	New(rpcplus.DefaultServer).HandleHTTP(rpcplus.DefaultRPCPath)
}

func Register(rcvr interface{}) error {
	return rpcplus.Register(rcvr)
}

func RegisterName(name string, rcvr interface{}) error {
	return rpcplus.RegisterName(name, rcvr)
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/otto/appfile"
	"github.com/hashicorp/otto/appfile/detect"
	"github.com/hashicorp/otto/ui"
)

// CompileCommand is the command that is responsible for "compiling" the
// Appfile into a set of data that is used by the other commands for
// execution.
type CompileCommand struct {
	Meta

	Detectors []*detect.Detector
}

func (c *CompileCommand) Run(args []string) int {
	var flagAppfile string
	fs := c.FlagSet("compile", FlagSetNone)
	fs.Usage = func() { c.Ui.Error(c.Help()) }
	fs.StringVar(&flagAppfile, "appfile", "", "")
	if err := fs.Parse(args); err != nil {
		return 1
	}

	// Load a UI
	ui := c.OttoUi()
	ui.Header("Loading Appfile...")

	// Determine all the Appfile paths
	//
	// First, if an Appfile was specified on the command-line, it must
	// exist so we validate that it exists.
	if flagAppfile != "" {
		fi, err := os.Stat(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}

		if fi.IsDir() {
			flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
		}
	}

	// If the Appfile is still blank, just use our current directory
	if flagAppfile == "" {
		var err error
		flagAppfile, err = os.Getwd()
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading working directory: %s", err))
			return 1
		}

		flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
	}

	// If we have the Appfile, then make sure it is an absoute path
	if flagAppfile != "" {
		var err error
		flagAppfile, err = filepath.Abs(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error getting Appfile path: %s", err))
			return 1
		}
	}

	// Load the appfile. This is the only time we ever load the
	// raw Appfile. All other commands load the compiled Appfile.
	var app *appfile.File
	if fi, err := os.Stat(flagAppfile); err == nil && !fi.IsDir() {
		app, err = appfile.ParseFile(flagAppfile)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
	}

	// Tell the user what is happening if they have no Appfile
	if app == nil {
		ui.Header("No Appfile found! Detecting project information...")
		ui.Message(fmt.Sprintf(
			"No Appfile was found. If there is no Appfile, Otto will do its best\n" +
				"to detect the type of application this is and set reasonable defaults.\n" +
				"This is a good way to get started with Otto, but over time we recommend\n" +
				"writing a real Appfile since this will allow more complex customizations,\n" +
				"the ability to reference dependencies, versioning, and more."))
	}

	// Parse the detectors
	dataDir, err := c.DataDir()
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}
	detectorDir := filepath.Join(dataDir, DefaultLocalDataDetectorDir)
	log.Printf("[DEBUG] loading detectors from: %s", detectorDir)
	detectConfig, err := detect.ParseDir(detectorDir)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}
	if detectConfig == nil {
		detectConfig = &detect.Config{}
	}
	err = detectConfig.Merge(&detect.Config{Detectors: c.Detectors})
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	// Load the default Appfile so we can merge in any defaults into
	// the loaded Appfile (if there is one).
	appDef, err := appfile.Default(filepath.Dir(flagAppfile), detectConfig)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading Appfile: %s", err))
		return 1
	}

	// If there was no loaded Appfile and we don't have an application
	// type then we weren't able to detect the type. Error.
	if app == nil && appDef.Application.Type == "" {
		c.Ui.Error(strings.TrimSpace(errCantDetectType))
		return 1
	}

	// Merge the appfiles
	if app != nil {
		if err := appDef.Merge(app); err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}
	}
	app = appDef

	// Compile the Appfile
	ui.Header("Fetching all Appfile dependencies...")
	capp, err := appfile.Compile(app, &appfile.CompileOpts{
		Dir: filepath.Join(
			filepath.Dir(app.Path), DefaultOutputDir, DefaultOutputDirCompiledAppfile),
		Detect:   detectConfig,
		Callback: c.compileCallback(ui),
	})
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling Appfile: %s", err))
		return 1
	}

	// Get a core
	core, err := c.Core(capp)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading core: %s", err))
		return 1
	}

	// Get the active infrastructure just for UI reasons
	infra := app.ActiveInfrastructure()

	// Before the compilation, output to the user what is going on
	ui.Header("Compiling...")
	ui.Message(fmt.Sprintf(
		"Application:    %s (%s)",
		app.Application.Name,
		app.Application.Type))
	ui.Message(fmt.Sprintf("Project:        %s", app.Project.Name))
	ui.Message(fmt.Sprintf(
		"Infrastructure: %s (%s)",
		infra.Type,
		infra.Flavor))
	ui.Message("")

	// Compile!
	if err := core.Compile(); err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling: %s", err))
		return 1
	}

	// Success!
	ui.Header("[green]Compilation success!")
	ui.Message(fmt.Sprintf(
		"[green]This means that Otto is now ready to start a development environment,\n" +
			"deploy this application, build the supporting infastructure, and\n" +
			"more. See the help for more information.\n\n" +
			"Supporting files to enable Otto to manage your application from\n" +
			"development to deployment have been placed in the output directory.\n" +
			"These files can be manually inspected to determine what Otto will do."))

	return 0
}

func (c *CompileCommand) Synopsis() string {
	return "Prepares your project for being run."
}

func (c *CompileCommand) Help() string {
	helpText := `
Usage: otto [options] [path]

  Compiles the Appfile into the set of supporting files used for
  development, deploy, etc. If path is not specified, the current directory
  is assumed.

  This command will download and update any dependencies as well as
  the import statements in your Appfile. This process only happens during
  compilation so that every other Otto operation begins executing much
  more quickly.

`

	return strings.TrimSpace(helpText)
}

func (c *CompileCommand) compileCallback(ui ui.Ui) func(appfile.CompileEvent) {
	return func(raw appfile.CompileEvent) {
		switch e := raw.(type) {
		case *appfile.CompileEventDep:
			ui.Message(fmt.Sprintf(
				"Fetching dependency: %s", e.Source))
		case *appfile.CompileEventImport:
			ui.Message(fmt.Sprintf(
				"Fetching import: %s", e.Source))
		}
	}
}

const errCantDetectType = `
No Appfile is present and Otto couldn't detect the project type automatically.
Otto does its best without an Appfile to detect what kind of project this is
automatically, but sometimes this fails if the project is in a structure
Otto doesn't recognize or its a project type that Otto doesn't yet support.

Please create an Appfile and specify at a minimum the project type. Below
is an example minimal Appfile specifying the "go" project type:

    application {
        type = "go"
    }

If you believe Otto should've been able to automatically detect your
project type, then please open an issue with the Otto project.
`
<commit_msg>Include application name to "errCantDetectType" error message text<commit_after>package command

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/otto/appfile"
	"github.com/hashicorp/otto/appfile/detect"
	"github.com/hashicorp/otto/ui"
)

// CompileCommand is the command that is responsible for "compiling" the
// Appfile into a set of data that is used by the other commands for
// execution.
type CompileCommand struct {
	Meta

	Detectors []*detect.Detector
}

func (c *CompileCommand) Run(args []string) int {
	var flagAppfile string
	fs := c.FlagSet("compile", FlagSetNone)
	fs.Usage = func() { c.Ui.Error(c.Help()) }
	fs.StringVar(&flagAppfile, "appfile", "", "")
	if err := fs.Parse(args); err != nil {
		return 1
	}

	// Load a UI
	ui := c.OttoUi()
	ui.Header("Loading Appfile...")

	// Determine all the Appfile paths
	//
	// First, if an Appfile was specified on the command-line, it must
	// exist so we validate that it exists.
	if flagAppfile != "" {
		fi, err := os.Stat(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}

		if fi.IsDir() {
			flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
		}
	}

	// If the Appfile is still blank, just use our current directory
	if flagAppfile == "" {
		var err error
		flagAppfile, err = os.Getwd()
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading working directory: %s", err))
			return 1
		}

		flagAppfile = filepath.Join(flagAppfile, DefaultAppfile)
	}

	// If we have the Appfile, then make sure it is an absoute path
	if flagAppfile != "" {
		var err error
		flagAppfile, err = filepath.Abs(flagAppfile)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error getting Appfile path: %s", err))
			return 1
		}
	}

	// Load the appfile. This is the only time we ever load the
	// raw Appfile. All other commands load the compiled Appfile.
	var app *appfile.File
	if fi, err := os.Stat(flagAppfile); err == nil && !fi.IsDir() {
		app, err = appfile.ParseFile(flagAppfile)
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
	}

	// Tell the user what is happening if they have no Appfile
	if app == nil {
		ui.Header("No Appfile found! Detecting project information...")
		ui.Message(fmt.Sprintf(
			"No Appfile was found. If there is no Appfile, Otto will do its best\n" +
				"to detect the type of application this is and set reasonable defaults.\n" +
				"This is a good way to get started with Otto, but over time we recommend\n" +
				"writing a real Appfile since this will allow more complex customizations,\n" +
				"the ability to reference dependencies, versioning, and more."))
	}

	// Parse the detectors
	dataDir, err := c.DataDir()
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}
	detectorDir := filepath.Join(dataDir, DefaultLocalDataDetectorDir)
	log.Printf("[DEBUG] loading detectors from: %s", detectorDir)
	detectConfig, err := detect.ParseDir(detectorDir)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}
	if detectConfig == nil {
		detectConfig = &detect.Config{}
	}
	err = detectConfig.Merge(&detect.Config{Detectors: c.Detectors})
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	// Load the default Appfile so we can merge in any defaults into
	// the loaded Appfile (if there is one).
	appDef, err := appfile.Default(filepath.Dir(flagAppfile), detectConfig)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading Appfile: %s", err))
		return 1
	}

	// If there was no loaded Appfile and we don't have an application
	// type then we weren't able to detect the type. Error.
	if app == nil && appDef.Application.Type == "" {
		c.Ui.Error(strings.TrimSpace(errCantDetectType))
		return 1
	}

	// Merge the appfiles
	if app != nil {
		if err := appDef.Merge(app); err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error loading Appfile: %s", err))
			return 1
		}
	}
	app = appDef

	// Compile the Appfile
	ui.Header("Fetching all Appfile dependencies...")
	capp, err := appfile.Compile(app, &appfile.CompileOpts{
		Dir: filepath.Join(
			filepath.Dir(app.Path), DefaultOutputDir, DefaultOutputDirCompiledAppfile),
		Detect:   detectConfig,
		Callback: c.compileCallback(ui),
	})
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling Appfile: %s", err))
		return 1
	}

	// Get a core
	core, err := c.Core(capp)
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error loading core: %s", err))
		return 1
	}

	// Get the active infrastructure just for UI reasons
	infra := app.ActiveInfrastructure()

	// Before the compilation, output to the user what is going on
	ui.Header("Compiling...")
	ui.Message(fmt.Sprintf(
		"Application:    %s (%s)",
		app.Application.Name,
		app.Application.Type))
	ui.Message(fmt.Sprintf("Project:        %s", app.Project.Name))
	ui.Message(fmt.Sprintf(
		"Infrastructure: %s (%s)",
		infra.Type,
		infra.Flavor))
	ui.Message("")

	// Compile!
	if err := core.Compile(); err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error compiling: %s", err))
		return 1
	}

	// Success!
	ui.Header("[green]Compilation success!")
	ui.Message(fmt.Sprintf(
		"[green]This means that Otto is now ready to start a development environment,\n" +
			"deploy this application, build the supporting infastructure, and\n" +
			"more. See the help for more information.\n\n" +
			"Supporting files to enable Otto to manage your application from\n" +
			"development to deployment have been placed in the output directory.\n" +
			"These files can be manually inspected to determine what Otto will do."))

	return 0
}

func (c *CompileCommand) Synopsis() string {
	return "Prepares your project for being run."
}

func (c *CompileCommand) Help() string {
	helpText := `
Usage: otto [options] [path]

  Compiles the Appfile into the set of supporting files used for
  development, deploy, etc. If path is not specified, the current directory
  is assumed.

  This command will download and update any dependencies as well as
  the import statements in your Appfile. This process only happens during
  compilation so that every other Otto operation begins executing much
  more quickly.

`

	return strings.TrimSpace(helpText)
}

func (c *CompileCommand) compileCallback(ui ui.Ui) func(appfile.CompileEvent) {
	return func(raw appfile.CompileEvent) {
		switch e := raw.(type) {
		case *appfile.CompileEventDep:
			ui.Message(fmt.Sprintf(
				"Fetching dependency: %s", e.Source))
		case *appfile.CompileEventImport:
			ui.Message(fmt.Sprintf(
				"Fetching import: %s", e.Source))
		}
	}
}

const errCantDetectType = `
No Appfile is present and Otto couldn't detect the project type automatically.
Otto does its best without an Appfile to detect what kind of project this is
automatically, but sometimes this fails if the project is in a structure
Otto doesn't recognize or its a project type that Otto doesn't yet support.

Please create an Appfile and specify at a minimum the project name and type. Below
is an example minimal Appfile specifying the "my-app" application name and "go"
project type:

    application {
	name = "my-app"
	type = "go"
    }

If you believe Otto should've been able to automatically detect your
project type, then please open an issue with the Otto project.
`
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/codegangsta/cli"

	"github.com/dockpit/debs"
	"github.com/dockpit/lang"
	"github.com/dockpit/pit/contract"
)

var tmpl_install = `Installation successful!`

type Install struct {
	*cmd
}

func NewInstall(out io.Writer) *Install {
	return &Install{
		cmd: newCmd(out),
	}
}

func (c *Install) Name() string {
	return "install"
}

func (c *Install) Description() string {
	return fmt.Sprintf("...")
}

func (c *Install) Usage() string {
	return "Install a dependency"
}

func (c *Install) Flags() []cli.Flag {

	//get working dir
	wd, err := os.Getwd()
	if err == nil {
		wd = filepath.Join(wd, ".dockpit", "examples")
	} else {
		wd = fmt.Sprintf("[%s]", err.Error())
	}

	return []cli.Flag{
		cli.StringFlag{Name: "path, p", Value: wd, Usage: fmt.Sprintf("Specify where to look for examples.")},
	}
}

func (c *Install) Action() func(ctx *cli.Context) {
	return c.templated(c.Run)
}

func (c *Install) Run(ctx *cli.Context) (*template.Template, interface{}, error) {

	//retrieve path
	path := strings.TrimSpace(ctx.String("path"))

	//get pit path
	pp := os.Getenv("PIT_PATH")
	if pp == "" {
		return nil, nil, fmt.Errorf("Couldn't read 'PIT_PATH' environment variable, is it set?")
	}

	//parse the spec
	p := lang.NewParser(path)
	cd, err := p.Parse()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("Failed to open .dockpit/examples in '%s', is this a Dockpit project?", path)
		}

		return nil, nil, fmt.Errorf("Parsing error: %s", err)
	}

	//create contract from data
	contract, err := contract.NewContract(cd)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to create contract from parsed data: %s", err)
	}

	//retrieve all dependencies
	deps, err := contract.Dependencies()
	if err != nil {
		return nil, nil, err
	}

	//use the manager to install all dependencies into pit path
	m := debs.NewManager(pp)
	for dep, _ := range deps {
		fmt.Fprintf(c.out, "Installing %s...", dep)

		err := m.Install(dep)
		if err != nil {
			fmt.Fprintf(c.out, "ERROR \n")
			return nil, nil, err
		}

		fmt.Fprintf(c.out, "done! \n")
	}

	return template.Must(template.New("install.success").Parse(tmpl_install)), nil, nil
}
<commit_msg>prettier command line output<commit_after>package command

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/codegangsta/cli"

	"github.com/dockpit/debs"
	"github.com/dockpit/lang"
	"github.com/dockpit/pit/contract"
)

var tmpl_install = `Installation successful!`

type Install struct {
	*cmd
}

func NewInstall(out io.Writer) *Install {
	return &Install{
		cmd: newCmd(out),
	}
}

func (c *Install) Name() string {
	return "install"
}

func (c *Install) Description() string {
	return fmt.Sprintf("Parse examples into a contract and extract all dependencies. Install each dependency into the Dockpit workspace which is expected to be specified as the PIT_PATH environment variable")
}

func (c *Install) Usage() string {
	return "Install a dependency"
}

func (c *Install) Flags() []cli.Flag {

	//get working dir
	wd, err := os.Getwd()
	if err == nil {
		wd = filepath.Join(wd, ".dockpit", "examples")
	} else {
		wd = fmt.Sprintf("[%s]", err.Error())
	}

	return []cli.Flag{
		cli.StringFlag{Name: "path, p", Value: wd, Usage: fmt.Sprintf(" Specify where to look for examples.")},
	}
}

func (c *Install) Action() func(ctx *cli.Context) {
	return c.templated(c.Run)
}

func (c *Install) Run(ctx *cli.Context) (*template.Template, interface{}, error) {

	//retrieve path
	path := strings.TrimSpace(ctx.String("path"))

	//get pit path
	pp := os.Getenv("PIT_PATH")
	if pp == "" {
		return nil, nil, fmt.Errorf("Couldn't read 'PIT_PATH' environment variable, is it set?")
	}

	//parse the spec
	p := lang.NewParser(path)
	cd, err := p.Parse()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("Failed to open .dockpit/examples in '%s', is this a Dockpit project?", path)
		}

		return nil, nil, fmt.Errorf("Parsing error: %s", err)
	}

	//create contract from data
	contract, err := contract.NewContract(cd)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to create contract from parsed data: %s", err)
	}

	//retrieve all dependencies
	deps, err := contract.Dependencies()
	if err != nil {
		return nil, nil, err
	}

	//use the manager to install all dependencies into pit path
	m := debs.NewManager(pp)
	for dep, _ := range deps {
		fmt.Fprintf(c.out, "Installing %s:\n", dep)

		err := m.Install(dep)
		if err != nil {
			fmt.Fprintf(c.out, "ERROR \n")
			return nil, nil, err
		}

		fmt.Fprintf(c.out, "done!\n\n")
	}

	return template.Must(template.New("install.success").Parse(tmpl_install)), nil, nil
}
<|endoftext|>


<commit_before>package buf

import (
	"io"
	"time"

	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/signal"
)

type errorHandler func(error) error
type dataHandler func(MultiBuffer)

type copyHandler struct {
	onReadError  []errorHandler
	onData       []dataHandler
	onWriteError []errorHandler
}

func (h *copyHandler) readFrom(reader Reader) (MultiBuffer, error) {
	mb, err := reader.ReadMultiBuffer()
	if err != nil {
		for _, handler := range h.onReadError {
			err = handler(err)
		}
	}
	return mb, err
}

func (h *copyHandler) writeTo(writer Writer, mb MultiBuffer) error {
	err := writer.WriteMultiBuffer(mb)
	if err != nil {
		for _, handler := range h.onWriteError {
			err = handler(err)
		}
	}
	return err
}

// SizeCounter is for counting bytes copied by Copy().
type SizeCounter struct {
	Size int64
}

// CopyOption is an option for copying data.
type CopyOption func(*copyHandler)

// IgnoreReaderError is a CopyOption that ignores errors from reader. Copy will continue in such case.
func IgnoreReaderError() CopyOption {
	return func(handler *copyHandler) {
		handler.onReadError = append(handler.onReadError, func(err error) error {
			return nil
		})
	}
}

// IgnoreWriterError is a CopyOption that ignores errors from writer. Copy will continue in such case.
func IgnoreWriterError() CopyOption {
	return func(handler *copyHandler) {
		handler.onWriteError = append(handler.onWriteError, func(err error) error {
			return nil
		})
	}
}

// UpdateActivity is a CopyOption to update activity on each data copy operation.
func UpdateActivity(timer signal.ActivityUpdater) CopyOption {
	return func(handler *copyHandler) {
		handler.onData = append(handler.onData, func(MultiBuffer) {
			timer.Update()
		})
	}
}

// CountSize is a CopyOption that sums the total size of data copied into the given SizeCounter.
func CountSize(sc *SizeCounter) CopyOption {
	return func(handler *copyHandler) {
		handler.onData = append(handler.onData, func(b MultiBuffer) {
			sc.Size += int64(b.Len())
		})
	}
}

func copyInternal(reader Reader, writer Writer, handler *copyHandler) error {
	for {
		buffer, err := handler.readFrom(reader)
		if !buffer.IsEmpty() {
			for _, handler := range handler.onData {
				handler(buffer)
			}

			if werr := handler.writeTo(writer, buffer); werr != nil {
				buffer.Release()
				return werr
			}
		}

		if err != nil {
			return err
		}
	}
}

// Copy dumps all payload from reader to writer or stops when an error occurs. It returns nil when EOF.
func Copy(reader Reader, writer Writer, options ...CopyOption) error {
	var handler copyHandler
	for _, option := range options {
		option(&handler)
	}
	err := copyInternal(reader, writer, &handler)
	if err != nil && errors.Cause(err) != io.EOF {
		return err
	}
	return nil
}

var ErrNotTimeoutReader = newError("not a TimeoutReader")

func CopyOnceTimeout(reader Reader, writer Writer, timeout time.Duration) error {
	timeoutReader, ok := reader.(TimeoutReader)
	if !ok {
		return ErrNotTimeoutReader
	}
	mb, err := timeoutReader.ReadMultiBufferTimeout(timeout)
	if err != nil {
		return err
	}
	return writer.WriteMultiBuffer(mb)
}
<commit_msg>don't release buffer as it is written<commit_after>package buf

import (
	"io"
	"time"

	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/signal"
)

type errorHandler func(error) error
type dataHandler func(MultiBuffer)

type copyHandler struct {
	onReadError  []errorHandler
	onData       []dataHandler
	onWriteError []errorHandler
}

func (h *copyHandler) readFrom(reader Reader) (MultiBuffer, error) {
	mb, err := reader.ReadMultiBuffer()
	if err != nil {
		for _, handler := range h.onReadError {
			err = handler(err)
		}
	}
	return mb, err
}

func (h *copyHandler) writeTo(writer Writer, mb MultiBuffer) error {
	err := writer.WriteMultiBuffer(mb)
	if err != nil {
		for _, handler := range h.onWriteError {
			err = handler(err)
		}
	}
	return err
}

// SizeCounter is for counting bytes copied by Copy().
type SizeCounter struct {
	Size int64
}

// CopyOption is an option for copying data.
type CopyOption func(*copyHandler)

// IgnoreReaderError is a CopyOption that ignores errors from reader. Copy will continue in such case.
func IgnoreReaderError() CopyOption {
	return func(handler *copyHandler) {
		handler.onReadError = append(handler.onReadError, func(err error) error {
			return nil
		})
	}
}

// IgnoreWriterError is a CopyOption that ignores errors from writer. Copy will continue in such case.
func IgnoreWriterError() CopyOption {
	return func(handler *copyHandler) {
		handler.onWriteError = append(handler.onWriteError, func(err error) error {
			return nil
		})
	}
}

// UpdateActivity is a CopyOption to update activity on each data copy operation.
func UpdateActivity(timer signal.ActivityUpdater) CopyOption {
	return func(handler *copyHandler) {
		handler.onData = append(handler.onData, func(MultiBuffer) {
			timer.Update()
		})
	}
}

// CountSize is a CopyOption that sums the total size of data copied into the given SizeCounter.
func CountSize(sc *SizeCounter) CopyOption {
	return func(handler *copyHandler) {
		handler.onData = append(handler.onData, func(b MultiBuffer) {
			sc.Size += int64(b.Len())
		})
	}
}

func copyInternal(reader Reader, writer Writer, handler *copyHandler) error {
	for {
		buffer, err := handler.readFrom(reader)
		if !buffer.IsEmpty() {
			for _, handler := range handler.onData {
				handler(buffer)
			}

			if werr := handler.writeTo(writer, buffer); werr != nil {
				return werr
			}
		}

		if err != nil {
			return err
		}
	}
}

// Copy dumps all payload from reader to writer or stops when an error occurs. It returns nil when EOF.
func Copy(reader Reader, writer Writer, options ...CopyOption) error {
	var handler copyHandler
	for _, option := range options {
		option(&handler)
	}
	err := copyInternal(reader, writer, &handler)
	if err != nil && errors.Cause(err) != io.EOF {
		return err
	}
	return nil
}

var ErrNotTimeoutReader = newError("not a TimeoutReader")

func CopyOnceTimeout(reader Reader, writer Writer, timeout time.Duration) error {
	timeoutReader, ok := reader.(TimeoutReader)
	if !ok {
		return ErrNotTimeoutReader
	}
	mb, err := timeoutReader.ReadMultiBufferTimeout(timeout)
	if err != nil {
		return err
	}
	return writer.WriteMultiBuffer(mb)
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"regexp"

	valid "gopkg.in/go-playground/validator.v9"
)

var (
	regexBraTagName     = regexp.MustCompile(`^[a-z,A-Z,0-9,\^,\.,\-,\_,\/,\(,\),\:,\ )]+$`)
	regexDBName         = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\(,\),\ ]+$`)
	regexDisplayName    = regexp.MustCompile(`^[a-z,A-Z,\.,\-,\,,\ ]+$`)
	regexFieldName      = regexp.MustCompile(`^[a-z,A-Z,0-9,\^,\.,\-,\_,\/,\(,\),\ )]+$`)
	regexFolder         = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\/]+$`)
	regexLicence        = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\(,\),\ ]+$`)
	regexMarkDownSource = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\/,\(,\),\[,\],\\,\!,\#,\',\",\@,\$,\*,\%,\^,\&,\+,\=,\:,\;,\<,\>,\,,\?,\~,\|,\ ,\012,\015]+$`)
	regexPGTable        = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\ ]+$`)
	regexUsername       = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_]+$`)

	// For input validation
	Validate *valid.Validate
)

func init() {
	// Load validation code
	Validate = valid.New()
	Validate.RegisterValidation("branchortagname", checkBranchOrTagName)
	Validate.RegisterValidation("dbname", checkDBName)
	Validate.RegisterValidation("displayname", checkDisplayName)
	Validate.RegisterValidation("fieldname", checkFieldName)
	Validate.RegisterValidation("folder", checkFolder)
	Validate.RegisterValidation("licence", checkLicence)
	Validate.RegisterValidation("markdownsource", checkMarkDownSource)
	Validate.RegisterValidation("pgtable", checkPGTableName)
	Validate.RegisterValidation("username", checkUsername)
}

// Custom validation function for branch and tag names.
// At the moment it just allows alphanumeric and "^.-_/(): " chars, though it should probably be extended to cover any
// valid file name
func checkBranchOrTagName(fl valid.FieldLevel) bool {
	return regexBraTagName.MatchString(fl.Field().String())
}

// Custom validation function for SQLite database names.
// At the moment it just allows alphanumeric and ".-_() " chars, though it should probably be extended to cover any
// valid file name
func checkDBName(fl valid.FieldLevel) bool {
	return regexDBName.MatchString(fl.Field().String())
}

// Custom validation function for display names.
// At the moment it just allows alpha and ".,- " chars
func checkDisplayName(fl valid.FieldLevel) bool {
	return regexDisplayName.MatchString(fl.Field().String())
}

// Custom validation function for SQLite field names
// At the moment it just allows alphanumeric and "^.-_/() " chars, though it should probably be extended to cover all
// valid SQLite field name characters
func checkFieldName(fl valid.FieldLevel) bool {
	return regexFieldName.MatchString(fl.Field().String())
}

// Custom validation function for folder names.
// At the moment it allows alphanumeric and ".-_/" chars.  Will probably need more characters added.
func checkFolder(fl valid.FieldLevel) bool {
	return regexFolder.MatchString(fl.Field().String())
}

// Custom validation function for licence names.
// At the moment it allows alphanumeric and ".-_() " chars.  Will probably need more characters added.
func checkLicence(fl valid.FieldLevel) bool {
	return regexLicence.MatchString(fl.Field().String())
}

// Custom validation function for Markdown source text.
// At the moment it allows Unicode alphanumeric, ".-_/()[]\#\!'"@$*%^&+=:;<>,?~| ", and "\r\n" chars.  Will probably need more characters added.
func checkMarkDownSource(fl valid.FieldLevel) bool {
	return regexMarkDownSource.MatchString(fl.Field().String())
}

// Custom validation function for PostgreSQL table names.
// At the moment it just allows alphanumeric and ".-_ " chars (may need to be expanded out at some point).
func checkPGTableName(fl valid.FieldLevel) bool {
	return regexPGTable.MatchString(fl.Field().String())
}

// Custom validation function for Usernames.
// At the moment it just allows alphanumeric and ".-_" chars (may need to be expanded out at some point).
func checkUsername(fl valid.FieldLevel) bool {
	return regexUsername.MatchString(fl.Field().String())
}

// Checks a username against the list of reserved ones.
func ReservedUsernamesCheck(userName string) error {
	reserved := []string{"about", "admin", "blog", "dbhub", "download", "downloadcsv", "forks", "legal", "login",
		"logout", "mail", "news", "pref", "printer", "public", "reference", "register", "root", "star",
		"stars", "system", "table", "upload", "uploaddata", "vis"}
	for _, word := range reserved {
		if userName == word {
			return fmt.Errorf("That username is not available: %s\n", userName)
		}
	}

	return nil
}

// Validate the SQLite field name
func ValidateFieldName(fieldName string) error {
	err := Validate.Var(fieldName, "required,fieldname,min=1,max=63") // 63 char limit seems reasonable
	if err != nil {
		return err
	}

	return nil
}

// Validate the database name.
func ValidateDB(dbName string) error {
	err := Validate.Var(dbName, "required,dbname,min=1,max=256") // 256 char limit seems reasonable
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided email address.
func ValidateEmail(email string) error {
	err := Validate.Var(email, "required,email")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided folder name.
func ValidateFolder(folder string) error {
	err := Validate.Var(folder, "folder,max=127")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided licence name.
func ValidateLicence(licence string) error {
	err := Validate.Var(licence, "licence,min=3,max=13") // 13 is the length of our longest licence name (thus far)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided PostgreSQL table name.
func ValidatePGTable(table string) error {
	// TODO: Improve this to work with all valid SQLite identifiers
	// TODO  Not seeing a definitive reference page for SQLite yet, so using the PostgreSQL one is
	// TODO  probably ok as a fallback:
	// TODO      https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
	// TODO: Should we exclude SQLite internal tables too? (eg "sqlite_*" https://sqlite.org/lang_createtable.html)
	err := Validate.Var(table, "required,pgtable,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username.
func ValidateUser(user string) error {
	err := Validate.Var(user, "required,username,min=2,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user and database name.
func ValidateUserDB(user string, db string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateDB(db)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user, database, and table name.
func ValidateUserDBTable(user string, db string, table string) error {
	err := ValidateUserDB(user, db)
	if err != nil {
		return err
	}

	err = ValidatePGTable(table)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username and email address.
func ValidateUserEmail(user string, email string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateEmail(email)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Add left & right single quotes to markdown source char white list<commit_after>package common

import (
	"fmt"
	"regexp"

	valid "gopkg.in/go-playground/validator.v9"
)

var (
	regexBraTagName     = regexp.MustCompile(`^[a-z,A-Z,0-9,\^,\.,\-,\_,\/,\(,\),\:,\ )]+$`)
	regexDBName         = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\(,\),\ ]+$`)
	regexDisplayName    = regexp.MustCompile(`^[a-z,A-Z,\.,\-,\,,\ ]+$`)
	regexFieldName      = regexp.MustCompile(`^[a-z,A-Z,0-9,\^,\.,\-,\_,\/,\(,\),\ )]+$`)
	regexFolder         = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\/]+$`)
	regexLicence        = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\(,\),\ ]+$`)
	regexMarkDownSource = regexp.MustCompile(`^[a-z,A-Z,0-9,‘,’,\.,\-,\_,\/,\(,\),\[,\],\\,\!,\#,\',\",\@,\$,\*,\%,\^,\&,\+,\=,\:,\;,\<,\>,\,,\?,\~,\|,\ ,\012,\015]+$`)
	regexPGTable        = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\ ]+$`)
	regexUsername       = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_]+$`)

	// For input validation
	Validate *valid.Validate
)

func init() {
	// Load validation code
	Validate = valid.New()
	Validate.RegisterValidation("branchortagname", checkBranchOrTagName)
	Validate.RegisterValidation("dbname", checkDBName)
	Validate.RegisterValidation("displayname", checkDisplayName)
	Validate.RegisterValidation("fieldname", checkFieldName)
	Validate.RegisterValidation("folder", checkFolder)
	Validate.RegisterValidation("licence", checkLicence)
	Validate.RegisterValidation("markdownsource", checkMarkDownSource)
	Validate.RegisterValidation("pgtable", checkPGTableName)
	Validate.RegisterValidation("username", checkUsername)
}

// Custom validation function for branch and tag names.
// At the moment it just allows alphanumeric and "^.-_/(): " chars, though it should probably be extended to cover any
// valid file name
func checkBranchOrTagName(fl valid.FieldLevel) bool {
	return regexBraTagName.MatchString(fl.Field().String())
}

// Custom validation function for SQLite database names.
// At the moment it just allows alphanumeric and ".-_() " chars, though it should probably be extended to cover any
// valid file name
func checkDBName(fl valid.FieldLevel) bool {
	return regexDBName.MatchString(fl.Field().String())
}

// Custom validation function for display names.
// At the moment it just allows alpha and ".,- " chars
func checkDisplayName(fl valid.FieldLevel) bool {
	return regexDisplayName.MatchString(fl.Field().String())
}

// Custom validation function for SQLite field names
// At the moment it just allows alphanumeric and "^.-_/() " chars, though it should probably be extended to cover all
// valid SQLite field name characters
func checkFieldName(fl valid.FieldLevel) bool {
	return regexFieldName.MatchString(fl.Field().String())
}

// Custom validation function for folder names.
// At the moment it allows alphanumeric and ".-_/" chars.  Will probably need more characters added.
func checkFolder(fl valid.FieldLevel) bool {
	return regexFolder.MatchString(fl.Field().String())
}

// Custom validation function for licence names.
// At the moment it allows alphanumeric and ".-_() " chars.  Will probably need more characters added.
func checkLicence(fl valid.FieldLevel) bool {
	return regexLicence.MatchString(fl.Field().String())
}

// Custom validation function for Markdown source text.
// At the moment it allows Unicode alphanumeric, "‘’.-_/()[]\#\!'"@$*%^&+=:;<>,?~| ", and "\r\n" chars.  Will probably need more characters added.
func checkMarkDownSource(fl valid.FieldLevel) bool {
	return regexMarkDownSource.MatchString(fl.Field().String())
}

// Custom validation function for PostgreSQL table names.
// At the moment it just allows alphanumeric and ".-_ " chars (may need to be expanded out at some point).
func checkPGTableName(fl valid.FieldLevel) bool {
	return regexPGTable.MatchString(fl.Field().String())
}

// Custom validation function for Usernames.
// At the moment it just allows alphanumeric and ".-_" chars (may need to be expanded out at some point).
func checkUsername(fl valid.FieldLevel) bool {
	return regexUsername.MatchString(fl.Field().String())
}

// Checks a username against the list of reserved ones.
func ReservedUsernamesCheck(userName string) error {
	reserved := []string{"about", "admin", "blog", "dbhub", "download", "downloadcsv", "forks", "legal", "login",
		"logout", "mail", "news", "pref", "printer", "public", "reference", "register", "root", "star",
		"stars", "system", "table", "upload", "uploaddata", "vis"}
	for _, word := range reserved {
		if userName == word {
			return fmt.Errorf("That username is not available: %s\n", userName)
		}
	}

	return nil
}

// Validate the SQLite field name
func ValidateFieldName(fieldName string) error {
	err := Validate.Var(fieldName, "required,fieldname,min=1,max=63") // 63 char limit seems reasonable
	if err != nil {
		return err
	}

	return nil
}

// Validate the database name.
func ValidateDB(dbName string) error {
	err := Validate.Var(dbName, "required,dbname,min=1,max=256") // 256 char limit seems reasonable
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided email address.
func ValidateEmail(email string) error {
	err := Validate.Var(email, "required,email")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided folder name.
func ValidateFolder(folder string) error {
	err := Validate.Var(folder, "folder,max=127")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided licence name.
func ValidateLicence(licence string) error {
	err := Validate.Var(licence, "licence,min=3,max=13") // 13 is the length of our longest licence name (thus far)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided PostgreSQL table name.
func ValidatePGTable(table string) error {
	// TODO: Improve this to work with all valid SQLite identifiers
	// TODO  Not seeing a definitive reference page for SQLite yet, so using the PostgreSQL one is
	// TODO  probably ok as a fallback:
	// TODO      https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
	// TODO: Should we exclude SQLite internal tables too? (eg "sqlite_*" https://sqlite.org/lang_createtable.html)
	err := Validate.Var(table, "required,pgtable,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username.
func ValidateUser(user string) error {
	err := Validate.Var(user, "required,username,min=2,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user and database name.
func ValidateUserDB(user string, db string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateDB(db)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user, database, and table name.
func ValidateUserDBTable(user string, db string, table string) error {
	err := ValidateUserDB(user, db)
	if err != nil {
		return err
	}

	err = ValidatePGTable(table)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username and email address.
func ValidateUserEmail(user string, email string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateEmail(email)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package repository

import (
	"time"

	"github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/model"
)

func (r *repository) UpdateOrCreateReferringTweets(e *model.Example) error {
	if e.ReferringTweets == nil || len((*e).ReferringTweets.Tweets) == 0 || (*e).ReferringTweets.Count == 0 {
		return nil
	}

	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	id := tmp.Id

	for _, t := range (*e).ReferringTweets.Tweets {
		t.ExampleId = id
		if _, err = r.db.NamedExec(`
INSERT INTO tweet
( example_id,  created_at,  id_str,  full_text,  favorite_count,  retweet_count,  lang,  screen_name,  name,  profile_image_url,  label,  score)
VALUES
(:example_id, :created_at, :id_str, :full_text, :favorite_count, :retweet_count, :lang, :screen_name, :name, :profile_image_url, :label, :score)
ON CONFLICT (example_id, id_str)
DO UPDATE SET
favorite_count = :favorite_count,  retweet_count = :retweet_count, label = :label
WHERE
EXCLUDED.label != 0 AND tweet.label != EXCLUDED.label
;`, t); err != nil {
			return err
		}
	}
	return nil
}

func (r *repository) UpdateTweetLabel(exampleId int, idStr string, label model.LabelType) error {
	if _, err := r.db.Exec(`UPDATE tweet SET label = $1 WHERE example_id = $2 AND id_str = $3;`, label, exampleId, idStr); err != nil {
		return err
	}
	return nil
}

type exampleIdWithTweetsCount struct {
	ExampleId   int `db:"example_id"`
	TweetsCount int `db:"tweets_count"`
}

func (r *repository) SearchReferringTweetsList(examples model.Examples, limitForEachExample int) (map[int]model.ReferringTweets, error) {
	referringTweetsByExampleId := make(map[int]model.ReferringTweets)

	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	exampleIdsWithTweetsCount := make([]exampleIdWithTweetsCount, 0)
	tweetsCountByExampleQuery := `SELECT example_id, COUNT(*) AS tweets_count FROM tweet WHERE example_id = ANY($1) GROUP BY example_id ORDER BY tweets_count DESC;`
	err := r.db.Select(&exampleIdsWithTweetsCount, tweetsCountByExampleQuery, pq.Array(exampleIds))
	if err != nil {
		return referringTweetsByExampleId, err
	}
	tweetsCountByExampleId := make(map[int]int)
	for _, e := range exampleIdsWithTweetsCount {
		tweetsCountByExampleId[e.ExampleId] = e.TweetsCount
	}

	if limitForEachExample == 0 {
		for _, exampleId := range exampleIds {
			referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
			if cnt, ok := tweetsCountByExampleId[exampleId]; ok {
				referringTweets.Count = cnt
			}
			referringTweetsByExampleId[exampleId] = referringTweets
		}
		return referringTweetsByExampleId, nil
	}

	tweets := make([]*model.Tweet, 0)
	query := `SELECT * FROM tweet WHERE example_id = ANY($1) AND label != -1 AND score > -1.0 AND (lang = 'en' OR lang = 'ja') ORDER BY favorite_count DESC LIMIT $2;`
	err = r.db.Select(&tweets, query, pq.Array(exampleIds), limitForEachExample)
	if err != nil {
		return referringTweetsByExampleId, err
	}
	tweetsByExampleId := make(map[int][]*model.Tweet)
	for _, t := range tweets {
		tweetsByExampleId[t.ExampleId] = append(tweetsByExampleId[t.ExampleId], t)
	}

	for _, exampleId := range exampleIds {
		referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
		if tweets, ok := tweetsByExampleId[exampleId]; ok {
			referringTweets.Tweets = tweets
		}
		if cnt, ok := tweetsCountByExampleId[exampleId]; ok {
			referringTweets.Count = cnt
		}
		referringTweetsByExampleId[exampleId] = referringTweets
	}
	return referringTweetsByExampleId, nil
}

func (r *repository) SearchReferringTweets(limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
	query := `SELECT * FROM tweet WHERE lang = 'en' OR lang = 'ja' ORDER BY created_at DESC LIMIT $1;`
	err := r.db.Select(&referringTweets.Tweets, query, limit)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = len(referringTweets.Tweets)
	return referringTweets, nil
}

func (r *repository) SearchRecentReferringTweetsWithHighScore(from time.Time, scoreThreshold float64, limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
	query := `
SELECT
	*
FROM
	tweet
INNER JOIN
	example ON example.id = example_id
WHERE
	tweet.created_at > $1 AND
	tweet.label != -1 AND
	example.label != -1 AND
	tweet.score > $2 AND
	(favorite_count > 0 OR retweet_count > 0) AND
	(lang = 'en' OR lang = 'ja')
ORDER BY tweet.score DESC
LIMIT $3
;
`
	err := r.db.Select(&referringTweets.Tweets, query, from, scoreThreshold, limit)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = len(referringTweets.Tweets)
	return referringTweets, nil
}

func (r *repository) searchReferringTweetsByLabel(label model.LabelType, limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
	query := `
SELECT * FROM tweet WHERE id IN
  (SELECT id FROM
    (SELECT tweet.id, ROW_NUMBER() OVER(partition BY example_id ORDER BY favorite_count DESC) AS rank
    FROM tweet
    INNER JOIN example ON tweet.example_id = example.id
    WHERE tweet.label = $1 AND (lang = 'en' OR lang = 'ja') AND (example.label = 1 OR example.label = 0)
  ) AS t WHERE rank < 4)
ORDER BY created_at DESC LIMIT $2
;`
	err := r.db.Select(&referringTweets.Tweets, query, label, limit)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = len(referringTweets.Tweets)
	return referringTweets, nil
}

func (r *repository) SearchPositiveReferringTweets(limit int) (model.ReferringTweets, error) {
	return r.searchReferringTweetsByLabel(model.POSITIVE, limit)
}

func (r *repository) SearchNegativeReferringTweets(limit int) (model.ReferringTweets, error) {
	return r.searchReferringTweetsByLabel(model.NEGATIVE, limit)
}

func (r *repository) SearchUnlabeledReferringTweets(limit int) (model.ReferringTweets, error) {
	return r.searchReferringTweetsByLabel(model.UNLABELED, limit)
}

type tweetsCount struct {
	Count int `db:"count"`
}

func (r *repository) FindReferringTweets(e *model.Example, limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}

	countQuery := `SELECT COUNT(*) AS count FROM tweet WHERE example_id = $1;`
	cnt := tweetsCount{}
	err := r.db.Get(&cnt, countQuery, e.Id)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = cnt.Count
	if limit == 0 {
		return referringTweets, err
	}

	query := `SELECT * FROM tweet WHERE example_id = $1 AND label != -1 AND score > 0.0 AND (lang = 'en' OR lang = 'ja') ORDER BY favorite_count DESC LIMIT $2;`
	err = r.db.Select(&referringTweets.Tweets, query, e.Id, limit)
	if err != nil {
		return referringTweets, err
	}
	return referringTweets, nil
}
<commit_msg>joinした時にexampleを無駄に引いているのでフィールドを絞る<commit_after>package repository

import (
	"time"

	"github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/model"
)

func (r *repository) UpdateOrCreateReferringTweets(e *model.Example) error {
	if e.ReferringTweets == nil || len((*e).ReferringTweets.Tweets) == 0 || (*e).ReferringTweets.Count == 0 {
		return nil
	}

	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	id := tmp.Id

	for _, t := range (*e).ReferringTweets.Tweets {
		t.ExampleId = id
		if _, err = r.db.NamedExec(`
INSERT INTO tweet
( example_id,  created_at,  id_str,  full_text,  favorite_count,  retweet_count,  lang,  screen_name,  name,  profile_image_url,  label,  score)
VALUES
(:example_id, :created_at, :id_str, :full_text, :favorite_count, :retweet_count, :lang, :screen_name, :name, :profile_image_url, :label, :score)
ON CONFLICT (example_id, id_str)
DO UPDATE SET
favorite_count = :favorite_count,  retweet_count = :retweet_count, label = :label
WHERE
EXCLUDED.label != 0 AND tweet.label != EXCLUDED.label
;`, t); err != nil {
			return err
		}
	}
	return nil
}

func (r *repository) UpdateTweetLabel(exampleId int, idStr string, label model.LabelType) error {
	if _, err := r.db.Exec(`UPDATE tweet SET label = $1 WHERE example_id = $2 AND id_str = $3;`, label, exampleId, idStr); err != nil {
		return err
	}
	return nil
}

type exampleIdWithTweetsCount struct {
	ExampleId   int `db:"example_id"`
	TweetsCount int `db:"tweets_count"`
}

func (r *repository) SearchReferringTweetsList(examples model.Examples, limitForEachExample int) (map[int]model.ReferringTweets, error) {
	referringTweetsByExampleId := make(map[int]model.ReferringTweets)

	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	exampleIdsWithTweetsCount := make([]exampleIdWithTweetsCount, 0)
	tweetsCountByExampleQuery := `SELECT example_id, COUNT(*) AS tweets_count FROM tweet WHERE example_id = ANY($1) GROUP BY example_id ORDER BY tweets_count DESC;`
	err := r.db.Select(&exampleIdsWithTweetsCount, tweetsCountByExampleQuery, pq.Array(exampleIds))
	if err != nil {
		return referringTweetsByExampleId, err
	}
	tweetsCountByExampleId := make(map[int]int)
	for _, e := range exampleIdsWithTweetsCount {
		tweetsCountByExampleId[e.ExampleId] = e.TweetsCount
	}

	if limitForEachExample == 0 {
		for _, exampleId := range exampleIds {
			referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
			if cnt, ok := tweetsCountByExampleId[exampleId]; ok {
				referringTweets.Count = cnt
			}
			referringTweetsByExampleId[exampleId] = referringTweets
		}
		return referringTweetsByExampleId, nil
	}

	tweets := make([]*model.Tweet, 0)
	query := `SELECT * FROM tweet WHERE example_id = ANY($1) AND label != -1 AND score > -1.0 AND (lang = 'en' OR lang = 'ja') ORDER BY favorite_count DESC LIMIT $2;`
	err = r.db.Select(&tweets, query, pq.Array(exampleIds), limitForEachExample)
	if err != nil {
		return referringTweetsByExampleId, err
	}
	tweetsByExampleId := make(map[int][]*model.Tweet)
	for _, t := range tweets {
		tweetsByExampleId[t.ExampleId] = append(tweetsByExampleId[t.ExampleId], t)
	}

	for _, exampleId := range exampleIds {
		referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
		if tweets, ok := tweetsByExampleId[exampleId]; ok {
			referringTweets.Tweets = tweets
		}
		if cnt, ok := tweetsCountByExampleId[exampleId]; ok {
			referringTweets.Count = cnt
		}
		referringTweetsByExampleId[exampleId] = referringTweets
	}
	return referringTweetsByExampleId, nil
}

func (r *repository) SearchReferringTweets(limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
	query := `SELECT * FROM tweet WHERE lang = 'en' OR lang = 'ja' ORDER BY created_at DESC LIMIT $1;`
	err := r.db.Select(&referringTweets.Tweets, query, limit)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = len(referringTweets.Tweets)
	return referringTweets, nil
}

func (r *repository) SearchRecentReferringTweetsWithHighScore(from time.Time, scoreThreshold float64, limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
	query := `
SELECT
	tweet.id,
	tweet.example_id,

	tweet.created_at,
	tweet.id_str,
	tweet.full_text,
	tweet.favorite_count,
	tweet.retweet_count,
	tweet.lang,

	tweet.screen_name,
	tweet.name,
	tweet.profile_image_url,
	tweet.label,
	tweet.score
FROM
	tweet
INNER JOIN
	example ON example.id = example_id
WHERE
	tweet.created_at > $1 AND
	tweet.label != -1 AND
	example.label != -1 AND
	tweet.score > $2 AND
	(favorite_count > 0 OR retweet_count > 0) AND
	(lang = 'en' OR lang = 'ja')
ORDER BY tweet.score DESC
LIMIT $3
;
`
	err := r.db.Select(&referringTweets.Tweets, query, from, scoreThreshold, limit)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = len(referringTweets.Tweets)
	return referringTweets, nil
}

func (r *repository) searchReferringTweetsByLabel(label model.LabelType, limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}
	query := `
SELECT * FROM tweet WHERE id IN
  (SELECT id FROM
    (SELECT tweet.id, ROW_NUMBER() OVER(partition BY example_id ORDER BY favorite_count DESC) AS rank
    FROM tweet
    INNER JOIN example ON tweet.example_id = example.id
    WHERE tweet.label = $1 AND (lang = 'en' OR lang = 'ja') AND (example.label = 1 OR example.label = 0)
  ) AS t WHERE rank < 4)
ORDER BY created_at DESC LIMIT $2
;`
	err := r.db.Select(&referringTweets.Tweets, query, label, limit)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = len(referringTweets.Tweets)
	return referringTweets, nil
}

func (r *repository) SearchPositiveReferringTweets(limit int) (model.ReferringTweets, error) {
	return r.searchReferringTweetsByLabel(model.POSITIVE, limit)
}

func (r *repository) SearchNegativeReferringTweets(limit int) (model.ReferringTweets, error) {
	return r.searchReferringTweetsByLabel(model.NEGATIVE, limit)
}

func (r *repository) SearchUnlabeledReferringTweets(limit int) (model.ReferringTweets, error) {
	return r.searchReferringTweetsByLabel(model.UNLABELED, limit)
}

type tweetsCount struct {
	Count int `db:"count"`
}

func (r *repository) FindReferringTweets(e *model.Example, limit int) (model.ReferringTweets, error) {
	referringTweets := model.ReferringTweets{Count: 0, Tweets: make([]*model.Tweet, 0)}

	countQuery := `SELECT COUNT(*) AS count FROM tweet WHERE example_id = $1;`
	cnt := tweetsCount{}
	err := r.db.Get(&cnt, countQuery, e.Id)
	if err != nil {
		return referringTweets, err
	}
	referringTweets.Count = cnt.Count
	if limit == 0 {
		return referringTweets, err
	}

	query := `SELECT * FROM tweet WHERE example_id = $1 AND label != -1 AND score > 0.0 AND (lang = 'en' OR lang = 'ja') ORDER BY favorite_count DESC LIMIT $2;`
	err = r.db.Select(&referringTweets.Tweets, query, e.Id, limit)
	if err != nil {
		return referringTweets, err
	}
	return referringTweets, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io"
)

type ObjectType int

func (t ObjectType) String() string {
	switch t {
	case COMMIT:
		return "commit"
	case TREE:
		return "tree"
	case BLOB:
		return "blob"
	case TAG:
		return "tag"
	case DELTA1:
		return "delta1"
	case DELTA2:
		return "delta2"
	}
	return "Unknown type"
}

const (
	_ = iota
	COMMIT
	TREE
	BLOB
	TAG
	_
	DELTA1
	DELTA2
)

func ReadPackedDataAtOffset(offset int64, in io.ReadSeeker) (ObjectType, int, []byte, error) {
	_, err := in.Seek(offset, 0)
	if err != nil {
		return 0, 0, nil, err
	}
	headByte := make([]byte, 1, 1)
	_, err = in.Read(headByte)
	if err != nil {
		return 0, 0, nil, err
	}

	objectType := (int(headByte[0]) & 0x70) >> 4

	size := (int(headByte[0])) & int(0x0f)

	var shiftBit uint = 4

	for {
		sizeByte := make([]byte, 1, 1)
		_, err = in.Read(sizeByte)
		if err != nil {
			return 0, 0, nil, err
		}
		sizeByteInInt := int(sizeByte[0])
		size = size + (sizeByteInInt << shiftBit)
		shiftBit += 8
		cont := (sizeByteInInt & 0x80) >> 7
		if cont == 0 {
			break
		}
	}

	buff := make([]byte, size, size)
	_, err = in.Read(buff)
	if err != nil {
		return 0, 0, nil, err
	}

	return ObjectType(objectType), size, buff, err
}

<commit_msg>Changed shiftbits from 8 to 7 for size computation<commit_after>package main

import (
	"io"
)

type ObjectType int

func (t ObjectType) String() string {
	switch t {
	case COMMIT:
		return "commit"
	case TREE:
		return "tree"
	case BLOB:
		return "blob"
	case TAG:
		return "tag"
	case DELTA1:
		return "delta1"
	case DELTA2:
		return "delta2"
	}
	return "Unknown type"
}

const (
	_ = iota
	COMMIT
	TREE
	BLOB
	TAG
	_
	DELTA1
	DELTA2
)

func ReadPackedDataAtOffset(offset int64, in io.ReadSeeker) (ObjectType, int, []byte, error) {
	_, err := in.Seek(offset, 0)
	if err != nil {
		return 0, 0, nil, err
	}
	headByte := make([]byte, 1, 1)
	_, err = in.Read(headByte)
	if err != nil {
		return 0, 0, nil, err
	}

	objectType := (int(headByte[0]) & 0x70) >> 4
	size := (int(headByte[0])) & int(0x0f)
	var shiftBit uint = 4
	for {
		sizeByte := make([]byte, 1, 1)
		_, err = in.Read(sizeByte)
		if err != nil {
			return 0, 0, nil, err
		}
		sizeByteInInt := int(sizeByte[0])
		size = size + ((sizeByteInInt & 0x7f) << shiftBit)
		shiftBit += 7
		cont := (sizeByteInInt & 0x80) >> 7
		if cont == 0 {
			break
		}
	}

	buff := make([]byte, size, size)
	_, err = in.Read(buff)
	if err != nil {
		return 0, 0, nil, err
	}

	return ObjectType(objectType), size, buff, err
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/parser"
	"go/token"
	"os"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		die("xgen FILE1.go FILE2.go ...")
	}

	fset := token.NewFileSet()
	mode := parser.ImportsOnly | parser.ParseComments
	for _, f := range os.Args[1:] {
		if !strings.HasSuffix(f, ".go") {
			fmt.Fprintln(os.Stderr, "ignoring:", f)
			continue
		}
		a, err := parser.ParseFile(fset, f, nil, mode)
		checkErr(err)
		pkg := a.Name.Name
		for _, c := range a.Comments {
			txt := c.Text()
			switch {
			case strings.HasPrefix(txt, "BaseAddr:"):
				mmio(pkg, f, txt)
			}
		}
	}
}
<commit_msg>xgen: Registers description must be before package keyword<commit_after>package main

import (
	"fmt"
	"go/parser"
	"go/token"
	"os"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		die("xgen FILE1.go FILE2.go ...")
	}

	fset := token.NewFileSet()
	mode := parser.PackageClauseOnly | parser.ParseComments
	for _, f := range os.Args[1:] {
		if !strings.HasSuffix(f, ".go") {
			fmt.Fprintln(os.Stderr, "ignoring:", f)
			continue
		}
		a, err := parser.ParseFile(fset, f, nil, mode)
		checkErr(err)
		pkg := a.Name.Name
		for _, cg := range a.Comments {
			for len(cg.List) > 0 {
				c := strings.TrimLeft(cg.List[0].Text, "/* \t")
				switch {
				case strings.HasPrefix(c, "BaseAddr:"):
					mmio(pkg, f, cg.Text())
				default:
					cg.List = cg.List[1:]
					continue
				}
				break
			}
		}
	}
}
<|endoftext|>