<commit_before>package instructions

import (
    . "jvmgo/any"
    "jvmgo/rtda"
)

// Store into reference array
type aastore struct {NoOperandsInstruction}
func (self *aastore) execute(thread *rtda.Thread) {
    // todo
}

// Store into byte or boolean array
type bastore struct {NoOperandsInstruction}
func (self *bastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    byteArr := arrRef.Fields().([]int8)
    checkArrIndex(index, len(byteArr))
    byteArr[index] = val.(int8)
}

// Store into char array
type castore struct {NoOperandsInstruction}
func (self *castore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    charArr := arrRef.Fields().([]uint16)
    checkArrIndex(index, len(charArr))
    charArr[index] = val.(uint16)
}

// Store into double array
type dastore struct {NoOperandsInstruction}
func (self *dastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    doubleArr := arrRef.Fields().([]float64)
    checkArrIndex(index, len(doubleArr))
    doubleArr[index] = val.(float64)
}

// Store into float array
type fastore struct {NoOperandsInstruction}
func (self *fastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    floatArr := arrRef.Fields().([]float32)
    checkArrIndex(index, len(floatArr))
    floatArr[index] = val.(float32)
}

// Store into int array
type iastore struct {NoOperandsInstruction}
func (self *iastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    intArr := arrRef.Fields().([]int32)
    checkArrIndex(index, len(intArr))
    intArr[index] = val.(int32)
}

// Store into long array
type lastore struct {NoOperandsInstruction}
func (self *lastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    longArr := arrRef.Fields().([]int64)
    checkArrIndex(index, len(longArr))
    longArr[index] = val.(int64)
}

// Store into short array
type sastore struct {NoOperandsInstruction}
func (self *sastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    shortArr := arrRef.Fields().([]int16)
    checkArrIndex(index, len(shortArr))
    shortArr[index] = val.(int16)
}

func popOperands(thread *rtda.Thread) (*rtda.Obj, int, Any) {
    stack := thread.CurrentFrame().OperandStack()
    arrRef := stack.PopRef()
    index := int(stack.PopInt())
    val := stack.Pop()
    if arrRef == nil {
        // todo
        panic("NullPointerException")
    }
    return arrRef, index, val
}
<commit_msg>aastore<commit_after>package instructions

import (
    . "jvmgo/any"
    "jvmgo/rtda"
)

// Store into reference array
type aastore struct {NoOperandsInstruction}
func (self *aastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    refArr := arrRef.Fields().([]*rtda.Obj)
    checkArrIndex(index, len(refArr))
    // todo
    ref := val.(*rtda.Obj)
    refArr[index] = ref
}

// Store into byte or boolean array
type bastore struct {NoOperandsInstruction}
func (self *bastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    byteArr := arrRef.Fields().([]int8)
    checkArrIndex(index, len(byteArr))
    byteArr[index] = val.(int8)
}

// Store into char array
type castore struct {NoOperandsInstruction}
func (self *castore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    charArr := arrRef.Fields().([]uint16)
    checkArrIndex(index, len(charArr))
    charArr[index] = val.(uint16)
}

// Store into double array
type dastore struct {NoOperandsInstruction}
func (self *dastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    doubleArr := arrRef.Fields().([]float64)
    checkArrIndex(index, len(doubleArr))
    doubleArr[index] = val.(float64)
}

// Store into float array
type fastore struct {NoOperandsInstruction}
func (self *fastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    floatArr := arrRef.Fields().([]float32)
    checkArrIndex(index, len(floatArr))
    floatArr[index] = val.(float32)
}

// Store into int array
type iastore struct {NoOperandsInstruction}
func (self *iastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    intArr := arrRef.Fields().([]int32)
    checkArrIndex(index, len(intArr))
    intArr[index] = val.(int32)
}

// Store into long array
type lastore struct {NoOperandsInstruction}
func (self *lastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    longArr := arrRef.Fields().([]int64)
    checkArrIndex(index, len(longArr))
    longArr[index] = val.(int64)
}

// Store into short array
type sastore struct {NoOperandsInstruction}
func (self *sastore) execute(thread *rtda.Thread) {
    arrRef, index, val := popOperands(thread)
    shortArr := arrRef.Fields().([]int16)
    checkArrIndex(index, len(shortArr))
    shortArr[index] = val.(int16)
}

func popOperands(thread *rtda.Thread) (*rtda.Obj, int, Any) {
    stack := thread.CurrentFrame().OperandStack()
    arrRef := stack.PopRef()
    index := int(stack.PopInt())
    val := stack.Pop()
    if arrRef == nil {
        // todo
        panic("NullPointerException")
    }
    return arrRef, index, val
}
<|endoftext|>


<commit_before>package models

type Person struct {
    Id        int    `name:"id" type:"int" null:"NOT NULL" extra:"PRIMARY"`
    FirstName string `name:"fname" type:"text" null:"NOT NULL" extra:""`
    LastName  string `name:"lname" type:"text" null:"NOT NULL" extra:""`
}

func (c *ModelManager) Persons() *PersonsModel {
    model := new(PersonsModel)

    model.TableName = "persons"
    model.Caption = "Персоны"

    model.Columns = []string{"id", "fname", "lname"}
    model.ColNames = []string{"ID", "Имя", "Фамилия"}

    model.Fields = new(Person)
    model.WherePart = make(map[string]interface{}, 0)
    model.OrderBy = "id"
    model.Limit = "ALL"
    model.Offset = 0

    model.Sub = true
    model.SubTable = []string{"teams_persons"}
    model.SubField = "person_id"

    return model
}

type PersonsModel struct {
    Entity
}
<commit_msg>person's model: fix fields<commit_after>package models

type Person struct {
    Id int `name:"id" type:"int" null:"NOT NULL" extra:"PRIMARY"`
}

func (c *ModelManager) Persons() *PersonsModel {
    model := new(PersonsModel)

    model.TableName = "persons"
    model.Caption = "Персоны"

    model.Columns = []string{"id"}
    model.ColNames = []string{"ID"}

    model.Fields = new(Person)
    model.WherePart = make(map[string]interface{}, 0)
    model.OrderBy = "id"
    model.Limit = "ALL"
    model.Offset = 0

    model.Sub = false
    model.SubTable = nil
    model.SubField = ""

    return model
}

type PersonsModel struct {
    Entity
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build darwin || dragonfly || freebsd || netbsd || openbsd
// +build darwin dragonfly freebsd netbsd openbsd

package net

import (
	"os"
	"runtime"
	"syscall"
)

func setDefaultSockopts(s, family, sotype int, ipv6only bool) error {
	if runtime.GOOS == "dragonfly" && sotype != syscall.SOCK_RAW {
		// On DragonFly BSD, we adjust the ephemeral port
		// range because unlike other BSD systems its default
		// port range doesn't conform to IANA recommendation
		// as described in RFC 6056 and is pretty narrow.
		switch family {
		case syscall.AF_INET:
			syscall.SetsockoptInt(s, syscall.IPPROTO_IP, syscall.IP_PORTRANGE, syscall.IP_PORTRANGE_HIGH)
		case syscall.AF_INET6:
			syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_PORTRANGE, syscall.IPV6_PORTRANGE_HIGH)
		}
	}
	if supportsIPv4map() && family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW {
		// Allow both IP versions even if the OS default
		// is otherwise. Note that some operating systems
		// never admit this option.
		syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only))
	}
	if (sotype == syscall.SOCK_DGRAM || sotype == syscall.SOCK_RAW) && family != syscall.AF_UNIX {
		// Allow broadcast.
		return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1))
	}
	return nil
}

func setDefaultListenerSockopts(s int) error {
	// Allow reuse of recently-used addresses.
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1))
}

func setDefaultMulticastSockopts(s int) error {
	// Allow multicast UDP and raw IP datagram sockets to listen
	// concurrently across multiple listeners.
	if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
		return os.NewSyscallError("setsockopt", err)
	}
	// Allow reuse of recently-used ports.
	// This option is supported only in descendants of 4.4BSD,
	// to make an effective multicast application that requires
	// quick draw possible.
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1))
}
<commit_msg>net: only perform IPv4 map check for AF_INET6 sockets<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build darwin || dragonfly || freebsd || netbsd || openbsd
// +build darwin dragonfly freebsd netbsd openbsd

package net

import (
	"os"
	"runtime"
	"syscall"
)

func setDefaultSockopts(s, family, sotype int, ipv6only bool) error {
	if runtime.GOOS == "dragonfly" && sotype != syscall.SOCK_RAW {
		// On DragonFly BSD, we adjust the ephemeral port
		// range because unlike other BSD systems its default
		// port range doesn't conform to IANA recommendation
		// as described in RFC 6056 and is pretty narrow.
		switch family {
		case syscall.AF_INET:
			syscall.SetsockoptInt(s, syscall.IPPROTO_IP, syscall.IP_PORTRANGE, syscall.IP_PORTRANGE_HIGH)
		case syscall.AF_INET6:
			syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_PORTRANGE, syscall.IPV6_PORTRANGE_HIGH)
		}
	}
	if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW && supportsIPv4map() {
		// Allow both IP versions even if the OS default
		// is otherwise. Note that some operating systems
		// never admit this option.
		syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only))
	}
	if (sotype == syscall.SOCK_DGRAM || sotype == syscall.SOCK_RAW) && family != syscall.AF_UNIX {
		// Allow broadcast.
		return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1))
	}
	return nil
}

func setDefaultListenerSockopts(s int) error {
	// Allow reuse of recently-used addresses.
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1))
}

func setDefaultMulticastSockopts(s int) error {
	// Allow multicast UDP and raw IP datagram sockets to listen
	// concurrently across multiple listeners.
	if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
		return os.NewSyscallError("setsockopt", err)
	}
	// Allow reuse of recently-used ports.
	// This option is supported only in descendants of 4.4BSD,
	// to make an effective multicast application that requires
	// quick draw possible.
	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1))
}
<|endoftext|>


<commit_before>package webhooks

import (
	log "github.com/Sirupsen/logrus"
	"github.com/majestrate/srndv2/lib/config"
	"github.com/majestrate/srndv2/lib/nntp"
	"github.com/majestrate/srndv2/lib/nntp/message"
	"github.com/majestrate/srndv2/lib/store"

	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"regexp"
	"strings"
)

// web hook implementation
type httpWebhook struct {
	conf    *config.WebhookConfig
	storage store.Storage
	hdr     *message.HeaderIO
}

func (h *httpWebhook) SentArticleVia(msgid nntp.MessageID, name string) {
	// web hooks don't care about feed state
}

// we got a new article
func (h *httpWebhook) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) {
	f, err := h.storage.OpenArticle(msgid.String())
	if err == nil {
		c := textproto.NewConn(f)
		var hdr textproto.MIMEHeader
		hdr, err = c.ReadMIMEHeader()
		if err == nil {
			ctype := hdr.Get("Content-Type")
			if ctype == "" {
				ctype = "text/plain"
			}
			ctype = strings.Replace(strings.ToLower(ctype), "multipart/mixed", "multipart/form-data", 1)
			u, _ := url.Parse(h.conf.URL)
			q := u.Query()
			for k, vs := range hdr {
				for _, v := range vs {
					q.Add(k, v)
				}
			}
			q.Set("Content-Type", ctype)
			u.RawQuery = q.Encode()

			var body io.Reader

			if strings.HasPrefix(ctype, "multipart") {
				pr, pw := io.Pipe()
				log.Debug("using pipe")
				body = pr
				go func(in io.Reader, out io.WriteCloser) {
					_, params, _ := mime.ParseMediaType(ctype)
					if params == nil {
						// send as whatever lol
						io.Copy(out, in)
					} else {
						boundary, _ := params["boundary"]
						mpr := multipart.NewReader(in, boundary)
						mpw := multipart.NewWriter(out)
						mpw.SetBoundary(boundary)
						for {
							part, err := mpr.NextPart()
							if err == io.EOF {
								err = nil
								break
							} else if err == nil {
								// get part header
								h := part.Header
								// rewrite header part for php
								cd := h.Get("Content-Disposition")
								r := regexp.MustCompile(`; name=".*"`)
								cd = r.ReplaceAllString(cd, `; name="attachment[]"`)
								log.Debug(cd)
								h.Set("Content-Disposition", cd)
								// make write part
								wp, err := mpw.CreatePart(h)
								if err == nil {
									// write part out
									io.Copy(wp, part)
								} else {
									log.Errorf("error writng webhook part: %s", err.Error())
								}
							}
							part.Close()
						}
						mpw.Close()
					}
					out.Close()
				}(c.R, pw)
			} else {
				body = c.R
			}

			var r *http.Response
			r, err = http.Post(u.String(), ctype, body)
			if err == nil {
				_, err = io.Copy(ioutil.Discard, r.Body)
				r.Body.Close()
				log.Infof("hook called for %s", msgid)
			}
		} else {
			f.Close()
		}
	}
	if err != nil {
		log.Errorf("error calling web hook %s: %s", h.conf.Name, err.Error())
	}
}
<commit_msg>try fixing webhook<commit_after>package webhooks

import (
	log "github.com/Sirupsen/logrus"
	"github.com/majestrate/srndv2/lib/config"
	"github.com/majestrate/srndv2/lib/nntp"
	"github.com/majestrate/srndv2/lib/nntp/message"
	"github.com/majestrate/srndv2/lib/store"

	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"regexp"
	"strings"
)

// web hook implementation
type httpWebhook struct {
	conf    *config.WebhookConfig
	storage store.Storage
	hdr     *message.HeaderIO
}

func (h *httpWebhook) SentArticleVia(msgid nntp.MessageID, name string) {
	// web hooks don't care about feed state
}

// we got a new article
func (h *httpWebhook) GotArticle(msgid nntp.MessageID, group nntp.Newsgroup) {
	f, err := h.storage.OpenArticle(msgid.String())
	if err == nil {
		c := textproto.NewConn(f)
		var hdr textproto.MIMEHeader
		hdr, err = c.ReadMIMEHeader()
		if err == nil {
			ctype := hdr.Get("Content-Type")
			if ctype == "" {
				ctype = "text/plain"
			}
			ctype = strings.Replace(strings.ToLower(ctype), "multipart/mixed", "multipart/form-data", 1)
			u, _ := url.Parse(h.conf.URL)
			q := u.Query()
			for k, vs := range hdr {
				for _, v := range vs {
					q.Add(k, v)
				}
			}
			q.Set("Content-Type", ctype)
			u.RawQuery = q.Encode()

			var body io.Reader

			if strings.HasPrefix(ctype, "multipart") {
				pr, pw := io.Pipe()
				log.Debug("using pipe")
				body = pr
				go func(in io.Reader, out io.WriteCloser) {
					_, params, _ := mime.ParseMediaType(ctype)
					if params == nil {
						// send as whatever lol
						io.Copy(out, in)
					} else {
						boundary, _ := params["boundary"]
						mpr := multipart.NewReader(in, boundary)
						mpw := multipart.NewWriter(out)
						mpw.SetBoundary(boundary)
						for {
							part, err := mpr.NextPart()
							if err == io.EOF {
								err = nil
								break
							} else if err == nil {
								// get part header
								h := part.Header
								// rewrite header part for php
								cd := h.Get("Content-Disposition")
								r := regexp.MustCompile(`; name=".*"`)
								cd = r.ReplaceAllString(cd, `; name="attachment[]; "`)
								log.Debug(cd)
								h.Set("Content-Disposition", cd)
								// make write part
								wp, err := mpw.CreatePart(h)
								if err == nil {
									// write part out
									io.Copy(wp, part)
								} else {
									log.Errorf("error writng webhook part: %s", err.Error())
								}
							}
							part.Close()
						}
						mpw.Close()
					}
					out.Close()
				}(c.R, pw)
			} else {
				body = c.R
			}

			var r *http.Response
			r, err = http.Post(u.String(), ctype, body)
			if err == nil {
				_, err = io.Copy(ioutil.Discard, r.Body)
				r.Body.Close()
				log.Infof("hook called for %s", msgid)
			}
		} else {
			f.Close()
		}
	}
	if err != nil {
		log.Errorf("error calling web hook %s: %s", h.conf.Name, err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"github.com/keybase/client/go/libkb"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/kbfssync"
	"github.com/keybase/kbfs/tlf"
	"golang.org/x/net/context"
)

const (
	disableFavoritesEnvVar       = "KEYBASE_DISABLE_FAVORITES"
	favoritesCacheExpirationTime = time.Hour * 24 * 7 // one week
)

type favToAdd struct {
	Favorite

	// created, if set to true, indicates that this is the first time the TLF has
	// ever existed. It is only used when adding the TLF to favorites
	created bool
}

func (f favToAdd) ToKBFolder() keybase1.Folder {
	return f.Favorite.ToKBFolder(f.created)
}

// favReq represents a request to access the logged-in user's
// favorites list.  A single request can do one or more of the
// following: refresh the current cached list, add a favorite, remove
// a favorite, and get all the favorites.  When the request is done,
// the resulting error (or nil) is sent over the done channel.  The
// given ctx is used for all network operations.
type favReq struct {
	// Request types
	refresh bool
	toAdd   []favToAdd
	toDel   []Favorite
	favs    chan<- []Favorite

	// Closed when the request is done.
	done chan struct{}
	// Set before done is closed
	err error

	// Context
	ctx context.Context
}

// Favorites manages a user's favorite list.
type Favorites struct {
	config   Config
	disabled bool

	// Channels for interacting with the favorites cache
	reqChan chan *favReq

	wg kbfssync.RepeatedWaitGroup

	// cache tracks the favorites for this user, that we know about.
	// It may not be consistent with the server's view of the user's
	// favorites list, if other devices have modified the list since
	// the last refresh.
	cache           map[Favorite]bool
	cacheExpireTime time.Time

	inFlightLock sync.Mutex
	inFlightAdds map[favToAdd]*favReq

	muShutdown sync.RWMutex
	shutdown   bool
}

func newFavoritesWithChan(config Config, reqChan chan *favReq) *Favorites {
	disableVal := strings.ToLower(os.Getenv(disableFavoritesEnvVar))
	if len(disableVal) > 0 && disableVal != "0" && disableVal != "false" &&
		disableVal != "no" {
		config.MakeLogger("").CDebugf(nil,
			"Disable favorites due to env var %s=%s",
			disableFavoritesEnvVar, disableVal)
		return &Favorites{
			config:   config,
			disabled: true,
		}
	}

	f := &Favorites{
		config:       config,
		reqChan:      reqChan,
		inFlightAdds: make(map[favToAdd]*favReq),
	}
	go f.loop()
	return f
}

// NewFavorites constructs a new Favorites instance.
func NewFavorites(config Config) *Favorites {
	return newFavoritesWithChan(config, make(chan *favReq, 100))
}

func (f *Favorites) closeReq(req *favReq, err error) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req.err = err
	close(req.done)
	for _, fav := range req.toAdd {
		delete(f.inFlightAdds, fav)
	}
}

func (f *Favorites) handleReq(req *favReq) (err error) {
	defer func() { f.closeReq(req, err) }()

	kbpki := f.config.KBPKI()
	// Fetch a new list if:
	//  * The user asked us to refresh
	//  * We haven't fetched it before
	//  * It's stale
	if req.refresh || f.cache == nil || time.Now().After(f.cacheExpireTime) {
		folders, err := kbpki.FavoriteList(req.ctx)
		if err != nil {
			if req.refresh {
				// if we're supposed to refresh the cache and it's not
				// working, try again in a few minutes.
				in5Minutes := time.Now().Add(5 * time.Minute)
				if in5Minutes.After(f.cacheExpireTime) {
					f.cacheExpireTime = in5Minutes
				}
			}
			return err
		}

		f.cache = make(map[Favorite]bool)
		f.cacheExpireTime = libkb.ForceWallClock(time.Now()).Add(favoritesCacheExpirationTime)
		for _, folder := range folders {
			f.cache[*NewFavoriteFromFolder(folder)] = true
		}
		session, err := f.config.KBPKI().GetCurrentSession(req.ctx)
		if err == nil {
			// Add favorites for the current user, that cannot be deleted.
			f.cache[Favorite{string(session.Name), tlf.Private}] = true
			f.cache[Favorite{string(session.Name), tlf.Public}] = true
		}
	}

	for _, fav := range req.toAdd {
		if !fav.created && f.cache[fav.Favorite] {
			continue
		}
		err := kbpki.FavoriteAdd(req.ctx, fav.ToKBFolder())
		if err != nil {
			f.config.MakeLogger("").CDebugf(req.ctx,
				"Failure adding favorite %v: %v", fav, err)
			return err
		}
		f.cache[fav.Favorite] = true
	}

	for _, fav := range req.toDel {
		// Since our cache isn't necessarily up-to-date, always delete
		// the favorite.
		folder := fav.ToKBFolder(false)
		err := kbpki.FavoriteDelete(req.ctx, folder)
		if err != nil {
			return err
		}
		delete(f.cache, fav)
	}

	if req.favs != nil {
		favorites := make([]Favorite, 0, len(f.cache))
		for fav := range f.cache {
			favorites = append(favorites, fav)
		}
		req.favs <- favorites
	}

	return nil
}

func (f *Favorites) loop() {
	for req := range f.reqChan {
		f.handleReq(req)
		f.wg.Done()
	}
}

// Shutdown shuts down this Favorites instance.
func (f *Favorites) Shutdown() error {
	if f.disabled {
		return nil
	}

	f.muShutdown.Lock()
	defer f.muShutdown.Unlock()
	f.shutdown = true
	close(f.reqChan)
	return f.wg.Wait(context.Background())
}

func (f *Favorites) hasShutdown() bool {
	f.muShutdown.RLock()
	defer f.muShutdown.RUnlock()
	return f.shutdown
}

func (f *Favorites) waitOnReq(ctx context.Context,
	req *favReq) (retry bool, err error) {
	select {
	case <-ctx.Done():
		return false, ctx.Err()
	case <-req.done:
		err = req.err
		// If the request was canceled due to a context timeout that
		// wasn't our own, try it again.
		if err == context.Canceled || err == context.DeadlineExceeded {
			select {
			case <-ctx.Done():
				return false, err
			default:
				return true, nil
			}
		}
		return false, err
	}
}

func (f *Favorites) sendReq(ctx context.Context, req *favReq) error {
	f.wg.Add(1)
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		f.wg.Done()
		err := ctx.Err()
		f.closeReq(req, err)
		return err
	}
	// With a direct sendReq call, we'll never have a shared request,
	// so no need to check the retry status.
	_, err := f.waitOnReq(ctx, req)
	return err
}

func (f *Favorites) startOrJoinAddReq(
	ctx context.Context, fav favToAdd) (req *favReq, doSend bool) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req, ok := f.inFlightAdds[fav]
	if !ok {
		req = &favReq{
			ctx:   ctx,
			toAdd: []favToAdd{fav},
			done:  make(chan struct{}),
		}
		f.inFlightAdds[fav] = req
		doSend = true
	}
	return req, doSend
}

// Add adds a favorite to your favorites list.
func (f *Favorites) Add(ctx context.Context, fav favToAdd) error {
	if f.disabled {
		return nil
	}
	if f.hasShutdown() {
		return ShutdownHappenedError{}
	}
	doAdd := true
	var err error
	// Retry until we get an error that wasn't related to someone
	// else's context being canceled.
	for doAdd {
		req, doSend := f.startOrJoinAddReq(ctx, fav)
		if doSend {
			return f.sendReq(ctx, req)
		}
		doAdd, err = f.waitOnReq(ctx, req)
	}
	return err
}

// AddAsync initiates a request to add this favorite to your favorites
// list, if one is not already in flight, but it doesn't wait for the
// result.  (It could block while kicking off the request, if lots of
// different favorite operations are in flight.)  The given context is
// used only for enqueuing the request on an internal queue, not for
// any resulting I/O.
func (f *Favorites) AddAsync(ctx context.Context, fav favToAdd) {
	if f.disabled || f.hasShutdown() {
		return
	}
	// Use a fresh context, since we want the request to succeed even
	// if the original context is canceled.
	req, doSend := f.startOrJoinAddReq(context.Background(), fav)
	if doSend {
		f.wg.Add(1)
		select {
		case f.reqChan <- req:
		case <-ctx.Done():
			f.wg.Done()
			err := ctx.Err()
			f.closeReq(req, err)
			return
		}
	}
}

// Delete deletes a favorite from the favorites list.  It is
// idempotent.
func (f *Favorites) Delete(ctx context.Context, fav Favorite) error {
	if f.disabled {
		return nil
	}
	if f.hasShutdown() {
		return ShutdownHappenedError{}
	}
	return f.sendReq(ctx, &favReq{
		ctx:   ctx,
		toDel: []Favorite{fav},
		done:  make(chan struct{}),
	})
}

// RefreshCache refreshes the cached list of favorites.
func (f *Favorites) RefreshCache(ctx context.Context) {
	if f.disabled || f.hasShutdown() {
		return
	}
	// This request is non-blocking, so use a throw-away done channel
	// and context.
	req := &favReq{
		refresh: true,
		done:    make(chan struct{}),
		ctx:     context.Background(),
	}
	f.wg.Add(1)
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		f.wg.Done()
		return
	}
}

// Get returns the logged-in user's list of favorites. It uses the cache.
func (f *Favorites) Get(ctx context.Context) ([]Favorite, error) {
	if f.disabled {
		session, err := f.config.KBPKI().GetCurrentSession(ctx)
		if err == nil {
			// Add favorites only for the current user.
			return []Favorite{
				{string(session.Name), tlf.Private},
				{string(session.Name), tlf.Public},
			}, nil
		}
		return nil, nil
	}
	if f.hasShutdown() {
		return nil, ShutdownHappenedError{}
	}
	favChan := make(chan []Favorite, 1)
	req := &favReq{
		ctx:  ctx,
		favs: favChan,
		done: make(chan struct{}),
	}
	err := f.sendReq(ctx, req)
	if err != nil {
		return nil, err
	}
	return <-favChan, nil
}
<commit_msg>Revert "favorites: shorten cache on refresh error"<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"github.com/keybase/client/go/libkb"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/kbfssync"
	"github.com/keybase/kbfs/tlf"
	"golang.org/x/net/context"
)

const (
	disableFavoritesEnvVar = "KEYBASE_DISABLE_FAVORITES"
)

type favToAdd struct {
	Favorite

	// created, if set to true, indicates that this is the first time the TLF has
	// ever existed. It is only used when adding the TLF to favorites
	created bool
}

func (f favToAdd) ToKBFolder() keybase1.Folder {
	return f.Favorite.ToKBFolder(f.created)
}

// favReq represents a request to access the logged-in user's
// favorites list.  A single request can do one or more of the
// following: refresh the current cached list, add a favorite, remove
// a favorite, and get all the favorites.  When the request is done,
// the resulting error (or nil) is sent over the done channel.  The
// given ctx is used for all network operations.
type favReq struct {
	// Request types
	refresh bool
	toAdd   []favToAdd
	toDel   []Favorite
	favs    chan<- []Favorite

	// Closed when the request is done.
	done chan struct{}
	// Set before done is closed
	err error

	// Context
	ctx context.Context
}

// Favorites manages a user's favorite list.
type Favorites struct {
	config   Config
	disabled bool

	// Channels for interacting with the favorites cache
	reqChan chan *favReq

	wg kbfssync.RepeatedWaitGroup

	// cache tracks the favorites for this user, that we know about.
	// It may not be consistent with the server's view of the user's
	// favorites list, if other devices have modified the list since
	// the last refresh.
	cache           map[Favorite]bool
	cacheExpireTime time.Time

	inFlightLock sync.Mutex
	inFlightAdds map[favToAdd]*favReq

	muShutdown sync.RWMutex
	shutdown   bool
}

func newFavoritesWithChan(config Config, reqChan chan *favReq) *Favorites {
	disableVal := strings.ToLower(os.Getenv(disableFavoritesEnvVar))
	if len(disableVal) > 0 && disableVal != "0" && disableVal != "false" &&
		disableVal != "no" {
		config.MakeLogger("").CDebugf(nil,
			"Disable favorites due to env var %s=%s",
			disableFavoritesEnvVar, disableVal)
		return &Favorites{
			config:   config,
			disabled: true,
		}
	}

	f := &Favorites{
		config:       config,
		reqChan:      reqChan,
		inFlightAdds: make(map[favToAdd]*favReq),
	}
	go f.loop()
	return f
}

// NewFavorites constructs a new Favorites instance.
func NewFavorites(config Config) *Favorites {
	return newFavoritesWithChan(config, make(chan *favReq, 100))
}

func (f *Favorites) closeReq(req *favReq, err error) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req.err = err
	close(req.done)
	for _, fav := range req.toAdd {
		delete(f.inFlightAdds, fav)
	}
}

func (f *Favorites) handleReq(req *favReq) (err error) {
	defer func() { f.closeReq(req, err) }()

	kbpki := f.config.KBPKI()
	// Fetch a new list if:
	//  * The user asked us to refresh
	//  * We haven't fetched it before
	//  * It's stale
	if req.refresh || f.cache == nil || time.Now().After(f.cacheExpireTime) {
		folders, err := kbpki.FavoriteList(req.ctx)
		if err != nil {
			return err
		}

		f.cache = make(map[Favorite]bool)
		f.cacheExpireTime = libkb.ForceWallClock(time.Now()).Add(time.
			Hour * 24 * 7)
		for _, folder := range folders {
			f.cache[*NewFavoriteFromFolder(folder)] = true
		}
		session, err := f.config.KBPKI().GetCurrentSession(req.ctx)
		if err == nil {
			// Add favorites for the current user, that cannot be deleted.
			f.cache[Favorite{string(session.Name), tlf.Private}] = true
			f.cache[Favorite{string(session.Name), tlf.Public}] = true
		}
	}

	for _, fav := range req.toAdd {
		if !fav.created && f.cache[fav.Favorite] {
			continue
		}
		err := kbpki.FavoriteAdd(req.ctx, fav.ToKBFolder())
		if err != nil {
			f.config.MakeLogger("").CDebugf(req.ctx,
				"Failure adding favorite %v: %v", fav, err)
			return err
		}
		f.cache[fav.Favorite] = true
	}

	for _, fav := range req.toDel {
		// Since our cache isn't necessarily up-to-date, always delete
		// the favorite.
		folder := fav.ToKBFolder(false)
		err := kbpki.FavoriteDelete(req.ctx, folder)
		if err != nil {
			return err
		}
		delete(f.cache, fav)
	}

	if req.favs != nil {
		favorites := make([]Favorite, 0, len(f.cache))
		for fav := range f.cache {
			favorites = append(favorites, fav)
		}
		req.favs <- favorites
	}

	return nil
}

func (f *Favorites) loop() {
	for req := range f.reqChan {
		f.handleReq(req)
		f.wg.Done()
	}
}

// Shutdown shuts down this Favorites instance.
func (f *Favorites) Shutdown() error {
	if f.disabled {
		return nil
	}

	f.muShutdown.Lock()
	defer f.muShutdown.Unlock()
	f.shutdown = true
	close(f.reqChan)
	return f.wg.Wait(context.Background())
}

func (f *Favorites) hasShutdown() bool {
	f.muShutdown.RLock()
	defer f.muShutdown.RUnlock()
	return f.shutdown
}

func (f *Favorites) waitOnReq(ctx context.Context,
	req *favReq) (retry bool, err error) {
	select {
	case <-ctx.Done():
		return false, ctx.Err()
	case <-req.done:
		err = req.err
		// If the request was canceled due to a context timeout that
		// wasn't our own, try it again.
		if err == context.Canceled || err == context.DeadlineExceeded {
			select {
			case <-ctx.Done():
				return false, err
			default:
				return true, nil
			}
		}
		return false, err
	}
}

func (f *Favorites) sendReq(ctx context.Context, req *favReq) error {
	f.wg.Add(1)
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		f.wg.Done()
		err := ctx.Err()
		f.closeReq(req, err)
		return err
	}
	// With a direct sendReq call, we'll never have a shared request,
	// so no need to check the retry status.
	_, err := f.waitOnReq(ctx, req)
	return err
}

func (f *Favorites) startOrJoinAddReq(
	ctx context.Context, fav favToAdd) (req *favReq, doSend bool) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req, ok := f.inFlightAdds[fav]
	if !ok {
		req = &favReq{
			ctx:   ctx,
			toAdd: []favToAdd{fav},
			done:  make(chan struct{}),
		}
		f.inFlightAdds[fav] = req
		doSend = true
	}
	return req, doSend
}

// Add adds a favorite to your favorites list.
func (f *Favorites) Add(ctx context.Context, fav favToAdd) error {
	if f.disabled {
		return nil
	}
	if f.hasShutdown() {
		return ShutdownHappenedError{}
	}
	doAdd := true
	var err error
	// Retry until we get an error that wasn't related to someone
	// else's context being canceled.
	for doAdd {
		req, doSend := f.startOrJoinAddReq(ctx, fav)
		if doSend {
			return f.sendReq(ctx, req)
		}
		doAdd, err = f.waitOnReq(ctx, req)
	}
	return err
}

// AddAsync initiates a request to add this favorite to your favorites
// list, if one is not already in flight, but it doesn't wait for the
// result.  (It could block while kicking off the request, if lots of
// different favorite operations are in flight.)  The given context is
// used only for enqueuing the request on an internal queue, not for
// any resulting I/O.
func (f *Favorites) AddAsync(ctx context.Context, fav favToAdd) {
	if f.disabled || f.hasShutdown() {
		return
	}
	// Use a fresh context, since we want the request to succeed even
	// if the original context is canceled.
	req, doSend := f.startOrJoinAddReq(context.Background(), fav)
	if doSend {
		f.wg.Add(1)
		select {
		case f.reqChan <- req:
		case <-ctx.Done():
			f.wg.Done()
			err := ctx.Err()
			f.closeReq(req, err)
			return
		}
	}
}

// Delete deletes a favorite from the favorites list.  It is
// idempotent.
func (f *Favorites) Delete(ctx context.Context, fav Favorite) error {
	if f.disabled {
		return nil
	}
	if f.hasShutdown() {
		return ShutdownHappenedError{}
	}
	return f.sendReq(ctx, &favReq{
		ctx:   ctx,
		toDel: []Favorite{fav},
		done:  make(chan struct{}),
	})
}

// RefreshCache refreshes the cached list of favorites.
func (f *Favorites) RefreshCache(ctx context.Context) {
	if f.disabled || f.hasShutdown() {
		return
	}
	// This request is non-blocking, so use a throw-away done channel
	// and context.
	req := &favReq{
		refresh: true,
		done:    make(chan struct{}),
		ctx:     context.Background(),
	}
	f.wg.Add(1)
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		f.wg.Done()
		return
	}
}

// Get returns the logged-in user's list of favorites. It uses the cache.
// TODO: decide on behavior if we've gotten an invalidation but not an updated
// list yet
func (f *Favorites) Get(ctx context.Context) ([]Favorite, error) {
	if f.disabled {
		session, err := f.config.KBPKI().GetCurrentSession(ctx)
		if err == nil {
			// Add favorites only for the current user.
			return []Favorite{
				{string(session.Name), tlf.Private},
				{string(session.Name), tlf.Public},
			}, nil
		}
		return nil, nil
	}
	if f.hasShutdown() {
		return nil, ShutdownHappenedError{}
	}
	favChan := make(chan []Favorite, 1)
	req := &favReq{
		ctx:  ctx,
		favs: favChan,
		done: make(chan struct{}),
	}
	err := f.sendReq(ctx, req)
	if err != nil {
		return nil, err
	}
	return <-favChan, nil
}
<|endoftext|>


<commit_before>package network

import (
    "fmt"
    "net"
    "bufio"
    "encoding/binary"
    "time"

)

const (
    ConnStateIn    = iota
    ConnStateDisc  = iota

    MAX_LEN_HEAD   = 1024 * 4
)

type ClientConnection struct {
    conn        net.Conn
    reader      *bufio.Reader
    writer      *bufio.Writer
    connState   int
}

func NewClientConnection(c net.Conn) *ClientConnection {
    cliConn := new(ClientConnection)
    cliConn.conn = c
    cliConn.reader = bufio.NewReader(c)
    cliConn.writer = bufio.NewWriter(c)
    return cliConn
}

func (this *ClientConnection) Send(buf []byte) {
    if this.connState == ConnStateDisc { return }

    head := make([]byte, 4)
    binary.LittleEndian.PutUint32(head, uint32(len(buf)))
    buf = append(head, buf...)

    this.writer.Write(buf)
}

//func (this *ClientConnection) blockSend(b []byte) bool {
//    for len(b) > 0 {
//        n, err := this.conn.Write(b)
//        if err == nil {
//            b = b[n:]
//        } else if e, ok := err.(*net.OpError); ok && e.Temporary() {
//            continue
//        } else {
//            if this.connState != ConnStateDisc {
//                this.connState = ConnStateDisc
//            }
//            fmt.Println("blockSend disconnect")
//            return false
//        }
//    }
//    return true
//}

func (this *ClientConnection) sendall() bool {
    if err := this.writer.Flush(); err != nil {
        if this.connState != ConnStateDisc {
            this.connState = ConnStateDisc
        }
        fmt.Println("send err:", err)
        return false
    }
    return true
}

func (this *ClientConnection) duplexRead(buff []byte) bool {
    var read_size int
    for {
        // write
        if !this.sendall() {
            return false
        }

        // read
        this.conn.SetReadDeadline(time.Now().Add(1e8))
        n, err := this.reader.Read(buff[read_size:])
        if err != nil {
            if e, ok := err.(*net.OpError); ok && e.Temporary() {
                read_size = n
                continue
            } else {
                //fmt.Println("read err, disconnect", err)
                return false
            }
        }

        if n == 0 { return true }
        if n < len(buff) {
            read_size += n
            continue
        }
        return true
    }
    return false
}

func (this *ClientConnection) duplexReadBody() (ret []byte,  ok bool) {
    buff_head := make([]byte, 4)
    if !this.duplexRead(buff_head) {
        return
    }
    len_head := binary.LittleEndian.Uint32(buff_head)
    if len_head > MAX_LEN_HEAD {
        fmt.Println("message len too long", len_head)
        return
    }
    ret = make([]byte, len_head)
    if !this.duplexRead(ret) {
        return
    }
    ok = true
    return
}

func (this *ClientConnection) Close() {
    this.connState  = ConnStateDisc
    this.conn.Close()
}

func (this* ClientConnection) IsClose() bool {
    return this.connState == ConnStateDisc
}
<commit_msg>send data race issuse<commit_after>package network

import (
    "fmt"
    "net"
    "bufio"
    "encoding/binary"
    "time"

)

const (
    ConnStateIn    = iota
    ConnStateDisc  = iota

    MAX_LEN_HEAD   = 1024 * 4
)

type ClientConnection struct {
    conn        net.Conn
    reader      *bufio.Reader
    writer      *bufio.Writer
    connState   int
}

func NewClientConnection(c net.Conn) *ClientConnection {
    cliConn := new(ClientConnection)
    cliConn.conn = c
    cliConn.reader = bufio.NewReader(c)
    cliConn.writer = bufio.NewWriter(c)
    return cliConn
}

func (this *ClientConnection) Send(buf []byte) {
    if this.connState == ConnStateDisc { return }

    head := make([]byte, 4)
    binary.LittleEndian.PutUint32(head, uint32(len(buf)))
    buf = append(head, buf...)

    if _, err := this.writer.Write(buf); err != nil {
        fmt.Println("Send err:", err)
    }
}

func (this *ClientConnection) sendall() bool {
    if err := this.writer.Flush(); err != nil {
        if this.connState != ConnStateDisc {
            this.connState = ConnStateDisc
        }
        fmt.Println("send err:", err)
        return false
    }
    return true
}

func (this *ClientConnection) duplexRead(buff []byte) bool {
    var read_size int
    for {
        //// write
        //if !this.sendall() {
        //    return false
        //}

        // read
        this.conn.SetReadDeadline(time.Now().Add(1e8))
        n, err := this.reader.Read(buff[read_size:])
        if err != nil {
            if e, ok := err.(*net.OpError); ok && e.Temporary() {
                read_size = n
                continue
            } else {
                fmt.Println("read err, disconnect", err)
                return false
            }
        }

        if n == 0 { return true }
        if n < len(buff) {
            read_size += n
            continue
        }
        return true
    }
    return false
}

func (this *ClientConnection) duplexReadBody() (ret []byte,  ok bool) {
    buff_head := make([]byte, 4)
    if !this.duplexRead(buff_head) {
        return
    }
    len_head := binary.LittleEndian.Uint32(buff_head)
    if len_head > MAX_LEN_HEAD {
        fmt.Println("message len too long", len_head)
        return
    }
    ret = make([]byte, len_head)
    if !this.duplexRead(ret) {
        return
    }
    ok = true
    return
}

func (this *ClientConnection) Close() {
    this.connState  = ConnStateDisc
    this.conn.Close()
}

<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"os/exec"

	"github.com/mgierok/monujo/config"

	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
)

var db *sqlx.DB

func MustInitialize() {
	dbinfo := fmt.Sprintf(
		"host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
		config.Db().Host,
		config.Db().Port,
		config.Db().User,
		config.Db().Password,
		config.Db().Dbname,
	)

	var err error
	db, err = sqlx.Connect("postgres", dbinfo)

	if err != nil {
		panic(err)
	}
}

func Connection() *sqlx.DB {
	return db
}

func Dump(dumptype string, file string) {
	if len(file) == 0 {
		fmt.Println("Output file is not set")
		return
	}

	var cmd *exec.Cmd
	if dumptype == "schema" {
		cmd = exec.Command(
			"/usr/bin/pg_dump",
			"--host",
			config.Db().Host,
			"--port",
			config.Db().Port,
			"--username",
			config.Db().User,
			"--no-password",
			"--format",
			"plain",
			"--schema-only",
			"--no-owner",
			"--no-privileges",
			"--no-tablespaces",
			"--no-unlogged-table-data",
			"--file",
			file,
			config.Db().Dbname,
		)
	} else if dumptype == "data" {
		cmd = exec.Command(
			"/usr/bin/pg_dump",
			"--host",
			config.Db().Host,
			"--port",
			config.Db().Port,
			"--username",
			config.Db().User,
			"--no-password",
			"--format",
			"plain",
			"--data-only",
			"--disable-triggers",
			"--no-owner",
			"--no-privileges",
			"--no-tablespaces",
			"--no-unlogged-table-data",
			"--file",
			file,
			config.Db().Dbname,
		)
	} else {
		fmt.Println("Invalid dump type, please specify 'schema' or 'data'")
		return
	}

	stdout, err := cmd.Output()
	if err != nil {
		fmt.Println(err.Error())
		return
	}

	fmt.Println(string(stdout))
}
<commit_msg>use inserts for data dumps<commit_after>package db

import (
	"fmt"
	"os/exec"

	"github.com/mgierok/monujo/config"

	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
)

var db *sqlx.DB

func MustInitialize() {
	dbinfo := fmt.Sprintf(
		"host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
		config.Db().Host,
		config.Db().Port,
		config.Db().User,
		config.Db().Password,
		config.Db().Dbname,
	)

	var err error
	db, err = sqlx.Connect("postgres", dbinfo)

	if err != nil {
		panic(err)
	}
}

func Connection() *sqlx.DB {
	return db
}

func Dump(dumptype string, file string) {
	if len(file) == 0 {
		fmt.Println("Output file is not set")
		return
	}

	var cmd *exec.Cmd
	if dumptype == "schema" {
		cmd = exec.Command(
			"/usr/bin/pg_dump",
			"--host",
			config.Db().Host,
			"--port",
			config.Db().Port,
			"--username",
			config.Db().User,
			"--no-password",
			"--format",
			"plain",
			"--schema-only",
			"--no-owner",
			"--no-privileges",
			"--no-tablespaces",
			"--no-unlogged-table-data",
			"--file",
			file,
			config.Db().Dbname,
		)
	} else if dumptype == "data" {
		cmd = exec.Command(
			"/usr/bin/pg_dump",
			"--host",
			config.Db().Host,
			"--port",
			config.Db().Port,
			"--username",
			config.Db().User,
			"--no-password",
			"--format",
			"plain",
			"--data-only",
			"--inserts",
			"--disable-triggers",
			"--no-owner",
			"--no-privileges",
			"--no-tablespaces",
			"--no-unlogged-table-data",
			"--file",
			file,
			config.Db().Dbname,
		)
	} else {
		fmt.Println("Invalid dump type, please specify 'schema' or 'data'")
		return
	}

	stdout, err := cmd.Output()
	if err != nil {
		fmt.Println(err.Error())
		return
	}

	fmt.Println(string(stdout))
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	chars      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

var (
	FileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	Hash    string
}

type FileData struct {
	Name string
	Data []byte
}

type FileTag uint8

const (
	File = FileTag(iota)
	Conflict
	Directory
	Deleted
)

// Parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version hash
			if len(fields) != 6 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			sum := fields[5]

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId, sum}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// Parses a TraDb from a file.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(FileNotFound.Error())
		tradb = New()

		return tradb, FileNotFound
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

// Creates a new TraDb.
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = chars[rand.Intn(len(chars))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
			Hash:    hash(filename),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

// Writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d %s",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
			hash(filename),
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

// Looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	// db.VersionVec[db.ReplicaId] += 1
	visitedFiles := make(map[string]bool)
	ourVersion := db.VersionVec[db.ReplicaId]

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if dbRecord.Version == 0 {
			log.Printf("found a new file: %s\n", filename)
			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
				Hash:    hash(filename),
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
			dbRecord.Replica = db.ReplicaId
			dbRecord.Hash = hash(filename)
			db.Files[filename] = dbRecord
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
		visitedFiles[filename] = true
	}

	// Check for deleted files.
	for filename, _ := range db.Files {
		if !visitedFiles[filename] {
			log.Printf("update: deleting entry for %s\n", filename)
		}
	}

	return nil
}

// Compares two TraDbs.
func (local *TraDb) Compare(remote *TraDb) map[string]FileTag {
	tags := make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			if state.Version < remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags[file] = Deleted
			}
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
				delete(tags, file) // in case we marked it for deletion
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags[file] = File
			} else {
				log.Printf("conflict: %s\n", file)
				tags[file] = Conflict
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags[file] = File
		}
	}

	return tags
}

func (db *TraDb) UpdateMTimes() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if mtime := file.ModTime().UTC().UnixNano(); mtime > dbRecord.MTime {
			log.Printf("updating mtime: %s\n", filename)
			dbRecord.MTime = mtime
			db.Files[filename] = dbRecord
		}
	}

	return nil
}

func CombineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func hash(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		log.Fatal(err)
	}

	return hex.EncodeToString(h.Sum(nil))
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		h1, err1 := hex.DecodeString(fs1.Hash)
		h2, err2 := hex.DecodeString(fs2.Hash)
		if err1 != nil || err2 != nil {
			log.Fatal(err1, err2)
		}

		return !bytes.Equal(h1, h2)
	}
	return false
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>actually delete entries for removed files<commit_after>package db

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	chars      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

var (
	FileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	Hash    string
}

type FileData struct {
	Name string
	Data []byte
}

type FileTag uint8

const (
	File = FileTag(iota)
	Conflict
	Directory
	Deleted
)

// Parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version hash
			if len(fields) != 6 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			sum := fields[5]

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId, sum}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// Parses a TraDb from a file.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(FileNotFound.Error())
		tradb = New()

		return tradb, FileNotFound
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

// Creates a new TraDb.
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = chars[rand.Intn(len(chars))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
			Hash:    hash(filename),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

// Writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d %s",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
			hash(filename),
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

// Looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	// db.VersionVec[db.ReplicaId] += 1
	visitedFiles := make(map[string]bool)
	ourVersion := db.VersionVec[db.ReplicaId]

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if dbRecord.Version == 0 {
			log.Printf("found a new file: %s\n", filename)
			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
				Hash:    hash(filename),
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
			dbRecord.Replica = db.ReplicaId
			dbRecord.Hash = hash(filename)
			db.Files[filename] = dbRecord
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
		visitedFiles[filename] = true
	}

	// Check for deleted files.
	for filename, _ := range db.Files {
		if !visitedFiles[filename] {
			log.Printf("update: deleting entry for %s\n", filename)
			delete(db.Files, filename)
		}
	}

	return nil
}

// Compares two TraDbs.
func (local *TraDb) Compare(remote *TraDb) map[string]FileTag {
	tags := make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			if state.Version < remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags[file] = Deleted
			}
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
				delete(tags, file) // in case we marked it for deletion
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags[file] = File
			} else {
				log.Printf("conflict: %s\n", file)
				tags[file] = Conflict
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags[file] = File
		}
	}

	return tags
}

func (db *TraDb) UpdateMTimes() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if mtime := file.ModTime().UTC().UnixNano(); mtime > dbRecord.MTime {
			log.Printf("updating mtime: %s\n", filename)
			dbRecord.MTime = mtime
			db.Files[filename] = dbRecord
		}
	}

	return nil
}

func CombineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func hash(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		log.Fatal(err)
	}

	return hex.EncodeToString(h.Sum(nil))
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		h1, err1 := hex.DecodeString(fs1.Hash)
		h2, err2 := hex.DecodeString(fs2.Hash)
		if err1 != nil || err2 != nil {
			log.Fatal(err1, err2)
		}

		return !bytes.Equal(h1, h2)
	}
	return false
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package couchbase

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type ViewDefinition struct {
	Map    string `json:"map"`
	Reduce string `json:"reduce"`
}

type DDocJSON struct {
	Language string                    `json:"language"`
	Views    map[string]ViewDefinition `json:"views"`
}

type DDoc struct {
	Meta map[string]interface{} `json:"meta"`
	Json DDocJSON               `json:"json"`
}

type DDocRow struct {
	DDoc DDoc `json:"doc"`
}

type DDocsResult struct {
	Rows []DDocRow `json:"rows"`
}

// Get the design documents
func (b *Bucket) GetDDocs() (DDocsResult, error) {
	var ddocsResult DDocsResult
	err := b.pool.client.parseURLResponse(b.DDocs.URI, &ddocsResult)
	if err != nil {
		return DDocsResult{}, err
	}
	return ddocsResult, nil
}

func (b *Bucket) ddocURL(docname string) (string, error) {
	u, err := b.randomBaseURL()
	if err != nil {
		return "", err
	}
	u.Path = fmt.Sprintf("/%s/_design/%s", b.Name, docname)
	return u.String(), nil
}

// Install a design document.
func (b *Bucket) PutDDoc(docname string, value interface{}) error {
	ddocU, err := b.ddocURL(docname)
	if err != nil {
		return err
	}

	j, err := json.Marshal(value)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", ddocU, bytes.NewReader(j))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	maybeAddAuth(req, b.authHandler())

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 201 {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("Error installing view: %v / %s",
			res.Status, body)
	}

	return nil
}

// Get a design doc.
func (b *Bucket) GetDDoc(docname string, into interface{}) error {
	ddocU, err := b.ddocURL(docname)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("GET", ddocU, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	maybeAddAuth(req, b.authHandler())

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("Error reading view: %v / %s",
			res.Status, body)
	}

	d := json.NewDecoder(res.Body)
	return d.Decode(into)
}

// Delete a design document.
func (b *Bucket) DeleteDDoc(docname string) error {
	ddocU, err := b.ddocURL(docname)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", ddocU, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	maybeAddAuth(req, b.authHandler())

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("Error deleting view: %v / %s",
			res.Status, body)
	}

	return nil
}
<commit_msg>Suppress empty reduce function.<commit_after>package couchbase

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type ViewDefinition struct {
	Map    string `json:"map"`
	Reduce string `json:"reduce,omitempty"`
}

type DDocJSON struct {
	Language string                    `json:"language,omitempty"`
	Views    map[string]ViewDefinition `json:"views"`
}

type DDoc struct {
	Meta map[string]interface{} `json:"meta"`
	Json DDocJSON               `json:"json"`
}

type DDocRow struct {
	DDoc DDoc `json:"doc"`
}

type DDocsResult struct {
	Rows []DDocRow `json:"rows"`
}

// Get the design documents
func (b *Bucket) GetDDocs() (DDocsResult, error) {
	var ddocsResult DDocsResult
	err := b.pool.client.parseURLResponse(b.DDocs.URI, &ddocsResult)
	if err != nil {
		return DDocsResult{}, err
	}
	return ddocsResult, nil
}

func (b *Bucket) ddocURL(docname string) (string, error) {
	u, err := b.randomBaseURL()
	if err != nil {
		return "", err
	}
	u.Path = fmt.Sprintf("/%s/_design/%s", b.Name, docname)
	return u.String(), nil
}

// Install a design document.
func (b *Bucket) PutDDoc(docname string, value interface{}) error {
	ddocU, err := b.ddocURL(docname)
	if err != nil {
		return err
	}

	j, err := json.Marshal(value)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", ddocU, bytes.NewReader(j))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	maybeAddAuth(req, b.authHandler())

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 201 {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("Error installing view: %v / %s",
			res.Status, body)
	}

	return nil
}

// Get a design doc.
func (b *Bucket) GetDDoc(docname string, into interface{}) error {
	ddocU, err := b.ddocURL(docname)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("GET", ddocU, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	maybeAddAuth(req, b.authHandler())

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("Error reading view: %v / %s",
			res.Status, body)
	}

	d := json.NewDecoder(res.Body)
	return d.Decode(into)
}

// Delete a design document.
func (b *Bucket) DeleteDDoc(docname string) error {
	ddocU, err := b.ddocURL(docname)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", ddocU, nil)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	maybeAddAuth(req, b.authHandler())

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("Error deleting view: %v / %s",
			res.Status, body)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Kochava
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/wvanbergen/kafka/consumergroup"
	"github.com/wvanbergen/kazoo-go"
)

// GetKafkaConsumer returns a new consumer
func GetKafkaConsumer(custConfig *Config) (*consumergroup.ConsumerGroup, error) {
	groupConfig := consumergroup.NewConfig()
	groupConfig.Offsets.Initial = sarama.OffsetNewest
	groupConfig.Offsets.ProcessingTimeout = 10 * time.Second

	zookeeperNodes := strings.Split(custConfig.SourceZookeepers, ",")
	groupConfig.Zookeeper.Chroot = ""

	topic := []string{custConfig.Topic}

	consumer, err := consumergroup.JoinConsumerGroup("firehose", topic, zookeeperNodes, groupConfig)
	if err != nil {
		log.Printf("GetKafkaConsumer - Failed to join consumer group: %v\n", err)
		return nil, err
	}

	// Create new consumer
	return consumer, nil

}

// GetConsumerErrors logs any consumer errors
func GetConsumerErrors(consumer *consumergroup.ConsumerGroup) {
	for err := range consumer.Errors() {
		log.Printf("GetConsumerErrors - %v\n", err)
	}
}

// GetKafkaProducer returns a new consumer
func GetKafkaProducer(custConfig *Config) (sarama.AsyncProducer, error) {
	config := consumergroup.NewConfig()

	config.Config.Producer.Retry.Max = 1
	config.Config.Producer.RequiredAcks = sarama.WaitForAll
	config.Config.Producer.Return.Errors = true
	config.Config.Producer.Return.Successes = true
	// config.Producer.Partitioner = sarama.NewManualPartitioner
	config.Config.ClientID = "firehose_realtime"

	log.Printf("GetKafkaProducer - client id %v\n", config.Config.ClientID)

	zookeepers := strings.Split(custConfig.DestinationZookeepers, ",")
	kz, err := kazoo.NewKazoo(zookeepers, config.Zookeeper)
	if err != nil {
		log.Printf("GetKafkaProducer - Unable to create zookeeper object: %v\n", err)
		return nil, err
	}

	// Create new consumer
	brokerList, bErr := kz.BrokerList()
	if bErr != nil {
		log.Printf("GetKafkaProducer - Unable to get broker list: %v\n", err)
		return nil, err
	}

	return sarama.NewAsyncProducer(brokerList, config.Config)
}

// PullFromTopic pulls messages from the topic partition
func PullFromTopic(consumer *consumergroup.ConsumerGroup, producer chan<- sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()

	for consumerMsg := range consumer.Messages() {
		producerMsg := sarama.ProducerMessage{
			Topic: consumerMsg.Topic,
			Key:   sarama.StringEncoder(consumerMsg.Key),
			Value: sarama.StringEncoder(consumerMsg.Value),
		}
		producer <- producerMsg
		consumer.CommitUpto(consumerMsg)
	}
	// 	// for {
	// 	// 	if len(signals) > 0 {
	// 	// 		log.Println("Consumer - Interrupt is detected - exiting")
	// 	// 		return
	// 	// 	}
	// 	// 	select {
	// 	// 	case err := <-consumer.Errors():
	// 	// 		log.Println(err)
	// 	// 		return
	// 	// 	case consumerMsg := <-consumer.Messages():
	// 	// 		producerMsg := sarama.ProducerMessage{
	// 	// 			Topic:     consumerMsg.Topic,
	// 	// 			Partition: consumerMsg.Partition,
	// 	// 			Key:       sarama.StringEncoder(consumerMsg.Key),
	// 	// 			Value:     sarama.StringEncoder(consumerMsg.Value),
	// 	// 		}
	// 	//
	// 	//
	// 	// 		if finalOffset > 0 && consumerMsg.Offset >= finalOffset {
	// 	// 			syncChan <- consumerMsg.Offset
	// 	// 			log.Println("Consumer - partition ", consumerMsg.Partition, " reached final offset, shutting down partition")
	// 	// 			return
	// 	// 		}
	// 	// 	}
	// 	// }
}

// PushToTopic pushes messages to topic
func PushToTopic(producer sarama.AsyncProducer, consumer <-chan sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()

	success := 1

	for {
		select {
		case err := <-producer.Errors():
			log.Println("Failed to produce message", err)
		case <-producer.Successes():
			success++
		case consumerMsg := <-consumer:
			producer.Input() <- &consumerMsg
		}
	}
}

// MonitorChan monitors the transfer channel
func MonitorChan(transferChan chan sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		log.Println("Transfer channel length: ", len(transferChan))
		time.Sleep(10 * time.Second)
	}
}
<commit_msg>use sync producer<commit_after>// Copyright 2016 Kochava
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/wvanbergen/kafka/consumergroup"
	"github.com/wvanbergen/kazoo-go"
)

// GetKafkaConsumer returns a new consumer
func GetKafkaConsumer(custConfig *Config) (*consumergroup.ConsumerGroup, error) {
	groupConfig := consumergroup.NewConfig()
	groupConfig.Offsets.Initial = sarama.OffsetNewest
	groupConfig.Offsets.ProcessingTimeout = 10 * time.Second

	zookeeperNodes := strings.Split(custConfig.SourceZookeepers, ",")
	groupConfig.Zookeeper.Chroot = ""

	topic := []string{custConfig.Topic}

	consumer, err := consumergroup.JoinConsumerGroup("firehose", topic, zookeeperNodes, groupConfig)
	if err != nil {
		log.Printf("GetKafkaConsumer - Failed to join consumer group: %v\n", err)
		return nil, err
	}

	// Create new consumer
	return consumer, nil

}

// GetConsumerErrors logs any consumer errors
func GetConsumerErrors(consumer *consumergroup.ConsumerGroup) {
	for err := range consumer.Errors() {
		log.Printf("GetConsumerErrors - %v\n", err)
	}
}

// GetKafkaProducer returns a new consumer
func GetKafkaProducer(custConfig *Config) (sarama.SyncProducer, error) {
	config := consumergroup.NewConfig()

	config.Config.Producer.Retry.Max = 1
	config.Config.Producer.RequiredAcks = sarama.WaitForLocal
	// config.Producer.Partitioner = sarama.NewManualPartitioner
	config.Config.ClientID = "firehose_realtime"

	log.Printf("GetKafkaProducer - client id %v\n", config.Config.ClientID)

	zookeepers := strings.Split(custConfig.DestinationZookeepers, ",")
	kz, err := kazoo.NewKazoo(zookeepers, config.Zookeeper)
	if err != nil {
		log.Printf("GetKafkaProducer - Unable to create zookeeper object: %v\n", err)
		return nil, err
	}

	// Create new consumer
	brokerList, bErr := kz.BrokerList()
	if bErr != nil {
		log.Printf("GetKafkaProducer - Unable to get broker list: %v\n", err)
		return nil, err
	}

	return sarama.NewSyncProducer(brokerList, config.Config)
}

// PullFromTopic pulls messages from the topic partition
func PullFromTopic(consumer *consumergroup.ConsumerGroup, producer chan<- sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()

	for consumerMsg := range consumer.Messages() {
		producerMsg := sarama.ProducerMessage{
			Topic: consumerMsg.Topic,
			Key:   sarama.StringEncoder(consumerMsg.Key),
			Value: sarama.StringEncoder(consumerMsg.Value),
		}
		producer <- producerMsg
		consumer.CommitUpto(consumerMsg)
	}
	// 	// for {
	// 	// 	if len(signals) > 0 {
	// 	// 		log.Println("Consumer - Interrupt is detected - exiting")
	// 	// 		return
	// 	// 	}
	// 	// 	select {
	// 	// 	case err := <-consumer.Errors():
	// 	// 		log.Println(err)
	// 	// 		return
	// 	// 	case consumerMsg := <-consumer.Messages():
	// 	// 		producerMsg := sarama.ProducerMessage{
	// 	// 			Topic:     consumerMsg.Topic,
	// 	// 			Partition: consumerMsg.Partition,
	// 	// 			Key:       sarama.StringEncoder(consumerMsg.Key),
	// 	// 			Value:     sarama.StringEncoder(consumerMsg.Value),
	// 	// 		}
	// 	//
	// 	//
	// 	// 		if finalOffset > 0 && consumerMsg.Offset >= finalOffset {
	// 	// 			syncChan <- consumerMsg.Offset
	// 	// 			log.Println("Consumer - partition ", consumerMsg.Partition, " reached final offset, shutting down partition")
	// 	// 			return
	// 	// 		}
	// 	// 	}
	// 	// }
}

// PushToTopic pushes messages to topic
func PushToTopic(producer sarama.SyncProducer, consumer <-chan sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		// case err := <-producer.Errors():
		// 	log.Println("Failed to produce message", err)
		// case <-producer.Successes():
		// 	success++
		case consumerMsg := <-consumer:
			partition, offset, err := producer.SendMessage(&consumerMsg)
			if err != nil {
				log.Printf("FAILED to send message: %s\n", err)
			} else {
				log.Printf("> message sent to partition %d at offset %d\n", partition, offset)
			}
		}
	}
}

// MonitorChan monitors the transfer channel
func MonitorChan(transferChan chan sarama.ProducerMessage, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		log.Println("Transfer channel length: ", len(transferChan))
		time.Sleep(10 * time.Second)
	}
}
<|endoftext|>


<commit_before>package collection

import "fmt"
import "log"
import "github.com/gosuri/uitable"

import "github.com/nlewo/contrail-introspect-cli/utils"

// Several representations of resources
type Shower interface {
	Long()
	Short()
	Xml()
}

func (e Element) Xml() {
	fmt.Printf("%s", e.Node)
}
func (elts Elements) Xml() {
	for _, e := range elts {
		e.Xml()
	}
}
func (c Collection) Xml() {
	fmt.Printf("%s", c.rootNode)
}

func (e Element) Short() {
	if e.desc.ShortDetailXpath == "" {
		log.Fatal("ShortDetailXpath is not defined. Use XML output instead (-x option)")
		return
	}
	s, _ := e.Node.Search(e.desc.ShortDetailXpath)
	if len(s) != 1 {
		log.Fatal("Xpath '" + e.desc.ShortDetailXpath + "' is not valid (check your ShortDetailXpath definition)")
	}
	fmt.Printf("%s\n", s[0])
}
func (col Collection) Short() {
	Elements(col.elements).Short()
}
func (elts Elements) Short() {
	for _, e := range elts {
		e.Short()
	}
}
func (e Element) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	e.desc.LongDetail.LongFormat(table, FORMAT_TEXT, e)
	fmt.Println(table)
}
func (col Collection) Long() {
	Elements(col.elements).Long()
}
func (elts Elements) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	for i, e := range elts {
		format := FORMAT_TABLE
		if i == 0 {
			format = FORMAT_TABLE_HEADER
		}
		e.desc.LongDetail.LongFormat(table, format, e)
	}
	fmt.Println(table)
}

// This is used to show the long version of an Element.
type LongFormatter interface {
	LongFormat(t *uitable.Table, f Format, e Element)
}

type LongFormatFn (func(*uitable.Table, Element))
type LongFormatXpaths []string

type Format uint8

const (
	FORMAT_TEXT         Format = 1
	FORMAT_TABLE_HEADER Format = 2
	FORMAT_TABLE        Format = 3
)

func (fn LongFormatFn) LongFormat(table *uitable.Table, format Format, e Element) {
	fn(table, e)
}

func (xpaths LongFormatXpaths) LongFormat(table *uitable.Table, format Format, e Element) {
	if format == FORMAT_TABLE_HEADER || format == FORMAT_TABLE {
		longFormatTable(table, format, e, xpaths)
	} else {
		for _, xpath := range xpaths {
			s, _ := e.Node.Search(xpath + "/text()")
			if len(s) == 1 {
				table.AddRow(utils.Pretty(s))
			}
		}
	}
}

func longFormatTable(table *uitable.Table, format Format, e Element, xpaths LongFormatXpaths) {
	if format == FORMAT_TABLE_HEADER {
		tmp := make([]interface{}, len(xpaths))
		for i, v := range xpaths {
			tmp[i] = v
		}
		table.AddRow(tmp...)
	}

	tmp := make([]interface{}, len(xpaths))
	for i, xpath := range xpaths {
		s, _ := e.Node.Search(xpath + "/text()")
		if len(s) == 1 {
			tmp[i] = utils.Pretty(s)
		}
	}
	table.AddRow(tmp...)
}
<commit_msg>Temporary remove debug message<commit_after>package collection

import "fmt"
import "log"
import "github.com/gosuri/uitable"

import "github.com/nlewo/contrail-introspect-cli/utils"

// Several representations of resources
type Shower interface {
	Long()
	Short()
	Xml()
}

func (e Element) Xml() {
	fmt.Printf("%s", e.Node)
}
func (elts Elements) Xml() {
	for _, e := range elts {
		e.Xml()
	}
}
func (c Collection) Xml() {
	fmt.Printf("%s", c.rootNode)
}

func (e Element) Short() {
	if e.desc.ShortDetailXpath == "" {
		log.Fatal("ShortDetailXpath is not defined. Use XML output instead (-x option)")
		return
	}
	s, _ := e.Node.Search(e.desc.ShortDetailXpath)
	if len(s) != 1 {
		return
		// TODO Debug mode
		//log.Fatal("Xpath '" + e.desc.ShortDetailXpath + "' is not valid (check your ShortDetailXpath definition)")
	}
	fmt.Printf("%s\n", s[0])
}
func (col Collection) Short() {
	Elements(col.elements).Short()
}
func (elts Elements) Short() {
	for _, e := range elts {
		e.Short()
	}
}
func (e Element) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	e.desc.LongDetail.LongFormat(table, FORMAT_TEXT, e)
	fmt.Println(table)
}
func (col Collection) Long() {
	Elements(col.elements).Long()
}
func (elts Elements) Long() {
	table := uitable.New()
	table.MaxColWidth = 80
	for i, e := range elts {
		format := FORMAT_TABLE
		if i == 0 {
			format = FORMAT_TABLE_HEADER
		}
		e.desc.LongDetail.LongFormat(table, format, e)
	}
	fmt.Println(table)
}

// This is used to show the long version of an Element.
type LongFormatter interface {
	LongFormat(t *uitable.Table, f Format, e Element)
}

type LongFormatFn (func(*uitable.Table, Element))
type LongFormatXpaths []string

type Format uint8

const (
	FORMAT_TEXT         Format = 1
	FORMAT_TABLE_HEADER Format = 2
	FORMAT_TABLE        Format = 3
)

func (fn LongFormatFn) LongFormat(table *uitable.Table, format Format, e Element) {
	fn(table, e)
}

func (xpaths LongFormatXpaths) LongFormat(table *uitable.Table, format Format, e Element) {
	if format == FORMAT_TABLE_HEADER || format == FORMAT_TABLE {
		longFormatTable(table, format, e, xpaths)
	} else {
		for _, xpath := range xpaths {
			s, _ := e.Node.Search(xpath + "/text()")
			if len(s) == 1 {
				table.AddRow(utils.Pretty(s))
			}
		}
	}
}

func longFormatTable(table *uitable.Table, format Format, e Element, xpaths LongFormatXpaths) {
	if format == FORMAT_TABLE_HEADER {
		tmp := make([]interface{}, len(xpaths))
		for i, v := range xpaths {
			tmp[i] = v
		}
		table.AddRow(tmp...)
	}

	tmp := make([]interface{}, len(xpaths))
	for i, xpath := range xpaths {
		s, _ := e.Node.Search(xpath + "/text()")
		if len(s) == 1 {
			tmp[i] = utils.Pretty(s)
		}
	}
	table.AddRow(tmp...)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"index/suffixarray"
	"regexp/syntax"
	"runtime"
	"sort"
	"sync"
)

type line struct {
	off   int    // Offset of line
	value []byte // One line from source slice
}

type delta struct {
	off   int // Offset of existing value
	index int // Index of replacement value
}

// TODO Are there any errors that need handling?
func producer(input <-chan line, index *replaceIndex) []delta {

	var wg sync.WaitGroup
	runtime.GOMAXPROCS(runtime.NumCPU()) // Maximum CPU utilisation please!
	maxGophers := runtime.GOMAXPROCS(0)
	wg.Add(maxGophers)

	deltaChan := make(chan []delta)
	done := make(chan bool)
	defer close(done)

	fmt.Printf("Launching %v gophers ...\n", maxGophers)

	for i := 0; i < maxGophers; i++ {
		go func() {
			for t := range input {
				select {
				case deltaChan <- makeDeltas(t, index, i):
				case <-done:
					return
				}
			}
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(deltaChan)
	}()

	fmt.Printf("Generating deltas... \n")
	deltas := make([]delta, 0)
	for d := range deltaChan {
		if len(d) > 0 {
			deltas = append(deltas, d...)
		}
	}

	fmt.Printf("Got %v deltas.\n", len(deltas))

	sort.Sort(ByLine(deltas))
	return deltas
}

// Make deltas
func makeDeltas(t line, index *replaceIndex, id int) []delta {

	s := make([]delta, 0)

	lineIndex := suffixarray.New(t.value)

	// Look for each replacement word in the line
	for i := 0; i < index.len(); i++ {
		results := lineIndex.Lookup(index.readItem(i).find, -1)
		if len(results) > 0 {
			for _, p := range results {

				// It's not a match if it's a partial word

				// TODO Is there a guarantee of left-to-right eval?
				// TODO Test this works and hasn't broken everything!!!
				// BUG Fails on matching last word of line with a terminating non-alpha, drops it
				// It's a full word match if it's end of line or word terminates in non-alpha
				if (p < len(t.value) && !syntax.IsWordChar(rune(t.value[p+len(index.readItem(i).find)])+1)) || (p == len(t.value)) {
					d := delta{off: t.off + p, index: i}
					s = append(s, d)
				}
			}
		}
	}

	// TODO Optional memory optimiser. Use less system RAM at the expense of CPU.
	// debug.FreeOSMemory()

	return s
}

type ByLine []delta

func (d ByLine) Len() int {
	return len(d)
}

func (d ByLine) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d ByLine) Less(i, j int) bool {
	return d[i].off < d[j].off
}
<commit_msg>Cleaned up comments.<commit_after>package main

import (
	"fmt"
	"index/suffixarray"
	"regexp/syntax"
	"runtime"
	"sort"
	"sync"
)

type line struct {
	off   int    // Offset of line
	value []byte // One line from source slice
}

type delta struct {
	off   int // Offset of existing value
	index int // Index of replacement value
}

// TODO Are there any errors that need handling?
func producer(input <-chan line, index *replaceIndex) []delta {

	var wg sync.WaitGroup
	runtime.GOMAXPROCS(runtime.NumCPU()) // Maximum CPU utilisation please!
	maxGophers := runtime.GOMAXPROCS(0)
	wg.Add(maxGophers)

	deltaChan := make(chan []delta)
	done := make(chan bool)
	defer close(done)

	fmt.Printf("Launching %v gophers ...\n", maxGophers)

	for i := 0; i < maxGophers; i++ {
		go func() {
			for t := range input {
				select {
				case deltaChan <- makeDeltas(t, index, i):
				case <-done:
					return
				}
			}
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(deltaChan)
	}()

	fmt.Printf("Generating deltas... \n")
	deltas := make([]delta, 0)
	for d := range deltaChan {
		if len(d) > 0 {
			deltas = append(deltas, d...)
		}
	}

	fmt.Printf("Got %v deltas.\n", len(deltas))

	sort.Sort(ByLine(deltas))
	return deltas
}

// Make deltas
func makeDeltas(t line, index *replaceIndex, id int) []delta {

	s := make([]delta, 0)

	lineIndex := suffixarray.New(t.value)

	// Look for each replacement word in the line
	for i := 0; i < index.len(); i++ {

		// Find this word in the line
		results := lineIndex.Lookup(index.readItem(i).find, -1)
		if len(results) > 0 {
			for _, p := range results {

				// Is it a full word match or not?
				if (p < len(t.value) && !syntax.IsWordChar(rune(t.value[p+len(index.readItem(i).find)])+1)) || (p == len(t.value)) {
					d := delta{off: t.off + p, index: i}
					s = append(s, d)
				}
			}
		}
	}

	// TODO Optional memory optimiser. Use less system RAM at the expense of CPU.
	// debug.FreeOSMemory()

	return s
}

type ByLine []delta

func (d ByLine) Len() int {
	return len(d)
}

func (d ByLine) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d ByLine) Less(i, j int) bool {
	return d[i].off < d[j].off
}
<|endoftext|>