<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
)

// A utility program for generating `project` and `variable` files for SemmleCore Go projects
//
// This program should not normally be run directly; it is usually executed as part of
// `odasa bootstrap`, and expects two files as arguments: a (partial) `variables` file and
// an empty file to be filled in with an `<autoupdate>` element containing build steps.
//
// The `variables` file is extended with a definition of `LGTM_SRC` and, if it defines the
// `repository` variable, `SEMMLE_REPO_URL`. The only build step is an invocation of the
// Go autobuilder.
func main() {
	vars := os.Args[1]
	buildSteps := os.Args[2]

	haveRepo := false
	content, err := ioutil.ReadFile(vars)
	if err != nil {
		log.Fatal(err)
	}
	re := regexp.MustCompile(`(^|\n)repository=`)
	haveRepo = re.Find(content) != nil

	additionalVars := "LGTM_SRC=${src}\n"
	if haveRepo {
		additionalVars += "SEMMLE_REPO_URL=${repository}\n"
	}
	content = append(content, []byte(additionalVars)...)
	err = ioutil.WriteFile(vars, content, 0644)
	if err != nil {
		log.Fatal(err)
	}

	export := "LGTM_SRC"
	if haveRepo {
		export += ",SEMMLE_REPO_URL"
	}
	content = []byte(fmt.Sprintf(`<autoupdate>
  <build export="%s">${semmle_dist}/language-packs/go/tools/platform/${semmle_platform}/bin/go-autobuilder</build>
</autoupdate>
`, export))
	err = ioutil.WriteFile(buildSteps, content, 0644)
}
<commit_msg>Fix a dead assignment.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
)

// A utility program for generating `project` and `variable` files for SemmleCore Go projects
//
// This program should not normally be run directly; it is usually executed as part of
// `odasa bootstrap`, and expects two files as arguments: a (partial) `variables` file and
// an empty file to be filled in with an `<autoupdate>` element containing build steps.
//
// The `variables` file is extended with a definition of `LGTM_SRC` and, if it defines the
// `repository` variable, `SEMMLE_REPO_URL`. The only build step is an invocation of the
// Go autobuilder.
func main() {
	vars := os.Args[1]
	buildSteps := os.Args[2]

	haveRepo := false
	content, err := ioutil.ReadFile(vars)
	if err != nil {
		log.Fatal(err)
	}
	re := regexp.MustCompile(`(^|\n)repository=`)
	haveRepo = re.Find(content) != nil

	additionalVars := "LGTM_SRC=${src}\n"
	if haveRepo {
		additionalVars += "SEMMLE_REPO_URL=${repository}\n"
	}
	content = append(content, []byte(additionalVars)...)
	err = ioutil.WriteFile(vars, content, 0644)
	if err != nil {
		log.Fatal(err)
	}

	export := "LGTM_SRC"
	if haveRepo {
		export += ",SEMMLE_REPO_URL"
	}
	content = []byte(fmt.Sprintf(`<autoupdate>
  <build export="%s">${semmle_dist}/language-packs/go/tools/platform/${semmle_platform}/bin/go-autobuilder</build>
</autoupdate>
`, export))
	err = ioutil.WriteFile(buildSteps, content, 0644)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * hprose/websocket_service.go                            *
 *                                                        *
 * hprose websocket service for Go.                       *
 *                                                        *
 * LastModified: May 22, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package hprose

import (
	"io"
	"net/http"
	"reflect"
	"strings"

	"github.com/gorilla/websocket"
)

// WebSocketContext is the hprose websocket context
type WebSocketContext struct {
	*HttpContext
	WebSocket *websocket.Conn
}

// WebSocketService is the hprose websocket service
type WebSocketService struct {
	*HttpService
	*websocket.Upgrader
}

type wsArgsFixer struct {
	httpArgsFixer
}

func (fixer wsArgsFixer) FixArgs(args []reflect.Value, lastParamType reflect.Type, context Context) []reflect.Value {
	if c, ok := context.(*WebSocketContext); ok {
		if lastParamType.String() == "*hprose.WebSocketContext" {
			return append(args, reflect.ValueOf(c))
		} else if lastParamType.String() == "*websocket.Conn" {
			return append(args, reflect.ValueOf(c.WebSocket))
		} else if lastParamType.String() == "*hprose.HttpContext" {
			return append(args, reflect.ValueOf(c.HttpContext))
		} else if lastParamType.String() == "*http.Request" {
			return append(args, reflect.ValueOf(c.Request))
		}
	}
	return fixer.httpArgsFixer.FixArgs(args, lastParamType, context)
}

// NewWebSocketService is the constructor of WebSocketService
func NewWebSocketService() *WebSocketService {
	service := &WebSocketService{HttpService: NewHttpService()}
	service.argsfixer = wsArgsFixer{}
	service.Upgrader = &websocket.Upgrader{
		CheckOrigin: func(r *http.Request) bool {
			origin := r.Header.Get("origin")
			if origin != "" && origin != "null" {
				if len(service.accessControlAllowOrigins) == 0 || service.accessControlAllowOrigins[origin] {
					return true
				}
				return false
			}
			return true
		},
	}
	return service
}

// ServeHTTP ...
func (service *WebSocketService) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if request.Method == "GET" && strings.ToLower(request.Header.Get("connection")) != "upgrade" || request.Method == "POST" {
		service.HttpService.ServeHTTP(response, request)
		return
	}
	conn, err := service.Upgrade(response, request, nil)
	if err != nil {
		context := &HttpContext{BaseContext: NewBaseContext(), Response: response, Request: request}
		service.fireErrorEvent(err, context)
		return
	}
	defer conn.Close()
	for {
		context := &WebSocketContext{HttpContext: &HttpContext{BaseContext: NewBaseContext(), Response: response, Request: request}, WebSocket: conn}
		_, data, err := conn.ReadMessage()
		if err != nil {
			if err != io.EOF {
				service.fireErrorEvent(err, context)
			}
			break
		}
		go func(conn *websocket.Conn, data []byte, context *WebSocketContext) {
			id := data[0:4]
			data = append(id, service.Handle(data[4:], context)...)
			if err := conn.WriteMessage(websocket.BinaryMessage, data); err != nil {
				service.fireErrorEvent(err, context)
				conn.Close()
			}
		}(conn, data, context)
	}
}
<commit_msg>Added lock for WriteMessage<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * hprose/websocket_service.go                            *
 *                                                        *
 * hprose websocket service for Go.                       *
 *                                                        *
 * LastModified: May 22, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package hprose

import (
	"io"
	"net/http"
	"reflect"
	"strings"
	"sync"

	"github.com/gorilla/websocket"
)

// WebSocketContext is the hprose websocket context
type WebSocketContext struct {
	*HttpContext
	WebSocket *websocket.Conn
}

// WebSocketService is the hprose websocket service
type WebSocketService struct {
	*HttpService
	*websocket.Upgrader
}

type wsArgsFixer struct {
	httpArgsFixer
}

func (fixer wsArgsFixer) FixArgs(args []reflect.Value, lastParamType reflect.Type, context Context) []reflect.Value {
	if c, ok := context.(*WebSocketContext); ok {
		if lastParamType.String() == "*hprose.WebSocketContext" {
			return append(args, reflect.ValueOf(c))
		} else if lastParamType.String() == "*websocket.Conn" {
			return append(args, reflect.ValueOf(c.WebSocket))
		} else if lastParamType.String() == "*hprose.HttpContext" {
			return append(args, reflect.ValueOf(c.HttpContext))
		} else if lastParamType.String() == "*http.Request" {
			return append(args, reflect.ValueOf(c.Request))
		}
	}
	return fixer.httpArgsFixer.FixArgs(args, lastParamType, context)
}

// NewWebSocketService is the constructor of WebSocketService
func NewWebSocketService() *WebSocketService {
	service := &WebSocketService{HttpService: NewHttpService()}
	service.argsfixer = wsArgsFixer{}
	service.Upgrader = &websocket.Upgrader{
		CheckOrigin: func(r *http.Request) bool {
			origin := r.Header.Get("origin")
			if origin != "" && origin != "null" {
				if len(service.accessControlAllowOrigins) == 0 || service.accessControlAllowOrigins[origin] {
					return true
				}
				return false
			}
			return true
		},
	}
	return service
}

// ServeHTTP ...
func (service *WebSocketService) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if request.Method == "GET" && strings.ToLower(request.Header.Get("connection")) != "upgrade" || request.Method == "POST" {
		service.HttpService.ServeHTTP(response, request)
		return
	}
	conn, err := service.Upgrade(response, request, nil)
	if err != nil {
		context := &HttpContext{BaseContext: NewBaseContext(), Response: response, Request: request}
		service.fireErrorEvent(err, context)
		return
	}
	defer conn.Close()
	mutex := new(sync.Mutex)
	for {
		context := &WebSocketContext{HttpContext: &HttpContext{BaseContext: NewBaseContext(), Response: response, Request: request}, WebSocket: conn}
		_, data, err := conn.ReadMessage()
		if err != nil {
			if err != io.EOF {
				service.fireErrorEvent(err, context)
			}
			break
		}
		go func(conn *websocket.Conn, data []byte, context *WebSocketContext) {
			id := data[0:4]
			data = append(id, service.Handle(data[4:], context)...)
			err := func() error {
				mutex.Lock()
				defer mutex.Unlock()
				return conn.WriteMessage(websocket.BinaryMessage, data)
			}()
			if err != nil {
				service.fireErrorEvent(err, context)
				conn.Close()
			}
		}(conn, data, context)
	}
}
<|endoftext|>


<commit_before>// Package cache implements a cache that supports single-flight value computation.
//
// Single-flight value computation means that for concurrent callers of the cache, only one caller
// will compute the value to avoid redundant work which could be system intensive.
package cache

import (
	"sync"
)

// SingleFlight is a cache that supports single-flight value computation.
type SingleFlight struct {
	mu    sync.RWMutex // protects `store` field itself
	store sync.Map
}

type entry struct {
	compute sync.Once
	val     interface{}
	err     error
}

// LoadOrStore is similar to a sync.Map except that it receives a function that computes the value
// to store instead of the value directly. It ensures that the function is only executed once for
// concurrent callers of the LoadOrStore function.
func (s *SingleFlight) LoadOrStore(key interface{}, valFn func() (interface{}, error)) (interface{}, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	eUntyped, _ := s.store.LoadOrStore(key, &entry{})
	e := eUntyped.(*entry)
	e.compute.Do(func() {
		e.val, e.err = valFn()
	})
	return e.val, e.err
}

// Load is similar to a sync.Map.Load.
func (s *SingleFlight) Load(key interface{}) (val interface{}, err error, loaded bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	var eUntyped interface{}
	eUntyped, loaded = s.store.Load(key)
	if loaded {
		e := eUntyped.(*entry)
		val = e.val
		err = e.err
	}
	return
}

// Store forcefully updates the given cache key with val. Note that unlike LoadOrStore,
// Store accepts a value instead of a valFn since it is intended to be only used in
// cases where updates are lightweight and do not involve computing the cache value.
func (s *SingleFlight) Store(key interface{}, val interface{}) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	e := &entry{val: val}
	e.compute.Do(func() {}) // mark as computed
	s.store.Store(key, e)
}

// Delete removes a key from the cache.
func (s *SingleFlight) Delete(key interface{}) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	s.store.Delete(key)
}

// Reset invalidates all cache entries.
func (s *SingleFlight) Reset() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.store = sync.Map{}
}
<commit_msg>Add return value names in singleflightcache (#356)<commit_after>// Package cache implements a cache that supports single-flight value computation.
//
// Single-flight value computation means that for concurrent callers of the cache, only one caller
// will compute the value to avoid redundant work which could be system intensive.
package cache

import (
	"sync"
)

// SingleFlight is a cache that supports single-flight value computation.
type SingleFlight struct {
	mu    sync.RWMutex // protects `store` field itself
	store sync.Map
}

type entry struct {
	compute sync.Once
	val     interface{}
	err     error
}

// LoadOrStore is similar to a sync.Map except that it receives a function that computes the value
// to store instead of the value directly. It ensures that the function is only executed once for
// concurrent callers of the LoadOrStore function.
func (s *SingleFlight) LoadOrStore(key interface{}, valFn func() (val interface{}, err error)) (interface{}, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	eUntyped, _ := s.store.LoadOrStore(key, &entry{})
	e := eUntyped.(*entry)
	e.compute.Do(func() {
		e.val, e.err = valFn()
	})
	return e.val, e.err
}

// Load is similar to a sync.Map.Load.
func (s *SingleFlight) Load(key interface{}) (val interface{}, err error, loaded bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	var eUntyped interface{}
	eUntyped, loaded = s.store.Load(key)
	if loaded {
		e := eUntyped.(*entry)
		val = e.val
		err = e.err
	}
	return
}

// Store forcefully updates the given cache key with val. Note that unlike LoadOrStore,
// Store accepts a value instead of a valFn since it is intended to be only used in
// cases where updates are lightweight and do not involve computing the cache value.
func (s *SingleFlight) Store(key interface{}, val interface{}) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	e := &entry{val: val}
	e.compute.Do(func() {}) // mark as computed
	s.store.Store(key, e)
}

// Delete removes a key from the cache.
func (s *SingleFlight) Delete(key interface{}) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	s.store.Delete(key)
}

// Reset invalidates all cache entries.
func (s *SingleFlight) Reset() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.store = sync.Map{}
}
<|endoftext|>


<commit_before>package request

import (
	"math"
	"net/url"
	"strconv"
	"time"

	"github.com/kennygrant/sanitize"
)

const (
	DEFAULT_REPLY_LIMIT = 3
	MAX_REPLY_LIMIT     = 25
	MAX_LIMIT           = 25
)

type Query struct {
	Id             int64     `url:"id,omitempty"`
	Skip           int       `url:"skip"`
	Limit          int       `url:"limit"`
	To             time.Time `url:"to"`
	From           time.Time `url:"from"`
	GroupName      string    `url:"groupName"`
	GroupChannelId int64     `url:"groupChannelId"`
	Type           string    `url:"type"`
	Privacy        string    `url:"privacy"`
	AccountId      int64     `url:"accountId"`
	Name           string    `url:"name"`
	Slug           string    `url:"slug"`
	ShowExempt     bool      `url:"showExempt"`
	ReplyLimit     int       `url:"replyLimit"`
	ReplySkip      int       `url:"replySkip"`
}

func NewQuery() *Query {
	return &Query{}
}

func (q *Query) MapURL(u *url.URL) *Query {
	urlQuery := u.Query()

	q.Skip, _ = strconv.Atoi(urlQuery.Get("skip"))
	q.Limit, _ = strconv.Atoi(urlQuery.Get("limit"))

	q.Name = urlQuery.Get("name")
	if q.Name != "" {
		q.Name = sanitize.Name(q.Name)
	}

	q.Slug = urlQuery.Get("slug")
	if q.Slug != "" {
		q.Slug = sanitize.Name(q.Slug)
	}

	q.GroupName = urlQuery.Get("groupName")
	if q.GroupName != "" {
		q.GroupName = sanitize.Name(q.GroupName)
	}

	q.Type = urlQuery.Get("type")
	if q.Type != "" {
		q.Type = sanitize.Name(q.Type)
	}

	q.Privacy = urlQuery.Get("privacy")
	if q.Privacy != "" {
		q.Privacy = sanitize.Name(q.Privacy)
	}

	q.Id, _ = GetURIInt64(u, "id")
	q.AccountId, _ = GetURIInt64(u, "accountId")
	q.GroupChannelId, _ = GetURIInt64(u, "groupChannelId")

	if to := urlQuery.Get("to"); to != "" {
		q.To, _ = time.Parse(time.RFC3339, to)
	}

	if from := urlQuery.Get("from"); from != "" {
		q.From, _ = time.Parse(time.RFC3339, from)
	}

	if showExempt := urlQuery.Get("showExempt"); showExempt != "" {
		isExempt, _ := strconv.ParseBool(showExempt)
		q.ShowExempt = isExempt
	}

	if replyLimit := urlQuery.Get("replyLimit"); replyLimit != "" {
		replyLimit, _ := strconv.Atoi(replyLimit)
		q.ReplyLimit = replyLimit
	}

	if replySkip := urlQuery.Get("replySkip"); replySkip != "" {
		replySkip, _ := strconv.Atoi(replySkip)
		q.ReplySkip = replySkip
	}

	return q
}

func (q *Query) Clone() *Query {
	cq := NewQuery()
	*cq = *q

	return cq
}

func (q *Query) SetDefaults() *Query {
	if q.Skip == 0 {
		// no need to do something
	}

	if q.Limit == 0 || q.Limit > MAX_LIMIT {
		q.Limit = MAX_LIMIT
	}

	if q.From.IsZero() {
		q.From = time.Now().UTC()
	}

	if q.GroupName == "" {
		q.GroupName = "koding"
	}

	if q.ReplyLimit == 0 {
		q.ReplyLimit = DEFAULT_REPLY_LIMIT
	} else {
		q.ReplyLimit = int(math.Min(float64(q.ReplyLimit), float64(MAX_REPLY_LIMIT)))
	}

	return q
}
<commit_msg>Social: added `AddIsInteracted` property<commit_after>package request

import (
	"math"
	"net/url"
	"strconv"
	"time"

	"github.com/kennygrant/sanitize"
)

const (
	DEFAULT_REPLY_LIMIT = 3
	MAX_REPLY_LIMIT     = 25
	MAX_LIMIT           = 25
)

type Query struct {
	Id              int64     `url:"id,omitempty"`
	Skip            int       `url:"skip"`
	Limit           int       `url:"limit"`
	To              time.Time `url:"to"`
	From            time.Time `url:"from"`
	GroupName       string    `url:"groupName"`
	GroupChannelId  int64     `url:"groupChannelId"`
	Type            string    `url:"type"`
	Privacy         string    `url:"privacy"`
	AccountId       int64     `url:"accountId"`
	Name            string    `url:"name"`
	Slug            string    `url:"slug"`
	ShowExempt      bool      `url:"showExempt"`
	ReplyLimit      int       `url:"replyLimit"`
	ReplySkip       int       `url:"replySkip"`
	AddIsInteracted bool      `url:"addIsInteracted"`
}

func NewQuery() *Query {
	return &Query{}
}

func (q *Query) MapURL(u *url.URL) *Query {
	urlQuery := u.Query()

	q.Skip, _ = strconv.Atoi(urlQuery.Get("skip"))
	q.Limit, _ = strconv.Atoi(urlQuery.Get("limit"))

	q.Name = urlQuery.Get("name")
	if q.Name != "" {
		q.Name = sanitize.Name(q.Name)
	}

	q.Slug = urlQuery.Get("slug")
	if q.Slug != "" {
		q.Slug = sanitize.Name(q.Slug)
	}

	q.GroupName = urlQuery.Get("groupName")
	if q.GroupName != "" {
		q.GroupName = sanitize.Name(q.GroupName)
	}

	q.Type = urlQuery.Get("type")
	if q.Type != "" {
		q.Type = sanitize.Name(q.Type)
	}

	q.Privacy = urlQuery.Get("privacy")
	if q.Privacy != "" {
		q.Privacy = sanitize.Name(q.Privacy)
	}

	q.Id, _ = GetURIInt64(u, "id")
	q.AccountId, _ = GetURIInt64(u, "accountId")
	q.GroupChannelId, _ = GetURIInt64(u, "groupChannelId")

	if to := urlQuery.Get("to"); to != "" {
		q.To, _ = time.Parse(time.RFC3339, to)
	}

	if from := urlQuery.Get("from"); from != "" {
		q.From, _ = time.Parse(time.RFC3339, from)
	}

	if showExempt := urlQuery.Get("showExempt"); showExempt != "" {
		isExempt, _ := strconv.ParseBool(showExempt)
		q.ShowExempt = isExempt
	}

	if addIsInteracted := urlQuery.Get("addIsInteracted"); addIsInteracted != "" {
		addIsInteracted, _ := strconv.ParseBool(addIsInteracted)
		q.AddIsInteracted = addIsInteracted
	}

	if replyLimit := urlQuery.Get("replyLimit"); replyLimit != "" {
		replyLimit, _ := strconv.Atoi(replyLimit)
		q.ReplyLimit = replyLimit
	}

	if replySkip := urlQuery.Get("replySkip"); replySkip != "" {
		replySkip, _ := strconv.Atoi(replySkip)
		q.ReplySkip = replySkip
	}

	return q
}

func (q *Query) Clone() *Query {
	cq := NewQuery()
	*cq = *q

	return cq
}

func (q *Query) SetDefaults() *Query {
	if q.Skip == 0 {
		// no need to do something
	}

	if q.Limit == 0 || q.Limit > MAX_LIMIT {
		q.Limit = MAX_LIMIT
	}

	if q.From.IsZero() {
		q.From = time.Now().UTC()
	}

	if q.GroupName == "" {
		q.GroupName = "koding"
	}

	if q.ReplyLimit == 0 {
		q.ReplyLimit = DEFAULT_REPLY_LIMIT
	} else {
		q.ReplyLimit = int(math.Min(float64(q.ReplyLimit), float64(MAX_REPLY_LIMIT)))
	}

	q.AddIsInteracted = true

	return q
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mysqlctl

import (
	"fmt"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/rpcwrap"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/dbconfigs"
	"github.com/youtube/vitess/go/vt/mysqlctl/proto"
)

/* API and config for UpdateStream Service */

const (
	DISABLED = iota
	ENABLED
)

type UpdateStream struct {
	mycnf *Mycnf

	actionLock     sync.Mutex
	state          sync2.AtomicInt64
	states         *stats.States
	mysqld         *Mysqld
	stateWaitGroup sync.WaitGroup
	dbname         string
	streams        streamList
}

type streamer interface {
	Stop()
}

type streamList struct {
	sync.Mutex
	streams map[streamer]bool
}

func (sl *streamList) Init() {
	sl.Lock()
	sl.streams = make(map[streamer]bool)
	sl.Unlock()
}

func (sl *streamList) Add(e streamer) {
	sl.Lock()
	sl.streams[e] = true
	sl.Unlock()
}

func (sl *streamList) Delete(e streamer) {
	sl.Lock()
	delete(sl.streams, e)
	sl.Unlock()
}

func (sl *streamList) Stop() {
	sl.Lock()
	for stream := range sl.streams {
		stream.Stop()
	}
	sl.Unlock()
}

var UpdateStreamRpcService *UpdateStream

func RegisterUpdateStreamService(mycnf *Mycnf) {
	if UpdateStreamRpcService != nil {
		panic("Update Stream service already initialized")
	}

	UpdateStreamRpcService = &UpdateStream{mycnf: mycnf}
	UpdateStreamRpcService.states = stats.NewStates("UpdateStreamState", []string{
		"Disabled",
		"Enabled",
	}, time.Now(), DISABLED)
	rpcwrap.RegisterAuthenticated(UpdateStreamRpcService)
}

func logError() {
	if x := recover(); x != nil {
		log.Errorf("%s", x.(error).Error())
	}
}

func EnableUpdateStreamService(dbcfgs dbconfigs.DBConfigs) {
	defer logError()
	UpdateStreamRpcService.enable(dbcfgs)
}

func DisableUpdateStreamService() {
	defer logError()
	UpdateStreamRpcService.disable()
}

func ServeUpdateStream(req *BinlogPosition, sendReply func(reply interface{}) error) error {
	return UpdateStreamRpcService.ServeUpdateStream(req, sendReply)
}

func IsUpdateStreamEnabled() bool {
	return UpdateStreamRpcService.isEnabled()
}

func GetReplicationPosition() (*proto.ReplicationCoordinates, error) {
	return UpdateStreamRpcService.getReplicationPosition()
}

func (updateStream *UpdateStream) enable(dbcfgs dbconfigs.DBConfigs) {
	updateStream.actionLock.Lock()
	defer updateStream.actionLock.Unlock()
	if updateStream.isEnabled() {
		return
	}

	if dbcfgs.Dba.UnixSocket == "" {
		log.Errorf("Missing dba socket connection, cannot enable update stream service")
		return
	}

	if dbcfgs.App.DbName == "" {
		log.Errorf("Missing db name, cannot enable update stream service")
		return
	}

	if updateStream.mycnf.BinLogPath == "" {
		log.Errorf("Update stream service requires binlogs enabled")
		return
	}

	updateStream.state.Set(ENABLED)
	updateStream.states.SetState(ENABLED)
	updateStream.mysqld = NewMysqld(updateStream.mycnf, dbcfgs.Dba, dbcfgs.Repl)
	updateStream.dbname = dbcfgs.App.DbName
	updateStream.streams.Init()
	log.Infof("Enabling update stream, dbname: %s, binlogpath: %s", updateStream.dbname, updateStream.mycnf.BinLogPath)
}

func (updateStream *UpdateStream) disable() {
	updateStream.actionLock.Lock()
	defer updateStream.actionLock.Unlock()
	if !updateStream.isEnabled() {
		return
	}

	updateStream.state.Set(DISABLED)
	updateStream.states.SetState(DISABLED)
	updateStream.streams.Stop()
	updateStream.stateWaitGroup.Wait()
	log.Infof("Update Stream Disabled")
}

func (updateStream *UpdateStream) isEnabled() bool {
	return updateStream.state.Get() == ENABLED
}

func (updateStream *UpdateStream) ServeUpdateStream(req *BinlogPosition, sendReply func(reply interface{}) error) (err error) {
	defer func() {
		if x := recover(); x != nil {
			err = x.(error)
		}
	}()

	updateStream.actionLock.Lock()
	if !updateStream.isEnabled() {
		updateStream.actionLock.Unlock()
		log.Errorf("Unable to serve client request: Update stream service is not enabled")
		return fmt.Errorf("update stream service is not enabled")
	}
	updateStream.stateWaitGroup.Add(1)
	updateStream.actionLock.Unlock()
	defer updateStream.stateWaitGroup.Done()

	rp, err := updateStream.mysqld.BinlogInfo(req.GroupId, req.ServerId)
	if err != nil {
		return fmt.Errorf("error computing start position: %v", err)
	}
	log.Infof("ServeUpdateStream starting @ %v", rp)

	evs := NewEventStreamer(updateStream.dbname, updateStream.mycnf.BinLogPath)
	updateStream.streams.Add(evs)
	defer updateStream.streams.Delete(evs)

	return evs.Stream(rp.MasterLogFile, int64(rp.MasterLogPosition), func(reply *StreamEvent) error {
		return sendReply(reply)
	})
}

func (updateStream *UpdateStream) getReplicationPosition() (*proto.ReplicationCoordinates, error) {
	updateStream.actionLock.Lock()
	defer updateStream.actionLock.Unlock()
	if !updateStream.isEnabled() {
		return nil, fmt.Errorf("update stream service is not enabled")
	}

	rp, err := updateStream.mysqld.MasterStatus()
	if err != nil {
		return nil, err
	}
	return proto.NewReplicationCoordinates(rp.MasterLogFile, uint64(rp.MasterLogPosition), rp.MasterLogGroupId), nil
}
<commit_msg>UpdateStream.StreamKeyrange: new binlog server<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package mysqlctl

import (
	"fmt"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/rpcwrap"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/dbconfigs"
	"github.com/youtube/vitess/go/vt/key"
	"github.com/youtube/vitess/go/vt/mysqlctl/proto"
)

/* API and config for UpdateStream Service */

const (
	DISABLED = iota
	ENABLED
)

type UpdateStream struct {
	mycnf *Mycnf

	actionLock     sync.Mutex
	state          sync2.AtomicInt64
	states         *stats.States
	mysqld         *Mysqld
	stateWaitGroup sync.WaitGroup
	dbname         string
	streams        streamList
}

type KeyrangeRequest struct {
	StartPosition BinlogPosition
	Keyrange      key.KeyRange
}

type streamer interface {
	Stop()
}

type streamList struct {
	sync.Mutex
	streams map[streamer]bool
}

func (sl *streamList) Init() {
	sl.Lock()
	sl.streams = make(map[streamer]bool)
	sl.Unlock()
}

func (sl *streamList) Add(e streamer) {
	sl.Lock()
	sl.streams[e] = true
	sl.Unlock()
}

func (sl *streamList) Delete(e streamer) {
	sl.Lock()
	delete(sl.streams, e)
	sl.Unlock()
}

func (sl *streamList) Stop() {
	sl.Lock()
	for stream := range sl.streams {
		stream.Stop()
	}
	sl.Unlock()
}

var UpdateStreamRpcService *UpdateStream

func RegisterUpdateStreamService(mycnf *Mycnf) {
	if UpdateStreamRpcService != nil {
		panic("Update Stream service already initialized")
	}

	UpdateStreamRpcService = &UpdateStream{mycnf: mycnf}
	UpdateStreamRpcService.states = stats.NewStates("UpdateStreamState", []string{
		"Disabled",
		"Enabled",
	}, time.Now(), DISABLED)
	rpcwrap.RegisterAuthenticated(UpdateStreamRpcService)
}

func logError() {
	if x := recover(); x != nil {
		log.Errorf("%s", x.(error).Error())
	}
}

func EnableUpdateStreamService(dbcfgs dbconfigs.DBConfigs) {
	defer logError()
	UpdateStreamRpcService.enable(dbcfgs)
}

func DisableUpdateStreamService() {
	defer logError()
	UpdateStreamRpcService.disable()
}

func ServeUpdateStream(req *BinlogPosition, sendReply func(reply interface{}) error) error {
	return UpdateStreamRpcService.ServeUpdateStream(req, sendReply)
}

func IsUpdateStreamEnabled() bool {
	return UpdateStreamRpcService.isEnabled()
}

func GetReplicationPosition() (*proto.ReplicationCoordinates, error) {
	return UpdateStreamRpcService.getReplicationPosition()
}

func (updateStream *UpdateStream) enable(dbcfgs dbconfigs.DBConfigs) {
	updateStream.actionLock.Lock()
	defer updateStream.actionLock.Unlock()
	if updateStream.isEnabled() {
		return
	}

	if dbcfgs.Dba.UnixSocket == "" {
		log.Errorf("Missing dba socket connection, cannot enable update stream service")
		return
	}

	if dbcfgs.App.DbName == "" {
		log.Errorf("Missing db name, cannot enable update stream service")
		return
	}

	if updateStream.mycnf.BinLogPath == "" {
		log.Errorf("Update stream service requires binlogs enabled")
		return
	}

	updateStream.state.Set(ENABLED)
	updateStream.states.SetState(ENABLED)
	updateStream.mysqld = NewMysqld(updateStream.mycnf, dbcfgs.Dba, dbcfgs.Repl)
	updateStream.dbname = dbcfgs.App.DbName
	updateStream.streams.Init()
	log.Infof("Enabling update stream, dbname: %s, binlogpath: %s", updateStream.dbname, updateStream.mycnf.BinLogPath)
}

func (updateStream *UpdateStream) disable() {
	updateStream.actionLock.Lock()
	defer updateStream.actionLock.Unlock()
	if !updateStream.isEnabled() {
		return
	}

	updateStream.state.Set(DISABLED)
	updateStream.states.SetState(DISABLED)
	updateStream.streams.Stop()
	updateStream.stateWaitGroup.Wait()
	log.Infof("Update Stream Disabled")
}

func (updateStream *UpdateStream) isEnabled() bool {
	return updateStream.state.Get() == ENABLED
}

func (updateStream *UpdateStream) ServeUpdateStream(req *BinlogPosition, sendReply func(reply interface{}) error) (err error) {
	defer func() {
		if x := recover(); x != nil {
			err = x.(error)
		}
	}()

	updateStream.actionLock.Lock()
	if !updateStream.isEnabled() {
		updateStream.actionLock.Unlock()
		log.Errorf("Unable to serve client request: Update stream service is not enabled")
		return fmt.Errorf("update stream service is not enabled")
	}
	updateStream.stateWaitGroup.Add(1)
	updateStream.actionLock.Unlock()
	defer updateStream.stateWaitGroup.Done()

	rp, err := updateStream.mysqld.BinlogInfo(req.GroupId, req.ServerId)
	if err != nil {
		return fmt.Errorf("error computing start position: %v", err)
	}
	log.Infof("ServeUpdateStream starting @ %v", rp)

	evs := NewEventStreamer(updateStream.dbname, updateStream.mycnf.BinLogPath)
	updateStream.streams.Add(evs)
	defer updateStream.streams.Delete(evs)

	// Calls cascade like this: BinlogStreamer->func(*StreamEvent)->sendReply
	return evs.Stream(rp.MasterLogFile, int64(rp.MasterLogPosition), func(reply *StreamEvent) error {
		return sendReply(reply)
	})
}

func (updateStream *UpdateStream) StreamKeyrange(req *KeyrangeRequest, sendReply func(reply interface{}) error) (err error) {
	defer func() {
		if x := recover(); x != nil {
			err = x.(error)
		}
	}()

	updateStream.actionLock.Lock()
	if !updateStream.isEnabled() {
		updateStream.actionLock.Unlock()
		log.Errorf("Unable to serve client request: Update stream service is not enabled")
		return fmt.Errorf("update stream service is not enabled")
	}
	updateStream.stateWaitGroup.Add(1)
	updateStream.actionLock.Unlock()
	defer updateStream.stateWaitGroup.Done()

	rp, err := updateStream.mysqld.BinlogInfo(req.StartPosition.GroupId, req.StartPosition.ServerId)
	if err != nil {
		return fmt.Errorf("error computing start position: %v", err)
	}
	log.Infof("ServeUpdateStream starting @ %v", rp)

	bls := NewBinlogStreamer(updateStream.dbname, updateStream.mycnf.BinLogPath)
	updateStream.streams.Add(bls)
	defer updateStream.streams.Delete(bls)

	// Calls cascade like this: BinlogStreamer->KeyrangeFilterFunc->func(*BinlogTransaction)->sendReply
	f := KeyrangeFilterFunc(req.Keyrange, func(reply *BinlogTransaction) error {
		return sendReply(reply)
	})
	return bls.Stream(rp.MasterLogFile, int64(rp.MasterLogPosition), f)
}

func (updateStream *UpdateStream) getReplicationPosition() (*proto.ReplicationCoordinates, error) {
	updateStream.actionLock.Lock()
	defer updateStream.actionLock.Unlock()
	if !updateStream.isEnabled() {
		return nil, fmt.Errorf("update stream service is not enabled")
	}

	rp, err := updateStream.mysqld.MasterStatus()
	if err != nil {
		return nil, err
	}
	return proto.NewReplicationCoordinates(rp.MasterLogFile, uint64(rp.MasterLogPosition), rp.MasterLogGroupId), nil
}
<|endoftext|>


<commit_before>package docker

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"
)

type Image struct {
	Id              string    `json:"id"`
	Parent          string    `json:"parent,omitempty"`
	Comment         string    `json:"comment,omitempty"`
	Created         time.Time `json:"created"`
	Container       string    `json:"container,omitempty"`
	ContainerConfig Config    `json:"container_config,omitempty"`
	DockerVersion   string    `json:"docker_version,omitempty"`
	graph           *Graph
}

func LoadImage(root string) (*Image, error) {
	// Load the json data
	jsonData, err := ioutil.ReadFile(jsonPath(root))
	if err != nil {
		return nil, err
	}
	var img Image
	if err := json.Unmarshal(jsonData, &img); err != nil {
		return nil, err
	}
	if err := ValidateId(img.Id); err != nil {
		return nil, err
	}
	// Check that the filesystem layer exists
	if stat, err := os.Stat(layerPath(root)); err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.Id)
		} else {
			return nil, err
		}
	} else if !stat.IsDir() {
		return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root))
	}
	return &img, nil
}

func StoreImage(img *Image, layerData Archive, root string) error {
	// Check that root doesn't already exist
	if _, err := os.Stat(root); err == nil {
		return fmt.Errorf("Image %s already exists", img.Id)
	} else if !os.IsNotExist(err) {
		return err
	}
	// Store the layer
	layer := layerPath(root)
	if err := os.MkdirAll(layer, 0700); err != nil {
		return err
	}
	if err := Untar(layerData, layer); err != nil {
		return err
	}
	// Store the json ball
	jsonData, err := json.Marshal(img)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
		return err
	}
	return nil
}

func layerPath(root string) string {
	return path.Join(root, "layer")
}

func jsonPath(root string) string {
	return path.Join(root, "json")
}

func MountAUFS(ro []string, rw string, target string) error {
	// FIXME: Now mount the layers
	rwBranch := fmt.Sprintf("%v=rw", rw)
	roBranches := ""
	for _, layer := range ro {
		roBranches += fmt.Sprintf("%v=ro:", layer)
	}
	branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches)
	return mount("none", target, "aufs", 0, branches)
}

func (image *Image) Mount(root, rw string) error {
	if mounted, err := Mounted(root); err != nil {
		return err
	} else if mounted {
		return fmt.Errorf("%s is already mounted", root)
	}
	layers, err := image.layers()
	if err != nil {
		return err
	}
	// Create the target directories if they don't exist
	if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) {
		return err
	}
	if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) {
		return err
	}
	// FIXME: @creack shouldn't we do this after going over changes?
	if err := MountAUFS(layers, rw, root); err != nil {
		return err
	}
	// FIXME: Create tests for deletion
	// FIXME: move this part to change.go
	// Retrieve the changeset from the parent and apply it to the container
	//  - Retrieve the changes
	changes, err := Changes(layers, layers[0])
	if err != nil {
		return err
	}
	// Iterate on changes
	for _, c := range changes {
		// If there is a delete
		if c.Kind == ChangeDelete {
			// Make sure the directory exists
			file_path, file_name := path.Dir(c.Path), path.Base(c.Path)
			if err := os.MkdirAll(path.Join(rw, file_path), 0755); err != nil {
				return err
			}
			// And create the whiteout (we just need to create empty file, discard the return)
			if _, err := os.Create(path.Join(path.Join(rw, file_path),
				".wh."+path.Base(file_name))); err != nil {
				return err
			}
		}
	}
	return nil
}

func (image *Image) Changes(rw string) ([]Change, error) {
	layers, err := image.layers()
	if err != nil {
		return nil, err
	}
	return Changes(layers, rw)
}

func (image *Image) ShortId() string {
	return TruncateId(image.Id)
}

func ValidateId(id string) error {
	if id == "" {
		return fmt.Errorf("Image id can't be empty")
	}
	if strings.Contains(id, ":") {
		return fmt.Errorf("Invalid character in image id: ':'")
	}
	return nil
}

func GenerateId() string {
	id := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, id)
	if err != nil {
		panic(err) // This shouldn't happen
	}
	return hex.EncodeToString(id)
}

// Image includes convenience proxy functions to its graph
// These functions will return an error if the image is not registered
// (ie. if image.graph == nil)
func (img *Image) History() ([]*Image, error) {
	var parents []*Image
	if err := img.WalkHistory(
		func(img *Image) error {
			parents = append(parents, img)
			return nil
		},
	); err != nil {
		return nil, err
	}
	return parents, nil
}

// layers returns all the filesystem layers needed to mount an image
// FIXME: @shykes refactor this function with the new error handling
//        (I'll do it if I have time tonight, I focus on the rest)
func (img *Image) layers() ([]string, error) {
	var list []string
	var e error
	if err := img.WalkHistory(
		func(img *Image) (err error) {
			if layer, err := img.layer(); err != nil {
				e = err
			} else if layer != "" {
				list = append(list, layer)
			}
			return err
		},
	); err != nil {
		return nil, err
	} else if e != nil { // Did an error occur inside the handler?
		return nil, e
	}
	if len(list) == 0 {
		return nil, fmt.Errorf("No layer found for image %s\n", img.Id)
	}
	return list, nil
}

func (img *Image) WalkHistory(handler func(*Image) error) (err error) {
	currentImg := img
	for currentImg != nil {
		if handler != nil {
			if err := handler(currentImg); err != nil {
				return err
			}
		}
		currentImg, err = currentImg.GetParent()
		if err != nil {
			return fmt.Errorf("Error while getting parent image: %v", err)
		}
	}
	return nil
}

func (img *Image) GetParent() (*Image, error) {
	if img.Parent == "" {
		return nil, nil
	}
	if img.graph == nil {
		return nil, fmt.Errorf("Can't lookup parent of unregistered image")
	}
	return img.graph.Get(img.Parent)
}

func (img *Image) root() (string, error) {
	if img.graph == nil {
		return "", fmt.Errorf("Can't lookup root of unregistered image")
	}
	return img.graph.imageRoot(img.Id), nil
}

// Return the path of an image's layer
func (img *Image) layer() (string, error) {
	root, err := img.root()
	if err != nil {
		return "", err
	}
	return layerPath(root), nil
}
<commit_msg>try to load aufs module, disply error on failure<commit_after>package docker

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

type Image struct {
	Id              string    `json:"id"`
	Parent          string    `json:"parent,omitempty"`
	Comment         string    `json:"comment,omitempty"`
	Created         time.Time `json:"created"`
	Container       string    `json:"container,omitempty"`
	ContainerConfig Config    `json:"container_config,omitempty"`
	DockerVersion   string    `json:"docker_version,omitempty"`
	graph           *Graph
}

func LoadImage(root string) (*Image, error) {
	// Load the json data
	jsonData, err := ioutil.ReadFile(jsonPath(root))
	if err != nil {
		return nil, err
	}
	var img Image
	if err := json.Unmarshal(jsonData, &img); err != nil {
		return nil, err
	}
	if err := ValidateId(img.Id); err != nil {
		return nil, err
	}
	// Check that the filesystem layer exists
	if stat, err := os.Stat(layerPath(root)); err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.Id)
		} else {
			return nil, err
		}
	} else if !stat.IsDir() {
		return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root))
	}
	return &img, nil
}

func StoreImage(img *Image, layerData Archive, root string) error {
	// Check that root doesn't already exist
	if _, err := os.Stat(root); err == nil {
		return fmt.Errorf("Image %s already exists", img.Id)
	} else if !os.IsNotExist(err) {
		return err
	}
	// Store the layer
	layer := layerPath(root)
	if err := os.MkdirAll(layer, 0700); err != nil {
		return err
	}
	if err := Untar(layerData, layer); err != nil {
		return err
	}
	// Store the json ball
	jsonData, err := json.Marshal(img)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
		return err
	}
	return nil
}

func layerPath(root string) string {
	return path.Join(root, "layer")
}

func jsonPath(root string) string {
	return path.Join(root, "json")
}

func MountAUFS(ro []string, rw string, target string) error {
	// FIXME: Now mount the layers
	rwBranch := fmt.Sprintf("%v=rw", rw)
	roBranches := ""
	for _, layer := range ro {
		roBranches += fmt.Sprintf("%v=ro:", layer)
	}
	branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches)

	//if error, try to load aufs kernel module
	if err := mount("none", target, "aufs", 0, branches); err != nil {
		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")
		if err := exec.Command("modprobe", "aufs").Run(); err != nil {
			return fmt.Errorf("Unable to load the AUFS module")
		}
		log.Printf("...module loaded.")
		if err := mount("none", target, "aufs", 0, branches); err != nil {
			return fmt.Errorf("Unable to mount using aufs")
		}
	}
	return nil
}

func (image *Image) Mount(root, rw string) error {
	if mounted, err := Mounted(root); err != nil {
		return err
	} else if mounted {
		return fmt.Errorf("%s is already mounted", root)
	}
	layers, err := image.layers()
	if err != nil {
		return err
	}
	// Create the target directories if they don't exist
	if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) {
		return err
	}
	if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) {
		return err
	}
	// FIXME: @creack shouldn't we do this after going over changes?
	if err := MountAUFS(layers, rw, root); err != nil {
		return err
	}
	// FIXME: Create tests for deletion
	// FIXME: move this part to change.go
	// Retrieve the changeset from the parent and apply it to the container
	//  - Retrieve the changes
	changes, err := Changes(layers, layers[0])
	if err != nil {
		return err
	}
	// Iterate on changes
	for _, c := range changes {
		// If there is a delete
		if c.Kind == ChangeDelete {
			// Make sure the directory exists
			file_path, file_name := path.Dir(c.Path), path.Base(c.Path)
			if err := os.MkdirAll(path.Join(rw, file_path), 0755); err != nil {
				return err
			}
			// And create the whiteout (we just need to create empty file, discard the return)
			if _, err := os.Create(path.Join(path.Join(rw, file_path),
				".wh."+path.Base(file_name))); err != nil {
				return err
			}
		}
	}
	return nil
}

func (image *Image) Changes(rw string) ([]Change, error) {
	layers, err := image.layers()
	if err != nil {
		return nil, err
	}
	return Changes(layers, rw)
}

func (image *Image) ShortId() string {
	return TruncateId(image.Id)
}

func ValidateId(id string) error {
	if id == "" {
		return fmt.Errorf("Image id can't be empty")
	}
	if strings.Contains(id, ":") {
		return fmt.Errorf("Invalid character in image id: ':'")
	}
	return nil
}

func GenerateId() string {
	id := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, id)
	if err != nil {
		panic(err) // This shouldn't happen
	}
	return hex.EncodeToString(id)
}

// Image includes convenience proxy functions to its graph
// These functions will return an error if the image is not registered
// (ie. if image.graph == nil)
func (img *Image) History() ([]*Image, error) {
	var parents []*Image
	if err := img.WalkHistory(
		func(img *Image) error {
			parents = append(parents, img)
			return nil
		},
	); err != nil {
		return nil, err
	}
	return parents, nil
}

// layers returns all the filesystem layers needed to mount an image
// FIXME: @shykes refactor this function with the new error handling
//        (I'll do it if I have time tonight, I focus on the rest)
func (img *Image) layers() ([]string, error) {
	var list []string
	var e error
	if err := img.WalkHistory(
		func(img *Image) (err error) {
			if layer, err := img.layer(); err != nil {
				e = err
			} else if layer != "" {
				list = append(list, layer)
			}
			return err
		},
	); err != nil {
		return nil, err
	} else if e != nil { // Did an error occur inside the handler?
		return nil, e
	}
	if len(list) == 0 {
		return nil, fmt.Errorf("No layer found for image %s\n", img.Id)
	}
	return list, nil
}

func (img *Image) WalkHistory(handler func(*Image) error) (err error) {
	currentImg := img
	for currentImg != nil {
		if handler != nil {
			if err := handler(currentImg); err != nil {
				return err
			}
		}
		currentImg, err = currentImg.GetParent()
		if err != nil {
			return fmt.Errorf("Error while getting parent image: %v", err)
		}
	}
	return nil
}

func (img *Image) GetParent() (*Image, error) {
	if img.Parent == "" {
		return nil, nil
	}
	if img.graph == nil {
		return nil, fmt.Errorf("Can't lookup parent of unregistered image")
	}
	return img.graph.Get(img.Parent)
}

func (img *Image) root() (string, error) {
	if img.graph == nil {
		return "", fmt.Errorf("Can't lookup root of unregistered image")
	}
	return img.graph.imageRoot(img.Id), nil
}

// Return the path of an image's layer
func (img *Image) layer() (string, error) {
	root, err := img.root()
	if err != nil {
		return "", err
	}
	return layerPath(root), nil
}
<|endoftext|>


<commit_before>package polygen

import (
	"fmt"
	"image"
	"image/draw"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png" // register PNG format
	"log"
	"math"
	"os"
)

func MustReadImage(file string) image.Image {
	infile, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer infile.Close()

	img, _, err := image.Decode(infile)
	if err != nil {
		log.Fatal(err)
	}

	return img
}

func Compare(img1, img2 *image.RGBA) (int64, error) {
	if img1.Bounds() != img2.Bounds() {
		return 0, fmt.Errorf("image bounds not equal: %+v, %+v", img1.Bounds(), img2.Bounds())
	}

	accumError := int64(0)
	bounds := img1.Bounds()

	for x := bounds.Min.X; x < bounds.Max.X; x++ {
		for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
			c1 := img1.At(x, y)
			c2 := img2.At(x, y)

			r1, g1, b1, a1 := c1.RGBA()
			r2, g2, b2, a2 := c2.RGBA()

			// TODO: consider ignoring the Alpha, since the colors are pre-multiplied
			sum := sqDiff(r1, r2) + sqDiff(g1, g2) + sqDiff(b1, b2) + sqDiff(a1, a2)
			accumError += int64(sum)
		}
	}

	return int64(math.Sqrt(float64(accumError))), nil
}

func FastCompare(img1, img2 *image.RGBA) (int64, error) {
	if img1.Bounds() != img2.Bounds() {
		return 0, fmt.Errorf("image bounds not equal: %+v, %+v", img1.Bounds(), img2.Bounds())
	}

	accumError := int64(0)

	for i := 0; i < len(img1.Pix); i++ {
		accumError += int64(sqDiffUInt8(img1.Pix[i], img2.Pix[i]))
	}

	return int64(math.Sqrt(float64(accumError))), nil
}

// from http://blog.golang.org/go-imagedraw-package ("Converting an Image to RGBA"),
// modified slightly to be a no-op if the src image is already RGBA
//
func ConvertToRGBA(img image.Image) (result *image.RGBA) {
	result, ok := img.(*image.RGBA)
	if ok {
		log.Printf("automatically converted to RGBA")
		return result
	} else {
		log.Printf("must convert manually to RGBA")
	}

	b := img.Bounds()
	result = image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	draw.Draw(result, result.Bounds(), img, b.Min, draw.Src)

	return
}

// taken directly from image/color/color.go:
//
// sqDiff returns the squared-difference of x and y, shifted by 2 so that
// adding four of those won't overflow a uint32.
//
// x and y are both assumed to be in the range [0, 0xffff].
func sqDiff(x, y uint32) uint32 {
	var d uint32
	if x > y {
		d = x - y
	} else {
		d = y - x
	}
	return (d * d) >> 2
}

func sqDiffUInt8(x, y uint8) uint8 {
	var d uint8
	if x > y {
		d = x - y
	} else {
		d = y - x
	}
	return (d * d)
}
<commit_msg>use row major order in (slow) Compare()<commit_after>package polygen

import (
	"fmt"
	"image"
	"image/draw"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png" // register PNG format
	"log"
	"math"
	"os"
)

func MustReadImage(file string) image.Image {
	infile, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer infile.Close()

	img, _, err := image.Decode(infile)
	if err != nil {
		log.Fatal(err)
	}

	return img
}

func Compare(img1, img2 *image.RGBA) (int64, error) {
	if img1.Bounds() != img2.Bounds() {
		return 0, fmt.Errorf("image bounds not equal: %+v, %+v", img1.Bounds(), img2.Bounds())
	}

	accumError := int64(0)
	bounds := img1.Bounds()

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			c1 := img1.At(x, y)
			c2 := img2.At(x, y)

			r1, g1, b1, a1 := c1.RGBA()
			r2, g2, b2, a2 := c2.RGBA()

			// TODO: consider ignoring the Alpha, since the colors are pre-multiplied
			sum := sqDiff(r1, r2) + sqDiff(g1, g2) + sqDiff(b1, b2) + sqDiff(a1, a2)
			accumError += int64(sum)
		}
	}

	return int64(math.Sqrt(float64(accumError))), nil
}

func FastCompare(img1, img2 *image.RGBA) (int64, error) {
	if img1.Bounds() != img2.Bounds() {
		return 0, fmt.Errorf("image bounds not equal: %+v, %+v", img1.Bounds(), img2.Bounds())
	}

	accumError := int64(0)

	for i := 0; i < len(img1.Pix); i++ {
		accumError += int64(sqDiffUInt8(img1.Pix[i], img2.Pix[i]))
	}

	return int64(math.Sqrt(float64(accumError))), nil
}

// from http://blog.golang.org/go-imagedraw-package ("Converting an Image to RGBA"),
// modified slightly to be a no-op if the src image is already RGBA
//
func ConvertToRGBA(img image.Image) (result *image.RGBA) {
	result, ok := img.(*image.RGBA)
	if ok {
		log.Printf("automatically converted to RGBA")
		return result
	} else {
		log.Printf("must convert manually to RGBA")
	}

	b := img.Bounds()
	result = image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
	draw.Draw(result, result.Bounds(), img, b.Min, draw.Src)

	return
}

// taken directly from image/color/color.go:
//
// sqDiff returns the squared-difference of x and y, shifted by 2 so that
// adding four of those won't overflow a uint32.
//
// x and y are both assumed to be in the range [0, 0xffff].
func sqDiff(x, y uint32) uint32 {
	var d uint32
	if x > y {
		d = x - y
	} else {
		d = y - x
	}
	return (d * d) >> 2
}

func sqDiffUInt8(x, y uint8) uint8 {
	var d uint8
	if x > y {
		d = x - y
	} else {
		d = y - x
	}
	return (d * d)
}
<|endoftext|>


<commit_before>package agent

import (
	"encoding/json"
	"errors"

	log "github.com/Sirupsen/logrus"
	"github.com/mistifyio/mistify-agent/rpc"
)

func imageMultiQuery(r *HTTPRequest, actionBaseName string, request *rpc.ImageRequest) ([]*rpc.Image, *HTTPErrorMessage) {
	// Determine the set of actions to query based on desired image type
	desiredImageType := r.Parameter("type")
	imageTypes := []string{"", "container"}
	if desiredImageType != "" {
		imageTypes = []string{desiredImageType}
	}
	n := len(imageTypes)

	// Create channels to aggregate results
	resps := make(chan *rpc.ImageResponse, n)
	errors := make(chan error, n)

	// Get the action runner
	runner, err := r.Context.GetAgentRunner()
	if err != nil {
		return nil, r.NewError(err, 500)
	}

	// Query in parallel
	for _, imageType := range imageTypes {
		actionName := prefixedActionName(imageType, actionBaseName)
		go imageQuery(runner, actionName, request, resps, errors)
	}

	// Wait for all to finish and aggregate results
	images := make([]*rpc.Image, 0)
	for i := 0; i < n; i++ {
		select {
		case resp := <-resps:
			images = append(images, resp.Images...)
		case err = <-errors:
			if err.Error() == "no such image" {
				err = nil
				continue
			}
			log.WithField("err", err).Info("image query error")
		}
	}
	if err != nil {
		return nil, r.NewError(err, 500)
	}

	return images, nil
}

func imageQuery(runner *GuestRunner, actionName string, request *rpc.ImageRequest, respChan chan *rpc.ImageResponse, errChan chan error) {
	response := &rpc.ImageResponse{}

	action, err := runner.Context.GetAction(actionName)
	if err != nil {
		respChan <- response
		return
	}
	pipeline := action.GeneratePipeline(request, response, nil, nil)

	if err = runner.Process(pipeline); err != nil {
		errChan <- err
		return
	}
	respChan <- response
	return
}

// listImages returns a list of images, optionally filtered by type
func listImages(r *HTTPRequest) *HTTPErrorMessage {
	request := &rpc.ImageRequest{}
	images, err := imageMultiQuery(r, "listImages", request)
	if err != nil {
		return err
	}
	return r.JSON(200, images)
}

func getImage(r *HTTPRequest) *HTTPErrorMessage {
	request := &rpc.ImageRequest{
		Id: r.Parameter("id"),
	}
	images, err := imageMultiQuery(r, "getImage", request)
	if err != nil {
		return err
	}

	if len(images) < 1 {
		return r.NewError(ErrNotFound, 404)
	}

	// This may happen if more than one backend have the image stored under the
	// same id, which is a problem. It will be much less likely when images are
	// all pulled from a central image server that assigns ids.
	if len(images) > 1 {
		log.WithFields(log.Fields{
			"imageID": request.Id,
			"images":  images,
		}).Error("more than one image share id")
	}

	return r.JSON(200, images[0])
}

func deleteImage(r *HTTPRequest) *HTTPErrorMessage {
	response := &rpc.ImageResponse{}
	request := &rpc.ImageRequest{
		Id: r.Parameter("id"),
	}

	// First find the image in order to know the type and, therefore, what
	// specific action to use to delete it
	images, mqErr := imageMultiQuery(r, "getImage", request)
	if mqErr != nil {
		return mqErr
	}

	if len(images) < 1 {
		return r.NewError(ErrNotFound, 404)
	}

	// This may happen if more than one backend have the image stored under the
	// same id, which is a problem. It will be much less likely when images are
	// all pulled from a central image server that assigns ids.
	if len(images) > 1 {
		log.WithFields(log.Fields{
			"imageID": request.Id,
			"images":  images,
		}).Error("more than one image share id")
	}

	// Go ahead with the delete
	action, err := r.Context.GetAction(prefixedActionName(images[0].Type, "deleteImage"))
	if err != nil {
		return r.NewError(err, 404)
	}

	pipeline := action.GeneratePipeline(request, response, r.ResponseWriter, nil)
	r.ResponseWriter.Header().Set("X-Guest-Job-ID", pipeline.ID)

	runner, err := r.Context.GetAgentRunner()
	if err != nil {
		return r.NewError(err, 500)
	}

	if err := runner.Process(pipeline); err != nil {
		// how to check for not found??
		return r.NewError(err, 500)
	}
	return r.JSON(202, struct{}{})
}

func fetchImage(r *HTTPRequest) *HTTPErrorMessage {
	request := &rpc.ImageRequest{}
	err := json.NewDecoder(r.Request.Body).Decode(request)
	if err != nil {
		return r.NewError(err, 400)
	}
	if request.Source == "" {
		return r.NewError(errors.New("missing source"), 400)
	}

	response := &rpc.ImageResponse{}
	action, err := r.Context.GetAction(prefixedActionName(request.Type, "fetchImage"))
	if err != nil {
		return r.NewError(err, 404)
	}
	pipeline := action.GeneratePipeline(request, response, r.ResponseWriter, nil)
	r.ResponseWriter.Header().Set("X-Guest-Job-ID", pipeline.ID)

	runner, err := r.Context.GetAgentRunner()
	if err != nil {
		return r.NewError(err, 500)
	}

	if err := runner.Process(pipeline); err != nil {
		return r.NewError(err, 500)
	}

	return r.JSON(202, response)
}
<commit_msg>Require image id instead of source for fetching an image<commit_after>package agent

import (
	"encoding/json"
	"errors"

	log "github.com/Sirupsen/logrus"
	"github.com/mistifyio/mistify-agent/rpc"
)

func imageMultiQuery(r *HTTPRequest, actionBaseName string, request *rpc.ImageRequest) ([]*rpc.Image, *HTTPErrorMessage) {
	// Determine the set of actions to query based on desired image type
	desiredImageType := r.Parameter("type")
	imageTypes := []string{"", "container"}
	if desiredImageType != "" {
		imageTypes = []string{desiredImageType}
	}
	n := len(imageTypes)

	// Create channels to aggregate results
	resps := make(chan *rpc.ImageResponse, n)
	errors := make(chan error, n)

	// Get the action runner
	runner, err := r.Context.GetAgentRunner()
	if err != nil {
		return nil, r.NewError(err, 500)
	}

	// Query in parallel
	for _, imageType := range imageTypes {
		actionName := prefixedActionName(imageType, actionBaseName)
		go imageQuery(runner, actionName, request, resps, errors)
	}

	// Wait for all to finish and aggregate results
	images := make([]*rpc.Image, 0)
	for i := 0; i < n; i++ {
		select {
		case resp := <-resps:
			images = append(images, resp.Images...)
		case err = <-errors:
			if err.Error() == "no such image" {
				err = nil
				continue
			}
			log.WithField("err", err).Info("image query error")
		}
	}
	if err != nil {
		return nil, r.NewError(err, 500)
	}

	return images, nil
}

func imageQuery(runner *GuestRunner, actionName string, request *rpc.ImageRequest, respChan chan *rpc.ImageResponse, errChan chan error) {
	response := &rpc.ImageResponse{}

	action, err := runner.Context.GetAction(actionName)
	if err != nil {
		respChan <- response
		return
	}
	pipeline := action.GeneratePipeline(request, response, nil, nil)

	if err = runner.Process(pipeline); err != nil {
		errChan <- err
		return
	}
	respChan <- response
	return
}

// listImages returns a list of images, optionally filtered by type
func listImages(r *HTTPRequest) *HTTPErrorMessage {
	request := &rpc.ImageRequest{}
	images, err := imageMultiQuery(r, "listImages", request)
	if err != nil {
		return err
	}
	return r.JSON(200, images)
}

func getImage(r *HTTPRequest) *HTTPErrorMessage {
	request := &rpc.ImageRequest{
		Id: r.Parameter("id"),
	}
	images, err := imageMultiQuery(r, "getImage", request)
	if err != nil {
		return err
	}

	if len(images) < 1 {
		return r.NewError(ErrNotFound, 404)
	}

	// This may happen if more than one backend have the image stored under the
	// same id, which is a problem. It will be much less likely when images are
	// all pulled from a central image server that assigns ids.
	if len(images) > 1 {
		log.WithFields(log.Fields{
			"imageID": request.Id,
			"images":  images,
		}).Error("more than one image share id")
	}

	return r.JSON(200, images[0])
}

func deleteImage(r *HTTPRequest) *HTTPErrorMessage {
	response := &rpc.ImageResponse{}
	request := &rpc.ImageRequest{
		Id: r.Parameter("id"),
	}

	// First find the image in order to know the type and, therefore, what
	// specific action to use to delete it
	images, mqErr := imageMultiQuery(r, "getImage", request)
	if mqErr != nil {
		return mqErr
	}

	if len(images) < 1 {
		return r.NewError(ErrNotFound, 404)
	}

	// This may happen if more than one backend have the image stored under the
	// same id, which is a problem. It will be much less likely when images are
	// all pulled from a central image server that assigns ids.
	if len(images) > 1 {
		log.WithFields(log.Fields{
			"imageID": request.Id,
			"images":  images,
		}).Error("more than one image share id")
	}

	// Go ahead with the delete
	action, err := r.Context.GetAction(prefixedActionName(images[0].Type, "deleteImage"))
	if err != nil {
		return r.NewError(err, 404)
	}

	pipeline := action.GeneratePipeline(request, response, r.ResponseWriter, nil)
	r.ResponseWriter.Header().Set("X-Guest-Job-ID", pipeline.ID)

	runner, err := r.Context.GetAgentRunner()
	if err != nil {
		return r.NewError(err, 500)
	}

	if err := runner.Process(pipeline); err != nil {
		// how to check for not found??
		return r.NewError(err, 500)
	}
	return r.JSON(202, struct{}{})
}

func fetchImage(r *HTTPRequest) *HTTPErrorMessage {
	request := &rpc.ImageRequest{}
	err := json.NewDecoder(r.Request.Body).Decode(request)
	if err != nil {
		return r.NewError(err, 400)
	}
	if request.Id == "" {
		return r.NewError(errors.New("missing id"), 400)
	}

	response := &rpc.ImageResponse{}
	action, err := r.Context.GetAction(prefixedActionName(request.Type, "fetchImage"))
	if err != nil {
		return r.NewError(err, 404)
	}
	pipeline := action.GeneratePipeline(request, response, r.ResponseWriter, nil)
	r.ResponseWriter.Header().Set("X-Guest-Job-ID", pipeline.ID)

	runner, err := r.Context.GetAgentRunner()
	if err != nil {
		return r.NewError(err, 500)
	}

	if err := runner.Process(pipeline); err != nil {
		return r.NewError(err, 500)
	}

	return r.JSON(202, response)
}
<|endoftext|>


<commit_before>package derpigo

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

/*
Image is an image on the Booru.
*/
type Image struct {
	ID               string             `json:"id"`
	CreatedAt        time.Time          `json:"created_at"`
	UpdatedAt        time.Time          `json:"updated_at"`
	DuplicateReports []*DuplicateReport `json:"duplicate_reports"`
	FirstSeenAt      time.Time          `json:"first_seen_at"`
	UploaderID       interface{}        `json:"uploader_id"`
	FileName         string             `json:"file_name"`
	Description      string             `json:"description"`
	Uploader         string             `json:"uploader"`
	Image            string             `json:"image"`
	Score            int                `json:"score"`
	Upvotes          int                `json:"upvotes"`
	Downvotes        int                `json:"downvotes"`
	Faves            int                `json:"faves"`
	CommentCount     int                `json:"comment_count"`
	Tags             string             `json:"tags"`
	TagIds           []string           `json:"tag_ids"`
	Width            int                `json:"width"`
	Height           int                `json:"height"`
	AspectRatio      float64            `json:"aspect_ratio"`
	OriginalFormat   string             `json:"original_format"`
	MimeType         string             `json:"mime_type"`
	Sha512Hash       string             `json:"sha512_hash"`
	OrigSha512Hash   string             `json:"orig_sha512_hash"`
	SourceURL        string             `json:"source_url"`
	Representations  struct {
		ThumbTiny  string `json:"thumb_tiny"`
		ThumbSmall string `json:"thumb_small"`
		Thumb      string `json:"thumb"`
		Small      string `json:"small"`
		Medium     string `json:"medium"`
		Large      string `json:"large"`
		Tall       string `json:"tall"`
		Full       string `json:"full"`
	} `json:"representations"`
	IsRendered   bool          `json:"is_rendered"`
	IsOptimized  bool          `json:"is_optimized"`
	Interactions []interface{} `json:"interactions"`
}

/*
DuplicateReport is a duplicate image report.
*/
type DuplicateReport struct {
	ID                  int64               `json:"id"`
	State               string              `json:"state"`
	Reason              string              `json:"reason"`
	ImageIDNumber       int                 `json:"image_id_number"`
	TargetImageIDNumber int                 `json:"target_image_id_number"`
	User                interface{}         `json:"user"`
	CreatedAt           string              `json:"created_at"`
	Modifier            *DupeReportModifier `json:"modifier"`
}

/*
DupeReportModifier is the weighting of the an image duplicate report.
*/
type DupeReportModifier struct {
	ID           string `json:"id"`
	Name         string `json:"name"`
	Avatar       string `json:"avatar"`
	CommentCount int    `json:"comment_count"`
	UploadCount  int    `json:"upload_count"`
	PostCount    int    `json:"post_count"`
	TopicCount   int    `json:"topic_count"`
}

/*
GetImage grabs image information with the api key of the recieving Connection.
If something fails it returns an error.
*/
func (c *Connection) GetImage(ctx context.Context, id int) (*Image, []Interaction, error) {
	data, is, err := c.apiCall(ctx, http.MethodGet, fmt.Sprintf("%d.json", id), url.Values{}, nil, http.StatusOK)
	if err != nil {
		return nil, nil, err
	}

	i := &Image{}

	err = json.Unmarshal(data, i)

	return i, is, err
}

/*
DeleteImage deletes an image from the booru. Needs assistant/mod/admin API key.

This is useful for handling morons scriptedly. There seems to be a lot of them
lately. This is a shame.
*/
func (c *Connection) DeleteImage(ctx context.Context, id, why string) error {
	q := url.Values{}
	q.Set("deletion_reason", why)

	_, _, err := c.apiCall(ctx, http.MethodDelete, fmt.Sprintf("https://derpibooru.org/images/%s.json", id), q, nil, 200)

	return err
}
<commit_msg>this one too<commit_after>package derpigo

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

/*
Image is an image on the Booru.
*/
type Image struct {
	ID               string             `json:"id"`
	CreatedAt        time.Time          `json:"created_at"`
	UpdatedAt        time.Time          `json:"updated_at"`
	DuplicateReports []*DuplicateReport `json:"duplicate_reports"`
	FirstSeenAt      time.Time          `json:"first_seen_at"`
	UploaderID       interface{}        `json:"uploader_id"`
	FileName         string             `json:"file_name"`
	Description      string             `json:"description"`
	Uploader         string             `json:"uploader"`
	Image            string             `json:"image"`
	Score            int                `json:"score"`
	Upvotes          int                `json:"upvotes"`
	Downvotes        int                `json:"downvotes"`
	Faves            int                `json:"faves"`
	CommentCount     int                `json:"comment_count"`
	Tags             string             `json:"tags"`
	TagIds           []string           `json:"tag_ids"`
	Width            int                `json:"width"`
	Height           int                `json:"height"`
	AspectRatio      float64            `json:"aspect_ratio"`
	OriginalFormat   string             `json:"original_format"`
	MimeType         string             `json:"mime_type"`
	Sha512Hash       string             `json:"sha512_hash"`
	OrigSha512Hash   string             `json:"orig_sha512_hash"`
	SourceURL        string             `json:"source_url"`
	Representations  struct {
		ThumbTiny  string `json:"thumb_tiny"`
		ThumbSmall string `json:"thumb_small"`
		Thumb      string `json:"thumb"`
		Small      string `json:"small"`
		Medium     string `json:"medium"`
		Large      string `json:"large"`
		Tall       string `json:"tall"`
		Full       string `json:"full"`
	} `json:"representations"`
	IsRendered   bool          `json:"is_rendered"`
	IsOptimized  bool          `json:"is_optimized"`
	Interactions []interface{} `json:"interactions"`
}

/*
DuplicateReport is a duplicate image report.
*/
type DuplicateReport struct {
	ID                  int64               `json:"id"`
	State               string              `json:"state"`
	Reason              string              `json:"reason"`
	ImageIDNumber       int                 `json:"image_id_number"`
	TargetImageIDNumber int                 `json:"target_image_id_number"`
	User                interface{}         `json:"user"`
	CreatedAt           string              `json:"created_at"`
	Modifier            *DupeReportModifier `json:"modifier"`
}

/*
DupeReportModifier is the weighting of the an image duplicate report.
*/
type DupeReportModifier struct {
	ID           int64  `json:"id"`
	Name         string `json:"name"`
	Avatar       string `json:"avatar"`
	CommentCount int    `json:"comment_count"`
	UploadCount  int    `json:"upload_count"`
	PostCount    int    `json:"post_count"`
	TopicCount   int    `json:"topic_count"`
}

/*
GetImage grabs image information with the api key of the recieving Connection.
If something fails it returns an error.
*/
func (c *Connection) GetImage(ctx context.Context, id int) (*Image, []Interaction, error) {
	data, is, err := c.apiCall(ctx, http.MethodGet, fmt.Sprintf("%d.json", id), url.Values{}, nil, http.StatusOK)
	if err != nil {
		return nil, nil, err
	}

	i := &Image{}

	err = json.Unmarshal(data, i)

	return i, is, err
}

/*
DeleteImage deletes an image from the booru. Needs assistant/mod/admin API key.

This is useful for handling morons scriptedly. There seems to be a lot of them
lately. This is a shame.
*/
func (c *Connection) DeleteImage(ctx context.Context, id, why string) error {
	q := url.Values{}
	q.Set("deletion_reason", why)

	_, _, err := c.apiCall(ctx, http.MethodDelete, fmt.Sprintf("https://derpibooru.org/images/%s.json", id), q, nil, 200)

	return err
}
<|endoftext|>


<commit_before>package gitbase

import (
	"bytes"
	"compress/zlib"
	"crypto/sha1"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"

	errors "gopkg.in/src-d/go-errors.v1"
	"gopkg.in/src-d/go-mysql-server.v0/sql"
)

var (
	// ErrColumnNotFound is returned when a given column is not found in the table's schema.
	ErrColumnNotFound = errors.NewKind("column %s not found for table %s")
	// ErrInvalidObjectType is returned when the received object is not of the correct type.
	ErrInvalidObjectType = errors.NewKind("got object of type %T, expecting %s")
)

// Indexable represents an indexable gitbase table.
type Indexable interface {
	sql.Indexable
	gitBase
}

func encodeIndexKey(k indexKey) ([]byte, error) {
	bs, err := k.encode()
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	gz := zlib.NewWriter(&buf)
	if _, err := gz.Write(bs); err != nil {
		return nil, err
	}

	if err := gz.Close(); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func decodeIndexKey(data []byte, k indexKey) error {
	gz, err := zlib.NewReader(bytes.NewReader(data))
	if err != nil {
		return err
	}

	bs, err := ioutil.ReadAll(gz)
	if err != nil {
		return err
	}

	return k.decode(bs)
}

func rowIndexValues(row sql.Row, columns []string, schema sql.Schema) ([]interface{}, error) {
	var values = make([]interface{}, len(columns))
	for i, col := range columns {
		var found bool
		for j, c := range schema {
			if c.Name == col {
				values[i] = row[j]
				found = true
				break
			}
		}

		if !found {
			return nil, ErrColumnNotFound.New(col, schema[0].Source)
		}
	}
	return values, nil
}

type rowKeyMapper interface {
	toRow([]byte) (sql.Row, error)
	fromRow(sql.Row) ([]byte, error)
}

type rowKeyValueIter struct {
	mapper  rowKeyMapper
	iter    sql.RowIter
	columns []string
	schema  sql.Schema
}

func (i *rowKeyValueIter) Next() ([]interface{}, []byte, error) {
	row, err := i.iter.Next()
	if err != nil {
		return nil, nil, err
	}

	key, err := i.mapper.fromRow(row)
	if err != nil {
		return nil, nil, err
	}

	values, err := rowIndexValues(row, i.columns, i.schema)
	if err != nil {
		return nil, nil, err
	}

	return values, key, nil
}

func (i *rowKeyValueIter) Close() error { return i.iter.Close() }

var (
	errRowKeyMapperRowLength = errors.NewKind("row should have %d columns, has: %d")
	errRowKeyMapperColType   = errors.NewKind("row column %d should have type %T, has: %T")
)

type rowIndexIter struct {
	mapper rowKeyMapper
	index  sql.IndexValueIter
}

func (i *rowIndexIter) Next() (sql.Row, error) {
	var err error
	var data []byte
	defer closeIndexOnError(&err, i.index)

	data, err = i.index.Next()
	if err != nil {
		return nil, err
	}

	row, err := i.mapper.toRow(data)
	if err != nil {
		return nil, err
	}

	return row, nil
}

func (i *rowIndexIter) Close() error { return i.index.Close() }

type indexKey interface {
	encode() ([]byte, error)
	decode([]byte) error
}

type packOffsetIndexKey struct {
	Repository string
	Packfile   string
	Offset     int64
	Hash       string
}

func (k *packOffsetIndexKey) decode(data []byte) error {
	buf := bytes.NewBuffer(data)
	var err error

	if k.Repository, err = readString(buf); err != nil {
		return err
	}

	if k.Packfile, err = readHash(buf); err != nil {
		return err
	}

	ok, err := readBool(buf)
	if err != nil {
		return err
	}

	if ok {
		if k.Offset, err = readInt64(buf); err != nil {
			return err
		}
		k.Hash = ""
	} else {
		k.Offset = -1
		if k.Hash, err = readHash(buf); err != nil {
			return err
		}
	}

	return nil
}

func (k *packOffsetIndexKey) encode() ([]byte, error) {
	var buf bytes.Buffer
	writeString(&buf, k.Repository)
	if err := writeHash(&buf, k.Packfile); err != nil {
		return nil, err
	}
	writeBool(&buf, k.Offset >= 0)
	if k.Offset >= 0 {
		writeInt64(&buf, k.Offset)
	} else {
		if err := writeHash(&buf, k.Hash); err != nil {
			return nil, err
		}
	}
	return buf.Bytes(), nil
}

func readInt64(buf *bytes.Buffer) (int64, error) {
	var bs = make([]byte, 8)
	_, err := io.ReadFull(buf, bs)
	if err != nil {
		return 0, fmt.Errorf("can't read int64: %s", err)
	}

	ux := binary.LittleEndian.Uint64(bs)
	x := int64(ux >> 1)
	if ux&1 != 0 {
		x = ^x
	}

	return x, nil
}

func readString(buf *bytes.Buffer) (string, error) {
	size, err := readInt64(buf)
	if err != nil {
		return "", fmt.Errorf("can't read string size: %s", err)
	}

	var b = make([]byte, int(size))
	if _, err = io.ReadFull(buf, b); err != nil {
		return "", fmt.Errorf("can't read string of size %d: %s", size, err)
	}

	return string(b), nil
}

func readBool(buf *bytes.Buffer) (bool, error) {
	b, err := buf.ReadByte()
	if err != nil {
		return false, fmt.Errorf("can't read bool: %s", err)
	}

	return b == 1, nil
}

func writeInt64(buf *bytes.Buffer, n int64) {
	ux := uint64(n) << 1
	if n < 0 {
		ux = ^ux
	}

	var bs = make([]byte, 8)
	binary.LittleEndian.PutUint64(bs, ux)
	buf.Write(bs)
}

func writeString(buf *bytes.Buffer, s string) {
	bs := []byte(s)
	writeInt64(buf, int64(len(bs)))
	buf.Write(bs)
}

var (
	hashSize           = 2 * sha1.Size
	errInvalidHashSize = errors.NewKind("invalid hash size: %d, expecting 40 bytes")
)

func writeHash(buf *bytes.Buffer, s string) error {
	bs := []byte(s)
	if len(bs) != hashSize {
		return errInvalidHashSize.New(len(bs))
	}
	buf.Write(bs)
	return nil
}

func readHash(buf *bytes.Buffer) (string, error) {
	bs := make([]byte, hashSize)
	n, err := io.ReadFull(buf, bs)
	if err != nil {
		return "", fmt.Errorf("can't read hash, only read %d: %s", n, err)
	}
	return string(bs), nil
}

func writeBool(buf *bytes.Buffer, b bool) {
	if b {
		buf.WriteByte(1)
	} else {
		buf.WriteByte(0)
	}
}

func closeIndexOnError(err *error, index sql.IndexValueIter) {
	if *err != nil {
		_ = index.Close()
	}
}
<commit_msg>gitbase: use a shared zlib encoder<commit_after>package gitbase

import (
	"bytes"
	"compress/zlib"
	"crypto/sha1"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"sync"

	errors "gopkg.in/src-d/go-errors.v1"
	"gopkg.in/src-d/go-mysql-server.v0/sql"
)

var (
	// ErrColumnNotFound is returned when a given column is not found in the table's schema.
	ErrColumnNotFound = errors.NewKind("column %s not found for table %s")
	// ErrInvalidObjectType is returned when the received object is not of the correct type.
	ErrInvalidObjectType = errors.NewKind("got object of type %T, expecting %s")
)

// Indexable represents an indexable gitbase table.
type Indexable interface {
	sql.Indexable
	gitBase
}

type zlibEncoder struct {
	w   *zlib.Writer
	mut sync.Mutex
}

func (e *zlibEncoder) encode(data []byte) ([]byte, error) {
	e.mut.Lock()
	defer e.mut.Unlock()

	var buf bytes.Buffer
	e.w.Reset(&buf)

	if _, err := e.w.Write(data); err != nil {
		return nil, err
	}

	if err := e.w.Close(); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

var encoder = func() *zlibEncoder {
	return &zlibEncoder{w: zlib.NewWriter(bytes.NewBuffer(nil))}
}()

func encodeIndexKey(k indexKey) ([]byte, error) {
	bs, err := k.encode()
	if err != nil {
		return nil, err
	}

	return encoder.encode(bs)
}

func decodeIndexKey(data []byte, k indexKey) error {
	gz, err := zlib.NewReader(bytes.NewReader(data))
	if err != nil {
		return err
	}

	bs, err := ioutil.ReadAll(gz)
	if err != nil {
		return err
	}

	return k.decode(bs)
}

func rowIndexValues(row sql.Row, columns []string, schema sql.Schema) ([]interface{}, error) {
	var values = make([]interface{}, len(columns))
	for i, col := range columns {
		var found bool
		for j, c := range schema {
			if c.Name == col {
				values[i] = row[j]
				found = true
				break
			}
		}

		if !found {
			return nil, ErrColumnNotFound.New(col, schema[0].Source)
		}
	}
	return values, nil
}

type rowKeyMapper interface {
	toRow([]byte) (sql.Row, error)
	fromRow(sql.Row) ([]byte, error)
}

type rowKeyValueIter struct {
	mapper  rowKeyMapper
	iter    sql.RowIter
	columns []string
	schema  sql.Schema
}

func (i *rowKeyValueIter) Next() ([]interface{}, []byte, error) {
	row, err := i.iter.Next()
	if err != nil {
		return nil, nil, err
	}

	key, err := i.mapper.fromRow(row)
	if err != nil {
		return nil, nil, err
	}

	values, err := rowIndexValues(row, i.columns, i.schema)
	if err != nil {
		return nil, nil, err
	}

	return values, key, nil
}

func (i *rowKeyValueIter) Close() error { return i.iter.Close() }

var (
	errRowKeyMapperRowLength = errors.NewKind("row should have %d columns, has: %d")
	errRowKeyMapperColType   = errors.NewKind("row column %d should have type %T, has: %T")
)

type rowIndexIter struct {
	mapper rowKeyMapper
	index  sql.IndexValueIter
}

func (i *rowIndexIter) Next() (sql.Row, error) {
	var err error
	var data []byte
	defer closeIndexOnError(&err, i.index)

	data, err = i.index.Next()
	if err != nil {
		return nil, err
	}

	row, err := i.mapper.toRow(data)
	if err != nil {
		return nil, err
	}

	return row, nil
}

func (i *rowIndexIter) Close() error { return i.index.Close() }

type indexKey interface {
	encode() ([]byte, error)
	decode([]byte) error
}

type packOffsetIndexKey struct {
	Repository string
	Packfile   string
	Offset     int64
	Hash       string
}

func (k *packOffsetIndexKey) decode(data []byte) error {
	buf := bytes.NewBuffer(data)
	var err error

	if k.Repository, err = readString(buf); err != nil {
		return err
	}

	if k.Packfile, err = readHash(buf); err != nil {
		return err
	}

	ok, err := readBool(buf)
	if err != nil {
		return err
	}

	if ok {
		if k.Offset, err = readInt64(buf); err != nil {
			return err
		}
		k.Hash = ""
	} else {
		k.Offset = -1
		if k.Hash, err = readHash(buf); err != nil {
			return err
		}
	}

	return nil
}

func (k *packOffsetIndexKey) encode() ([]byte, error) {
	var buf bytes.Buffer
	writeString(&buf, k.Repository)
	if err := writeHash(&buf, k.Packfile); err != nil {
		return nil, err
	}
	writeBool(&buf, k.Offset >= 0)
	if k.Offset >= 0 {
		writeInt64(&buf, k.Offset)
	} else {
		if err := writeHash(&buf, k.Hash); err != nil {
			return nil, err
		}
	}
	return buf.Bytes(), nil
}

func readInt64(buf *bytes.Buffer) (int64, error) {
	var bs = make([]byte, 8)
	_, err := io.ReadFull(buf, bs)
	if err != nil {
		return 0, fmt.Errorf("can't read int64: %s", err)
	}

	ux := binary.LittleEndian.Uint64(bs)
	x := int64(ux >> 1)
	if ux&1 != 0 {
		x = ^x
	}

	return x, nil
}

func readString(buf *bytes.Buffer) (string, error) {
	size, err := readInt64(buf)
	if err != nil {
		return "", fmt.Errorf("can't read string size: %s", err)
	}

	var b = make([]byte, int(size))
	if _, err = io.ReadFull(buf, b); err != nil {
		return "", fmt.Errorf("can't read string of size %d: %s", size, err)
	}

	return string(b), nil
}

func readBool(buf *bytes.Buffer) (bool, error) {
	b, err := buf.ReadByte()
	if err != nil {
		return false, fmt.Errorf("can't read bool: %s", err)
	}

	return b == 1, nil
}

func writeInt64(buf *bytes.Buffer, n int64) {
	ux := uint64(n) << 1
	if n < 0 {
		ux = ^ux
	}

	var bs = make([]byte, 8)
	binary.LittleEndian.PutUint64(bs, ux)
	buf.Write(bs)
}

func writeString(buf *bytes.Buffer, s string) {
	bs := []byte(s)
	writeInt64(buf, int64(len(bs)))
	buf.Write(bs)
}

var (
	hashSize           = 2 * sha1.Size
	errInvalidHashSize = errors.NewKind("invalid hash size: %d, expecting 40 bytes")
)

func writeHash(buf *bytes.Buffer, s string) error {
	bs := []byte(s)
	if len(bs) != hashSize {
		return errInvalidHashSize.New(len(bs))
	}
	buf.Write(bs)
	return nil
}

func readHash(buf *bytes.Buffer) (string, error) {
	bs := make([]byte, hashSize)
	n, err := io.ReadFull(buf, bs)
	if err != nil {
		return "", fmt.Errorf("can't read hash, only read %d: %s", n, err)
	}
	return string(bs), nil
}

func writeBool(buf *bytes.Buffer, b bool) {
	if b {
		buf.WriteByte(1)
	} else {
		buf.WriteByte(0)
	}
}

func closeIndexOnError(err *error, index sql.IndexValueIter) {
	if *err != nil {
		_ = index.Close()
	}
}
<|endoftext|>