<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"code.google.com/p/jra-go/mqtt"
	"github.com/eclesh/welford"
	proto "github.com/huin/mqtt"
)

var conns = flag.Int("conns", 100, "how many connections")
var wsubs = flag.Int("wsubs", 20, "how many wildcard subscribers")
var messages = flag.Int("messages", 1000, "how many messages")
var host = flag.String("host", "localhost:1883", "hostname of broker")
var dump = flag.Bool("dump", false, "dump messages?")
var id = flag.String("id", "", "client id")
var user = flag.String("user", "", "username")
var pass = flag.String("pass", "", "password")

var cwg sync.WaitGroup

type sd struct {
	mu sync.Mutex
	w  welford.Stats
}

func (s *sd) add(sample time.Duration) {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.w.Add(float64(sample))
}

func (s *sd) calculate() (min, max, mean, stddev time.Duration) {
	s.mu.Lock()
	defer s.mu.Unlock()

	min = time.Duration(s.w.Min())
	max = time.Duration(s.w.Max())
	mean = time.Duration(s.w.Mean())
	stddev = time.Duration(s.w.Stddev())

	return
}

var stddev sd

func main() {
	log.SetFlags(log.Lmicroseconds)
	flag.Parse()

	timeStart := time.Now()

	// a system to check how long connection establishment takes
	cwg.Add(*conns + *wsubs)
	go func() {
		cwg.Wait()
		log.Print("all connections made")
	}()

	// start the wildcard subscribers
	wsubWg.Add(*wsubs)
	for i := 0; i < *wsubs; i++ {
		go wsub()
	}

	var wg sync.WaitGroup
	publishers := *conns / 2
	for i := 0; i < publishers; i++ {
		wg.Add(1)
		go func(i int) {
			ping(i)
			wg.Done()
		}(i)
	}
	log.Print("all started")
	wg.Wait()
	log.Print("all finished")

	timeEnd := time.Now()

	log.Print("checking wildcard subscribers:")
	wsubWg.Wait()
	log.Print("ok")

	elapsed := timeEnd.Sub(timeStart)
	totmsg := float64(*messages * 2 * publishers)
	msgpersec := totmsg / float64(elapsed/time.Second)

	log.Print("elapsed time: ", elapsed)
	log.Print("messages    : ", totmsg)
	log.Print("messages/sec: ", msgpersec)

	log.Print("round-trip latency")
	min, max, mean, stddev := stddev.calculate()
	log.Print("min   : ", min)
	log.Print("max   : ", max)
	log.Print("mean  : ", mean)
	log.Print("stddev: ", stddev)
}

func ping(i int) {
	topic := fmt.Sprintf("pingtest/%v/request", i)
	topic2 := fmt.Sprintf("pingtest/%v/reply", i)

	start := make(chan struct{})
	stop := make(chan struct{})

	// the goroutine to reply to pings for this pair
	go func() {
		payload := []byte("ok")
		cc := connect()
		ack := cc.Subscribe([]proto.TopicQos{
			{topic, proto.QosAtLeastOnce},
		})
		if *dump {
			fmt.Printf("suback: %#v\n", ack)
		}

		close(start)
		for {
			select {
			// for each incoming message, send it back unchanged
			case in := <-cc.Incoming:
				if *dump {
					fmt.Printf("request: %#v\n", in)
				}
				in.TopicName = topic2
				in.Payload = proto.BytesPayload(payload)
				cc.Publish(in)
			case _ = <-stop:
				cc.Disconnect()
				return
			}
		}
	}()

	_ = <-start

	cc := connect()

	ack := cc.Subscribe([]proto.TopicQos{
		{topic2, proto.QosAtLeastOnce},
	})
	if *dump {
		fmt.Printf("suback: %#v\n", ack)
	}

	for i := 0; i < *messages; i++ {
		payload := []byte(fmt.Sprintf("ping %v", i))

		timeStart := time.Now()
		cc.Publish(&proto.Publish{
			Header:    proto.Header{QosLevel: proto.QosAtMostOnce},
			TopicName: topic,
			Payload:   proto.BytesPayload(payload),
		})

		in := <-cc.Incoming
		if *dump {
			fmt.Printf("reply: %#v\n", in)
		}

		elapsed := time.Now().Sub(timeStart)
		stddev.add(elapsed)

		buf := &bytes.Buffer{}
		err := in.Payload.WritePayload(buf)
		if err != nil {
			log.Println("payload data:", err)
			panic("ugh")
		}

		if !bytes.Equal(buf.Bytes(), []byte("ok")) {
			log.Println("unexpected reply: ", string(buf.Bytes()))
			break
		}
	}

	cc.Disconnect()
	close(stop)
}

func connect() *mqtt.ClientConn {
	conn, err := net.Dial("tcp", *host)
	if err != nil {
		fmt.Fprintf(os.Stderr, "dial: %v\n", err)
		os.Exit(2)
	}
	cc := mqtt.NewClientConn(conn)
	cc.Dump = *dump
	cc.ClientId = *id

	err = cc.Connect(*user, *pass)
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	cwg.Done()
	return cc
}

var wsubWg sync.WaitGroup

func wsub() {
	topic := "pingtest/1/#"

	cc := connect()
	defer cc.Disconnect()

	ack := cc.Subscribe([]proto.TopicQos{
		{topic, proto.QosAtLeastOnce},
	})
	if *dump {
		fmt.Printf("suback: %#v\n", ack)
	}

	count := 0
	for _ = range cc.Incoming {
		count++
		if count == (2 * *messages) {
			wsubWg.Done()
			return
		}
	}
}
<commit_msg>Improve ui. Fix bug with wsubs=1.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"code.google.com/p/jra-go/mqtt"
	"github.com/eclesh/welford"
	proto "github.com/huin/mqtt"
)

var pairs = flag.Int("pairs", 100, "how many ping/reply pairs")
var wsubs = flag.Int("wsubs", 20, "how many wildcard subscribers")
var messages = flag.Int("messages", 1000, "how many messages")
var host = flag.String("host", "localhost:1883", "hostname of broker")
var dump = flag.Bool("dump", false, "dump messages?")
var id = flag.String("id", "", "client id (default: use a random one)")
var user = flag.String("user", "", "username")
var pass = flag.String("pass", "", "password")

var cwg sync.WaitGroup

type sd struct {
	mu sync.Mutex
	w  welford.Stats
}

func (s *sd) add(sample time.Duration) {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.w.Add(float64(sample))
}

func (s *sd) calculate() (min, max, mean, stddev time.Duration) {
	s.mu.Lock()
	defer s.mu.Unlock()

	min = time.Duration(s.w.Min())
	max = time.Duration(s.w.Max())
	mean = time.Duration(s.w.Mean())
	stddev = time.Duration(s.w.Stddev())

	return
}

var stddev sd

func main() {
	log.SetFlags(log.Lmicroseconds)
	flag.Parse()

	timeStart := time.Now()

	// a system to check how long connection establishment takes
	cwg.Add(2 * *pairs + *wsubs)
	go func() {
		cwg.Wait()
		log.Print("all connections made")
	}()

	// start the wildcard subscribers
	wsubExit.Add(*wsubs)
	wsubReady.Add(*wsubs)
	for i := 0; i < *wsubs; i++ {
		go wsub()
	}

	var wg sync.WaitGroup
	for i := 0; i < *pairs; i++ {
		wg.Add(1)
		go func(i int) {
			ping(i)
			wg.Done()
		}(i)
	}
	log.Print("all started")
	wg.Wait()
	log.Print("all finished")

	timeEnd := time.Now()

	log.Print("checking wildcard subscribers:")
	wsubExit.Wait()
	log.Print("ok")

	elapsed := timeEnd.Sub(timeStart)
	totmsg := float64(*messages * 2 * *pairs)
	msgpersec := totmsg / float64(elapsed/time.Second)

	log.Print("elapsed time: ", elapsed)
	log.Print("messages    : ", totmsg)
	log.Print("messages/sec: ", msgpersec)

	log.Print("round-trip latency")
	min, max, mean, stddev := stddev.calculate()
	log.Print("min   : ", min)
	log.Print("max   : ", max)
	log.Print("mean  : ", mean)
	log.Print("stddev: ", stddev)
}

func ping(i int) {
	wsubReady.Wait()

	topic := fmt.Sprintf("pingtest/%v/request", i)
	topic2 := fmt.Sprintf("pingtest/%v/reply", i)

	start := make(chan struct{})
	stop := make(chan struct{})

	// the goroutine to reply to pings for this pair
	go func() {
		payload := []byte("ok")
		cc := connect()
		ack := cc.Subscribe([]proto.TopicQos{
			{topic, proto.QosAtLeastOnce},
		})
		if *dump {
			fmt.Printf("suback: %#v\n", ack)
		}

		close(start)
		for {
			select {
			// for each incoming message, send it back unchanged
			case in := <-cc.Incoming:
				if *dump {
					fmt.Printf("request: %#v\n", in)
				}
				in.TopicName = topic2
				in.Payload = proto.BytesPayload(payload)
				cc.Publish(in)
			case _ = <-stop:
				cc.Disconnect()
				return
			}
		}
	}()

	_ = <-start

	cc := connect()

	ack := cc.Subscribe([]proto.TopicQos{
		{topic2, proto.QosAtLeastOnce},
	})
	if *dump {
		fmt.Printf("suback: %#v\n", ack)
	}

	for i := 0; i < *messages; i++ {
		payload := []byte(fmt.Sprintf("ping %v", i))

		timeStart := time.Now()
		cc.Publish(&proto.Publish{
			Header:    proto.Header{QosLevel: proto.QosAtMostOnce},
			TopicName: topic,
			Payload:   proto.BytesPayload(payload),
		})

		in := <-cc.Incoming
		if *dump {
			fmt.Printf("reply: %#v\n", in)
		}

		elapsed := time.Now().Sub(timeStart)
		stddev.add(elapsed)

		buf := &bytes.Buffer{}
		err := in.Payload.WritePayload(buf)
		if err != nil {
			log.Println("payload data:", err)
			panic("ugh")
		}

		if !bytes.Equal(buf.Bytes(), []byte("ok")) {
			log.Println("unexpected reply: ", string(buf.Bytes()))
			break
		}
	}

	cc.Disconnect()
	close(stop)
}

func connect() *mqtt.ClientConn {
	conn, err := net.Dial("tcp", *host)
	if err != nil {
		fmt.Fprintf(os.Stderr, "dial: %v\n", err)
		os.Exit(2)
	}
	cc := mqtt.NewClientConn(conn)
	cc.Dump = *dump
	cc.ClientId = *id

	err = cc.Connect(*user, *pass)
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	cwg.Done()
	return cc
}

var wsubExit sync.WaitGroup
var wsubReady sync.WaitGroup

func wsub() {
	topic := "pingtest/0/#"

	cc := connect()
	defer cc.Disconnect()

	ack := cc.Subscribe([]proto.TopicQos{
		{topic, proto.QosAtLeastOnce},
	})
	if *dump {
		fmt.Printf("suback: %#v\n", ack)
	}

	wsubReady.Done()

	count := 0
	for _ = range cc.Incoming {
		count++
		if count == (2 * *messages) {
			wsubExit.Done()
			return
		}
	}
}
<|endoftext|>


<commit_before>package pinyingo

import (
  "testing"
)

func TestConvert(t *testing.T) {
  pinyinMap := map[string][]string{
    "hello中国": []string{"hello", "zhōng", "guó"},
    "中国hello": []string{"zhōng", "guó", "hello"},
    "中国":      []string{"zhōng", "guó"},
    "123qwe":  []string{"123qwe"},
  }
  pinyinMap1 := map[string][]string{
    "hello中国": []string{"hello", "zh", "g"},
    "中国":      []string{"zh", "g"},
    "123qwe":  []string{"123qwe"},
  }
  pinyinMap2 := map[string][]string{
    "hello中国": []string{"hello", "zhong", "guo"},
    "中国":      []string{"zhong", "guo"},
    "123qwe":  []string{"123qwe"},
  }
  for k, v := range pinyinMap {
    py := NewPy(STYLE_TONE)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }

  for k, v := range pinyinMap1 {
    py := NewPy(STYLE_INITIALS)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }

  for k, v := range pinyinMap2 {
    py := NewPy(STYLE_NORMAL)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }
}
<commit_msg>update test file<commit_after>package pinyingo

import (
  "testing"
)

func TestConvert(t *testing.T) {
  pinyinMap := map[string][]string{
    "hello中国": []string{"hello", "zhōng", "guó"},
    "中国hello": []string{"zhōng", "guó", "hello"},
    "中国":      []string{"zhōng", "guó"},
    "123qwe":  []string{"123qwe"},
  }
  pinyinMap1 := map[string][]string{
    "hello中国": []string{"hello", "zh", "g"},
    "中国":      []string{"zh", "g"},
    "123qwe":  []string{"123qwe"},
  }
  pinyinMap2 := map[string][]string{
    "hello中国": []string{"hello", "zhong", "guo"},
    "中国":      []string{"zhong", "guo"},
    "123qwe":  []string{"123qwe"},
  }

  pinyinMap3 := map[string][]string{
    "重阳": []string{"chóng", "yáng"},
    "重点": []string{"zhòng", "diǎn"},
  }

  for k, v := range pinyinMap {
    py := NewPy(STYLE_TONE, NO_SEGMENT)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }

  for k, v := range pinyinMap1 {
    py := NewPy(STYLE_INITIALS, NO_SEGMENT)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }

  for k, v := range pinyinMap2 {
    py := NewPy(STYLE_NORMAL, NO_SEGMENT)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }

  for k, v := range pinyinMap3 {
    py := NewPy(STYLE_TONE, USE_SEGMENT)
    converted := py.Convert(k)
    for i := 0; i < len(converted); i++ {
      if converted[i] != v[i] {
        t.Errorf("%s is not equal %s", converted, v)
      }
    }
  }
}
<|endoftext|>


<commit_before>/**
This package is used to provide an implementation of the
Chord network protocol. It can be used to provide a DHT
which is tolerant to churn in the member ndoes.
*/
package chord

import (
	"crypto/sha1"
	"hash"
	"sort"
	"time"
)

// Implements the methods needed for a Chord ring
type Transport interface {
	// Ping a Vnode, check for liveness
	Ping(*Vnode) (bool, error)

	// Request a nodes predecessor
	GetPredecessor(*Vnode) (*Vnode, error)

	// Notify our successor of ourselves
	Notify(target, self *Vnode) ([]*Vnode, error)

	// Find a successor
	FindSuccessor(*Vnode, []byte) (*Vnode, error)

	// Register for an RPC callbacks
	Register(*Vnode, VnodeRPC)
}

// These are the methods to invoke on the registered vnodes
type VnodeRPC interface {
	GetPredecessor() (*Vnode, error)
	Notify(*Vnode) ([]*Vnode, error)
	FindSuccessor([]byte) (*Vnode, error)
}

// Delegate to notify on ring events
type Delegate interface {
	NewSuccessor(local *Vnode, remoteNew *Vnode, remotePrev *Vnode)
	NewPredecessor(local *Vnode, remoteNew *Vnode, remotePrev *Vnode)
	PredecessorLeaving(local *Vnode, remote *Vnode)
	SuccessorLeaving(local *Vnode, remote *Vnode)
}

// Configuration for Chord nodes
type Config struct {
	Hostname      string           // Local host name
	NumVnodes     int              // Number of vnodes per physical node
	HashFunc      func() hash.Hash // Hash function to use
	HashBits      int              // Bit size of the hash function
	StabilizeMin  time.Duration    // Minimum stabilization time
	StabilizeMax  time.Duration    // Maximum stabilization time
	NumSuccessors int              // Number of successors to maintain
	Delegate      Delegate         // Invoked to handle ring events
}

// Represents an Vnode, local or remote
type Vnode struct {
	Id   []byte // Virtual ID
	Host string // Host identifier
}

// Represents a local Vnode
type localVnode struct {
	Vnode
	ring        *Ring
	successors  []*Vnode
	finger      []*Vnode
	last_finger int
	predecessor *Vnode
	stabilized  time.Time
	timer       *time.Timer
}

// Stores the state required for a Chord ring
type Ring struct {
	config    *Config
	transport Transport
	vnodes    []localVnode
	shutdown  bool
}

// Creates an iterator over Vnodes
type VnodeIterator interface {
	Next() (*Vnode, error) // Returns the next vnode
	Done() bool            // Returns true if all vnodes exhausted
}

// Returns the default Ring configuration
func DefaultConfig(hostname string) *Config {
	return &Config{
		hostname,
		8,        // 8 vnodes
		sha1.New, // SHA1
		160,      // 160bit hash function
		time.Duration(15 * time.Second),
		time.Duration(45 * time.Second),
		8,   // 8 successors
		nil, // No delegate
	}
}

// Creates a new Chord ring given the config and transport
func Create(conf *Config, trans Transport) (*Ring, error) {
	vnodes := make([]localVnode, conf.NumVnodes)
	trans = InitLocalTransport(trans)
	ring := &Ring{conf, trans, vnodes, false}
	for i := 0; i < conf.NumVnodes; i++ {
		vn := &vnodes[i]
		vn.ring = ring
		if err := vn.init(i); err != nil {
			return nil, err
		}
	}

	// Sort the vnodes
	sort.Sort(ring)

	// Setup each vnode successors
	for idx, vn := range vnodes {
		if idx == len(vnodes) {
			vn.successors[0] = &vnodes[0].Vnode
		} else {
			vn.successors[0] = &vnodes[idx+1].Vnode
		}
	}

	// Schedule each Vnode
	for _, vn := range vnodes {
		vn.schedule()
	}
	return ring, nil
}

// Joins an existing Chord ring
func Join(conf *Config, trans Transport, existing string) (*Ring, error) {
	return nil, nil
}

// Leaves a given Chord ring
func (*Ring) Leave() error {
	return nil
}

// Shutdown shuts down the local processes in a given Chord ring
func (r *Ring) Shutdown() error {
	r.shutdown = true
	return nil
}

// Does a key lookup
func (r *Ring) Lookup(key []byte) (VnodeIterator, error) {
	// Hash the key
	h := r.config.HashFunc()
	h.Write(key)
	key_hash := h.Sum(nil)

	// Find the nearest local vnode
	nearest := r.nearestVnode(key_hash)

	// Use the nearest node for the lookup
	_, err := nearest.FindSuccessor(key_hash)
	if err != nil {
		return nil, err
	}

	// TODO: Iterator over results
	return nil, nil
}

// Does a key lookup, returning only the primary Vnode
func (r *Ring) LookupPrimary(key []byte) (*Vnode, error) {
	iter, err := r.Lookup(key)
	if err != nil {
		return nil, err
	}
	prim, err := iter.Next()
	if err != nil {
		return nil, err
	}
	return prim, nil
}
<commit_msg>Fixing index<commit_after>/**
This package is used to provide an implementation of the
Chord network protocol. It can be used to provide a DHT
which is tolerant to churn in the member ndoes.
*/
package chord

import (
	"crypto/sha1"
	"hash"
	"sort"
	"time"
)

// Implements the methods needed for a Chord ring
type Transport interface {
	// Ping a Vnode, check for liveness
	Ping(*Vnode) (bool, error)

	// Request a nodes predecessor
	GetPredecessor(*Vnode) (*Vnode, error)

	// Notify our successor of ourselves
	Notify(target, self *Vnode) ([]*Vnode, error)

	// Find a successor
	FindSuccessor(*Vnode, []byte) (*Vnode, error)

	// Register for an RPC callbacks
	Register(*Vnode, VnodeRPC)
}

// These are the methods to invoke on the registered vnodes
type VnodeRPC interface {
	GetPredecessor() (*Vnode, error)
	Notify(*Vnode) ([]*Vnode, error)
	FindSuccessor([]byte) (*Vnode, error)
}

// Delegate to notify on ring events
type Delegate interface {
	NewSuccessor(local *Vnode, remoteNew *Vnode, remotePrev *Vnode)
	NewPredecessor(local *Vnode, remoteNew *Vnode, remotePrev *Vnode)
	PredecessorLeaving(local *Vnode, remote *Vnode)
	SuccessorLeaving(local *Vnode, remote *Vnode)
}

// Configuration for Chord nodes
type Config struct {
	Hostname      string           // Local host name
	NumVnodes     int              // Number of vnodes per physical node
	HashFunc      func() hash.Hash // Hash function to use
	HashBits      int              // Bit size of the hash function
	StabilizeMin  time.Duration    // Minimum stabilization time
	StabilizeMax  time.Duration    // Maximum stabilization time
	NumSuccessors int              // Number of successors to maintain
	Delegate      Delegate         // Invoked to handle ring events
}

// Represents an Vnode, local or remote
type Vnode struct {
	Id   []byte // Virtual ID
	Host string // Host identifier
}

// Represents a local Vnode
type localVnode struct {
	Vnode
	ring        *Ring
	successors  []*Vnode
	finger      []*Vnode
	last_finger int
	predecessor *Vnode
	stabilized  time.Time
	timer       *time.Timer
}

// Stores the state required for a Chord ring
type Ring struct {
	config    *Config
	transport Transport
	vnodes    []localVnode
	shutdown  bool
}

// Creates an iterator over Vnodes
type VnodeIterator interface {
	Next() (*Vnode, error) // Returns the next vnode
	Done() bool            // Returns true if all vnodes exhausted
}

// Returns the default Ring configuration
func DefaultConfig(hostname string) *Config {
	return &Config{
		hostname,
		8,        // 8 vnodes
		sha1.New, // SHA1
		160,      // 160bit hash function
		time.Duration(15 * time.Second),
		time.Duration(45 * time.Second),
		8,   // 8 successors
		nil, // No delegate
	}
}

// Creates a new Chord ring given the config and transport
func Create(conf *Config, trans Transport) (*Ring, error) {
	vnodes := make([]localVnode, conf.NumVnodes)
	trans = InitLocalTransport(trans)
	ring := &Ring{conf, trans, vnodes, false}
	for i := 0; i < conf.NumVnodes; i++ {
		vn := &vnodes[i]
		vn.ring = ring
		if err := vn.init(i); err != nil {
			return nil, err
		}
	}

	// Sort the vnodes
	sort.Sort(ring)

	// Setup each vnode successors
	for idx, vn := range vnodes {
		if idx == len(vnodes)-1 {
			vn.successors[0] = &vnodes[0].Vnode
		} else {
			vn.successors[0] = &vnodes[idx+1].Vnode
		}
	}

	// Schedule each Vnode
	for _, vn := range vnodes {
		vn.schedule()
	}
	return ring, nil
}

// Joins an existing Chord ring
func Join(conf *Config, trans Transport, existing string) (*Ring, error) {
	return nil, nil
}

// Leaves a given Chord ring
func (*Ring) Leave() error {
	return nil
}

// Shutdown shuts down the local processes in a given Chord ring
func (r *Ring) Shutdown() error {
	r.shutdown = true
	return nil
}

// Does a key lookup
func (r *Ring) Lookup(key []byte) (VnodeIterator, error) {
	// Hash the key
	h := r.config.HashFunc()
	h.Write(key)
	key_hash := h.Sum(nil)

	// Find the nearest local vnode
	nearest := r.nearestVnode(key_hash)

	// Use the nearest node for the lookup
	_, err := nearest.FindSuccessor(key_hash)
	if err != nil {
		return nil, err
	}

	// TODO: Iterator over results
	return nil, nil
}

// Does a key lookup, returning only the primary Vnode
func (r *Ring) LookupPrimary(key []byte) (*Vnode, error) {
	iter, err := r.Lookup(key)
	if err != nil {
		return nil, err
	}
	prim, err := iter.Next()
	if err != nil {
		return nil, err
	}
	return prim, nil
}
<|endoftext|>


<commit_before>package nzb

import (
    "sync"
)

// Chunks list of all chunks within NZB
type Chunks struct {
    c       []*Chunk
    mu      *sync.Mutex
    Total   int
    Marker  int
}

func (c *Chunks) addChunks(groups []string, segments []*Segment) {
    for _, s := range segments {
        cnk := &Chunk {
            Groups: groups,
            Segment: s,
        }

        c.c = append(c.c, cnk)
    }
}

// GetChunks limited by max
func (c *Chunks) GetChunks(max int) []*Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    chunks := make([]*Chunk, max)

    if c.Marker < len(c.c) {
        for i := 0; i < max; i++ {
            chunks[i] = c.c[c.Marker]
            c.Marker++
        }
    }

    return chunks
}

// GetChunk get next chunk from Chunks
func (c *Chunks) GetChunk() *Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    if c.Marker < len(c.c) {
        cnk := c.c[c.Marker]
        c.Marker++
        return cnk
    }

    return nil
}

// Chunk of data to download
type Chunk struct {
    Groups []string
    Segment *Segment
}

// EOF
<commit_msg>code format<commit_after>package nzb

import (
    "sync"
)

// Chunk of data to download
type Chunk struct {
    Groups []string
    Segment *Segment
}

// Chunks list of all chunks within NZB
type Chunks struct {
    c       []*Chunk
    mu      *sync.Mutex
    Total   int
    Marker  int
}

func (c *Chunks) addChunks(groups []string, segments []*Segment) {
    for _, s := range segments {
        cnk := &Chunk {
            Groups: groups,
            Segment: s,
        }

        c.c = append(c.c, cnk)
    }
}

// GetChunks limited by max
func (c *Chunks) GetChunks(max int) []*Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    chunks := make([]*Chunk, max)

    if c.Marker < len(c.c) {
        for i := 0; i < max; i++ {
            chunks[i] = c.c[c.Marker]
            c.Marker++
        }
    }

    return chunks
}

// GetChunk get next chunk from Chunks
func (c *Chunks) GetChunk() *Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    if c.Marker < len(c.c) {
        cnk := c.c[c.Marker]
        c.Marker++
        return cnk
    }

    return nil
}

// EOF
<|endoftext|>


<commit_before>package main

import (
	"github.com/dynport/dgtk/cli"
	"github.com/phrase/phraseapp-go/phraseapp"
)

func ApplyNonRestRoutes(r *cli.Router, cfg *phraseapp.Config) {
	r.Register("pull", &PullCommand{Config: *cfg}, "Download locales from your PhraseApp project.\n  You can provide parameters supported by the locales#download endpoint http://docs.phraseapp.com/api/v2/locales/#download\n  in your configuration (.phraseapp.yml) for each source.\n  See our configuration guide for more information http://docs.phraseapp.com/developers/cli/configuration/")

	r.Register("push", &PushCommand{Config: *cfg}, "Upload locales to your PhraseApp project.\n  You can provide parameters supported by the uploads#create endpoint http://docs.phraseapp.com/api/v2/uploads/#create\n  in your configuration (.phraseapp.yml) for each source.\n  See our configuration guide for more information http://docs.phraseapp.com/developers/cli/configuration/")

	r.Register("init", &InitCommand{Config: *cfg}, "Configure your PhraseApp client.")

	r.Register("upload/cleanup", &UploadCleanupCommand{Config: *cfg}, "Delete unmentioned keys for given upload")

	r.RegisterFunc("info", infoCommand, "Info about version and revision of this client")
}
<commit_msg>Use new api docs urls<commit_after>package main

import (
	"github.com/dynport/dgtk/cli"
	"github.com/phrase/phraseapp-go/phraseapp"
)

func ApplyNonRestRoutes(r *cli.Router, cfg *phraseapp.Config) {
	r.Register("pull", &PullCommand{Config: *cfg}, "Download locales from your PhraseApp project.\n  You can provide parameters supported by the locales#download endpoint https://developers.phraseapp.com/api/#locales_download\n  in your configuration (.phraseapp.yml) for each source.\n  See our configuration guide for more information https://phraseapp.com/docs/developers/cli/configuration/")

	r.Register("push", &PushCommand{Config: *cfg}, "Upload locales to your PhraseApp project.\n  You can provide parameters supported by the uploads#create endpoint https://developers.phraseapp.com/api/#uploads_create\n  in your configuration (.phraseapp.yml) for each source.\n  See our configuration guide for more information https://phraseapp.com/docs/developers/cli/configuration/")

	r.Register("init", &InitCommand{Config: *cfg}, "Configure your PhraseApp client.")

	r.Register("upload/cleanup", &UploadCleanupCommand{Config: *cfg}, "Delete unmentioned keys for given upload")

	r.RegisterFunc("info", infoCommand, "Info about version and revision of this client")
}
<|endoftext|>


<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// This plugin imports influxdbbackend to register the influxdbbackend stats backend.

import (
	_ "github.com/youtube/vitess/go/stats/influxdbbackend"
)
<commit_msg>Remove influxdb plugin from vtctlclient<commit_after><|endoftext|>


<commit_before>package main

import (
    "npl/bu/edu/unzip"
    "os"
    "os/exec"
    "log"
    "path/filepath"
    "io"
)
func cwdAbs() string {
    cwd, err := filepath.Abs("")
    if err != nil {
        log.Fatalln(err)
    }
    return cwd
}

func main() {
    logf, err := os.OpenFile("boost-win.log", os.O_WRONLY|os.O_CREATE,0640)
    if err != nil {
        log.Fatalln("Failed to create boost-win.log", err)
    }
    log.SetOutput(io.MultiWriter(logf, os.Stdout))

    log.Println(" Version 0.91 ")

    u := &unzip.Unzip{"boost_1_54_0.zip", "", nil}
    if err := u.Expand(); err != nil {
        log.Fatalln("Failed to expand boost_1_54_0.zip ", err)
    }

    boost_dir := cwdAbs()

    if err := os.Chdir("boost_1_54_0"); err != nil {
        log.Fatalln("Failed to build Boost; the directory boost_1_54_0 does not exist ", err)
    }

    build_dir := cwdAbs()
   bootstrap_cmd := "/C "+build_dir+"\\"+"bootstrap"
   log.Println("Executing: "+bootstrap_cmd)
    cmd := exec.Command("cmd", bootstrap_cmd)
    if err := cmd.Run(); err != nil {
        log.Fatalln("Failed to execute boostrap command: "+bootstrap_cmd, err)
    }
    log.Println("Success: "+bootstrap_cmd)

    b2_cmd :=  "/C "+build_dir+"\\"+"b2 install --prefix="+boost_dir+"\\win-x86  address-model=32  runtime-link=shared link=shared"
    log.Println("Executing: "+b2_cmd)
    cmd = exec.Command("cmd", b2_cmd)
    if err := cmd.Run(); err != nil {
        log.Fatalln("Failed to execute b2: ", err)
    }

}
<commit_msg>Cosmetics<commit_after>package main

import (
    "npl/bu/edu/unzip"
    "os"
    "os/exec"
    "log"
    "path/filepath"
    "io"
)

func cwdAbs() string {
    cwd, err := filepath.Abs("")
    if err != nil {
        log.Fatalln(err)
    }
    return cwd
}

func main() {
    logf, err := os.OpenFile("boost-win.log", os.O_WRONLY|os.O_CREATE,0640)
    if err != nil {
        log.Fatalln("Failed to create boost-win.log", err)
    }
    log.SetOutput(io.MultiWriter(logf, os.Stdout))

    log.Println(" Version 0.91 ")

    u := &unzip.Unzip{"boost_1_54_0.zip", "", nil}
    if err := u.Expand(); err != nil {
        log.Fatalln("Failed to expand boost_1_54_0.zip ", err)
    }

    boost_dir := cwdAbs()

    if err := os.Chdir("boost_1_54_0"); err != nil {
        log.Fatalln("Failed to build Boost; the directory boost_1_54_0 does not exist ", err)
    }

    build_dir := cwdAbs()
   bootstrap_cmd := "/C "+build_dir+"\\"+"bootstrap"
   log.Println("Executing: "+bootstrap_cmd)
    cmd := exec.Command("cmd", bootstrap_cmd)
    if err := cmd.Run(); err != nil {
        log.Fatalln("Failed to execute boostrap command: "+bootstrap_cmd, err)
    }
    log.Println("Success: "+bootstrap_cmd)

    b2_cmd :=  "/C "+build_dir+"\\"+"b2 install --prefix="+boost_dir+"\\win-x86  address-model=32  runtime-link=shared link=shared"
    log.Println("Executing: "+b2_cmd)
    cmd = exec.Command("cmd", b2_cmd)
    if err := cmd.Run(); err != nil {
        log.Fatalln("Failed to execute b2: ", err)
    }

}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"

	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/fuzz"
)

var (
	eventPollInterval = 15 * time.Second
	eventPollTimeout  = 3 * time.Minute
)

func TestBackendConfigNotExist(t *testing.T) {
	t.Parallel()

	Framework.RunWithSandbox("BackendConfig not exist", t, func(t *testing.T, s *e2e.Sandbox) {
		testBackendConfigAnnotation := map[string]string{
			annotations.BackendConfigKey: `{"default":"not-exist"}`,
		}
		if _, _, err := e2e.CreateEchoService(s, "service-1", testBackendConfigAnnotation); err != nil {
			t.Fatalf("e2e.CreateEchoService(s, service-1, %q) = _, _, %v, want _, _, nil", testBackendConfigAnnotation, err)
		}

		port80 := intstr.FromInt(80)
		testIng := fuzz.NewIngressBuilder("", "ingress-1", "").
			AddPath("test.com", "/", "service-1", port80).
			Build()
		testIng, err := Framework.Clientset.Extensions().Ingresses(s.Namespace).Create(testIng)
		if err != nil {
			t.Fatalf("error creating Ingress spec: %v", err)
		}
		t.Logf("Ingress %s/%s created", s.Namespace, testIng.Name)

		t.Logf("Waiting for BackendConfig warning event to be emitted")
		if err := wait.Poll(eventPollInterval, eventPollTimeout, func() (bool, error) {
			events, err := Framework.Clientset.CoreV1().Events(s.Namespace).List(metav1.ListOptions{})
			if err != nil {
				return false, fmt.Errorf("error in listing events: %s", err)
			}
			for _, event := range events.Items {
				if event.InvolvedObject.Kind != "Ingress" ||
					event.InvolvedObject.Name != "ingress-1" ||
					event.Type != v1.EventTypeWarning {
					continue
				}
				if strings.Contains(event.Message, "no BackendConfig") {
					t.Logf("BackendConfig warning event emitted")
					return true, nil
				}
			}
			t.Logf("No BackendConfig warning event is emitted yet")
			return false, nil
		}); err != nil {
			t.Fatalf("error waiting for BackendConfig warning event: %v", err)
		}

		testIng, err = Framework.Clientset.Extensions().Ingresses(s.Namespace).Get(testIng.Name, metav1.GetOptions{})
		if err != nil {
			t.Fatalf("error retrieving Ingress %q: %v", testIng.Name, err)
		}
		if len(testIng.Status.LoadBalancer.Ingress) > 0 {
			t.Fatalf("Ingress should not have an IP: %+v", testIng.Status)
		}
	})
}
<commit_msg>Add couple more backend config negative cases<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"

	"k8s.io/ingress-gce/pkg/annotations"
	backendconfigv1beta1 "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/fuzz"
)

var (
	eventPollInterval = 15 * time.Second
	eventPollTimeout  = 3 * time.Minute
)

func TestBackendConfigNegatives(t *testing.T) {
	t.Parallel()

	for _, tc := range []struct {
		desc           string
		svcAnnotations map[string]string
		backendConfig  *backendconfigv1beta1.BackendConfig
		expectedMsg    string
	}{
		{
			desc: "backend config not exist",
			svcAnnotations: map[string]string{
				annotations.BackendConfigKey: `{"default":"backendconfig-1"}`,
			},
			expectedMsg: "no BackendConfig",
		},
		{
			desc: "invalid format in backend config annotation",
			svcAnnotations: map[string]string{
				annotations.BackendConfigKey: `invalid`,
			},
			expectedMsg: fmt.Sprintf("%v", annotations.ErrBackendConfigInvalidJSON),
		},
		{
			desc: "enable both IAP and CDN in backend config",
			svcAnnotations: map[string]string{
				annotations.BackendConfigKey: `{"default":"backendconfig-1"}`,
			},
			backendConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").
				EnableCDN(true).
				SetIAPConfig(true, "bar").
				Build(),
			expectedMsg: "is not valid",
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			t.Parallel()

			if tc.backendConfig != nil {
				if _, err := Framework.BackendConfigClient.CloudV1beta1().BackendConfigs(s.Namespace).Create(tc.backendConfig); err != nil {
					t.Fatalf("Error creating backend config: %v", err)
				}
				t.Logf("Backend config %s/%s created", s.Namespace, tc.backendConfig.Name)
			}

			if _, _, err := e2e.CreateEchoService(s, "service-1", tc.svcAnnotations); err != nil {
				t.Fatalf("e2e.CreateEchoService(s, service-1, %q) = _, _, %v, want _, _, nil", tc.svcAnnotations, err)
			}

			port80 := intstr.FromInt(80)
			testIng := fuzz.NewIngressBuilder("", "ingress-1", "").
				AddPath("test.com", "/", "service-1", port80).
				Build()
			testIng, err := Framework.Clientset.Extensions().Ingresses(s.Namespace).Create(testIng)
			if err != nil {
				t.Fatalf("error creating Ingress spec: %v", err)
			}
			t.Logf("Ingress %s/%s created", s.Namespace, testIng.Name)

			t.Logf("Waiting for warning event to be emitted")
			if err := wait.Poll(eventPollInterval, eventPollTimeout, func() (bool, error) {
				events, err := Framework.Clientset.CoreV1().Events(s.Namespace).List(metav1.ListOptions{})
				if err != nil {
					return false, fmt.Errorf("error in listing events: %s", err)
				}
				for _, event := range events.Items {
					if event.InvolvedObject.Kind != "Ingress" ||
						event.InvolvedObject.Name != "ingress-1" ||
						event.Type != v1.EventTypeWarning {
						continue
					}
					if strings.Contains(event.Message, tc.expectedMsg) {
						t.Logf("Warning event emitted")
						return true, nil
					}
				}
				t.Logf("No warning event is emitted yet")
				return false, nil
			}); err != nil {
				t.Fatalf("error waiting for BackendConfig warning event: %v", err)
			}

			testIng, err = Framework.Clientset.Extensions().Ingresses(s.Namespace).Get(testIng.Name, metav1.GetOptions{})
			if err != nil {
				t.Fatalf("error retrieving Ingress %q: %v", testIng.Name, err)
			}
			if len(testIng.Status.LoadBalancer.Ingress) > 0 {
				t.Fatalf("Ingress should not have an IP: %+v", testIng.Status)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Cobrateam members. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sqlgen

import (
	"reflect"
	"testing"
)

type Person struct {
	Id   int
	Name string
	Age  int
}

func TestFieldNames(t *testing.T) {
	var p Person
	expected := []string{"Id", "Name", "Age"}
	got := fieldNames(reflect.TypeOf(p))
	if !reflect.DeepEqual(expected, got) {
		t.Errorf("Expected %q. Got %q.", expected, got)
	}
}

func TestFieldValues(t *testing.T) {
	p := Person{Id: 1, Name: "Umi", Age: 6}
	expected := []string{"id=1", `name="Umi"`, "age=6"}
	got := fieldValues(reflect.ValueOf(p), []string{"id", "name", "age"})

	if !reflect.DeepEqual(expected, got) {
		t.Errorf("Expected %q. Got %q.", expected, got)
	}
}

func TestGenerateSelectFromStruct(t *testing.T) {
	var p Person
	expected := "select id, name, age from person"
	got := Select(p)
	if expected != got {
		t.Errorf(`SELECT generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestGenerateInsertFromStruct(t *testing.T) {
	var p Person
	expected := "insert into person (id, name, age) values (?, ?, ?)"
	got := Insert(p)
	if expected != got {
		t.Errorf(`INSERT generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestSimpleDeleteFromStruct(t *testing.T) {
	p := Person{1, "Chuck", 32}
	expected := "delete from person where name = Chuck"
	got := Delete(p, []string{"Name"})
	if expected != got {
		t.Errorf(`DELETE generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestMultipleFilterDeleteFromStruct(t *testing.T) {
	p := Person{"Chuck", 32}
	expected := "delete from person where name = Chuck and age = 32"
	got := Delete(p, []string{"Name", "Age"})
	if expected != got {
		t.Errorf(`DELETE generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestGenerateUpdateFromStruct(t *testing.T) {
	p := Person{Id: 1, Name: "Umi", Age: 6}
	expected := `update person name="Umi", age=6 where id=1`
	got := Update(p, []string{"name", "age"}, []string{"id"})

	if expected != got {
		t.Errorf(`UPDATE generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}
<commit_msg>fix on test<commit_after>// Copyright 2012 Cobrateam members. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sqlgen

import (
	"reflect"
	"testing"
)

type Person struct {
	Id   int
	Name string
	Age  int
}

func TestFieldNames(t *testing.T) {
	var p Person
	expected := []string{"Id", "Name", "Age"}
	got := fieldNames(reflect.TypeOf(p))
	if !reflect.DeepEqual(expected, got) {
		t.Errorf("Expected %q. Got %q.", expected, got)
	}
}

func TestFieldValues(t *testing.T) {
	p := Person{Id: 1, Name: "Umi", Age: 6}
	expected := []string{"id=1", `name="Umi"`, "age=6"}
	got := fieldValues(reflect.ValueOf(p), []string{"id", "name", "age"})

	if !reflect.DeepEqual(expected, got) {
		t.Errorf("Expected %q. Got %q.", expected, got)
	}
}

func TestGenerateSelectFromStruct(t *testing.T) {
	var p Person
	expected := "select id, name, age from person"
	got := Select(p)
	if expected != got {
		t.Errorf(`SELECT generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestGenerateInsertFromStruct(t *testing.T) {
	var p Person
	expected := "insert into person (id, name, age) values (?, ?, ?)"
	got := Insert(p)
	if expected != got {
		t.Errorf(`INSERT generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestSimpleDeleteFromStruct(t *testing.T) {
	p := Person{1, "Chuck", 32}
	expected := "delete from person where name = Chuck"
	got := Delete(p, []string{"Name"})
	if expected != got {
		t.Errorf(`DELETE generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestMultipleFilterDeleteFromStruct(t *testing.T) {
	p := Person{1, "Chuck", 32}
	expected := "delete from person where name = Chuck and age = 32"
	got := Delete(p, []string{"Name", "Age"})
	if expected != got {
		t.Errorf(`DELETE generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}

func TestGenerateUpdateFromStruct(t *testing.T) {
	p := Person{Id: 1, Name: "Umi", Age: 6}
	expected := `update person name="Umi", age=6 where id=1`
	got := Update(p, []string{"name", "age"}, []string{"id"})

	if expected != got {
		t.Errorf(`UPDATE generation for %q. Was expecting "%s", got %s.`, reflect.TypeOf(p), expected, got)
	}
}
<|endoftext|>


<commit_before>package kafka

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("kafka.zombiecg", func() monitor.Watcher {
		return &WatchZombieConsumerGroups{
			Tick: time.Minute,
		}
	})
}

type WatchZombieConsumerGroups struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup
}

func (this *WatchZombieConsumerGroups) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
}

func (this *WatchZombieConsumerGroups) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	zombies := metrics.NewRegisteredGauge("consumer.zombie", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kafka.zombiecg stopped")
			return

		case <-ticker.C:
			n := 0
			this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				zbcg := zkcluster.ZombieConsumerGroups(false)
				log.Warn("cluster[%s] zombies: %+v", zkcluster.Name(), zbcg)

				n += len(zbcg)
			})

			zombies.Update(int64(n))
		}
	}

}
<commit_msg>prod consumers are too complex, too many kinds of client sdk<commit_after>package kafka

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("kafka.zombiecg", func() monitor.Watcher {
		return &WatchZombieConsumerGroups{
			Tick: time.Minute,
		}
	})
}

type WatchZombieConsumerGroups struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup
}

func (this *WatchZombieConsumerGroups) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
}

func (this *WatchZombieConsumerGroups) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	zombies := metrics.NewRegisteredGauge("consumer.zombie", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kafka.zombiecg stopped")
			return

		case <-ticker.C:
			n := 0
			this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				if !zkcluster.Public {
					continue
				}

				zbcg := zkcluster.ZombieConsumerGroups(false)
				log.Warn("cluster[%s] zombies: %+v", zkcluster.Name(), zbcg)

				n += len(zbcg)
			})

			zombies.Update(int64(n))
		}
	}

}
<|endoftext|>


<commit_before>package cmd

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/opalmer/gerrittest"
	. "gopkg.in/check.v1"
)

type IntegrationTest struct{}

var _ = Suite(&IntegrationTest{})

func (s *IntegrationTest) Test_StartStop(c *C) {
	if testing.Short() {
		c.Skip("-short set")
	}
	privateFile, err := ioutil.TempFile("", "")
	c.Assert(err, IsNil)
	defer os.Remove(privateFile.Name()) // nolint: errcheck

	// Generate a custom private key and pass it to Start(). This is
	// important to do because we want to replicate a case where a user
	// provides their own key and wants to cleanup after a test run. The
	// expectation is that gerrittest *does not* delete the key during
	// cleanup.
	private, err := gerrittest.GenerateRSAKey()
	c.Assert(err, IsNil)
	c.Assert(gerrittest.WriteRSAKey(private, privateFile), IsNil)

	file, err := ioutil.TempFile("", "")
	c.Assert(err, IsNil)
	defer os.Remove(file.Name()) // nolint: errcheck
	flags := []string{"--json", file.Name(), "--private-key", privateFile.Name()}

	// Call the start subcommand and write the struct to disk.
	c.Assert(file.Close(), IsNil)
	c.Assert(Start.ParseFlags(flags), IsNil)
	c.Assert(Start.RunE(Start, []string{}), IsNil)

	// Load information from json and then perform a few common
	// operations.
	gerrit, err := gerrittest.NewFromJSON(file.Name())
	c.Assert(err, IsNil)
	defer gerrit.Container.Terminate()         // nolint: errcheck
	defer os.RemoveAll(gerrit.Config.RepoRoot) // nolint: errcheck

	client, err := gerrit.HTTP.Gerrit()
	c.Assert(err, IsNil)

	_, _, err = client.Projects.CreateProject("testing", nil)
	c.Assert(err, IsNil)

	// Add a remote based on the container then push a commit to it.
	c.Assert(gerrit.Repo.AddRemoteFromContainer(gerrit.Container, "", "testing"), ErrorMatches, "remote not provided")
	c.Assert(gerrit.Repo.AddRemoteFromContainer(gerrit.Container, "testing", "testing"), IsNil)
	c.Assert(gerrit.Repo.AddContent("foo/bar.txt", 0600, []byte("Hello")), IsNil)
	c.Assert(gerrit.Repo.Commit("42: hello"), IsNil) // Note, the number is an attempt to mess up Push()
	c.Assert(gerrit.Repo.Push("", ""), ErrorMatches, "remote note provided")
	c.Assert(gerrit.Repo.Push("testing", ""), IsNil)
	change, err := gerrit.Repo.ChangeID()
	c.Assert(err, IsNil)

	// Use the change id retrieved from git to retrieve information
	// about the change.
	_, _, err = client.Changes.GetChangeDetail(change, nil)
	c.Assert(err, IsNil)

	// Read the struct from disk and use it to stop the container. This should
	// also cleanup the repository.
	c.Assert(Stop.ParseFlags([]string{"--json", file.Name()}), IsNil)
	c.Assert(Stop.RunE(Stop, []string{}), IsNil)

	// The below test to make sure that the custom private key and
	// repository were not removed when cleanup was run.
	_, err = os.Stat(privateFile.Name())
	c.Assert(err, IsNil)
	_, err = os.Stat(gerrit.Config.RepoRoot)
	c.Assert(os.IsNotExist(err), Equals, true)
}
<commit_msg>fixing tests<commit_after>package cmd

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/opalmer/gerrittest"
	. "gopkg.in/check.v1"
)

type IntegrationTest struct{}

var _ = Suite(&IntegrationTest{})

func (s *IntegrationTest) Test_StartStop(c *C) {
	if testing.Short() {
		c.Skip("-short set")
	}
	privateFile, err := ioutil.TempFile("", "")
	c.Assert(err, IsNil)
	defer os.Remove(privateFile.Name()) // nolint: errcheck

	// Generate a custom private key and pass it to Start(). This is
	// important to do because we want to replicate a case where a user
	// provides their own key and wants to cleanup after a test run. The
	// expectation is that gerrittest *does not* delete the key during
	// cleanup.
	private, err := gerrittest.GenerateRSAKey()
	c.Assert(err, IsNil)
	c.Assert(gerrittest.WriteRSAKey(private, privateFile), IsNil)

	file, err := ioutil.TempFile("", "")
	c.Assert(err, IsNil)
	defer os.Remove(file.Name()) // nolint: errcheck
	flags := []string{"--json", file.Name(), "--private-key", privateFile.Name()}

	// Call the start subcommand and write the struct to disk.
	c.Assert(file.Close(), IsNil)
	c.Assert(Start.ParseFlags(flags), IsNil)
	c.Assert(Start.RunE(Start, []string{}), IsNil)

	// Load information from json and then perform a few common
	// operations.
	gerrit, err := gerrittest.NewFromJSON(file.Name())
	c.Assert(err, IsNil)
	defer gerrit.Container.Terminate()         // nolint: errcheck
	defer os.RemoveAll(gerrit.Config.RepoRoot) // nolint: errcheck

	client, err := gerrit.HTTP.Gerrit()
	c.Assert(err, IsNil)

	_, _, err = client.Projects.CreateProject("testing", nil)
	c.Assert(err, IsNil)

	// Add a remote based on the container then push a commit to it.
	c.Assert(gerrit.Repo.AddRemoteFromContainer(gerrit.Container, "", "testing"), ErrorMatches, gerrittest.ErrRemoteNotProvided.Error())
	c.Assert(gerrit.Repo.AddRemoteFromContainer(gerrit.Container, "testing", "testing"), IsNil)
	c.Assert(gerrit.Repo.AddContent("foo/bar.txt", 0600, []byte("Hello")), IsNil)
	c.Assert(gerrit.Repo.Commit("42: hello"), IsNil) // Note, the number is an attempt to mess up Push()
	c.Assert(gerrit.Repo.Push("", ""), ErrorMatches, gerrittest.ErrRemoteNotProvided.Error())
	c.Assert(gerrit.Repo.Push("testing", ""), IsNil)
	change, err := gerrit.Repo.ChangeID()
	c.Assert(err, IsNil)

	// Use the change id retrieved from git to retrieve information
	// about the change.
	_, _, err = client.Changes.GetChangeDetail(change, nil)
	c.Assert(err, IsNil)

	// Read the struct from disk and use it to stop the container. This should
	// also cleanup the repository.
	c.Assert(Stop.ParseFlags([]string{"--json", file.Name()}), IsNil)
	c.Assert(Stop.RunE(Stop, []string{}), IsNil)

	// The below test to make sure that the custom private key and
	// repository were not removed when cleanup was run.
	_, err = os.Stat(privateFile.Name())
	c.Assert(err, IsNil)
	_, err = os.Stat(gerrit.Config.RepoRoot)
	c.Assert(os.IsNotExist(err), Equals, true)
}
<|endoftext|>


<commit_before>package imaging

import (
	"image"
	"image/color"
)

// Clone returns a copy of the given image.
func Clone(img image.Image) *image.NRGBA {
	dstBounds := img.Bounds().Sub(img.Bounds().Min)
	dst := image.NewNRGBA(dstBounds)

	switch src := img.(type) {
	case *image.NRGBA:
		copyNRGBA(dst, src)
	case *image.NRGBA64:
		copyNRGBA64(dst, src)
	case *image.RGBA:
		copyRGBA(dst, src)
	case *image.RGBA64:
		copyRGBA64(dst, src)
	case *image.Gray:
		copyGray(dst, src)
	case *image.Gray16:
		copyGray16(dst, src)
	case *image.YCbCr:
		copyYCbCr(dst, src)
	case *image.Paletted:
		copyPaletted(dst, src)
	default:
		copyImage(dst, src)
	}

	return dst
}

func copyNRGBA(dst *image.NRGBA, src *image.NRGBA) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	rowSize := dstW * 4
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			copy(dst.Pix[di:di+rowSize], src.Pix[si:si+rowSize])
		}
	})
}

func copyNRGBA64(dst *image.NRGBA, src *image.NRGBA64) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				dst.Pix[di+0] = src.Pix[si+0]
				dst.Pix[di+1] = src.Pix[si+2]
				dst.Pix[di+2] = src.Pix[si+4]
				dst.Pix[di+3] = src.Pix[si+6]
				di += 4
				si += 8
			}
		}
	})
}

func copyRGBA(dst *image.NRGBA, src *image.RGBA) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				a := src.Pix[si+3]
				dst.Pix[di+3] = a

				switch a {
				case 0:
					dst.Pix[di+0] = 0
					dst.Pix[di+1] = 0
					dst.Pix[di+2] = 0
				case 0xff:
					dst.Pix[di+0] = src.Pix[si+0]
					dst.Pix[di+1] = src.Pix[si+1]
					dst.Pix[di+2] = src.Pix[si+2]
				default:
					var tmp uint16
					tmp = uint16(src.Pix[si+0]) * 0xff / uint16(a)
					dst.Pix[di+0] = uint8(tmp)
					tmp = uint16(src.Pix[si+1]) * 0xff / uint16(a)
					dst.Pix[di+1] = uint8(tmp)
					tmp = uint16(src.Pix[si+2]) * 0xff / uint16(a)
					dst.Pix[di+2] = uint8(tmp)
				}

				di += 4
				si += 4
			}
		}
	})
}

func copyRGBA64(dst *image.NRGBA, src *image.RGBA64) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				a := src.Pix[si+6]
				dst.Pix[di+3] = a

				switch a {
				case 0:
					dst.Pix[di+0] = 0
					dst.Pix[di+1] = 0
					dst.Pix[di+2] = 0
				case 0xff:
					dst.Pix[di+0] = src.Pix[si+0]
					dst.Pix[di+1] = src.Pix[si+2]
					dst.Pix[di+2] = src.Pix[si+4]
				default:
					var tmp uint16
					tmp = uint16(src.Pix[si+0]) * 0xff / uint16(a)
					dst.Pix[di+0] = uint8(tmp)
					tmp = uint16(src.Pix[si+2]) * 0xff / uint16(a)
					dst.Pix[di+1] = uint8(tmp)
					tmp = uint16(src.Pix[si+4]) * 0xff / uint16(a)
					dst.Pix[di+2] = uint8(tmp)
				}

				di += 4
				si += 8
			}
		}
	})
}

func copyGray(dst *image.NRGBA, src *image.Gray) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := src.Pix[si]
				dst.Pix[di+0] = c
				dst.Pix[di+1] = c
				dst.Pix[di+2] = c
				dst.Pix[di+3] = 0xff
				di += 4
				si++
			}
		}
	})
}

func copyGray16(dst *image.NRGBA, src *image.Gray16) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := src.Pix[si]
				dst.Pix[di+0] = c
				dst.Pix[di+1] = c
				dst.Pix[di+2] = c
				dst.Pix[di+3] = 0xff
				di += 4
				si += 2
			}
		}
	})
}

func copyYCbCr(dst *image.NRGBA, src *image.YCbCr) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				srcX := srcMinX + dstX
				srcY := srcMinY + dstY
				siy := src.YOffset(srcX, srcY)
				sic := src.COffset(srcX, srcY)
				r, g, b := color.YCbCrToRGB(src.Y[siy], src.Cb[sic], src.Cr[sic])
				dst.Pix[di+0] = r
				dst.Pix[di+1] = g
				dst.Pix[di+2] = b
				dst.Pix[di+3] = 0xff
				di += 4
			}
		}
	})
}

func copyPaletted(dst *image.NRGBA, src *image.Paletted) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	plen := len(src.Palette)
	pnew := make([]color.NRGBA, plen)
	for i := 0; i < plen; i++ {
		pnew[i] = color.NRGBAModel.Convert(src.Palette[i]).(color.NRGBA)
	}
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := pnew[src.Pix[si]]
				dst.Pix[di+0] = c.R
				dst.Pix[di+1] = c.G
				dst.Pix[di+2] = c.B
				dst.Pix[di+3] = c.A
				di += 4
				si++
			}
		}
	})
}

func copyImage(dst *image.NRGBA, src image.Image) {
	srcMinX := src.Bounds().Min.X
	srcMinY := src.Bounds().Min.Y
	dstW := dst.Bounds().Dx()
	dstH := dst.Bounds().Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := color.NRGBAModel.Convert(src.At(srcMinX+dstX, srcMinY+dstY)).(color.NRGBA)
				dst.Pix[di+0] = c.R
				dst.Pix[di+1] = c.G
				dst.Pix[di+2] = c.B
				dst.Pix[di+3] = c.A
				di += 4
			}
		}
	})
}

// toNRGBA converts any image type to *image.NRGBA with min-point at (0, 0).
func toNRGBA(img image.Image) *image.NRGBA {
	if img, ok := img.(*image.NRGBA); ok && img.Bounds().Min.Eq(image.ZP) {
		return img
	}
	return Clone(img)
}
<commit_msg>clone: inline YCbCr color conversion<commit_after>package imaging

import (
	"image"
	"image/color"
)

// Clone returns a copy of the given image.
func Clone(img image.Image) *image.NRGBA {
	dstBounds := img.Bounds().Sub(img.Bounds().Min)
	dst := image.NewNRGBA(dstBounds)

	switch src := img.(type) {
	case *image.NRGBA:
		copyNRGBA(dst, src)
	case *image.NRGBA64:
		copyNRGBA64(dst, src)
	case *image.RGBA:
		copyRGBA(dst, src)
	case *image.RGBA64:
		copyRGBA64(dst, src)
	case *image.Gray:
		copyGray(dst, src)
	case *image.Gray16:
		copyGray16(dst, src)
	case *image.YCbCr:
		copyYCbCr(dst, src)
	case *image.Paletted:
		copyPaletted(dst, src)
	default:
		copyImage(dst, src)
	}

	return dst
}

func copyNRGBA(dst *image.NRGBA, src *image.NRGBA) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	rowSize := dstW * 4
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			copy(dst.Pix[di:di+rowSize], src.Pix[si:si+rowSize])
		}
	})
}

func copyNRGBA64(dst *image.NRGBA, src *image.NRGBA64) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				dst.Pix[di+0] = src.Pix[si+0]
				dst.Pix[di+1] = src.Pix[si+2]
				dst.Pix[di+2] = src.Pix[si+4]
				dst.Pix[di+3] = src.Pix[si+6]
				di += 4
				si += 8
			}
		}
	})
}

func copyRGBA(dst *image.NRGBA, src *image.RGBA) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				a := src.Pix[si+3]
				dst.Pix[di+3] = a

				switch a {
				case 0:
					dst.Pix[di+0] = 0
					dst.Pix[di+1] = 0
					dst.Pix[di+2] = 0
				case 0xff:
					dst.Pix[di+0] = src.Pix[si+0]
					dst.Pix[di+1] = src.Pix[si+1]
					dst.Pix[di+2] = src.Pix[si+2]
				default:
					var tmp uint16
					tmp = uint16(src.Pix[si+0]) * 0xff / uint16(a)
					dst.Pix[di+0] = uint8(tmp)
					tmp = uint16(src.Pix[si+1]) * 0xff / uint16(a)
					dst.Pix[di+1] = uint8(tmp)
					tmp = uint16(src.Pix[si+2]) * 0xff / uint16(a)
					dst.Pix[di+2] = uint8(tmp)
				}

				di += 4
				si += 4
			}
		}
	})
}

func copyRGBA64(dst *image.NRGBA, src *image.RGBA64) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				a := src.Pix[si+6]
				dst.Pix[di+3] = a

				switch a {
				case 0:
					dst.Pix[di+0] = 0
					dst.Pix[di+1] = 0
					dst.Pix[di+2] = 0
				case 0xff:
					dst.Pix[di+0] = src.Pix[si+0]
					dst.Pix[di+1] = src.Pix[si+2]
					dst.Pix[di+2] = src.Pix[si+4]
				default:
					var tmp uint16
					tmp = uint16(src.Pix[si+0]) * 0xff / uint16(a)
					dst.Pix[di+0] = uint8(tmp)
					tmp = uint16(src.Pix[si+2]) * 0xff / uint16(a)
					dst.Pix[di+1] = uint8(tmp)
					tmp = uint16(src.Pix[si+4]) * 0xff / uint16(a)
					dst.Pix[di+2] = uint8(tmp)
				}

				di += 4
				si += 8
			}
		}
	})
}

func copyGray(dst *image.NRGBA, src *image.Gray) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := src.Pix[si]
				dst.Pix[di+0] = c
				dst.Pix[di+1] = c
				dst.Pix[di+2] = c
				dst.Pix[di+3] = 0xff
				di += 4
				si++
			}
		}
	})
}

func copyGray16(dst *image.NRGBA, src *image.Gray16) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := src.Pix[si]
				dst.Pix[di+0] = c
				dst.Pix[di+1] = c
				dst.Pix[di+2] = c
				dst.Pix[di+3] = 0xff
				di += 4
				si += 2
			}
		}
	})
}

func copyYCbCr(dst *image.NRGBA, src *image.YCbCr) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				srcX := srcMinX + dstX
				srcY := srcMinY + dstY

				siy := (srcY-src.Rect.Min.Y)*src.YStride + (srcX - src.Rect.Min.X)

				var sic int
				switch src.SubsampleRatio {
				case image.YCbCrSubsampleRatio444:
					sic = (srcY-src.Rect.Min.Y)*src.CStride + (srcX - src.Rect.Min.X)
				case image.YCbCrSubsampleRatio422:
					sic = (srcY-src.Rect.Min.Y)*src.CStride + (srcX/2 - src.Rect.Min.X/2)
				case image.YCbCrSubsampleRatio420:
					sic = (srcY/2-src.Rect.Min.Y/2)*src.CStride + (srcX/2 - src.Rect.Min.X/2)
				case image.YCbCrSubsampleRatio440:
					sic = (srcY/2-src.Rect.Min.Y/2)*src.CStride + (srcX - src.Rect.Min.X)
				default:
					sic = src.COffset(srcX, srcY)
				}

				yy1 := int32(src.Y[siy]) * 0x10101
				cb1 := int32(src.Cb[sic]) - 128
				cr1 := int32(src.Cr[sic]) - 128

				r := yy1 + 91881*cr1
				if uint32(r)&0xff000000 == 0 {
					r >>= 16
				} else {
					r = ^(r >> 31)
				}

				g := yy1 - 22554*cb1 - 46802*cr1
				if uint32(g)&0xff000000 == 0 {
					g >>= 16
				} else {
					g = ^(g >> 31)
				}

				b := yy1 + 116130*cb1
				if uint32(b)&0xff000000 == 0 {
					b >>= 16
				} else {
					b = ^(b >> 31)
				}

				dst.Pix[di+0] = uint8(r)
				dst.Pix[di+1] = uint8(g)
				dst.Pix[di+2] = uint8(b)
				dst.Pix[di+3] = 0xff

				di += 4
			}
		}
	})
}

func copyPaletted(dst *image.NRGBA, src *image.Paletted) {
	srcMinX := src.Rect.Min.X
	srcMinY := src.Rect.Min.Y
	dstW := dst.Rect.Dx()
	dstH := dst.Rect.Dy()
	plen := len(src.Palette)
	pnew := make([]color.NRGBA, plen)
	for i := 0; i < plen; i++ {
		pnew[i] = color.NRGBAModel.Convert(src.Palette[i]).(color.NRGBA)
	}
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			si := src.PixOffset(srcMinX, srcMinY+dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := pnew[src.Pix[si]]
				dst.Pix[di+0] = c.R
				dst.Pix[di+1] = c.G
				dst.Pix[di+2] = c.B
				dst.Pix[di+3] = c.A
				di += 4
				si++
			}
		}
	})
}

func copyImage(dst *image.NRGBA, src image.Image) {
	srcMinX := src.Bounds().Min.X
	srcMinY := src.Bounds().Min.Y
	dstW := dst.Bounds().Dx()
	dstH := dst.Bounds().Dy()
	parallel(dstH, func(partStart, partEnd int) {
		for dstY := partStart; dstY < partEnd; dstY++ {
			di := dst.PixOffset(0, dstY)
			for dstX := 0; dstX < dstW; dstX++ {
				c := color.NRGBAModel.Convert(src.At(srcMinX+dstX, srcMinY+dstY)).(color.NRGBA)
				dst.Pix[di+0] = c.R
				dst.Pix[di+1] = c.G
				dst.Pix[di+2] = c.B
				dst.Pix[di+3] = c.A
				di += 4
			}
		}
	})
}

// toNRGBA converts any image type to *image.NRGBA with min-point at (0, 0).
func toNRGBA(img image.Image) *image.NRGBA {
	if img, ok := img.(*image.NRGBA); ok && img.Bounds().Min.Eq(image.ZP) {
		return img
	}
	return Clone(img)
}
<|endoftext|>


<commit_before>package xmpp

// This file contains data structures.

import (
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"log"
	"reflect"
	"regexp"
	"strings"
)

// BUG(cjyar): Doesn't use stringprep. Could try the implementation at
// "code.google.com/p/go-idn/src/stringprep"

// JID represents an entity that can communicate with other
// entities. It looks like node@domain/resource. Node and resource are
// sometimes optional.
type JID struct {
	Node     string
	Domain   string
	Resource string
}

var _ fmt.Stringer = &JID{}
var _ flag.Value = &JID{}

// XMPP's <stream:stream> XML element
type stream struct {
	XMLName xml.Name `xml:"stream=http://etherx.jabber.org/streams stream"`
	To      string   `xml:"to,attr"`
	From    string   `xml:"from,attr"`
	Id      string   `xml:"id,attr"`
	Lang    string   `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`
	Version string   `xml:"version,attr"`
}

var _ fmt.Stringer = &stream{}

// <stream:error>
type streamError struct {
	XMLName xml.Name `xml:"http://etherx.jabber.org/streams error"`
	Any     Generic  `xml:",any"`
	Text    *errText
}

type errText struct {
	XMLName xml.Name `xml:"urn:ietf:params:xml:ns:xmpp-streams text"`
	Lang    string   `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`
	Text    string   `xml:",chardata"`
}

type Features struct {
	Starttls   *starttls `xml:"urn:ietf:params:xml:ns:xmpp-tls starttls"`
	Mechanisms mechs     `xml:"urn:ietf:params:xml:ns:xmpp-sasl mechanisms"`
	Bind       *bindIq
	Session    *Generic
	Any        *Generic
}

type starttls struct {
	XMLName  xml.Name
	Required *string
}

type mechs struct {
	Mechanism []string `xml:"urn:ietf:params:xml:ns:xmpp-sasl mechanism"`
}

type auth struct {
	XMLName   xml.Name
	Chardata  string `xml:",chardata"`
	Mechanism string `xml:"mechanism,attr,omitempty"`
	Any       *Generic
}

type Stanza interface {
	GetHeader() *Header
}

// One of the three core XMPP stanza types: iq, message, presence. See
// RFC3920, section 9.
type Header struct {
	To       string `xml:"to,attr,omitempty"`
	From     string `xml:"from,attr,omitempty"`
	Id       string `xml:"id,attr,omitempty"`
	Type     string `xml:"type,attr,omitempty"`
	Lang     string `xml:"http://www.w3.org/XML/1998/namespace lang,attr,omitempty"`
	Innerxml string `xml:",innerxml"`
	Error    *Error
	Nested   []interface{}
}

// message stanza
type Message struct {
	XMLName xml.Name `xml:"jabber:client message"`
	Header
	Subject []Text   `xml:"jabber:client subject"`
	Body    []Text   `xml:"jabber:client body"`
	Thread  *Data  `xml:"jabber:client thread"`
}

var _ Stanza = &Message{}

// presence stanza
type Presence struct {
	XMLName xml.Name `xml:"presence"`
	Header
	Show     *Data   `xml:"jabber:client show"`
	Status   []Text  `xml:"jabber:client status"`
	Priority *Data   `xml:"jabber:client priority"`
}

var _ Stanza = &Presence{}

// iq stanza
type Iq struct {
	XMLName xml.Name `xml:"iq"`
	Header
}

var _ Stanza = &Iq{}

// Describes an XMPP stanza error. See RFC 3920, Section 9.3.
type Error struct {
	XMLName xml.Name `xml:"error"`
	// The error type attribute.
	Type string `xml:"type,attr"`
	// Any nested element, if present.
	Any *Generic
}

var _ error = &Error{}

// Used for resource binding as a nested element inside <iq/>.
type bindIq struct {
	XMLName  xml.Name `xml:"urn:ietf:params:xml:ns:xmpp-bind bind"`
	Resource *string  `xml:"resource"`
	Jid      *string  `xml:"jid"`
}

// Holds human-readable text, with an optional language
// specification. Generally multiple instances of these can be found
// together, allowing the software to choose which language to present
// to the user.
type Text struct {
	XMLName xml.Name
	Lang     string `xml:"http://www.w3.org/XML/1998/namespace lang,attr,omitempty"`
	Chardata string `xml:",chardata"`
}

// Non-human-readable content of some sort, used by the protocol.
type Data struct {
	XMLName xml.Name
	Chardata string `xml:",chardata"`
}

// Holds an XML element not described by the more specific types.
type Generic struct {
	XMLName  xml.Name
	Any      *Generic `xml:",any"`
	Chardata string   `xml:",chardata"`
}

var _ fmt.Stringer = &Generic{}

func (jid *JID) String() string {
	result := jid.Domain
	if jid.Node != "" {
		result = jid.Node + "@" + result
	}
	if jid.Resource != "" {
		result = result + "/" + jid.Resource
	}
	return result
}

// Set implements flag.Value.
func (jid *JID) Set(val string) error {
	r := regexp.MustCompile("^(([^@/]+)@)?([^@/]+)(/([^@/]+))?$")
	parts := r.FindStringSubmatch(val)
	if parts == nil {
		return fmt.Errorf("%s doesn't match user@domain/resource", val)
	}
	// jid.Node = stringprep.Nodeprep(parts[2])
	// jid.Domain = stringprep.Nodeprep(parts[3])
	// jid.Resource = stringprep.Resourceprep(parts[5])
	jid.Node = parts[2]
	jid.Domain = parts[3]
	jid.Resource = parts[5]
	return nil
}

func (s *stream) String() string {
	var buf bytes.Buffer
	buf.WriteString(`<stream:stream xmlns="`)
	buf.WriteString(NsClient)
	buf.WriteString(`" xmlns:stream="`)
	buf.WriteString(NsStream)
	buf.WriteString(`"`)
	if s.To != "" {
		buf.WriteString(` to="`)
		xml.Escape(&buf, []byte(s.To))
		buf.WriteString(`"`)
	}
	if s.From != "" {
		buf.WriteString(` from="`)
		xml.Escape(&buf, []byte(s.From))
		buf.WriteString(`"`)
	}
	if s.Id != "" {
		buf.WriteString(` id="`)
		xml.Escape(&buf, []byte(s.Id))
		buf.WriteString(`"`)
	}
	if s.Lang != "" {
		buf.WriteString(` xml:lang="`)
		xml.Escape(&buf, []byte(s.Lang))
		buf.WriteString(`"`)
	}
	if s.Version != "" {
		buf.WriteString(` version="`)
		xml.Escape(&buf, []byte(s.Version))
		buf.WriteString(`"`)
	}
	buf.WriteString(">")
	return buf.String()
}

func parseStream(se xml.StartElement) (*stream, error) {
	s := &stream{}
	for _, attr := range se.Attr {
		switch strings.ToLower(attr.Name.Local) {
		case "to":
			s.To = attr.Value
		case "from":
			s.From = attr.Value
		case "id":
			s.Id = attr.Value
		case "lang":
			s.Lang = attr.Value
		case "version":
			s.Version = attr.Value
		}
	}
	return s, nil
}

func (iq *Iq) GetHeader() *Header {
	return &iq.Header
}

func (m *Message) GetHeader() *Header {
	return &m.Header
}

func (p *Presence) GetHeader() *Header {
	return &p.Header
}

func (u *Generic) String() string {
	if u == nil {
		return "nil"
	}
	var sub string
	if u.Any != nil {
		sub = u.Any.String()
	}
	return fmt.Sprintf("<%s %s>%s%s</%s %s>", u.XMLName.Space,
		u.XMLName.Local, sub, u.Chardata, u.XMLName.Space,
		u.XMLName.Local)
}

func (er *Error) Error() string {
	buf, err := xml.Marshal(er)
	if err != nil {
		log.Println("double bad error: couldn't marshal error")
		return "unreadable error"
	}
	return string(buf)
}

var bindExt Extension = Extension{}

func init() {
	bindExt.StanzaTypes = make(map[xml.Name]reflect.Type)
	bName := xml.Name{Space: NsBind, Local: "bind"}
	bindExt.StanzaTypes[bName] = reflect.TypeOf(bindIq{})
}
<commit_msg>go fmt<commit_after>package xmpp

// This file contains data structures.

import (
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"log"
	"reflect"
	"regexp"
	"strings"
)

// BUG(cjyar): Doesn't use stringprep. Could try the implementation at
// "code.google.com/p/go-idn/src/stringprep"

// JID represents an entity that can communicate with other
// entities. It looks like node@domain/resource. Node and resource are
// sometimes optional.
type JID struct {
	Node     string
	Domain   string
	Resource string
}

var _ fmt.Stringer = &JID{}
var _ flag.Value = &JID{}

// XMPP's <stream:stream> XML element
type stream struct {
	XMLName xml.Name `xml:"stream=http://etherx.jabber.org/streams stream"`
	To      string   `xml:"to,attr"`
	From    string   `xml:"from,attr"`
	Id      string   `xml:"id,attr"`
	Lang    string   `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`
	Version string   `xml:"version,attr"`
}

var _ fmt.Stringer = &stream{}

// <stream:error>
type streamError struct {
	XMLName xml.Name `xml:"http://etherx.jabber.org/streams error"`
	Any     Generic  `xml:",any"`
	Text    *errText
}

type errText struct {
	XMLName xml.Name `xml:"urn:ietf:params:xml:ns:xmpp-streams text"`
	Lang    string   `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`
	Text    string   `xml:",chardata"`
}

type Features struct {
	Starttls   *starttls `xml:"urn:ietf:params:xml:ns:xmpp-tls starttls"`
	Mechanisms mechs     `xml:"urn:ietf:params:xml:ns:xmpp-sasl mechanisms"`
	Bind       *bindIq
	Session    *Generic
	Any        *Generic
}

type starttls struct {
	XMLName  xml.Name
	Required *string
}

type mechs struct {
	Mechanism []string `xml:"urn:ietf:params:xml:ns:xmpp-sasl mechanism"`
}

type auth struct {
	XMLName   xml.Name
	Chardata  string `xml:",chardata"`
	Mechanism string `xml:"mechanism,attr,omitempty"`
	Any       *Generic
}

type Stanza interface {
	GetHeader() *Header
}

// One of the three core XMPP stanza types: iq, message, presence. See
// RFC3920, section 9.
type Header struct {
	To       string `xml:"to,attr,omitempty"`
	From     string `xml:"from,attr,omitempty"`
	Id       string `xml:"id,attr,omitempty"`
	Type     string `xml:"type,attr,omitempty"`
	Lang     string `xml:"http://www.w3.org/XML/1998/namespace lang,attr,omitempty"`
	Innerxml string `xml:",innerxml"`
	Error    *Error
	Nested   []interface{}
}

// message stanza
type Message struct {
	XMLName xml.Name `xml:"jabber:client message"`
	Header
	Subject []Text `xml:"jabber:client subject"`
	Body    []Text `xml:"jabber:client body"`
	Thread  *Data  `xml:"jabber:client thread"`
}

var _ Stanza = &Message{}

// presence stanza
type Presence struct {
	XMLName xml.Name `xml:"presence"`
	Header
	Show     *Data  `xml:"jabber:client show"`
	Status   []Text `xml:"jabber:client status"`
	Priority *Data  `xml:"jabber:client priority"`
}

var _ Stanza = &Presence{}

// iq stanza
type Iq struct {
	XMLName xml.Name `xml:"iq"`
	Header
}

var _ Stanza = &Iq{}

// Describes an XMPP stanza error. See RFC 3920, Section 9.3.
type Error struct {
	XMLName xml.Name `xml:"error"`
	// The error type attribute.
	Type string `xml:"type,attr"`
	// Any nested element, if present.
	Any *Generic
}

var _ error = &Error{}

// Used for resource binding as a nested element inside <iq/>.
type bindIq struct {
	XMLName  xml.Name `xml:"urn:ietf:params:xml:ns:xmpp-bind bind"`
	Resource *string  `xml:"resource"`
	Jid      *string  `xml:"jid"`
}

// Holds human-readable text, with an optional language
// specification. Generally multiple instances of these can be found
// together, allowing the software to choose which language to present
// to the user.
type Text struct {
	XMLName  xml.Name
	Lang     string `xml:"http://www.w3.org/XML/1998/namespace lang,attr,omitempty"`
	Chardata string `xml:",chardata"`
}

// Non-human-readable content of some sort, used by the protocol.
type Data struct {
	XMLName  xml.Name
	Chardata string `xml:",chardata"`
}

// Holds an XML element not described by the more specific types.
type Generic struct {
	XMLName  xml.Name
	Any      *Generic `xml:",any"`
	Chardata string   `xml:",chardata"`
}

var _ fmt.Stringer = &Generic{}

func (jid *JID) String() string {
	result := jid.Domain
	if jid.Node != "" {
		result = jid.Node + "@" + result
	}
	if jid.Resource != "" {
		result = result + "/" + jid.Resource
	}
	return result
}

// Set implements flag.Value.
func (jid *JID) Set(val string) error {
	r := regexp.MustCompile("^(([^@/]+)@)?([^@/]+)(/([^@/]+))?$")
	parts := r.FindStringSubmatch(val)
	if parts == nil {
		return fmt.Errorf("%s doesn't match user@domain/resource", val)
	}
	// jid.Node = stringprep.Nodeprep(parts[2])
	// jid.Domain = stringprep.Nodeprep(parts[3])
	// jid.Resource = stringprep.Resourceprep(parts[5])
	jid.Node = parts[2]
	jid.Domain = parts[3]
	jid.Resource = parts[5]
	return nil
}

func (s *stream) String() string {
	var buf bytes.Buffer
	buf.WriteString(`<stream:stream xmlns="`)
	buf.WriteString(NsClient)
	buf.WriteString(`" xmlns:stream="`)
	buf.WriteString(NsStream)
	buf.WriteString(`"`)
	if s.To != "" {
		buf.WriteString(` to="`)
		xml.Escape(&buf, []byte(s.To))
		buf.WriteString(`"`)
	}
	if s.From != "" {
		buf.WriteString(` from="`)
		xml.Escape(&buf, []byte(s.From))
		buf.WriteString(`"`)
	}
	if s.Id != "" {
		buf.WriteString(` id="`)
		xml.Escape(&buf, []byte(s.Id))
		buf.WriteString(`"`)
	}
	if s.Lang != "" {
		buf.WriteString(` xml:lang="`)
		xml.Escape(&buf, []byte(s.Lang))
		buf.WriteString(`"`)
	}
	if s.Version != "" {
		buf.WriteString(` version="`)
		xml.Escape(&buf, []byte(s.Version))
		buf.WriteString(`"`)
	}
	buf.WriteString(">")
	return buf.String()
}

func parseStream(se xml.StartElement) (*stream, error) {
	s := &stream{}
	for _, attr := range se.Attr {
		switch strings.ToLower(attr.Name.Local) {
		case "to":
			s.To = attr.Value
		case "from":
			s.From = attr.Value
		case "id":
			s.Id = attr.Value
		case "lang":
			s.Lang = attr.Value
		case "version":
			s.Version = attr.Value
		}
	}
	return s, nil
}

func (iq *Iq) GetHeader() *Header {
	return &iq.Header
}

func (m *Message) GetHeader() *Header {
	return &m.Header
}

func (p *Presence) GetHeader() *Header {
	return &p.Header
}

func (u *Generic) String() string {
	if u == nil {
		return "nil"
	}
	var sub string
	if u.Any != nil {
		sub = u.Any.String()
	}
	return fmt.Sprintf("<%s %s>%s%s</%s %s>", u.XMLName.Space,
		u.XMLName.Local, sub, u.Chardata, u.XMLName.Space,
		u.XMLName.Local)
}

func (er *Error) Error() string {
	buf, err := xml.Marshal(er)
	if err != nil {
		log.Println("double bad error: couldn't marshal error")
		return "unreadable error"
	}
	return string(buf)
}

var bindExt Extension = Extension{}

func init() {
	bindExt.StanzaTypes = make(map[xml.Name]reflect.Type)
	bName := xml.Name{Space: NsBind, Local: "bind"}
	bindExt.StanzaTypes[bName] = reflect.TypeOf(bindIq{})
}
<|endoftext|>


<commit_before>// +build linux

package mtree

import (
	"archive/tar"
	"encoding/base64"
	"fmt"
	"io"
	"os"
	"os/user"
	"strings"
	"syscall"

	"github.com/vbatts/go-mtree/xattr"
)

var (
	// this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
	flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		return emptyKV, nil
	}

	unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
		}

		stat := info.Sys().(*syscall.Stat_t)
		u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
		if err != nil {
			return emptyKV, err
		}
		return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil
	}
	uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
		}
		stat := info.Sys().(*syscall.Stat_t)
		return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil
	}
	gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
		}
		if stat, ok := info.Sys().(*syscall.Stat_t); ok {
			return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil
		}
		return emptyKV, nil
	}
	nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if stat, ok := info.Sys().(*syscall.Stat_t); ok {
			return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil
		}
		return emptyKV, nil
	}
	xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			if len(hdr.Xattrs) == 0 {
				return emptyKV, nil
			}
			klist := []KeyVal{}
			for k, v := range hdr.Xattrs {
				klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", k, base64.StdEncoding.EncodeToString([]byte(v)))))
			}
			return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
		}
		if !info.Mode().IsRegular() && !info.Mode().IsDir() {
			return emptyKV, nil
		}

		xlist, err := xattr.List(path)
		if err != nil {
			return emptyKV, err
		}
		klist := make([]KeyVal, len(xlist))
		for i := range xlist {
			data, err := xattr.Get(path, xlist[i])
			if err != nil {
				return emptyKV, err
			}
			klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", xlist[i], base64.StdEncoding.EncodeToString(data)))
		}
		return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
	}
)
<commit_msg>keywords: encode xattr.* keywords with Vis<commit_after>// +build linux

package mtree

import (
	"archive/tar"
	"encoding/base64"
	"fmt"
	"io"
	"os"
	"os/user"
	"strings"
	"syscall"

	"github.com/vbatts/go-mtree/xattr"
)

var (
	// this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
	flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		return emptyKV, nil
	}

	unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			return KeyVal(fmt.Sprintf("uname=%s", hdr.Uname)), nil
		}

		stat := info.Sys().(*syscall.Stat_t)
		u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
		if err != nil {
			return emptyKV, err
		}
		return KeyVal(fmt.Sprintf("uname=%s", u.Username)), nil
	}
	uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			return KeyVal(fmt.Sprintf("uid=%d", hdr.Uid)), nil
		}
		stat := info.Sys().(*syscall.Stat_t)
		return KeyVal(fmt.Sprintf("uid=%d", stat.Uid)), nil
	}
	gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			return KeyVal(fmt.Sprintf("gid=%d", hdr.Gid)), nil
		}
		if stat, ok := info.Sys().(*syscall.Stat_t); ok {
			return KeyVal(fmt.Sprintf("gid=%d", stat.Gid)), nil
		}
		return emptyKV, nil
	}
	nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if stat, ok := info.Sys().(*syscall.Stat_t); ok {
			return KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink)), nil
		}
		return emptyKV, nil
	}
	xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) (KeyVal, error) {
		if hdr, ok := info.Sys().(*tar.Header); ok {
			if len(hdr.Xattrs) == 0 {
				return emptyKV, nil
			}
			klist := []KeyVal{}
			for k, v := range hdr.Xattrs {
				encKey, err := Vis(k, DefaultVisFlags)
				if err != nil {
					return emptyKV, err
				}
				klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString([]byte(v)))))
			}
			return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
		}
		if !info.Mode().IsRegular() && !info.Mode().IsDir() {
			return emptyKV, nil
		}

		xlist, err := xattr.List(path)
		if err != nil {
			return emptyKV, err
		}
		klist := make([]KeyVal, len(xlist))
		for i := range xlist {
			data, err := xattr.Get(path, xlist[i])
			if err != nil {
				return emptyKV, err
			}
			encKey, err := Vis(xlist[i], DefaultVisFlags)
			if err != nil {
				return emptyKV, err
			}
			klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString(data)))
		}
		return KeyVal(strings.Join(KeyValToString(klist), " ")), nil
	}
)
<|endoftext|>


<commit_before>package goridge

import (
	"errors"
	json "github.com/json-iterator/go"
	"io"
	"net/rpc"
	"reflect"
)

// Codec represent net/rpc bridge over Goridge socket relay.
type Codec struct {
	relay  Relay
	closed bool
}

// NewCodec initiates new server rpc codec over socket connection.
func NewCodec(rwc io.ReadWriteCloser) *Codec {
	return &Codec{relay: NewSocketRelay(rwc)}
}

// ReadRequestHeader receives
func (c *Codec) ReadRequestHeader(r *rpc.Request) error {
	data, p, err := c.relay.Receive()
	if err != nil {
		return err
	}

	if !p.HasFlag(PayloadControl) {
		return errors.New("invalid rpc header, control flag is missing")
	}

	if !p.HasFlag(PayloadRaw) {
		return errors.New("rpc response header must be in {rawData}")
	}

	if !p.HasPayload() {
		return errors.New("rpc request header can't be empty")
	}

	return unpack(data, &r.ServiceMethod, &r.Seq)
}

// ReadRequestBody fetches prefixed body data and automatically unmarshal it as json. RawBody flag will populate
// []byte lice argument for rpc method.
func (c *Codec) ReadRequestBody(out interface{}) error {
	data, p, err := c.relay.Receive()
	if err != nil {
		return err
	}

	if out == nil {
		// discarding
		return nil
	}

	if !p.HasPayload() {
		return nil
	}

	if p.HasFlag(PayloadRaw) {
		if bin, ok := out.(*[]byte); ok {
			*bin = append(*bin, data...)
			return nil
		}

		return errors.New("{rawData} request for " + reflect.ValueOf(out).String())
	}

	return json.Unmarshal(data, out)
}

// WriteResponse marshals response, byte slice or error to remote party.
func (c *Codec) WriteResponse(r *rpc.Response, body interface{}) error {
	data := make([]byte, len(r.ServiceMethod)+Uint64Size)
	pack(r.ServiceMethod, r.Seq, data)
	if err := c.relay.Send(data, PayloadControl|PayloadRaw); err != nil {
		return err
	}

	if r.Error != "" {
		return c.relay.Send([]byte(r.Error), PayloadError|PayloadRaw)
	}

	switch bin := body.(type) {
	case *[]byte:
		return c.relay.Send(*bin, PayloadRaw)
	case []byte:
		return c.relay.Send(bin, PayloadRaw)
	}

	packed, err := json.Marshal(body)
	if err != nil {
		return err
	}

	return c.relay.Send(packed, 0)
}

// Close underlying socket.
func (c *Codec) Close() error {
	if c.closed {
		return nil
	}

	c.closed = true
	return c.relay.Close()
}
<commit_msg>feat: allow user to customize codec relay<commit_after>package goridge

import (
	"errors"
	json "github.com/json-iterator/go"
	"io"
	"net/rpc"
	"reflect"
)

// Codec represent net/rpc bridge over Goridge socket relay.
type Codec struct {
	relay  Relay
	closed bool
}

// NewCodec initiates new server rpc codec over socket connection.
func NewCodec(rwc io.ReadWriteCloser) *Codec {
	return &Codec{relay: NewSocketRelay(rwc)}
}

func NewCodecWithRelay(relay Relay) *Codec {
	return &Codec{relay: relay}
}

// ReadRequestHeader receives
func (c *Codec) ReadRequestHeader(r *rpc.Request) error {
	data, p, err := c.relay.Receive()
	if err != nil {
		return err
	}

	if !p.HasFlag(PayloadControl) {
		return errors.New("invalid rpc header, control flag is missing")
	}

	if !p.HasFlag(PayloadRaw) {
		return errors.New("rpc response header must be in {rawData}")
	}

	if !p.HasPayload() {
		return errors.New("rpc request header can't be empty")
	}

	return unpack(data, &r.ServiceMethod, &r.Seq)
}

// ReadRequestBody fetches prefixed body data and automatically unmarshal it as json. RawBody flag will populate
// []byte lice argument for rpc method.
func (c *Codec) ReadRequestBody(out interface{}) error {
	data, p, err := c.relay.Receive()
	if err != nil {
		return err
	}

	if out == nil {
		// discarding
		return nil
	}

	if !p.HasPayload() {
		return nil
	}

	if p.HasFlag(PayloadRaw) {
		if bin, ok := out.(*[]byte); ok {
			*bin = append(*bin, data...)
			return nil
		}

		return errors.New("{rawData} request for " + reflect.ValueOf(out).String())
	}

	return json.Unmarshal(data, out)
}

// WriteResponse marshals response, byte slice or error to remote party.
func (c *Codec) WriteResponse(r *rpc.Response, body interface{}) error {
	data := make([]byte, len(r.ServiceMethod)+Uint64Size)
	pack(r.ServiceMethod, r.Seq, data)
	if err := c.relay.Send(data, PayloadControl|PayloadRaw); err != nil {
		return err
	}

	if r.Error != "" {
		return c.relay.Send([]byte(r.Error), PayloadError|PayloadRaw)
	}

	switch bin := body.(type) {
	case *[]byte:
		return c.relay.Send(*bin, PayloadRaw)
	case []byte:
		return c.relay.Send(bin, PayloadRaw)
	}

	packed, err := json.Marshal(body)
	if err != nil {
		return err
	}

	return c.relay.Send(packed, 0)
}

// Close underlying socket.
func (c *Codec) Close() error {
	if c.closed {
		return nil
	}

	c.closed = true
	return c.relay.Close()
}
<|endoftext|>


<commit_before>package goridge

import (
	"bytes"
	"io"
	"net/rpc"
	"sync"

	j "github.com/json-iterator/go"
	"github.com/spiral/errors"
	"go.uber.org/multierr"
)

var json = j.ConfigCompatibleWithStandardLibrary

// Codec represent net/rpc bridge over Goridge socket relay.
type Codec struct {
	relay  Relay
	closed bool
	frame  *Frame
	codec  sync.Map
}

// NewCodec initiates new server rpc codec over socket connection.
func NewCodec(rwc io.ReadWriteCloser) *Codec {
	return &Codec{
		relay: NewSocketRelay(rwc),
		codec: sync.Map{},
	}
}

// NewCodecWithRelay initiates new server rpc codec with a relay of choice.
func NewCodecWithRelay(relay Relay) *Codec {
	return &Codec{relay: relay}
}

// WriteResponse marshals response, byte slice or error to remote party.
func (c *Codec) WriteResponse(r *rpc.Response, body interface{}) error {
	frame := NewFrame()

	// SEQ_ID + METHOD_NAME_LEN
	frame.WriteOptions(uint32(r.Seq), uint32(len(r.ServiceMethod)))
	// Write protocol version
	frame.WriteVersion(VERSION_1)

	// Store response flags
	codec, ok := c.codec.Load(r.Seq)
	if !ok {
		// fallback codec
		frame.WriteFlags(CODEC_GOB)
	} else {
		frame.WriteFlags(codec.(FrameFlag))
	}

	// initialize buffer
	buf := new(bytes.Buffer)
	// writeServiceMethod to the buffer
	buf.WriteString(r.ServiceMethod)

	// if error returned, we sending it via relay and return error from WriteResponse
	if r.Error != "" {
		// Append error flag
		return c.handleError(r, frame, buf, errors.Str(r.Error))
	}

	flags := frame.ReadFlags()

	if flags&byte(CODEC_JSON) != 0 {
		err := encodeJSON(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
	}

	if flags&byte(CODEC_GOB) != 0 {
		err := encodeGob(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
	}

	if flags&byte(CODEC_RAW) != 0 {
		err := encodeRaw(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
	}

	if flags&byte(CODEC_MSGPACK) != 0 {
		err := encodeMsgPack(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
	}

	// everything written correct
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()

	return c.relay.Send(frame)
}

func (c *Codec) handleError(r *rpc.Response, frame *Frame, buf *bytes.Buffer, err error) error {
	// just to be sure, remove all data from buffer and write new
	buf.Truncate(0)
	// write all possible errors
	err = multierr.Append(err, errors.Str(r.Error))
	buf.WriteString(r.ServiceMethod)

	const op = errors.Op("handle codec error")
	frame.WriteFlags(ERROR)
	// error should be here
	if err != nil {
		buf.WriteString(err.Error())
	}
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()
	_ = c.relay.Send(frame)
	return errors.E(op, errors.Str(r.Error))
}

// ReadRequestHeader receives
func (c *Codec) ReadRequestHeader(r *rpc.Request) error {
	const op = errors.Op("codec: read request header")
	frame := NewFrame()
	err := c.relay.Receive(frame)
	if err != nil {
		return err
	}

	// opts[0] sequence ID
	// opts[1] service method name offset from payload in bytes
	opts := frame.ReadOptions()
	if len(opts) != 2 {
		return errors.E(op, errors.Str("should be 2 options. SEQ_ID and METHOD_LEN"))
	}

	r.Seq = uint64(opts[0])
	r.ServiceMethod = string(frame.Payload()[:opts[1]])
	c.frame = frame
	return c.storeCodec(r, frame.ReadFlags())
}

func (c *Codec) storeCodec(r *rpc.Request, flag byte) error {
	if flag&byte(CODEC_JSON) != 0 {
		c.codec.Store(r.Seq, CODEC_JSON)
		return nil
	}

	if flag&byte(CODEC_GOB) != 0 {
		c.codec.Store(r.Seq, CODEC_GOB)
		return nil
	}

	if flag&byte(CODEC_RAW) != 0 {
		c.codec.Store(r.Seq, CODEC_RAW)
		return nil
	}

	if flag&byte(CODEC_MSGPACK) != 0 {
		c.codec.Store(r.Seq, CODEC_MSGPACK)
		return nil
	}

	// If no flags provided, store GOB
	c.codec.Store(r.Seq, CODEC_GOB)
	return nil
}

// ReadRequestBody fetches prefixed body data and automatically unmarshal it as json. RawBody flag will populate
// []byte lice argument for rpc method.
func (c *Codec) ReadRequestBody(out interface{}) error {
	const op = errors.Op("codec read request body")
	if out == nil {
		return nil
	}

	defer func() {
		c.frame = nil
	}()

	flags := c.frame.ReadFlags()

	if flags&byte(CODEC_JSON) != byte(0) {
		return decodeJSON(out, c.frame)
	}

	if flags&byte(CODEC_GOB) != byte(0) {
		return decodeGob(out, c.frame)
	}

	if flags&byte(CODEC_RAW) != byte(0) {
		return decodeRaw(out, c.frame)
	}

	if flags&byte(CODEC_MSGPACK) != byte(0) {
		return decodeMsgPack(out, c.frame)
	}

	return errors.E(op, errors.Str("unknown decoder used in frame"))
}

// Close underlying socket.
func (c *Codec) Close() error {
	if c.closed {
		return nil
	}

	c.closed = true
	return c.relay.Close()
}
<commit_msg>Fix potential bug with storing a unlimited number of seq_id to map. Now we clean the syncMap when seq_id read. Use switch instead of checking all if's.<commit_after>package goridge

import (
	"bytes"
	"io"
	"net/rpc"
	"sync"

	j "github.com/json-iterator/go"
	"github.com/spiral/errors"
	"go.uber.org/multierr"
)

var json = j.ConfigCompatibleWithStandardLibrary

// Codec represent net/rpc bridge over Goridge socket relay.
type Codec struct {
	relay  Relay
	closed bool
	frame  *Frame
	codec  sync.Map
}

// NewCodec initiates new server rpc codec over socket connection.
func NewCodec(rwc io.ReadWriteCloser) *Codec {
	return &Codec{
		relay: NewSocketRelay(rwc),
		codec: sync.Map{},
	}
}

// NewCodecWithRelay initiates new server rpc codec with a relay of choice.
func NewCodecWithRelay(relay Relay) *Codec {
	return &Codec{relay: relay}
}

// WriteResponse marshals response, byte slice or error to remote party.
func (c *Codec) WriteResponse(r *rpc.Response, body interface{}) error {
	const op = errors.Op("codec: write response")
	frame := NewFrame()

	// SEQ_ID + METHOD_NAME_LEN
	frame.WriteOptions(uint32(r.Seq), uint32(len(r.ServiceMethod)))
	// Write protocol version
	frame.WriteVersion(VERSION_1)

	// load and delete associated codec to not waste memory
	// because we write it to the frame and don't need more information about it
	codec, ok := c.codec.LoadAndDelete(r.Seq)
	if !ok {
		// fallback codec
		frame.WriteFlags(CODEC_GOB)
	} else {
		frame.WriteFlags(codec.(FrameFlag))
	}

	// initialize buffer
	buf := new(bytes.Buffer)
	// writeServiceMethod to the buffer
	buf.WriteString(r.ServiceMethod)

	// if error returned, we sending it via relay and return error from WriteResponse
	if r.Error != "" {
		// Append error flag
		return c.handleError(r, frame, buf, errors.Str(r.Error))
	}

	// read flag previously written
	// TODO might be better to save it to local variable
	flags := frame.ReadFlags()

	switch {
	case flags&byte(CODEC_JSON) != 0:
		err := encodeJSON(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// everything written correct
		frame.WritePayloadLen(uint32(buf.Len()))
		frame.WritePayload(buf.Bytes())

		frame.WriteCRC()
		return c.relay.Send(frame)
	case flags&byte(CODEC_MSGPACK) != 0:
		err := encodeMsgPack(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// everything written correct
		frame.WritePayloadLen(uint32(buf.Len()))
		frame.WritePayload(buf.Bytes())

		frame.WriteCRC()
		return c.relay.Send(frame)
	case flags&byte(CODEC_RAW) != 0:
		err := encodeRaw(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// everything written correct
		frame.WritePayloadLen(uint32(buf.Len()))
		frame.WritePayload(buf.Bytes())

		frame.WriteCRC()
		return c.relay.Send(frame)
	case flags&byte(CODEC_GOB) != 0:
		err := encodeGob(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// everything written correct
		frame.WritePayloadLen(uint32(buf.Len()))
		frame.WritePayload(buf.Bytes())

		frame.WriteCRC()
		return c.relay.Send(frame)
	default:
		return errors.E(op, errors.Str("unknown codec"))
	}
}

func (c *Codec) handleError(r *rpc.Response, frame *Frame, buf *bytes.Buffer, err error) error {
	// just to be sure, remove all data from buffer and write new
	buf.Truncate(0)
	// write all possible errors
	err = multierr.Append(err, errors.Str(r.Error))
	buf.WriteString(r.ServiceMethod)

	const op = errors.Op("handle codec error")
	frame.WriteFlags(ERROR)
	// error should be here
	if err != nil {
		buf.WriteString(err.Error())
	}
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()
	_ = c.relay.Send(frame)
	return errors.E(op, errors.Str(r.Error))
}

// ReadRequestHeader receives
func (c *Codec) ReadRequestHeader(r *rpc.Request) error {
	const op = errors.Op("codec: read request header")
	frame := NewFrame()
	err := c.relay.Receive(frame)
	if err != nil {
		return err
	}

	// opts[0] sequence ID
	// opts[1] service method name offset from payload in bytes
	opts := frame.ReadOptions()
	if len(opts) != 2 {
		return errors.E(op, errors.Str("should be 2 options. SEQ_ID and METHOD_LEN"))
	}

	r.Seq = uint64(opts[0])
	r.ServiceMethod = string(frame.Payload()[:opts[1]])
	c.frame = frame
	return c.storeCodec(r, frame.ReadFlags())
}

func (c *Codec) storeCodec(r *rpc.Request, flag byte) error {
	switch {
	case flag&byte(CODEC_JSON) != 0:
		c.codec.Store(r.Seq, CODEC_JSON)
	case flag&byte(CODEC_RAW) != 0:
		c.codec.Store(r.Seq, CODEC_RAW)
	case flag&byte(CODEC_MSGPACK) != 0:
		c.codec.Store(r.Seq, CODEC_MSGPACK)
	case flag&byte(CODEC_GOB) != 0:
		c.codec.Store(r.Seq, CODEC_GOB)
	default:
		c.codec.Store(r.Seq, CODEC_GOB)
	}

	return nil
}

// ReadRequestBody fetches prefixed body data and automatically unmarshal it as json. RawBody flag will populate
// []byte lice argument for rpc method.
func (c *Codec) ReadRequestBody(out interface{}) error {
	const op = errors.Op("codec read request body")
	if out == nil {
		return nil
	}

	defer func() {
		c.frame = nil
	}()

	flags := c.frame.ReadFlags()

	if flags&byte(CODEC_JSON) != byte(0) {
		return decodeJSON(out, c.frame)
	}

	if flags&byte(CODEC_GOB) != byte(0) {
		return decodeGob(out, c.frame)
	}

	if flags&byte(CODEC_RAW) != byte(0) {
		return decodeRaw(out, c.frame)
	}

	if flags&byte(CODEC_MSGPACK) != byte(0) {
		return decodeMsgPack(out, c.frame)
	}

	return errors.E(op, errors.Str("unknown decoder used in frame"))
}

// Close underlying socket.
func (c *Codec) Close() error {
	if c.closed {
		return nil
	}

	c.closed = true
	return c.relay.Close()
}
<|endoftext|>