<commit_before>package debos

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func CleanPathAt(path, at string) string {
	if filepath.IsAbs(path) {
		return filepath.Clean(path)
	}

	return filepath.Join(at, path)
}

func CleanPath(path string) string {
	cwd, _ := os.Getwd()
	return CleanPathAt(path, cwd)
}

func CopyFile(src, dst string, mode os.FileMode) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	tmp, err := ioutil.TempFile(filepath.Dir(dst), "")
	if err != nil {
		return err
	}
	_, err = io.Copy(tmp, in)
	if err != nil {
		tmp.Close()
		os.Remove(tmp.Name())
		return err
	}
	if err = tmp.Close(); err != nil {
		os.Remove(tmp.Name())
		return err
	}
	if err = os.Chmod(tmp.Name(), mode); err != nil {
		os.Remove(tmp.Name())
		return err
	}
	return os.Rename(tmp.Name(), dst)
}

func CopyTree(sourcetree, desttree string) error {
	fmt.Printf("Overlaying %s on %s\n", sourcetree, desttree)
	walker := func(p string, info os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		suffix, _ := filepath.Rel(sourcetree, p)
		target := path.Join(desttree, suffix)
		switch info.Mode() & os.ModeType {
		case 0:
			CopyFile(p, target, info.Mode())
		case os.ModeDir:
			os.Mkdir(target, info.Mode())
		case os.ModeSymlink:
			link, err := os.Readlink(p)
			if err != nil {
				log.Panicf("Failed to read symlink %s: %v", suffix, err)
			}
			os.Symlink(link, target)
		default:
			log.Panicf("Not handled /%s %v", suffix, info.Mode())
		}

		return nil
	}

	return filepath.Walk(sourcetree, walker)
}

func RestrictedPath(prefix, dest string) (string, error) {
	var err error
	destination := path.Join(prefix, dest)
	destination, err = filepath.Abs(destination)
	if err != nil {
		return "", err
	}
	if !strings.HasPrefix(destination, prefix) {
		return "", fmt.Errorf("The resulting path points outside of prefix '%s': '%s'\n", prefix, destination)
	}
	return destination, nil
}
<commit_msg>Add realpath filesystem helper<commit_after>package debos

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func CleanPathAt(path, at string) string {
	if filepath.IsAbs(path) {
		return filepath.Clean(path)
	}

	return filepath.Join(at, path)
}

func CleanPath(path string) string {
	cwd, _ := os.Getwd()
	return CleanPathAt(path, cwd)
}

func CopyFile(src, dst string, mode os.FileMode) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	tmp, err := ioutil.TempFile(filepath.Dir(dst), "")
	if err != nil {
		return err
	}
	_, err = io.Copy(tmp, in)
	if err != nil {
		tmp.Close()
		os.Remove(tmp.Name())
		return err
	}
	if err = tmp.Close(); err != nil {
		os.Remove(tmp.Name())
		return err
	}
	if err = os.Chmod(tmp.Name(), mode); err != nil {
		os.Remove(tmp.Name())
		return err
	}
	return os.Rename(tmp.Name(), dst)
}

func CopyTree(sourcetree, desttree string) error {
	fmt.Printf("Overlaying %s on %s\n", sourcetree, desttree)
	walker := func(p string, info os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		suffix, _ := filepath.Rel(sourcetree, p)
		target := path.Join(desttree, suffix)
		switch info.Mode() & os.ModeType {
		case 0:
			CopyFile(p, target, info.Mode())
		case os.ModeDir:
			os.Mkdir(target, info.Mode())
		case os.ModeSymlink:
			link, err := os.Readlink(p)
			if err != nil {
				log.Panicf("Failed to read symlink %s: %v", suffix, err)
			}
			os.Symlink(link, target)
		default:
			log.Panicf("Not handled /%s %v", suffix, info.Mode())
		}

		return nil
	}

	return filepath.Walk(sourcetree, walker)
}

func RealPath(path string) (string, error) {
	p, err := filepath.EvalSymlinks(path)
	if err != nil {
		return "", err
	}

	return filepath.Abs(p)
}

func RestrictedPath(prefix, dest string) (string, error) {
	var err error
	destination := path.Join(prefix, dest)
	destination, err = filepath.Abs(destination)
	if err != nil {
		return "", err
	}
	if !strings.HasPrefix(destination, prefix) {
		return "", fmt.Errorf("The resulting path points outside of prefix '%s': '%s'\n", prefix, destination)
	}
	return destination, nil
}
<|endoftext|>


<commit_before>package omise

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

type CardInfo struct {
	Name            string
	Number          string
	ExpirationMonth int
	ExpirationYear  int
	City            string
	PostalCode      string
	SecurityCode    int
}

type Card struct {
	ID                string `json:"id"`
	LiveMode          bool   `json:"livemode"`
	Country           string `json:"country"`
	City              string `json:"city"`
	PostalCode        string `json:"postal_code"`
	Financing         string `json:"financing"`
	LastDigits        string `json:"last_digits"`
	Brand             string `json:"brand"`
	ExpirationMonth   int    `json:"expiration_month"`
	ExpirationYear    int    `json:"expiration_year"`
	Fingerprint       string `json:"fingerprint"`
	Name              string `json:"name"`
	SecurityCodeCheck bool   `json:"security_code_check"`
}

type Token struct {
	ID       string `json:"id"`
	LiveMode bool   `json:"livemode"`
	Used     bool   `json:"used"`
	Location string `json:"location"`
	Card     *Card  `json:"card"`
}

type NotFoundError struct {
	Location string `json:"location"`
	Code string `json:"code"`
	Message string `json:"message"`
}

func (err *NotFoundError) Error() string {
	return ""
}

type BadRequestError struct {
	Location string `json:"location"`
	Code string `json:"code"`
	Message string `json:"message"`
}

func (err *BadRequestError) Error() string {
	return ""
}

type TokensService struct {
	Key string
	URL string
}

func (ts *TokensService) Create(ci *CardInfo) (*Token, error) {
	var data = url.Values{}
	data.Set("card[name]", ci.Name)
	data.Set("card[number]", ci.Number)
	data.Set("card[expiration_month]", strconv.Itoa(ci.ExpirationMonth))
	data.Set("card[expiration_year]", strconv.Itoa(ci.ExpirationYear))
	data.Set("card[city]", ci.City)
	data.Set("card[postal_code]", ci.PostalCode)
	data.Set("card[security_code]", strconv.Itoa(ci.SecurityCode))

	req, _ := http.NewRequest("POST", ts.URL+"/tokens", strings.NewReader(data.Encode()))
	req.SetBasicAuth(ts.Key, "")

	c := &http.Client{}

	resp, _ := c.Do(req)

	b, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	var e BadRequestError
	if resp.StatusCode == http.StatusBadRequest {
		json.Unmarshal(b, &e)
		return nil, &e
	}

	var t Token
	err := json.Unmarshal(b, &t)

	return &t, err
}

func (ts *TokensService) Get(key string) (*Token, error) {
	req, _ := http.NewRequest("GET", ts.URL+"/tokens/"+key, nil)
	req.SetBasicAuth(ts.Key, "")
	c := &http.Client{}
	resp, _ := c.Do(req)
	b, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	var (
		t Token
		e NotFoundError
	)

	if resp.StatusCode == http.StatusNotFound {
		json.Unmarshal(b, &e)
		return nil, &e
	}
	err := json.Unmarshal(b, &t)
	return &t, err
}
<commit_msg>reformat tokens.go<commit_after>package omise

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

type CardInfo struct {
	Name            string
	Number          string
	ExpirationMonth int
	ExpirationYear  int
	City            string
	PostalCode      string
	SecurityCode    int
}

type Card struct {
	ID                string `json:"id"`
	LiveMode          bool   `json:"livemode"`
	Country           string `json:"country"`
	City              string `json:"city"`
	PostalCode        string `json:"postal_code"`
	Financing         string `json:"financing"`
	LastDigits        string `json:"last_digits"`
	Brand             string `json:"brand"`
	ExpirationMonth   int    `json:"expiration_month"`
	ExpirationYear    int    `json:"expiration_year"`
	Fingerprint       string `json:"fingerprint"`
	Name              string `json:"name"`
	SecurityCodeCheck bool   `json:"security_code_check"`
}

type Token struct {
	ID       string `json:"id"`
	LiveMode bool   `json:"livemode"`
	Used     bool   `json:"used"`
	Location string `json:"location"`
	Card     *Card  `json:"card"`
}

type NotFoundError struct {
	Location string `json:"location"`
	Code     string `json:"code"`
	Message  string `json:"message"`
}

func (err *NotFoundError) Error() string {
	return ""
}

type BadRequestError struct {
	Location string `json:"location"`
	Code     string `json:"code"`
	Message  string `json:"message"`
}

func (err *BadRequestError) Error() string {
	return ""
}

type TokensService struct {
	Key string
	URL string
}

func (ts *TokensService) Create(ci *CardInfo) (*Token, error) {
	var data = url.Values{}
	data.Set("card[name]", ci.Name)
	data.Set("card[number]", ci.Number)
	data.Set("card[expiration_month]", strconv.Itoa(ci.ExpirationMonth))
	data.Set("card[expiration_year]", strconv.Itoa(ci.ExpirationYear))
	data.Set("card[city]", ci.City)
	data.Set("card[postal_code]", ci.PostalCode)
	data.Set("card[security_code]", strconv.Itoa(ci.SecurityCode))

	req, _ := http.NewRequest("POST", ts.URL+"/tokens", strings.NewReader(data.Encode()))
	req.SetBasicAuth(ts.Key, "")

	c := &http.Client{}

	resp, _ := c.Do(req)

	b, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	var e BadRequestError
	if resp.StatusCode == http.StatusBadRequest {
		json.Unmarshal(b, &e)
		return nil, &e
	}

	var t Token
	err := json.Unmarshal(b, &t)

	return &t, err
}

func (ts *TokensService) Get(key string) (*Token, error) {
	req, _ := http.NewRequest("GET", ts.URL+"/tokens/"+key, nil)
	req.SetBasicAuth(ts.Key, "")
	c := &http.Client{}
	resp, _ := c.Do(req)
	b, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	var (
		t Token
		e NotFoundError
	)

	if resp.StatusCode == http.StatusNotFound {
		json.Unmarshal(b, &e)
		return nil, &e
	}
	err := json.Unmarshal(b, &t)
	return &t, err
}
<|endoftext|>


<commit_before>package mux

import (
	"context"
	"sync"
	"time"

	"v2ray.com/core/app"
	"v2ray.com/core/app/dispatcher"
	"v2ray.com/core/app/log"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/ray"
)

const (
	maxParallel = 8
	maxTotal    = 128
)

type manager interface {
	remove(id uint16)
}

type session struct {
	sync.Mutex
	input          ray.InputStream
	output         ray.OutputStream
	parent         manager
	id             uint16
	uplinkClosed   bool
	downlinkClosed bool
}

func (s *session) closeUplink() {
	var allDone bool
	s.Lock()
	s.uplinkClosed = true
	allDone = s.uplinkClosed && s.downlinkClosed
	s.Unlock()
	if allDone {
		s.parent.remove(s.id)
	}
}

func (s *session) closeDownlink() {
	var allDone bool
	s.Lock()
	s.downlinkClosed = true
	allDone = s.uplinkClosed && s.downlinkClosed
	s.Unlock()
	if allDone {
		s.parent.remove(s.id)
	}
}

type ClientManager struct {
	access  sync.Mutex
	clients []*Client
	proxy   proxy.Outbound
	dialer  proxy.Dialer
}

func NewClientManager(p proxy.Outbound, d proxy.Dialer) *ClientManager {
	return &ClientManager{
		proxy:  p,
		dialer: d,
	}
}

func (m *ClientManager) Dispatch(ctx context.Context, outboundRay ray.OutboundRay) error {
	m.access.Lock()
	defer m.access.Unlock()

	for _, client := range m.clients {
		if client.Dispatch(ctx, outboundRay) {
			return nil
		}
	}

	client, err := NewClient(m.proxy, m.dialer, m)
	if err != nil {
		return errors.Base(err).Message("Proxyman|Mux|ClientManager: Failed to create client.")
	}
	m.clients = append(m.clients, client)
	client.Dispatch(ctx, outboundRay)
	return nil
}

func (m *ClientManager) onClientFinish() {
	m.access.Lock()
	defer m.access.Unlock()

	if len(m.clients) < 10 {
		return
	}

	activeClients := make([]*Client, 0, len(m.clients))

	for _, client := range m.clients {
		if !client.Closed() {
			activeClients = append(activeClients, client)
		}
	}
	m.clients = activeClients
}

type Client struct {
	access     sync.RWMutex
	count      uint16
	sessions   map[uint16]*session
	inboundRay ray.InboundRay
	ctx        context.Context
	cancel     context.CancelFunc
	manager    *ClientManager
}

var muxCoolDestination = net.TCPDestination(net.DomainAddress("v1.mux.cool"), net.Port(9527))

func NewClient(p proxy.Outbound, dialer proxy.Dialer, m *ClientManager) (*Client, error) {
	ctx, cancel := context.WithCancel(context.Background())
	ctx = proxy.ContextWithTarget(ctx, muxCoolDestination)
	pipe := ray.NewRay(ctx)
	go p.Process(ctx, pipe, dialer)
	c := &Client{
		sessions:   make(map[uint16]*session, 256),
		inboundRay: pipe,
		ctx:        ctx,
		cancel:     cancel,
		manager:    m,
		count:      0,
	}
	go c.fetchOutput()
	return c, nil
}

func (m *Client) remove(id uint16) {
	m.access.Lock()
	defer m.access.Unlock()

	delete(m.sessions, id)

	if len(m.sessions) == 0 {
		m.cancel()
		m.inboundRay.InboundInput().Close()
		go m.manager.onClientFinish()
	}
}

func (m *Client) Closed() bool {
	select {
	case <-m.ctx.Done():
		return true
	default:
		return false
	}
}

func fetchInput(ctx context.Context, s *session, output buf.Writer) {
	dest, _ := proxy.TargetFromContext(ctx)
	writer := &Writer{
		dest:   dest,
		id:     s.id,
		writer: output,
	}
	defer writer.Close()
	defer s.closeUplink()

	log.Info("Proxyman|Mux|Client: Dispatching request to ", dest)
	data, _ := s.input.ReadTimeout(time.Millisecond * 500)
	if data != nil {
		if err := writer.Write(data); err != nil {
			log.Info("Proxyman|Mux|Client: Failed to write first payload: ", err)
			return
		}
	}
	if err := buf.PipeUntilEOF(signal.BackgroundTimer(), s.input, writer); err != nil {
		log.Info("Proxyman|Mux|Client: Failed to fetch all input: ", err)
	}
}

func (m *Client) Dispatch(ctx context.Context, outboundRay ray.OutboundRay) bool {
	m.access.Lock()
	defer m.access.Unlock()

	if len(m.sessions) >= maxParallel {
		return false
	}

	if m.count >= maxTotal {
		return false
	}

	select {
	case <-m.ctx.Done():
		return false
	default:
	}

	m.count++
	id := m.count
	s := &session{
		input:  outboundRay.OutboundInput(),
		output: outboundRay.OutboundOutput(),
		parent: m,
		id:     id,
	}
	m.sessions[id] = s
	go fetchInput(ctx, s, m.inboundRay.InboundInput())
	return true
}

func drain(reader *Reader) error {
	for {
		data, more, err := reader.Read()
		if err != nil {
			return err
		}
		data.Release()
		if !more {
			return nil
		}
	}
}

func pipe(reader *Reader, writer buf.Writer) error {
	for {
		data, more, err := reader.Read()
		if err != nil {
			return err
		}
		if err := writer.Write(data); err != nil {
			return err
		}
		if !more {
			return nil
		}
	}
}

func (m *Client) fetchOutput() {
	reader := NewReader(m.inboundRay.InboundOutput())
	for {
		meta, err := reader.ReadMetadata()
		if err != nil {
			log.Info("Proxyman|Mux|Client: Failed to read metadata: ", err)
			break
		}
		m.access.RLock()
		s, found := m.sessions[meta.SessionID]
		m.access.RUnlock()
		if found && meta.SessionStatus == SessionStatusEnd {
			s.closeDownlink()
			s.output.Close()
		}
		if !meta.Option.Has(OptionData) {
			continue
		}

		if found {
			err = pipe(reader, s.output)
		} else {
			err = drain(reader)
		}

		if err != nil {
			log.Info("Proxyman|Mux|Client: Failed to read data: ", err)
			break
		}
	}

	// Close all downlinks
	m.access.RLock()
	for _, s := range m.sessions {
		s.closeUplink()
		s.closeDownlink()
		s.output.CloseError()
	}
	m.access.RUnlock()
}

type Server struct {
	dispatcher dispatcher.Interface
}

func NewServer(ctx context.Context) *Server {
	s := &Server{}
	space := app.SpaceFromContext(ctx)
	space.OnInitialize(func() error {
		d := dispatcher.FromSpace(space)
		if d == nil {
			return errors.New("Proxyman|Mux: No dispatcher in space.")
		}
		s.dispatcher = d
		return nil
	})
	return s
}

func (s *Server) Dispatch(ctx context.Context, dest net.Destination) (ray.InboundRay, error) {
	if dest != muxCoolDestination {
		return s.dispatcher.Dispatch(ctx, dest)
	}

	ray := ray.NewRay(ctx)
	worker := &ServerWorker{
		dispatcher:  s.dispatcher,
		outboundRay: ray,
		sessions:    make(map[uint16]*session),
	}
	go worker.run(ctx)
	return ray, nil
}

type ServerWorker struct {
	dispatcher  dispatcher.Interface
	outboundRay ray.OutboundRay
	sessions    map[uint16]*session
	access      sync.RWMutex
}

func (w *ServerWorker) remove(id uint16) {
	w.access.Lock()
	delete(w.sessions, id)
	w.access.Unlock()
}

func handle(ctx context.Context, s *session, output buf.Writer) {
	writer := NewResponseWriter(s.id, output)
	if err := buf.PipeUntilEOF(signal.BackgroundTimer(), s.input, writer); err != nil {
		log.Info("Proxyman|Mux|ServerWorker: Session ", s.id, " ends: ", err)
	}
	writer.Close()
	s.closeDownlink()
}

func (w *ServerWorker) run(ctx context.Context) {
	input := w.outboundRay.OutboundInput()
	reader := NewReader(input)
	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		meta, err := reader.ReadMetadata()
		if err != nil {
			return
		}

		w.access.RLock()
		s, found := w.sessions[meta.SessionID]
		w.access.RUnlock()

		if found && meta.SessionStatus == SessionStatusEnd {
			s.closeUplink()
			s.output.Close()
		}

		if meta.SessionStatus == SessionStatusNew {
			log.Info("Proxyman|Mux|Server: Received request for ", meta.Target)
			inboundRay, err := w.dispatcher.Dispatch(ctx, meta.Target)
			if err != nil {
				log.Info("Proxyman|Mux: Failed to dispatch request: ", err)
				continue
			}
			s = &session{
				input:  inboundRay.InboundOutput(),
				output: inboundRay.InboundInput(),
				parent: w,
				id:     meta.SessionID,
			}
			w.access.Lock()
			w.sessions[meta.SessionID] = s
			w.access.Unlock()
			go handle(ctx, s, w.outboundRay.OutboundOutput())
		}

		if !meta.Option.Has(OptionData) {
			continue
		}

		if s != nil {
			err = pipe(reader, s.output)
		} else {
			err = drain(reader)
		}

		if err != nil {
			log.Info("Proxyman|Mux|ServerWorker: Failed to read data: ", err)
			break
		}
	}
}
<commit_msg>fix dead lock<commit_after>package mux

import (
	"context"
	"sync"
	"time"

	"v2ray.com/core/app"
	"v2ray.com/core/app/dispatcher"
	"v2ray.com/core/app/log"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/proxy"
	"v2ray.com/core/transport/ray"
)

const (
	maxParallel = 8
	maxTotal    = 128
)

type manager interface {
	remove(id uint16)
}

type session struct {
	sync.Mutex
	input          ray.InputStream
	output         ray.OutputStream
	parent         manager
	id             uint16
	uplinkClosed   bool
	downlinkClosed bool
}

func (s *session) closeUplink() {
	var allDone bool
	s.Lock()
	s.uplinkClosed = true
	allDone = s.uplinkClosed && s.downlinkClosed
	s.Unlock()
	if allDone {
		go s.parent.remove(s.id)
	}
}

func (s *session) closeDownlink() {
	var allDone bool
	s.Lock()
	s.downlinkClosed = true
	allDone = s.uplinkClosed && s.downlinkClosed
	s.Unlock()
	if allDone {
		go s.parent.remove(s.id)
	}
}

type ClientManager struct {
	access  sync.Mutex
	clients []*Client
	proxy   proxy.Outbound
	dialer  proxy.Dialer
}

func NewClientManager(p proxy.Outbound, d proxy.Dialer) *ClientManager {
	return &ClientManager{
		proxy:  p,
		dialer: d,
	}
}

func (m *ClientManager) Dispatch(ctx context.Context, outboundRay ray.OutboundRay) error {
	m.access.Lock()
	defer m.access.Unlock()

	for _, client := range m.clients {
		if client.Dispatch(ctx, outboundRay) {
			return nil
		}
	}

	client, err := NewClient(m.proxy, m.dialer, m)
	if err != nil {
		return errors.Base(err).Message("Proxyman|Mux|ClientManager: Failed to create client.")
	}
	m.clients = append(m.clients, client)
	client.Dispatch(ctx, outboundRay)
	return nil
}

func (m *ClientManager) onClientFinish() {
	m.access.Lock()
	defer m.access.Unlock()

	if len(m.clients) < 10 {
		return
	}

	activeClients := make([]*Client, 0, len(m.clients))

	for _, client := range m.clients {
		if !client.Closed() {
			activeClients = append(activeClients, client)
		}
	}
	m.clients = activeClients
}

type Client struct {
	access     sync.RWMutex
	count      uint16
	sessions   map[uint16]*session
	inboundRay ray.InboundRay
	ctx        context.Context
	cancel     context.CancelFunc
	manager    *ClientManager
}

var muxCoolDestination = net.TCPDestination(net.DomainAddress("v1.mux.cool"), net.Port(9527))

func NewClient(p proxy.Outbound, dialer proxy.Dialer, m *ClientManager) (*Client, error) {
	ctx, cancel := context.WithCancel(context.Background())
	ctx = proxy.ContextWithTarget(ctx, muxCoolDestination)
	pipe := ray.NewRay(ctx)
	go p.Process(ctx, pipe, dialer)
	c := &Client{
		sessions:   make(map[uint16]*session, 256),
		inboundRay: pipe,
		ctx:        ctx,
		cancel:     cancel,
		manager:    m,
		count:      0,
	}
	go c.fetchOutput()
	return c, nil
}

func (m *Client) remove(id uint16) {
	m.access.Lock()
	defer m.access.Unlock()

	delete(m.sessions, id)

	if len(m.sessions) == 0 {
		m.cancel()
		m.inboundRay.InboundInput().Close()
		go m.manager.onClientFinish()
	}
}

func (m *Client) Closed() bool {
	select {
	case <-m.ctx.Done():
		return true
	default:
		return false
	}
}

func fetchInput(ctx context.Context, s *session, output buf.Writer) {
	dest, _ := proxy.TargetFromContext(ctx)
	writer := &Writer{
		dest:   dest,
		id:     s.id,
		writer: output,
	}
	defer writer.Close()
	defer s.closeUplink()

	log.Info("Proxyman|Mux|Client: Dispatching request to ", dest)
	data, _ := s.input.ReadTimeout(time.Millisecond * 500)
	if data != nil {
		if err := writer.Write(data); err != nil {
			log.Info("Proxyman|Mux|Client: Failed to write first payload: ", err)
			return
		}
	}
	if err := buf.PipeUntilEOF(signal.BackgroundTimer(), s.input, writer); err != nil {
		log.Info("Proxyman|Mux|Client: Failed to fetch all input: ", err)
	}
}

func (m *Client) Dispatch(ctx context.Context, outboundRay ray.OutboundRay) bool {
	m.access.Lock()
	defer m.access.Unlock()

	if len(m.sessions) >= maxParallel {
		return false
	}

	if m.count >= maxTotal {
		return false
	}

	select {
	case <-m.ctx.Done():
		return false
	default:
	}

	m.count++
	id := m.count
	s := &session{
		input:  outboundRay.OutboundInput(),
		output: outboundRay.OutboundOutput(),
		parent: m,
		id:     id,
	}
	m.sessions[id] = s
	go fetchInput(ctx, s, m.inboundRay.InboundInput())
	return true
}

func drain(reader *Reader) error {
	for {
		data, more, err := reader.Read()
		if err != nil {
			return err
		}
		data.Release()
		if !more {
			return nil
		}
	}
}

func pipe(reader *Reader, writer buf.Writer) error {
	for {
		data, more, err := reader.Read()
		if err != nil {
			return err
		}
		if err := writer.Write(data); err != nil {
			return err
		}
		if !more {
			return nil
		}
	}
}

func (m *Client) fetchOutput() {
	reader := NewReader(m.inboundRay.InboundOutput())
	for {
		meta, err := reader.ReadMetadata()
		if err != nil {
			log.Info("Proxyman|Mux|Client: Failed to read metadata: ", err)
			break
		}
		m.access.RLock()
		s, found := m.sessions[meta.SessionID]
		m.access.RUnlock()
		if found && meta.SessionStatus == SessionStatusEnd {
			s.closeDownlink()
			s.output.Close()
		}
		if !meta.Option.Has(OptionData) {
			continue
		}

		if found {
			err = pipe(reader, s.output)
		} else {
			err = drain(reader)
		}

		if err != nil {
			log.Info("Proxyman|Mux|Client: Failed to read data: ", err)
			break
		}
	}

	// Close all downlinks
	m.access.RLock()
	for _, s := range m.sessions {
		s.closeUplink()
		s.closeDownlink()
		s.output.CloseError()
	}
	m.access.RUnlock()
}

type Server struct {
	dispatcher dispatcher.Interface
}

func NewServer(ctx context.Context) *Server {
	s := &Server{}
	space := app.SpaceFromContext(ctx)
	space.OnInitialize(func() error {
		d := dispatcher.FromSpace(space)
		if d == nil {
			return errors.New("Proxyman|Mux: No dispatcher in space.")
		}
		s.dispatcher = d
		return nil
	})
	return s
}

func (s *Server) Dispatch(ctx context.Context, dest net.Destination) (ray.InboundRay, error) {
	if dest != muxCoolDestination {
		return s.dispatcher.Dispatch(ctx, dest)
	}

	ray := ray.NewRay(ctx)
	worker := &ServerWorker{
		dispatcher:  s.dispatcher,
		outboundRay: ray,
		sessions:    make(map[uint16]*session),
	}
	go worker.run(ctx)
	return ray, nil
}

type ServerWorker struct {
	dispatcher  dispatcher.Interface
	outboundRay ray.OutboundRay
	sessions    map[uint16]*session
	access      sync.RWMutex
}

func (w *ServerWorker) remove(id uint16) {
	w.access.Lock()
	delete(w.sessions, id)
	w.access.Unlock()
}

func handle(ctx context.Context, s *session, output buf.Writer) {
	writer := NewResponseWriter(s.id, output)
	if err := buf.PipeUntilEOF(signal.BackgroundTimer(), s.input, writer); err != nil {
		log.Info("Proxyman|Mux|ServerWorker: Session ", s.id, " ends: ", err)
	}
	writer.Close()
	s.closeDownlink()
}

func (w *ServerWorker) run(ctx context.Context) {
	input := w.outboundRay.OutboundInput()
	reader := NewReader(input)
	for {
		select {
		case <-ctx.Done():
			return
		default:
		}

		meta, err := reader.ReadMetadata()
		if err != nil {
			return
		}

		w.access.RLock()
		s, found := w.sessions[meta.SessionID]
		w.access.RUnlock()

		if found && meta.SessionStatus == SessionStatusEnd {
			s.closeUplink()
			s.output.Close()
		}

		if meta.SessionStatus == SessionStatusNew {
			log.Info("Proxyman|Mux|Server: Received request for ", meta.Target)
			inboundRay, err := w.dispatcher.Dispatch(ctx, meta.Target)
			if err != nil {
				log.Info("Proxyman|Mux: Failed to dispatch request: ", err)
				continue
			}
			s = &session{
				input:  inboundRay.InboundOutput(),
				output: inboundRay.InboundInput(),
				parent: w,
				id:     meta.SessionID,
			}
			w.access.Lock()
			w.sessions[meta.SessionID] = s
			w.access.Unlock()
			go handle(ctx, s, w.outboundRay.OutboundOutput())
		}

		if !meta.Option.Has(OptionData) {
			continue
		}

		if s != nil {
			err = pipe(reader, s.output)
		} else {
			err = drain(reader)
		}

		if err != nil {
			log.Info("Proxyman|Mux|ServerWorker: Failed to read data: ", err)
			break
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package mailer

import (
	"encoding/hex"
	"errors"
	"fmt"
	"path"

	"github.com/Unknwon/com"
	"github.com/Unknwon/macaron"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

const (
	AUTH_ACTIVE           base.TplName = "mail/auth/active"
	AUTH_ACTIVATE_EMAIL   base.TplName = "mail/auth/activate_email"
	AUTH_REGISTER_SUCCESS base.TplName = "mail/auth/register_success"
	AUTH_RESET_PASSWORD   base.TplName = "mail/auth/reset_passwd"

	NOTIFY_COLLABORATOR base.TplName = "mail/notify/collaborator"
	NOTIFY_MENTION      base.TplName = "mail/notify/mention"
)

// Create New mail message use MailFrom and MailUser
func NewMailMessageFrom(To []string, from, subject, body string) Message {
	return NewHtmlMessage(To, from, subject, body)
}

// Create New mail message use MailFrom and MailUser
func NewMailMessage(To []string, subject, body string) Message {
	return NewMailMessageFrom(To, setting.MailService.From, subject, body)
}

func GetMailTmplData(u *models.User) map[interface{}]interface{} {
	data := make(map[interface{}]interface{}, 10)
	data["AppName"] = setting.AppName
	data["AppVer"] = setting.AppVer
	data["AppUrl"] = setting.AppUrl
	data["ActiveCodeLives"] = setting.Service.ActiveCodeLives / 60
	data["ResetPwdCodeLives"] = setting.Service.ResetPwdCodeLives / 60
	if u != nil {
		data["User"] = u
	}
	return data
}

// create a time limit code for user active
func CreateUserActiveCode(u *models.User, startInf interface{}) string {
	minutes := setting.Service.ActiveCodeLives
	data := com.ToStr(u.Id) + u.Email + u.LowerName + u.Passwd + u.Rands
	code := base.CreateTimeLimitCode(data, minutes, startInf)

	// add tail hex username
	code += hex.EncodeToString([]byte(u.LowerName))
	return code
}

// create a time limit code for user active
func CreateUserEmailActivateCode(u *models.User, e *models.EmailAddress, startInf interface{}) string {
	minutes := setting.Service.ActiveCodeLives
	data := com.ToStr(u.Id) + e.Email + u.LowerName + u.Passwd + u.Rands
	code := base.CreateTimeLimitCode(data, minutes, startInf)

	// add tail hex username
	code += hex.EncodeToString([]byte(u.LowerName))
	return code
}

// Send user register mail with active code
func SendRegisterMail(r macaron.Render, u *models.User) {
	code := CreateUserActiveCode(u, nil)
	subject := "Register success, Welcome"

	data := GetMailTmplData(u)
	data["Code"] = code
	body, err := r.HTMLString(string(AUTH_REGISTER_SUCCESS), data)
	if err != nil {
		log.Error(4, "mail.SendRegisterMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send register mail", u.Id)

	SendAsync(&msg)
}

// Send email verify active email.
func SendActiveMail(r macaron.Render, u *models.User) {
	code := CreateUserActiveCode(u, nil)

	subject := "Verify your e-mail address"

	data := GetMailTmplData(u)
	data["Code"] = code
	body, err := r.HTMLString(string(AUTH_ACTIVE), data)
	if err != nil {
		log.Error(4, "mail.SendActiveMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send active mail", u.Id)

	SendAsync(&msg)
}

// Send email to verify secondary email.
func SendActivateEmail(r macaron.Render, user *models.User, email *models.EmailAddress) {
	code := CreateUserEmailActivateCode(user, email, nil)

	subject := "Verify your e-mail address"

	data := GetMailTmplData(user)
	data["Code"] = code
	data["Email"] = email.Email
	body, err := r.HTMLString(string(AUTH_ACTIVATE_EMAIL), data)
	if err != nil {
		log.Error(4, "mail.SendActiveMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{email.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send activate email to %s", user.Id, email.Email)

	SendAsync(&msg)
}

// Send reset password email.
func SendResetPasswdMail(r macaron.Render, u *models.User) {
	code := CreateUserActiveCode(u, nil)

	subject := "Reset your password"

	data := GetMailTmplData(u)
	data["Code"] = code
	body, err := r.HTMLString(string(AUTH_RESET_PASSWORD), data)
	if err != nil {
		log.Error(4, "mail.SendResetPasswdMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send reset password email", u.Id)

	SendAsync(&msg)
}

// SendIssueNotifyMail sends mail notification of all watchers of repository.
func SendIssueNotifyMail(u, owner *models.User, repo *models.Repository, issue *models.Issue) ([]string, error) {
	ws, err := models.GetWatchers(repo.ID)
	if err != nil {
		return nil, errors.New("mail.NotifyWatchers(GetWatchers): " + err.Error())
	}

	tos := make([]string, 0, len(ws))
	for i := range ws {
		uid := ws[i].UserID
		if u.Id == uid {
			continue
		}
		u, err := models.GetUserByID(uid)
		if err != nil {
			return nil, errors.New("mail.NotifyWatchers(GetUserById): " + err.Error())
		}
		tos = append(tos, u.Email)
	}

	if len(tos) == 0 {
		return tos, nil
	}

	subject := fmt.Sprintf("[%s] %s(#%d)", repo.Name, issue.Name, issue.Index)
	content := fmt.Sprintf("%s
-
 View it on Gogs.",
		base.RenderSpecialLink([]byte(issue.Content), owner.Name+"/"+repo.Name),
		setting.AppUrl, owner.Name, repo.Name, issue.Index)
	msg := NewMailMessageFrom(tos, u.Email, subject, content)
	msg.Info = fmt.Sprintf("Subject: %s, send issue notify emails", subject)
	SendAsync(&msg)
	return tos, nil
}

// SendIssueMentionMail sends mail notification for who are mentioned in issue.
func SendIssueMentionMail(r macaron.Render, u, owner *models.User,
	repo *models.Repository, issue *models.Issue, tos []string) error {

	if len(tos) == 0 {
		return nil
	}

	subject := fmt.Sprintf("[%s] %s(#%d)", repo.Name, issue.Name, issue.Index)

	data := GetMailTmplData(nil)
	data["IssueLink"] = fmt.Sprintf("%s/%s/issues/%d", owner.Name, repo.Name, issue.Index)
	data["Subject"] = subject

	body, err := r.HTMLString(string(NOTIFY_MENTION), data)
	if err != nil {
		return fmt.Errorf("mail.SendIssueMentionMail(fail to render): %v", err)
	}

	msg := NewMailMessageFrom(tos, u.Email, subject, body)
	msg.Info = fmt.Sprintf("Subject: %s, send issue mention emails", subject)
	SendAsync(&msg)
	return nil
}

// SendCollaboratorMail sends mail notification to new collaborator.
func SendCollaboratorMail(r macaron.Render, u, owner *models.User,
	repo *models.Repository) error {

	subject := fmt.Sprintf("%s added you to %s", owner.Name, repo.Name)

	data := GetMailTmplData(nil)
	data["RepoLink"] = path.Join(owner.Name, repo.Name)
	data["Subject"] = subject

	body, err := r.HTMLString(string(NOTIFY_COLLABORATOR), data)
	if err != nil {
		return fmt.Errorf("mail.SendCollaboratorMail(fail to render): %v", err)
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send register mail", u.Id)

	SendAsync(&msg)
	return nil
}
<commit_msg>Fix missing space in issue emails subject<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package mailer

import (
	"encoding/hex"
	"errors"
	"fmt"
	"path"

	"github.com/Unknwon/com"
	"github.com/Unknwon/macaron"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/base"
	"github.com/gogits/gogs/modules/log"
	"github.com/gogits/gogs/modules/setting"
)

const (
	AUTH_ACTIVE           base.TplName = "mail/auth/active"
	AUTH_ACTIVATE_EMAIL   base.TplName = "mail/auth/activate_email"
	AUTH_REGISTER_SUCCESS base.TplName = "mail/auth/register_success"
	AUTH_RESET_PASSWORD   base.TplName = "mail/auth/reset_passwd"

	NOTIFY_COLLABORATOR base.TplName = "mail/notify/collaborator"
	NOTIFY_MENTION      base.TplName = "mail/notify/mention"
)

// Create New mail message use MailFrom and MailUser
func NewMailMessageFrom(To []string, from, subject, body string) Message {
	return NewHtmlMessage(To, from, subject, body)
}

// Create New mail message use MailFrom and MailUser
func NewMailMessage(To []string, subject, body string) Message {
	return NewMailMessageFrom(To, setting.MailService.From, subject, body)
}

func GetMailTmplData(u *models.User) map[interface{}]interface{} {
	data := make(map[interface{}]interface{}, 10)
	data["AppName"] = setting.AppName
	data["AppVer"] = setting.AppVer
	data["AppUrl"] = setting.AppUrl
	data["ActiveCodeLives"] = setting.Service.ActiveCodeLives / 60
	data["ResetPwdCodeLives"] = setting.Service.ResetPwdCodeLives / 60
	if u != nil {
		data["User"] = u
	}
	return data
}

// create a time limit code for user active
func CreateUserActiveCode(u *models.User, startInf interface{}) string {
	minutes := setting.Service.ActiveCodeLives
	data := com.ToStr(u.Id) + u.Email + u.LowerName + u.Passwd + u.Rands
	code := base.CreateTimeLimitCode(data, minutes, startInf)

	// add tail hex username
	code += hex.EncodeToString([]byte(u.LowerName))
	return code
}

// create a time limit code for user active
func CreateUserEmailActivateCode(u *models.User, e *models.EmailAddress, startInf interface{}) string {
	minutes := setting.Service.ActiveCodeLives
	data := com.ToStr(u.Id) + e.Email + u.LowerName + u.Passwd + u.Rands
	code := base.CreateTimeLimitCode(data, minutes, startInf)

	// add tail hex username
	code += hex.EncodeToString([]byte(u.LowerName))
	return code
}

// Send user register mail with active code
func SendRegisterMail(r macaron.Render, u *models.User) {
	code := CreateUserActiveCode(u, nil)
	subject := "Register success, Welcome"

	data := GetMailTmplData(u)
	data["Code"] = code
	body, err := r.HTMLString(string(AUTH_REGISTER_SUCCESS), data)
	if err != nil {
		log.Error(4, "mail.SendRegisterMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send register mail", u.Id)

	SendAsync(&msg)
}

// Send email verify active email.
func SendActiveMail(r macaron.Render, u *models.User) {
	code := CreateUserActiveCode(u, nil)

	subject := "Verify your e-mail address"

	data := GetMailTmplData(u)
	data["Code"] = code
	body, err := r.HTMLString(string(AUTH_ACTIVE), data)
	if err != nil {
		log.Error(4, "mail.SendActiveMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send active mail", u.Id)

	SendAsync(&msg)
}

// Send email to verify secondary email.
func SendActivateEmail(r macaron.Render, user *models.User, email *models.EmailAddress) {
	code := CreateUserEmailActivateCode(user, email, nil)

	subject := "Verify your e-mail address"

	data := GetMailTmplData(user)
	data["Code"] = code
	data["Email"] = email.Email
	body, err := r.HTMLString(string(AUTH_ACTIVATE_EMAIL), data)
	if err != nil {
		log.Error(4, "mail.SendActiveMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{email.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send activate email to %s", user.Id, email.Email)

	SendAsync(&msg)
}

// Send reset password email.
func SendResetPasswdMail(r macaron.Render, u *models.User) {
	code := CreateUserActiveCode(u, nil)

	subject := "Reset your password"

	data := GetMailTmplData(u)
	data["Code"] = code
	body, err := r.HTMLString(string(AUTH_RESET_PASSWORD), data)
	if err != nil {
		log.Error(4, "mail.SendResetPasswdMail(fail to render): %v", err)
		return
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send reset password email", u.Id)

	SendAsync(&msg)
}

// SendIssueNotifyMail sends mail notification of all watchers of repository.
func SendIssueNotifyMail(u, owner *models.User, repo *models.Repository, issue *models.Issue) ([]string, error) {
	ws, err := models.GetWatchers(repo.ID)
	if err != nil {
		return nil, errors.New("mail.NotifyWatchers(GetWatchers): " + err.Error())
	}

	tos := make([]string, 0, len(ws))
	for i := range ws {
		uid := ws[i].UserID
		if u.Id == uid {
			continue
		}
		u, err := models.GetUserByID(uid)
		if err != nil {
			return nil, errors.New("mail.NotifyWatchers(GetUserById): " + err.Error())
		}
		tos = append(tos, u.Email)
	}

	if len(tos) == 0 {
		return tos, nil
	}

	subject := fmt.Sprintf("[%s] %s (#%d)", repo.Name, issue.Name, issue.Index)
	content := fmt.Sprintf("%s
-
 View it on Gogs.",
		base.RenderSpecialLink([]byte(issue.Content), owner.Name+"/"+repo.Name),
		setting.AppUrl, owner.Name, repo.Name, issue.Index)
	msg := NewMailMessageFrom(tos, u.Email, subject, content)
	msg.Info = fmt.Sprintf("Subject: %s, send issue notify emails", subject)
	SendAsync(&msg)
	return tos, nil
}

// SendIssueMentionMail sends mail notification for who are mentioned in issue.
func SendIssueMentionMail(r macaron.Render, u, owner *models.User,
	repo *models.Repository, issue *models.Issue, tos []string) error {

	if len(tos) == 0 {
		return nil
	}

	subject := fmt.Sprintf("[%s] %s (#%d)", repo.Name, issue.Name, issue.Index)

	data := GetMailTmplData(nil)
	data["IssueLink"] = fmt.Sprintf("%s/%s/issues/%d", owner.Name, repo.Name, issue.Index)
	data["Subject"] = subject

	body, err := r.HTMLString(string(NOTIFY_MENTION), data)
	if err != nil {
		return fmt.Errorf("mail.SendIssueMentionMail(fail to render): %v", err)
	}

	msg := NewMailMessageFrom(tos, u.Email, subject, body)
	msg.Info = fmt.Sprintf("Subject: %s, send issue mention emails", subject)
	SendAsync(&msg)
	return nil
}

// SendCollaboratorMail sends mail notification to new collaborator.
func SendCollaboratorMail(r macaron.Render, u, owner *models.User,
	repo *models.Repository) error {

	subject := fmt.Sprintf("%s added you to %s", owner.Name, repo.Name)

	data := GetMailTmplData(nil)
	data["RepoLink"] = path.Join(owner.Name, repo.Name)
	data["Subject"] = subject

	body, err := r.HTMLString(string(NOTIFY_COLLABORATOR), data)
	if err != nil {
		return fmt.Errorf("mail.SendCollaboratorMail(fail to render): %v", err)
	}

	msg := NewMailMessage([]string{u.Email}, subject, body)
	msg.Info = fmt.Sprintf("UID: %d, send register mail", u.Id)

	SendAsync(&msg)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package install

import (
	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/sample-apiserver/pkg/apis/wardle"
	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
)

// Install registers the API group and adds types to a scheme
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	if err := announced.NewGroupMetaFactory(
		&announced.GroupMetaFactoryArgs{
			GroupName:                  wardle.GroupName,
			RootScopedKinds:            sets.NewString("Fischer", "FischerList"),
			VersionPreferenceOrder:     []string{v1alpha1.SchemeGroupVersion.Version},
			ImportPrefix:               "k8s.io/sample-apiserver/pkg/apis/wardle",
			AddInternalObjectsToScheme: wardle.AddToScheme,
		},
		announced.VersionToSchemeFunc{
			v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme,
		},
	).Announce(groupFactoryRegistry).RegisterAndEnable(registry, scheme); err != nil {
		panic(err)
	}
}
<commit_msg>apimachinery: remove pre-apigroups import prefix logic<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package install

import (
	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/sample-apiserver/pkg/apis/wardle"
	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
)

// Install registers the API group and adds types to a scheme
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	if err := announced.NewGroupMetaFactory(
		&announced.GroupMetaFactoryArgs{
			GroupName:                  wardle.GroupName,
			RootScopedKinds:            sets.NewString("Fischer", "FischerList"),
			VersionPreferenceOrder:     []string{v1alpha1.SchemeGroupVersion.Version},
			AddInternalObjectsToScheme: wardle.AddToScheme,
		},
		announced.VersionToSchemeFunc{
			v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme,
		},
	).Announce(groupFactoryRegistry).RegisterAndEnable(registry, scheme); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package querier

import (
	"log"
	"sync"

	"github.com/digitalocean/vulcan/bus"
	"github.com/digitalocean/vulcan/convert"
	"github.com/digitalocean/vulcan/storage"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/metric"
)

// DatapointReader is able to create SeriesIterator that talk to a DatapointReader.
// This bridges the new series iterator prometheus interface to our older cassandra
// code that was built for a different interface. Eventually, a cassandra storage
// object should just be able to create iterators directly.
type DatapointReader struct {
	DR storage.DatapointReader
}

// Iterator returns a new SeriesIterator
func (dr *DatapointReader) Iterator(m metric.Metric, from, through model.Time) (local.SeriesIterator, error) {
	key, err := convert.MetricToKey(toBusMetric(m.Metric))
	if err != nil {
		return nil, err
	}
	si := &SeriesIterator{
		After:    from,
		Before:   through,
		Key:      key,
		M:        m,
		DPReader: dr.DR,

		points: []bus.Datapoint{},
	}

	return si, nil
}

// SeriesIterator enables efficient access of sample values in a series. Its
// methods are not goroutine-safe. A SeriesIterator iterates over a snapshot of
// a series, i.e. it is safe to continue using a SeriesIterator after or during
// modifying the corresponding series, but the iterator will represent the state
// of the series prior to the modification.
type SeriesIterator struct {
	After, Before model.Time
	M             metric.Metric
	Key           string
	DPReader      storage.DatapointReader

	l      sync.Mutex
	ready  bool
	points []bus.Datapoint
}

// ValueAtOrBeforeTime gets the value that is closest before the given time. In case a value
// exists at precisely the given time, that value is returned. If no
// applicable value exists, ZeroSamplePair is returned.
func (si *SeriesIterator) ValueAtOrBeforeTime(t model.Time) model.SamplePair {
	si.fetch()
	for _, point := range si.points {
		if int64(point.Timestamp) <= int64(t) {
			return model.SamplePair{
				Timestamp: model.Time(point.Timestamp),
				Value:     model.SampleValue(point.Value),
			}
		}
	}
	return local.ZeroSamplePair
}

// RangeValues gets all values contained within a given interval.
func (si *SeriesIterator) RangeValues(r metric.Interval) []model.SamplePair {
	si.fetch()
	result := []model.SamplePair{}
	for i := len(si.points) - 1; i > 0; i-- {
		point := si.points[i]
		if int64(r.OldestInclusive) <= int64(point.Timestamp) && int64(point.Timestamp) <= int64(r.NewestInclusive) {
			result = append(result, model.SamplePair{
				Timestamp: model.Time(point.Timestamp),
				Value:     model.SampleValue(point.Value),
			})
		}
	}
	return result
}

// Metric returns the metric of the series that the iterator corresponds to.
func (si *SeriesIterator) Metric() metric.Metric {
	return si.M
}

// Close closes the iterator and releases the underlying data.
func (si *SeriesIterator) Close() {
	return
}

func (si *SeriesIterator) fetch() {
	// early unsynchronized exit
	if si.ready {
		return
	}
	si.l.Lock()
	defer si.l.Unlock()
	// recheck condition
	if si.ready {
		return
	}
	points, err := si.DPReader.ReadDatapoints(si.Key, bus.Timestamp(si.After), bus.Timestamp(si.Before))
	if err != nil {
		log.Println(err)
	}
	si.points = points
	si.ready = true
}
<commit_msg>adds license<commit_after>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package querier

import (
	"log"
	"sync"

	"github.com/digitalocean/vulcan/bus"
	"github.com/digitalocean/vulcan/convert"
	"github.com/digitalocean/vulcan/storage"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/prometheus/prometheus/storage/metric"
)

// DatapointReader is able to create SeriesIterator that talk to a DatapointReader.
// This bridges the new series iterator prometheus interface to our older cassandra
// code that was built for a different interface. Eventually, a cassandra storage
// object should just be able to create iterators directly.
type DatapointReader struct {
	DR storage.DatapointReader
}

// Iterator returns a new SeriesIterator
func (dr *DatapointReader) Iterator(m metric.Metric, from, through model.Time) (local.SeriesIterator, error) {
	key, err := convert.MetricToKey(toBusMetric(m.Metric))
	if err != nil {
		return nil, err
	}
	si := &SeriesIterator{
		After:    from,
		Before:   through,
		Key:      key,
		M:        m,
		DPReader: dr.DR,

		points: []bus.Datapoint{},
	}

	return si, nil
}

// SeriesIterator enables efficient access of sample values in a series. Its
// methods are not goroutine-safe. A SeriesIterator iterates over a snapshot of
// a series, i.e. it is safe to continue using a SeriesIterator after or during
// modifying the corresponding series, but the iterator will represent the state
// of the series prior to the modification.
type SeriesIterator struct {
	After, Before model.Time
	M             metric.Metric
	Key           string
	DPReader      storage.DatapointReader

	l      sync.Mutex
	ready  bool
	points []bus.Datapoint
}

// ValueAtOrBeforeTime gets the value that is closest before the given time. In case a value
// exists at precisely the given time, that value is returned. If no
// applicable value exists, ZeroSamplePair is returned.
func (si *SeriesIterator) ValueAtOrBeforeTime(t model.Time) model.SamplePair {
	si.fetch()
	for _, point := range si.points {
		if int64(point.Timestamp) <= int64(t) {
			return model.SamplePair{
				Timestamp: model.Time(point.Timestamp),
				Value:     model.SampleValue(point.Value),
			}
		}
	}
	return local.ZeroSamplePair
}

// RangeValues gets all values contained within a given interval.
func (si *SeriesIterator) RangeValues(r metric.Interval) []model.SamplePair {
	si.fetch()
	result := []model.SamplePair{}
	for i := len(si.points) - 1; i > 0; i-- {
		point := si.points[i]
		if int64(r.OldestInclusive) <= int64(point.Timestamp) && int64(point.Timestamp) <= int64(r.NewestInclusive) {
			result = append(result, model.SamplePair{
				Timestamp: model.Time(point.Timestamp),
				Value:     model.SampleValue(point.Value),
			})
		}
	}
	return result
}

// Metric returns the metric of the series that the iterator corresponds to.
func (si *SeriesIterator) Metric() metric.Metric {
	return si.M
}

// Close closes the iterator and releases the underlying data.
func (si *SeriesIterator) Close() {
	return
}

func (si *SeriesIterator) fetch() {
	// early unsynchronized exit
	if si.ready {
		return
	}
	si.l.Lock()
	defer si.l.Unlock()
	// recheck condition
	if si.ready {
		return
	}
	points, err := si.DPReader.ReadDatapoints(si.Key, bus.Timestamp(si.After), bus.Timestamp(si.Before))
	if err != nil {
		log.Println(err)
	}
	si.points = points
	si.ready = true
}
<|endoftext|>


<commit_before><commit_msg>enable 3 delete strategies<commit_after><|endoftext|>


<commit_before>package confetto

import (
	"fmt"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

var (
	regs       map[string]interface{}
	configured bool
	errors     []string
)

func saveError(format string, a ...interface{}) {
	m := fmt.Sprintf(format, a...)
	errors = append(errors, m)
}

func Register(path string, dest interface{}) {
	// FIXME in realtà basterebbe verificare che sia settabile
	if reflect.TypeOf(dest).Kind() != reflect.Ptr {
		// FIXME
		panic("destination is not a pointer type")
	}
	if regs == nil {
		regs = make(map[string]interface{})
	}
	if _, ok := regs[path]; ok {
		saveError("%s: path already registerd")
	}
	regs[path] = dest
}

func Configure(yamlData []byte) {
	if configured {
		panic("already configured")
	}
	configured = true
	conf := make(map[interface{}]interface{})
	err := yaml.Unmarshal(yamlData, &conf)
	if err != nil {
		panic("cannot decode yaml data")
	}

	for path, dest := range regs {
		subConf := getSubConf(path, conf)
		if subConf != nil {
			configPath(path, reflect.ValueOf(dest), subConf)
		}
	}

	//FIXME
	fmt.Printf("%d error(s)\n", len(errors))
	for _, m := range errors {
		fmt.Println(m)
	}
}

func getSubConf(path string, conf interface{}) interface{} {
	subConf := conf
	for _, p := range strings.Split(path, ".") {
		m, ok := subConf.(map[interface{}]interface{})
		if !ok {
			saveError("%s: path not found", path)
			return nil
		}

		subConf, ok = m[p]
		if !ok {
			return nil
		}
	}
	return subConf
}

func configStruct(path string, dest reflect.Value, conf interface{}) {
	if conf == nil {
		return
	}
	confMap, ok := conf.(map[interface{}]interface{})
	if !ok {
		saveError("%s: expected map not found", path)
		return
	}
	for k, subConf := range confMap {
		kk, ok := k.(string)
		if !ok {
			saveError("%s.%v: map key is not a string", path, k)
			return
		}
		// TODO Title is arbitrary
		fieldName := strings.Title(kk)
		fieldVal := dest.FieldByName(fieldName)
		if fieldVal.Kind() == reflect.Invalid {
			saveError("%s.%v: field not present in target struct", path, k)
			return
		}
		configPath(path+"."+kk, dest.FieldByName(fieldName), subConf)
	}
}

func configPath(path string, dest reflect.Value, conf interface{}) {
	fmt.Printf("path=%s %v\n", path, dest)
	destKind := dest.Kind()
	switch destKind {
	case reflect.Ptr:
		configPath(path, dest.Elem(), conf)
	case reflect.Struct:
		configStruct(path, dest, conf)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.String, reflect.Bool:
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			saveError("%s: target type %v != conf type %v", dType, cType)
			return
		}
		dest.Set(confValue)
	default:
		saveError("%s: conf type %v not handled", dest.Type())
		return
	}
}
<commit_msg>first cut<commit_after>package confoo

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

const confVar = "CONFOO_CONFIG_FILE"

func errorPanic(format string, a ...interface{}) {
	m := fmt.Sprintf(format, a...)
	m2 := "CONFOO - " + m
	panic(m2)
}

func Configure(path string, target interface{}) {
	confFile := os.Getenv("CONFOO_CONFIG_FILE")
	if confFile == "" {
		errorPanic(confVar + " is not set")
	}
	data, err := ioutil.ReadFile(confFile)
	if err != nil {
		errorPanic(err.Error())
		panic(err)
	}

	conf := make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &conf)
	if err != nil {
		errorPanic("cannot decode yaml data")
	}

	subConf := getSubConf(path, conf)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
}

func getSubConf(path string, conf interface{}) interface{} {
	subConf := conf
	for _, p := range strings.Split(path, ".") {
		m, ok := subConf.(map[interface{}]interface{})
		if !ok {
			errorPanic("%s: path not found", path)
		}

		subConf, ok = m[p]
		if !ok {
			return nil
		}
	}
	return subConf
}

func configStruct(path string, dest reflect.Value, conf interface{}) {
	if conf == nil {
		return
	}
	confMap, ok := conf.(map[interface{}]interface{})
	if !ok {
		errorPanic("%s: expected map not found", path)
	}
	for k, subConf := range confMap {
		kk, ok := k.(string)
		if !ok {
			errorPanic("%s.%v: map key is not a string", path, k)
		}
		// TODO Title is arbitrary
		fieldName := strings.Title(kk)
		fieldVal := dest.FieldByName(fieldName)
		if fieldVal.Kind() == reflect.Invalid {
			errorPanic("%s.%v: field not present in target struct", path, k)
		}
		configPath(path+"."+kk, dest.FieldByName(fieldName), subConf)
	}
}

func configPath(path string, dest reflect.Value, conf interface{}) {
	destKind := dest.Kind()
	switch destKind {
	case reflect.Ptr:
		configPath(path, dest.Elem(), conf)
	case reflect.Struct:
		configStruct(path, dest, conf)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.String, reflect.Bool:
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	default:
		errorPanic("%s: conf type %v not handled", dest.Type())
	}
}
<|endoftext|>


<commit_before>package miner

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errNilCS     = errors.New("miner cannot use a nil consensus set")
	errNilTpool  = errors.New("miner cannot use a nil transaction pool")
	errNilWallet = errors.New("miner cannot use a nil wallet")

	// HeaderMemory is the number of previous calls to 'header'
	// that are remembered. Additionally, 'header' will only poll for a
	// new block every 'headerMemory / blockMemory' times it is
	// called. This reduces the amount of memory used, but comes at the cost of
	// not always having the most recent transactions.
	HeaderMemory = func() int {
		if build.Release == "dev" {
			return 500
		}
		if build.Release == "standard" {
			return 10000
		}
		if build.Release == "testing" {
			return 50
		}
		panic("unrecognized build.Release")
	}()

	// BlockMemory is the maximum number of blocks the miner will store
	// Blocks take up to 2 megabytes of memory, which is why this number is
	// limited.
	BlockMemory = func() int {
		if build.Release == "dev" {
			return 10
		}
		if build.Release == "standard" {
			return 50
		}
		if build.Release == "testing" {
			return 5
		}
		panic("unrecognized build.Release")
	}()

	// MaxSourceBlockAge is the maximum amount of time that is allowed to
	// elapse between generating source blocks.
	MaxSourceBlockAge = func() time.Duration {
		if build.Release == "dev" {
			return 5 * time.Second
		}
		if build.Release == "standard" {
			return 30 * time.Second
		}
		if build.Release == "testing" {
			return 1 * time.Second
		}
		panic("unrecognized build.Release")
	}()
)

// Miner struct contains all variables the miner needs
// in order to create and submit blocks.
type Miner struct {
	// Module dependencies.
	cs     modules.ConsensusSet
	tpool  modules.TransactionPool
	wallet modules.Wallet

	// BlockManager variables. Becaues blocks are large, one block is used to
	// make many headers which can be used by miners. Headers include an
	// arbitrary data transaction (appended to the block) to make the merkle
	// roots unique (preventing miners from doing redundant work). Every N
	// requests or M seconds, a new block is used to create headers.
	//
	// Only 'blocksMemory' blocks are kept in memory at a time, which
	// keeps ram usage reasonable. Miners may request many headers in parallel,
	// and thus may be working on different blocks. When they submit the solved
	// header to the block manager, the rest of the block needs to be found in
	// a lookup.
	blockMem        map[types.BlockHeader]*types.Block             // Mappings from headers to the blocks they are derived from.
	arbDataMem      map[types.BlockHeader][crypto.EntropySize]byte // Mappings from the headers to their unique arb data.
	headerMem       []types.BlockHeader                            // A circular list of headers that have been given out from the api recently.
	sourceBlock     *types.Block                                   // The block from which new headers for mining are created.
	sourceBlockTime time.Time                                      // How long headers have been using the same block (different from 'recent block').
	memProgress     int                                            // The index of the most recent header used in headerMem.

	// CPUMiner variables.
	miningOn bool  // indicates if the miner is supposed to be running
	mining   bool  // indicates if the miner is actually running
	hashRate int64 // indicates hashes per second

	// Utils
	log        *persist.Logger
	mu         sync.RWMutex
	persist    persistence
	persistDir string
}

// startupRescan will rescan the blockchain in the event that the miner
// persistance layer has become desynchronized from the consensus persistance
// layer. This might happen if a user replaces any of the folders with backups
// or deletes any of the folders.
func (m *Miner) startupRescan() error {
	// Reset all of the variables that have relevance to the consensus set. The
	// operations are wrapped by an anonymous function so that the locking can
	// be handled using a defer statement.
	err := func() error {
		m.mu.Lock()
		defer m.mu.Unlock()

		m.persist.RecentChange = modules.ConsensusChangeID{}
		m.persist.Height = 0
		m.persist.Target = types.Target{}
		return m.save()
	}()
	if err != nil {
		return err
	}

	// Subscribe to the consensus set. This is a blocking call that will not
	// return until the miner has fully caught up to the current block.
	return m.cs.ConsensusSetPersistentSubscribe(m, modules.ConsensusChangeID{})
}

// New returns a ready-to-go miner that is not mining.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, w modules.Wallet, persistDir string) (*Miner, error) {
	// Create the miner and its dependencies.
	if cs == nil {
		return nil, errNilCS
	}
	if tpool == nil {
		return nil, errNilTpool
	}
	if w == nil {
		return nil, errNilWallet
	}

	// Assemble the miner. The miner is assembled without an address because
	// the wallet is likely not unlocked yet. The miner will grab an address
	// after the miner is unlocked (this must be coded manually for each
	// function that potentially requires the miner to have an address.
	m := &Miner{
		cs:     cs,
		tpool:  tpool,
		wallet: w,

		blockMem:   make(map[types.BlockHeader]*types.Block),
		arbDataMem: make(map[types.BlockHeader][crypto.EntropySize]byte),
		headerMem:  make([]types.BlockHeader, HeaderMemory),

		persistDir: persistDir,
	}

	err := m.initPersist()
	if err != nil {
		return nil, errors.New("miner persistence startup failed: " + err.Error())
	}

	err = m.cs.ConsensusSetPersistentSubscribe(m, m.persist.RecentChange)
	if err == modules.ErrInvalidConsensusChangeID {
		// Perform a rescan of the consensus set if the change id is not found.
		// The id will only be not found if there has been desynchronization
		// between the miner and the consensus package.
		err = m.startupRescan()
		if err != nil {
			return nil, errors.New("miner startup failed - rescanning failed: " + err.Error())
		}
	} else if err != nil {
		return nil, errors.New("miner subscription failed: " + err.Error())
	}

	m.tpool.TransactionPoolSubscribe(m)
	return m, nil
}

// Close terminates all ongoing processes involving the miner, enabling garbage
// collection.
func (m *Miner) Close() error {
	// Save the latest miner state.
	err := func() error {
		m.mu.Lock()
		defer m.mu.Unlock()
		return m.save()
	}()
	if err != nil {
		return err
	}

	m.cs.Unsubscribe(m)
	return m.log.Close()
}

// checkAddress checks that the miner has an address, fetching an address from
// the wallet if not.
func (m *Miner) checkAddress() error {
	if m.persist.Address != (types.UnlockHash{}) {
		return nil
	}
	uc, err := m.wallet.NextAddress()
	if err != nil {
		return err
	}
	m.persist.Address = uc.UnlockHash()
	return nil
}

// BlocksMined returns the number of good blocks and stale blocks that have
// been mined by the miner.
func (m *Miner) BlocksMined() (goodBlocks, staleBlocks int) {
	m.mu.Lock()
	defer m.mu.Unlock()

	for _, blockID := range m.persist.BlocksFound {
		if m.cs.InCurrentPath(blockID) {
			goodBlocks++
		} else {
			staleBlocks++
		}
	}
	return
}
<commit_msg>remove an unneeded anonymous function from miner<commit_after>package miner

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errNilCS     = errors.New("miner cannot use a nil consensus set")
	errNilTpool  = errors.New("miner cannot use a nil transaction pool")
	errNilWallet = errors.New("miner cannot use a nil wallet")

	// HeaderMemory is the number of previous calls to 'header'
	// that are remembered. Additionally, 'header' will only poll for a
	// new block every 'headerMemory / blockMemory' times it is
	// called. This reduces the amount of memory used, but comes at the cost of
	// not always having the most recent transactions.
	HeaderMemory = func() int {
		if build.Release == "dev" {
			return 500
		}
		if build.Release == "standard" {
			return 10000
		}
		if build.Release == "testing" {
			return 50
		}
		panic("unrecognized build.Release")
	}()

	// BlockMemory is the maximum number of blocks the miner will store
	// Blocks take up to 2 megabytes of memory, which is why this number is
	// limited.
	BlockMemory = func() int {
		if build.Release == "dev" {
			return 10
		}
		if build.Release == "standard" {
			return 50
		}
		if build.Release == "testing" {
			return 5
		}
		panic("unrecognized build.Release")
	}()

	// MaxSourceBlockAge is the maximum amount of time that is allowed to
	// elapse between generating source blocks.
	MaxSourceBlockAge = func() time.Duration {
		if build.Release == "dev" {
			return 5 * time.Second
		}
		if build.Release == "standard" {
			return 30 * time.Second
		}
		if build.Release == "testing" {
			return 1 * time.Second
		}
		panic("unrecognized build.Release")
	}()
)

// Miner struct contains all variables the miner needs
// in order to create and submit blocks.
type Miner struct {
	// Module dependencies.
	cs     modules.ConsensusSet
	tpool  modules.TransactionPool
	wallet modules.Wallet

	// BlockManager variables. Becaues blocks are large, one block is used to
	// make many headers which can be used by miners. Headers include an
	// arbitrary data transaction (appended to the block) to make the merkle
	// roots unique (preventing miners from doing redundant work). Every N
	// requests or M seconds, a new block is used to create headers.
	//
	// Only 'blocksMemory' blocks are kept in memory at a time, which
	// keeps ram usage reasonable. Miners may request many headers in parallel,
	// and thus may be working on different blocks. When they submit the solved
	// header to the block manager, the rest of the block needs to be found in
	// a lookup.
	blockMem        map[types.BlockHeader]*types.Block             // Mappings from headers to the blocks they are derived from.
	arbDataMem      map[types.BlockHeader][crypto.EntropySize]byte // Mappings from the headers to their unique arb data.
	headerMem       []types.BlockHeader                            // A circular list of headers that have been given out from the api recently.
	sourceBlock     *types.Block                                   // The block from which new headers for mining are created.
	sourceBlockTime time.Time                                      // How long headers have been using the same block (different from 'recent block').
	memProgress     int                                            // The index of the most recent header used in headerMem.

	// CPUMiner variables.
	miningOn bool  // indicates if the miner is supposed to be running
	mining   bool  // indicates if the miner is actually running
	hashRate int64 // indicates hashes per second

	// Utils
	log        *persist.Logger
	mu         sync.RWMutex
	persist    persistence
	persistDir string
}

// startupRescan will rescan the blockchain in the event that the miner
// persistance layer has become desynchronized from the consensus persistance
// layer. This might happen if a user replaces any of the folders with backups
// or deletes any of the folders.
func (m *Miner) startupRescan() error {
	// Reset all of the variables that have relevance to the consensus set. The
	// operations are wrapped by an anonymous function so that the locking can
	// be handled using a defer statement.
	err := func() error {
		m.mu.Lock()
		defer m.mu.Unlock()

		m.persist.RecentChange = modules.ConsensusChangeID{}
		m.persist.Height = 0
		m.persist.Target = types.Target{}
		return m.save()
	}()
	if err != nil {
		return err
	}

	// Subscribe to the consensus set. This is a blocking call that will not
	// return until the miner has fully caught up to the current block.
	return m.cs.ConsensusSetPersistentSubscribe(m, modules.ConsensusChangeID{})
}

// New returns a ready-to-go miner that is not mining.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, w modules.Wallet, persistDir string) (*Miner, error) {
	// Create the miner and its dependencies.
	if cs == nil {
		return nil, errNilCS
	}
	if tpool == nil {
		return nil, errNilTpool
	}
	if w == nil {
		return nil, errNilWallet
	}

	// Assemble the miner. The miner is assembled without an address because
	// the wallet is likely not unlocked yet. The miner will grab an address
	// after the miner is unlocked (this must be coded manually for each
	// function that potentially requires the miner to have an address.
	m := &Miner{
		cs:     cs,
		tpool:  tpool,
		wallet: w,

		blockMem:   make(map[types.BlockHeader]*types.Block),
		arbDataMem: make(map[types.BlockHeader][crypto.EntropySize]byte),
		headerMem:  make([]types.BlockHeader, HeaderMemory),

		persistDir: persistDir,
	}

	err := m.initPersist()
	if err != nil {
		return nil, errors.New("miner persistence startup failed: " + err.Error())
	}

	err = m.cs.ConsensusSetPersistentSubscribe(m, m.persist.RecentChange)
	if err == modules.ErrInvalidConsensusChangeID {
		// Perform a rescan of the consensus set if the change id is not found.
		// The id will only be not found if there has been desynchronization
		// between the miner and the consensus package.
		err = m.startupRescan()
		if err != nil {
			return nil, errors.New("miner startup failed - rescanning failed: " + err.Error())
		}
	} else if err != nil {
		return nil, errors.New("miner subscription failed: " + err.Error())
	}

	m.tpool.TransactionPoolSubscribe(m)
	return m, nil
}

// Close terminates all ongoing processes involving the miner, enabling garbage
// collection.
func (m *Miner) Close() error {
	// Save the latest miner state.
	m.mu.RLock()
	err := m.save()
	m.mu.RUnlock()
	if err != nil {
		return err
	}

	m.cs.Unsubscribe(m)
	return m.log.Close()
}

// checkAddress checks that the miner has an address, fetching an address from
// the wallet if not.
func (m *Miner) checkAddress() error {
	if m.persist.Address != (types.UnlockHash{}) {
		return nil
	}
	uc, err := m.wallet.NextAddress()
	if err != nil {
		return err
	}
	m.persist.Address = uc.UnlockHash()
	return nil
}

// BlocksMined returns the number of good blocks and stale blocks that have
// been mined by the miner.
func (m *Miner) BlocksMined() (goodBlocks, staleBlocks int) {
	m.mu.Lock()
	defer m.mu.Unlock()

	for _, blockID := range m.persist.BlocksFound {
		if m.cs.InCurrentPath(blockID) {
			goodBlocks++
		} else {
			staleBlocks++
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/docker/go-plugins-helpers/volume"
	quobyte_api "github.com/quobyte/api"
)

type quobyteDriver struct {
	client       *quobyte_api.QuobyteClient
	quobyteMount string
	m            *sync.Mutex
}

func newQuobyteDriver(apiURL string, username string, password string, quobyteMount string) quobyteDriver {
	driver := quobyteDriver{
		client:       quobyte_api.NewQuobyteClient(apiURL, username, password),
		quobyteMount: quobyteMount,
		m:            &sync.Mutex{},
	}

	return driver
}

func (driver quobyteDriver) Create(request volume.Request) volume.Response {
	log.Printf("Creating volume %s\n", request.Name)
	driver.m.Lock()
	defer driver.m.Unlock()

	user, group := "root", "root"

	if usr, ok := request.Options["user"]; ok {
		user = usr
	}

	if grp, ok := request.Options["group"]; ok {
		group = grp
	}

	if _, err := driver.client.CreateVolume(&quobyte_api.CreateVolumeRequest{
		Name:        request.Name,
		RootUserID:  user,
		RootGroupID: group,
	}); err != nil {
		log.Println(err)
		return volume.Response{Err: err.Error()}
	}

	return volume.Response{Err: ""}
}

func (driver quobyteDriver) Remove(request volume.Request) volume.Response {
	log.Printf("Removing volume %s\n", request.Name)
	driver.m.Lock()
	defer driver.m.Unlock()

	if err := driver.client.DeleteVolumeByName(request.Name, ""); err != nil {
		log.Println(err)
		return volume.Response{Err: err.Error()}
	}

	return volume.Response{Err: ""}
}

func (driver quobyteDriver) Mount(request volume.MountRequest) volume.Response {
	driver.m.Lock()
	defer driver.m.Unlock()
	mPoint := filepath.Join(driver.quobyteMount, request.Name)
	log.Printf("Mounting volume %s on %s\n", request.Name, mPoint)
	if fi, err := os.Lstat(mPoint); err != nil || !fi.IsDir() {
		log.Println(err)
		return volume.Response{Err: fmt.Sprintf("%v not mounted", mPoint)}
	}

	return volume.Response{Err: "", Mountpoint: mPoint}
}

func (driver quobyteDriver) Path(request volume.Request) volume.Response {
	return volume.Response{Mountpoint: filepath.Join(driver.quobyteMount, request.Name)}
}

func (driver quobyteDriver) Unmount(request volume.UnmountRequest) volume.Response {
	return volume.Response{}
}

func (driver quobyteDriver) Get(request volume.Request) volume.Response {
	driver.m.Lock()
	defer driver.m.Unlock()

	mPoint := filepath.Join(driver.quobyteMount, request.Name)

	if fi, err := os.Lstat(mPoint); err != nil || !fi.IsDir() {
		log.Println(err)
		return volume.Response{Err: fmt.Sprintf("%v not mounted", mPoint)}
	}

	return volume.Response{Volume: &volume.Volume{Name: request.Name, Mountpoint: mPoint}}
}

func (driver quobyteDriver) List(request volume.Request) volume.Response {
	driver.m.Lock()
	defer driver.m.Unlock()

	var vols []*volume.Volume
	files, err := ioutil.ReadDir(driver.quobyteMount)
	if err != nil {
		log.Println(err)
		return volume.Response{Err: err.Error()}
	}

	for _, entry := range files {
		if entry.IsDir() {
			vols = append(vols, &volume.Volume{Name: entry.Name(), Mountpoint: filepath.Join(driver.quobyteMount, entry.Name())})
		}
	}

	return volume.Response{Volumes: vols}
}

func (driver quobyteDriver) Capabilities(request volume.Request) volume.Response {
	return volume.Response{Capabilities: volume.Capability{Scope: "global"}}
}
<commit_msg>Make create calls idempotent<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/docker/go-plugins-helpers/volume"
	quobyte_api "github.com/quobyte/api"
)

type quobyteDriver struct {
	client       *quobyte_api.QuobyteClient
	quobyteMount string
	m            *sync.Mutex
}

func newQuobyteDriver(apiURL string, username string, password string, quobyteMount string) quobyteDriver {
	driver := quobyteDriver{
		client:       quobyte_api.NewQuobyteClient(apiURL, username, password),
		quobyteMount: quobyteMount,
		m:            &sync.Mutex{},
	}

	return driver
}

func (driver quobyteDriver) Create(request volume.Request) volume.Response {
	log.Printf("Creating volume %s\n", request.Name)
	driver.m.Lock()
	defer driver.m.Unlock()

	user, group := "root", "root"

	if usr, ok := request.Options["user"]; ok {
		user = usr
	}

	if grp, ok := request.Options["group"]; ok {
		group = grp
	}

	if _, err := driver.client.CreateVolume(&quobyte_api.CreateVolumeRequest{
		Name:        request.Name,
		RootUserID:  user,
		RootGroupID: group,
	}); err != nil {
		log.Println(err)

		if !strings.Contains(err.Error(), "ENTITY_EXISTS_ALREADY/POSIX_ERROR_NONE") {
			return volume.Response{Err: err.Error()}
		}
	}

	return volume.Response{Err: ""}
}

func (driver quobyteDriver) Remove(request volume.Request) volume.Response {
	log.Printf("Removing volume %s\n", request.Name)
	driver.m.Lock()
	defer driver.m.Unlock()

	if err := driver.client.DeleteVolumeByName(request.Name, ""); err != nil {
		log.Println(err)
		return volume.Response{Err: err.Error()}
	}

	return volume.Response{Err: ""}
}

func (driver quobyteDriver) Mount(request volume.MountRequest) volume.Response {
	driver.m.Lock()
	defer driver.m.Unlock()
	mPoint := filepath.Join(driver.quobyteMount, request.Name)
	log.Printf("Mounting volume %s on %s\n", request.Name, mPoint)
	if fi, err := os.Lstat(mPoint); err != nil || !fi.IsDir() {
		log.Println(err)
		return volume.Response{Err: fmt.Sprintf("%v not mounted", mPoint)}
	}

	return volume.Response{Err: "", Mountpoint: mPoint}
}

func (driver quobyteDriver) Path(request volume.Request) volume.Response {
	return volume.Response{Mountpoint: filepath.Join(driver.quobyteMount, request.Name)}
}

func (driver quobyteDriver) Unmount(request volume.UnmountRequest) volume.Response {
	return volume.Response{}
}

func (driver quobyteDriver) Get(request volume.Request) volume.Response {
	driver.m.Lock()
	defer driver.m.Unlock()

	mPoint := filepath.Join(driver.quobyteMount, request.Name)

	if fi, err := os.Lstat(mPoint); err != nil || !fi.IsDir() {
		log.Println(err)
		return volume.Response{Err: fmt.Sprintf("%v not mounted", mPoint)}
	}

	return volume.Response{Volume: &volume.Volume{Name: request.Name, Mountpoint: mPoint}}
}

func (driver quobyteDriver) List(request volume.Request) volume.Response {
	driver.m.Lock()
	defer driver.m.Unlock()

	var vols []*volume.Volume
	files, err := ioutil.ReadDir(driver.quobyteMount)
	if err != nil {
		log.Println(err)
		return volume.Response{Err: err.Error()}
	}

	for _, entry := range files {
		if entry.IsDir() {
			vols = append(vols, &volume.Volume{Name: entry.Name(), Mountpoint: filepath.Join(driver.quobyteMount, entry.Name())})
		}
	}

	return volume.Response{Volumes: vols}
}

func (driver quobyteDriver) Capabilities(request volume.Request) volume.Response {
	return volume.Response{Capabilities: volume.Capability{Scope: "global"}}
}
<|endoftext|>


<commit_before>package api

import (
	"testing"
	"github.com/stretchr/testify/assert"
)

func TestRedisResource(t *testing.T) {
	t.Run("Replicas should be 1 when not prod", func(t *testing.T) {
		deploymentRequest := NaisDeploymentRequest{
			Application: "redisTest",
			FasitEnvironment: "t",
			Namespace: "default",
		}
		redisFailoverDef := createRedisFailoverDef(deploymentRequest, "teamBeam")
		assert.Equal(t, int32(1), redisFailoverDef.Spec.Redis.Replicas, "")
	})

	t.Run("Replicas should be 3 when prod", func(t *testing.T) {
		deploymentRequest := NaisDeploymentRequest{
			Application: "redisTest",
			FasitEnvironment: ENVIRONMENT_P,
			Namespace: "default",
		}
		redisFailoverDef := createRedisFailoverDef(deploymentRequest, "teamBeam")
		assert.Equal(t, int32(3), redisFailoverDef.Spec.Redis.Replicas, "")
	})
}<commit_msg>Simple formatting of file<commit_after>package api

import (
	"testing"
	"github.com/stretchr/testify/assert"
)

func TestRedisResource(t *testing.T) {
	t.Run("Replicas should be 1 when not prod", func(t *testing.T) {
		deploymentRequest := NaisDeploymentRequest{
			Application:      "redisTest",
			FasitEnvironment: "t",
			Namespace:        "default",
		}
		redisFailoverDef := createRedisFailoverDef(deploymentRequest, "teamBeam")
		assert.Equal(t, int32(1), redisFailoverDef.Spec.Redis.Replicas, "")
	})

	t.Run("Replicas should be 3 when prod", func(t *testing.T) {
		deploymentRequest := NaisDeploymentRequest{
			Application:      "redisTest",
			FasitEnvironment: ENVIRONMENT_P,
			Namespace:        "default",
		}
		redisFailoverDef := createRedisFailoverDef(deploymentRequest, "teamBeam")
		assert.Equal(t, int32(3), redisFailoverDef.Spec.Redis.Replicas, "")
	})
}
<|endoftext|>


<commit_before>package cuckoo

import (
	"encoding/binary"
	"encoding/gob"
	"fmt"
	"hash"
	"io"
	"math/rand"
	"sync"

	"github.com/spaolacci/murmur3"
)

const (
	defaultBucketSize   = 8
	maxBucketSize       = 16
	defaultTotalBuckets = 4 << 20
	defaultMaxKicks     = 500
	seed                = 59053
)

// fingerprint of the item
type fingerprint uint16

// emptyFingerprint
var emptyFingerprint fingerprint

// bucket with n fingerprints
type bucket struct {
	Track uint16
	FPs   []fingerprint
}

// Filter is the cuckoo-filter
type Filter struct {
	count        uint32
	buckets      []bucket
	bucketSize   uint8
	totalBuckets uint32
	hash         hash.Hash32
	maxKicks     uint16

	// protects above fields
	L sync.RWMutex
}

// gobFilter for encoding and decoding the Filter
type gobFilter struct {
	Count        uint32
	Buckets      []bucket
	BucketSize   uint8
	TotalBuckets uint32
	MaxKicks     uint16
}

// initBuckets initialises the buckets
func initBuckets(totalBuckets uint32, bucketSize uint8) []bucket {
	buckets := make([]bucket, totalBuckets, totalBuckets)
	for i := range buckets {
		buckets[i] = bucket{FPs: make([]fingerprint, bucketSize, bucketSize)}
	}

	return buckets
}

// StdFilter returns Standard Cuckoo-Filter
func StdFilter() *Filter {
	return newFilter(defaultTotalBuckets, defaultBucketSize, murmur3.New32WithSeed(seed))
}

func newFilter(tb uint32, bs uint8, hash hash.Hash32) *Filter {
	return &Filter{
		buckets:      initBuckets(tb, bs),
		bucketSize:   bs,
		totalBuckets: tb,
		hash:         hash,
		maxKicks:     defaultMaxKicks,
	}
}

func NewFilter(count uint32) *Filter {
	b := nextPowerOf2(count) / defaultBucketSize
	return newFilter(b, defaultBucketSize, murmur3.New32WithSeed(seed))
}

func NewFilterWithBucketSize(count uint32, bs uint8) (*Filter, error) {
	if bs > maxBucketSize {
		return nil, fmt.Errorf("doesn't support %d bucket size. Max bucket size is %d", bs, maxBucketSize)
	}

	b := nextPowerOf2(count) / uint32(bs)
	return newFilter(b, bs, murmur3.New32WithSeed(seed)), nil
}

// nextPowerOf2 returns the next power 2 >= v
func nextPowerOf2(v uint32) (n uint32) {
	var i uint32
	for i = 2; i < 32; i++ {
		n = 1 << i
		if n >= v {
			break
		}
	}

	return n
}

// isSet returns true if the i th bit in the Track is 1
func isSet(track uint16, i uint8) bool {
	return track|(1<<uint8(i)) == track
}

// unSet un sets the i th bit in Track
func unSet(track uint16, i uint8) uint16 {
	return track ^ 1<<i
}

func set(track uint16, i uint8) uint16 {
	return track | 1<<i
}

// deleteFrom deletes fingerprint from bucket if exists
func deleteFrom(b *bucket, bs uint8, fp fingerprint) bool {
	for i := uint8(0); i < bs; i++ {
		if !isSet(b.Track, i) || b.FPs[i] != fp {
			continue
		}

		b.FPs[i] = emptyFingerprint
		b.Track = unSet(b.Track, i)
		return true
	}

	return false
}

// containsIn returns if the given fingerprint exists in bucket
func containsIn(b bucket, bs uint8, fp fingerprint) bool {
	for i := uint8(0); i < bs; i++ {
		if isSet(b.Track, i) && b.FPs[i] == fp {
			return true
		}
	}

	return false
}

// addToBucket will add fp to the bucket i in filter
func addToBucket(b *bucket, bs uint8, fp fingerprint) bool {
	for i := uint8(0); i < bs; i++ {
		if isSet(b.Track, i) {
			continue
		}

		b.FPs[i] = fp
		b.Track = set(b.Track, i)
		return true
	}

	return false
}

// hashOf returns the 32-bit hash
func hashOf(x []byte, hash hash.Hash32) (uint32, []byte) {
	hash.Reset()
	hash.Write(x)
	h := hash.Sum32()
	return h, hash.Sum(nil)
}

// fingerprintOf returns the fingerprint of x with size using hash
func fingerprintOf(xb []byte) (fp fingerprint) {
	return fingerprint(binary.BigEndian.Uint16(xb))
}

func fingerprintHash(fp fingerprint, hash hash.Hash32) (fph uint32) {
	b := make([]byte, 2, 2)
	binary.BigEndian.PutUint16(b, uint16(fp))
	fph, _ = hashOf(b, hash)
	return fph
}

// indicesOf returns the indices of item x using given hash
func indicesOf(xh, fph, totalBuckets uint32) (i1, i2 uint32) {
	i1 = xh % totalBuckets
	i2 = alternateIndex(totalBuckets, i1, fph)
	return i1, i2
}

// alternateIndex returns the alternate index of i
func alternateIndex(totalBuckets, i, fph uint32) (j uint32) {
	return (i ^ fph) % totalBuckets
}

// estimatedLoadFactor returns an estimated max load factor based on bucket size
func estimatedLoadFactor(bucketSize uint8) float64 {
	switch {
	case bucketSize < 8:
		return 0.955
	case bucketSize < 16:
		return 0.985
	default:
		return 0.994
	}
}

// isReliable returns if the filter is reliable for another insert
func isReliable(f *Filter) bool {
	clf := f.ULoadFactor()
	elf := estimatedLoadFactor(f.bucketSize)
	if clf < elf {
		return true
	}

	return false
}

// swapFingerprint swaps a random fp from the bucket with provided fp
func swapFingerprint(b *bucket, fp fingerprint) fingerprint {
	var sfp fingerprint
	k := rand.Intn(len(b.FPs))
	sfp, b.FPs[k] = b.FPs[k], fp
	return sfp
}

// insert inserts the item into filter
func insert(f *Filter, x []byte) (ok bool) {
	xh, xb := hashOf(x, f.hash)
	fp := fingerprintOf(xb)
	fph := fingerprintHash(fp, f.hash)
	i1, i2 := indicesOf(xh, fph, f.totalBuckets)

	defer func() {
		if ok {
			f.count++
		}
	}()

	if addToBucket(&f.buckets[i1], f.bucketSize, fp) || addToBucket(&f.buckets[i2], f.bucketSize, fp) {
		return true
	}

	ri := []uint32{i1, i2}[rand.Intn(2)]
	var k uint16
	for k = 0; k < f.maxKicks; k++ {
		fp = swapFingerprint(&f.buckets[ri], fp)
		fph = fingerprintHash(fp, f.hash)
		ri = alternateIndex(f.totalBuckets, ri, fph)
		if addToBucket(&f.buckets[ri], f.bucketSize, fp) {
			return true
		}
	}

	return false
}

// lookup checks if the item x existence in filter
func lookup(f *Filter, x []byte) bool {
	xh, xb := hashOf(x, f.hash)
	fp := fingerprintOf(xb)
	fph := fingerprintHash(fp, f.hash)
	i1, i2 := indicesOf(xh, fph, f.totalBuckets)

	if containsIn(f.buckets[i1], f.bucketSize, fp) || containsIn(f.buckets[i2], f.bucketSize, fp) {
		return true
	}

	return false
}

// deleteItem deletes item if present from the filter
func deleteItem(f *Filter, x []byte) (ok bool) {
	xh, xb := hashOf(x, f.hash)
	fp := fingerprintOf(xb)
	fph := fingerprintHash(fp, f.hash)
	i1, i2 := indicesOf(xh, fph, f.totalBuckets)

	defer func() {
		if ok {
			f.count--
		}
	}()

	if deleteFrom(&f.buckets[i1], f.bucketSize, fp) || deleteFrom(&f.buckets[i2], f.bucketSize, fp) {
		return true
	}

	return false
}

// sanitize the bytes
func sanitize(x []byte) ([]byte, bool) {
	if len(x) == 0 {
		return nil, false
	}

	if len(x) == 1 {
		x = []byte{0, x[0]}
	}

	return x, true
}

// Insert inserts the item to the filter
func (f *Filter) Insert(x []byte) bool {
	f.L.Lock()
	defer f.L.Unlock()

	return f.UInsert(x)
}

// UInsert inserts the item to the filter. Not thread safe
func (f *Filter) UInsert(x []byte) bool {
	if !isReliable(f) {
		return false
	}

	x, ok := sanitize(x)
	if !ok {
		return false
	}

	return insert(f, x)
}

// InsertUnique inserts only unique items
func (f *Filter) InsertUnique(x []byte) bool {
	f.L.Lock()
	defer f.L.Unlock()

	return f.UInsertUnique(x)
}

// UInsertUnique inserts only unique items. Not thread safe
func (f *Filter) UInsertUnique(x []byte) bool {
	if !isReliable(f) {
		return false
	}

	x, ok := sanitize(x)
	if !ok {
		return false
	}

	if f.ULookup(x) {
		return true
	}

	return insert(f, x)
}

// Lookup checks if item exists in filter
func (f *Filter) Lookup(x []byte) bool {
	f.L.RLock()
	defer f.L.RUnlock()

	return f.ULookup(x)
}

// ULookup checks if item exists in filter. Not thread safe
func (f *Filter) ULookup(x []byte) bool {
	x, ok := sanitize(x)
	if !ok {
		return false
	}

	return lookup(f, x)
}

// Delete deletes the item from the filter
func (f *Filter) Delete(x []byte) bool {
	f.L.Lock()
	defer f.L.Unlock()

	return f.UDelete(x)
}

// UDelete deletes the item from the filter. Not thread safe
func (f *Filter) UDelete(x []byte) bool {
	x, ok := sanitize(x)
	if !ok {
		return false
	}

	return deleteItem(f, x)
}

// Count returns total inserted items into filter
func (f *Filter) Count() uint32 {
	f.L.RLock()
	defer f.L.RUnlock()

	return f.UCount()
}

// UCount returns total inserted items into filter. Not thread safe
func (f *Filter) UCount() uint32 {
	return f.count
}

// LoadFactor returns the load factor of the filter
func (f *Filter) LoadFactor() float64 {
	f.L.RLock()
	defer f.L.RUnlock()
	return f.ULoadFactor()
}

// ULoadFactor returns the load factor of the filter. Not thread safe
func (f *Filter) ULoadFactor() float64 {
	return float64(f.count) / (float64(uint32(f.bucketSize) * f.totalBuckets))
}

// Encode gob encodes the filter to passed writer
func (f *Filter) Encode(w io.Writer) error {
	f.L.RLock()
	gf := &gobFilter{
		Count:        f.count,
		Buckets:      f.buckets,
		BucketSize:   f.bucketSize,
		TotalBuckets: f.totalBuckets,
		MaxKicks:     f.maxKicks,
	}
	f.L.RUnlock()

	ge := gob.NewEncoder(w)
	return ge.Encode(gf)
}

// Decode decodes and returns the filter instance
func Decode(r io.Reader) (*Filter, error) {
	gd := gob.NewDecoder(r)
	gf := &gobFilter{}
	err := gd.Decode(gf)
	if err != nil {
		return nil, fmt.Errorf("failed to decode filter: %v", err)
	}

	f := &Filter{
		count:        gf.Count,
		buckets:      gf.Buckets,
		bucketSize:   gf.BucketSize,
		totalBuckets: gf.TotalBuckets,
		hash:         murmur3.New32WithSeed(seed),
		maxKicks:     gf.MaxKicks,
	}

	return f, nil
}
<commit_msg>remove unneeded conversion<commit_after>package cuckoo

import (
	"encoding/binary"
	"encoding/gob"
	"fmt"
	"hash"
	"io"
	"math/rand"
	"sync"

	"github.com/spaolacci/murmur3"
)

const (
	defaultBucketSize   = 8
	maxBucketSize       = 16
	defaultTotalBuckets = 4 << 20
	defaultMaxKicks     = 500
	seed                = 59053
)

// fingerprint of the item
type fingerprint uint16

// emptyFingerprint
var emptyFingerprint fingerprint

// bucket with n fingerprints
type bucket struct {
	Track uint16
	FPs   []fingerprint
}

// Filter is the cuckoo-filter
type Filter struct {
	count        uint32
	buckets      []bucket
	bucketSize   uint8
	totalBuckets uint32
	hash         hash.Hash32
	maxKicks     uint16

	// protects above fields
	L sync.RWMutex
}

// gobFilter for encoding and decoding the Filter
type gobFilter struct {
	Count        uint32
	Buckets      []bucket
	BucketSize   uint8
	TotalBuckets uint32
	MaxKicks     uint16
}

// initBuckets initialises the buckets
func initBuckets(totalBuckets uint32, bucketSize uint8) []bucket {
	buckets := make([]bucket, totalBuckets, totalBuckets)
	for i := range buckets {
		buckets[i] = bucket{FPs: make([]fingerprint, bucketSize, bucketSize)}
	}

	return buckets
}

// StdFilter returns Standard Cuckoo-Filter
func StdFilter() *Filter {
	return newFilter(defaultTotalBuckets, defaultBucketSize, murmur3.New32WithSeed(seed))
}

func newFilter(tb uint32, bs uint8, hash hash.Hash32) *Filter {
	return &Filter{
		buckets:      initBuckets(tb, bs),
		bucketSize:   bs,
		totalBuckets: tb,
		hash:         hash,
		maxKicks:     defaultMaxKicks,
	}
}

func NewFilter(count uint32) *Filter {
	b := nextPowerOf2(count) / defaultBucketSize
	return newFilter(b, defaultBucketSize, murmur3.New32WithSeed(seed))
}

func NewFilterWithBucketSize(count uint32, bs uint8) (*Filter, error) {
	if bs > maxBucketSize {
		return nil, fmt.Errorf("doesn't support %d bucket size. Max bucket size is %d", bs, maxBucketSize)
	}

	b := nextPowerOf2(count) / uint32(bs)
	return newFilter(b, bs, murmur3.New32WithSeed(seed)), nil
}

// nextPowerOf2 returns the next power 2 >= v
func nextPowerOf2(v uint32) (n uint32) {
	var i uint32
	for i = 2; i < 32; i++ {
		n = 1 << i
		if n >= v {
			break
		}
	}

	return n
}

// isSet returns true if the i th bit in the Track is 1
func isSet(track uint16, i uint8) bool {
	return track|(1<<i) == track
}

// unSet un sets the i th bit in Track
func unSet(track uint16, i uint8) uint16 {
	return track ^ 1<<i
}

func set(track uint16, i uint8) uint16 {
	return track | 1<<i
}

// deleteFrom deletes fingerprint from bucket if exists
func deleteFrom(b *bucket, bs uint8, fp fingerprint) bool {
	for i := uint8(0); i < bs; i++ {
		if !isSet(b.Track, i) || b.FPs[i] != fp {
			continue
		}

		b.FPs[i] = emptyFingerprint
		b.Track = unSet(b.Track, i)
		return true
	}

	return false
}

// containsIn returns if the given fingerprint exists in bucket
func containsIn(b bucket, bs uint8, fp fingerprint) bool {
	for i := uint8(0); i < bs; i++ {
		if isSet(b.Track, i) && b.FPs[i] == fp {
			return true
		}
	}

	return false
}

// addToBucket will add fp to the bucket i in filter
func addToBucket(b *bucket, bs uint8, fp fingerprint) bool {
	for i := uint8(0); i < bs; i++ {
		if isSet(b.Track, i) {
			continue
		}

		b.FPs[i] = fp
		b.Track = set(b.Track, i)
		return true
	}

	return false
}

// hashOf returns the 32-bit hash
func hashOf(x []byte, hash hash.Hash32) (uint32, []byte) {
	hash.Reset()
	hash.Write(x)
	h := hash.Sum32()
	return h, hash.Sum(nil)
}

// fingerprintOf returns the fingerprint of x with size using hash
func fingerprintOf(xb []byte) (fp fingerprint) {
	return fingerprint(binary.BigEndian.Uint16(xb))
}

func fingerprintHash(fp fingerprint, hash hash.Hash32) (fph uint32) {
	b := make([]byte, 2, 2)
	binary.BigEndian.PutUint16(b, uint16(fp))
	fph, _ = hashOf(b, hash)
	return fph
}

// indicesOf returns the indices of item x using given hash
func indicesOf(xh, fph, totalBuckets uint32) (i1, i2 uint32) {
	i1 = xh % totalBuckets
	i2 = alternateIndex(totalBuckets, i1, fph)
	return i1, i2
}

// alternateIndex returns the alternate index of i
func alternateIndex(totalBuckets, i, fph uint32) (j uint32) {
	return (i ^ fph) % totalBuckets
}

// estimatedLoadFactor returns an estimated max load factor based on bucket size
func estimatedLoadFactor(bucketSize uint8) float64 {
	switch {
	case bucketSize < 8:
		return 0.955
	case bucketSize < 16:
		return 0.985
	default:
		return 0.994
	}
}

// isReliable returns if the filter is reliable for another insert
func isReliable(f *Filter) bool {
	clf := f.ULoadFactor()
	elf := estimatedLoadFactor(f.bucketSize)
	if clf < elf {
		return true
	}

	return false
}

// swapFingerprint swaps a random fp from the bucket with provided fp
func swapFingerprint(b *bucket, fp fingerprint) fingerprint {
	var sfp fingerprint
	k := rand.Intn(len(b.FPs))
	sfp, b.FPs[k] = b.FPs[k], fp
	return sfp
}

// insert inserts the item into filter
func insert(f *Filter, x []byte) (ok bool) {
	xh, xb := hashOf(x, f.hash)
	fp := fingerprintOf(xb)
	fph := fingerprintHash(fp, f.hash)
	i1, i2 := indicesOf(xh, fph, f.totalBuckets)

	defer func() {
		if ok {
			f.count++
		}
	}()

	if addToBucket(&f.buckets[i1], f.bucketSize, fp) || addToBucket(&f.buckets[i2], f.bucketSize, fp) {
		return true
	}

	ri := []uint32{i1, i2}[rand.Intn(2)]
	var k uint16
	for k = 0; k < f.maxKicks; k++ {
		fp = swapFingerprint(&f.buckets[ri], fp)
		fph = fingerprintHash(fp, f.hash)
		ri = alternateIndex(f.totalBuckets, ri, fph)
		if addToBucket(&f.buckets[ri], f.bucketSize, fp) {
			return true
		}
	}

	return false
}

// lookup checks if the item x existence in filter
func lookup(f *Filter, x []byte) bool {
	xh, xb := hashOf(x, f.hash)
	fp := fingerprintOf(xb)
	fph := fingerprintHash(fp, f.hash)
	i1, i2 := indicesOf(xh, fph, f.totalBuckets)

	if containsIn(f.buckets[i1], f.bucketSize, fp) || containsIn(f.buckets[i2], f.bucketSize, fp) {
		return true
	}

	return false
}

// deleteItem deletes item if present from the filter
func deleteItem(f *Filter, x []byte) (ok bool) {
	xh, xb := hashOf(x, f.hash)
	fp := fingerprintOf(xb)
	fph := fingerprintHash(fp, f.hash)
	i1, i2 := indicesOf(xh, fph, f.totalBuckets)

	defer func() {
		if ok {
			f.count--
		}
	}()

	if deleteFrom(&f.buckets[i1], f.bucketSize, fp) || deleteFrom(&f.buckets[i2], f.bucketSize, fp) {
		return true
	}

	return false
}

// sanitize the bytes
func sanitize(x []byte) ([]byte, bool) {
	if len(x) == 0 {
		return nil, false
	}

	if len(x) == 1 {
		x = []byte{0, x[0]}
	}

	return x, true
}

// Insert inserts the item to the filter
func (f *Filter) Insert(x []byte) bool {
	f.L.Lock()
	defer f.L.Unlock()

	return f.UInsert(x)
}

// UInsert inserts the item to the filter. Not thread safe
func (f *Filter) UInsert(x []byte) bool {
	if !isReliable(f) {
		return false
	}

	x, ok := sanitize(x)
	if !ok {
		return false
	}

	return insert(f, x)
}

// InsertUnique inserts only unique items
func (f *Filter) InsertUnique(x []byte) bool {
	f.L.Lock()
	defer f.L.Unlock()

	return f.UInsertUnique(x)
}

// UInsertUnique inserts only unique items. Not thread safe
func (f *Filter) UInsertUnique(x []byte) bool {
	if !isReliable(f) {
		return false
	}

	x, ok := sanitize(x)
	if !ok {
		return false
	}

	if f.ULookup(x) {
		return true
	}

	return insert(f, x)
}

// Lookup checks if item exists in filter
func (f *Filter) Lookup(x []byte) bool {
	f.L.RLock()
	defer f.L.RUnlock()

	return f.ULookup(x)
}

// ULookup checks if item exists in filter. Not thread safe
func (f *Filter) ULookup(x []byte) bool {
	x, ok := sanitize(x)
	if !ok {
		return false
	}

	return lookup(f, x)
}

// Delete deletes the item from the filter
func (f *Filter) Delete(x []byte) bool {
	f.L.Lock()
	defer f.L.Unlock()

	return f.UDelete(x)
}

// UDelete deletes the item from the filter. Not thread safe
func (f *Filter) UDelete(x []byte) bool {
	x, ok := sanitize(x)
	if !ok {
		return false
	}

	return deleteItem(f, x)
}

// Count returns total inserted items into filter
func (f *Filter) Count() uint32 {
	f.L.RLock()
	defer f.L.RUnlock()

	return f.UCount()
}

// UCount returns total inserted items into filter. Not thread safe
func (f *Filter) UCount() uint32 {
	return f.count
}

// LoadFactor returns the load factor of the filter
func (f *Filter) LoadFactor() float64 {
	f.L.RLock()
	defer f.L.RUnlock()
	return f.ULoadFactor()
}

// ULoadFactor returns the load factor of the filter. Not thread safe
func (f *Filter) ULoadFactor() float64 {
	return float64(f.count) / (float64(uint32(f.bucketSize) * f.totalBuckets))
}

// Encode gob encodes the filter to passed writer
func (f *Filter) Encode(w io.Writer) error {
	f.L.RLock()
	gf := &gobFilter{
		Count:        f.count,
		Buckets:      f.buckets,
		BucketSize:   f.bucketSize,
		TotalBuckets: f.totalBuckets,
		MaxKicks:     f.maxKicks,
	}
	f.L.RUnlock()

	ge := gob.NewEncoder(w)
	return ge.Encode(gf)
}

// Decode decodes and returns the filter instance
func Decode(r io.Reader) (*Filter, error) {
	gd := gob.NewDecoder(r)
	gf := &gobFilter{}
	err := gd.Decode(gf)
	if err != nil {
		return nil, fmt.Errorf("failed to decode filter: %v", err)
	}

	f := &Filter{
		count:        gf.Count,
		buckets:      gf.Buckets,
		bucketSize:   gf.BucketSize,
		totalBuckets: gf.TotalBuckets,
		hash:         murmur3.New32WithSeed(seed),
		maxKicks:     gf.MaxKicks,
	}

	return f, nil
}
<|endoftext|>