<commit_before>/********************************
*** Web server API for Go     ***
*** Code is under MIT license ***
*** Code by CodingFerret      ***
*** github.com/squiidz        ***
*********************************/

package furserv

import (
	"fmt"
	"log"
	"net/http"
)

// Simple Server structure for a web server.
type Server struct {
	Host string
	Port string
	Log  bool
	Mux  *http.ServeMux
}

type MiddleWare func(http.Handler) http.Handler

// Create a NewServer instance with the given value.
// Host: "localhost"
// Port: ":8080"
// Log: true/false
// Options: functions to run on the server instance who's gonna be return.
func NewServer(host string, port string, log bool, options ...func(s *Server)) *Server {
	svr := Server{host, port, log, http.NewServeMux()}
	for _, option := range options {
		option(&svr)
	}
	return &svr
}

// Start Listening on host and port of the Server.
// Log the request if the log was initiated as true in NewServer.
func (s *Server) Start() {
	fmt.Printf("[+] Server Running on %s ... \n", s.Port)
	if s.Log {
		http.ListenAndServe(s.Host+s.Port, s.logger(s.Mux))
	}
	http.ListenAndServe(s.Host+s.Port, s.Mux)
}

// Add function with the right sigature to the Server Mux
// and chain the provided middlewares on it.
func (s *Server) AddRoute(pat string, f func(rw http.ResponseWriter, req *http.Request), middles ...MiddleWare) {
	if middles != nil {
		var stack http.Handler
		for i := len(middles) - 1; i >= 0; i-- {
			if i == len(middles)-1 {
				stack = middles[i](http.HandlerFunc(f))
			} else {
				stack = middles[i](stack)
			}
		}
		s.Mux.Handle(pat, stack)
	} else {
		s.Mux.Handle(pat, http.HandlerFunc(f))
	}

}

// Log request to the Server.
func (s *Server) logger(mux http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		log.Printf("%s %s %s", req.RemoteAddr, req.Method, req.URL)
		mux.ServeHTTP(rw, req)
	})
}
<commit_msg>Delete furserv.go<commit_after><|endoftext|>


<commit_before>package types

import (
    "errors"
    "fmt"
    "strconv"
    "strings"

    "github.com/elliotchance/c2go/program"
    "github.com/elliotchance/c2go/util"
)

func removePrefix(s, prefix string) string {
    if strings.HasPrefix(s, prefix) {
        s = s[len(prefix):]
    }

    return s
}

// SizeOf returns the number of bytes for a type. This the same as using the
// sizeof operator/function in C.
func SizeOf(p *program.Program, cType string) (int, error) {
    // Remove keywords that do not effect the size.
    cType = removePrefix(cType, "signed ")
    cType = removePrefix(cType, "unsigned ")
    cType = removePrefix(cType, "const ")
    cType = removePrefix(cType, "volatile ")

    // FIXME: The pointer size will be different on different platforms. We
    // should find out the correct size at runtime.
    pointerSize := 8

    // A structure will be the sum of its parts.
    if strings.HasPrefix(cType, "struct ") {
        totalBytes := 0

        s := p.Structs[cType[7:]]
        if s == nil {
            return 0, errors.New(fmt.Sprintf("could not sizeof: %s", cType))
        }

        for _, t := range s.Fields {
            var bytes int
            var err error

            switch f := t.(type) {
            case string:
                bytes, err = SizeOf(p, f)

            case *program.Struct:
                bytes, err = SizeOf(p, f.Name)
            }

            if err != nil {
                return 0, err
            }
            totalBytes += bytes
        }

        // The size of a struct is rounded up to fit the size of the pointer of
        // the OS.
        if totalBytes%pointerSize != 0 {
            totalBytes += pointerSize - (totalBytes % pointerSize)
        }

        return totalBytes, nil
    }

    // An union will be the max size of its parts.
    if strings.HasPrefix(cType, "union ") {
        byte_count := 0

        s := p.Unions[cType[6:]]
        if s == nil {
            return 0, errors.New(fmt.Sprintf("could not sizeof: %s", cType))
        }

        for _, t := range s.Fields {
            var bytes int
            var err error

            switch f := t.(type) {
            case string:
                bytes, err = SizeOf(p, f)

            case *program.Struct:
                bytes, err = SizeOf(p, f.Name)
            }

            if err != nil {
                return 0, err
            }

            if byte_count < bytes {
                byte_count = bytes
            }
        }

        // The size of an union is rounded up to fit the size of the pointer of
        // the OS.
        if byte_count%pointerSize != 0 {
            byte_count += pointerSize - (byte_count % pointerSize)
        }

        return byte_count, nil
    }

    // Function pointers are one byte?
    if strings.Index(cType, "(") >= 0 {
        return 1, nil
    }

    if strings.HasSuffix(cType, "*") {
        return pointerSize, nil
    }

    switch cType {
    case "char", "void":
        return 1, nil

    case "short":
        return 2, nil

    case "int", "float":
        return 4, nil

    case "long", "double":
        return 8, nil

    case "long double":
        return 16, nil
    }

    // Get size for array types like: `base_type [count]`
    groups := util.GroupsFromRegex(`^(?P<type>.+) ?[(?P<count>\d+)\]$`, cType)
    fmt.Println("Gr:", groups)

    if groups == nil {
        return pointerSize, errors.New(
            fmt.Sprintf("cannot determine size of: %s", cType))
    }

    base_size, err := SizeOf(p, groups["type"])
    if err != nil {
        return 0, err
    }

    count, err := strconv.Atoi(groups["count"])
    if err != nil {
        return 0, err
    }

    return base_size * count, nil
}
<commit_msg>Fixup for types.Sizeof() function<commit_after>package types

import (
    "errors"
    "fmt"
    "strconv"
    "strings"

    "github.com/elliotchance/c2go/program"
    "github.com/elliotchance/c2go/util"
)

func removePrefix(s, prefix string) string {
    if strings.HasPrefix(s, prefix) {
        s = s[len(prefix):]
    }

    return s
}

// SizeOf returns the number of bytes for a type. This the same as using the
// sizeof operator/function in C.
func SizeOf(p *program.Program, cType string) (int, error) {
    // Remove keywords that do not effect the size.
    cType = removePrefix(cType, "signed ")
    cType = removePrefix(cType, "unsigned ")
    cType = removePrefix(cType, "const ")
    cType = removePrefix(cType, "volatile ")

    // FIXME: The pointer size will be different on different platforms. We
    // should find out the correct size at runtime.
    pointerSize := 8

    // A structure will be the sum of its parts.
    if strings.HasPrefix(cType, "struct ") {
        totalBytes := 0

        s := p.Structs[cType[7:]]
        if s == nil {
            return 0, errors.New(fmt.Sprintf("could not sizeof: %s", cType))
        }

        for _, t := range s.Fields {
            var bytes int
            var err error

            switch f := t.(type) {
            case string:
                bytes, err = SizeOf(p, f)

            case *program.Struct:
                bytes, err = SizeOf(p, f.Name)
            }

            if err != nil {
                return 0, err
            }
            totalBytes += bytes
        }

        // The size of a struct is rounded up to fit the size of the pointer of
        // the OS.
        if totalBytes%pointerSize != 0 {
            totalBytes += pointerSize - (totalBytes % pointerSize)
        }

        return totalBytes, nil
    }

    // An union will be the max size of its parts.
    if strings.HasPrefix(cType, "union ") {
        byte_count := 0

        s := p.Unions[cType[6:]]
        if s == nil {
            return 0, errors.New(fmt.Sprintf("could not sizeof: %s", cType))
        }

        for _, t := range s.Fields {
            var bytes int
            var err error

            switch f := t.(type) {
            case string:
                bytes, err = SizeOf(p, f)

            case *program.Struct:
                bytes, err = SizeOf(p, f.Name)
            }

            if err != nil {
                return 0, err
            }

            if byte_count < bytes {
                byte_count = bytes
            }
        }

        // The size of an union is rounded up to fit the size of the pointer of
        // the OS.
        if byte_count%pointerSize != 0 {
            byte_count += pointerSize - (byte_count % pointerSize)
        }

        return byte_count, nil
    }

    // Function pointers are one byte?
    if strings.Index(cType, "(") >= 0 {
        return 1, nil
    }

    if strings.HasSuffix(cType, "*") {
        return pointerSize, nil
    }

    switch cType {
    case "char", "void":
        return 1, nil

    case "short":
        return 2, nil

    case "int", "float":
        return 4, nil

    case "long", "double":
        return 8, nil

    case "long double":
        return 16, nil
    }

    // Get size for array types like: `base_type [count]`
    groups := util.GroupsFromRegex(`^(?P<type>[^ ]+) *\[(?P<count>\d+)\]$`, cType)
    if groups == nil {
        return pointerSize, errors.New(
            fmt.Sprintf("cannot determine size of: `%s`", cType))
    }

    base_size, err := SizeOf(p, groups["type"])
    if err != nil {
        return 0, err
    }

    count, err := strconv.Atoi(groups["count"])
    if err != nil {
        return 0, err
    }

    return base_size * count, nil
}
<|endoftext|>


<commit_before>package discgo

import (
	"encoding/json"
	"time"

	"sync"

	"runtime"

	"github.com/gorilla/websocket"
	"github.com/nhooyr/log"
)

type Game struct {
	Name string  `json:"name"`
	Type *int    `json:"type"`
	URL  *string `json:"url"`
}

type endpointGateway struct {
	*endpoint
}

func (c *Client) gateway() endpointGateway {
	e2 := c.e.appendMajor("gateway")
	return endpointGateway{e2}
}

func (g endpointGateway) get() (url string, err error) {
	var urlStruct struct {
		URL string `json:"url"`
	}
	return urlStruct.URL, g.doMethod("GET", nil, &urlStruct)
}

type Conn struct {
	token      string
	userAgent  string
	gatewayURL string

	sessionID string

	closeOnce           sync.Once
	closeChan           chan struct{}
	closeConfirmChannel chan struct{}

	wsConn *websocket.Conn

	mu                    sync.Mutex
	heartbeatAcknowledged bool
	sequenceNumber        int
}

func NewConn(apiClient *Client) (*Conn, error) {
	gatewayURL, err := apiClient.gateway().get()
	if err != nil {
		return nil, err
	}
	gatewayURL += "?v=" + apiVersion + "&encoding=json"
	return &Conn{
		token:      apiClient.Token,
		userAgent:  apiClient.UserAgent,
		gatewayURL: gatewayURL,
	}, nil
}

const (
	dispatchOperation            = iota
	heartbeatOperation
	identifyOperation
	statusUpdateOperation
	voiceStateUpdateOperation
	voiceServerPingOperation
	resumeOperation
	reconnectOperation
	requestGuildMembersOperation
	invalidSessionOperation
	helloOperation
	heartbeatACKOperation
)

func (c *Conn) close() error {
	close(c.closeChan)
	closeMsg := websocket.FormatCloseMessage(websocket.CloseNoStatusReceived, "no heartbeat acknowledgment")
	err := c.wsConn.WriteMessage(websocket.CloseMessage, closeMsg)
	err2 := c.wsConn.Close()
	if err != nil {
		return err
	}
	return err2
}

func (c *Conn) Close() error {
	err := c.close()
	// Wait for eventloop and heartbeat goroutines.
	<-c.closeConfirmChannel
	<-c.closeConfirmChannel
	return err
}

func (c *Conn) reconnect() {
	c.closeOnce.Do(func() {
		err := c.close()
		if err != nil {
			log.Print(err)
		}
		// Wait for eventloop or heartbeat goroutine.
		<-c.closeConfirmChannel
		err = c.Dial()
		if err != nil {
			log.Print(err)
		}
	})
}

type helloOPData struct {
	HeartbeatInterval int      `json:"heartbeat_interval"`
	Trace             []string `json:"_trace"`
}

func (c *Conn) Dial() (err error) {
	c.heartbeatAcknowledged = true

	c.closeOnce = sync.Once{}
	c.closeChan = make(chan struct{})
	c.closeConfirmChannel = make(chan struct{})

	c.wsConn, _, err = websocket.DefaultDialer.Dial(c.gatewayURL, nil)
	if err != nil {
		return err
	}

	if c.sessionID == "" {
		err = c.identify()
	} else {
		err = c.resume()
	}
	if err != nil {
		return err
	}

	go c.eventLoop()

	return nil
}

type resumeOPData struct {
	Token     string `json:"token"`
	SessionID string `json:"session_id"`
	Seq       int    `json:"seq"`
}

func (c *Conn) resume() error {
	c.mu.Lock()
	p := &sendPayload{
		Operation: resumeOperation,
		Data: resumeOPData{
			Token:     c.token,
			SessionID: c.sessionID,
			Seq:       c.sequenceNumber,
		},
	}
	c.mu.Unlock()
	return c.wsConn.WriteJSON(p)
}

type readyEvent struct {
	V               int        `json:"v"`
	User            *User      `json:"user"`
	PrivateChannels []*Channel `json:"private_channels"`
	SessionID       string     `json:"session_id"`
	Trace           []string   `json:"_trace"`
}

func (c *Conn) eventLoop() {
	for {
		p, err := c.nextPayload()
		if err != nil {
			select {
			case <-c.closeChan:
				c.closeConfirmChannel <- struct{}{}
			default:
				// TODO use sync.Once to prevent race condition?
				c.reconnect()
			}
			return
		}

		switch p.Operation {
		case helloOperation:
			var hello helloOPData
			err = json.Unmarshal(p.Data, &hello)
			if err != nil {
				err = c.close()
				if err != nil {
					log.Print(err)
				}
				return
			}
			go c.heartbeat(&hello)
		case heartbeatACKOperation:
			c.mu.Lock()
			c.heartbeatAcknowledged = true
			c.mu.Unlock()
		case invalidSessionOperation:
			// TODO only once do this or sleep too? not sure, confusing docs
			err := c.identify()
			if err != nil {
				log.Print(err)
				continue
			}
		case dispatchOperation:
			c.mu.Lock()
			c.sequenceNumber = p.SequenceNumber
			c.mu.Unlock()

			switch p.Type {
			case "READY":
				var ready readyEvent
				err = json.Unmarshal(p.Data, &ready)
				if err != nil {
					err = c.close()
					if err != nil {
						log.Print(err)
					}
					return
				}
				c.sessionID = ready.SessionID
			}
			// TODO state tracking
		}
		log.Print(p.Operation)
		log.Print(p.Type)
		log.Printf("%s", p.Data)
		log.Print(p.SequenceNumber)
		log.Print()
	}
}

type identifyOPData struct {
	Token          string             `json:"token"`
	Properties     identifyProperties `json:"properties"`
	Compress       bool               `json:"compress"`
	LargeThreshold int                `json:"large_threshold"`
}

type identifyProperties struct {
	OS              string `json:"$os,omitempty"`
	Browser         string `json:"$browser,omitempty"`
	Device          string `json:"$device,omitempty"`
	Referrer        string `json:"$referrer,omitempty"`
	ReferringDomain string `json:"$referring_domain,omitempty"`
}

func (c *Conn) identify() error {
	p := &sendPayload{
		Operation: identifyOperation,
		Data: identifyOPData{
			Token: c.token,
			Properties: identifyProperties{
				OS:      runtime.GOOS,
				Browser: c.userAgent,
			},
			// TODO COMPRESS!!!
			Compress:       false,
			LargeThreshold: 250,
		},
	}
	return c.wsConn.WriteJSON(p)
}

func (c *Conn) heartbeat(hello *helloOPData) {
	ticker := time.NewTicker(time.Duration(hello.HeartbeatInterval) * time.Millisecond)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
		case <-c.closeChan:
			c.closeConfirmChannel <- struct{}{}
			return
		}
		c.mu.Lock()
		if !c.heartbeatAcknowledged {
			c.mu.Unlock()
			c.reconnect()
			return
		}
		sequenceNumber := c.sequenceNumber
		c.heartbeatAcknowledged = false
		c.mu.Unlock()

		p := &sendPayload{Operation: heartbeatOperation, Data: sequenceNumber}
		err := c.wsConn.WriteJSON(p)
		if err != nil {
			log.Print(err)
			err = c.close()
			if err != nil {
				log.Print(err)
			}
			return
		}
	}
}

type sendPayload struct {
	Operation int         `json:"op"`
	Data      interface{} `json:"d,omitempty"`
	Sequence  int         `json:"s,omitempty"`
}

type receivePayload struct {
	Operation      int             `json:"op"`
	Data           json.RawMessage `json:"d"`
	SequenceNumber int             `json:"s"`
	Type           string          `json:"t"`
}

func (c *Conn) nextPayload() (*receivePayload, error) {
	var p receivePayload
	// TODO compression, see how discordgo does it
	err := c.wsConn.ReadJSON(&p)
	return &p, err
}
<commit_msg>robust resumption. Need to review though.<commit_after>package discgo

import (
	"encoding/json"
	"time"

	"sync"

	"runtime"

	"errors"
	"net"
	"os"

	"github.com/gorilla/websocket"
	"github.com/nhooyr/log"
)

type Game struct {
	Name string  `json:"name"`
	Type *int    `json:"type"`
	URL  *string `json:"url"`
}

type endpointGateway struct {
	*endpoint
}

func (c *Client) gateway() endpointGateway {
	e2 := c.e.appendMajor("gateway")
	return endpointGateway{e2}
}

func (g endpointGateway) get() (url string, err error) {
	var urlStruct struct {
		URL string `json:"url"`
	}
	return urlStruct.URL, g.doMethod("GET", nil, &urlStruct)
}

type Conn struct {
	token      string
	userAgent  string
	gatewayURL string

	sessionID string

	closeChan         chan struct{}
	confirmClosedChan chan struct{}
	reconnectChan     chan struct{}

	wsConn *websocket.Conn

	mu                    sync.Mutex
	heartbeatAcknowledged bool
	sequenceNumber        int
}

func NewConn(apiClient *Client) (*Conn, error) {
	gatewayURL, err := apiClient.gateway().get()
	if err != nil {
		return nil, err
	}
	gatewayURL += "?v=" + apiVersion + "&encoding=json"
	return &Conn{
		token:             apiClient.Token,
		userAgent:         apiClient.UserAgent,
		gatewayURL:        gatewayURL,
		confirmClosedChan: make(chan struct{}),
		reconnectChan:     make(chan struct{}),
	}, nil
}

const (
	dispatchOperation            = iota
	heartbeatOperation
	identifyOperation
	statusUpdateOperation
	voiceStateUpdateOperation
	voiceServerPingOperation
	resumeOperation
	reconnectOperation
	requestGuildMembersOperation
	invalidSessionOperation
	helloOperation
	heartbeatACKOperation
)

func (c *Conn) close() error {
	closeMsg := websocket.FormatCloseMessage(websocket.CloseNoStatusReceived, "no heartbeat acknowledgment")
	err := c.wsConn.WriteMessage(websocket.CloseMessage, closeMsg)
	err2 := c.wsConn.Close()
	if err != nil {
		return err
	}
	return err2
}

func (c *Conn) Close() error {
	close(c.closeChan)
	err := c.close()
	<-c.confirmClosedChan
	<-c.confirmClosedChan
	return err
}

type helloOPData struct {
	HeartbeatInterval int      `json:"heartbeat_interval"`
	Trace             []string `json:"_trace"`
}

func (c *Conn) Dial() (err error) {
	c.heartbeatAcknowledged = true

	c.closeChan = make(chan struct{})

	// TODO Need to set read deadline for hello packet and I also need to set write deadlines.
	// TODO also max message
	c.wsConn, _, err = websocket.DefaultDialer.Dial(c.gatewayURL, nil)
	if err != nil {
		return err
	}

	if c.sessionID == "" {
		err = c.identify()
	} else {
		err = c.resume()
	}
	if err != nil {
		return err
	}

	go c.readLoop()

	return nil
}

type resumeOPData struct {
	Token     string `json:"token"`
	SessionID string `json:"session_id"`
	Seq       int    `json:"seq"`
}

func (c *Conn) resume() error {
	c.mu.Lock()
	p := &sendPayload{
		Operation: resumeOperation,
		Data: resumeOPData{
			Token:     c.token,
			SessionID: c.sessionID,
			Seq:       c.sequenceNumber,
		},
	}
	c.mu.Unlock()
	return c.wsConn.WriteJSON(p)
}

type readyEvent struct {
	V               int        `json:"v"`
	User            *User      `json:"user"`
	PrivateChannels []*Channel `json:"private_channels"`
	SessionID       string     `json:"session_id"`
	Trace           []string   `json:"_trace"`
}

type receivePayload struct {
	Operation      int             `json:"op"`
	Data           json.RawMessage `json:"d"`
	SequenceNumber int             `json:"s"`
	Type           string          `json:"t"`
}

func (c *Conn) nextPayload() (*receivePayload, error) {
	var p receivePayload
	// TODO compression, see how discordgo does it
	err := c.wsConn.ReadJSON(&p)
	return &p, err
}

func (c *Conn) readLoop() {
	for {
		// TODO somehow reuse payload
		p, err := c.nextPayload()
		if err != nil {
			if err, ok := err.(net.OpError); ok && err.Err == os.ErrClosed {
				c.confirmClosedChan <- struct{}{}
			} else {
				log.Print(err)
				c.reconnectChan <- struct{}{}
			}
			return
		}
		c.onEvent(p)
	}
}

type sendPayload struct {
	Operation int         `json:"op"`
	Data      interface{} `json:"d,omitempty"`
	Sequence  int         `json:"s,omitempty"`
}

func (c *Conn) onEvent(p *receivePayload) error {
	switch p.Operation {
	case helloOperation:
		var hello helloOPData
		err := json.Unmarshal(p.Data, &hello)
		if err != nil {
			return err
		}
		go c.eventLoop(hello.HeartbeatInterval)
	case heartbeatACKOperation:
		c.mu.Lock()
		c.heartbeatAcknowledged = true
		c.mu.Unlock()
	case invalidSessionOperation:
		// TODO only once do this or sleep too? not sure, confusing docs
		err := c.identify()
		if err != nil {
			return err
		}
	case dispatchOperation:
		c.mu.Lock()
		c.sequenceNumber = p.SequenceNumber
		c.mu.Unlock()

		switch p.Type {
		case "READY":
			var ready readyEvent
			err := json.Unmarshal(p.Data, &ready)
			if err != nil {
				return err
			}
			c.sessionID = ready.SessionID
		}
		// TODO state tracking
	default:
		panic("discord gone crazy")
	}
	return nil
}

func (c *Conn) eventLoop(heartbeatInterval int) {
	ticker := time.NewTicker(time.Duration(heartbeatInterval) * time.Millisecond)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			err := c.heartbeat()
			if err != nil {
				log.Print(err)
				err := c.close()
				if err != nil {
					log.Print(err)
				}

				// Wait for readLoop to exit.
				// It's possible that it is trying to reconnect so receive on that channel too.
				select {
				case <-c.confirmClosedChan:
				case <-c.reconnectChan:
				}

				err = c.Dial()
				if err != nil {
					log.Print(err)
				}
				return
			}
		case <-c.reconnectChan:
			err := c.close()
			if err != nil {
				log.Print(err)
			}

			err = c.Dial()
			if err != nil {
				log.Print(err)
			}
			return
		case <-c.closeChan:
			c.confirmClosedChan <- struct{}{}
			return
		}
	}
}

func (c *Conn) heartbeat() error {
	c.mu.Lock()
	if !c.heartbeatAcknowledged {
		c.mu.Unlock()
		return errors.New("heartbeat not acknowledged")
	}
	sequenceNumber := c.sequenceNumber
	c.heartbeatAcknowledged = false
	c.mu.Unlock()

	p := &sendPayload{Operation: heartbeatOperation, Data: sequenceNumber}
	return c.wsConn.WriteJSON(p)
}

type identifyOPData struct {
	Token          string             `json:"token"`
	Properties     identifyProperties `json:"properties"`
	Compress       bool               `json:"compress"`
	LargeThreshold int                `json:"large_threshold"`
}

type identifyProperties struct {
	OS              string `json:"$os,omitempty"`
	Browser         string `json:"$browser,omitempty"`
	Device          string `json:"$device,omitempty"`
	Referrer        string `json:"$referrer,omitempty"`
	ReferringDomain string `json:"$referring_domain,omitempty"`
}

func (c *Conn) identify() error {
	p := &sendPayload{
		Operation: identifyOperation,
		Data: identifyOPData{
			Token: c.token,
			Properties: identifyProperties{
				OS:      runtime.GOOS,
				Browser: c.userAgent,
			},
			// TODO COMPRESS!!!
			Compress:       false,
			LargeThreshold: 250,
		},
	}
	return c.wsConn.WriteJSON(p)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

/*
Package gcs provides an API for building and using a Golomb-coded set filter.

Golomb-Coded Set

A Golomb-coded set is a probabilistic data structure used similarly to a Bloom
filter. A filter uses constant-size overhead plus on average n+2 bits per
item added to the filter, where 2^-1 is the desired false positive (collision)
probability.

GCS use in Bitcoin

GCS filters are a proposed mechanism for storing and transmitting per-block
filters in Bitcoin. The usage is intended to be the inverse of Bloom filters:
a full node would send an SPV node the GCS filter for a block, which the SPV
node would check against its list of relevant items. The suggested collision
probability for Bitcoin use is 2^-20.
*/
package gcs
<commit_msg>gcs: fix slight typo in docs, 2^-n not 2^-1<commit_after>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

/*
Package gcs provides an API for building and using a Golomb-coded set filter.

Golomb-Coded Set

A Golomb-coded set is a probabilistic data structure used similarly to a Bloom
filter. A filter uses constant-size overhead plus on average n+2 bits per
item added to the filter, where 2^-n is the desired false positive (collision)
probability.

GCS use in Bitcoin

GCS filters are a proposed mechanism for storing and transmitting per-block
filters in Bitcoin. The usage is intended to be the inverse of Bloom filters:
a full node would send an SPV node the GCS filter for a block, which the SPV
node would check against its list of relevant items. The suggested collision
probability for Bitcoin use is 2^-20.
*/
package gcs
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs

import (
	"encoding/binary"
	"fmt"
	"io"
	"sort"

	"github.com/aead/siphash"
	"github.com/kkdai/bstream"
)

// Inspired by https://github.com/rasky/gcs

var (
	// ErrNTooBig signifies that the filter can't handle N items.
	ErrNTooBig = fmt.Errorf("N is too big to fit in uint32")

	// ErrPTooBig signifies that the filter can't handle `1/2**P`
	// collision probability.
	ErrPTooBig = fmt.Errorf("P is too big to fit in uint32")

	// ErrNoData signifies that an empty slice was passed.
	ErrNoData = fmt.Errorf("No data provided")
)

const (
	// KeySize is the size of the byte array required for key material for
	// the SipHash keyed hash function.
	KeySize = 16
)

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
// include N or P to allow the user to encode the metadata separately if
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type Filter struct {
	n          uint32
	p          uint8
	modulusP   uint64
	modulusNP  uint64
	filterData []byte
}

// BuildGCSFilter builds a new GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
	// Some initial parameter checks: make sure we have data from which to
	// build the filter, and make sure our parameters will fit the hash
	// function we're using.
	if len(data) == 0 {
		return nil, ErrNoData
	}
	if len(data) > ((1 << 32) - 1) {
		return nil, ErrNTooBig
	}
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := Filter{
		n: uint32(len(data)),
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Build the filter.
	values := make(uint64Slice, 0, len(data))
	b := bstream.NewBStreamWriter(0)

	// Insert the hash (modulo N*P) of each data element into a slice and
	// sort the slice.
	for _, d := range data {
		v := siphash.Sum64(d, &key) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort(values)

	// Write the sorted list of values into the filter bitstream,
	// compressing it using Golomb coding.
	var value, lastValue, remainder uint64
	for _, v := range values {
		// Calculate the difference between this value and the last,
		// modulo P.
		remainder = (v - lastValue) & (f.modulusP - 1)

		// Calculate the difference between this value and the last,
		// divided by P.
		value = (v - lastValue - remainder) >> f.p
		lastValue = v

		// Write the P multiple into the bitstream in unary; the
		// average should be around 1 (2 bits - 0b10).
		for value > 0 {
			b.WriteBit(true)
			value--
		}
		b.WriteBit(false)

		// Write the remainder as a big-endian integer with enough bits
		// to represent the appropriate collision probability.
		b.WriteBits(remainder, int(f.p))
	}

	// Copy the bitstream into the filter object and return the object.
	f.filterData = b.Bytes()

	return &f, nil
}

// FromBytes deserializes a GCS filter from a known N, P, and serialized filter
// as returned by Bytes().
func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {

	// Basic sanity check.
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := &Filter{
		n: N,
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Copy the filter.
	f.filterData = make([]byte, len(d))
	copy(f.filterData, d)

	return f, nil
}

// FromNBytes deserializes a GCS filter from a known P, and serialized N and
// filter as returned by NBytes().
func FromNBytes(P uint8, d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), P, d[4:])
}

// FromPBytes deserializes a GCS filter from a known N, and serialized P and
// filter as returned by NBytes().
func FromPBytes(N uint32, d []byte) (*Filter, error) {
	return FromBytes(N, d[0], d[1:])
}

// FromNPBytes deserializes a GCS filter from a serialized N, P, and filter as
// returned by NPBytes().
func FromNPBytes(d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), d[4], d[5:])
}

// Bytes returns the serialized format of the GCS filter, which does not
// include N or P (returned by separate methods) or the key used by SipHash.
func (f *Filter) Bytes() []byte {
	filterData := make([]byte, len(f.filterData))
	copy(filterData, f.filterData)
	return filterData
}

// NBytes returns the serialized format of the GCS filter with N, which does
// not include P (returned by a separate method) or the key used by SipHash.
func (f *Filter) NBytes() []byte {
	filterData := make([]byte, len(f.filterData)+4)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	copy(filterData[4:], f.filterData)
	return filterData
}

// PBytes returns the serialized format of the GCS filter with P, which does
// not include N (returned by a separate method) or the key used by SipHash.
func (f *Filter) PBytes() []byte {
	filterData := make([]byte, len(f.filterData)+1)
	filterData[0] = f.p
	copy(filterData[1:], f.filterData)
	return filterData
}

// NPBytes returns the serialized format of the GCS filter with N and P, which
// does not include the key used by SipHash.
func (f *Filter) NPBytes() []byte {
	filterData := make([]byte, len(f.filterData)+5)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	filterData[4] = f.p
	copy(filterData[5:], f.filterData)
	return filterData
}

// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *Filter) P() uint8 {
	return f.p
}

// N returns the size of the data set used to build the filter.
func (f *Filter) N() uint32 {
	return f.n
}

// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Hash our search term with the same parameters as the filter.
	term := siphash.Sum64(data, &key) % f.modulusNP

	// Go through the search filter and look for the desired value.
	var lastValue uint64
	for lastValue < term {

		// Read the difference between previous and new value from
		// bitstream.
		value, err := f.readFullUint64(b)
		if err != nil {
			if err == io.EOF {
				return false, nil
			}
			return false, err
		}

		// Add the previous value to it.
		value += lastValue
		if value == term {
			return true, nil
		}

		lastValue = value
	}

	return false, nil
}

// MatchAny returns checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {

	// Basic sanity check.
	if len(data) == 0 {
		return false, ErrNoData
	}

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Create an uncompressed filter of the search values.
	values := make(uint64Slice, 0, len(data))
	for _, d := range data {
		v := siphash.Sum64(d, &key) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort(values)

	// Zip down the filters, comparing values until we either run out of
	// values to compare in one of the filters or we reach a matching
	// value.
	var lastValue1, lastValue2 uint64
	lastValue2 = values[0]
	i := 1
	for lastValue1 != lastValue2 {
		// Check which filter to advance to make sure we're comparing
		// the right values.
		switch {
		case lastValue1 > lastValue2:
			// Advance filter created from search terms or return
			// false if we're at the end because nothing matched.
			if i < len(values) {
				lastValue2 = values[i]
				i++
			} else {
				return false, nil
			}
		case lastValue2 > lastValue1:
			// Advance filter we're searching or return false if
			// we're at the end because nothing matched.
			value, err := f.readFullUint64(b)
			if err != nil {
				if err == io.EOF {
					return false, nil
				}
				return false, err
			}
			lastValue1 += value
		}
	}

	// If we've made it this far, an element matched between filters so we
	// return true.
	return true, nil
}

// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) {
	var v uint64

	// Count the 1s until we reach a 0.
	c, err := b.ReadBit()
	if err != nil {
		return 0, err
	}
	for c {
		v++
		c, err = b.ReadBit()
		if err != nil {
			return 0, err
		}
	}

	// Read P bits.
	remainder, err := b.ReadBits(int(f.p))
	if err != nil {
		return 0, err
	}

	// Add the multiple and the remainder.
	v = v*f.modulusP + remainder
	return v, nil
}
<commit_msg>gcs: fix constant overflow for 32-bit systems<commit_after>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"sort"

	"github.com/aead/siphash"
	"github.com/kkdai/bstream"
)

// Inspired by https://github.com/rasky/gcs

var (
	// ErrNTooBig signifies that the filter can't handle N items.
	ErrNTooBig = fmt.Errorf("N is too big to fit in uint32")

	// ErrPTooBig signifies that the filter can't handle `1/2**P`
	// collision probability.
	ErrPTooBig = fmt.Errorf("P is too big to fit in uint32")

	// ErrNoData signifies that an empty slice was passed.
	ErrNoData = fmt.Errorf("No data provided")
)

const (
	// KeySize is the size of the byte array required for key material for
	// the SipHash keyed hash function.
	KeySize = 16
)

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
// include N or P to allow the user to encode the metadata separately if
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type Filter struct {
	n          uint32
	p          uint8
	modulusP   uint64
	modulusNP  uint64
	filterData []byte
}

// BuildGCSFilter builds a new GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
	// Some initial parameter checks: make sure we have data from which to
	// build the filter, and make sure our parameters will fit the hash
	// function we're using.
	if len(data) == 0 {
		return nil, ErrNoData
	}
	if len(data) > math.MaxInt32 {
		return nil, ErrNTooBig
	}
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := Filter{
		n: uint32(len(data)),
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Build the filter.
	values := make(uint64Slice, 0, len(data))
	b := bstream.NewBStreamWriter(0)

	// Insert the hash (modulo N*P) of each data element into a slice and
	// sort the slice.
	for _, d := range data {
		v := siphash.Sum64(d, &key) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort(values)

	// Write the sorted list of values into the filter bitstream,
	// compressing it using Golomb coding.
	var value, lastValue, remainder uint64
	for _, v := range values {
		// Calculate the difference between this value and the last,
		// modulo P.
		remainder = (v - lastValue) & (f.modulusP - 1)

		// Calculate the difference between this value and the last,
		// divided by P.
		value = (v - lastValue - remainder) >> f.p
		lastValue = v

		// Write the P multiple into the bitstream in unary; the
		// average should be around 1 (2 bits - 0b10).
		for value > 0 {
			b.WriteBit(true)
			value--
		}
		b.WriteBit(false)

		// Write the remainder as a big-endian integer with enough bits
		// to represent the appropriate collision probability.
		b.WriteBits(remainder, int(f.p))
	}

	// Copy the bitstream into the filter object and return the object.
	f.filterData = b.Bytes()

	return &f, nil
}

// FromBytes deserializes a GCS filter from a known N, P, and serialized filter
// as returned by Bytes().
func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {

	// Basic sanity check.
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := &Filter{
		n: N,
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Copy the filter.
	f.filterData = make([]byte, len(d))
	copy(f.filterData, d)

	return f, nil
}

// FromNBytes deserializes a GCS filter from a known P, and serialized N and
// filter as returned by NBytes().
func FromNBytes(P uint8, d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), P, d[4:])
}

// FromPBytes deserializes a GCS filter from a known N, and serialized P and
// filter as returned by NBytes().
func FromPBytes(N uint32, d []byte) (*Filter, error) {
	return FromBytes(N, d[0], d[1:])
}

// FromNPBytes deserializes a GCS filter from a serialized N, P, and filter as
// returned by NPBytes().
func FromNPBytes(d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), d[4], d[5:])
}

// Bytes returns the serialized format of the GCS filter, which does not
// include N or P (returned by separate methods) or the key used by SipHash.
func (f *Filter) Bytes() []byte {
	filterData := make([]byte, len(f.filterData))
	copy(filterData, f.filterData)
	return filterData
}

// NBytes returns the serialized format of the GCS filter with N, which does
// not include P (returned by a separate method) or the key used by SipHash.
func (f *Filter) NBytes() []byte {
	filterData := make([]byte, len(f.filterData)+4)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	copy(filterData[4:], f.filterData)
	return filterData
}

// PBytes returns the serialized format of the GCS filter with P, which does
// not include N (returned by a separate method) or the key used by SipHash.
func (f *Filter) PBytes() []byte {
	filterData := make([]byte, len(f.filterData)+1)
	filterData[0] = f.p
	copy(filterData[1:], f.filterData)
	return filterData
}

// NPBytes returns the serialized format of the GCS filter with N and P, which
// does not include the key used by SipHash.
func (f *Filter) NPBytes() []byte {
	filterData := make([]byte, len(f.filterData)+5)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	filterData[4] = f.p
	copy(filterData[5:], f.filterData)
	return filterData
}

// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *Filter) P() uint8 {
	return f.p
}

// N returns the size of the data set used to build the filter.
func (f *Filter) N() uint32 {
	return f.n
}

// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Hash our search term with the same parameters as the filter.
	term := siphash.Sum64(data, &key) % f.modulusNP

	// Go through the search filter and look for the desired value.
	var lastValue uint64
	for lastValue < term {

		// Read the difference between previous and new value from
		// bitstream.
		value, err := f.readFullUint64(b)
		if err != nil {
			if err == io.EOF {
				return false, nil
			}
			return false, err
		}

		// Add the previous value to it.
		value += lastValue
		if value == term {
			return true, nil
		}

		lastValue = value
	}

	return false, nil
}

// MatchAny returns checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {

	// Basic sanity check.
	if len(data) == 0 {
		return false, ErrNoData
	}

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Create an uncompressed filter of the search values.
	values := make(uint64Slice, 0, len(data))
	for _, d := range data {
		v := siphash.Sum64(d, &key) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort(values)

	// Zip down the filters, comparing values until we either run out of
	// values to compare in one of the filters or we reach a matching
	// value.
	var lastValue1, lastValue2 uint64
	lastValue2 = values[0]
	i := 1
	for lastValue1 != lastValue2 {
		// Check which filter to advance to make sure we're comparing
		// the right values.
		switch {
		case lastValue1 > lastValue2:
			// Advance filter created from search terms or return
			// false if we're at the end because nothing matched.
			if i < len(values) {
				lastValue2 = values[i]
				i++
			} else {
				return false, nil
			}
		case lastValue2 > lastValue1:
			// Advance filter we're searching or return false if
			// we're at the end because nothing matched.
			value, err := f.readFullUint64(b)
			if err != nil {
				if err == io.EOF {
					return false, nil
				}
				return false, err
			}
			lastValue1 += value
		}
	}

	// If we've made it this far, an element matched between filters so we
	// return true.
	return true, nil
}

// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) {
	var v uint64

	// Count the 1s until we reach a 0.
	c, err := b.ReadBit()
	if err != nil {
		return 0, err
	}
	for c {
		v++
		c, err = b.ReadBit()
		if err != nil {
			return 0, err
		}
	}

	// Read P bits.
	remainder, err := b.ReadBits(int(f.p))
	if err != nil {
		return 0, err
	}

	// Add the multiple and the remainder.
	v = v*f.modulusP + remainder
	return v, nil
}
<|endoftext|>


<commit_before>package builder

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/configwatch"
	libjson "github.com/Symantec/Dominator/lib/json"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/slavedriver"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/url/urlutil"
)

func imageStreamsDecoder(reader io.Reader) (interface{}, error) {
	var config imageStreamsConfigurationType
	decoder := json.NewDecoder(bufio.NewReader(reader))
	if err := decoder.Decode(&config); err != nil {
		return nil, fmt.Errorf("error reading image streams: %s", err)
	}
	return &config, nil
}

func load(confUrl, variablesFile, stateDir, imageServerAddress string,
	imageRebuildInterval time.Duration, slaveDriver *slavedriver.SlaveDriver,
	logger log.DebugLogger) (*Builder, error) {
	err := syscall.Mount("none", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
	if err != nil {
		return nil, fmt.Errorf("error making mounts private: %s", err)
	}
	masterConfiguration, err := masterConfiguration(confUrl)
	if err != nil {
		return nil, fmt.Errorf("error getting master configuration: %s", err)
	}
	imageStreamsToAutoRebuild := make([]string, 0)
	for name := range masterConfiguration.BootstrapStreams {
		imageStreamsToAutoRebuild = append(imageStreamsToAutoRebuild, name)
	}
	sort.Strings(imageStreamsToAutoRebuild)
	for _, name := range masterConfiguration.ImageStreamsToAutoRebuild {
		imageStreamsToAutoRebuild = append(imageStreamsToAutoRebuild, name)
	}
	var variables map[string]string
	if variablesFile != "" {
		if err := libjson.ReadFromFile(variablesFile, &variables); err != nil {
			return nil, err
		}
	}
	if variables == nil {
		variables = make(map[string]string)
	}
	b := &Builder{
		stateDir:                  stateDir,
		imageServerAddress:        imageServerAddress,
		logger:                    logger,
		imageStreamsUrl:           masterConfiguration.ImageStreamsUrl,
		bootstrapStreams:          masterConfiguration.BootstrapStreams,
		imageStreamsToAutoRebuild: imageStreamsToAutoRebuild,
		slaveDriver:               slaveDriver,
		currentBuildLogs:          make(map[string]*bytes.Buffer),
		lastBuildResults:          make(map[string]buildResultType),
		packagerTypes:             masterConfiguration.PackagerTypes,
		variables:                 variables,
	}
	for name, stream := range b.bootstrapStreams {
		stream.builder = b
		stream.name = name
	}
	imageStreamsConfigChannel, err := configwatch.WatchWithCache(
		masterConfiguration.ImageStreamsUrl,
		time.Second*time.Duration(
			masterConfiguration.ImageStreamsCheckInterval), imageStreamsDecoder,
		filepath.Join(stateDir, "image-streams.json"),
		time.Second*5, logger)
	if err != nil {
		return nil, err
	}
	go b.watchConfigLoop(imageStreamsConfigChannel)
	go b.rebuildImages(imageRebuildInterval)
	return b, nil
}

func loadImageStreams(url string) (*imageStreamsConfigurationType, error) {
	if url == "" {
		return &imageStreamsConfigurationType{}, nil
	}
	file, err := urlutil.Open(url)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	var configuration imageStreamsConfigurationType
	decoder := json.NewDecoder(bufio.NewReader(file))
	if err := decoder.Decode(&configuration); err != nil {
		return nil, fmt.Errorf("error decoding image streams from: %s: %s",
			url, err)
	}
	return &configuration, nil
}

func masterConfiguration(url string) (*masterConfigurationType, error) {
	file, err := urlutil.Open(url)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	var configuration masterConfigurationType
	decoder := json.NewDecoder(bufio.NewReader(file))
	if err := decoder.Decode(&configuration); err != nil {
		return nil, fmt.Errorf("error reading configuration from: %s: %s",
			url, err)
	}
	for _, stream := range configuration.BootstrapStreams {
		if _, ok := configuration.PackagerTypes[stream.PackagerType]; !ok {
			return nil, fmt.Errorf("packager type: \"%s\" unknown",
				stream.PackagerType)
		}
		if stream.Filter != nil {
			if err := stream.Filter.Compile(); err != nil {
				return nil, err
			}
		}
	}
	return &configuration, nil
}

func (b *Builder) makeRequiredDirectories() error {
	imageServer, err := srpc.DialHTTP("tcp", b.imageServerAddress, 0)
	if err != nil {
		b.logger.Println(err)
		return nil
	}
	defer imageServer.Close()
	directoryList, err := client.ListDirectories(imageServer)
	if err != nil {
		b.logger.Println(err)
		return nil
	}
	directories := make(map[string]struct{}, len(directoryList))
	for _, directory := range directoryList {
		directories[directory.Name] = struct{}{}
	}
	streamNames := b.listAllStreamNames()
	for _, streamName := range streamNames {
		if _, ok := directories[streamName]; ok {
			continue
		}
		pathComponents := strings.Split(streamName, "/")
		for index := range pathComponents {
			partPath := strings.Join(pathComponents[0:index+1], "/")
			if _, ok := directories[partPath]; ok {
				continue
			}
			if err := client.MakeDirectory(imageServer, partPath); err != nil {
				return err
			}
			b.logger.Printf("Created missing directory: %s\n", partPath)
			directories[partPath] = struct{}{}
		}
	}
	return nil
}

func (b *Builder) reloadNormalStreamsConfiguration() error {
	imageStreamsConfiguration, err := loadImageStreams(b.imageStreamsUrl)
	if err != nil {
		return err
	}
	b.logger.Println("Reloaded streams streams configuration")
	return b.updateImageStreams(imageStreamsConfiguration)
}

func (b *Builder) updateImageStreams(
	imageStreamsConfiguration *imageStreamsConfigurationType) error {
	for name, stream := range imageStreamsConfiguration.Streams {
		stream.builder = b
		stream.name = name
	}
	b.streamsLock.Lock()
	b.imageStreams = imageStreamsConfiguration.Streams
	b.streamsLock.Unlock()
	return b.makeRequiredDirectories()
}

func (b *Builder) watchConfigLoop(configChannel <-chan interface{}) {
	for rawConfig := range configChannel {
		imageStreamsConfig, ok := rawConfig.(*imageStreamsConfigurationType)
		if !ok {
			b.logger.Printf("received unknown type over config channel")
			continue
		}
		b.logger.Println("received new image streams configuration")
		b.updateImageStreams(imageStreamsConfig)
	}
}
<commit_msg>imaginator: make missing directories even when streams config not present.<commit_after>package builder

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/configwatch"
	libjson "github.com/Symantec/Dominator/lib/json"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/slavedriver"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/url/urlutil"
)

func imageStreamsDecoder(reader io.Reader) (interface{}, error) {
	var config imageStreamsConfigurationType
	decoder := json.NewDecoder(bufio.NewReader(reader))
	if err := decoder.Decode(&config); err != nil {
		return nil, fmt.Errorf("error reading image streams: %s", err)
	}
	return &config, nil
}

func load(confUrl, variablesFile, stateDir, imageServerAddress string,
	imageRebuildInterval time.Duration, slaveDriver *slavedriver.SlaveDriver,
	logger log.DebugLogger) (*Builder, error) {
	err := syscall.Mount("none", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
	if err != nil {
		return nil, fmt.Errorf("error making mounts private: %s", err)
	}
	masterConfiguration, err := masterConfiguration(confUrl)
	if err != nil {
		return nil, fmt.Errorf("error getting master configuration: %s", err)
	}
	imageStreamsToAutoRebuild := make([]string, 0)
	for name := range masterConfiguration.BootstrapStreams {
		imageStreamsToAutoRebuild = append(imageStreamsToAutoRebuild, name)
	}
	sort.Strings(imageStreamsToAutoRebuild)
	for _, name := range masterConfiguration.ImageStreamsToAutoRebuild {
		imageStreamsToAutoRebuild = append(imageStreamsToAutoRebuild, name)
	}
	var variables map[string]string
	if variablesFile != "" {
		if err := libjson.ReadFromFile(variablesFile, &variables); err != nil {
			return nil, err
		}
	}
	if variables == nil {
		variables = make(map[string]string)
	}
	b := &Builder{
		stateDir:                  stateDir,
		imageServerAddress:        imageServerAddress,
		logger:                    logger,
		imageStreamsUrl:           masterConfiguration.ImageStreamsUrl,
		bootstrapStreams:          masterConfiguration.BootstrapStreams,
		imageStreamsToAutoRebuild: imageStreamsToAutoRebuild,
		slaveDriver:               slaveDriver,
		currentBuildLogs:          make(map[string]*bytes.Buffer),
		lastBuildResults:          make(map[string]buildResultType),
		packagerTypes:             masterConfiguration.PackagerTypes,
		variables:                 variables,
	}
	for name, stream := range b.bootstrapStreams {
		stream.builder = b
		stream.name = name
	}
	imageStreamsConfigChannel, err := configwatch.WatchWithCache(
		masterConfiguration.ImageStreamsUrl,
		time.Second*time.Duration(
			masterConfiguration.ImageStreamsCheckInterval), imageStreamsDecoder,
		filepath.Join(stateDir, "image-streams.json"),
		time.Second*5, logger)
	if err != nil {
		return nil, err
	}
	go b.watchConfigLoop(imageStreamsConfigChannel)
	go b.rebuildImages(imageRebuildInterval)
	return b, nil
}

func loadImageStreams(url string) (*imageStreamsConfigurationType, error) {
	if url == "" {
		return &imageStreamsConfigurationType{}, nil
	}
	file, err := urlutil.Open(url)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	var configuration imageStreamsConfigurationType
	decoder := json.NewDecoder(bufio.NewReader(file))
	if err := decoder.Decode(&configuration); err != nil {
		return nil, fmt.Errorf("error decoding image streams from: %s: %s",
			url, err)
	}
	return &configuration, nil
}

func masterConfiguration(url string) (*masterConfigurationType, error) {
	file, err := urlutil.Open(url)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	var configuration masterConfigurationType
	decoder := json.NewDecoder(bufio.NewReader(file))
	if err := decoder.Decode(&configuration); err != nil {
		return nil, fmt.Errorf("error reading configuration from: %s: %s",
			url, err)
	}
	for _, stream := range configuration.BootstrapStreams {
		if _, ok := configuration.PackagerTypes[stream.PackagerType]; !ok {
			return nil, fmt.Errorf("packager type: \"%s\" unknown",
				stream.PackagerType)
		}
		if stream.Filter != nil {
			if err := stream.Filter.Compile(); err != nil {
				return nil, err
			}
		}
	}
	return &configuration, nil
}

func (b *Builder) delayMakeRequiredDirectories(abortNotifier <-chan struct{}) {
	timer := time.NewTimer(time.Second * 5)
	select {
	case <-abortNotifier:
		if !timer.Stop() {
			<-timer.C
		}
	case <-timer.C:
		b.makeRequiredDirectories()
	}
}

func (b *Builder) makeRequiredDirectories() error {
	imageServer, err := srpc.DialHTTP("tcp", b.imageServerAddress, 0)
	if err != nil {
		b.logger.Println(err)
		return nil
	}
	defer imageServer.Close()
	directoryList, err := client.ListDirectories(imageServer)
	if err != nil {
		b.logger.Println(err)
		return nil
	}
	directories := make(map[string]struct{}, len(directoryList))
	for _, directory := range directoryList {
		directories[directory.Name] = struct{}{}
	}
	streamNames := b.listAllStreamNames()
	for _, streamName := range streamNames {
		if _, ok := directories[streamName]; ok {
			continue
		}
		pathComponents := strings.Split(streamName, "/")
		for index := range pathComponents {
			partPath := strings.Join(pathComponents[0:index+1], "/")
			if _, ok := directories[partPath]; ok {
				continue
			}
			if err := client.MakeDirectory(imageServer, partPath); err != nil {
				return err
			}
			b.logger.Printf("Created missing directory: %s\n", partPath)
			directories[partPath] = struct{}{}
		}
	}
	return nil
}

func (b *Builder) reloadNormalStreamsConfiguration() error {
	imageStreamsConfiguration, err := loadImageStreams(b.imageStreamsUrl)
	if err != nil {
		return err
	}
	b.logger.Println("Reloaded streams streams configuration")
	return b.updateImageStreams(imageStreamsConfiguration)
}

func (b *Builder) updateImageStreams(
	imageStreamsConfiguration *imageStreamsConfigurationType) error {
	for name, stream := range imageStreamsConfiguration.Streams {
		stream.builder = b
		stream.name = name
	}
	b.streamsLock.Lock()
	b.imageStreams = imageStreamsConfiguration.Streams
	b.streamsLock.Unlock()
	return b.makeRequiredDirectories()
}

func (b *Builder) watchConfigLoop(configChannel <-chan interface{}) {
	firstLoadNotifier := make(chan struct{})
	go b.delayMakeRequiredDirectories(firstLoadNotifier)
	for rawConfig := range configChannel {
		imageStreamsConfig, ok := rawConfig.(*imageStreamsConfigurationType)
		if !ok {
			b.logger.Printf("received unknown type over config channel")
			continue
		}
		if firstLoadNotifier != nil {
			firstLoadNotifier <- struct{}{}
			close(firstLoadNotifier)
			firstLoadNotifier = nil
		}
		b.logger.Println("received new image streams configuration")
		b.updateImageStreams(imageStreamsConfig)
	}
}
<|endoftext|>


<commit_before>package gddoexp

import (
	"sync"
	"time"

	"github.com/golang/gddo/database"
	"github.com/juju/ratelimit"
)

// unused stores the time that an unmodified project is considered unused.
const unused = 2 * 365 * 24 * time.Hour

// commitsLimit is the maximum number of commits made in the fork so we could
// identify as a fast fork.
const commitsLimit = 2

// commitsPeriod is the period after the fork creation date that we will
// consider the commits a fast fork.
const commitsPeriod = 7 * 24 * time.Hour

// agents contains the number of concurrent go routines that will process
// a list of packages
const agents = 4

// RateLimit controls the number of requests sent to the Github API for
// authenticated and unauthenticated scenarios using the token bucket
// strategy. For more information on the values, please check:
// https://developer.github.com/v3/#rate-limiting
var RateLimit = struct {
	FillInterval     time.Duration
	Capacity         int64
	AuthFillInterval time.Duration
	AuthCapacity     int64
}{
	FillInterval:     time.Minute,
	Capacity:         agents,
	AuthFillInterval: time.Second,
	AuthCapacity:     agents,
}

// gddoDB contains all used methods from Database type of
// github.com/golang/gddo/database. This is useful for mocking and building
// tests.
type gddoDB interface {
	ImporterCount(string) (int, error)
}

// ArchiveResponse stores the information of a path verification on an
// asynchronous check.
type ArchiveResponse struct {
	Path    string
	Archive bool
	Cache   bool
	Error   error
}

// ShouldArchivePackage determinate if a package should be archived or not.
// It's necessary to inform the GoDoc database to retrieve current stored
// package information. An optional argument with the Github authentication
// can be informed to allow more checks per minute in Github API.
func ShouldArchivePackage(p database.Package, db gddoDB, auth *GithubAuth) (archive, cache bool, err error) {
	count, err := db.ImporterCount(p.Path)
	if err != nil {
		// as we didn't perform any request yet, we can return a cache hit to
		// reuse the token
		return false, true, NewError(p.Path, ErrorCodeRetrieveImportCounts, err)
	}

	// don't archive the package if there's a reference to it from other
	// projects (let's avoid send a request to Github if we already no that we
	// don't need to archive it). We return cache hit as no request was made to
	// Github API.
	if count > 0 {
		return false, true, nil
	}

	repository, cache, err := getGithubRepository(p.Path, auth)
	if err != nil {
		return false, cache, err
	}

	// we only archive the package if there's no reference to it from other
	// projects (checked above) and if there's no updates in Github on the last
	// 2 years
	return time.Now().Sub(repository.UpdatedAt) >= unused, cache, nil
}

// ShouldArchivePackages determinate if a package should be archived or not,
// but unlike ShouldArchivePackage, it can process a list of packages
// concurrently. It's necessary to inform the GoDoc database to retrieve
// current stored package information. An optional argument with the Github
// authentication can be informed to allow more checks per minute in Github
// API (we will use token bucket strategy to don't exceed the rate limit).
func ShouldArchivePackages(packages []database.Package, db gddoDB, auth *GithubAuth) <-chan ArchiveResponse {
	out := make(chan ArchiveResponse)

	go func() {
		var bucket *ratelimit.Bucket
		if auth == nil {
			bucket = ratelimit.NewBucket(RateLimit.FillInterval, RateLimit.Capacity)
		} else {
			bucket = ratelimit.NewBucket(RateLimit.AuthFillInterval, RateLimit.AuthCapacity)
		}

		var wg sync.WaitGroup
		wg.Add(agents)

		in := make(chan database.Package)

		for i := 0; i < agents; i++ {
			go func() {
				// if the go routine retrieve a response from cache, it can run again
				// without waiting for a token, as no hit was made in the Github API
				wait := true
				for p := range in {
					if wait {
						bucket.Wait(1)
					} else {
						wait = true
					}

					archive, cache, err := ShouldArchivePackage(p, db, auth)
					out <- ArchiveResponse{
						Path:    p.Path,
						Archive: archive,
						Cache:   cache,
						Error:   err,
					}

					if cache {
						wait = false
					}
				}

				wg.Done()
			}()
		}

		for _, pkg := range packages {
			in <- pkg
		}

		close(in)
		wg.Wait()
		close(out)
	}()

	return out
}

// FastForkResponse stores the information of a path verification on an
// asynchronous check.
type FastForkResponse struct {
	Path     string
	FastFork bool
	Cache    bool
	Error    error
}

// IsFastForkPackage identifies if a package is a fork created only to make
// small changes for a pull request. An optional argument with the Github
// authentication can be informed to allow more checks per minute in Github API.
func IsFastForkPackage(p database.Package, auth *GithubAuth) (fastFork, cache bool, err error) {
	repository, cacheRepository, err := getGithubRepository(p.Path, auth)
	if err != nil {
		return false, cacheRepository, err
	}

	// if the repository is not a fork we don't need to check the commits
	if !repository.Fork {
		return false, cacheRepository, nil
	}

	commits, cacheCommits, err := getCommits(p.Path, auth)
	if err != nil {
		return false, cacheRepository && cacheCommits, err
	}

	forkLimitDate := repository.CreatedAt.Add(commitsPeriod)
	commitCounts := 0
	fastFork = true

	for _, commit := range commits {
		if commit.Commit.Author.Date.After(forkLimitDate) {
			fastFork = false
			break
		}

		if commit.Commit.Author.Date.After(repository.CreatedAt) {
			commitCounts++
		}
	}

	if commitCounts > commitsLimit {
		fastFork = false
	}

	return fastFork, cacheRepository && cacheCommits, nil
}

// AreFastForkPackages determinate if a package is a fast fork or not,
// but unlike IsFastForkPackage, it can process a list of packages
// concurrently. An optional argument with the Github authentication can be
// informed to allow more checks per minute in Github API (we will use token
// bucket strategy to don't exceed the rate limit).
func AreFastForkPackages(packages []database.Package, auth *GithubAuth) <-chan FastForkResponse {
	out := make(chan FastForkResponse)

	go func() {
		var bucket *ratelimit.Bucket
		if auth == nil {
			bucket = ratelimit.NewBucket(RateLimit.FillInterval, RateLimit.Capacity)
		} else {
			bucket = ratelimit.NewBucket(RateLimit.AuthFillInterval, RateLimit.AuthCapacity)
		}

		var wg sync.WaitGroup
		wg.Add(agents)

		in := make(chan database.Package)

		for i := 0; i < agents; i++ {
			go func() {
				// if the go routine retrieve a response from cache, it can run again
				// without waiting for a token, as no hit was made in the Github API
				wait := true
				for p := range in {
					if wait {
						bucket.Wait(1)
					} else {
						wait = true
					}

					fastFork, cache, err := IsFastForkPackage(p, auth)
					out <- FastForkResponse{
						Path:     p.Path,
						FastFork: fastFork,
						Cache:    cache,
						Error:    err,
					}

					if cache {
						wait = false
					}
				}

				wg.Done()
			}()
		}

		for _, pkg := range packages {
			in <- pkg
		}

		close(in)
		wg.Wait()
		close(out)
	}()

	return out
}
<commit_msg>Avoid wait buffering channels<commit_after>package gddoexp

import (
	"sync"
	"time"

	"github.com/golang/gddo/database"
	"github.com/juju/ratelimit"
)

// unused stores the time that an unmodified project is considered unused.
const unused = 2 * 365 * 24 * time.Hour

// commitsLimit is the maximum number of commits made in the fork so we could
// identify as a fast fork.
const commitsLimit = 2

// commitsPeriod is the period after the fork creation date that we will
// consider the commits a fast fork.
const commitsPeriod = 7 * 24 * time.Hour

// agents contains the number of concurrent go routines that will process
// a list of packages
const agents = 4

// RateLimit controls the number of requests sent to the Github API for
// authenticated and unauthenticated scenarios using the token bucket
// strategy. For more information on the values, please check:
// https://developer.github.com/v3/#rate-limiting
var RateLimit = struct {
	FillInterval     time.Duration
	Capacity         int64
	AuthFillInterval time.Duration
	AuthCapacity     int64
}{
	FillInterval:     time.Minute,
	Capacity:         agents,
	AuthFillInterval: time.Second,
	AuthCapacity:     agents,
}

// gddoDB contains all used methods from Database type of
// github.com/golang/gddo/database. This is useful for mocking and building
// tests.
type gddoDB interface {
	ImporterCount(string) (int, error)
}

// ArchiveResponse stores the information of a path verification on an
// asynchronous check.
type ArchiveResponse struct {
	Path    string
	Archive bool
	Cache   bool
	Error   error
}

// ShouldArchivePackage determinate if a package should be archived or not.
// It's necessary to inform the GoDoc database to retrieve current stored
// package information. An optional argument with the Github authentication
// can be informed to allow more checks per minute in Github API.
func ShouldArchivePackage(p database.Package, db gddoDB, auth *GithubAuth) (archive, cache bool, err error) {
	count, err := db.ImporterCount(p.Path)
	if err != nil {
		// as we didn't perform any request yet, we can return a cache hit to
		// reuse the token
		return false, true, NewError(p.Path, ErrorCodeRetrieveImportCounts, err)
	}

	// don't archive the package if there's a reference to it from other
	// projects (let's avoid send a request to Github if we already no that we
	// don't need to archive it). We return cache hit as no request was made to
	// Github API.
	if count > 0 {
		return false, true, nil
	}

	repository, cache, err := getGithubRepository(p.Path, auth)
	if err != nil {
		return false, cache, err
	}

	// we only archive the package if there's no reference to it from other
	// projects (checked above) and if there's no updates in Github on the last
	// 2 years
	return time.Now().Sub(repository.UpdatedAt) >= unused, cache, nil
}

// ShouldArchivePackages determinate if a package should be archived or not,
// but unlike ShouldArchivePackage, it can process a list of packages
// concurrently. It's necessary to inform the GoDoc database to retrieve
// current stored package information. An optional argument with the Github
// authentication can be informed to allow more checks per minute in Github
// API (we will use token bucket strategy to don't exceed the rate limit).
func ShouldArchivePackages(packages []database.Package, db gddoDB, auth *GithubAuth) <-chan ArchiveResponse {
	out := make(chan ArchiveResponse, agents)

	go func() {
		var bucket *ratelimit.Bucket
		if auth == nil {
			bucket = ratelimit.NewBucket(RateLimit.FillInterval, RateLimit.Capacity)
		} else {
			bucket = ratelimit.NewBucket(RateLimit.AuthFillInterval, RateLimit.AuthCapacity)
		}

		var wg sync.WaitGroup
		wg.Add(agents)

		in := make(chan database.Package)

		for i := 0; i < agents; i++ {
			go func() {
				// if the go routine retrieve a response from cache, it can run again
				// without waiting for a token, as no hit was made in the Github API
				wait := true
				for p := range in {
					if wait {
						bucket.Wait(1)
					} else {
						wait = true
					}

					archive, cache, err := ShouldArchivePackage(p, db, auth)
					out <- ArchiveResponse{
						Path:    p.Path,
						Archive: archive,
						Cache:   cache,
						Error:   err,
					}

					if cache {
						wait = false
					}
				}

				wg.Done()
			}()
		}

		for _, pkg := range packages {
			in <- pkg
		}

		close(in)
		wg.Wait()
		close(out)
	}()

	return out
}

// FastForkResponse stores the information of a path verification on an
// asynchronous check.
type FastForkResponse struct {
	Path     string
	FastFork bool
	Cache    bool
	Error    error
}

// IsFastForkPackage identifies if a package is a fork created only to make
// small changes for a pull request. An optional argument with the Github
// authentication can be informed to allow more checks per minute in Github API.
func IsFastForkPackage(p database.Package, auth *GithubAuth) (fastFork, cache bool, err error) {
	repository, cacheRepository, err := getGithubRepository(p.Path, auth)
	if err != nil {
		return false, cacheRepository, err
	}

	// if the repository is not a fork we don't need to check the commits
	if !repository.Fork {
		return false, cacheRepository, nil
	}

	commits, cacheCommits, err := getCommits(p.Path, auth)
	if err != nil {
		return false, cacheRepository && cacheCommits, err
	}

	forkLimitDate := repository.CreatedAt.Add(commitsPeriod)
	commitCounts := 0
	fastFork = true

	for _, commit := range commits {
		if commit.Commit.Author.Date.After(forkLimitDate) {
			fastFork = false
			break
		}

		if commit.Commit.Author.Date.After(repository.CreatedAt) {
			commitCounts++
		}
	}

	if commitCounts > commitsLimit {
		fastFork = false
	}

	return fastFork, cacheRepository && cacheCommits, nil
}

// AreFastForkPackages determinate if a package is a fast fork or not,
// but unlike IsFastForkPackage, it can process a list of packages
// concurrently. An optional argument with the Github authentication can be
// informed to allow more checks per minute in Github API (we will use token
// bucket strategy to don't exceed the rate limit).
func AreFastForkPackages(packages []database.Package, auth *GithubAuth) <-chan FastForkResponse {
	out := make(chan FastForkResponse, agents)

	go func() {
		var bucket *ratelimit.Bucket
		if auth == nil {
			bucket = ratelimit.NewBucket(RateLimit.FillInterval, RateLimit.Capacity)
		} else {
			bucket = ratelimit.NewBucket(RateLimit.AuthFillInterval, RateLimit.AuthCapacity)
		}

		var wg sync.WaitGroup
		wg.Add(agents)

		in := make(chan database.Package)

		for i := 0; i < agents; i++ {
			go func() {
				// if the go routine retrieve a response from cache, it can run again
				// without waiting for a token, as no hit was made in the Github API
				wait := true
				for p := range in {
					if wait {
						bucket.Wait(1)
					} else {
						wait = true
					}

					fastFork, cache, err := IsFastForkPackage(p, auth)
					out <- FastForkResponse{
						Path:     p.Path,
						FastFork: fastFork,
						Cache:    cache,
						Error:    err,
					}

					if cache {
						wait = false
					}
				}

				wg.Done()
			}()
		}

		for _, pkg := range packages {
			in <- pkg
		}

		close(in)
		wg.Wait()
		close(out)
	}()

	return out
}
<|endoftext|>


<commit_before>package rpcd

import (
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/imageserver"
)

func (t *srpcType) GetImage(conn *srpc.Conn,
	request imageserver.GetImageRequest,
	reply *imageserver.GetImageResponse) error {
	var response imageserver.GetImageResponse
	response.Image = t.imageDataBase.GetImage(request.ImageName)
	*reply = response
	return nil
}
<commit_msg>Add timeout support to ImageServer.GetImage SRPC method.<commit_after>package rpcd

import (
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/imageserver"
	"time"
)

func (t *srpcType) GetImage(conn *srpc.Conn,
	request imageserver.GetImageRequest,
	reply *imageserver.GetImageResponse) error {
	var response imageserver.GetImageResponse
	response.Image = t.imageDataBase.GetImage(request.ImageName)
	*reply = response
	if response.Image != nil || request.Timeout == 0 {
		return nil
	}
	// Image not found yet and willing to wait.
	addCh := t.imageDataBase.RegisterAddNotifier()
	defer func() {
		t.imageDataBase.UnregisterAddNotifier(addCh)
		select {
		case <-addCh:
		default:
		}
	}()
	timer := time.NewTimer(request.Timeout)
	for {
		select {
		case imageName := <-addCh:
			if imageName == request.ImageName {
				if !timer.Stop() {
					<-timer.C
				}
				response.Image = t.imageDataBase.GetImage(request.ImageName)
				*reply = response
				return nil
			}
		case <-timer.C:
			return nil
		}
	}
}
<|endoftext|>


<commit_before>package endless

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	// "github.com/fvbock/uds-go/introspect"
)

const (
	PRE_SIGNAL  = 0
	POST_SIGNAL = 1
)

var (
	runningServerReg     sync.Mutex
	runningServers       map[string]*endlessServer
	runningServersOrder  map[int]string
	runningServersForked bool

	DefaultReadTimeOut    time.Duration
	DefaultWriteTimeOut   time.Duration
	DefaultMaxHeaderBytes int

	isChild bool
)

func init() {
	flag.BoolVar(&isChild, "continue", false, "listen on open fd (after forking)")
	flag.Parse()

	runningServerReg = sync.Mutex{}
	runningServers = make(map[string]*endlessServer)
	runningServersOrder = make(map[int]string)
}

type endlessServer struct {
	http.Server
	EndlessListener  net.Listener
	tlsInnerListener *endlessListener
	wg               sync.WaitGroup
	sigChan          chan os.Signal
	isChild          bool
	SignalHooks      map[int]map[os.Signal][]func()
}

func NewServer(addr string, handler http.Handler) (srv *endlessServer) {
	srv = &endlessServer{
		wg:      sync.WaitGroup{},
		sigChan: make(chan os.Signal),
		isChild: isChild,
		SignalHooks: map[int]map[os.Signal][]func(){
			PRE_SIGNAL: map[os.Signal][]func(){
				syscall.SIGHUP:  []func(){},
				syscall.SIGUSR1: []func(){},
				syscall.SIGUSR2: []func(){},
				syscall.SIGINT:  []func(){},
				syscall.SIGTERM: []func(){},
				syscall.SIGTSTP: []func(){},
			},
			POST_SIGNAL: map[os.Signal][]func(){
				syscall.SIGHUP:  []func(){},
				syscall.SIGUSR1: []func(){},
				syscall.SIGUSR2: []func(){},
				syscall.SIGINT:  []func(){},
				syscall.SIGTERM: []func(){},
				syscall.SIGTSTP: []func(){},
			},
		},
	}

	srv.Server.Addr = addr
	srv.Server.ReadTimeout = DefaultReadTimeOut
	srv.Server.WriteTimeout = DefaultWriteTimeOut
	srv.Server.MaxHeaderBytes = DefaultMaxHeaderBytes // or 1 << 16? rather implent the hammerTime func
	srv.Server.Handler = handler

	runningServerReg.Lock()
	runningServersOrder[len(runningServers)] = addr
	runningServers[addr] = srv
	runningServerReg.Unlock()

	return
}

func ListenAndServe(addr string, handler http.Handler) error {
	server := NewServer(addr, handler)
	return server.ListenAndServe()
}

func (srv *endlessServer) ListenAndServe() (err error) {
	addr := srv.Addr
	if addr == "" {
		addr = ":http"
	}

	go srv.handleSignals()

	l, err := srv.getListener(addr)
	if err != nil {
		log.Println(err)
		return
	}

	srv.EndlessListener = newEndlessListener(l, srv)

	if srv.isChild {
		syscall.Kill(syscall.Getppid(), syscall.SIGTERM)
	}

	log.Println(syscall.Getpid(), srv.Addr)
	return srv.Serve()
}

func (srv *endlessServer) Serve() (err error) {
	err = srv.Server.Serve(srv.EndlessListener)
	log.Println(syscall.Getpid(), "Waiting for connections to finish...")
	srv.wg.Wait()
	return
}

func ListenAndServeTLS(addr string, certFile string, keyFile string, handler http.Handler) error {
	server := NewServer(addr, handler)
	return server.ListenAndServeTLS(certFile, keyFile)
}

func (srv *endlessServer) ListenAndServeTLS(certFile, keyFile string) (err error) {
	addr := srv.Addr
	if addr == "" {
		addr = ":https"
	}

	config := &tls.Config{}
	if srv.TLSConfig != nil {
		*config = *srv.TLSConfig
	}
	if config.NextProtos == nil {
		config.NextProtos = []string{"http/1.1"}
	}

	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return
	}

	go srv.handleSignals()

	l, err := srv.getListener(addr)
	if err != nil {
		log.Println(err)
		return
	}

	srv.tlsInnerListener = newEndlessListener(l, srv)
	srv.EndlessListener = tls.NewListener(srv.tlsInnerListener, config)

	if srv.isChild {
		syscall.Kill(syscall.Getppid(), syscall.SIGTERM)
	}

	log.Println(syscall.Getpid(), srv.Addr)
	return srv.Serve()
}

func (srv *endlessServer) getListener(laddr string) (l net.Listener, err error) {
	if srv.isChild {
		var ptrOffset uint = 0
		for i, addr := range runningServersOrder {
			if addr == laddr {
				ptrOffset = uint(i)
				break
			}
		}
		f := os.NewFile(uintptr(3+ptrOffset), "")
		l, err = net.FileListener(f)
		if err != nil {
			err = fmt.Errorf("net.FileListener error:", err)
			return
		}
	} else {
		l, err = net.Listen("tcp", laddr)
		if err != nil {
			err = fmt.Errorf("net.Listen error:", err)
			return
		}
	}
	return
}

func (srv *endlessServer) handleSignals() {
	var sig os.Signal

	signal.Notify(
		srv.sigChan,
		syscall.SIGHUP,
		syscall.SIGUSR1,
		syscall.SIGUSR2,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGTSTP,
	)

	pid := syscall.Getpid()
	for {
		sig = <-srv.sigChan
		srv.signalHooks(PRE_SIGNAL, sig)
		switch sig {
		case syscall.SIGHUP:
			log.Println(pid, "Received SIGHUP. forking.")
			err := srv.fork()
			if err != nil {
				log.Println("Fork err:", err)
			}
		case syscall.SIGUSR1:
			log.Println(pid, "Received SIGUSR1.")
		case syscall.SIGUSR2:
			log.Println(pid, "Received SIGUSR2.")
		case syscall.SIGINT:
			log.Println(pid, "Received SIGINT.")
			srv.shutdown()
		case syscall.SIGTERM:
			log.Println(pid, "Received SIGTERM.")
			srv.shutdown()
		case syscall.SIGTSTP:
			log.Println(pid, "Received SIGTSTP.")
		default:
			log.Printf("Received %v: nothing i care about...\n", sig)
		}
		srv.signalHooks(POST_SIGNAL, sig)
	}
}

func (srv *endlessServer) signalHooks(ppFlag int, sig os.Signal) {
	if _, notSet := srv.SignalHooks[ppFlag][sig]; !notSet {
		return
	}
	for _, f := range srv.SignalHooks[ppFlag][sig] {
		f()
	}
	return
}

func (srv *endlessServer) shutdown() {
	err := srv.EndlessListener.Close()
	if err != nil {
		log.Println(syscall.Getpid(), "srv.EndlessListener.Close() error:", err)
	} else {
		log.Println(syscall.Getpid(), srv.EndlessListener.Addr(), "srv.EndlessListener closed.")
	}
}

// /* TODO: add this
// hammerTime forces the server to shutdown in a given timeout - whether it
// finished outstanding requests or not. if Read/WriteTimeout are not set or the
// max header size is 0 a connection could hang...
// */
// func (srv *endlessServer) hammerTime(d time.Duration) (err error) {
// 	log.Println("[STOP - HAMMER TIME] Forcefully shutting down parent.")
// 	return
// }

func (srv *endlessServer) fork() (err error) {
	// only one server isntance should fork!
	runningServerReg.Lock()
	defer runningServerReg.Unlock()
	if runningServersForked {
		return
	}
	runningServersForked = true

	var files []*os.File
	// get the accessor socket fds for _all_ server instances
	for _, srvPtr := range runningServers {
		// introspect.PrintTypeDump(srvPtr.EndlessListener)
		switch srvPtr.EndlessListener.(type) {
		case *endlessListener:
			// log.Println("normal listener")
			files = append(files, srvPtr.EndlessListener.(*endlessListener).File()) // returns a dup(2) - FD_CLOEXEC flag *not* set
		default:
			// log.Println("tls listener")
			files = append(files, srvPtr.tlsInnerListener.File()) // returns a dup(2) - FD_CLOEXEC flag *not* set
		}
	}

	path := os.Args[0]
	args := []string{"-continue"}

	cmd := exec.Command(path, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.ExtraFiles = files

	err = cmd.Start()
	if err != nil {
		log.Fatalf("Restart: Failed to launch, error: %v", err)
	}

	return
}

type endlessListener struct {
	net.Listener
	stop    chan error
	stopped bool
	server  *endlessServer
}

func (el *endlessListener) Accept() (c net.Conn, err error) {
	tc, err := el.Listener.(*net.TCPListener).AcceptTCP()
	if err != nil {
		return
	}

	tc.SetKeepAlive(true)                  // see http.tcpKeepAliveListener
	tc.SetKeepAlivePeriod(3 * time.Minute) // see http.tcpKeepAliveListener

	c = endlessConn{
		Conn:   tc,
		server: el.server,
	}

	el.server.wg.Add(1)
	return
}

func newEndlessListener(l net.Listener, srv *endlessServer) (el *endlessListener) {
	el = &endlessListener{
		Listener: l,
		stop:     make(chan error),
		server:   srv,
	}

	go func() {
		_ = <-el.stop
		el.stopped = true
		el.stop <- el.Listener.Close()
	}()
	return
}

func (el *endlessListener) Close() error {
	if el.stopped {
		return syscall.EINVAL
	}
	el.stop <- nil
	return <-el.stop
}

func (el *endlessListener) File() *os.File {
	tl := el.Listener.(*net.TCPListener)
	fl, _ := tl.File()
	return fl
}

type endlessConn struct {
	net.Conn
	server *endlessServer
}

func (w endlessConn) Close() error {
	w.server.wg.Done()
	return w.Conn.Close()
}
<commit_msg>add hammerTime to force shutdown<commit_after>package endless

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	// "github.com/fvbock/uds-go/introspect"
)

const (
	PRE_SIGNAL  = 0
	POST_SIGNAL = 1
)

var (
	runningServerReg     sync.Mutex
	runningServers       map[string]*endlessServer
	runningServersOrder  map[int]string
	runningServersForked bool

	DefaultReadTimeOut    time.Duration
	DefaultWriteTimeOut   time.Duration
	DefaultMaxHeaderBytes int
	DefaultHammerTime     time.Duration

	isChild bool
)

const (
	STATE_INIT = iota
	STATE_RUNNING
	STATE_SHUTTING_DOWN
	STATE_TERMINATE
)

func init() {
	flag.BoolVar(&isChild, "continue", false, "listen on open fd (after forking)")
	flag.Parse()

	runningServerReg = sync.Mutex{}
	runningServers = make(map[string]*endlessServer)
	runningServersOrder = make(map[int]string)

	DefaultMaxHeaderBytes = 0 // use http.DefaultMaxHeaderBytes - which currently is 1 << 20 (1MB)

	// after a restart the parent will finish ongoing requests before
	// shutting down. set to a negative value to disable
	DefaultHammerTime = 60 * time.Second
}

type endlessServer struct {
	http.Server
	EndlessListener  net.Listener
	tlsInnerListener *endlessListener
	wg               sync.WaitGroup
	sigChan          chan os.Signal
	isChild          bool
	SignalHooks      map[int]map[os.Signal][]func()
	state            uint8
}

func NewServer(addr string, handler http.Handler) (srv *endlessServer) {
	srv = &endlessServer{
		wg:      sync.WaitGroup{},
		sigChan: make(chan os.Signal),
		isChild: isChild,
		SignalHooks: map[int]map[os.Signal][]func(){
			PRE_SIGNAL: map[os.Signal][]func(){
				syscall.SIGHUP:  []func(){},
				syscall.SIGUSR1: []func(){},
				syscall.SIGUSR2: []func(){},
				syscall.SIGINT:  []func(){},
				syscall.SIGTERM: []func(){},
				syscall.SIGTSTP: []func(){},
			},
			POST_SIGNAL: map[os.Signal][]func(){
				syscall.SIGHUP:  []func(){},
				syscall.SIGUSR1: []func(){},
				syscall.SIGUSR2: []func(){},
				syscall.SIGINT:  []func(){},
				syscall.SIGTERM: []func(){},
				syscall.SIGTSTP: []func(){},
			},
		},
		state: STATE_INIT,
	}

	srv.Server.Addr = addr
	srv.Server.ReadTimeout = DefaultReadTimeOut
	srv.Server.WriteTimeout = DefaultWriteTimeOut
	srv.Server.MaxHeaderBytes = DefaultMaxHeaderBytes
	srv.Server.Handler = handler

	runningServerReg.Lock()
	runningServersOrder[len(runningServers)] = addr
	runningServers[addr] = srv
	runningServerReg.Unlock()

	return
}

func (srv *endlessServer) Serve() (err error) {
	srv.state = STATE_RUNNING
	err = srv.Server.Serve(srv.EndlessListener)
	log.Println(syscall.Getpid(), "Waiting for connections to finish...")
	srv.wg.Wait()
	srv.state = STATE_TERMINATE
	return
}

func ListenAndServe(addr string, handler http.Handler) error {
	server := NewServer(addr, handler)
	return server.ListenAndServe()
}

func (srv *endlessServer) ListenAndServe() (err error) {
	addr := srv.Addr
	if addr == "" {
		addr = ":http"
	}

	go srv.handleSignals()

	l, err := srv.getListener(addr)
	if err != nil {
		log.Println(err)
		return
	}

	srv.EndlessListener = newEndlessListener(l, srv)

	if srv.isChild {
		syscall.Kill(syscall.Getppid(), syscall.SIGTERM)
	}

	log.Println(syscall.Getpid(), srv.Addr)
	return srv.Serve()
}

func ListenAndServeTLS(addr string, certFile string, keyFile string, handler http.Handler) error {
	server := NewServer(addr, handler)
	return server.ListenAndServeTLS(certFile, keyFile)
}

func (srv *endlessServer) ListenAndServeTLS(certFile, keyFile string) (err error) {
	addr := srv.Addr
	if addr == "" {
		addr = ":https"
	}

	config := &tls.Config{}
	if srv.TLSConfig != nil {
		*config = *srv.TLSConfig
	}
	if config.NextProtos == nil {
		config.NextProtos = []string{"http/1.1"}
	}

	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return
	}

	go srv.handleSignals()

	l, err := srv.getListener(addr)
	if err != nil {
		log.Println(err)
		return
	}

	srv.tlsInnerListener = newEndlessListener(l, srv)
	srv.EndlessListener = tls.NewListener(srv.tlsInnerListener, config)

	if srv.isChild {
		syscall.Kill(syscall.Getppid(), syscall.SIGTERM)
	}

	log.Println(syscall.Getpid(), srv.Addr)
	return srv.Serve()
}

func (srv *endlessServer) getListener(laddr string) (l net.Listener, err error) {
	if srv.isChild {
		var ptrOffset uint = 0
		for i, addr := range runningServersOrder {
			if addr == laddr {
				ptrOffset = uint(i)
				break
			}
		}
		f := os.NewFile(uintptr(3+ptrOffset), "")
		l, err = net.FileListener(f)
		if err != nil {
			err = fmt.Errorf("net.FileListener error: %v", err)
			return
		}
	} else {
		l, err = net.Listen("tcp", laddr)
		if err != nil {
			err = fmt.Errorf("net.Listen error: %v", err)
			return
		}
	}
	return
}

func (srv *endlessServer) handleSignals() {
	var sig os.Signal

	signal.Notify(
		srv.sigChan,
		syscall.SIGHUP,
		syscall.SIGUSR1,
		syscall.SIGUSR2,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGTSTP,
	)

	pid := syscall.Getpid()
	for {
		sig = <-srv.sigChan
		srv.signalHooks(PRE_SIGNAL, sig)
		switch sig {
		case syscall.SIGHUP:
			log.Println(pid, "Received SIGHUP. forking.")
			err := srv.fork()
			if err != nil {
				log.Println("Fork err:", err)
			}
		case syscall.SIGUSR1:
			log.Println(pid, "Received SIGUSR1.")
		case syscall.SIGUSR2:
			log.Println(pid, "Received SIGUSR2.")
			srv.hammerTime(0 * time.Second)
		case syscall.SIGINT:
			log.Println(pid, "Received SIGINT.")
			srv.shutdown()
		case syscall.SIGTERM:
			log.Println(pid, "Received SIGTERM.")
			srv.shutdown()
		case syscall.SIGTSTP:
			log.Println(pid, "Received SIGTSTP.")
		default:
			log.Printf("Received %v: nothing i care about...\n", sig)
		}
		srv.signalHooks(POST_SIGNAL, sig)
	}
}

func (srv *endlessServer) signalHooks(ppFlag int, sig os.Signal) {
	if _, notSet := srv.SignalHooks[ppFlag][sig]; !notSet {
		return
	}
	for _, f := range srv.SignalHooks[ppFlag][sig] {
		f()
	}
	return
}

func (srv *endlessServer) shutdown() {
	if srv.state != STATE_RUNNING {
		return
	}
	srv.state = STATE_SHUTTING_DOWN
	if DefaultHammerTime >= 0 {
		go srv.hammerTime(DefaultHammerTime)
	}
	err := srv.EndlessListener.Close()
	if err != nil {
		log.Println(syscall.Getpid(), "Listener.Close() error:", err)
	} else {
		log.Println(syscall.Getpid(), srv.EndlessListener.Addr(), "Listener closed.")
	}
}

/*
hammerTime forces the server to shutdown in a given timeout - whether it
finished outstanding requests or not. if Read/WriteTimeout are not set or the
max header size is very big a connection could hang...

srv.Serve() will not return until all connections are served. this will
unblock the srv.wg.Wait() in Serve() thus causing ListenAndServe(TLS) to
return.
*/
func (srv *endlessServer) hammerTime(d time.Duration) {
	defer func() {
		// we are calling srv.wg.Done() until it panics which means we called
		// Done() when the counter was already at 0 and we're done.
		// (and thus Serve() will return and the parent will exit)
		if r := recover(); r != nil {
			log.Println("WaitGroup at 0", r)
		}
	}()
	if srv.state != STATE_SHUTTING_DOWN {
		return
	}
	time.Sleep(d)
	log.Println("[STOP - Hammer Time] Forcefully shutting down parent")
	for {
		if srv.state == STATE_TERMINATE {
			break
		}
		srv.wg.Done()
	}
}

func (srv *endlessServer) fork() (err error) {
	// only one server isntance should fork!
	runningServerReg.Lock()
	defer runningServerReg.Unlock()
	if runningServersForked {
		return
	}
	runningServersForked = true

	var files []*os.File
	// get the accessor socket fds for _all_ server instances
	for _, srvPtr := range runningServers {
		// introspect.PrintTypeDump(srvPtr.EndlessListener)
		switch srvPtr.EndlessListener.(type) {
		case *endlessListener:
			// log.Println("normal listener")
			files = append(files, srvPtr.EndlessListener.(*endlessListener).File()) // returns a dup(2) - FD_CLOEXEC flag *not* set
		default:
			// log.Println("tls listener")
			files = append(files, srvPtr.tlsInnerListener.File()) // returns a dup(2) - FD_CLOEXEC flag *not* set
		}
	}

	path := os.Args[0]
	args := []string{"-continue"}

	cmd := exec.Command(path, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.ExtraFiles = files

	err = cmd.Start()
	if err != nil {
		log.Fatalf("Restart: Failed to launch, error: %v", err)
	}

	return
}

type endlessListener struct {
	net.Listener
	stop    chan error
	stopped bool
	server  *endlessServer
}

func (el *endlessListener) Accept() (c net.Conn, err error) {
	tc, err := el.Listener.(*net.TCPListener).AcceptTCP()
	if err != nil {
		return
	}

	tc.SetKeepAlive(true)                  // see http.tcpKeepAliveListener
	tc.SetKeepAlivePeriod(3 * time.Minute) // see http.tcpKeepAliveListener

	c = endlessConn{
		Conn:   tc,
		server: el.server,
	}

	el.server.wg.Add(1)
	return
}

func newEndlessListener(l net.Listener, srv *endlessServer) (el *endlessListener) {
	el = &endlessListener{
		Listener: l,
		stop:     make(chan error),
		server:   srv,
	}

	go func() {
		_ = <-el.stop
		el.stopped = true
		el.stop <- el.Listener.Close()
	}()
	return
}

func (el *endlessListener) Close() error {
	if el.stopped {
		return syscall.EINVAL
	}
	el.stop <- nil
	return <-el.stop
}

func (el *endlessListener) File() *os.File {
	tl := el.Listener.(*net.TCPListener)
	fl, _ := tl.File()
	return fl
}

type endlessConn struct {
	net.Conn
	server *endlessServer
}

func (w endlessConn) Close() error {
	w.server.wg.Done()
	return w.Conn.Close()
}
<|endoftext|>