<commit_before>package atlas

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
)

const atlasURL = "https://atlas.hashicorp.com"
const userAgent = "HashiCorp Atlas Go Client v1"

// If this is set to true, verbose debug data will be output
var Debug = false

// ErrAuth is the error returned if a 401 is returned by an API request.
var ErrAuth = fmt.Errorf("authentication failed")

// ErrNotFound is the error returned if a 404 is returned by an API request.
var ErrNotFound = fmt.Errorf("resource not found")

// RailsError represents an error that was returned from the Rails server.
type RailsError struct {
	Errors []string `json:"errors"`
}

// Error collects all of the errors in the RailsError and returns a comma-
// separated list of the errors that were returned from the server.
func (re *RailsError) Error() string {
	return strings.Join(re.Errors, ", ")
}

// Client represents a single connection to a Atlas API endpoint.
type Client struct {
	// URL is the full endpoint address to the Atlas server including the
	// protocol, port, and path.
	URL *url.URL

	// Token is the Atlas authentication token
	Token string

	// HTTPClient is the underlying http client with which to make requests.
	HTTPClient *http.Client
}

// DefaultClient returns a client that connects to the Atlas API.
func DefaultClient() *Client {
	client, err := NewClient(atlasURL)
	if err != nil {
		panic(err)
	}

	return client
}

// NewClient creates a new Atlas Client from the given URL (as a string). If
// the URL cannot be parsed, an error is returned. The HTTPClient is set to
// http.DefaultClient, but this can be changed programmatically by setting
// client.HTTPClient. The user can also programmatically set the URL as a
// *url.URL.
func NewClient(urlString string) (*Client, error) {
	if len(urlString) == 0 {
		return nil, fmt.Errorf("client: missing url")
	}

	parsedURL, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	token := os.Getenv("ATLAS_TOKEN")
	if token != "" {
		log.Printf("[DEBUG] using ATLAS_TOKEN (%s)", maskString(token))
	}

	client := &Client{
		URL:   parsedURL,
		Token: token,
	}

	if err := client.init(); err != nil {
		return nil, err
	}

	return client, nil
}

// init() sets defaults on the client.
func (c *Client) init() error {
	c.HTTPClient = http.DefaultClient
	return nil
}

// RequestOptions is the list of options to pass to the request.
type RequestOptions struct {
	// Params is a map of key-value pairs that will be added to the Request.
	Params map[string]string

	// Headers is a map of key-value pairs that will be added to the Request.
	Headers map[string]string

	// Body is an io.Reader object that will be streamed or uploaded with the
	// Request. BodyLength is the final size of the Body.
	Body       io.Reader
	BodyLength int64
}

// Request creates a new HTTP request using the given verb and sub path.
func (c *Client) Request(verb, spath string, ro *RequestOptions) (*http.Request, error) {
	log.Printf("[INFO] request: %s %s", verb, spath)

	// Ensure we have a RequestOptions struct (passing nil is an acceptable)
	if ro == nil {
		ro = new(RequestOptions)
	}

	// Create a new URL with the appended path
	u := *c.URL
	u.Path = path.Join(c.URL.Path, spath)

	// Add the token and other params
	if c.Token != "" {
		log.Printf("[DEBUG] request: appending token (%s)", maskString(c.Token))
		if ro.Params == nil {
			ro.Params = make(map[string]string)
		}

		ro.Params["access_token"] = c.Token
	}

	return c.rawRequest(verb, &u, ro)
}

func (c *Client) putFile(rawURL string, r io.Reader, size int64) error {
	log.Printf("[INFO] putting file: %s", rawURL)

	url, err := url.Parse(rawURL)
	if err != nil {
		return err
	}

	request, err := c.rawRequest("PUT", url, &RequestOptions{
		Body:       r,
		BodyLength: size,
	})
	if err != nil {
		return err
	}

	if _, err := checkResp(c.HTTPClient.Do(request)); err != nil {
		return err
	}

	return nil
}

// rawRequest accepts a verb, URL, and RequestOptions struct and returns the
// constructed http.Request and any errors that occurred
func (c *Client) rawRequest(verb string, u *url.URL, ro *RequestOptions) (*http.Request, error) {
	if verb == "" {
		return nil, fmt.Errorf("client: missing verb")
	}

	if u == nil {
		return nil, fmt.Errorf("client: missing URL.url")
	}

	if ro == nil {
		return nil, fmt.Errorf("client: missing RequestOptions")
	}

	// Add the token and other params
	var params = make(url.Values)
	for k, v := range ro.Params {
		params.Add(k, v)
	}
	u.RawQuery = params.Encode()

	// Create the request object
	request, err := http.NewRequest(verb, u.String(), ro.Body)
	if err != nil {
		return nil, err
	}

	// Set the User-Agent
	request.Header.Set("User-Agent", userAgent)

	// Add any headers
	for k, v := range ro.Headers {
		request.Header.Add(k, v)
	}

	// Add content-length if we have it
	if ro.BodyLength > 0 {
		request.ContentLength = ro.BodyLength
	}

	log.Printf("[DEBUG] raw request: %#v", request)

	return request, nil
}

// checkResp wraps http.Client.Do() and verifies that the request was
// successful. A non-200 request returns an error formatted to included any
// validation problems or otherwise.
func checkResp(resp *http.Response, err error) (*http.Response, error) {
	// If the err is already there, there was an error higher up the chain, so
	// just return that
	if err != nil {
		return resp, err
	}

	log.Printf("[INFO] response: %d (%s)", resp.StatusCode, resp.Status)
	if Debug {
		var buf bytes.Buffer
		if _, err := io.Copy(&buf, resp.Body); err != nil {
			log.Printf("[ERR] response: error copying response body")
		} else {
			log.Printf("[DEBUG] response: %s", buf.String())

			// We are going to reset the response body, so we need to close the old
			// one or else it will leak.
			resp.Body.Close()
			resp.Body = &bytesReadCloser{&buf}
		}
	}

	switch resp.StatusCode {
	case 200:
		return resp, nil
	case 201:
		return resp, nil
	case 202:
		return resp, nil
	case 204:
		return resp, nil
	case 400:
		return nil, parseErr(resp)
	case 401:
		return nil, ErrAuth
	case 404:
		return nil, ErrNotFound
	case 422:
		return nil, parseErr(resp)
	default:
		return nil, fmt.Errorf("client: %s", resp.Status)
	}
}

// parseErr is used to take an error JSON response and return a single string
// for use in error messages.
func parseErr(r *http.Response) error {
	re := &RailsError{}

	if err := decodeJSON(r, &re); err != nil {
		return fmt.Errorf("error decoding JSON body: %s", err)
	}

	return re
}

// decodeJSON is used to JSON decode a body into an interface.
func decodeJSON(resp *http.Response, out interface{}) error {
	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	return dec.Decode(out)
}

// bytesReadCloser is a simple wrapper around a bytes buffer that implements
// Close as a noop.
type bytesReadCloser struct {
	*bytes.Buffer
}

func (nrc *bytesReadCloser) Close() error {
	// we don't actually have to do anything here, since the buffer is just some
	// data in memory  and the error is initialized to no-error
	return nil
}
<commit_msg>Remove Debug flag<commit_after>package atlas

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
)

const atlasURL = "https://atlas.hashicorp.com"
const userAgent = "HashiCorp Atlas Go Client v1"

// ErrAuth is the error returned if a 401 is returned by an API request.
var ErrAuth = fmt.Errorf("authentication failed")

// ErrNotFound is the error returned if a 404 is returned by an API request.
var ErrNotFound = fmt.Errorf("resource not found")

// RailsError represents an error that was returned from the Rails server.
type RailsError struct {
	Errors []string `json:"errors"`
}

// Error collects all of the errors in the RailsError and returns a comma-
// separated list of the errors that were returned from the server.
func (re *RailsError) Error() string {
	return strings.Join(re.Errors, ", ")
}

// Client represents a single connection to a Atlas API endpoint.
type Client struct {
	// URL is the full endpoint address to the Atlas server including the
	// protocol, port, and path.
	URL *url.URL

	// Token is the Atlas authentication token
	Token string

	// HTTPClient is the underlying http client with which to make requests.
	HTTPClient *http.Client
}

// DefaultClient returns a client that connects to the Atlas API.
func DefaultClient() *Client {
	client, err := NewClient(atlasURL)
	if err != nil {
		panic(err)
	}

	return client
}

// NewClient creates a new Atlas Client from the given URL (as a string). If
// the URL cannot be parsed, an error is returned. The HTTPClient is set to
// http.DefaultClient, but this can be changed programmatically by setting
// client.HTTPClient. The user can also programmatically set the URL as a
// *url.URL.
func NewClient(urlString string) (*Client, error) {
	if len(urlString) == 0 {
		return nil, fmt.Errorf("client: missing url")
	}

	parsedURL, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	token := os.Getenv("ATLAS_TOKEN")
	if token != "" {
		log.Printf("[DEBUG] using ATLAS_TOKEN (%s)", maskString(token))
	}

	client := &Client{
		URL:   parsedURL,
		Token: token,
	}

	if err := client.init(); err != nil {
		return nil, err
	}

	return client, nil
}

// init() sets defaults on the client.
func (c *Client) init() error {
	c.HTTPClient = http.DefaultClient
	return nil
}

// RequestOptions is the list of options to pass to the request.
type RequestOptions struct {
	// Params is a map of key-value pairs that will be added to the Request.
	Params map[string]string

	// Headers is a map of key-value pairs that will be added to the Request.
	Headers map[string]string

	// Body is an io.Reader object that will be streamed or uploaded with the
	// Request. BodyLength is the final size of the Body.
	Body       io.Reader
	BodyLength int64
}

// Request creates a new HTTP request using the given verb and sub path.
func (c *Client) Request(verb, spath string, ro *RequestOptions) (*http.Request, error) {
	log.Printf("[INFO] request: %s %s", verb, spath)

	// Ensure we have a RequestOptions struct (passing nil is an acceptable)
	if ro == nil {
		ro = new(RequestOptions)
	}

	// Create a new URL with the appended path
	u := *c.URL
	u.Path = path.Join(c.URL.Path, spath)

	// Add the token and other params
	if c.Token != "" {
		log.Printf("[DEBUG] request: appending token (%s)", maskString(c.Token))
		if ro.Params == nil {
			ro.Params = make(map[string]string)
		}

		ro.Params["access_token"] = c.Token
	}

	return c.rawRequest(verb, &u, ro)
}

func (c *Client) putFile(rawURL string, r io.Reader, size int64) error {
	log.Printf("[INFO] putting file: %s", rawURL)

	url, err := url.Parse(rawURL)
	if err != nil {
		return err
	}

	request, err := c.rawRequest("PUT", url, &RequestOptions{
		Body:       r,
		BodyLength: size,
	})
	if err != nil {
		return err
	}

	if _, err := checkResp(c.HTTPClient.Do(request)); err != nil {
		return err
	}

	return nil
}

// rawRequest accepts a verb, URL, and RequestOptions struct and returns the
// constructed http.Request and any errors that occurred
func (c *Client) rawRequest(verb string, u *url.URL, ro *RequestOptions) (*http.Request, error) {
	if verb == "" {
		return nil, fmt.Errorf("client: missing verb")
	}

	if u == nil {
		return nil, fmt.Errorf("client: missing URL.url")
	}

	if ro == nil {
		return nil, fmt.Errorf("client: missing RequestOptions")
	}

	// Add the token and other params
	var params = make(url.Values)
	for k, v := range ro.Params {
		params.Add(k, v)
	}
	u.RawQuery = params.Encode()

	// Create the request object
	request, err := http.NewRequest(verb, u.String(), ro.Body)
	if err != nil {
		return nil, err
	}

	// Set the User-Agent
	request.Header.Set("User-Agent", userAgent)

	// Add any headers
	for k, v := range ro.Headers {
		request.Header.Add(k, v)
	}

	// Add content-length if we have it
	if ro.BodyLength > 0 {
		request.ContentLength = ro.BodyLength
	}

	log.Printf("[DEBUG] raw request: %#v", request)

	return request, nil
}

// checkResp wraps http.Client.Do() and verifies that the request was
// successful. A non-200 request returns an error formatted to included any
// validation problems or otherwise.
func checkResp(resp *http.Response, err error) (*http.Response, error) {
	// If the err is already there, there was an error higher up the chain, so
	// just return that
	if err != nil {
		return resp, err
	}

	log.Printf("[INFO] response: %d (%s)", resp.StatusCode, resp.Status)
	var buf bytes.Buffer
	if _, err := io.Copy(&buf, resp.Body); err != nil {
		log.Printf("[ERR] response: error copying response body")
	} else {
		log.Printf("[DEBUG] response: %s", buf.String())

		// We are going to reset the response body, so we need to close the old
		// one or else it will leak.
		resp.Body.Close()
		resp.Body = &bytesReadCloser{&buf}
	}

	switch resp.StatusCode {
	case 200:
		return resp, nil
	case 201:
		return resp, nil
	case 202:
		return resp, nil
	case 204:
		return resp, nil
	case 400:
		return nil, parseErr(resp)
	case 401:
		return nil, ErrAuth
	case 404:
		return nil, ErrNotFound
	case 422:
		return nil, parseErr(resp)
	default:
		return nil, fmt.Errorf("client: %s", resp.Status)
	}
}

// parseErr is used to take an error JSON response and return a single string
// for use in error messages.
func parseErr(r *http.Response) error {
	re := &RailsError{}

	if err := decodeJSON(r, &re); err != nil {
		return fmt.Errorf("error decoding JSON body: %s", err)
	}

	return re
}

// decodeJSON is used to JSON decode a body into an interface.
func decodeJSON(resp *http.Response, out interface{}) error {
	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	return dec.Decode(out)
}

// bytesReadCloser is a simple wrapper around a bytes buffer that implements
// Close as a noop.
type bytesReadCloser struct {
	*bytes.Buffer
}

func (nrc *bytesReadCloser) Close() error {
	// we don't actually have to do anything here, since the buffer is just some
	// data in memory  and the error is initialized to no-error
	return nil
}
<|endoftext|>


<commit_before>package openstack

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/images"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// Find image ID from image name
func (h *HatcheryOpenstack) imageID(ctx context.Context, img string) (string, error) {
	for _, i := range h.getImages(ctx) {
		if i.Name == img {
			return i.ID, nil
		}
	}
	return "", sdk.WithStack(fmt.Errorf("image '%s' not found", img))
}

// Find flavor ID from flavor name
func (h *HatcheryOpenstack) flavor(flavor string) (flavors.Flavor, error) {
	for i := range h.flavors {
		if h.flavors[i].Name == flavor {
			return h.flavors[i], nil
		}
	}
	return flavors.Flavor{}, sdk.WithStack(fmt.Errorf("flavor '%s' not found", flavor))
}

// Find flavor ID from flavor name
func (h *HatcheryOpenstack) getSmallerFlavorThan(flavor flavors.Flavor) flavors.Flavor {
	var smaller *flavors.Flavor
	for i := range h.flavors {
		// If the flavor is not the given one and need less CPUs its
		if h.flavors[i].ID != flavor.ID && h.flavors[i].VCPUs < flavor.VCPUs && (smaller == nil || smaller.VCPUs < h.flavors[i].VCPUs) {
			smaller = &h.flavors[i]
		}
	}
	if smaller == nil {
		return flavor
	}
	return *smaller
}

//This a embedded cache for images list
var limages = struct {
	mu   sync.RWMutex
	list []images.Image
}{
	mu:   sync.RWMutex{},
	list: []images.Image{},
}

func (h *HatcheryOpenstack) getImages(ctx context.Context) []images.Image {
	t := time.Now()
	defer log.Debug("getImages(): %fs", time.Since(t).Seconds())

	limages.mu.RLock()
	nbImages := len(limages.list)
	limages.mu.RUnlock()

	if nbImages == 0 {
		all, err := images.ListDetail(h.openstackClient, nil).AllPages()
		if err != nil {
			log.Error(ctx, "getImages> error on listDetail: %s", err)
			return limages.list
		}
		imgs, err := images.ExtractImages(all)
		if err != nil {
			log.Error(ctx, "getImages> error on images.ExtractImages: %s", err)
			return limages.list
		}

		activeImages := []images.Image{}
		for i := range imgs {
			log.Debug("getImages> image %s status %s progress %d all:%+v", imgs[i].Name, imgs[i].Status, imgs[i].Progress, imgs[i])
			if imgs[i].Status == "ACTIVE" {
				log.Debug("getImages> add %s to activeImages", imgs[i].Name)
				activeImages = append(activeImages, imgs[i])
			}
		}

		limages.mu.Lock()
		limages.list = activeImages
		limages.mu.Unlock()
		//Remove data from the cache after 2 seconds
		go func() {
			time.Sleep(10 * time.Minute)
			h.resetImagesCache()
		}()
	}

	return limages.list
}

func (h *HatcheryOpenstack) resetImagesCache() {
	limages.mu.Lock()
	limages.list = []images.Image{}
	limages.mu.Unlock()
}

//This a embedded cache for servers list
var lservers = struct {
	mu   sync.RWMutex
	list []servers.Server
}{
	mu:   sync.RWMutex{},
	list: []servers.Server{},
}

func (h *HatcheryOpenstack) getServers(ctx context.Context) []servers.Server {
	t := time.Now()
	defer log.Debug("getServers() : %fs", time.Since(t).Seconds())

	lservers.mu.RLock()
	nbServers := len(lservers.list)
	lservers.mu.RUnlock()

	if nbServers == 0 {
		all, err := servers.List(h.openstackClient, nil).AllPages()
		if err != nil {
			log.Error(ctx, "getServers> error on servers.List: %s", err)
			return lservers.list
		}
		serverList, err := servers.ExtractServers(all)
		if err != nil {
			log.Error(ctx, "getServers> error on servers.ExtractServers: %s", err)
			return lservers.list
		}

		srvs := []servers.Server{}
		for _, s := range serverList {
			_, worker := s.Metadata["worker"]
			if !worker {
				continue
			}
			workerHatcheryName := s.Metadata["hatchery_name"]
			if workerHatcheryName == "" || workerHatcheryName != h.Name() {
				continue
			}
			srvs = append(srvs, s)
		}

		lservers.mu.Lock()
		lservers.list = srvs
		lservers.mu.Unlock()
		//Remove data from the cache after 2 seconds
		go func() {
			time.Sleep(2 * time.Second)
			lservers.mu.Lock()
			lservers.list = []servers.Server{}
			lservers.mu.Unlock()
		}()
	}

	return lservers.list
}

func (h *HatcheryOpenstack) getConsoleLog(s servers.Server) (string, error) {
	result := servers.ShowConsoleOutput(h.openstackClient, s.ID, servers.ShowConsoleOutputOpts{})
	info, err := result.Extract()
	return info, sdk.WrapError(err, "unable to get console log from %s", s.ID)
}
<commit_msg>fix(hatchery/openstack): add retry on server list (#5492)<commit_after>package openstack

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/images"
	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// Find image ID from image name
func (h *HatcheryOpenstack) imageID(ctx context.Context, img string) (string, error) {
	for _, i := range h.getImages(ctx) {
		if i.Name == img {
			return i.ID, nil
		}
	}
	return "", sdk.WithStack(fmt.Errorf("image '%s' not found", img))
}

// Find flavor ID from flavor name
func (h *HatcheryOpenstack) flavor(flavor string) (flavors.Flavor, error) {
	for i := range h.flavors {
		if h.flavors[i].Name == flavor {
			return h.flavors[i], nil
		}
	}
	return flavors.Flavor{}, sdk.WithStack(fmt.Errorf("flavor '%s' not found", flavor))
}

// Find flavor ID from flavor name
func (h *HatcheryOpenstack) getSmallerFlavorThan(flavor flavors.Flavor) flavors.Flavor {
	var smaller *flavors.Flavor
	for i := range h.flavors {
		// If the flavor is not the given one and need less CPUs its
		if h.flavors[i].ID != flavor.ID && h.flavors[i].VCPUs < flavor.VCPUs && (smaller == nil || smaller.VCPUs < h.flavors[i].VCPUs) {
			smaller = &h.flavors[i]
		}
	}
	if smaller == nil {
		return flavor
	}
	return *smaller
}

//This a embedded cache for images list
var limages = struct {
	mu   sync.RWMutex
	list []images.Image
}{
	mu:   sync.RWMutex{},
	list: []images.Image{},
}

func (h *HatcheryOpenstack) getImages(ctx context.Context) []images.Image {
	t := time.Now()
	defer log.Debug("getImages(): %fs", time.Since(t).Seconds())

	limages.mu.RLock()
	nbImages := len(limages.list)
	limages.mu.RUnlock()

	if nbImages == 0 {
		all, err := images.ListDetail(h.openstackClient, nil).AllPages()
		if err != nil {
			log.Error(ctx, "getImages> error on listDetail: %s", err)
			return limages.list
		}
		imgs, err := images.ExtractImages(all)
		if err != nil {
			log.Error(ctx, "getImages> error on images.ExtractImages: %s", err)
			return limages.list
		}

		activeImages := []images.Image{}
		for i := range imgs {
			log.Debug("getImages> image %s status %s progress %d all:%+v", imgs[i].Name, imgs[i].Status, imgs[i].Progress, imgs[i])
			if imgs[i].Status == "ACTIVE" {
				log.Debug("getImages> add %s to activeImages", imgs[i].Name)
				activeImages = append(activeImages, imgs[i])
			}
		}

		limages.mu.Lock()
		limages.list = activeImages
		limages.mu.Unlock()
		//Remove data from the cache after 2 seconds
		go func() {
			time.Sleep(10 * time.Minute)
			h.resetImagesCache()
		}()
	}

	return limages.list
}

func (h *HatcheryOpenstack) resetImagesCache() {
	limages.mu.Lock()
	limages.list = []images.Image{}
	limages.mu.Unlock()
}

//This a embedded cache for servers list
var lservers = struct {
	mu   sync.RWMutex
	list []servers.Server
}{
	mu:   sync.RWMutex{},
	list: []servers.Server{},
}

func (h *HatcheryOpenstack) getServers(ctx context.Context) []servers.Server {
	t := time.Now()
	defer log.Debug("getServers() : %fs", time.Since(t).Seconds())

	lservers.mu.RLock()
	nbServers := len(lservers.list)
	lservers.mu.RUnlock()

	if nbServers == 0 {
		var serverList []servers.Server
		var isOk bool
		for i := 0; i <= 5; i++ {
			all, err := servers.List(h.openstackClient, nil).AllPages()
			if err != nil {
				log.Error(ctx, "getServers> error on servers.List: %s", err)
				continue
			}
			serverList, err = servers.ExtractServers(all)
			if err != nil {
				log.Error(ctx, "getServers> error on servers.ExtractServers: %s", err)
				continue
			}
			isOk = true
			break
		}
		if !isOk {
			return lservers.list
		}

		srvs := []servers.Server{}
		for _, s := range serverList {
			_, worker := s.Metadata["worker"]
			if !worker {
				continue
			}
			workerHatcheryName := s.Metadata["hatchery_name"]
			if workerHatcheryName == "" || workerHatcheryName != h.Name() {
				continue
			}
			srvs = append(srvs, s)
		}

		lservers.mu.Lock()
		lservers.list = srvs
		lservers.mu.Unlock()
		//Remove data from the cache after 2 seconds
		go func() {
			time.Sleep(2 * time.Second)
			lservers.mu.Lock()
			lservers.list = []servers.Server{}
			lservers.mu.Unlock()
		}()
	}

	return lservers.list
}

func (h *HatcheryOpenstack) getConsoleLog(s servers.Server) (string, error) {
	result := servers.ShowConsoleOutput(h.openstackClient, s.ID, servers.ShowConsoleOutputOpts{})
	info, err := result.Extract()
	return info, sdk.WrapError(err, "unable to get console log from %s", s.ID)
}
<|endoftext|>


<commit_before>package cssminify

import (
	"fmt"
)

func Minify(blocks []Block) {
	for _, block := range blocks {
		showSelectors(string(block.selector))
		fmt.Print("{")
		showPropVals(block.pairs)
		fmt.Print("}")
	}
}

func showSelectors(selector string) {
}

func showPropVals(pairs []Pair) {
}
<commit_msg>It finally minifies<commit_after>package cssminify

import (
	"fmt"
	"regexp"
	"strings"
)

func Minify(blocks []Block) {
	for _, block := range blocks {
		showSelectors(string(block.selector))
		fmt.Print("{")
		showPropVals(block.pairs)
		fmt.Print("}")
	}
}

func showSelectors(selector string) {
	selectors := strings.Split(selector, ",")
	for i, sel := range selectors {
		fmt.Printf("%s", minifySelector(sel))
		if i != len(selectors)-1 {
			fmt.Print(",")
		}
	}
}

func minifySelector(sel string) string {
	return cleanSpaces(sel)
}

func showPropVals(pairs []Pair) {
	for i, pair := range pairs {
		fmt.Printf("%s:%s", minifyProp(string(pair.property)), minifyVal(string(pair.value)))

		// Let's gain some space: semicolons are optional for the last value
		if i != len(pairs)-1 {
			fmt.Print(";")
		}
	}
}

func minifyProp(property string) string {
	return cleanSpaces(property)
}

func minifyVal(value string) string {
	value = cleanSpaces(value)

	// Values need special care
	value = cleanHex(value)
	value = cleanUrl(value)
	return value
}

func cleanHex(value string) string {
	return value
}

func cleanUrl(value string) string {
	return value
}

func cleanSpaces(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`\s\s`)
	for str = re.ReplaceAllString(str, " "); re.Find([]byte(str)) != nil; {
		str = re.ReplaceAllString(str, " ")
	}
	return str
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013-2014 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"bytes"
	"container/list"
	"encoding/binary"
	"net/url"
	"strconv"
	"strings"
	"time"
)

func str_to_bytes(s string) []byte {
	return bytes.NewBufferString(s).Bytes()
}

func int32_to_bytes(i32 int32) []byte {
	bs := []byte{
		byte(i32 & 0xFF),
		byte(i32 >> 8 & 0xFF),
		byte(i32 >> 16 & 0xFF),
		byte(i32 >> 24 & 0xFF),
	}
	return bs
}

func bint32_to_bytes(i32 int32) []byte {
	bs := []byte{
		byte(i32 >> 24 & 0xFF),
		byte(i32 >> 16 & 0xFF),
		byte(i32 >> 8 & 0xFF),
		byte(i32 & 0xFF),
	}
	return bs
}

func int16_to_bytes(i16 int16) []byte {
	bs := []byte{
		byte(i16 & 0xFF),
		byte(i16 >> 8 & 0xFF),
	}
	return bs
}
func bytes_to_str(b []byte) string {
	return bytes.NewBuffer(b).String()
}

func bytes_to_bint32(b []byte) int32 {
	var i32 int32
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.BigEndian, &i32)
	return i32
}

func bytes_to_int32(b []byte) int32 {
	var i32 int32
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.LittleEndian, &i32)
	return i32
}

func bytes_to_bint16(b []byte) int16 {
	var i int16
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.BigEndian, &i)
	return i
}

func bytes_to_int16(b []byte) int16 {
	var i int16
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.LittleEndian, &i)
	return i
}

func bytes_to_bint64(b []byte) int64 {
	var i int64
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.BigEndian, &i)
	return i
}

func bytes_to_int64(b []byte) int64 {
	var i int64
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.LittleEndian, &i)
	return i
}

func xdrBytes(bs []byte) []byte {
	// XDR encoding bytes
	n := len(bs)
	padding := 0
	if n%4 != 0 {
		padding = 4 - n%4
	}
	buf := make([]byte, 4+n+padding)
	buf[0] = byte(n >> 24 & 0xFF)
	buf[1] = byte(n >> 16 & 0xFF)
	buf[2] = byte(n >> 8 & 0xFF)
	buf[3] = byte(n & 0xFF)
	for i, b := range bs {
		buf[4+i] = b
	}
	return buf
}

func xdrString(s string) []byte {
	// XDR encoding string
	bs := bytes.NewBufferString(s).Bytes()
	return xdrBytes(bs)
}

func flattenBytes(l *list.List) []byte {
	n := 0
	for e := l.Front(); e != nil; e = e.Next() {
		n += len((e.Value).([]byte))
	}

	bs := make([]byte, n)

	n = 0
	for e := l.Front(); e != nil; e = e.Next() {
		for i, b := range (e.Value).([]byte) {
			bs[n+i] = b
		}
		n += len((e.Value).([]byte))
	}

	return bs
}

func _int32ToBlr(i32 int32) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		bint32_to_bytes(i32),
	}, nil)
	blr := []byte{8, 0}

	return blr, v
}

func _bytesToBlr(v []byte) ([]byte, []byte) {
	nbytes := len(v)
	pad_length := ((4 - nbytes) & 3)
	padding := make([]byte, pad_length)
	v = bytes.Join([][]byte{
		v,
		padding,
	}, nil)
	blr := []byte{14, byte(nbytes & 255), byte(nbytes >> 8)}
	return blr, v
}

func _convert_date(t time.Time) []byte {
	i := int(t.Month()) + 9
	jy := t.Year() + (i / 12) - 1
	jm := i % 12
	c := jy / 100
	jy -= 100 * c
	j := (146097*c)/4 + (1461*jy)/4 + (153*jm+2)/5 + t.Day() - 678882
	return bint32_to_bytes(int32(j))
}

func _convert_time(t time.Time) []byte {
	v := (t.Hour()*3600+t.Minute()*60+t.Second())*10000 + t.Nanosecond()/100000
	return bint32_to_bytes(int32(v))
}

func _dateToBlr(t time.Time) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		_convert_date(t),
	}, nil)
	blr := []byte{12}
	return blr, v
}

func _timeToBlr(t time.Time) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		_convert_time(t),
	}, nil)
	blr := []byte{13}
	return blr, v
}

func _timestampToBlr(t time.Time) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		_convert_date(t),
		_convert_time(t),
	}, nil)

	blr := []byte{35}
	return blr, v
}

func split1(src string, delm string) (string, string) {
	for i := 0; i < len(src); i++ {
		if src[i:i+1] == delm {
			s1 := src[0:i]
			s2 := src[i+1:]
			return s1, s2
		}
	}
	return src, ""
}

func parseDSN(dsn string) (addr string, dbName string, user string, passwd string, role string, authPluginName string, wireCrypt bool, err error) {
	u, err := url.Parse("firebird://" + dsn)
	if err != nil {
		return
	}
	user = u.User.Username()
	passwd, _ = u.User.Password()
	addr = u.Host
	if !strings.ContainsRune(addr, ':') {
		addr += ":3050"
	}
	dbName = u.Path
	if !strings.ContainsRune(dbName[1:], '/') {
		dbName = dbName[1:]
	}

	m, _ := url.ParseQuery(u.RawQuery)

	values, ok := m["role"]
	if ok {
		role = values[0]
	} else {
		role = ""
	}

	values, ok = m["auth_plugin_name"]
	if ok {
		authPluginName = values[0]
	} else {
		authPluginName = "Srp"
	}

	values, ok = m["wire_crypt"]
	if ok {
		wireCrypt, _ = strconv.ParseBool(values[0])
	} else {
		wireCrypt = true
	}

	return
}

func calcBlr(xsqlda []xSQLVAR) []byte {
	// Calculate  BLR from XSQLVAR array.
	ln := len(xsqlda) * 2
	blr := make([]byte, (ln*4)+8)
	blr[0] = 5
	blr[1] = 2
	blr[2] = 4
	blr[3] = 0
	blr[4] = byte(ln & 255)
	blr[5] = byte(ln >> 8)
	n := 6

	for _, x := range xsqlda {
		sqlscale := x.sqlscale
		if sqlscale < 0 {
			sqlscale += 256
		}
		switch x.sqltype {
		case SQL_TYPE_VARYING:
			blr[n] = 37
			blr[n+1] = byte(x.sqllen & 255)
			blr[n+2] = byte(x.sqllen >> 8)
			n += 3
		case SQL_TYPE_TEXT:
			blr[n] = 14
			blr[n+1] = byte(x.sqllen & 255)
			blr[n+2] = byte(x.sqllen >> 8)
			n += 3
		case SQL_TYPE_LONG:
			blr[n] = 8
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_SHORT:
			blr[n] = 7
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_INT64:
			blr[n] = 16
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_QUAD:
			blr[n] = 9
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_BLOB:
			blr[n] = 9
			blr[n+1] = 0
			n += 2
		case SQL_TYPE_ARRAY:
			blr[n] = 9
			blr[n+1] = 0
			n += 2
		case SQL_TYPE_DOUBLE:
			blr[n] = 27
			n += 1
		case SQL_TYPE_FLOAT:
			blr[n] = 10
			n += 1
		case SQL_TYPE_D_FLOAT:
			blr[n] = 11
			n += 1
		case SQL_TYPE_DATE:
			blr[n] = 12
			n += 1
		case SQL_TYPE_TIME:
			blr[n] = 13
			n += 1
		case SQL_TYPE_TIMESTAMP:
			blr[n] = 35
			n += 1
		case SQL_TYPE_BOOLEAN:
			blr[n] = 23
			n += 1
		}
		// [blr_short, 0]
		blr[n] = 7
		blr[n+1] = 0
		n += 2
	}
	// [blr_end, blr_eoc]
	blr[n] = 255
	blr[n+1] = 76
	n += 2

	return blr[:n]
}
<commit_msg>Add Windows path for connect<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013-2014 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"bytes"
	"container/list"
	"encoding/binary"
	"net/url"
	"strconv"
	"strings"
	"time"
)

func str_to_bytes(s string) []byte {
	return bytes.NewBufferString(s).Bytes()
}

func int32_to_bytes(i32 int32) []byte {
	bs := []byte{
		byte(i32 & 0xFF),
		byte(i32 >> 8 & 0xFF),
		byte(i32 >> 16 & 0xFF),
		byte(i32 >> 24 & 0xFF),
	}
	return bs
}

func bint32_to_bytes(i32 int32) []byte {
	bs := []byte{
		byte(i32 >> 24 & 0xFF),
		byte(i32 >> 16 & 0xFF),
		byte(i32 >> 8 & 0xFF),
		byte(i32 & 0xFF),
	}
	return bs
}

func int16_to_bytes(i16 int16) []byte {
	bs := []byte{
		byte(i16 & 0xFF),
		byte(i16 >> 8 & 0xFF),
	}
	return bs
}
func bytes_to_str(b []byte) string {
	return bytes.NewBuffer(b).String()
}

func bytes_to_bint32(b []byte) int32 {
	var i32 int32
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.BigEndian, &i32)
	return i32
}

func bytes_to_int32(b []byte) int32 {
	var i32 int32
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.LittleEndian, &i32)
	return i32
}

func bytes_to_bint16(b []byte) int16 {
	var i int16
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.BigEndian, &i)
	return i
}

func bytes_to_int16(b []byte) int16 {
	var i int16
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.LittleEndian, &i)
	return i
}

func bytes_to_bint64(b []byte) int64 {
	var i int64
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.BigEndian, &i)
	return i
}

func bytes_to_int64(b []byte) int64 {
	var i int64
	buffer := bytes.NewBuffer(b)
	binary.Read(buffer, binary.LittleEndian, &i)
	return i
}

func xdrBytes(bs []byte) []byte {
	// XDR encoding bytes
	n := len(bs)
	padding := 0
	if n%4 != 0 {
		padding = 4 - n%4
	}
	buf := make([]byte, 4+n+padding)
	buf[0] = byte(n >> 24 & 0xFF)
	buf[1] = byte(n >> 16 & 0xFF)
	buf[2] = byte(n >> 8 & 0xFF)
	buf[3] = byte(n & 0xFF)
	for i, b := range bs {
		buf[4+i] = b
	}
	return buf
}

func xdrString(s string) []byte {
	// XDR encoding string
	bs := bytes.NewBufferString(s).Bytes()
	return xdrBytes(bs)
}

func flattenBytes(l *list.List) []byte {
	n := 0
	for e := l.Front(); e != nil; e = e.Next() {
		n += len((e.Value).([]byte))
	}

	bs := make([]byte, n)

	n = 0
	for e := l.Front(); e != nil; e = e.Next() {
		for i, b := range (e.Value).([]byte) {
			bs[n+i] = b
		}
		n += len((e.Value).([]byte))
	}

	return bs
}

func _int32ToBlr(i32 int32) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		bint32_to_bytes(i32),
	}, nil)
	blr := []byte{8, 0}

	return blr, v
}

func _bytesToBlr(v []byte) ([]byte, []byte) {
	nbytes := len(v)
	pad_length := ((4 - nbytes) & 3)
	padding := make([]byte, pad_length)
	v = bytes.Join([][]byte{
		v,
		padding,
	}, nil)
	blr := []byte{14, byte(nbytes & 255), byte(nbytes >> 8)}
	return blr, v
}

func _convert_date(t time.Time) []byte {
	i := int(t.Month()) + 9
	jy := t.Year() + (i / 12) - 1
	jm := i % 12
	c := jy / 100
	jy -= 100 * c
	j := (146097*c)/4 + (1461*jy)/4 + (153*jm+2)/5 + t.Day() - 678882
	return bint32_to_bytes(int32(j))
}

func _convert_time(t time.Time) []byte {
	v := (t.Hour()*3600+t.Minute()*60+t.Second())*10000 + t.Nanosecond()/100000
	return bint32_to_bytes(int32(v))
}

func _dateToBlr(t time.Time) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		_convert_date(t),
	}, nil)
	blr := []byte{12}
	return blr, v
}

func _timeToBlr(t time.Time) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		_convert_time(t),
	}, nil)
	blr := []byte{13}
	return blr, v
}

func _timestampToBlr(t time.Time) ([]byte, []byte) {
	v := bytes.Join([][]byte{
		_convert_date(t),
		_convert_time(t),
	}, nil)

	blr := []byte{35}
	return blr, v
}

func split1(src string, delm string) (string, string) {
	for i := 0; i < len(src); i++ {
		if src[i:i+1] == delm {
			s1 := src[0:i]
			s2 := src[i+1:]
			return s1, s2
		}
	}
	return src, ""
}

func parseDSN(dsn string) (addr string, dbName string, user string, passwd string, role string, authPluginName string, wireCrypt bool, err error) {
	u, err := url.Parse("firebird://" + dsn)
	if err != nil {
		return
	}
	user = u.User.Username()
	passwd, _ = u.User.Password()
	addr = u.Host
	if !strings.ContainsRune(addr, ':') {
		addr += ":3050"
	}
	dbName = u.Path
	if !strings.ContainsRune(dbName[1:], '/') {
		dbName = dbName[1:]
	}

	//Windows Path
	if strings.ContainsRune(dbName[2:], ':') {
		dbName = dbName[1:]
	}

	m, _ := url.ParseQuery(u.RawQuery)

	values, ok := m["role"]
	if ok {
		role = values[0]
	} else {
		role = ""
	}

	values, ok = m["auth_plugin_name"]
	if ok {
		authPluginName = values[0]
	} else {
		authPluginName = "Srp"
	}

	values, ok = m["wire_crypt"]
	if ok {
		wireCrypt, _ = strconv.ParseBool(values[0])
	} else {
		wireCrypt = true
	}

	return
}

func calcBlr(xsqlda []xSQLVAR) []byte {
	// Calculate  BLR from XSQLVAR array.
	ln := len(xsqlda) * 2
	blr := make([]byte, (ln*4)+8)
	blr[0] = 5
	blr[1] = 2
	blr[2] = 4
	blr[3] = 0
	blr[4] = byte(ln & 255)
	blr[5] = byte(ln >> 8)
	n := 6

	for _, x := range xsqlda {
		sqlscale := x.sqlscale
		if sqlscale < 0 {
			sqlscale += 256
		}
		switch x.sqltype {
		case SQL_TYPE_VARYING:
			blr[n] = 37
			blr[n+1] = byte(x.sqllen & 255)
			blr[n+2] = byte(x.sqllen >> 8)
			n += 3
		case SQL_TYPE_TEXT:
			blr[n] = 14
			blr[n+1] = byte(x.sqllen & 255)
			blr[n+2] = byte(x.sqllen >> 8)
			n += 3
		case SQL_TYPE_LONG:
			blr[n] = 8
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_SHORT:
			blr[n] = 7
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_INT64:
			blr[n] = 16
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_QUAD:
			blr[n] = 9
			blr[n+1] = byte(sqlscale)
			n += 2
		case SQL_TYPE_BLOB:
			blr[n] = 9
			blr[n+1] = 0
			n += 2
		case SQL_TYPE_ARRAY:
			blr[n] = 9
			blr[n+1] = 0
			n += 2
		case SQL_TYPE_DOUBLE:
			blr[n] = 27
			n += 1
		case SQL_TYPE_FLOAT:
			blr[n] = 10
			n += 1
		case SQL_TYPE_D_FLOAT:
			blr[n] = 11
			n += 1
		case SQL_TYPE_DATE:
			blr[n] = 12
			n += 1
		case SQL_TYPE_TIME:
			blr[n] = 13
			n += 1
		case SQL_TYPE_TIMESTAMP:
			blr[n] = 35
			n += 1
		case SQL_TYPE_BOOLEAN:
			blr[n] = 23
			n += 1
		}
		// [blr_short, 0]
		blr[n] = 7
		blr[n+1] = 0
		n += 2
	}
	// [blr_end, blr_eoc]
	blr[n] = 255
	blr[n+1] = 76
	n += 2

	return blr[:n]
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// More information about Google Distance Matrix API is available on
// https://developers.google.com/maps/documentation/distancematrix/
package maps // import "google.golang.org/maps"

import (
	"net/http"

	"golang.org/x/net/context"
	"google.golang.org/maps/internal"
)

func httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error {
	// Run the HTTP request in a goroutine and pass the response to f.
	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan error, 1)
	go func() { c <- f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		<-c // Wait for f to return.
		return ctx.Err()
	case err := <-c:
		return err
	}
}

func rawService(ctx context.Context) *http.Client {
	return internal.Service(ctx, "directions", func(hc *http.Client) interface{} {
		// TODO(brettmorgan): Introduce a rate limiting wrapper for hc here.
		return hc
	}).(*http.Client)
}
<commit_msg>Dropping unused code<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// More information about Google Distance Matrix API is available on
// https://developers.google.com/maps/documentation/distancematrix/

package maps // import "google.golang.org/maps"

import (
	"net/http"

	"golang.org/x/net/context"
)

func httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error {
	// Run the HTTP request in a goroutine and pass the response to f.
	tr := &http.Transport{}
	client := &http.Client{Transport: tr}
	c := make(chan error, 1)
	go func() { c <- f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		<-c // Wait for f to return.
		return ctx.Err()
	case err := <-c:
		return err
	}
}
<|endoftext|>


<commit_before>package gobot

import (
	"math"
	"math/rand"
	"reflect"
	"time"
)

func Every(t string, f func()) {
	dur := parseDuration(t)
	go func() {
		for {
			time.Sleep(dur)
			go f()
		}
	}()
}

func After(t string, f func()) {
	dur := parseDuration(t)
	go func() {
		time.Sleep(dur)
		f()
	}()
}

func Publish(c chan interface{}, val interface{}) {
	select {
	case c <- val:
	default:
	}
}

func On(c chan interface{}, f func(s interface{})) {
	go func() {
		for s := range c {
			f(s)
		}
	}()
}

func Rand(max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max)
}

func Call(thing interface{}, method string, params ...interface{}) []reflect.Value {
	in := make([]reflect.Value, len(params))
	for k, param := range params {
		in[k] = reflect.ValueOf(param)
	}
	return reflect.ValueOf(thing).MethodByName(method).Call(in)
}

func FieldByName(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).FieldByName(field)
}
func FieldByNamePtr(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).Elem().FieldByName(field)
}

func FromScale(input, min, max float64) float64 {
	return (input - math.Min(min, max)) / (math.Max(min, max) - math.Min(min, max))
}

func ToScale(input, min, max float64) float64 {
	i := input*(math.Max(min, max)-math.Min(min, max)) + math.Min(min, max)
	if i < math.Min(min, max) {
		return math.Min(min, max)
	} else if i > math.Max(min, max) {
		return math.Max(min, max)
	} else {
		return i
	}
}

func parseDuration(t string) time.Duration {
	dur, err := time.ParseDuration(t)
	if err != nil {
		panic(err)
	}
	return dur
}
<commit_msg>removed sleeps from every and after funcs<commit_after>package gobot

import (
	"math"
	"math/rand"
	"reflect"
	"time"
)

// Every triggers f every `t` time until the end of days.
func Every(t string, f func()) {
	c := time.Tick(parseDuration(t))
	// start a go routine to not bloc the function
	go func() {
		for {
			// wait for the ticker to tell us to run
			<-c
			// run the passed function in another go routine
			// so we don't slow down the loop.
			go f()
		}
	}()
}

// After triggers the passed function after `t` duration.
func After(t string, f func()) {
	time.AfterFunc(parseDuration(t), f)
}

func Publish(c chan interface{}, val interface{}) {
	select {
	case c <- val:
	default:
	}
}

func On(c chan interface{}, f func(s interface{})) {
	go func() {
		for s := range c {
			f(s)
		}
	}()
}

func Rand(max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max)
}

func Call(thing interface{}, method string, params ...interface{}) []reflect.Value {
	in := make([]reflect.Value, len(params))
	for k, param := range params {
		in[k] = reflect.ValueOf(param)
	}
	return reflect.ValueOf(thing).MethodByName(method).Call(in)
}

func FieldByName(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).FieldByName(field)
}
func FieldByNamePtr(thing interface{}, field string) reflect.Value {
	return reflect.ValueOf(thing).Elem().FieldByName(field)
}

func FromScale(input, min, max float64) float64 {
	return (input - math.Min(min, max)) / (math.Max(min, max) - math.Min(min, max))
}

func ToScale(input, min, max float64) float64 {
	i := input*(math.Max(min, max)-math.Min(min, max)) + math.Min(min, max)
	if i < math.Min(min, max) {
		return math.Min(min, max)
	} else if i > math.Max(min, max) {
		return math.Max(min, max)
	} else {
		return i
	}
}

func parseDuration(t string) time.Duration {
	dur, err := time.ParseDuration(t)
	if err != nil {
		panic(err)
	}
	return dur
}
<|endoftext|>


<commit_before>package couchdb

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"path/filepath"
	"net/url"
	"strconv"
)

// Get mime type from file name.
func mimeType(name string) string {
	ext := filepath.Ext(name)
	return mime.TypeByExtension(ext)
}

// Make HTTP request.
// Treat status code other than 2xx as Error.
func request(method, url string, data io.Reader, contentType string) ([]byte, error) {
	req, err := http.NewRequest(method, url, data)
	if err != nil {
		return nil, err
	}
	if contentType != "" {
		req.Header.Set("Content-Type", contentType)
	}
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	// handle CouchDB http errors
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, newError(res, body)
	}
	return body, nil
}

// Convert HTTP response from CouchDB into Error.
func newError(res *http.Response, body []byte) error {
	var error *Error
	err := json.Unmarshal(body, &error)
	if err != nil {
		return err
	}
	error.Method = res.Request.Method
	error.Url = res.Request.URL.String()
	error.StatusCode = res.StatusCode
	return error
}

// Create new CouchDB response for any document method.
func newDocumentResponse(body []byte) (*DocumentResponse, error) {
	var response *DocumentResponse
	return response, json.Unmarshal(body, &response)
}

// Create new CouchDB response for any database method.
func newDatabaseResponse(body []byte) (*DatabaseResponse, error) {
	var response *DatabaseResponse
	return response, json.Unmarshal(body, &response)
}

// Write JSON to multipart/related.
func writeJSON(document *Document, writer *multipart.Writer, file *os.File) error {
	partHeaders := textproto.MIMEHeader{}
	partHeaders.Set("Content-Type", "application/json")
	part, err := writer.CreatePart(partHeaders)
	if err != nil {
		return err
	}

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	path := file.Name()

	// make empty map
	document.Attachments = make(map[string]Attachment)
	attachment := Attachment{
		Follows:     true,
		ContentType: mimeType(path),
		Length:      stat.Size(),
	}
	// add attachment to map
	filename := filepath.Base(path)
	document.Attachments[filename] = attachment

	bytes, err := json.Marshal(document)
	if err != nil {
		return err
	}

	_, err = part.Write(bytes)
	if err != nil {
		return err
	}
	return nil
}

// Write actual file content to multipart/related.
func writeMultipart(writer *multipart.Writer, file *os.File) error {
	part, err := writer.CreatePart(textproto.MIMEHeader{})
	if err != nil {
		return err
	}

	// copy file content into multipart message
	_, err = io.Copy(part, file)
	if err != nil {
		return err
	}

	return nil
}

// Quote all values because CouchDB needs those double quotes in query params.
func quote(values url.Values) url.Values {
	for key, value := range values {
		values.Set(key, strconv.Quote(value[0]))
	}
	return values
}
<commit_msg>go fmt utils.go<commit_after>package couchdb

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
)

// Get mime type from file name.
func mimeType(name string) string {
	ext := filepath.Ext(name)
	return mime.TypeByExtension(ext)
}

// Make HTTP request.
// Treat status code other than 2xx as Error.
func request(method, url string, data io.Reader, contentType string) ([]byte, error) {
	req, err := http.NewRequest(method, url, data)
	if err != nil {
		return nil, err
	}
	if contentType != "" {
		req.Header.Set("Content-Type", contentType)
	}
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	// handle CouchDB http errors
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, newError(res, body)
	}
	return body, nil
}

// Convert HTTP response from CouchDB into Error.
func newError(res *http.Response, body []byte) error {
	var error *Error
	err := json.Unmarshal(body, &error)
	if err != nil {
		return err
	}
	error.Method = res.Request.Method
	error.Url = res.Request.URL.String()
	error.StatusCode = res.StatusCode
	return error
}

// Create new CouchDB response for any document method.
func newDocumentResponse(body []byte) (*DocumentResponse, error) {
	var response *DocumentResponse
	return response, json.Unmarshal(body, &response)
}

// Create new CouchDB response for any database method.
func newDatabaseResponse(body []byte) (*DatabaseResponse, error) {
	var response *DatabaseResponse
	return response, json.Unmarshal(body, &response)
}

// Write JSON to multipart/related.
func writeJSON(document *Document, writer *multipart.Writer, file *os.File) error {
	partHeaders := textproto.MIMEHeader{}
	partHeaders.Set("Content-Type", "application/json")
	part, err := writer.CreatePart(partHeaders)
	if err != nil {
		return err
	}

	stat, err := file.Stat()
	if err != nil {
		return err
	}

	path := file.Name()

	// make empty map
	document.Attachments = make(map[string]Attachment)
	attachment := Attachment{
		Follows:     true,
		ContentType: mimeType(path),
		Length:      stat.Size(),
	}
	// add attachment to map
	filename := filepath.Base(path)
	document.Attachments[filename] = attachment

	bytes, err := json.Marshal(document)
	if err != nil {
		return err
	}

	_, err = part.Write(bytes)
	if err != nil {
		return err
	}
	return nil
}

// Write actual file content to multipart/related.
func writeMultipart(writer *multipart.Writer, file *os.File) error {
	part, err := writer.CreatePart(textproto.MIMEHeader{})
	if err != nil {
		return err
	}

	// copy file content into multipart message
	_, err = io.Copy(part, file)
	if err != nil {
		return err
	}

	return nil
}

// Quote all values because CouchDB needs those double quotes in query params.
func quote(values url.Values) url.Values {
	for key, value := range values {
		values.Set(key, strconv.Quote(value[0]))
	}
	return values
}
<|endoftext|>


<commit_before>package libnetwork

import (
	"fmt"
	"strings"

	"github.com/docker/libkv/store/boltdb"
	"github.com/docker/libkv/store/consul"
	"github.com/docker/libkv/store/etcd"
	"github.com/docker/libkv/store/zookeeper"
	"github.com/docker/libnetwork/datastore"
	"github.com/sirupsen/logrus"
)

func registerKVStores() {
	consul.Register()
	zookeeper.Register()
	etcd.Register()
	boltdb.Register()
}

func (c *controller) initScopedStore(scope string, scfg *datastore.ScopeCfg) error {
	store, err := datastore.NewDataStore(scope, scfg)
	if err != nil {
		return err
	}
	c.Lock()
	c.stores = append(c.stores, store)
	c.Unlock()

	return nil
}

func (c *controller) initStores() error {
	registerKVStores()

	c.Lock()
	if c.cfg == nil {
		c.Unlock()
		return nil
	}
	scopeConfigs := c.cfg.Scopes
	c.stores = nil
	c.Unlock()

	for scope, scfg := range scopeConfigs {
		if err := c.initScopedStore(scope, scfg); err != nil {
			return err
		}
	}

	c.startWatch()
	return nil
}

func (c *controller) closeStores() {
	for _, store := range c.getStores() {
		store.Close()
	}
}

func (c *controller) getStore(scope string) datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	for _, store := range c.stores {
		if store.Scope() == scope {
			return store
		}
	}

	return nil
}

func (c *controller) getStores() []datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	return c.stores
}

func (c *controller) getNetworkFromStore(nid string) (*network, error) {
	for _, store := range c.getStores() {
		n := &network{id: nid, ctrlr: c}
		err := store.GetObject(datastore.Key(n.Key()...), n)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Debugf("could not find network %s: %v", nid, err)
			}
			continue
		}

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			return nil, fmt.Errorf("could not find endpoint count for network %s: %v", n.Name(), err)
		}

		n.epCnt = ec
		if n.scope == "" {
			n.scope = store.Scope()
		}
		return n, nil
	}

	return nil, fmt.Errorf("network %s not found", nid)
}

func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
	var nl []*network

	store := c.getStore(scope)
	if store == nil {
		return nil, nil
	}

	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
		&network{ctrlr: c})
	if err != nil && err != datastore.ErrKeyNotFound {
		return nil, fmt.Errorf("failed to get networks for scope %s: %v",
			scope, err)
	}

	for _, kvo := range kvol {
		n := kvo.(*network)
		n.ctrlr = c

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			logrus.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
			continue
		}

		n.epCnt = ec
		if n.scope == "" {
			n.scope = scope
		}
		nl = append(nl, n)
	}

	return nl, nil
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network

	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
			}
			continue
		}

		kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Warnf("failed to get endpoint_count map for scope %s: %v", store.Scope(), err)
			}
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.Lock()
			n.ctrlr = c
			ec := &endpointCnt{n: n}
			// Trim the leading & trailing "/" to make it consistent across all stores
			if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
				ec = val.(*endpointCnt)
				ec.n = n
				n.epCnt = ec
			}
			if n.scope == "" {
				n.scope = store.Scope()
			}
			n.Unlock()
			nl = append(nl, n)
		}
	}

	return nl, nil
}

func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
	var errors []string
	for _, store := range n.ctrlr.getStores() {
		ep := &endpoint{id: eid, network: n}
		err := store.GetObject(datastore.Key(ep.Key()...), ep)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
				logrus.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
			}
			continue
		}
		return ep, nil
	}
	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
}

func (n *network) getEndpointsFromStore() ([]*endpoint, error) {
	var epl []*endpoint

	tmp := endpoint{network: n}
	for _, store := range n.getController().getStores() {
		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &endpoint{network: n})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Debugf("failed to get endpoints for network %s scope %s: %v",
					n.Name(), store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			ep := kvo.(*endpoint)
			epl = append(epl, ep)
		}
	}

	return epl, nil
}

func (c *controller) updateToStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return ErrDataStoreNotInitialized(kvObject.DataScope())
	}

	if err := cs.PutObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			return err
		}
		return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err)
	}

	return nil
}

func (c *controller) deleteFromStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return ErrDataStoreNotInitialized(kvObject.DataScope())
	}

retry:
	if err := cs.DeleteObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			if err := cs.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
			}
			logrus.Warnf("Error (%v) deleting object %v, retrying....", err, kvObject.Key())
			goto retry
		}
		return err
	}

	return nil
}

type netWatch struct {
	localEps  map[string]*endpoint
	remoteEps map[string]*endpoint
	stopCh    chan struct{}
}

func (c *controller) getLocalEps(nw *netWatch) []*endpoint {
	c.Lock()
	defer c.Unlock()

	var epl []*endpoint
	for _, ep := range nw.localEps {
		epl = append(epl, ep)
	}

	return epl
}

func (c *controller) watchSvcRecord(ep *endpoint) {
	c.watchCh <- ep
}

func (c *controller) unWatchSvcRecord(ep *endpoint) {
	c.unWatchCh <- ep
}

func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan datastore.KVObject) {
	for {
		select {
		case <-nw.stopCh:
			return
		case o := <-ecCh:
			ec := o.(*endpointCnt)

			epl, err := ec.n.getEndpointsFromStore()
			if err != nil {
				break
			}

			c.Lock()
			var addEp []*endpoint

			delEpMap := make(map[string]*endpoint)
			renameEpMap := make(map[string]bool)
			for k, v := range nw.remoteEps {
				delEpMap[k] = v
			}

			for _, lEp := range epl {
				if _, ok := nw.localEps[lEp.ID()]; ok {
					continue
				}

				if ep, ok := nw.remoteEps[lEp.ID()]; ok {
					// On a container rename EP ID will remain
					// the same but the name will change. service
					// records should reflect the change.
					// Keep old EP entry in the delEpMap and add
					// EP from the store (which has the new name)
					// into the new list
					if lEp.name == ep.name {
						delete(delEpMap, lEp.ID())
						continue
					}
					renameEpMap[lEp.ID()] = true
				}
				nw.remoteEps[lEp.ID()] = lEp
				addEp = append(addEp, lEp)
			}

			// EPs whose name are to be deleted from the svc records
			// should also be removed from nw's remote EP list, except
			// the ones that are getting renamed.
			for _, lEp := range delEpMap {
				if !renameEpMap[lEp.ID()] {
					delete(nw.remoteEps, lEp.ID())
				}
			}
			c.Unlock()

			for _, lEp := range delEpMap {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), false)

			}
			for _, lEp := range addEp {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), true)
			}
		}
	}
}

func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) {
	n := ep.getNetwork()
	if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() {
		return
	}

	c.Lock()
	nw, ok := nmap[n.ID()]
	c.Unlock()

	if ok {
		// Update the svc db for the local endpoint join right away
		n.updateSvcRecord(ep, c.getLocalEps(nw), true)

		c.Lock()
		nw.localEps[ep.ID()] = ep

		// If we had learned that from the kv store remove it
		// from remote ep list now that we know that this is
		// indeed a local endpoint
		delete(nw.remoteEps, ep.ID())
		c.Unlock()
		return
	}

	nw = &netWatch{
		localEps:  make(map[string]*endpoint),
		remoteEps: make(map[string]*endpoint),
	}

	// Update the svc db for the local endpoint join right away
	// Do this before adding this ep to localEps so that we don't
	// try to update this ep's container's svc records
	n.updateSvcRecord(ep, c.getLocalEps(nw), true)

	c.Lock()
	nw.localEps[ep.ID()] = ep
	nmap[n.ID()] = nw
	nw.stopCh = make(chan struct{})
	c.Unlock()

	store := c.getStore(n.DataScope())
	if store == nil {
		return
	}

	if !store.Watchable() {
		return
	}

	ch, err := store.Watch(n.getEpCnt(), nw.stopCh)
	if err != nil {
		logrus.Warnf("Error creating watch for network: %v", err)
		return
	}

	go c.networkWatchLoop(nw, ep, ch)
}

func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) {
	n := ep.getNetwork()
	if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() {
		return
	}

	c.Lock()
	nw, ok := nmap[n.ID()]

	if ok {
		delete(nw.localEps, ep.ID())
		c.Unlock()

		// Update the svc db about local endpoint leave right away
		// Do this after we remove this ep from localEps so that we
		// don't try to remove this svc record from this ep's container.
		n.updateSvcRecord(ep, c.getLocalEps(nw), false)

		c.Lock()
		if len(nw.localEps) == 0 {
			close(nw.stopCh)

			// This is the last container going away for the network. Destroy
			// this network's svc db entry
			delete(c.svcRecords, n.ID())

			delete(nmap, n.ID())
		}
	}
	c.Unlock()
}

func (c *controller) watchLoop() {
	for {
		select {
		case ep := <-c.watchCh:
			c.processEndpointCreate(c.nmap, ep)
		case ep := <-c.unWatchCh:
			c.processEndpointDelete(c.nmap, ep)
		}
	}
}

func (c *controller) startWatch() {
	if c.watchCh != nil {
		return
	}
	c.watchCh = make(chan *endpoint)
	c.unWatchCh = make(chan *endpoint)
	c.nmap = make(map[string]*netWatch)

	go c.watchLoop()
}

func (c *controller) networkCleanup() {
	networks, err := c.getNetworksFromStore()
	if err != nil {
		logrus.Warnf("Could not retrieve networks from store(s) during network cleanup: %v", err)
		return
	}

	for _, n := range networks {
		if n.inDelete {
			logrus.Infof("Removing stale network %s (%s)", n.Name(), n.ID())
			if err := n.delete(true, true); err != nil {
				logrus.Debugf("Error while removing stale network: %v", err)
			}
		}
	}
}

var populateSpecial NetworkWalker = func(nw Network) bool {
	if n := nw.(*network); n.hasSpecialDriver() && !n.ConfigOnly() {
		if err := n.getController().addNetwork(n); err != nil {
			logrus.Warnf("Failed to populate network %q with driver %q", nw.Name(), nw.Type())
		}
	}
	return false
}
<commit_msg>Fixed getNetworkFromStore, which returned incorrect network information - notably, the 'resolver' field was empty. This fixes https://github.com/moby/moby/issues/38901<commit_after>package libnetwork

import (
	"fmt"
	"strings"

	"github.com/docker/libkv/store/boltdb"
	"github.com/docker/libkv/store/consul"
	"github.com/docker/libkv/store/etcd"
	"github.com/docker/libkv/store/zookeeper"
	"github.com/docker/libnetwork/datastore"
	"github.com/sirupsen/logrus"
)

func registerKVStores() {
	consul.Register()
	zookeeper.Register()
	etcd.Register()
	boltdb.Register()
}

func (c *controller) initScopedStore(scope string, scfg *datastore.ScopeCfg) error {
	store, err := datastore.NewDataStore(scope, scfg)
	if err != nil {
		return err
	}
	c.Lock()
	c.stores = append(c.stores, store)
	c.Unlock()

	return nil
}

func (c *controller) initStores() error {
	registerKVStores()

	c.Lock()
	if c.cfg == nil {
		c.Unlock()
		return nil
	}
	scopeConfigs := c.cfg.Scopes
	c.stores = nil
	c.Unlock()

	for scope, scfg := range scopeConfigs {
		if err := c.initScopedStore(scope, scfg); err != nil {
			return err
		}
	}

	c.startWatch()
	return nil
}

func (c *controller) closeStores() {
	for _, store := range c.getStores() {
		store.Close()
	}
}

func (c *controller) getStore(scope string) datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	for _, store := range c.stores {
		if store.Scope() == scope {
			return store
		}
	}

	return nil
}

func (c *controller) getStores() []datastore.DataStore {
	c.Lock()
	defer c.Unlock()

	return c.stores
}

func (c *controller) getNetworkFromStore(nid string) (*network, error) {
	ns, err := c.getNetworksFromStore()
	for _, n := range ns {
		if n.id == nid {
			return n, err
		}
	}
	return nil, fmt.Errorf("network %s not found", nid)
}

func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
	var nl []*network

	store := c.getStore(scope)
	if store == nil {
		return nil, nil
	}

	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
		&network{ctrlr: c})
	if err != nil && err != datastore.ErrKeyNotFound {
		return nil, fmt.Errorf("failed to get networks for scope %s: %v",
			scope, err)
	}

	for _, kvo := range kvol {
		n := kvo.(*network)
		n.ctrlr = c

		ec := &endpointCnt{n: n}
		err = store.GetObject(datastore.Key(ec.Key()...), ec)
		if err != nil && !n.inDelete {
			logrus.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
			continue
		}

		n.epCnt = ec
		if n.scope == "" {
			n.scope = scope
		}
		nl = append(nl, n)
	}

	return nl, nil
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network

	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
			}
			continue
		}

		kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Warnf("failed to get endpoint_count map for scope %s: %v", store.Scope(), err)
			}
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.Lock()
			n.ctrlr = c
			ec := &endpointCnt{n: n}
			// Trim the leading & trailing "/" to make it consistent across all stores
			if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
				ec = val.(*endpointCnt)
				ec.n = n
				n.epCnt = ec
			}
			if n.scope == "" {
				n.scope = store.Scope()
			}
			n.Unlock()
			nl = append(nl, n)
		}
	}

	return nl, nil
}

func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
	var errors []string
	for _, store := range n.ctrlr.getStores() {
		ep := &endpoint{id: eid, network: n}
		err := store.GetObject(datastore.Key(ep.Key()...), ep)
		// Continue searching in the next store if the key is not found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
				logrus.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
			}
			continue
		}
		return ep, nil
	}
	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
}

func (n *network) getEndpointsFromStore() ([]*endpoint, error) {
	var epl []*endpoint

	tmp := endpoint{network: n}
	for _, store := range n.getController().getStores() {
		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &endpoint{network: n})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Debugf("failed to get endpoints for network %s scope %s: %v",
					n.Name(), store.Scope(), err)
			}
			continue
		}

		for _, kvo := range kvol {
			ep := kvo.(*endpoint)
			epl = append(epl, ep)
		}
	}

	return epl, nil
}

func (c *controller) updateToStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return ErrDataStoreNotInitialized(kvObject.DataScope())
	}

	if err := cs.PutObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			return err
		}
		return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err)
	}

	return nil
}

func (c *controller) deleteFromStore(kvObject datastore.KVObject) error {
	cs := c.getStore(kvObject.DataScope())
	if cs == nil {
		return ErrDataStoreNotInitialized(kvObject.DataScope())
	}

retry:
	if err := cs.DeleteObjectAtomic(kvObject); err != nil {
		if err == datastore.ErrKeyModified {
			if err := cs.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
			}
			logrus.Warnf("Error (%v) deleting object %v, retrying....", err, kvObject.Key())
			goto retry
		}
		return err
	}

	return nil
}

type netWatch struct {
	localEps  map[string]*endpoint
	remoteEps map[string]*endpoint
	stopCh    chan struct{}
}

func (c *controller) getLocalEps(nw *netWatch) []*endpoint {
	c.Lock()
	defer c.Unlock()

	var epl []*endpoint
	for _, ep := range nw.localEps {
		epl = append(epl, ep)
	}

	return epl
}

func (c *controller) watchSvcRecord(ep *endpoint) {
	c.watchCh <- ep
}

func (c *controller) unWatchSvcRecord(ep *endpoint) {
	c.unWatchCh <- ep
}

func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan datastore.KVObject) {
	for {
		select {
		case <-nw.stopCh:
			return
		case o := <-ecCh:
			ec := o.(*endpointCnt)

			epl, err := ec.n.getEndpointsFromStore()
			if err != nil {
				break
			}

			c.Lock()
			var addEp []*endpoint

			delEpMap := make(map[string]*endpoint)
			renameEpMap := make(map[string]bool)
			for k, v := range nw.remoteEps {
				delEpMap[k] = v
			}

			for _, lEp := range epl {
				if _, ok := nw.localEps[lEp.ID()]; ok {
					continue
				}

				if ep, ok := nw.remoteEps[lEp.ID()]; ok {
					// On a container rename EP ID will remain
					// the same but the name will change. service
					// records should reflect the change.
					// Keep old EP entry in the delEpMap and add
					// EP from the store (which has the new name)
					// into the new list
					if lEp.name == ep.name {
						delete(delEpMap, lEp.ID())
						continue
					}
					renameEpMap[lEp.ID()] = true
				}
				nw.remoteEps[lEp.ID()] = lEp
				addEp = append(addEp, lEp)
			}

			// EPs whose name are to be deleted from the svc records
			// should also be removed from nw's remote EP list, except
			// the ones that are getting renamed.
			for _, lEp := range delEpMap {
				if !renameEpMap[lEp.ID()] {
					delete(nw.remoteEps, lEp.ID())
				}
			}
			c.Unlock()

			for _, lEp := range delEpMap {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), false)

			}
			for _, lEp := range addEp {
				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), true)
			}
		}
	}
}

func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) {
	n := ep.getNetwork()
	if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() {
		return
	}

	c.Lock()
	nw, ok := nmap[n.ID()]
	c.Unlock()

	if ok {
		// Update the svc db for the local endpoint join right away
		n.updateSvcRecord(ep, c.getLocalEps(nw), true)

		c.Lock()
		nw.localEps[ep.ID()] = ep

		// If we had learned that from the kv store remove it
		// from remote ep list now that we know that this is
		// indeed a local endpoint
		delete(nw.remoteEps, ep.ID())
		c.Unlock()
		return
	}

	nw = &netWatch{
		localEps:  make(map[string]*endpoint),
		remoteEps: make(map[string]*endpoint),
	}

	// Update the svc db for the local endpoint join right away
	// Do this before adding this ep to localEps so that we don't
	// try to update this ep's container's svc records
	n.updateSvcRecord(ep, c.getLocalEps(nw), true)

	c.Lock()
	nw.localEps[ep.ID()] = ep
	nmap[n.ID()] = nw
	nw.stopCh = make(chan struct{})
	c.Unlock()

	store := c.getStore(n.DataScope())
	if store == nil {
		return
	}

	if !store.Watchable() {
		return
	}

	ch, err := store.Watch(n.getEpCnt(), nw.stopCh)
	if err != nil {
		logrus.Warnf("Error creating watch for network: %v", err)
		return
	}

	go c.networkWatchLoop(nw, ep, ch)
}

func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) {
	n := ep.getNetwork()
	if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() {
		return
	}

	c.Lock()
	nw, ok := nmap[n.ID()]

	if ok {
		delete(nw.localEps, ep.ID())
		c.Unlock()

		// Update the svc db about local endpoint leave right away
		// Do this after we remove this ep from localEps so that we
		// don't try to remove this svc record from this ep's container.
		n.updateSvcRecord(ep, c.getLocalEps(nw), false)

		c.Lock()
		if len(nw.localEps) == 0 {
			close(nw.stopCh)

			// This is the last container going away for the network. Destroy
			// this network's svc db entry
			delete(c.svcRecords, n.ID())

			delete(nmap, n.ID())
		}
	}
	c.Unlock()
}

func (c *controller) watchLoop() {
	for {
		select {
		case ep := <-c.watchCh:
			c.processEndpointCreate(c.nmap, ep)
		case ep := <-c.unWatchCh:
			c.processEndpointDelete(c.nmap, ep)
		}
	}
}

func (c *controller) startWatch() {
	if c.watchCh != nil {
		return
	}
	c.watchCh = make(chan *endpoint)
	c.unWatchCh = make(chan *endpoint)
	c.nmap = make(map[string]*netWatch)

	go c.watchLoop()
}

func (c *controller) networkCleanup() {
	networks, err := c.getNetworksFromStore()
	if err != nil {
		logrus.Warnf("Could not retrieve networks from store(s) during network cleanup: %v", err)
		return
	}

	for _, n := range networks {
		if n.inDelete {
			logrus.Infof("Removing stale network %s (%s)", n.Name(), n.ID())
			if err := n.delete(true, true); err != nil {
				logrus.Debugf("Error while removing stale network: %v", err)
			}
		}
	}
}

var populateSpecial NetworkWalker = func(nw Network) bool {
	if n := nw.(*network); n.hasSpecialDriver() && !n.ConfigOnly() {
		if err := n.getController().addNetwork(n); err != nil {
			logrus.Warnf("Failed to populate network %q with driver %q", nw.Name(), nw.Type())
		}
	}
	return false
}
<|endoftext|>


<commit_before>package libipvs

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"syscall"

	"github.com/hkwi/nlgo"
)

// Helper to build an nlgo.Attr
func nlattr(typ uint16, value nlgo.NlaValue) nlgo.Attr {
	return nlgo.Attr{Header: syscall.NlAttr{Type: typ}, Value: value}
}

// Helpers for struct <-> nlgo.Binary
func unpack(value nlgo.Binary, out interface{}) error {
	return binary.Read(bytes.NewReader(([]byte)(value)), binary.BigEndian, out)
}

func pack(in interface{}) nlgo.Binary {
	var buf bytes.Buffer

	if err := binary.Write(&buf, binary.BigEndian, in); err != nil {
		panic(err)
	}

	return nlgo.Binary(buf.Bytes())
}

// Helpers for net.IP <-> nlgo.Binary
func unpackAddr(value nlgo.Binary, af AddressFamily) (net.IP, error) {
	buf := ([]byte)(value)
	size := 0

	switch af {
	case syscall.AF_INET:
		size = 4
	case syscall.AF_INET6:
		size = 16
	default:
		return nil, fmt.Errorf("ipvs: unknown af=%d addr=%v", af, buf)
	}

	if size > len(buf) {
		return nil, fmt.Errorf("ipvs: short af=%d addr=%v", af, buf)
	}

	return (net.IP)(buf[:size]), nil
}

func packAddr(af AddressFamily, addr net.IP) nlgo.Binary {
	var ip net.IP

	switch af {
	case syscall.AF_INET:
		ip = addr.To4()
	case syscall.AF_INET6:
		ip = addr.To16()
	default:
		panic(fmt.Errorf("ipvs:packAddr: unknown af=%d addr=%v", af, addr))
	}

	if ip == nil {
		panic(fmt.Errorf("ipvs:packAddr: invalid af=%d addr=%v", af, addr))
	}

	return (nlgo.Binary)(ip)
}

// Helpers for uint16 port <-> nlgo.U16
func htons(value uint16) uint16 {
	return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}
func ntohs(value uint16) uint16 {
	return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}

func unpackPort(val nlgo.U16) uint16 {
	return ntohs((uint16)(val))
}
func packPort(port uint16) nlgo.U16 {
	return nlgo.U16(htons(port))
}

func unpackService(attrs nlgo.AttrMap) (Service, error) {
	var service Service

	var addr nlgo.Binary
	var flags nlgo.Binary

	for _, attr := range attrs.Slice() {
		switch attr.Field() {
		case IPVS_SVC_ATTR_AF:
			service.AddressFamily = (AddressFamily)(attr.Value.(nlgo.U16))
		case IPVS_SVC_ATTR_PROTOCOL:
			service.Protocol = (Protocol)(attr.Value.(nlgo.U16))
		case IPVS_SVC_ATTR_ADDR:
			addr = attr.Value.(nlgo.Binary)
		case IPVS_SVC_ATTR_PORT:
			service.Port = unpackPort(attr.Value.(nlgo.U16))
		case IPVS_SVC_ATTR_FWMARK:
			service.FWMark = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_SVC_ATTR_SCHED_NAME:
			service.SchedName = (string)(attr.Value.(nlgo.NulString))
		case IPVS_SVC_ATTR_FLAGS:
			flags = attr.Value.(nlgo.Binary)
		case IPVS_SVC_ATTR_TIMEOUT:
			service.Timeout = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_SVC_ATTR_NETMASK:
			service.Netmask = (uint32)(attr.Value.(nlgo.U32))
		}
	}

	if addrIP, err := unpackAddr(addr, service.AddressFamily); err != nil {
		return service, fmt.Errorf("ipvs:Service.unpack: addr: %s", err)
	} else {
		service.Address = addrIP
	}

	if err := unpack(flags, &service.Flags); err != nil {
		return service, fmt.Errorf("ipvs:Service.unpack: flags: %s", err)
	}

	return service, nil
}

func unpackDest(attrs nlgo.AttrMap) (Destination, error) {
	var dest Destination
	var addr []byte

	for _, attr := range attrs.Slice() {
		switch attr.Field() {
		case IPVS_DEST_ATTR_ADDR_FAMILY:
			dest.AddressFamily = (AddressFamily)(attr.Value.(nlgo.U16))
		case IPVS_DEST_ATTR_ADDR:
			addr = ([]byte)(attr.Value.(nlgo.Binary))
		case IPVS_DEST_ATTR_PORT:
			dest.Port = unpackPort(attr.Value.(nlgo.U16))
		case IPVS_DEST_ATTR_FWD_METHOD:
			dest.FwdMethod = (FwdMethod)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_WEIGHT:
			dest.Weight = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_U_THRESH:
			dest.UThresh = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_L_THRESH:
			dest.LThresh = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_ACTIVE_CONNS:
			dest.ActiveConns = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_INACT_CONNS:
			dest.InactConns = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_PERSIST_CONNS:
			dest.PersistConns = (uint32)(attr.Value.(nlgo.U32))
		}
	}

	if addrIP, err := unpackAddr(addr, dest.AddressFamily); err != nil {
		return dest, fmt.Errorf("ipvs:Dest.unpack: addr: %s", err)
	} else {
		dest.Address = addrIP
	}

	return dest, nil
}

func unpackInfo(attrs nlgo.AttrMap) (info Info, err error) {
	for _, attr := range attrs.Slice() {
		switch attr.Field() {
		case IPVS_INFO_ATTR_VERSION:
			info.Version = (Version)(attr.Value.(nlgo.U32))
		case IPVS_INFO_ATTR_CONN_TAB_SIZE:
			info.ConnTabSize = (uint32)(attr.Value.(nlgo.U32))
		}
	}

	return
}
<commit_msg>bug fix: ignore the address unpack error when ipvs Service has Fwmark<commit_after>package libipvs

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"syscall"

	"github.com/hkwi/nlgo"
)

// Helper to build an nlgo.Attr
func nlattr(typ uint16, value nlgo.NlaValue) nlgo.Attr {
	return nlgo.Attr{Header: syscall.NlAttr{Type: typ}, Value: value}
}

// Helpers for struct <-> nlgo.Binary
func unpack(value nlgo.Binary, out interface{}) error {
	return binary.Read(bytes.NewReader(([]byte)(value)), binary.BigEndian, out)
}

func pack(in interface{}) nlgo.Binary {
	var buf bytes.Buffer

	if err := binary.Write(&buf, binary.BigEndian, in); err != nil {
		panic(err)
	}

	return nlgo.Binary(buf.Bytes())
}

// Helpers for net.IP <-> nlgo.Binary
func unpackAddr(value nlgo.Binary, af AddressFamily) (net.IP, error) {
	buf := ([]byte)(value)
	size := 0

	switch af {
	case syscall.AF_INET:
		size = 4
	case syscall.AF_INET6:
		size = 16
	default:
		return nil, fmt.Errorf("ipvs: unknown af=%d addr=%v", af, buf)
	}

	if size > len(buf) {
		return nil, fmt.Errorf("ipvs: short af=%d addr=%v", af, buf)
	}

	return (net.IP)(buf[:size]), nil
}

func packAddr(af AddressFamily, addr net.IP) nlgo.Binary {
	var ip net.IP

	switch af {
	case syscall.AF_INET:
		ip = addr.To4()
	case syscall.AF_INET6:
		ip = addr.To16()
	default:
		panic(fmt.Errorf("ipvs:packAddr: unknown af=%d addr=%v", af, addr))
	}

	if ip == nil {
		panic(fmt.Errorf("ipvs:packAddr: invalid af=%d addr=%v", af, addr))
	}

	return (nlgo.Binary)(ip)
}

// Helpers for uint16 port <-> nlgo.U16
func htons(value uint16) uint16 {
	return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}
func ntohs(value uint16) uint16 {
	return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}

func unpackPort(val nlgo.U16) uint16 {
	return ntohs((uint16)(val))
}
func packPort(port uint16) nlgo.U16 {
	return nlgo.U16(htons(port))
}

func unpackService(attrs nlgo.AttrMap) (Service, error) {
	var service Service

	var addr nlgo.Binary
	var flags nlgo.Binary

	for _, attr := range attrs.Slice() {
		switch attr.Field() {
		case IPVS_SVC_ATTR_AF:
			service.AddressFamily = (AddressFamily)(attr.Value.(nlgo.U16))
		case IPVS_SVC_ATTR_PROTOCOL:
			service.Protocol = (Protocol)(attr.Value.(nlgo.U16))
		case IPVS_SVC_ATTR_ADDR:
			addr = attr.Value.(nlgo.Binary)
		case IPVS_SVC_ATTR_PORT:
			service.Port = unpackPort(attr.Value.(nlgo.U16))
		case IPVS_SVC_ATTR_FWMARK:
			service.FWMark = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_SVC_ATTR_SCHED_NAME:
			service.SchedName = (string)(attr.Value.(nlgo.NulString))
		case IPVS_SVC_ATTR_FLAGS:
			flags = attr.Value.(nlgo.Binary)
		case IPVS_SVC_ATTR_TIMEOUT:
			service.Timeout = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_SVC_ATTR_NETMASK:
			service.Netmask = (uint32)(attr.Value.(nlgo.U32))
		}
	}

	// TOTE: ipvs Service with Fwmarks has no Address, so we just ignore the error
	if addrIP, err := unpackAddr(addr, service.AddressFamily); err != nil && service.FWMark != 0 {
		return service, fmt.Errorf("ipvs:Service.unpack: addr: %s", err)
	} else {
		service.Address = addrIP
	}

	if err := unpack(flags, &service.Flags); err != nil {
		return service, fmt.Errorf("ipvs:Service.unpack: flags: %s", err)
	}

	return service, nil
}

func unpackDest(attrs nlgo.AttrMap) (Destination, error) {
	var dest Destination
	var addr []byte

	for _, attr := range attrs.Slice() {
		switch attr.Field() {
		case IPVS_DEST_ATTR_ADDR_FAMILY:
			dest.AddressFamily = (AddressFamily)(attr.Value.(nlgo.U16))
		case IPVS_DEST_ATTR_ADDR:
			addr = ([]byte)(attr.Value.(nlgo.Binary))
		case IPVS_DEST_ATTR_PORT:
			dest.Port = unpackPort(attr.Value.(nlgo.U16))
		case IPVS_DEST_ATTR_FWD_METHOD:
			dest.FwdMethod = (FwdMethod)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_WEIGHT:
			dest.Weight = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_U_THRESH:
			dest.UThresh = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_L_THRESH:
			dest.LThresh = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_ACTIVE_CONNS:
			dest.ActiveConns = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_INACT_CONNS:
			dest.InactConns = (uint32)(attr.Value.(nlgo.U32))
		case IPVS_DEST_ATTR_PERSIST_CONNS:
			dest.PersistConns = (uint32)(attr.Value.(nlgo.U32))
		}
	}

	if addrIP, err := unpackAddr(addr, dest.AddressFamily); err != nil {
		return dest, fmt.Errorf("ipvs:Dest.unpack: addr: %s", err)
	} else {
		dest.Address = addrIP
	}

	return dest, nil
}

func unpackInfo(attrs nlgo.AttrMap) (info Info, err error) {
	for _, attr := range attrs.Slice() {
		switch attr.Field() {
		case IPVS_INFO_ATTR_VERSION:
			info.Version = (Version)(attr.Value.(nlgo.U32))
		case IPVS_INFO_ATTR_CONN_TAB_SIZE:
			info.ConnTabSize = (uint32)(attr.Value.(nlgo.U32))
		}
	}

	return
}
<|endoftext|>


<commit_before>// Package store automatically configures a database to store structured information in an sql database
package store

import (
	"database/sql"
	"errors"
	"reflect"
	"strings"
	"sync"
)

const (
	add = iota
	get
	update
	remove
	getPage
	count
)

type field struct {
	name      string
	pos       int
	isPointer bool
	isStruct  bool
}

type typeInfo struct {
	primary    int
	fields     []field
	statements []*sql.Stmt
}

type Store struct {
	db    *sql.DB
	types map[string]typeInfo
	mutex sync.Mutex
}

func New(driverName, dataSourceName string) (*Store, error) {
	db, err := sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return &Store{
		db:    db,
		types: make(map[string]typeInfo),
	}, nil
}

func (s *Store) Close() error {
	err := s.db.Close()
	s.db = nil
	return err
}

func isPointerStruct(i interface{}) bool {
	t := reflect.TypeOf(i)
	return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
}

func (s *Store) Register(i interface{}) error {
	if s.db == nil {
		return DBClosed
	} else if !isPointerStruct(i) {
		return NoPointerStruct
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.defineType(i)
}

func (s *Store) defineType(i interface{}) error {
	name := typeName(i)
	if _, ok := s.types[name]; ok {
		return nil
	}

	s.types[name] = typeInfo{}

	v := reflect.ValueOf(i).Elem()
	numFields := v.Type().NumField()
	fields := make([]field, 0, numFields)
	id := 0
	idType := 0

	for n := 0; n < numFields; n++ {
		f := v.Type().Field(n)
		if f.PkgPath != "" { // not exported
			continue
		}
		fieldName := f.Name
		if fn := f.Tag.Get("store"); fn != "" {
			fieldName = fn
		}
		if fieldName == "-" { // Skip field
			continue
		}
		tmp := strings.ToLower(fieldName)
		for _, tf := range fields {
			if strings.ToLower(tf.name) == tmp {
				return DuplicateColumn
			}
		}
		isPointer := f.Type.Kind() == reflect.Ptr
		var iface interface{}
		if isPointer {
			iface = v.Field(n).Interface()
		} else {
			iface = v.Field(n).Addr().Interface()
		}
		isStruct := false
		if isPointerStruct(iface) {
			s.defineType(iface)
			isStruct = true
		} else if !isValidType(iface) {
			continue
		}
		if isValidKeyType(iface) {
			if idType < 3 && f.Tag.Get("key") == "1" {
				idType = 3
				id = len(fields)
			} else if idType < 2 && strings.ToLower(fieldName) == "id" {
				idType = 2
				id = len(fields)
			} else if idType < 1 {
				idType = 1
				id = len(fields)
			}
		}
		fields = append(fields, field{
			fieldName,
			n,
			isPointer,
			isStruct,
		})
	}
	if idType == 0 {
		return NoKey
	}
	s.types[name] = typeInfo{
		primary: id,
	}

	// create statements
	var (
		sqlVars, sqlParams, setSQLParams, tableVars string
		doneFirst, doneFirstNonKey                  bool
	)

	for pos, f := range fields {
		if doneFirst {
			tableVars += ", "
		} else {
			doneFirst = true
		}
		if pos != id {
			if doneFirstNonKey {
				sqlVars += ", "
				setSQLParams += ", "
				sqlParams += ", "
			} else {
				doneFirstNonKey = true
			}
		}
		var varType string
		if f.isStruct {
			varType = "INTEGER"
		} else {
			varType = getType(i, f.pos)
		}
		tableVars += "[" + f.name + "] " + varType
		if pos == id {
			tableVars += " PRIMARY KEY AUTOINCREMENT"
		} else {
			sqlVars += "[" + f.name + "]"
			setSQLParams += "[" + f.name + "] = ?"
			sqlParams += "?"
		}
	}

	statements := make([]*sql.Stmt, 6)

	sql := "CREATE TABLE IF NOT EXISTS [" + name + "](" + tableVars + ");"
	_, err := s.db.Exec(sql)
	if err != nil {
		return err
	}

	sql = "INSERT INTO [" + name + "] (" + sqlVars + ") VALUES (" + sqlParams + ");"
	stmt, err := s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[add] = stmt

	sql = "SELECT " + sqlVars + " FROM [" + name + "] WHERE [" + fields[id].name + "] = ? LIMIT 1;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[get] = stmt

	sql = "UPDATE [" + name + "] SET " + setSQLParams + " WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[update] = stmt

	sql = "DELETE FROM [" + name + "] WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[remove] = stmt

	sql = "SELECT " + sqlVars + ", [" + fields[id].name + "] FROM [" + name + "] ORDER BY [" + fields[id].name + "] LIMIT ? OFFSET ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[getPage] = stmt

	sql = "SELECT COUNT(1) FROM [" + name + "];"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[count] = stmt

	s.types[name] = typeInfo{
		primary:    id,
		fields:     fields,
		statements: statements,
	}
	return nil
}

func (s *Store) Set(is ...interface{}) error {
	var toSet []interface{}
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return UnregisteredType
		}
		toSet = toSet[:0]
		err := s.Set(i, &t, &toSet)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) set(i interface{}, t *typeInfo, toSet *[]interface{}) error {
	for _, oi := range *toSet {
		if oi == i {
			return nil
		}
	}
	(*toSet) = append(*toSet, i)
	id := t.GetID(i)
	isUpdate := id != 0
	vars := make([]interface{}, 0, len(t.fields))
	for pos, f := range t.fields {
		if pos == t.primary {
			continue
		}
		if f.isStruct {
			ni := getFieldPointer(i, f.pos)
			nt := s.types[typeName(ni)]
			err := s.Set(ni, &nt, toSet)
			if err != nil {
				return err
			}
			vars = append(vars, getField(ni, nt.fields[nt.primary].pos))
		} else {
			vars = append(vars, getField(i, f.pos))
		}
	}
	if isUpdate {
		vars = append(vars, id)
		_, err := t.statements[update].Exec(vars...)
		return err
	}
	r, err := t.statements[add].Exec(vars...)
	if err != nil {
		return err
	}
	lid, err := r.LastInsertId()
	if err != nil {
		return err
	}
	t.SetID(i, lid)
	return nil
}

func (s *Store) Get(is ...interface{}) error {
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return UnregisteredType
		}
		id := t.GetID(i)
		if id == 0 {
			continue
		}
		vars := make([]interface{}, 0, len(t.fields))
		var toGet []interface{}
		for pos, f := range t.fields {
			if pos == t.primary {
				continue
			}
			if f.isStruct {
				ni := getFieldPointer(i, f.pos)
				nt := s.types[typeName(ni)]
				toGet = append(toGet, ni)
				vars = append(vars, getFieldPointer(ni, nt.fields[nt.primary].pos))
			} else {
				vars = append(vars, getFieldPointer(i, f.pos))
			}
		}
		row := t.statements[get].QueryRow(id)
		err := row.Scan(vars...)
		if err != nil {
			return err
		}
		if len(toGet) > 0 {
			if err = s.Get(toGet...); err != nil {
				return err
			}
		}
	}
	return nil
}

func (s *Store) Count(i interface{}) (int, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if !isPointerStruct(i) {
		return 0, NoPointerStruct
	}
	name := typeName(i)
	stmt := s.types[name].statements[count]
	res, err := stmt.Query()
	if err != nil {
		return 0, err
	}
	num := 0
	err = res.Scan(&num)
	return num, err
}

// Errors

var (
	DBClosed         = errors.New("database already closed")
	NoPointerStruct  = errors.New("given variable is not a pointer to a struct")
	NoKey            = errors.New("could not determine key")
	DuplicateColumn  = errors.New("duplicate column name found")
	UnregisteredType = errors.New("type not registered")
)
<commit_msg>Removed uneeded isPointer field<commit_after>// Package store automatically configures a database to store structured information in an sql database
package store

import (
	"database/sql"
	"errors"
	"reflect"
	"strings"
	"sync"
)

const (
	add = iota
	get
	update
	remove
	getPage
	count
)

type field struct {
	isStruct bool
	pos      int
	name     string
}

type typeInfo struct {
	primary    int
	fields     []field
	statements []*sql.Stmt
}

type Store struct {
	db    *sql.DB
	types map[string]typeInfo
	mutex sync.Mutex
}

func New(driverName, dataSourceName string) (*Store, error) {
	db, err := sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return &Store{
		db:    db,
		types: make(map[string]typeInfo),
	}, nil
}

func (s *Store) Close() error {
	err := s.db.Close()
	s.db = nil
	return err
}

func isPointerStruct(i interface{}) bool {
	t := reflect.TypeOf(i)
	return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
}

func (s *Store) Register(i interface{}) error {
	if s.db == nil {
		return DBClosed
	} else if !isPointerStruct(i) {
		return NoPointerStruct
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.defineType(i)
}

func (s *Store) defineType(i interface{}) error {
	name := typeName(i)
	if _, ok := s.types[name]; ok {
		return nil
	}

	s.types[name] = typeInfo{}

	v := reflect.ValueOf(i).Elem()
	numFields := v.Type().NumField()
	fields := make([]field, 0, numFields)
	id := 0
	idType := 0

	for n := 0; n < numFields; n++ {
		f := v.Type().Field(n)
		if f.PkgPath != "" { // not exported
			continue
		}
		fieldName := f.Name
		if fn := f.Tag.Get("store"); fn != "" {
			fieldName = fn
		}
		if fieldName == "-" { // Skip field
			continue
		}
		tmp := strings.ToLower(fieldName)
		for _, tf := range fields {
			if strings.ToLower(tf.name) == tmp {
				return DuplicateColumn
			}
		}
		isPointer := f.Type.Kind() == reflect.Ptr
		var iface interface{}
		if isPointer {
			iface = v.Field(n).Interface()
		} else {
			iface = v.Field(n).Addr().Interface()
		}
		isStruct := false
		if isPointerStruct(iface) {
			s.defineType(iface)
			isStruct = true
		} else if !isValidType(iface) {
			continue
		}
		if isValidKeyType(iface) {
			if idType < 3 && f.Tag.Get("key") == "1" {
				idType = 3
				id = len(fields)
			} else if idType < 2 && strings.ToLower(fieldName) == "id" {
				idType = 2
				id = len(fields)
			} else if idType < 1 {
				idType = 1
				id = len(fields)
			}
		}
		fields = append(fields, field{
			isStruct,
			n,
			fieldName,
		})
	}
	if idType == 0 {
		return NoKey
	}
	s.types[name] = typeInfo{
		primary: id,
	}

	// create statements
	var (
		sqlVars, sqlParams, setSQLParams, tableVars string
		doneFirst, doneFirstNonKey                  bool
	)

	for pos, f := range fields {
		if doneFirst {
			tableVars += ", "
		} else {
			doneFirst = true
		}
		if pos != id {
			if doneFirstNonKey {
				sqlVars += ", "
				setSQLParams += ", "
				sqlParams += ", "
			} else {
				doneFirstNonKey = true
			}
		}
		var varType string
		if f.isStruct {
			varType = "INTEGER"
		} else {
			varType = getType(i, f.pos)
		}
		tableVars += "[" + f.name + "] " + varType
		if pos == id {
			tableVars += " PRIMARY KEY AUTOINCREMENT"
		} else {
			sqlVars += "[" + f.name + "]"
			setSQLParams += "[" + f.name + "] = ?"
			sqlParams += "?"
		}
	}

	statements := make([]*sql.Stmt, 6)

	sql := "CREATE TABLE IF NOT EXISTS [" + name + "](" + tableVars + ");"
	_, err := s.db.Exec(sql)
	if err != nil {
		return err
	}

	sql = "INSERT INTO [" + name + "] (" + sqlVars + ") VALUES (" + sqlParams + ");"
	stmt, err := s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[add] = stmt

	sql = "SELECT " + sqlVars + " FROM [" + name + "] WHERE [" + fields[id].name + "] = ? LIMIT 1;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[get] = stmt

	sql = "UPDATE [" + name + "] SET " + setSQLParams + " WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[update] = stmt

	sql = "DELETE FROM [" + name + "] WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[remove] = stmt

	sql = "SELECT " + sqlVars + ", [" + fields[id].name + "] FROM [" + name + "] ORDER BY [" + fields[id].name + "] LIMIT ? OFFSET ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[getPage] = stmt

	sql = "SELECT COUNT(1) FROM [" + name + "];"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[count] = stmt

	s.types[name] = typeInfo{
		primary:    id,
		fields:     fields,
		statements: statements,
	}
	return nil
}

func (s *Store) Set(is ...interface{}) error {
	var toSet []interface{}
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return UnregisteredType
		}
		toSet = toSet[:0]
		err := s.Set(i, &t, &toSet)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) set(i interface{}, t *typeInfo, toSet *[]interface{}) error {
	for _, oi := range *toSet {
		if oi == i {
			return nil
		}
	}
	(*toSet) = append(*toSet, i)
	id := t.GetID(i)
	isUpdate := id != 0
	vars := make([]interface{}, 0, len(t.fields))
	for pos, f := range t.fields {
		if pos == t.primary {
			continue
		}
		if f.isStruct {
			ni := getFieldPointer(i, f.pos)
			nt := s.types[typeName(ni)]
			err := s.Set(ni, &nt, toSet)
			if err != nil {
				return err
			}
			vars = append(vars, getField(ni, nt.fields[nt.primary].pos))
		} else {
			vars = append(vars, getField(i, f.pos))
		}
	}
	if isUpdate {
		vars = append(vars, id)
		_, err := t.statements[update].Exec(vars...)
		return err
	}
	r, err := t.statements[add].Exec(vars...)
	if err != nil {
		return err
	}
	lid, err := r.LastInsertId()
	if err != nil {
		return err
	}
	t.SetID(i, lid)
	return nil
}

func (s *Store) Get(is ...interface{}) error {
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return UnregisteredType
		}
		id := t.GetID(i)
		if id == 0 {
			continue
		}
		vars := make([]interface{}, 0, len(t.fields))
		var toGet []interface{}
		for pos, f := range t.fields {
			if pos == t.primary {
				continue
			}
			if f.isStruct {
				ni := getFieldPointer(i, f.pos)
				nt := s.types[typeName(ni)]
				toGet = append(toGet, ni)
				vars = append(vars, getFieldPointer(ni, nt.fields[nt.primary].pos))
			} else {
				vars = append(vars, getFieldPointer(i, f.pos))
			}
		}
		row := t.statements[get].QueryRow(id)
		err := row.Scan(vars...)
		if err != nil {
			return err
		}
		if len(toGet) > 0 {
			if err = s.Get(toGet...); err != nil {
				return err
			}
		}
	}
	return nil
}

func (s *Store) Count(i interface{}) (int, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if !isPointerStruct(i) {
		return 0, NoPointerStruct
	}
	name := typeName(i)
	stmt := s.types[name].statements[count]
	res, err := stmt.Query()
	if err != nil {
		return 0, err
	}
	num := 0
	err = res.Scan(&num)
	return num, err
}

// Errors

var (
	DBClosed         = errors.New("database already closed")
	NoPointerStruct  = errors.New("given variable is not a pointer to a struct")
	NoKey            = errors.New("could not determine key")
	DuplicateColumn  = errors.New("duplicate column name found")
	UnregisteredType = errors.New("type not registered")
)
<|endoftext|>