<commit_before>package mackerel

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Host struct {
	Id            string      `json:"id,omitempty"`
	Name          string      `json:"name,omitempty"`
	Type          string      `json:"type,omitempty"`
	Status        string      `json:"status,omitempty"`
	Memo          string      `json:"memo,omitempty"`
	Roles         Roles       `json:"roles,omitempty"`
	RoleFullnames []string    `json:"roleFullnames,omitempty"`
	IsRetired     bool        `json:"isRetired,omitempty"`
	CreatedAt     int32       `json:"createdAt,omitempty"`
	Meta          HostMeta    `json:"meta,omitempty"`
	Interfaces    []Interface `json:"interfaces,omitempty"`
}

type Roles map[string][]string

type HostMeta struct {
	AgentRevision string      `json:"agent-revision,omitempty"`
	AgentVersion  string      `json:"agent-version,omitempty"`
	BlockDevice   BlockDevice `json:"block_device,omitempty"`
	Cpu           CPU         `json:"cpu,omitempty"`
	Filesystem    FileSystem  `json:"filesystem,omitempty"`
	Kernel        Kernel      `json:"kernel,omitempty"`
	Memory        Memory      `json:"memory,omitempty"`
}

type BlockDevice map[string]map[string]interface{}
type CPU []map[string]interface{}
type FileSystem map[string]interface{}
type Kernel map[string]string
type Memory map[string]string

type Interface struct {
	Name       string `json:"name,omitempty"`
	IPAddress  string `json:"ipAddress,omitempty"`
	MacAddress string `json:"macAddress,omitempty"`
}

type FindHostsParam struct {
	Service  string
	Roles    []string
	Name     string
	Statuses []string
}

type CreateHostParam struct {
	Name          string      `json:"name,omitempty"`
	Meta          HostMeta    `json:"meta,omitempty"`
	Interfaces    []Interface `json:"interfaces,omitempty"`
	RoleFullnames []string    `json:"roleFullnames,omitempty"`
}

type UpdateHostParam CreateHostParam

func (h *Host) GetRoleFullnames() []string {
	if len(h.Roles) < 1 {
		return nil
	}

	var fullnames []string
	for service, roles := range h.Roles {
		for _, role := range roles {
			fullname := strings.Join([]string{service, role}, ":")
			fullnames = append(fullnames, fullname)
		}
	}

	return fullnames
}

func (h *Host) DateFromCreatedAt() time.Time {
	return time.Unix(int64(h.CreatedAt), 0)
}

func (h *Host) DateStringFromCreatedAt() string {
	const layout = "Jan 2, 2006 at 3:04pm (MST)"
	return h.DateFromCreatedAt().Format(layout)
}

func (c *Client) FindHost(id string) (*Host, error) {
	req, err := http.NewRequest("GET", c.urlFor(fmt.Sprintf("/api/v0/hosts/%s", id)).String(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New("status code is not 200")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var data struct {
		Host *Host `json:"host"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return nil, err
	}

	return data.Host, err
}

func (c *Client) FindHosts(param *FindHostsParam) ([]*Host, error) {
	v := url.Values{}
	if param.Service != "" {
		v.Set("service", param.Service)
	}
	if len(param.Roles) >= 1 {
		for _, role := range param.Roles {
			v.Add("role", role)
		}
	}
	if param.Name != "" {
		v.Set("name", param.Name)
	}
	if len(param.Statuses) >= 1 {
		for _, status := range param.Statuses {
			v.Add("status", status)
		}
	}

	req, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", c.urlFor("/api/v0/hosts.json").String(), v.Encode()), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New("status code is not 200")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var data struct {
		Hosts []*(Host) `json:"hosts"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return nil, err
	}

	return data.Hosts, err
}

func (c *Client) CreateHost(param *CreateHostParam) (string, error) {
	requestJson, err := json.Marshal(param)
	if err != nil {
		return "", err
	}

	req, err := http.NewRequest(
		"POST",
		c.urlFor("/api/v0/hosts").String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return "", err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var data struct {
		Id string `json:"id"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return "", err
	}

	return data.Id, nil
}

func (c *Client) UpdateHost(hostId string, param *UpdateHostParam) (string, error) {
	requestJson, err := json.Marshal(param)
	if err != nil {
		return "", err
	}

	req, err := http.NewRequest(
		"PUT",
		c.urlFor(fmt.Sprintf("/api/v0/hosts/%s", hostId)).String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return "", err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var data struct {
		Id string `json:"id"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return "", err
	}

	return data.Id, nil
}

func (c *Client) UpdateHostStatus(hostId string, status string) error {
	requestJson, err := json.Marshal(map[string]string{
		"status": status,
	})
	if err != nil {
		return err
	}

	req, err := http.NewRequest(
		"POST",
		c.urlFor(fmt.Sprintf("/api/v0/hosts/%s/status", hostId)).String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

func (c *Client) RetireHost(id string) error {
	requestJson, _ := json.Marshal("{}")

	req, err := http.NewRequest(
		"POST",
		c.urlFor(fmt.Sprintf("/api/v0/hosts/%s/retire", id)).String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New("status code is not 200")
	}

	return nil
}
<commit_msg>Impl Host.IpAddresses<commit_after>package mackerel

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Host struct {
	Id            string      `json:"id,omitempty"`
	Name          string      `json:"name,omitempty"`
	Type          string      `json:"type,omitempty"`
	Status        string      `json:"status,omitempty"`
	Memo          string      `json:"memo,omitempty"`
	Roles         Roles       `json:"roles,omitempty"`
	RoleFullnames []string    `json:"roleFullnames,omitempty"`
	IsRetired     bool        `json:"isRetired,omitempty"`
	CreatedAt     int32       `json:"createdAt,omitempty"`
	Meta          HostMeta    `json:"meta,omitempty"`
	Interfaces    []Interface `json:"interfaces,omitempty"`
}

type Roles map[string][]string

type HostMeta struct {
	AgentRevision string      `json:"agent-revision,omitempty"`
	AgentVersion  string      `json:"agent-version,omitempty"`
	BlockDevice   BlockDevice `json:"block_device,omitempty"`
	Cpu           CPU         `json:"cpu,omitempty"`
	Filesystem    FileSystem  `json:"filesystem,omitempty"`
	Kernel        Kernel      `json:"kernel,omitempty"`
	Memory        Memory      `json:"memory,omitempty"`
}

type BlockDevice map[string]map[string]interface{}
type CPU []map[string]interface{}
type FileSystem map[string]interface{}
type Kernel map[string]string
type Memory map[string]string

type Interface struct {
	Name       string `json:"name,omitempty"`
	IPAddress  string `json:"ipAddress,omitempty"`
	MacAddress string `json:"macAddress,omitempty"`
}

type FindHostsParam struct {
	Service  string
	Roles    []string
	Name     string
	Statuses []string
}

type CreateHostParam struct {
	Name          string      `json:"name,omitempty"`
	Meta          HostMeta    `json:"meta,omitempty"`
	Interfaces    []Interface `json:"interfaces,omitempty"`
	RoleFullnames []string    `json:"roleFullnames,omitempty"`
}

type UpdateHostParam CreateHostParam

func (h *Host) GetRoleFullnames() []string {
	if len(h.Roles) < 1 {
		return nil
	}

	var fullnames []string
	for service, roles := range h.Roles {
		for _, role := range roles {
			fullname := strings.Join([]string{service, role}, ":")
			fullnames = append(fullnames, fullname)
		}
	}

	return fullnames
}

func (h *Host) DateFromCreatedAt() time.Time {
	return time.Unix(int64(h.CreatedAt), 0)
}

func (h *Host) DateStringFromCreatedAt() string {
	const layout = "Jan 2, 2006 at 3:04pm (MST)"
	return h.DateFromCreatedAt().Format(layout)
}

func (h *Host) IpAddresses() map[string]string {
	if len(h.Interfaces) < 1 {
		return nil
	}

	ipAddresses := make(map[string]string, 0)
	for _, iface := range h.Interfaces {
		ipAddresses[iface.Name] = iface.IPAddress
	}
	return ipAddresses
}

func (c *Client) FindHost(id string) (*Host, error) {
	req, err := http.NewRequest("GET", c.urlFor(fmt.Sprintf("/api/v0/hosts/%s", id)).String(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New("status code is not 200")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var data struct {
		Host *Host `json:"host"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return nil, err
	}

	return data.Host, err
}

func (c *Client) FindHosts(param *FindHostsParam) ([]*Host, error) {
	v := url.Values{}
	if param.Service != "" {
		v.Set("service", param.Service)
	}
	if len(param.Roles) >= 1 {
		for _, role := range param.Roles {
			v.Add("role", role)
		}
	}
	if param.Name != "" {
		v.Set("name", param.Name)
	}
	if len(param.Statuses) >= 1 {
		for _, status := range param.Statuses {
			v.Add("status", status)
		}
	}

	req, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", c.urlFor("/api/v0/hosts.json").String(), v.Encode()), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New("status code is not 200")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var data struct {
		Hosts []*(Host) `json:"hosts"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return nil, err
	}

	return data.Hosts, err
}

func (c *Client) CreateHost(param *CreateHostParam) (string, error) {
	requestJson, err := json.Marshal(param)
	if err != nil {
		return "", err
	}

	req, err := http.NewRequest(
		"POST",
		c.urlFor("/api/v0/hosts").String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return "", err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var data struct {
		Id string `json:"id"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return "", err
	}

	return data.Id, nil
}

func (c *Client) UpdateHost(hostId string, param *UpdateHostParam) (string, error) {
	requestJson, err := json.Marshal(param)
	if err != nil {
		return "", err
	}

	req, err := http.NewRequest(
		"PUT",
		c.urlFor(fmt.Sprintf("/api/v0/hosts/%s", hostId)).String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return "", err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var data struct {
		Id string `json:"id"`
	}

	err = json.Unmarshal(body, &data)
	if err != nil {
		return "", err
	}

	return data.Id, nil
}

func (c *Client) UpdateHostStatus(hostId string, status string) error {
	requestJson, err := json.Marshal(map[string]string{
		"status": status,
	})
	if err != nil {
		return err
	}

	req, err := http.NewRequest(
		"POST",
		c.urlFor(fmt.Sprintf("/api/v0/hosts/%s/status", hostId)).String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

func (c *Client) RetireHost(id string) error {
	requestJson, _ := json.Marshal("{}")

	req, err := http.NewRequest(
		"POST",
		c.urlFor(fmt.Sprintf("/api/v0/hosts/%s/retire", id)).String(),
		bytes.NewReader(requestJson),
	)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New("status code is not 200")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Process etc.

package os

import "syscall"

// Args hold the command-line arguments, starting with the program name.
var Args []string

func init() {
	Args = runtime_args()
}

func runtime_args() []string // in package runtime

// Getuid returns the numeric user id of the caller.
func Getuid() int { return syscall.Getuid() }

// Geteuid returns the numeric effective user id of the caller.
func Geteuid() int { return syscall.Geteuid() }

// Getgid returns the numeric group id of the caller.
func Getgid() int { return syscall.Getgid() }

// Getegid returns the numeric effective group id of the caller.
func Getegid() int { return syscall.Getegid() }

// Getgroups returns a list of the numeric ids of groups that the caller belongs to.
func Getgroups() ([]int, error) {
	gids, e := syscall.Getgroups()
	return gids, NewSyscallError("getgroups", e)
}

// Exit causes the current program to exit with the given status code.
// Conventionally, code zero indicates success, non-zero an error.
// The program terminates immediately; deferred functions are
// not run.
func Exit(code int) { syscall.Exit(code) }
<commit_msg>os: fix Args setup on Windows<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Process etc.

package os

import (
	"runtime"
	"syscall"
)

// Args hold the command-line arguments, starting with the program name.
var Args []string

func init() {
	if runtime.GOOS == "windows" {
		// Initialized in exec_windows.go.
		return
	}
	Args = runtime_args()
}

func runtime_args() []string // in package runtime

// Getuid returns the numeric user id of the caller.
func Getuid() int { return syscall.Getuid() }

// Geteuid returns the numeric effective user id of the caller.
func Geteuid() int { return syscall.Geteuid() }

// Getgid returns the numeric group id of the caller.
func Getgid() int { return syscall.Getgid() }

// Getegid returns the numeric effective group id of the caller.
func Getegid() int { return syscall.Getegid() }

// Getgroups returns a list of the numeric ids of groups that the caller belongs to.
func Getgroups() ([]int, error) {
	gids, e := syscall.Getgroups()
	return gids, NewSyscallError("getgroups", e)
}

// Exit causes the current program to exit with the given status code.
// Conventionally, code zero indicates success, non-zero an error.
// The program terminates immediately; deferred functions are
// not run.
func Exit(code int) { syscall.Exit(code) }
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package mux

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/issue9/assert"
)

// 一些预定义的处理函数
var (
	f1 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(1)
	}
	f2 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(2)
	}
	f3 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(3)
	}
	f4 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(4)
	}
	f5 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(5)
	}
	f6 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(6)
	}
)

func request(a *assert.Assertion, srvmux *ServeMux, method, url string, status int) {
	w := httptest.NewRecorder()
	a.NotNil(w)

	r, err := http.NewRequest(method, url, nil)
	a.NotError(err).NotNil(r)

	srvmux.ServeHTTP(w, r)
	a.Equal(w.Code, status)
}

func TestClearPath(t *testing.T) {
	a := assert.New(t)

	a.Equal(cleanPath(""), "/")
	a.Equal(cleanPath("/api//"), "/api/")

	a.Equal(cleanPath("/api/"), "/api/")
	a.Equal(cleanPath("/api/./"), "/api/")

	a.Equal(cleanPath("/api/.."), "/")
	a.Equal(cleanPath("/api/../"), "/")

	a.Equal(cleanPath("/api/../../"), "/")
	a.Equal(cleanPath("/api../"), "/api../")
}

func TestServeMux_Add_Remove_1(t *testing.T) {
	a := assert.New(t)

	srvmux := NewServeMux(false)
	a.NotNil(srvmux)

	// 添加 delete /api/1
	a.NotPanic(func() {
		srvmux.DeleteFunc("/api/1", f1)
	})
	a.Equal(srvmux.entries.Len(), 1)

	// 添加 patch /api/1
	a.NotPanic(func() {
		srvmux.PatchFunc("/api/1", f1)
	})
	a.Equal(srvmux.entries.Len(), 1) // 加在同一个 Entry 下，所以数量不变

	// 添加 post /api/2
	a.NotPanic(func() {
		srvmux.PostFunc("/api/2", f1)
	})
	a.Equal(srvmux.entries.Len(), 2)

	// 删除 any /api/2
	srvmux.Remove("/api/2")
	a.Equal(srvmux.entries.Len(), 1)

	// 删除 delete /api/1
	srvmux.Remove("/api/1", http.MethodDelete)
	a.Equal(srvmux.entries.Len(), 1)

	// 删除 patch /api/1
	srvmux.Remove("/api/1", http.MethodPatch)
	a.Equal(srvmux.entries.Len(), 0)
}

func TestServeMux_Add_Remove_2(t *testing.T) {
	a := assert.New(t)
	srvmux := NewServeMux(false)
	a.NotNil(srvmux)

	// 添加 GET /api/1
	// 添加 PUT /api/1
	// 添加 GET /api/2
	a.NotError(srvmux.AddFunc("/api/1", f1, http.MethodGet))
	a.NotPanic(func() {
		srvmux.PutFunc("/api/1", f1)
	})
	a.NotPanic(func() {
		srvmux.GetFunc("/api/2", f2)
	})
	request(a, srvmux, http.MethodGet, "/api/1", 1)
	request(a, srvmux, http.MethodPut, "/api/1", 1)
	request(a, srvmux, http.MethodGet, "/api/2", 2)
	request(a, srvmux, http.MethodDelete, "/api/1", http.StatusMethodNotAllowed) // 未实现

	// 删除 GET /api/1
	srvmux.Remove("/api/1", http.MethodGet)
	request(a, srvmux, http.MethodGet, "/api/1", http.StatusMethodNotAllowed)
	request(a, srvmux, http.MethodPut, "/api/1", 1) // 不影响 PUT
	request(a, srvmux, http.MethodGet, "/api/2", 2)

	// 删除 GET /api/2，只有一个，所以相当于整个 Entry 被删除
	srvmux.Remove("/api/2", http.MethodGet)
	request(a, srvmux, http.MethodGet, "/api/1", http.StatusMethodNotAllowed)
	request(a, srvmux, http.MethodPut, "/api/1", 1)                   // 不影响 PUT
	request(a, srvmux, http.MethodGet, "/api/2", http.StatusNotFound) // 整个 entry 被删除

	// 添加 POST /api/1
	a.NotPanic(func() {
		srvmux.PostFunc("/api/1", f1)
	})
	request(a, srvmux, http.MethodPost, "/api/1", 1)

	// 删除 ANY /api/1
	srvmux.Remove("/api/1")
	request(a, srvmux, http.MethodPost, "/api/1", http.StatusNotFound) // 404 表示整个 entry 都没了
}

// 测试匹配顺序是否正确
func TestServeMux_ServeHTTP_Order(t *testing.T) {
	a := assert.New(t)

	test := func(m *ServeMux, method, host, path string, code int) {
		r, err := http.NewRequest(method, path, nil)
		if len(host) > 0 {
			r.Host = host
		}
		a.NotError(err).NotNil(r)
		w := httptest.NewRecorder()
		a.NotNil(w)
		m.ServeHTTP(w, r)
		a.Equal(w.Code, code)
	}

	serveMux := NewServeMux(false)
	a.NotNil(serveMux)
	a.NotError(serveMux.AddFunc("/post/", f1, "GET"))          // f1
	a.NotError(serveMux.AddFunc("/post/{id:\\d+}", f2, "GET")) // f2
	a.NotError(serveMux.AddFunc("/post/1", f3, "GET"))         // f3

	test(serveMux, "GET", "", "/post/1", 3)   // f3 静态路由项完全匹配
	test(serveMux, "GET", "", "/post/2", 2)   // f2 正则完全匹配
	test(serveMux, "GET", "", "/post/abc", 1) // f1 匹配度最高
}

func TestMethodIsSupported(t *testing.T) {
	a := assert.New(t)

	a.True(MethodIsSupported("get"))
	a.True(MethodIsSupported("POST"))
	a.False(MethodIsSupported("not exists"))
}
<commit_msg>整理部分测试代码<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package mux

import (
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/issue9/assert"
)

// 一些预定义的处理函数
var (
	f1 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(1)
	}
	f2 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(2)
	}
	f3 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(3)
	}
	f4 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(4)
	}
	f5 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(5)
	}
	f6 = func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(6)
	}
)

func request(a *assert.Assertion, srvmux *ServeMux, method, url string, status int) {
	w := httptest.NewRecorder()
	a.NotNil(w)

	r, err := http.NewRequest(method, url, nil)
	a.NotError(err).NotNil(r)

	srvmux.ServeHTTP(w, r)
	a.Equal(w.Code, status)
}

func TestClearPath(t *testing.T) {
	a := assert.New(t)

	a.Equal(cleanPath(""), "/")
	a.Equal(cleanPath("/api//"), "/api/")

	a.Equal(cleanPath("/api/"), "/api/")
	a.Equal(cleanPath("/api/./"), "/api/")

	a.Equal(cleanPath("/api/.."), "/")
	a.Equal(cleanPath("/api/../"), "/")

	a.Equal(cleanPath("/api/../../"), "/")
	a.Equal(cleanPath("/api../"), "/api../")
}

func TestServeMux_Add_Remove_1(t *testing.T) {
	a := assert.New(t)

	srvmux := NewServeMux(false)
	a.NotNil(srvmux)

	// 添加 delete /api/1
	a.NotPanic(func() {
		srvmux.DeleteFunc("/api/1", f1)
	})
	a.Equal(srvmux.entries.Len(), 1)

	// 添加 patch /api/1
	a.NotPanic(func() {
		srvmux.PatchFunc("/api/1", f1)
	})
	a.Equal(srvmux.entries.Len(), 1) // 加在同一个 Entry 下，所以数量不变

	// 添加 post /api/2
	a.NotPanic(func() {
		srvmux.PostFunc("/api/2", f1)
	})
	a.Equal(srvmux.entries.Len(), 2)

	// 删除 any /api/2
	srvmux.Remove("/api/2")
	a.Equal(srvmux.entries.Len(), 1)

	// 删除 delete /api/1
	srvmux.Remove("/api/1", http.MethodDelete)
	a.Equal(srvmux.entries.Len(), 1)

	// 删除 patch /api/1
	srvmux.Remove("/api/1", http.MethodPatch)
	a.Equal(srvmux.entries.Len(), 0)
}

func TestServeMux_Add_Remove_2(t *testing.T) {
	a := assert.New(t)
	srvmux := NewServeMux(false)
	a.NotNil(srvmux)

	// 添加 GET /api/1
	// 添加 PUT /api/1
	// 添加 GET /api/2
	a.NotError(srvmux.AddFunc("/api/1", f1, http.MethodGet))
	a.NotPanic(func() {
		srvmux.PutFunc("/api/1", f1)
	})
	a.NotPanic(func() {
		srvmux.GetFunc("/api/2", f2)
	})
	request(a, srvmux, http.MethodGet, "/api/1", 1)
	request(a, srvmux, http.MethodPut, "/api/1", 1)
	request(a, srvmux, http.MethodGet, "/api/2", 2)
	request(a, srvmux, http.MethodDelete, "/api/1", http.StatusMethodNotAllowed) // 未实现

	// 删除 GET /api/1
	srvmux.Remove("/api/1", http.MethodGet)
	request(a, srvmux, http.MethodGet, "/api/1", http.StatusMethodNotAllowed)
	request(a, srvmux, http.MethodPut, "/api/1", 1) // 不影响 PUT
	request(a, srvmux, http.MethodGet, "/api/2", 2)

	// 删除 GET /api/2，只有一个，所以相当于整个 Entry 被删除
	srvmux.Remove("/api/2", http.MethodGet)
	request(a, srvmux, http.MethodGet, "/api/1", http.StatusMethodNotAllowed)
	request(a, srvmux, http.MethodPut, "/api/1", 1)                   // 不影响 PUT
	request(a, srvmux, http.MethodGet, "/api/2", http.StatusNotFound) // 整个 entry 被删除

	// 添加 POST /api/1
	a.NotPanic(func() {
		srvmux.PostFunc("/api/1", f1)
	})
	request(a, srvmux, http.MethodPost, "/api/1", 1)

	// 删除 ANY /api/1
	srvmux.Remove("/api/1")
	request(a, srvmux, http.MethodPost, "/api/1", http.StatusNotFound) // 404 表示整个 entry 都没了
}

func TestServeMux_Options(t *testing.T) {
	a := assert.New(t)
	srvmux := NewServeMux(false)
	a.NotNil(srvmux)

	// TODO
}

func TestServeMux_Params(t *testing.T) {
	a := assert.New(t)
	srvmux := NewServeMux(false)
	a.NotNil(srvmux)

	// TODO
}

// 测试匹配顺序是否正确
func TestServeMux_ServeHTTP_Order(t *testing.T) {
	a := assert.New(t)
	serveMux := NewServeMux(false)
	a.NotNil(serveMux)

	a.NotError(serveMux.AddFunc("/post/", f1, "GET"))          // f1
	a.NotError(serveMux.AddFunc("/post/{id:\\d+}", f2, "GET")) // f2
	a.NotError(serveMux.AddFunc("/post/1", f3, "GET"))         // f3

	request(a, serveMux, "GET", "/post/1", 3)   // f3 静态路由项完全匹配
	request(a, serveMux, "GET", "/post/2", 2)   // f2 正则完全匹配
	request(a, serveMux, "GET", "/post/abc", 1) // f1 匹配度最高
}

func TestMethodIsSupported(t *testing.T) {
	a := assert.New(t)

	a.True(MethodIsSupported("get"))
	a.True(MethodIsSupported("POST"))
	a.False(MethodIsSupported("not exists"))
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package domain

import (
	"sync"
	"time"
)

// SchemaValidator is the interface for checking the validity of schema version.
type SchemaValidator interface {
	// Update the schema validator, add a new item, delete the expired items.
	// The schemaVer is valid within leaseGrantTime plus lease duration.
	Update(leaseGrantTime uint64, schemaVer int64)
	// Check is it valid for a transaction to use schemaVer, at timestamp txnTS.
	Check(txnTS uint64, schemaVer int64) bool
	// Latest returns the latest schema version it knows, but not necessary a valid one.
	Latest() int64
}

type schemaValidator struct {
	mux             sync.RWMutex
	lease           time.Duration
	items           map[int64]time.Time
	latestSchemaVer int64
}

func newSchemaValidator(lease time.Duration) SchemaValidator {
	return &schemaValidator{
		lease: lease,
		items: make(map[int64]time.Time),
	}
}

func (s *schemaValidator) Update(leaseGrantTS uint64, schemaVer int64) {
	s.mux.Lock()

	s.latestSchemaVer = schemaVer
	leaseGrantTime := extractPhysicalTime(leaseGrantTS)
	leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond)

	// Renewal lease.
	s.items[schemaVer] = leaseExpire

	// Delete expired items, leaseGrantTime is server current time, actually.
	for k, expire := range s.items {
		if leaseGrantTime.After(expire) {
			delete(s.items, k)
		}
	}

	s.mux.Unlock()
}

// Check checks schema validity, returns true if use schemaVer at txnTS is legal.
func (s *schemaValidator) Check(txnTS uint64, schemaVer int64) bool {
	s.mux.RLock()
	defer s.mux.RUnlock()

	if s.lease == 0 {
		return true
	}

	expire, ok := s.items[schemaVer]
	if !ok {
		// Can't find schema version means it's already expired.
		return false
	}

	t := extractPhysicalTime(txnTS)
	if t.After(expire) {
		return false
	}

	return true
}

// Latest returns the latest schema version it knows.
func (s *schemaValidator) Latest() int64 {
	return s.latestSchemaVer
}

func extractPhysicalTime(ts uint64) time.Time {
	t := int64(ts >> 18) // 18 for physicalShiftBits
	return time.Unix(t/1e3, (t%1e3)*1e6)
}
<commit_msg>domain: fix a data race (#2397)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package domain

import (
	"sync"
	"time"
)

// SchemaValidator is the interface for checking the validity of schema version.
type SchemaValidator interface {
	// Update the schema validator, add a new item, delete the expired items.
	// The schemaVer is valid within leaseGrantTime plus lease duration.
	Update(leaseGrantTime uint64, schemaVer int64)
	// Check is it valid for a transaction to use schemaVer, at timestamp txnTS.
	Check(txnTS uint64, schemaVer int64) bool
	// Latest returns the latest schema version it knows, but not necessary a valid one.
	Latest() int64
}

type schemaValidator struct {
	mux             sync.RWMutex
	lease           time.Duration
	items           map[int64]time.Time
	latestSchemaVer int64
}

func newSchemaValidator(lease time.Duration) SchemaValidator {
	return &schemaValidator{
		lease: lease,
		items: make(map[int64]time.Time),
	}
}

func (s *schemaValidator) Update(leaseGrantTS uint64, schemaVer int64) {
	s.mux.Lock()

	s.latestSchemaVer = schemaVer
	leaseGrantTime := extractPhysicalTime(leaseGrantTS)
	leaseExpire := leaseGrantTime.Add(s.lease - time.Millisecond)

	// Renewal lease.
	s.items[schemaVer] = leaseExpire

	// Delete expired items, leaseGrantTime is server current time, actually.
	for k, expire := range s.items {
		if leaseGrantTime.After(expire) {
			delete(s.items, k)
		}
	}

	s.mux.Unlock()
}

// Check checks schema validity, returns true if use schemaVer at txnTS is legal.
func (s *schemaValidator) Check(txnTS uint64, schemaVer int64) bool {
	s.mux.RLock()
	defer s.mux.RUnlock()

	if s.lease == 0 {
		return true
	}

	expire, ok := s.items[schemaVer]
	if !ok {
		// Can't find schema version means it's already expired.
		return false
	}

	t := extractPhysicalTime(txnTS)
	if t.After(expire) {
		return false
	}

	return true
}

// Latest returns the latest schema version it knows.
func (s *schemaValidator) Latest() int64 {
	s.mux.RLock()
	ret := s.latestSchemaVer
	s.mux.RUnlock()
	return ret
}

func extractPhysicalTime(ts uint64) time.Time {
	t := int64(ts >> 18) // 18 for physicalShiftBits
	return time.Unix(t/1e3, (t%1e3)*1e6)
}
<|endoftext|>


<commit_before>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"fmt"
	"io"
)

const (
	HeaderSize = 10
)

// Tag represents an ID3v2 tag
type Tag struct {
	Header
	frames  map[string][]Framer
	padding uint
}

// Creates a new tag
func NewTag(reader io.Reader) *Tag {
	t := &Tag{NewHeader(reader), make(map[string][]Framer), 0}
	if t.Header == nil {
		return nil
	}

	var frame Framer
	size := t.Header.Size()
	for size > 0 {
		switch t.Header.Version() {
		case "2.3.0":
			frame = NewV3Frame(reader)
		default:
			frame = NewV3Frame(reader)
		}

		if frame == nil {
			break
		}

		id := frame.Id()
		t.frames[id] = append(t.frames[id], frame)

		size -= FrameHeaderSize + frame.Size()
	}

	t.padding = uint(size)
	nAdvance := int(t.padding - FrameHeaderSize)
	if n, err := io.ReadFull(reader, make([]byte, nAdvance)); n != nAdvance || err != nil {
		return nil
	}

	return t
}

// Size of the tag
// Recalculated as frames and padding can be changed
func (t Tag) Size() int {
	size := 0
	for _, v := range t.frames {
		for _, f := range v {
			size += FrameHeaderSize + f.Size()
		}
	}

	headerSize := t.Header.Size()
	if padding := headerSize - size; padding < 0 {
		t.padding = 0
		head := t.Header.(Head)
		head.size = int32(size)
		return size
	} else {
		t.padding = uint(padding)
		return headerSize
	}
}

func (t Tag) Bytes() []byte {
	data := make([]byte, t.Size())

	index := 0
	for _, v := range t.frames {
		for _, f := range v {
			size := FrameHeaderSize + f.Size()

			switch t.Header.Version() {
			case "2.3":
				copy(data[index:index+size], V3Bytes(f))
			default:
				copy(data[index:index+size], V3Bytes(f))
			}

			index += size
		}
	}

	return append(t.Header.Bytes(), data...)
}

// All frames
func (t Tag) Frames(id string) []Framer {
	if frames, ok := t.frames[id]; ok && frames != nil {
		return frames
	}

	return nil
}

// First frame
func (t Tag) Frame(id string) Framer {
	if frames := t.Frames(id); frames != nil {
		return frames[0]
	}

	return nil
}

// Delete and return all frames
func (t *Tag) DeleteFrames(id string) []Framer {
	frames := t.Frames(id)
	if frames == nil {
		return nil
	}

	delete(t.frames, id)

	return frames
}

// Add frame
func (t *Tag) AddFrame(frame Framer) {
	id := frame.Id()
	t.frames[id] = append(t.frames[id], frame)
}

// Header represents the useful information contained in the data
type Header interface {
	Version() string
	Size() int
	Bytes() []byte
}

func (t Tag) Title() string {
	return t.textFrameText("TIT2")
}

func (t Tag) Artist() string {
	return t.textFrameText("TPE1")
}

func (t Tag) Album() string {
	return t.textFrameText("TALB")
}

func (t Tag) Genre() string {
	return t.textFrameText("TCON")
}

func (t *Tag) SetTitle(text string) {
	t.setTextFrameText("TIT2", text)
}

func (t *Tag) SetArtist(text string) {
	t.setTextFrameText("TPE1", text)
}

func (t *Tag) SetAlbum(text string) {
	t.setTextFrameText("TALB", text)
}

func (t *Tag) SetGenre(text string) {
	t.setTextFrameText("TCON", text)
}

func (t Tag) textFrameText(id string) string {
	if frame := t.Frame(id); frame != nil {
		switch frame.(type) {
		case (*TextFrame):
			return frame.(*TextFrame).Text()
		default:
		}
	}

	return ""
}

func (t *Tag) setTextFrameText(id, text string) {
	if frame := t.Frame(id); frame != nil {
		switch frame.(type) {
		case (*TextFrame):
			frame.(*TextFrame).SetText(text)
		default:
		}
	}
}

func NewHeader(reader io.Reader) Header {
	data := make([]byte, HeaderSize)
	n, err := io.ReadFull(reader, data)
	if n < HeaderSize || err != nil || string(data[:3]) != "ID3" {
		return nil
	}

	size, err := synchint(data[6:])
	if err != nil {
		return nil
	}

	return Head{
		version:  data[3],
		revision: data[4],
		flags:    data[5],
		size:     size,
	}
}

// Head represents the data of the header of the entire tag
type Head struct {
	version, revision byte
	flags             byte
	size              int32
}

func (h Head) Version() string {
	return fmt.Sprintf("2.%d.%d", h.version, h.revision)
}

func (h Head) Size() int {
	return int(h.size)
}

func (h Head) Bytes() []byte {
	data := make([]byte, HeaderSize)

	copy(data[:3], []byte("ID3"))
	copy(data[6:], synchbytes(h.size))
	data[3] = h.version
	data[4] = h.revision
	data[5] = h.flags

	return data
}
<commit_msg>Add year convenience method<commit_after>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"fmt"
	"io"
)

const (
	HeaderSize = 10
)

// Tag represents an ID3v2 tag
type Tag struct {
	Header
	frames  map[string][]Framer
	padding uint
}

// Creates a new tag
func NewTag(reader io.Reader) *Tag {
	t := &Tag{NewHeader(reader), make(map[string][]Framer), 0}
	if t.Header == nil {
		return nil
	}

	var frame Framer
	size := t.Header.Size()
	for size > 0 {
		switch t.Header.Version() {
		case "2.3.0":
			frame = NewV3Frame(reader)
		default:
			frame = NewV3Frame(reader)
		}

		if frame == nil {
			break
		}

		id := frame.Id()
		t.frames[id] = append(t.frames[id], frame)

		size -= FrameHeaderSize + frame.Size()
	}

	t.padding = uint(size)
	nAdvance := int(t.padding - FrameHeaderSize)
	if n, err := io.ReadFull(reader, make([]byte, nAdvance)); n != nAdvance || err != nil {
		return nil
	}

	return t
}

// Size of the tag
// Recalculated as frames and padding can be changed
func (t Tag) Size() int {
	size := 0
	for _, v := range t.frames {
		for _, f := range v {
			size += FrameHeaderSize + f.Size()
		}
	}

	headerSize := t.Header.Size()
	if padding := headerSize - size; padding < 0 {
		t.padding = 0
		head := t.Header.(Head)
		head.size = int32(size)
		return size
	} else {
		t.padding = uint(padding)
		return headerSize
	}
}

func (t Tag) Bytes() []byte {
	data := make([]byte, t.Size())

	index := 0
	for _, v := range t.frames {
		for _, f := range v {
			size := FrameHeaderSize + f.Size()

			switch t.Header.Version() {
			case "2.3":
				copy(data[index:index+size], V3Bytes(f))
			default:
				copy(data[index:index+size], V3Bytes(f))
			}

			index += size
		}
	}

	return append(t.Header.Bytes(), data...)
}

// All frames
func (t Tag) Frames(id string) []Framer {
	if frames, ok := t.frames[id]; ok && frames != nil {
		return frames
	}

	return nil
}

// First frame
func (t Tag) Frame(id string) Framer {
	if frames := t.Frames(id); frames != nil {
		return frames[0]
	}

	return nil
}

// Delete and return all frames
func (t *Tag) DeleteFrames(id string) []Framer {
	frames := t.Frames(id)
	if frames == nil {
		return nil
	}

	delete(t.frames, id)

	return frames
}

// Add frame
func (t *Tag) AddFrame(frame Framer) {
	id := frame.Id()
	t.frames[id] = append(t.frames[id], frame)
}

// Header represents the useful information contained in the data
type Header interface {
	Version() string
	Size() int
	Bytes() []byte
}

func (t Tag) Title() string {
	return t.textFrameText("TIT2")
}

func (t Tag) Artist() string {
	return t.textFrameText("TPE1")
}

func (t Tag) Album() string {
	return t.textFrameText("TALB")
}

func (t Tag) Year() string {
	return t.textFrameText("TYER")
}

func (t Tag) Genre() string {
	return t.textFrameText("TCON")
}

func (t *Tag) SetTitle(text string) {
	t.setTextFrameText("TIT2", text)
}

func (t *Tag) SetArtist(text string) {
	t.setTextFrameText("TPE1", text)
}

func (t *Tag) SetAlbum(text string) {
	t.setTextFrameText("TALB", text)
}

func (t *Tag) SetGenre(text string) {
	t.setTextFrameText("TCON", text)
}

func (t *Tag) SetYear(text string) {
	t.setTextFrameText("TYER", text)
}

func (t Tag) textFrameText(id string) string {
	if frame := t.Frame(id); frame != nil {
		switch frame.(type) {
		case (*TextFrame):
			return frame.(*TextFrame).Text()
		default:
		}
	}

	return ""
}

func (t *Tag) setTextFrameText(id, text string) {
	if frame := t.Frame(id); frame != nil {
		switch frame.(type) {
		case (*TextFrame):
			frame.(*TextFrame).SetText(text)
		default:
		}
	}
}

func NewHeader(reader io.Reader) Header {
	data := make([]byte, HeaderSize)
	n, err := io.ReadFull(reader, data)
	if n < HeaderSize || err != nil || string(data[:3]) != "ID3" {
		return nil
	}

	size, err := synchint(data[6:])
	if err != nil {
		return nil
	}

	return Head{
		version:  data[3],
		revision: data[4],
		flags:    data[5],
		size:     size,
	}
}

// Head represents the data of the header of the entire tag
type Head struct {
	version, revision byte
	flags             byte
	size              int32
}

func (h Head) Version() string {
	return fmt.Sprintf("2.%d.%d", h.version, h.revision)
}

func (h Head) Size() int {
	return int(h.size)
}

func (h Head) Bytes() []byte {
	data := make([]byte, HeaderSize)

	copy(data[:3], []byte("ID3"))
	copy(data[6:], synchbytes(h.size))
	data[3] = h.version
	data[4] = h.revision
	data[5] = h.flags

	return data
}
<|endoftext|>


<commit_before>// Copyright 2014 go-dockerclient authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

// This work with api verion < v1.7 and > v1.9
type APIImages struct {
	ID          string   `json:"Id"`
	RepoTags    []string `json:",omitempty"`
	Created     int64
	Size        int64
	VirtualSize int64
	ParentId    string `json:",omitempty"`
	Repository  string `json:",omitempty"`
	Tag         string `json:",omitempty"`
}

// Error returned when the image does not exist.
var ErrNoSuchImage = errors.New("No such image")
var ErrMissingRepo = errors.New("Missing remote repository e.g. 'github.com/user/repo'")
var ErrMissingOutputStream = errors.New("Missing output-stream")

// ListImages returns the list of available images in the server.
//
// See http://goo.gl/dkMrwP for more details.
func (c *Client) ListImages(all bool) ([]APIImages, error) {
	path := "/images/json?all="
	if all {
		path += "1"
	} else {
		path += "0"
	}
	body, _, err := c.do("GET", path, nil)
	if err != nil {
		return nil, err
	}
	var images []APIImages
	err = json.Unmarshal(body, &images)
	if err != nil {
		return nil, err
	}
	return images, nil
}

// RemoveImage removes an image by its name or ID.
//
// See http://goo.gl/7hjHHy for more details.
func (c *Client) RemoveImage(name string) error {
	_, status, err := c.do("DELETE", "/images/"+name, nil)
	if status == http.StatusNotFound {
		return ErrNoSuchImage
	}
	return err
}

// InspectImage returns an image by its name or ID.
//
// See http://goo.gl/pHEbma for more details.
func (c *Client) InspectImage(name string) (*Image, error) {
	body, status, err := c.do("GET", "/images/"+name+"/json", nil)
	if status == http.StatusNotFound {
		return nil, ErrNoSuchImage
	}
	if err != nil {
		return nil, err
	}
	var image Image
	err = json.Unmarshal(body, &image)
	if err != nil {
		return nil, err
	}
	return &image, nil
}

// PushImageOptions represents options to use in the PushImage method.
//
// See http://goo.gl/GBmyhc for more details.
type PushImageOptions struct {
	// Name of the image
	Name string

	// Registry server to push the image
	Registry string
}

// AuthConfiguration represents authentication options to use in the PushImage
// method. It represents the authencation in the Docker index server.
type AuthConfiguration struct {
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
	Email    string `json:"email,omitempty"`
}

// PushImage pushes an image to a remote registry, logging progress to w.
//
// An empty instance of AuthConfiguration may be used for unauthenticated
// pushes.
//
// See http://goo.gl/GBmyhc for more details.
func (c *Client) PushImage(opts PushImageOptions, auth AuthConfiguration, w io.Writer) error {
	if opts.Name == "" {
		return ErrNoSuchImage
	}
	name := opts.Name
	opts.Name = ""
	path := "/images/" + name + "/push?" + queryString(&opts)
	var buf bytes.Buffer
	json.NewEncoder(&buf).Encode(auth)
	return c.stream("POST", path, &buf, w)
}

// PullImageOptions present the set of options available for pulling an image
// from a registry.
//
// See http://goo.gl/PhBKnS for more details.
type PullImageOptions struct {
	Repository string `qs:"fromImage"`
	Registry   string
}

// PullImage pulls an image from a remote registry, logging progress to w.
//
// See http://goo.gl/PhBKnS for more details.
func (c *Client) PullImage(opts PullImageOptions, w io.Writer) error {
	if opts.Repository == "" {
		return ErrNoSuchImage
	}
	return c.createImage(queryString(&opts), nil, w)
}

func (c *Client) createImage(qs string, in io.Reader, w io.Writer) error {
	path := "/images/create?" + qs
	return c.stream("POST", path, in, w)
}

// ImportImageOptions present the set of informations available for importing
// an image from a source file or the stdin.
//
// See http://goo.gl/PhBKnS for more details.
type ImportImageOptions struct {
	Repository string `qs:"repo"`
	Source     string `qs:"fromSrc"`
}

// ImportImage imports an image from a url, a file or stdin
//
// See http://goo.gl/PhBKnS for more details.
func (c *Client) ImportImage(opts ImportImageOptions, in io.Reader, out io.Writer) error {
	if opts.Repository == "" {
		return ErrNoSuchImage
	}
	if opts.Source != "-" {
		in = nil
	}
	if opts.Source != "-" && !isUrl(opts.Source) {
		f, err := os.Open(opts.Source)
		if err != nil {
			return err
		}
		b, err := ioutil.ReadAll(f)
		in = bytes.NewBuffer(b)
		opts.Source = "-"
	}
	return c.createImage(queryString(&opts), in, out)
}

// BuildImageOptions present the set of informations available for building
// an image from a tarball's url.
type BuildImageOptions struct {
	Name           string    `qs:"t"`
	Remote         string    `qs:"remote"`
	SuppressOutput string    `qs:"q"`
	OutputStream   io.Writer `qs:"-"`
}

// BuildImage builds an image from a tarball's url.
func (c *Client) BuildImage(opts BuildImageOptions) error {
	if opts.Remote == "" {
		return ErrMissingRepo
	}

	// Name the image by default with the repository identifier e.g.
	// "github.com/user/repo"
	if opts.Name == "" {
		opts.Name = opts.Remote
	}

	// Suppress output by default.
	if opts.SuppressOutput == "" || opts.SuppressOutput != "0" || opts.SuppressOutput != "1" {
		opts.SuppressOutput = "1"
	}

	if opts.OutputStream == nil {
		return ErrMissingOutputStream
	}

	// Call api server.
	err := c.stream("POST", fmt.Sprintf("/build?%s", queryString(&opts)), nil, opts.OutputStream)
	return err
}

func isUrl(u string) bool {
	p, err := url.Parse(u)
	if err != nil {
		return false
	}
	return p.Scheme == "http" || p.Scheme == "https"
}
<commit_msg>removed blank lines inside the BuildImage method<commit_after>// Copyright 2014 go-dockerclient authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

// This work with api verion < v1.7 and > v1.9
type APIImages struct {
	ID          string   `json:"Id"`
	RepoTags    []string `json:",omitempty"`
	Created     int64
	Size        int64
	VirtualSize int64
	ParentId    string `json:",omitempty"`
	Repository  string `json:",omitempty"`
	Tag         string `json:",omitempty"`
}

// Error returned when the image does not exist.
var ErrNoSuchImage = errors.New("No such image")
var ErrMissingRepo = errors.New("Missing remote repository e.g. 'github.com/user/repo'")
var ErrMissingOutputStream = errors.New("Missing output-stream")

// ListImages returns the list of available images in the server.
//
// See http://goo.gl/dkMrwP for more details.
func (c *Client) ListImages(all bool) ([]APIImages, error) {
	path := "/images/json?all="
	if all {
		path += "1"
	} else {
		path += "0"
	}
	body, _, err := c.do("GET", path, nil)
	if err != nil {
		return nil, err
	}
	var images []APIImages
	err = json.Unmarshal(body, &images)
	if err != nil {
		return nil, err
	}
	return images, nil
}

// RemoveImage removes an image by its name or ID.
//
// See http://goo.gl/7hjHHy for more details.
func (c *Client) RemoveImage(name string) error {
	_, status, err := c.do("DELETE", "/images/"+name, nil)
	if status == http.StatusNotFound {
		return ErrNoSuchImage
	}
	return err
}

// InspectImage returns an image by its name or ID.
//
// See http://goo.gl/pHEbma for more details.
func (c *Client) InspectImage(name string) (*Image, error) {
	body, status, err := c.do("GET", "/images/"+name+"/json", nil)
	if status == http.StatusNotFound {
		return nil, ErrNoSuchImage
	}
	if err != nil {
		return nil, err
	}
	var image Image
	err = json.Unmarshal(body, &image)
	if err != nil {
		return nil, err
	}
	return &image, nil
}

// PushImageOptions represents options to use in the PushImage method.
//
// See http://goo.gl/GBmyhc for more details.
type PushImageOptions struct {
	// Name of the image
	Name string

	// Registry server to push the image
	Registry string
}

// AuthConfiguration represents authentication options to use in the PushImage
// method. It represents the authencation in the Docker index server.
type AuthConfiguration struct {
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
	Email    string `json:"email,omitempty"`
}

// PushImage pushes an image to a remote registry, logging progress to w.
//
// An empty instance of AuthConfiguration may be used for unauthenticated
// pushes.
//
// See http://goo.gl/GBmyhc for more details.
func (c *Client) PushImage(opts PushImageOptions, auth AuthConfiguration, w io.Writer) error {
	if opts.Name == "" {
		return ErrNoSuchImage
	}
	name := opts.Name
	opts.Name = ""
	path := "/images/" + name + "/push?" + queryString(&opts)
	var buf bytes.Buffer
	json.NewEncoder(&buf).Encode(auth)
	return c.stream("POST", path, &buf, w)
}

// PullImageOptions present the set of options available for pulling an image
// from a registry.
//
// See http://goo.gl/PhBKnS for more details.
type PullImageOptions struct {
	Repository string `qs:"fromImage"`
	Registry   string
}

// PullImage pulls an image from a remote registry, logging progress to w.
//
// See http://goo.gl/PhBKnS for more details.
func (c *Client) PullImage(opts PullImageOptions, w io.Writer) error {
	if opts.Repository == "" {
		return ErrNoSuchImage
	}
	return c.createImage(queryString(&opts), nil, w)
}

func (c *Client) createImage(qs string, in io.Reader, w io.Writer) error {
	path := "/images/create?" + qs
	return c.stream("POST", path, in, w)
}

// ImportImageOptions present the set of informations available for importing
// an image from a source file or the stdin.
//
// See http://goo.gl/PhBKnS for more details.
type ImportImageOptions struct {
	Repository string `qs:"repo"`
	Source     string `qs:"fromSrc"`
}

// ImportImage imports an image from a url, a file or stdin
//
// See http://goo.gl/PhBKnS for more details.
func (c *Client) ImportImage(opts ImportImageOptions, in io.Reader, out io.Writer) error {
	if opts.Repository == "" {
		return ErrNoSuchImage
	}
	if opts.Source != "-" {
		in = nil
	}
	if opts.Source != "-" && !isUrl(opts.Source) {
		f, err := os.Open(opts.Source)
		if err != nil {
			return err
		}
		b, err := ioutil.ReadAll(f)
		in = bytes.NewBuffer(b)
		opts.Source = "-"
	}
	return c.createImage(queryString(&opts), in, out)
}

// BuildImageOptions present the set of informations available for building
// an image from a tarball's url.
type BuildImageOptions struct {
	Name           string    `qs:"t"`
	Remote         string    `qs:"remote"`
	SuppressOutput string    `qs:"q"`
	OutputStream   io.Writer `qs:"-"`
}

// BuildImage builds an image from a tarball's url.
func (c *Client) BuildImage(opts BuildImageOptions) error {
	if opts.Remote == "" {
		return ErrMissingRepo
	}
	// Name the image by default with the repository identifier e.g.
	// "github.com/user/repo"
	if opts.Name == "" {
		opts.Name = opts.Remote
	}
	// Suppress output by default.
	if opts.SuppressOutput == "" || opts.SuppressOutput != "0" || opts.SuppressOutput != "1" {
		opts.SuppressOutput = "1"
	}
	if opts.OutputStream == nil {
		return ErrMissingOutputStream
	}
	// Call api server.
	err := c.stream("POST", fmt.Sprintf("/build?%s", queryString(&opts)), nil, opts.OutputStream)
	return err
}

func isUrl(u string) bool {
	p, err := url.Parse(u)
	if err != nil {
		return false
	}
	return p.Scheme == "http" || p.Scheme == "https"
}
<|endoftext|>


<commit_before>package types

import "fmt"

type Kind byte

const (
	NativeFit Kind = 0x00 // Standard 	-> Fit base type/alias
	TimeUTC   Kind = 0x01 // Time UTC 	-> time.Time
	TimeLocal Kind = 0x02 // Time Local 	-> time.Time with Location
	Lat       Kind = 0x03 // Latitude 	-> fit.Latitude
	Lng       Kind = 0x04 // Longitude 	-> fit.Longitude
)

func (k Kind) String() string {
	if k > 0x04 {
		return fmt.Sprintf("unknown kind (%d)", k)
	}
	return kname[k]
}

var kname = [...]string{
	"NativeFit",
	"TimeUTC",
	"TimeLocal",
	"Lat",
	"Lng",
}

func Make(kind Kind, array bool) Fit {
	var f Fit
	if array {
		f = f.setArray()
	}
	if kind == NativeFit {
		return f
	}
	f = f.setKind(kind)
	switch kind {
	case TimeUTC, TimeLocal:
		f = f.setBase(BaseUint32)
	case Lat, Lng:
		f = f.setBase(BaseSint32)
	}
	return f
}

func MakeNative(b Base, array bool) Fit {
	var f Fit
	f = f.setBase(b)
	if array {
		f = f.setArray()
	}
	return f
}

// Fit bit-packing layout:
//
// ----------------------------------------
// Three most significant bits: Kind
// Forth most significant bit:  Array flag
// Four least significant bits: Base type
// ----------------------------------------
//
type Fit byte

func (f Fit) setKind(k Kind) Fit {
	k = k << 5
	return Fit(byte(f) | byte(k))
}

func (f Fit) setArray() Fit {
	f = f | 0x10
	return f
}

func (f Fit) setBase(b Base) Fit {
	b = b & 0x0F
	return Fit(byte(f) | byte(b))
}
func (f Fit) Kind() Kind {
	return Kind((f & 0xE0) >> 5)
}

func (f Fit) Array() bool {
	return (f&0x10)>>4 == 1
}

func (f Fit) BaseType() Base {
	return Base(f & 0x0F)
}

func (f Fit) Valid() bool {
	return int(f.Kind()) < len(fgotype) && f.BaseType().Known()
}

func (f Fit) GoInvalidValue() string {
	if !f.Valid() {
		return "invalid type: " + f.String()
	}
	if f.Array() {
		return "nil"
	}
	if f.Kind() == NativeFit {
		return f.BaseType().GoInvalidValue()
	}
	return fgoinvalid[f.Kind()]
}

func (f Fit) GoType() string {
	if !f.Valid() {
		return "invalid type: " + f.String()
	}
	var gt string
	if f.Kind() == NativeFit {
		gt = f.BaseType().GoType()
	} else {
		gt = fgotype[f.Kind()]
	}
	if f.Array() {
		return "[]" + gt
	}
	return gt
}

func (f Fit) String() string {
	return fmt.Sprintf(
		"kind: %v |  base type: %v | array: %t",
		f.Kind(), f.BaseType(), f.Array(),
	)
}

func (f Fit) ValueString() string {
	return fmt.Sprintf("types.Fit(%d)", f)
}

var fgoinvalid = [...]string{
	"",
	"timeBase",
	"timeBase",
	"NewLatitudeInvalid()",
	"NewLongitudeInvalid()",
}

var fgotype = [...]string{
	"",
	"time.Time",
	"time.Time",
	"Latitude",
	"Longitude",
}
<commit_msg>internal/types: adjust Fit type to allow space for new base types<commit_after>package types

import "fmt"

type Kind byte

const (
	NativeFit Kind = 0x00 // Standard 	-> Fit base type/alias
	TimeUTC   Kind = 0x01 // Time UTC 	-> time.Time
	TimeLocal Kind = 0x02 // Time Local 	-> time.Time with Location
	Lat       Kind = 0x03 // Latitude 	-> fit.Latitude
	Lng       Kind = 0x04 // Longitude 	-> fit.Longitude
)

func (k Kind) String() string {
	if k > 0x04 {
		return fmt.Sprintf("unknown kind (%d)", k)
	}
	return kname[k]
}

var kname = [...]string{
	"NativeFit",
	"TimeUTC",
	"TimeLocal",
	"Lat",
	"Lng",
}

func Make(kind Kind, array bool) Fit {
	var f Fit
	if array {
		f = f.setArray()
	}
	if kind == NativeFit {
		return f
	}
	f = f.setKind(kind)
	switch kind {
	case TimeUTC, TimeLocal:
		f = f.setBase(BaseUint32)
	case Lat, Lng:
		f = f.setBase(BaseSint32)
	}
	return f
}

func MakeNative(b Base, array bool) Fit {
	var f Fit
	f = f.setBase(b)
	if array {
		f = f.setArray()
	}
	return f
}

// type Fit uint16 - bit-packing layout:
//
// +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
// | bF | bE | bD | bC | bB | bA | b9 | b8 | b7 | b6 | b5 | b4 | b3 | b2 | b1 | b0 |
// +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
//  \________________________________/\______________/\__/\________________________/
//
//                Unused                    Kind      Array       Base type
type Fit uint16

func (f Fit) setKind(k Kind) Fit {
	return Fit(uint16(f) | uint16(k)<<6)
}

func (f Fit) setArray() Fit {
	f = f | 0x20
	return f
}

func (f Fit) setBase(b Base) Fit {
	return Fit(uint16(f) | uint16(b&0x1F))
}

func (f Fit) Kind() Kind {
	return Kind((f & 0x1C0) >> 6)
}

func (f Fit) Array() bool {
	return (f&0x20)>>5 == 1
}

func (f Fit) BaseType() Base {
	return Base(f & 0x1F)
}

func (f Fit) Valid() bool {
	return int(f.Kind()) < len(fgotype) && f.BaseType().Known()
}

func (f Fit) GoInvalidValue() string {
	if !f.Valid() {
		return "invalid type: " + f.String()
	}
	if f.Array() {
		return "nil"
	}
	if f.Kind() == NativeFit {
		return f.BaseType().GoInvalidValue()
	}
	return fgoinvalid[f.Kind()]
}

func (f Fit) GoType() string {
	if !f.Valid() {
		return "invalid type: " + f.String()
	}
	var gt string
	if f.Kind() == NativeFit {
		gt = f.BaseType().GoType()
	} else {
		gt = fgotype[f.Kind()]
	}
	if f.Array() {
		return "[]" + gt
	}
	return gt
}

func (f Fit) String() string {
	return fmt.Sprintf(
		"kind: %v |  base type: %v | array: %t",
		f.Kind(), f.BaseType(), f.Array(),
	)
}

func (f Fit) ValueString() string {
	return fmt.Sprintf("types.Fit(%d)", f)
}

var fgoinvalid = [...]string{
	"",
	"timeBase",
	"timeBase",
	"NewLatitudeInvalid()",
	"NewLongitudeInvalid()",
}

var fgotype = [...]string{
	"",
	"time.Time",
	"time.Time",
	"Latitude",
	"Longitude",
}
<|endoftext|>


<commit_before>package ipam

import (
	"context"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/docker/libnetwork/ipamapi"
	"golang.org/x/sync/semaphore"
	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
)

const (
	all = iota
	even
	odd
)

type releaseMode uint

type testContext struct {
	a      *Allocator
	opts   map[string]string
	ipList []*net.IPNet
	ipMap  map[string]bool
	pid    string
	maxIP  int
}

func newTestContext(t *testing.T, mask int, options map[string]string) *testContext {
	a, err := getAllocator(false)
	if err != nil {
		t.Fatal(err)
	}
	a.addrSpaces["giallo"] = &addrSpace{
		id:      dsConfigKey + "/" + "giallo",
		ds:      a.addrSpaces[localAddressSpace].ds,
		alloc:   a.addrSpaces[localAddressSpace].alloc,
		scope:   a.addrSpaces[localAddressSpace].scope,
		subnets: map[SubnetKey]*PoolData{},
	}

	network := fmt.Sprintf("192.168.100.0/%d", mask)
	// total ips 2^(32-mask) - 2 (network and broadcast)
	totalIps := 1<<uint(32-mask) - 2

	pid, _, _, err := a.RequestPool("giallo", network, "", nil, false)
	if err != nil {
		t.Fatal(err)
	}

	return &testContext{
		a:      a,
		opts:   options,
		ipList: make([]*net.IPNet, 0, totalIps),
		ipMap:  make(map[string]bool),
		pid:    pid,
		maxIP:  totalIps,
	}
}

func TestDebug(t *testing.T) {
	tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
}

type op struct {
	id   int32
	add  bool
	name string
}

func (o *op) String() string {
	return fmt.Sprintf("%+v", *o)
}

func TestRequestPoolParallel(t *testing.T) {
	a, err := getAllocator(false)
	if err != nil {
		t.Fatal(err)
	}
	var operationIndex int32
	ch := make(chan *op, 240)
	for i := 0; i < 120; i++ {
		go func(t *testing.T, a *Allocator, ch chan *op) {
			name, _, _, err := a.RequestPool("GlobalDefault", "", "", nil, false)
			if err != nil {
				t.Fatalf("request error %v", err)
			}
			idx := atomic.AddInt32(&operationIndex, 1)
			ch <- &op{idx, true, name}
			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
			idx = atomic.AddInt32(&operationIndex, 1)
			err = a.ReleasePool(name)
			if err != nil {
				t.Fatalf("relase error %v", err)
			}
			ch <- &op{idx, false, name}
		}(t, a, ch)
	}

	// map of events
	m := make(map[string][]*op)
	for i := 0; i < 240; i++ {
		x := <-ch
		ops, ok := m[x.name]
		if !ok {
			ops = make([]*op, 0, 10)
		}
		ops = append(ops, x)
		m[x.name] = ops
	}

	// Post processing to avoid event reordering on the channel
	for pool, ops := range m {
		sort.Slice(ops[:], func(i, j int) bool {
			return ops[i].id < ops[j].id
		})
		expected := true
		for _, op := range ops {
			if op.add != expected {
				t.Fatalf("Operations for %v not valid %v, operations %v", pool, op, ops)
			}
			expected = !expected
		}
	}
}

func TestFullAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, all, parallelism)
		}
	}
}

func TestOddAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, odd, parallelism)
		}
	}
}

func TestFullAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, all, parallelism)
	}
}

func TestOddAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, odd, parallelism)
	}
}

func TestEvenAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, even, parallelism)
	}
}

func allocate(t *testing.T, tctx *testContext, parallel int64) {
	// Allocate the whole space
	parallelExec := semaphore.NewWeighted(parallel)
	routineNum := tctx.maxIP + 10
	ch := make(chan *net.IPNet, routineNum)
	var id int
	var wg sync.WaitGroup
	// routine loop
	for {
		wg.Add(1)
		go func(id int) {
			parallelExec.Acquire(context.Background(), 1)
			ip, _, _ := tctx.a.RequestAddress(tctx.pid, nil, tctx.opts)
			ch <- ip
			parallelExec.Release(1)
			wg.Done()
		}(id)
		id++
		if id == routineNum {
			break
		}
	}

	// give time to all the go routines to finish
	wg.Wait()

	// process results
	for i := 0; i < routineNum; i++ {
		ip := <-ch
		if ip == nil {
			continue
		}
		if there, ok := tctx.ipMap[ip.String()]; ok && there {
			t.Fatalf("Got duplicate IP %s", ip.String())
			break
		}
		tctx.ipList = append(tctx.ipList, ip)
		tctx.ipMap[ip.String()] = true
	}

	assert.Check(t, is.Len(tctx.ipList, tctx.maxIP))
	if len(tctx.ipList) != tctx.maxIP {
		t.Fatal("missmatch number allocation")
	}
}

func release(t *testing.T, tctx *testContext, mode releaseMode, parallel int64) {
	var startIndex, increment, stopIndex, length int
	switch mode {
	case all:
		startIndex = 0
		increment = 1
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP
	case odd, even:
		if mode == odd {
			startIndex = 1
		}
		increment = 2
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP / 2
		if tctx.maxIP%2 > 0 {
			length++
		}
	default:
		t.Fatal("unsupported mode yet")
	}

	ipIndex := make([]int, 0, length)
	// calculate the index to release from the ipList
	for i := startIndex; ; i += increment {
		ipIndex = append(ipIndex, i)
		if i+increment > stopIndex {
			break
		}
	}

	var id int
	parallelExec := semaphore.NewWeighted(parallel)
	ch := make(chan *net.IPNet, len(ipIndex))
	wg := sync.WaitGroup{}
	for index := range ipIndex {
		wg.Add(1)
		go func(id, index int) {
			parallelExec.Acquire(context.Background(), 1)
			// logrus.Errorf("index %v", index)
			// logrus.Errorf("list %v", tctx.ipList)
			err := tctx.a.ReleaseAddress(tctx.pid, tctx.ipList[index].IP)
			if err != nil {
				t.Fatalf("routine %d got %v", id, err)
			}
			ch <- tctx.ipList[index]
			parallelExec.Release(1)
			wg.Done()
		}(id, index)
		id++
	}
	wg.Wait()

	for i := 0; i < len(ipIndex); i++ {
		ip := <-ch

		// check if it is really free
		_, _, err := tctx.a.RequestAddress(tctx.pid, ip.IP, nil)
		assert.Check(t, err, "ip %v not properly released", ip)
		if err != nil {
			t.Fatalf("ip %v not properly released, error:%v", ip, err)
		}
		err = tctx.a.ReleaseAddress(tctx.pid, ip.IP)
		assert.NilError(t, err)

		if there, ok := tctx.ipMap[ip.String()]; !ok || !there {
			t.Fatalf("ip %v got double deallocated", ip)
		}
		tctx.ipMap[ip.String()] = false
		for j, v := range tctx.ipList {
			if v == ip {
				tctx.ipList = append(tctx.ipList[:j], tctx.ipList[j+1:]...)
				break
			}
		}
	}

	assert.Check(t, is.Len(tctx.ipList, tctx.maxIP-length))
}
<commit_msg>Fix some typos<commit_after>package ipam

import (
	"context"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/docker/libnetwork/ipamapi"
	"golang.org/x/sync/semaphore"
	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
)

const (
	all = iota
	even
	odd
)

type releaseMode uint

type testContext struct {
	a      *Allocator
	opts   map[string]string
	ipList []*net.IPNet
	ipMap  map[string]bool
	pid    string
	maxIP  int
}

func newTestContext(t *testing.T, mask int, options map[string]string) *testContext {
	a, err := getAllocator(false)
	if err != nil {
		t.Fatal(err)
	}
	a.addrSpaces["giallo"] = &addrSpace{
		id:      dsConfigKey + "/" + "giallo",
		ds:      a.addrSpaces[localAddressSpace].ds,
		alloc:   a.addrSpaces[localAddressSpace].alloc,
		scope:   a.addrSpaces[localAddressSpace].scope,
		subnets: map[SubnetKey]*PoolData{},
	}

	network := fmt.Sprintf("192.168.100.0/%d", mask)
	// total ips 2^(32-mask) - 2 (network and broadcast)
	totalIps := 1<<uint(32-mask) - 2

	pid, _, _, err := a.RequestPool("giallo", network, "", nil, false)
	if err != nil {
		t.Fatal(err)
	}

	return &testContext{
		a:      a,
		opts:   options,
		ipList: make([]*net.IPNet, 0, totalIps),
		ipMap:  make(map[string]bool),
		pid:    pid,
		maxIP:  totalIps,
	}
}

func TestDebug(t *testing.T) {
	tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
}

type op struct {
	id   int32
	add  bool
	name string
}

func (o *op) String() string {
	return fmt.Sprintf("%+v", *o)
}

func TestRequestPoolParallel(t *testing.T) {
	a, err := getAllocator(false)
	if err != nil {
		t.Fatal(err)
	}
	var operationIndex int32
	ch := make(chan *op, 240)
	for i := 0; i < 120; i++ {
		go func(t *testing.T, a *Allocator, ch chan *op) {
			name, _, _, err := a.RequestPool("GlobalDefault", "", "", nil, false)
			if err != nil {
				t.Fatalf("request error %v", err)
			}
			idx := atomic.AddInt32(&operationIndex, 1)
			ch <- &op{idx, true, name}
			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
			idx = atomic.AddInt32(&operationIndex, 1)
			err = a.ReleasePool(name)
			if err != nil {
				t.Fatalf("release error %v", err)
			}
			ch <- &op{idx, false, name}
		}(t, a, ch)
	}

	// map of events
	m := make(map[string][]*op)
	for i := 0; i < 240; i++ {
		x := <-ch
		ops, ok := m[x.name]
		if !ok {
			ops = make([]*op, 0, 10)
		}
		ops = append(ops, x)
		m[x.name] = ops
	}

	// Post processing to avoid event reordering on the channel
	for pool, ops := range m {
		sort.Slice(ops[:], func(i, j int) bool {
			return ops[i].id < ops[j].id
		})
		expected := true
		for _, op := range ops {
			if op.add != expected {
				t.Fatalf("Operations for %v not valid %v, operations %v", pool, op, ops)
			}
			expected = !expected
		}
	}
}

func TestFullAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, all, parallelism)
		}
	}
}

func TestOddAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, odd, parallelism)
		}
	}
}

func TestFullAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, all, parallelism)
	}
}

func TestOddAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, odd, parallelism)
	}
}

func TestEvenAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, even, parallelism)
	}
}

func allocate(t *testing.T, tctx *testContext, parallel int64) {
	// Allocate the whole space
	parallelExec := semaphore.NewWeighted(parallel)
	routineNum := tctx.maxIP + 10
	ch := make(chan *net.IPNet, routineNum)
	var id int
	var wg sync.WaitGroup
	// routine loop
	for {
		wg.Add(1)
		go func(id int) {
			parallelExec.Acquire(context.Background(), 1)
			ip, _, _ := tctx.a.RequestAddress(tctx.pid, nil, tctx.opts)
			ch <- ip
			parallelExec.Release(1)
			wg.Done()
		}(id)
		id++
		if id == routineNum {
			break
		}
	}

	// give time to all the go routines to finish
	wg.Wait()

	// process results
	for i := 0; i < routineNum; i++ {
		ip := <-ch
		if ip == nil {
			continue
		}
		if there, ok := tctx.ipMap[ip.String()]; ok && there {
			t.Fatalf("Got duplicate IP %s", ip.String())
			break
		}
		tctx.ipList = append(tctx.ipList, ip)
		tctx.ipMap[ip.String()] = true
	}

	assert.Check(t, is.Len(tctx.ipList, tctx.maxIP))
	if len(tctx.ipList) != tctx.maxIP {
		t.Fatal("missmatch number allocation")
	}
}

func release(t *testing.T, tctx *testContext, mode releaseMode, parallel int64) {
	var startIndex, increment, stopIndex, length int
	switch mode {
	case all:
		startIndex = 0
		increment = 1
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP
	case odd, even:
		if mode == odd {
			startIndex = 1
		}
		increment = 2
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP / 2
		if tctx.maxIP%2 > 0 {
			length++
		}
	default:
		t.Fatal("unsupported mode yet")
	}

	ipIndex := make([]int, 0, length)
	// calculate the index to release from the ipList
	for i := startIndex; ; i += increment {
		ipIndex = append(ipIndex, i)
		if i+increment > stopIndex {
			break
		}
	}

	var id int
	parallelExec := semaphore.NewWeighted(parallel)
	ch := make(chan *net.IPNet, len(ipIndex))
	wg := sync.WaitGroup{}
	for index := range ipIndex {
		wg.Add(1)
		go func(id, index int) {
			parallelExec.Acquire(context.Background(), 1)
			// logrus.Errorf("index %v", index)
			// logrus.Errorf("list %v", tctx.ipList)
			err := tctx.a.ReleaseAddress(tctx.pid, tctx.ipList[index].IP)
			if err != nil {
				t.Fatalf("routine %d got %v", id, err)
			}
			ch <- tctx.ipList[index]
			parallelExec.Release(1)
			wg.Done()
		}(id, index)
		id++
	}
	wg.Wait()

	for i := 0; i < len(ipIndex); i++ {
		ip := <-ch

		// check if it is really free
		_, _, err := tctx.a.RequestAddress(tctx.pid, ip.IP, nil)
		assert.Check(t, err, "ip %v not properly released", ip)
		if err != nil {
			t.Fatalf("ip %v not properly released, error:%v", ip, err)
		}
		err = tctx.a.ReleaseAddress(tctx.pid, ip.IP)
		assert.NilError(t, err)

		if there, ok := tctx.ipMap[ip.String()]; !ok || !there {
			t.Fatalf("ip %v got double deallocated", ip)
		}
		tctx.ipMap[ip.String()] = false
		for j, v := range tctx.ipList {
			if v == ip {
				tctx.ipList = append(tctx.ipList[:j], tctx.ipList[j+1:]...)
				break
			}
		}
	}

	assert.Check(t, is.Len(tctx.ipList, tctx.maxIP-length))
}
<|endoftext|>


<commit_before>package shortstr

import (
	"reflect"

	"github.com/armon/go-radix"
)

// Shortener is a helper to return short, unique substrings
// when given a set of data to work with and the full value of
// the string to shorten. This can be useful to make indexes
// more human-friendly while still retaining their uniqueness
// and identifiability.
//
// A good example of where to use this library is with user-
// facing UUID's. It is often much easier to return a 6- or
// 7-character string and pass it around than it is to use
// the full 128-bit value.
type Shortener struct {
	tree *radix.Tree
}

// NewStrings creates a new Shortener from a string slice.
func NewStrings(data []string) *Shortener {
	tree := radix.New()
	for _, s := range data {
		tree.Insert(s, struct{}{})
	}
	return &Shortener{tree}
}

// New creates a new Shortener given a set of structs and a
// field name to use for comparison. If the input is not a
// slice of structs (or struct pointers), or the specified
// field does not exist in the struct, New will panic.
func New(data interface{}, field string) *Shortener {
	// Check that we have a slice
	v := reflect.ValueOf(data)
	if v.Kind() != reflect.Slice {
		panic("not a slice")
	}
	elem := v.Type().Elem()

	// Check the slice type
	switch elem.Kind() {
	case reflect.Struct:
	case reflect.Ptr:
		elem = elem.Elem()
		if elem.Kind() == reflect.Struct {
			break
		}
		fallthrough
	default:
		panic("not a struct slice")
	}

	// Make sure our structs actually have the field
	fieldVal, ok := elem.FieldByName(field)
	if !ok {
		panic("invalid struct field")
	}
	if fieldVal.Type.Kind() != reflect.String {
		panic("struct field must be type string")
	}

	// Create the tree
	tree := radix.New()

	// Go over all of the data and insert our keys into
	// the tree.
	for i := 0; i < v.Len(); i++ {
		val := reflect.Indirect(v.Index(i)).FieldByName(field)
		tree.Insert(val.String(), struct{}{})
	}
	return &Shortener{tree}
}

// min is the internal method used to retrieve the shortest
// possible string, given the length constraint.
func (s *Shortener) min(in string, l int) string {
	var result string
	for i := 0; ; i++ {
		// Add the next chunk of characters
		lidx := (i + 1) * l
		if lidx > len(in) {
			break
		}
		result += in[i*l : lidx]

		// Walk the tree. If we find more than a single
		// result, then the result would be ambiguous.
		var ambiguous, found bool
		s.tree.WalkPrefix(result, func(s string, _ interface{}) bool {
			if found {
				ambiguous = true
				return true
			}
			found = true
			return false
		})

		// If the prefix didn't match anything, then return
		// early as the prefix isn't in the data set.
		if !found {
			return ""
		}

		// If multiple entries were found for the prefix,
		// continue to add more characters to disambiguate.
		if ambiguous {
			continue
		}

		// We got an unambiguous result, so return it
		return result
	}
	return ""
}

// ShortestChunk is used to return the shortest substring in
// the chunk size provided. This means the minimum returned
// length is l, and the max is a multiple thereof. This is
// useful for keeping churn rate low with a frequently
// changing data set.
//
// If the result is an empty string, then shortening would
// create an ambiguous result (non-unique in the set).
func (s *Shortener) ShortestChunk(in string, l int) string {
	return s.min(in, l)
}

// Shortest is used to return the shortest possible unique
// match from the data set.
//
// If the result is an empty string, then shortening would
// create an ambiguous result (non-unique in the set).
func (s *Shortener) Shortest(in string) string {
	return s.min(in, 1)
}

// Expand is used to look up the full value of a given short
// string in the data set.
//
// If the result is an empty string, then expanding is not
// possible due to either the provided prefix missing in the
// data set, or multiple entries sharing the same prefix.
func (s *Shortener) Expand(in string) string {
	var ambiguous bool
	var full string

	// Walk the prefix of the given short string. If a single
	// entry is found we can return safely, but if we find
	// more then the lookup cannot resolve.
	s.tree.WalkPrefix(in, func(s string, _ interface{}) bool {
		if full != "" {
			ambiguous = true
			return true
		}
		full = s
		return false
	})

	// Check if we found multiple entries by the same prefix.
	if ambiguous {
		return ""
	}

	// A single match was found, so return it.
	return full
}
<commit_msg>Format.<commit_after>package shortstr

import (
	"reflect"

	"github.com/armon/go-radix"
)

// Shortener is a helper to return short, unique substrings when given a set of
// data to work with and the full value of the string to shorten. This can be
// useful to make indexes more human-friendly while still retaining their
// uniqueness and identifiability.
//
// A good example of where to use this library is with user-facing UUID's. It
// is often much easier to return a 6- or 7-character string and pass it around
// than it is to use the full 128-bit value.
type Shortener struct {
	tree *radix.Tree
}

// New creates a new Shortener given a set of structs and a field name to use
// for comparison. If the input is not a slice of structs (or struct pointers),
// or the specified field does not exist in the struct, New will panic.
func New(data interface{}, field string) *Shortener {
	// Check that we have a slice
	v := reflect.ValueOf(data)
	if v.Kind() != reflect.Slice {
		panic("not a slice")
	}
	elem := v.Type().Elem()

	// Check the slice type
	switch elem.Kind() {
	case reflect.Struct:
	case reflect.Ptr:
		elem = elem.Elem()
		if elem.Kind() == reflect.Struct {
			break
		}
		fallthrough
	default:
		panic("not a struct slice")
	}

	// Make sure our structs actually have the field
	fieldVal, ok := elem.FieldByName(field)
	if !ok {
		panic("invalid struct field")
	}
	if fieldVal.Type.Kind() != reflect.String {
		panic("struct field must be type string")
	}

	// Create the tree
	tree := radix.New()

	// Go over all of the data and insert our keys into
	// the tree.
	for i := 0; i < v.Len(); i++ {
		val := reflect.Indirect(v.Index(i)).FieldByName(field)
		tree.Insert(val.String(), struct{}{})
	}
	return &Shortener{tree}
}

// NewStrings creates a new Shortener from a string slice.
func NewStrings(data []string) *Shortener {
	tree := radix.New()
	for _, s := range data {
		tree.Insert(s, struct{}{})
	}
	return &Shortener{tree}
}

// min is the internal method used to retrieve the shortest possible string,
// given the length constraint.
func (s *Shortener) min(in string, l int) string {
	var result string
	for i := 0; ; i++ {
		// Add the next chunk of characters
		lidx := (i + 1) * l
		if lidx > len(in) {
			break
		}
		result += in[i*l : lidx]

		// Walk the tree. If we find more than a single result, then the
		// result would be ambiguous.
		var ambiguous, found bool
		s.tree.WalkPrefix(result, func(s string, _ interface{}) bool {
			if found {
				ambiguous = true
				return true
			}
			found = true
			return false
		})

		// If the prefix didn't match anything, then return early as the
		// prefix isn't in the data set.
		if !found {
			return ""
		}

		// If multiple entries were found for the prefix, continue to add more
		// characters to disambiguate.
		if ambiguous {
			continue
		}

		// We got an unambiguous result, so return it
		return result
	}
	return ""
}

// ShortestChunk is used to return the shortest substring in the chunk size
// provided. This means the minimum returned length is l, and the max is a
// multiple thereof. This is useful for keeping churn rate low with a
// frequently changing data set.
//
// If the result is an empty string, then shortening would create an ambiguous
// result (non-unique in the set).
func (s *Shortener) ShortestChunk(in string, l int) string {
	return s.min(in, l)
}

// Shortest is used to return the shortest possible unique match from the
// data set. If the result is an empty string, then shortening would create
// an ambiguous result (non-unique in the set).
func (s *Shortener) Shortest(in string) string {
	return s.min(in, 1)
}

// Expand is used to look up the full value of a given short string in the data
// set. If the result is an empty string, then expanding is not possible due to
// either the provided prefix missing in the data set, or multiple entries
// sharing the same prefix.
func (s *Shortener) Expand(in string) string {
	var ambiguous bool
	var full string

	// Walk the prefix of the given short string. If a single entry is found we
	// can return safely, but if we find more then the lookup cannot resolve.
	s.tree.WalkPrefix(in, func(s string, _ interface{}) bool {
		if full != "" {
			ambiguous = true
			return true
		}
		full = s
		return false
	})

	// Check if we found multiple entries by the same prefix.
	if ambiguous {
		return ""
	}

	// A single match was found, so return it.
	return full
}
<|endoftext|>


<commit_before>package build

import (
	"net/url"
	"path"
	"reflect"

	"github.com/flant/dapp/pkg/build/builder"
	"github.com/flant/dapp/pkg/build/stage"
	"github.com/flant/dapp/pkg/config"
	"github.com/flant/dapp/pkg/git_repo"
	"github.com/flant/dapp/pkg/slug"
)

type InitializationPhase struct{}

func NewInitializationPhase() *InitializationPhase {
	return &InitializationPhase{}
}

func (p *InitializationPhase) Run(c *Conveyor) error {
	dimgsInOrder, err := generateDimgsInOrder(c.Dappfile, c)
	if err != nil {
		return err
	}

	c.DimgsInOrder = dimgsInOrder

	return nil
}

func generateDimgsInOrder(dappfile []*config.Dimg, c *Conveyor) ([]*Dimg, error) {
	var dimgs []*Dimg
	for _, dimgConfig := range getDimgConfigsInOrder(dappfile) {
		dimg := &Dimg{}

		dimgBaseConfig, dimgName, dimgArtifact := processDimgConfig(dimgConfig)
		from, fromDimgName := getFromAndFromDimgName(dimgBaseConfig)

		dimg.name = dimgName
		dimg.baseImageName = from
		dimg.baseImageDimgName = fromDimgName
		dimg.isArtifact = dimgArtifact

		stages, err := generateStages(dimgConfig, c)
		if err != nil {
			return nil, err
		}

		dimg.SetStages(stages)

		dimgs = append(dimgs, dimg)
	}

	return dimgs, nil
}

func getFromAndFromDimgName(dimgBaseConfig *config.DimgBase) (string, string) {
	var from string
	var fromDimgName string

	if dimgBaseConfig.From != "" {
		from = dimgBaseConfig.From
	} else {
		fromDimg := dimgBaseConfig.FromDimg
		fromDimgArtifact := dimgBaseConfig.FromDimgArtifact

		if fromDimg != nil {
			fromDimgName = fromDimg.Name
		} else {
			fromDimgName = fromDimgArtifact.Name
		}
	}

	return from, fromDimgName
}

func getDimgConfigsInOrder(dappfile []*config.Dimg) []config.DimgInterface {
	var dimgConfigs []config.DimgInterface
	for _, dimg := range dappfile {
		dimgsInBuildOrder := dimg.DimgTree()
		for i := 0; i < len(dimgsInBuildOrder); i++ {
			if isNotInArr(dimgConfigs, dimgsInBuildOrder[i]) {
				dimgConfigs = append(dimgConfigs, dimgsInBuildOrder[i])
			}
		}
	}

	return dimgConfigs
}

func isNotInArr(arr []config.DimgInterface, obj config.DimgInterface) bool {
	for _, elm := range arr {
		if reflect.DeepEqual(elm, obj) {
			return false
		}
	}

	return true
}

func generateStages(dimgConfig config.DimgInterface, c *Conveyor) ([]stage.Interface, error) {
	var stages []stage.Interface

	dimgBaseConfig, dimgName, dimgArtifact := processDimgConfig(dimgConfig)

	baseStageOptions := &stage.NewBaseStageOptions{
		DimgName:         dimgName,
		DimgTmpDir:       c.GetDimgTmpDir(dimgBaseConfig.Name),
		ContainerDappDir: c.ContainerDappDir,
		ProjectBuildDir:  c.ProjectBuildDir,
	}

	gitArtifacts, err := generateGitArtifacts(dimgBaseConfig, c)
	if err != nil {
		return nil, err
	}

	// from
	stages = appendIfExist(stages, stage.GenerateFromStage(dimgBaseConfig, baseStageOptions))

	// before_install
	stages = appendIfExist(stages, stage.GenerateBeforeInstallStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// before_install_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportBeforeInstallStage(dimgBaseConfig, baseStageOptions))

	// g_a_archive_stage
	stages = append(stages, stage.NewGAArchiveStage(baseStageOptions))

	// install
	stages = appendIfExist(stages, stage.GenerateInstallStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// after_install_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportAfterInstallStage(dimgBaseConfig, baseStageOptions))

	// before_setup
	stages = appendIfExist(stages, stage.GenerateBeforeSetupStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// before_setup_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportBeforeSetupStage(dimgBaseConfig, baseStageOptions))

	// setup
	stages = appendIfExist(stages, stage.GenerateSetupStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// after_setup_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportAfterSetupStage(dimgBaseConfig, baseStageOptions))

	if !dimgArtifact {
		// g_a_post_setup_patch
		stages = append(stages, stage.NewGAPostSetupPatchStage(baseStageOptions))

		// g_a_latest_patch
		stages = append(stages, stage.NewGALatestPatchStage(baseStageOptions))

		// docker_instructions
		stages = appendIfExist(stages, stage.GenerateDockerInstructionsStage(dimgConfig.(*config.Dimg), baseStageOptions))
	}

	for _, s := range stages {
		s.SetGitArtifacts(gitArtifacts)
	}

	return stages, nil
}

func generateGitArtifacts(dimgBaseConfig *config.DimgBase, c *Conveyor) ([]*stage.GitArtifact, error) {
	var gitArtifacts, nonEmptyGitArtifacts []*stage.GitArtifact

	var localGitRepo *git_repo.Local
	if len(dimgBaseConfig.Git.Local) != 0 {
		localGitRepo = &git_repo.Local{
			Base:   git_repo.Base{Name: "own"},
			Path:   c.ProjectDir,
			GitDir: path.Join(c.ProjectDir, ".git"),
		}
	}

	for _, localGAConfig := range dimgBaseConfig.Git.Local {
		gitArtifacts = append(gitArtifacts, gitLocalArtifactInit(localGAConfig, localGitRepo, dimgBaseConfig.Name, c))
	}

	remoteGitRepos := map[string]*git_repo.Remote{}
	for _, remoteGAConfig := range dimgBaseConfig.Git.Remote {
		var remoteGitRepo *git_repo.Remote
		if len(dimgBaseConfig.Git.Remote) != 0 {
			_, exist := remoteGitRepos[remoteGAConfig.Name]
			if !exist {
				clonePath, err := getRemoteGitRepoClonePath(remoteGAConfig, c)
				if err != nil {
					return nil, err
				}

				remoteGitRepo = &git_repo.Remote{
					Base:      git_repo.Base{Name: remoteGAConfig.Name},
					Url:       remoteGAConfig.Url,
					ClonePath: clonePath,
				}
				remoteGitRepos[remoteGAConfig.Name] = remoteGitRepo
			}
		}

		gitArtifacts = append(gitArtifacts, gitRemoteArtifactInit(remoteGAConfig, remoteGitRepo, dimgBaseConfig.Name, c))
	}

	for _, ga := range gitArtifacts {
		if empty, err := ga.IsEmpty(); err != nil {
			return nil, err
		} else if !empty {
			nonEmptyGitArtifacts = append(nonEmptyGitArtifacts, ga)
		}
	}

	return nonEmptyGitArtifacts, nil
}

func getRemoteGitRepoClonePath(remoteGaConfig *config.GitRemote, c *Conveyor) (string, error) {
	scheme, err := urlScheme(remoteGaConfig.Url)
	if err != nil {
		return "", err
	}

	clonePath := path.Join(
		c.ProjectBuildDir,
		"remote_git_repo",
		string(git_repo.RemoteGitRepoCacheVersion),
		slug.Slug(remoteGaConfig.Name),
		scheme,
	)

	return clonePath, nil
}

func urlScheme(urlString string) (string, error) {
	u, err := url.Parse(urlString)
	if err != nil {
		return "", err
	}

	return u.Scheme, nil
}

func gitRemoteArtifactInit(remoteGAConfig *config.GitRemote, remoteGitRepo *git_repo.Remote, dimgName string, c *Conveyor) *stage.GitArtifact {
	ga := baseGitArtifactInit(remoteGAConfig.GitLocalExport, dimgName, c)

	ga.Tag = remoteGAConfig.Tag
	ga.Commit = remoteGAConfig.Commit
	ga.Branch = remoteGAConfig.Branch

	ga.Name = remoteGAConfig.Name

	ga.GitRepoInterface = remoteGitRepo

	return ga
}

func gitLocalArtifactInit(localGAConfig *config.GitLocal, localGitRepo *git_repo.Local, dimgName string, c *Conveyor) *stage.GitArtifact {
	ga := baseGitArtifactInit(localGAConfig.GitLocalExport, dimgName, c)

	ga.As = localGAConfig.As

	ga.Name = "own"

	ga.GitRepoInterface = localGitRepo

	return ga
}

func baseGitArtifactInit(local *config.GitLocalExport, dimgName string, c *Conveyor) *stage.GitArtifact {
	var stageDependencies map[stage.StageName][]string
	if local.StageDependencies != nil {
		stageDependencies = stageDependenciesToMap(local.StageDependencies)
	}

	ga := &stage.GitArtifact{
		PatchesDir:           getDimgPatchesDir(dimgName, c),
		ContainerPatchesDir:  getDimgPatchesContainerDir(c),
		ArchivesDir:          getDimgArchivesDir(dimgName, c),
		ContainerArchivesDir: getDimgArchivesContainerDir(c),

		RepoPath: path.Join("/", local.Add),

		Cwd:                local.Add,
		To:                 local.To,
		ExcludePaths:       local.ExcludePaths,
		IncludePaths:       local.IncludePaths,
		Owner:              local.Owner,
		Group:              local.Group,
		StagesDependencies: stageDependencies,
	}

	return ga
}

func getDimgPatchesDir(dimgName string, c *Conveyor) string {
	return path.Join(c.TmpDir, dimgName, "patch")
}

func getDimgPatchesContainerDir(c *Conveyor) string {
	return path.Join(c.ContainerDappDir, "patch")
}

func getDimgArchivesDir(dimgName string, c *Conveyor) string {
	return path.Join(c.TmpDir, dimgName, "archive")
}

func getDimgArchivesContainerDir(c *Conveyor) string {
	return path.Join(c.ContainerDappDir, "archive")
}

func stageDependenciesToMap(sd *config.StageDependencies) map[stage.StageName][]string {
	result := map[stage.StageName][]string{
		stage.Install:     sd.Install,
		stage.BeforeSetup: sd.BeforeSetup,
		stage.Setup:       sd.Setup,
	}

	return result
}

func processDimgConfig(dimgConfig config.DimgInterface) (*config.DimgBase, string, bool) {
	var dimgBase *config.DimgBase
	var dimgArtifact bool
	switch dimgConfig.(type) {
	case *config.Dimg:
		dimgBase = dimgConfig.(*config.Dimg).DimgBase
		dimgArtifact = false
	case *config.DimgArtifact:
		dimgBase = dimgConfig.(*config.DimgArtifact).DimgBase
		dimgArtifact = true
	}

	return dimgBase, dimgBase.Name, dimgArtifact
}

func ansibleBuilderExtra(c *Conveyor) *builder.Extra {
	ansibleBuilderExtra := &builder.Extra{
		TmpPath:           c.TmpDir,
		ContainerDappPath: c.ContainerDappDir,
	}

	return ansibleBuilderExtra
}

func appendIfExist(stages []stage.Interface, stage stage.Interface) []stage.Interface {
	if !reflect.ValueOf(stage).IsNil() {
		return append(stages, stage)
	}

	return stages
}
<commit_msg>[go dapp] fix remote git artifact initialization<commit_after>package build

import (
	"fmt"
	"net/url"
	"path"
	"reflect"
	"strings"

	"github.com/flant/dapp/pkg/build/builder"
	"github.com/flant/dapp/pkg/build/stage"
	"github.com/flant/dapp/pkg/config"
	"github.com/flant/dapp/pkg/git_repo"
	"github.com/flant/dapp/pkg/slug"
)

type InitializationPhase struct{}

func NewInitializationPhase() *InitializationPhase {
	return &InitializationPhase{}
}

func (p *InitializationPhase) Run(c *Conveyor) error {
	dimgsInOrder, err := generateDimgsInOrder(c.Dappfile, c)
	if err != nil {
		return err
	}

	c.DimgsInOrder = dimgsInOrder

	return nil
}

func generateDimgsInOrder(dappfile []*config.Dimg, c *Conveyor) ([]*Dimg, error) {
	var dimgs []*Dimg
	for _, dimgConfig := range getDimgConfigsInOrder(dappfile) {
		dimg := &Dimg{}

		dimgBaseConfig, dimgName, dimgArtifact := processDimgConfig(dimgConfig)
		from, fromDimgName := getFromAndFromDimgName(dimgBaseConfig)

		dimg.name = dimgName
		dimg.baseImageName = from
		dimg.baseImageDimgName = fromDimgName
		dimg.isArtifact = dimgArtifact

		stages, err := generateStages(dimgConfig, c)
		if err != nil {
			return nil, err
		}

		dimg.SetStages(stages)

		dimgs = append(dimgs, dimg)
	}

	return dimgs, nil
}

func getFromAndFromDimgName(dimgBaseConfig *config.DimgBase) (string, string) {
	var from string
	var fromDimgName string

	if dimgBaseConfig.From != "" {
		from = dimgBaseConfig.From
	} else {
		fromDimg := dimgBaseConfig.FromDimg
		fromDimgArtifact := dimgBaseConfig.FromDimgArtifact

		if fromDimg != nil {
			fromDimgName = fromDimg.Name
		} else {
			fromDimgName = fromDimgArtifact.Name
		}
	}

	return from, fromDimgName
}

func getDimgConfigsInOrder(dappfile []*config.Dimg) []config.DimgInterface {
	var dimgConfigs []config.DimgInterface
	for _, dimg := range dappfile {
		dimgsInBuildOrder := dimg.DimgTree()
		for i := 0; i < len(dimgsInBuildOrder); i++ {
			if isNotInArr(dimgConfigs, dimgsInBuildOrder[i]) {
				dimgConfigs = append(dimgConfigs, dimgsInBuildOrder[i])
			}
		}
	}

	return dimgConfigs
}

func isNotInArr(arr []config.DimgInterface, obj config.DimgInterface) bool {
	for _, elm := range arr {
		if reflect.DeepEqual(elm, obj) {
			return false
		}
	}

	return true
}

func generateStages(dimgConfig config.DimgInterface, c *Conveyor) ([]stage.Interface, error) {
	var stages []stage.Interface

	dimgBaseConfig, dimgName, dimgArtifact := processDimgConfig(dimgConfig)

	baseStageOptions := &stage.NewBaseStageOptions{
		DimgName:         dimgName,
		DimgTmpDir:       c.GetDimgTmpDir(dimgBaseConfig.Name),
		ContainerDappDir: c.ContainerDappDir,
		ProjectBuildDir:  c.ProjectBuildDir,
	}

	gitArtifacts, err := generateGitArtifacts(dimgBaseConfig, c)
	if err != nil {
		return nil, err
	}

	// from
	stages = appendIfExist(stages, stage.GenerateFromStage(dimgBaseConfig, baseStageOptions))

	// before_install
	stages = appendIfExist(stages, stage.GenerateBeforeInstallStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// before_install_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportBeforeInstallStage(dimgBaseConfig, baseStageOptions))

	// g_a_archive_stage
	stages = append(stages, stage.NewGAArchiveStage(baseStageOptions))

	// install
	stages = appendIfExist(stages, stage.GenerateInstallStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// after_install_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportAfterInstallStage(dimgBaseConfig, baseStageOptions))

	// before_setup
	stages = appendIfExist(stages, stage.GenerateBeforeSetupStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// before_setup_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportBeforeSetupStage(dimgBaseConfig, baseStageOptions))

	// setup
	stages = appendIfExist(stages, stage.GenerateSetupStage(dimgBaseConfig, ansibleBuilderExtra(c), baseStageOptions))

	// after_setup_artifact
	stages = appendIfExist(stages, stage.GenerateArtifactImportAfterSetupStage(dimgBaseConfig, baseStageOptions))

	if !dimgArtifact {
		// g_a_post_setup_patch
		stages = append(stages, stage.NewGAPostSetupPatchStage(baseStageOptions))

		// g_a_latest_patch
		stages = append(stages, stage.NewGALatestPatchStage(baseStageOptions))

		// docker_instructions
		stages = appendIfExist(stages, stage.GenerateDockerInstructionsStage(dimgConfig.(*config.Dimg), baseStageOptions))
	}

	for _, s := range stages {
		s.SetGitArtifacts(gitArtifacts)
	}

	return stages, nil
}

func generateGitArtifacts(dimgBaseConfig *config.DimgBase, c *Conveyor) ([]*stage.GitArtifact, error) {
	var gitArtifacts, nonEmptyGitArtifacts []*stage.GitArtifact

	var localGitRepo *git_repo.Local
	if len(dimgBaseConfig.Git.Local) != 0 {
		localGitRepo = &git_repo.Local{
			Base:   git_repo.Base{Name: "own"},
			Path:   c.ProjectDir,
			GitDir: path.Join(c.ProjectDir, ".git"),
		}
	}

	for _, localGAConfig := range dimgBaseConfig.Git.Local {
		gitArtifacts = append(gitArtifacts, gitLocalArtifactInit(localGAConfig, localGitRepo, dimgBaseConfig.Name, c))
	}

	remoteGitRepos := map[string]*git_repo.Remote{}
	for _, remoteGAConfig := range dimgBaseConfig.Git.Remote {
		var remoteGitRepo *git_repo.Remote
		if len(dimgBaseConfig.Git.Remote) != 0 {
			_, exist := remoteGitRepos[remoteGAConfig.Name]
			if !exist {
				clonePath, err := getRemoteGitRepoClonePath(remoteGAConfig, c)
				if err != nil {
					return nil, err
				}

				remoteGitRepo = &git_repo.Remote{
					Base:      git_repo.Base{Name: remoteGAConfig.Name},
					Url:       remoteGAConfig.Url,
					ClonePath: clonePath,
				}

				if err := remoteGitRepo.CloneAndFetch(); err != nil {
					return nil, err
				}

				remoteGitRepos[remoteGAConfig.Name] = remoteGitRepo
			}
		}

		gitArtifacts = append(gitArtifacts, gitRemoteArtifactInit(remoteGAConfig, remoteGitRepo, dimgBaseConfig.Name, c))
	}

	for _, ga := range gitArtifacts {
		if empty, err := ga.IsEmpty(); err != nil {
			return nil, err
		} else if !empty {
			nonEmptyGitArtifacts = append(nonEmptyGitArtifacts, ga)
		}
	}

	return nonEmptyGitArtifacts, nil
}

func getRemoteGitRepoClonePath(remoteGaConfig *config.GitRemote, c *Conveyor) (string, error) {
	scheme, err := urlScheme(remoteGaConfig.Url)
	if err != nil {
		return "", err
	}

	clonePath := path.Join(
		c.ProjectBuildDir,
		"remote_git_repo",
		fmt.Sprintf("%v", git_repo.RemoteGitRepoCacheVersion),
		slug.Slug(remoteGaConfig.Name),
		scheme,
	)

	return clonePath, nil
}

func urlScheme(urlString string) (string, error) {
	u, err := url.Parse(urlString)
	if err != nil {
		if strings.HasSuffix(err.Error(), "first path segment in URL cannot contain colon") {
			for _, protocol := range []string{"git", "ssh"} {
				if strings.HasPrefix(urlString, fmt.Sprintf("%s@", protocol)) {
					return "ssh", nil
				}
			}
		}
		return "", err
	}

	return u.Scheme, nil
}

func gitRemoteArtifactInit(remoteGAConfig *config.GitRemote, remoteGitRepo *git_repo.Remote, dimgName string, c *Conveyor) *stage.GitArtifact {
	ga := baseGitArtifactInit(remoteGAConfig.GitLocalExport, dimgName, c)

	ga.Tag = remoteGAConfig.Tag
	ga.Commit = remoteGAConfig.Commit
	ga.Branch = remoteGAConfig.Branch

	ga.Name = remoteGAConfig.Name

	ga.GitRepoInterface = remoteGitRepo

	return ga
}

func gitLocalArtifactInit(localGAConfig *config.GitLocal, localGitRepo *git_repo.Local, dimgName string, c *Conveyor) *stage.GitArtifact {
	ga := baseGitArtifactInit(localGAConfig.GitLocalExport, dimgName, c)

	ga.As = localGAConfig.As

	ga.Name = "own"

	ga.GitRepoInterface = localGitRepo

	return ga
}

func baseGitArtifactInit(local *config.GitLocalExport, dimgName string, c *Conveyor) *stage.GitArtifact {
	var stageDependencies map[stage.StageName][]string
	if local.StageDependencies != nil {
		stageDependencies = stageDependenciesToMap(local.StageDependencies)
	}

	ga := &stage.GitArtifact{
		PatchesDir:           getDimgPatchesDir(dimgName, c),
		ContainerPatchesDir:  getDimgPatchesContainerDir(c),
		ArchivesDir:          getDimgArchivesDir(dimgName, c),
		ContainerArchivesDir: getDimgArchivesContainerDir(c),

		RepoPath: path.Join("/", local.Add),

		Cwd:                local.Add,
		To:                 local.To,
		ExcludePaths:       local.ExcludePaths,
		IncludePaths:       local.IncludePaths,
		Owner:              local.Owner,
		Group:              local.Group,
		StagesDependencies: stageDependencies,
	}

	return ga
}

func getDimgPatchesDir(dimgName string, c *Conveyor) string {
	return path.Join(c.TmpDir, dimgName, "patch")
}

func getDimgPatchesContainerDir(c *Conveyor) string {
	return path.Join(c.ContainerDappDir, "patch")
}

func getDimgArchivesDir(dimgName string, c *Conveyor) string {
	return path.Join(c.TmpDir, dimgName, "archive")
}

func getDimgArchivesContainerDir(c *Conveyor) string {
	return path.Join(c.ContainerDappDir, "archive")
}

func stageDependenciesToMap(sd *config.StageDependencies) map[stage.StageName][]string {
	result := map[stage.StageName][]string{
		stage.Install:     sd.Install,
		stage.BeforeSetup: sd.BeforeSetup,
		stage.Setup:       sd.Setup,
	}

	return result
}

func processDimgConfig(dimgConfig config.DimgInterface) (*config.DimgBase, string, bool) {
	var dimgBase *config.DimgBase
	var dimgArtifact bool
	switch dimgConfig.(type) {
	case *config.Dimg:
		dimgBase = dimgConfig.(*config.Dimg).DimgBase
		dimgArtifact = false
	case *config.DimgArtifact:
		dimgBase = dimgConfig.(*config.DimgArtifact).DimgBase
		dimgArtifact = true
	}

	return dimgBase, dimgBase.Name, dimgArtifact
}

func ansibleBuilderExtra(c *Conveyor) *builder.Extra {
	ansibleBuilderExtra := &builder.Extra{
		TmpPath:           c.TmpDir,
		ContainerDappPath: c.ContainerDappDir,
	}

	return ansibleBuilderExtra
}

func appendIfExist(stages []stage.Interface, stage stage.Interface) []stage.Interface {
	if !reflect.ValueOf(stage).IsNil() {
		return append(stages, stage)
	}

	return stages
}
<|endoftext|>