<commit_before>package azure

import (
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/azure/go-autorest/autorest"
)

const (
	defaultRefresh = 5 * time.Minute
	oauthURL       = "https://login.microsoftonline.com/{tenantID}/oauth2/{requestType}?api-version=1.0"
	tokenBaseDate  = "1970-01-01T00:00:00Z"
)

var expirationBase time.Time

func init() {
	expirationBase, _ = time.Parse(time.RFC3339, tokenBaseDate)
}

// Token encapsulates the access token used to authorize Azure requests.
type Token struct {
	AccessToken string `json:"access_token"`

	ExpiresIn string `json:"expires_in"`
	ExpiresOn string `json:"expires_on"`
	NotBefore string `json:"not_before"`

	Resource string `json:"resource"`
	Type     string `json:"token_type"`
}

// Expires returns the time.Time when the Token expires.
func (t Token) Expires() time.Time {
	s, err := strconv.Atoi(t.ExpiresOn)
	if err != nil {
		s = -3600
	}
	return expirationBase.Add(time.Duration(s) * time.Second).UTC()
}

// IsExpired returns true if the Token is expired, false otherwise.
func (t Token) IsExpired() bool {
	return t.WillExpireIn(0)
}

// WillExpireIn returns true if the Token will expire after the passed time.Duration interval
// from now, false otherwise.
func (t Token) WillExpireIn(d time.Duration) bool {
	return !t.Expires().After(time.Now().Add(d))
}

// WithAuthorization returns a PrepareDecorator that adds an HTTP Authorization header whose
// value is "Bearer " followed by the AccessToken of the Token.
func (t *Token) WithAuthorization() autorest.PrepareDecorator {
	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			return (autorest.WithBearerAuthorization(t.AccessToken)(p)).Prepare(r)
		})
	}
}

// ServicePrincipalToken encapsulates a Token created for a Service Principal.
type ServicePrincipalToken struct {
	Token

	clientID      string
	clientSecret  string
	resource      string
	tenantID      string
	autoRefresh   bool
	refreshWithin time.Duration
	sender        autorest.Sender
}

// NewServicePrincipalToken creates a ServicePrincipalToken from the supplied Service Principal
// credentials scoped to the named resource.
func NewServicePrincipalToken(id string, secret string, tenantID string, resource string) (*ServicePrincipalToken, error) {
	spt := &ServicePrincipalToken{
		clientID:      id,
		clientSecret:  secret,
		resource:      resource,
		tenantID:      tenantID,
		autoRefresh:   true,
		refreshWithin: defaultRefresh,
		sender:        &http.Client{}}
	return spt, nil
}

// EnsureFresh will refresh the token if it will expire within the refresh window (as set by
// RefreshWithin).
func (spt *ServicePrincipalToken) EnsureFresh() error {
	if spt.WillExpireIn(spt.refreshWithin) {
		return spt.Refresh()
	}
	return nil
}

// Refresh obtains a fresh token for the Service Principal.
func (spt *ServicePrincipalToken) Refresh() error {
	p := map[string]interface{}{
		"tenantID":    spt.tenantID,
		"requestType": "token",
	}

	v := url.Values{}
	v.Set("client_id", spt.clientID)
	v.Set("client_secret", spt.clientSecret)
	v.Set("grant_type", "client_credentials")
	v.Set("resource", spt.resource)

	req, _ := autorest.Prepare(&http.Request{},
		autorest.AsPost(),
		autorest.AsFormURLEncoded(),
		autorest.WithBaseURL(oauthURL),
		autorest.WithPathParameters(p),
		autorest.WithFormData(v))

	resp, err := autorest.SendWithSender(spt.sender, req)
	if err != nil {
		return autorest.NewErrorWithError(err,
			"azure.ServicePrincipalToken", "Refresh", "Failure sending request for Service Principal %s",
			spt.clientID)
	}

	err = autorest.Respond(resp,
		autorest.WithErrorUnlessOK(),
		autorest.ByUnmarshallingJSON(spt),
		autorest.ByClosing())
	if err != nil {
		return autorest.NewErrorWithError(err,
			"azure.ServicePrincipalToken", "Refresh", "Failure handling response to Service Principal %s request",
			spt.clientID)
	}

	return nil
}

// SetAutoRefresh enables or disables automatic refreshing of stale tokens.
func (spt *ServicePrincipalToken) SetAutoRefresh(autoRefresh bool) {
	spt.autoRefresh = autoRefresh
}

// SetRefreshWithin sets the interval within which if the token will expire, EnsureFresh will
// refresh the token.
func (spt *ServicePrincipalToken) SetRefreshWithin(d time.Duration) {
	spt.refreshWithin = d
	return
}

// SetSender sets the autorest.Sender used when obtaining the Service Principal token. An
// undecorated http.Client is used by default.
func (spt *ServicePrincipalToken) SetSender(s autorest.Sender) {
	spt.sender = s
}

// WithAuthorization returns a PrepareDecorator that adds an HTTP Authorization header whose
// value is "Bearer " followed by the AccessToken of the ServicePrincipalToken.
//
// By default, the token will automatically refresh if nearly expired (as determined by the
// RefreshWithin interval). Use the AutoRefresh method to enable or disable automatically refreshing
// tokens.
func (spt *ServicePrincipalToken) WithAuthorization() autorest.PrepareDecorator {
	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			if spt.autoRefresh {
				err := spt.EnsureFresh()
				if err != nil {
					return r, autorest.NewErrorWithError(err,
						"azure.ServicePrincipalToken", "WithAuthorization", "Failed to refresh Service Principal Token for request to %s",
						r.URL)
				}
			}
			return (autorest.WithBearerAuthorization(spt.AccessToken)(p)).Prepare(r)
		})
	}
}
<commit_msg>Added Azure Resource Manager scope constant<commit_after>package azure

import (
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/azure/go-autorest/autorest"
)

const (
	defaultRefresh = 5 * time.Minute
	oauthURL       = "https://login.microsoftonline.com/{tenantID}/oauth2/{requestType}?api-version=1.0"
	tokenBaseDate  = "1970-01-01T00:00:00Z"

	// AzureResourceManagerScope is the OAuth scope for the Azure Resource Manager.
	AzureResourceManagerScope = "https://management.azure.com/"
)

var expirationBase time.Time

func init() {
	expirationBase, _ = time.Parse(time.RFC3339, tokenBaseDate)
}

// Token encapsulates the access token used to authorize Azure requests.
type Token struct {
	AccessToken string `json:"access_token"`

	ExpiresIn string `json:"expires_in"`
	ExpiresOn string `json:"expires_on"`
	NotBefore string `json:"not_before"`

	Resource string `json:"resource"`
	Type     string `json:"token_type"`
}

// Expires returns the time.Time when the Token expires.
func (t Token) Expires() time.Time {
	s, err := strconv.Atoi(t.ExpiresOn)
	if err != nil {
		s = -3600
	}
	return expirationBase.Add(time.Duration(s) * time.Second).UTC()
}

// IsExpired returns true if the Token is expired, false otherwise.
func (t Token) IsExpired() bool {
	return t.WillExpireIn(0)
}

// WillExpireIn returns true if the Token will expire after the passed time.Duration interval
// from now, false otherwise.
func (t Token) WillExpireIn(d time.Duration) bool {
	return !t.Expires().After(time.Now().Add(d))
}

// WithAuthorization returns a PrepareDecorator that adds an HTTP Authorization header whose
// value is "Bearer " followed by the AccessToken of the Token.
func (t *Token) WithAuthorization() autorest.PrepareDecorator {
	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			return (autorest.WithBearerAuthorization(t.AccessToken)(p)).Prepare(r)
		})
	}
}

// ServicePrincipalToken encapsulates a Token created for a Service Principal.
type ServicePrincipalToken struct {
	Token

	clientID      string
	clientSecret  string
	resource      string
	tenantID      string
	autoRefresh   bool
	refreshWithin time.Duration
	sender        autorest.Sender
}

// NewServicePrincipalToken creates a ServicePrincipalToken from the supplied Service Principal
// credentials scoped to the named resource.
func NewServicePrincipalToken(id string, secret string, tenantID string, resource string) (*ServicePrincipalToken, error) {
	spt := &ServicePrincipalToken{
		clientID:      id,
		clientSecret:  secret,
		resource:      resource,
		tenantID:      tenantID,
		autoRefresh:   true,
		refreshWithin: defaultRefresh,
		sender:        &http.Client{}}
	return spt, nil
}

// EnsureFresh will refresh the token if it will expire within the refresh window (as set by
// RefreshWithin).
func (spt *ServicePrincipalToken) EnsureFresh() error {
	if spt.WillExpireIn(spt.refreshWithin) {
		return spt.Refresh()
	}
	return nil
}

// Refresh obtains a fresh token for the Service Principal.
func (spt *ServicePrincipalToken) Refresh() error {
	p := map[string]interface{}{
		"tenantID":    spt.tenantID,
		"requestType": "token",
	}

	v := url.Values{}
	v.Set("client_id", spt.clientID)
	v.Set("client_secret", spt.clientSecret)
	v.Set("grant_type", "client_credentials")
	v.Set("resource", spt.resource)

	req, _ := autorest.Prepare(&http.Request{},
		autorest.AsPost(),
		autorest.AsFormURLEncoded(),
		autorest.WithBaseURL(oauthURL),
		autorest.WithPathParameters(p),
		autorest.WithFormData(v))

	resp, err := autorest.SendWithSender(spt.sender, req)
	if err != nil {
		return autorest.NewErrorWithError(err,
			"azure.ServicePrincipalToken", "Refresh", "Failure sending request for Service Principal %s",
			spt.clientID)
	}

	err = autorest.Respond(resp,
		autorest.WithErrorUnlessOK(),
		autorest.ByUnmarshallingJSON(spt),
		autorest.ByClosing())
	if err != nil {
		return autorest.NewErrorWithError(err,
			"azure.ServicePrincipalToken", "Refresh", "Failure handling response to Service Principal %s request",
			spt.clientID)
	}

	return nil
}

// SetAutoRefresh enables or disables automatic refreshing of stale tokens.
func (spt *ServicePrincipalToken) SetAutoRefresh(autoRefresh bool) {
	spt.autoRefresh = autoRefresh
}

// SetRefreshWithin sets the interval within which if the token will expire, EnsureFresh will
// refresh the token.
func (spt *ServicePrincipalToken) SetRefreshWithin(d time.Duration) {
	spt.refreshWithin = d
	return
}

// SetSender sets the autorest.Sender used when obtaining the Service Principal token. An
// undecorated http.Client is used by default.
func (spt *ServicePrincipalToken) SetSender(s autorest.Sender) {
	spt.sender = s
}

// WithAuthorization returns a PrepareDecorator that adds an HTTP Authorization header whose
// value is "Bearer " followed by the AccessToken of the ServicePrincipalToken.
//
// By default, the token will automatically refresh if nearly expired (as determined by the
// RefreshWithin interval). Use the AutoRefresh method to enable or disable automatically refreshing
// tokens.
func (spt *ServicePrincipalToken) WithAuthorization() autorest.PrepareDecorator {
	return func(p autorest.Preparer) autorest.Preparer {
		return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
			if spt.autoRefresh {
				err := spt.EnsureFresh()
				if err != nil {
					return r, autorest.NewErrorWithError(err,
						"azure.ServicePrincipalToken", "WithAuthorization", "Failed to refresh Service Principal Token for request to %s",
						r.URL)
				}
			}
			return (autorest.WithBearerAuthorization(spt.AccessToken)(p)).Prepare(r)
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/engine-api/types"
	"github.com/opencontainers/runtime-spec/specs-go"
	"gopkg.in/yaml.v2"
)

// Moby is the type of a Moby config file
type Moby struct {
	Kernel struct {
		Image   string
		Cmdline string
	}
	Init   string
	System []MobyImage
	Daemon []MobyImage
	Files  []struct {
		Path     string
		Contents string
	}
	Outputs []struct {
		Format  string
		Project string
		Bucket  string
		Family  string
		Keys    string
		Public  bool
		Replace bool
	}
}

// MobyImage is the type of an image config
type MobyImage struct {
	Name             string
	Image            string
	Capabilities     []string
	Mounts           []specs.Mount
	Binds            []string
	Tmpfs            []string
	Command          []string
	Env              []string
	Cwd              string
	Net              string
	Pid              string
	Ipc              string
	Uts              string
	Readonly         bool
	UID              uint32   `yaml:"uid"`
	GID              uint32   `yaml:"gid"`
	AdditionalGids   []uint32 `yaml:"additionalGids"`
	NoNewPrivileges  bool     `yaml:"noNewPrivileges"`
	Hostname         string
	OomScoreAdj      int  `yaml:"oomScoreAdj"`
	DisableOOMKiller bool `yaml:"disableOOMKiller"`
}

// NewConfig parses a config file
func NewConfig(config []byte) (*Moby, error) {
	m := Moby{}

	err := yaml.Unmarshal(config, &m)
	if err != nil {
		return &m, err
	}

	return &m, nil
}

// ConfigToOCI converts a config specification to an OCI config file
func ConfigToOCI(image *MobyImage) ([]byte, error) {

	// TODO pass through same docker client to all functions
	cli, err := dockerClient()
	if err != nil {
		return []byte{}, err
	}

	inspect, err := dockerInspectImage(cli, image.Image)
	if err != nil {
		return []byte{}, err
	}

	return ConfigInspectToOCI(image, inspect)
}

func defaultMountpoint(tp string) string {
	switch tp {
	case "proc":
		return "/proc"
	case "devpts":
		return "/dev/pts"
	case "sysfs":
		return "/sys"
	case "cgroup":
		return "/sys/fs/cgroup"
	case "mqueue":
		return "/dev/mqueue"
	default:
		return ""
	}
}

// Sort mounts by number of path components so /dev/pts is listed after /dev
type mlist []specs.Mount

func (m mlist) Len() int {
	return len(m)
}
func (m mlist) Less(i, j int) bool {
	return m.parts(i) < m.parts(j)
}
func (m mlist) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}
func (m mlist) parts(i int) int {
	return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator))
}

// ConfigInspectToOCI converts a config and the output of image inspect to an OCI config file
func ConfigInspectToOCI(image *MobyImage, inspect types.ImageInspect) ([]byte, error) {
	oci := specs.Spec{}

	config := inspect.Config
	if config == nil {
		return []byte{}, errors.New("empty image config")
	}

	args := append(config.Entrypoint, config.Cmd...)
	if len(image.Command) != 0 {
		args = image.Command
	}
	env := config.Env
	if len(image.Env) != 0 {
		env = image.Env
	}
	cwd := config.WorkingDir
	if image.Cwd != "" {
		cwd = image.Cwd
	}
	if cwd == "" {
		cwd = "/"
	}
	// default options match what Docker does
	procOptions := []string{"nosuid", "nodev", "noexec", "relatime"}
	devOptions := []string{"nosuid", "strictatime", "mode=755", "size=65536k"}
	if image.Readonly {
		devOptions = append(devOptions, "ro")
	}
	ptsOptions := []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}
	sysOptions := []string{"nosuid", "noexec", "nodev"}
	if image.Readonly {
		sysOptions = append(sysOptions, "ro")
	}
	cgroupOptions := []string{"nosuid", "noexec", "nodev", "relatime", "ro"}
	// note omits "standard" /dev/shm and /dev/mqueue
	mounts := map[string]specs.Mount{
		"/proc":          {Destination: "/proc", Type: "proc", Source: "proc", Options: procOptions},
		"/dev":           {Destination: "/dev", Type: "tmpfs", Source: "tmpfs", Options: devOptions},
		"/dev/pts":       {Destination: "/dev/pts", Type: "devpts", Source: "devpts", Options: ptsOptions},
		"/sys":           {Destination: "/sys", Type: "sysfs", Source: "sysfs", Options: sysOptions},
		"/sys/fs/cgroup": {Destination: "/sys/fs/cgroup", Type: "cgroup", Source: "cgroup", Options: cgroupOptions},
	}
	for _, t := range image.Tmpfs {
		parts := strings.Split(t, ":")
		if len(parts) > 2 {
			return []byte{}, fmt.Errorf("Cannot parse tmpfs, too many ':': %s", t)
		}
		dest := parts[0]
		opts := []string{}
		if len(parts) == 2 {
			opts = strings.Split(parts[2], ",")
		}
		mounts[dest] = specs.Mount{Destination: dest, Type: "tmpfs", Source: "tmpfs", Options: opts}
	}
	for _, b := range image.Binds {
		parts := strings.Split(b, ":")
		if len(parts) < 2 {
			return []byte{}, fmt.Errorf("Cannot parse bind, missing ':': %s", b)
		}
		if len(parts) > 3 {
			return []byte{}, fmt.Errorf("Cannot parse bind, too many ':': %s", b)
		}
		src := parts[0]
		dest := parts[1]
		opts := []string{"rw", "rbind", "rprivate"}
		if len(parts) == 3 {
			opts = strings.Split(parts[2], ",")
		}
		mounts[dest] = specs.Mount{Destination: dest, Type: "bind", Source: src, Options: opts}
	}
	for _, m := range image.Mounts {
		tp := m.Type
		src := m.Source
		dest := m.Destination
		opts := m.Options
		if tp == "" {
			switch src {
			case "mqueue", "devpts", "proc", "sysfs", "cgroup":
				tp = src
			}
		}
		if tp == "" && dest == "/dev" {
			tp = "tmpfs"
		}
		if tp == "" {
			return []byte{}, fmt.Errorf("Mount for destination %s is missing type", dest)
		}
		if src == "" {
			// usually sane, eg proc, tmpfs etc
			src = tp
		}
		if dest == "" {
			dest = defaultMountpoint(tp)
		}
		if dest == "" {
			return []byte{}, fmt.Errorf("Mount type %s is missing destination", tp)
		}
		mounts[dest] = specs.Mount{Destination: dest, Type: tp, Source: src, Options: opts}
	}
	mountList := mlist{}
	for _, m := range mounts {
		mountList = append(mountList, m)
	}
	sort.Sort(mountList)
	namespaces := []specs.LinuxNamespace{}
	if image.Net != "" && image.Net != "host" {
		return []byte{}, fmt.Errorf("invalid net namespace: %s", image.Net)
	}
	if image.Net == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace})
	}
	if image.Pid != "" && image.Pid != "host" {
		return []byte{}, fmt.Errorf("invalid pid namespace: %s", image.Pid)
	}
	if image.Pid == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace})
	}
	if image.Ipc != "" && image.Ipc != "host" {
		return []byte{}, fmt.Errorf("invalid ipc namespace: %s", image.Ipc)
	}
	if image.Ipc == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.IPCNamespace})
	}
	if image.Uts != "" && image.Uts != "host" {
		return []byte{}, fmt.Errorf("invalid uts namespace: %s", image.Uts)
	}
	if image.Uts == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.UTSNamespace})
	}
	// TODO user, cgroup namespaces, maybe mount=host if useful
	namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.MountNamespace})
	caps := image.Capabilities
	if len(caps) == 1 && strings.ToLower(caps[0]) == "all" {
		caps = []string{
			"CAP_AUDIT_CONTROL",
			"CAP_AUDIT_READ",
			"CAP_AUDIT_WRITE",
			"CAP_BLOCK_SUSPEND",
			"CAP_CHOWN",
			"CAP_DAC_OVERRIDE",
			"CAP_DAC_READ_SEARCH",
			"CAP_FOWNER",
			"CAP_FSETID",
			"CAP_IPC_LOCK",
			"CAP_IPC_OWNER",
			"CAP_KILL",
			"CAP_LEASE",
			"CAP_LINUX_IMMUTABLE",
			"CAP_MAC_ADMIN",
			"CAP_MAC_OVERRIDE",
			"CAP_MKNOD",
			"CAP_NET_ADMIN",
			"CAP_NET_BIND_SERVICE",
			"CAP_NET_BROADCAST",
			"CAP_NET_RAW",
			"CAP_SETFCAP",
			"CAP_SETGID",
			"CAP_SETPCAP",
			"CAP_SETUID",
			"CAP_SYSLOG",
			"CAP_SYS_ADMIN",
			"CAP_SYS_BOOT",
			"CAP_SYS_CHROOT",
			"CAP_SYS_MODULE",
			"CAP_SYS_NICE",
			"CAP_SYS_PACCT",
			"CAP_SYS_PTRACE",
			"CAP_SYS_RAWIO",
			"CAP_SYS_RESOURCE",
			"CAP_SYS_TIME",
			"CAP_SYS_TTY_CONFIG",
			"CAP_WAKE_ALARM",
		}
	}

	oci.Version = specs.Version

	oci.Platform = specs.Platform{
		OS:   inspect.Os,
		Arch: inspect.Architecture,
	}

	oci.Process = specs.Process{
		Terminal: false,
		//ConsoleSize
		User: specs.User{
			UID:            image.UID,
			GID:            image.GID,
			AdditionalGids: image.AdditionalGids,
			// Username (Windows)
		},
		Args: args,
		Env:  env,
		Cwd:  cwd,
		Capabilities: &specs.LinuxCapabilities{
			Bounding:    caps,
			Effective:   caps,
			Inheritable: caps,
			Permitted:   caps,
			Ambient:     []string{},
		},
		Rlimits:         []specs.LinuxRlimit{},
		NoNewPrivileges: image.NoNewPrivileges,
		// ApparmorProfile
		// SelinuxLabel
	}

	oci.Root = specs.Root{
		Path:     "rootfs",
		Readonly: image.Readonly,
	}

	oci.Hostname = image.Hostname
	oci.Mounts = mountList

	oci.Linux = &specs.Linux{
		// UIDMappings
		// GIDMappings
		// Sysctl
		Resources: &specs.LinuxResources{
			// Devices
			DisableOOMKiller: &image.DisableOOMKiller,
			// Memory
			// CPU
			// Pids
			// BlockIO
			// HugepageLimits
			// Network
		},
		// CgroupsPath
		Namespaces: namespaces,
		// Devices
		// Seccomp
		// RootfsPropagation
		// MaskedPaths
		// ReadonlyPaths
		// MountLabel
		// IntelRdt
	}

	return json.MarshalIndent(oci, "", "    ")
}

func filesystem(m *Moby) (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	defer tw.Close()

	log.Infof("Add files:")
	for _, f := range m.Files {
		log.Infof("  %s", f.Path)
		if f.Path == "" {
			return buf, errors.New("Did not specify path for file")
		}
		if f.Contents == "" {
			return buf, errors.New("Contents of file not specified")
		}
		// we need all the leading directories
		parts := strings.Split(path.Dir(f.Path), "/")
		root := ""
		for _, p := range parts {
			if p == "." || p == "/" {
				continue
			}
			if root == "" {
				root = p
			} else {
				root = root + "/" + p
			}
			hdr := &tar.Header{
				Name:     root,
				Typeflag: tar.TypeDir,
				Mode:     0700,
			}
			err := tw.WriteHeader(hdr)
			if err != nil {
				return buf, err
			}
		}
		hdr := &tar.Header{
			Name: f.Path,
			Mode: 0600,
			Size: int64(len(f.Contents)),
		}
		err := tw.WriteHeader(hdr)
		if err != nil {
			return buf, err
		}
		_, err = tw.Write([]byte(f.Contents))
		if err != nil {
			return buf, err
		}
	}
	return buf, nil
}
<commit_msg>Add more OCI options<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/engine-api/types"
	"github.com/opencontainers/runtime-spec/specs-go"
	"gopkg.in/yaml.v2"
)

// Moby is the type of a Moby config file
type Moby struct {
	Kernel struct {
		Image   string
		Cmdline string
	}
	Init   string
	System []MobyImage
	Daemon []MobyImage
	Files  []struct {
		Path     string
		Contents string
	}
	Outputs []struct {
		Format  string
		Project string
		Bucket  string
		Family  string
		Keys    string
		Public  bool
		Replace bool
	}
}

// MobyImage is the type of an image config
type MobyImage struct {
	Name              string
	Image             string
	Capabilities      []string
	Mounts            []specs.Mount
	Binds             []string
	Tmpfs             []string
	Command           []string
	Env               []string
	Cwd               string
	Net               string
	Pid               string
	Ipc               string
	Uts               string
	Readonly          bool
	UID               uint32   `yaml:"uid"`
	GID               uint32   `yaml:"gid"`
	AdditionalGids    []uint32 `yaml:"additionalGids"`
	NoNewPrivileges   bool     `yaml:"noNewPrivileges"`
	Hostname          string
	OomScoreAdj       int    `yaml:"oomScoreAdj"`
	DisableOOMKiller  bool   `yaml:"disableOOMKiller"`
	RootfsPropagation string `yaml:"rootfsPropagation"`
	CgroupsPath       string `yaml:"cgroupsPath"`
	Sysctl            map[string]string
}

// NewConfig parses a config file
func NewConfig(config []byte) (*Moby, error) {
	m := Moby{}

	err := yaml.Unmarshal(config, &m)
	if err != nil {
		return &m, err
	}

	return &m, nil
}

// ConfigToOCI converts a config specification to an OCI config file
func ConfigToOCI(image *MobyImage) ([]byte, error) {

	// TODO pass through same docker client to all functions
	cli, err := dockerClient()
	if err != nil {
		return []byte{}, err
	}

	inspect, err := dockerInspectImage(cli, image.Image)
	if err != nil {
		return []byte{}, err
	}

	return ConfigInspectToOCI(image, inspect)
}

func defaultMountpoint(tp string) string {
	switch tp {
	case "proc":
		return "/proc"
	case "devpts":
		return "/dev/pts"
	case "sysfs":
		return "/sys"
	case "cgroup":
		return "/sys/fs/cgroup"
	case "mqueue":
		return "/dev/mqueue"
	default:
		return ""
	}
}

// Sort mounts by number of path components so /dev/pts is listed after /dev
type mlist []specs.Mount

func (m mlist) Len() int {
	return len(m)
}
func (m mlist) Less(i, j int) bool {
	return m.parts(i) < m.parts(j)
}
func (m mlist) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}
func (m mlist) parts(i int) int {
	return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator))
}

// ConfigInspectToOCI converts a config and the output of image inspect to an OCI config file
func ConfigInspectToOCI(image *MobyImage, inspect types.ImageInspect) ([]byte, error) {
	oci := specs.Spec{}

	config := inspect.Config
	if config == nil {
		return []byte{}, errors.New("empty image config")
	}

	args := append(config.Entrypoint, config.Cmd...)
	if len(image.Command) != 0 {
		args = image.Command
	}
	env := config.Env
	if len(image.Env) != 0 {
		env = image.Env
	}
	cwd := config.WorkingDir
	if image.Cwd != "" {
		cwd = image.Cwd
	}
	if cwd == "" {
		cwd = "/"
	}
	// default options match what Docker does
	procOptions := []string{"nosuid", "nodev", "noexec", "relatime"}
	devOptions := []string{"nosuid", "strictatime", "mode=755", "size=65536k"}
	if image.Readonly {
		devOptions = append(devOptions, "ro")
	}
	ptsOptions := []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}
	sysOptions := []string{"nosuid", "noexec", "nodev"}
	if image.Readonly {
		sysOptions = append(sysOptions, "ro")
	}
	cgroupOptions := []string{"nosuid", "noexec", "nodev", "relatime", "ro"}
	// note omits "standard" /dev/shm and /dev/mqueue
	mounts := map[string]specs.Mount{
		"/proc":          {Destination: "/proc", Type: "proc", Source: "proc", Options: procOptions},
		"/dev":           {Destination: "/dev", Type: "tmpfs", Source: "tmpfs", Options: devOptions},
		"/dev/pts":       {Destination: "/dev/pts", Type: "devpts", Source: "devpts", Options: ptsOptions},
		"/sys":           {Destination: "/sys", Type: "sysfs", Source: "sysfs", Options: sysOptions},
		"/sys/fs/cgroup": {Destination: "/sys/fs/cgroup", Type: "cgroup", Source: "cgroup", Options: cgroupOptions},
	}
	for _, t := range image.Tmpfs {
		parts := strings.Split(t, ":")
		if len(parts) > 2 {
			return []byte{}, fmt.Errorf("Cannot parse tmpfs, too many ':': %s", t)
		}
		dest := parts[0]
		opts := []string{}
		if len(parts) == 2 {
			opts = strings.Split(parts[2], ",")
		}
		mounts[dest] = specs.Mount{Destination: dest, Type: "tmpfs", Source: "tmpfs", Options: opts}
	}
	for _, b := range image.Binds {
		parts := strings.Split(b, ":")
		if len(parts) < 2 {
			return []byte{}, fmt.Errorf("Cannot parse bind, missing ':': %s", b)
		}
		if len(parts) > 3 {
			return []byte{}, fmt.Errorf("Cannot parse bind, too many ':': %s", b)
		}
		src := parts[0]
		dest := parts[1]
		opts := []string{"rw", "rbind", "rprivate"}
		if len(parts) == 3 {
			opts = strings.Split(parts[2], ",")
		}
		mounts[dest] = specs.Mount{Destination: dest, Type: "bind", Source: src, Options: opts}
	}
	for _, m := range image.Mounts {
		tp := m.Type
		src := m.Source
		dest := m.Destination
		opts := m.Options
		if tp == "" {
			switch src {
			case "mqueue", "devpts", "proc", "sysfs", "cgroup":
				tp = src
			}
		}
		if tp == "" && dest == "/dev" {
			tp = "tmpfs"
		}
		if tp == "" {
			return []byte{}, fmt.Errorf("Mount for destination %s is missing type", dest)
		}
		if src == "" {
			// usually sane, eg proc, tmpfs etc
			src = tp
		}
		if dest == "" {
			dest = defaultMountpoint(tp)
		}
		if dest == "" {
			return []byte{}, fmt.Errorf("Mount type %s is missing destination", tp)
		}
		mounts[dest] = specs.Mount{Destination: dest, Type: tp, Source: src, Options: opts}
	}
	mountList := mlist{}
	for _, m := range mounts {
		mountList = append(mountList, m)
	}
	sort.Sort(mountList)
	namespaces := []specs.LinuxNamespace{}
	if image.Net != "" && image.Net != "host" {
		return []byte{}, fmt.Errorf("invalid net namespace: %s", image.Net)
	}
	if image.Net == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace})
	}
	if image.Pid != "" && image.Pid != "host" {
		return []byte{}, fmt.Errorf("invalid pid namespace: %s", image.Pid)
	}
	if image.Pid == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.PIDNamespace})
	}
	if image.Ipc != "" && image.Ipc != "host" {
		return []byte{}, fmt.Errorf("invalid ipc namespace: %s", image.Ipc)
	}
	if image.Ipc == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.IPCNamespace})
	}
	if image.Uts != "" && image.Uts != "host" {
		return []byte{}, fmt.Errorf("invalid uts namespace: %s", image.Uts)
	}
	if image.Uts == "" {
		namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.UTSNamespace})
	}
	// TODO user, cgroup namespaces, maybe mount=host if useful
	namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.MountNamespace})
	caps := image.Capabilities
	if len(caps) == 1 && strings.ToLower(caps[0]) == "all" {
		caps = []string{
			"CAP_AUDIT_CONTROL",
			"CAP_AUDIT_READ",
			"CAP_AUDIT_WRITE",
			"CAP_BLOCK_SUSPEND",
			"CAP_CHOWN",
			"CAP_DAC_OVERRIDE",
			"CAP_DAC_READ_SEARCH",
			"CAP_FOWNER",
			"CAP_FSETID",
			"CAP_IPC_LOCK",
			"CAP_IPC_OWNER",
			"CAP_KILL",
			"CAP_LEASE",
			"CAP_LINUX_IMMUTABLE",
			"CAP_MAC_ADMIN",
			"CAP_MAC_OVERRIDE",
			"CAP_MKNOD",
			"CAP_NET_ADMIN",
			"CAP_NET_BIND_SERVICE",
			"CAP_NET_BROADCAST",
			"CAP_NET_RAW",
			"CAP_SETFCAP",
			"CAP_SETGID",
			"CAP_SETPCAP",
			"CAP_SETUID",
			"CAP_SYSLOG",
			"CAP_SYS_ADMIN",
			"CAP_SYS_BOOT",
			"CAP_SYS_CHROOT",
			"CAP_SYS_MODULE",
			"CAP_SYS_NICE",
			"CAP_SYS_PACCT",
			"CAP_SYS_PTRACE",
			"CAP_SYS_RAWIO",
			"CAP_SYS_RESOURCE",
			"CAP_SYS_TIME",
			"CAP_SYS_TTY_CONFIG",
			"CAP_WAKE_ALARM",
		}
	}

	oci.Version = specs.Version

	oci.Platform = specs.Platform{
		OS:   inspect.Os,
		Arch: inspect.Architecture,
	}

	oci.Process = specs.Process{
		Terminal: false,
		//ConsoleSize
		User: specs.User{
			UID:            image.UID,
			GID:            image.GID,
			AdditionalGids: image.AdditionalGids,
			// Username (Windows)
		},
		Args: args,
		Env:  env,
		Cwd:  cwd,
		Capabilities: &specs.LinuxCapabilities{
			Bounding:    caps,
			Effective:   caps,
			Inheritable: caps,
			Permitted:   caps,
			Ambient:     []string{},
		},
		Rlimits:         []specs.LinuxRlimit{},
		NoNewPrivileges: image.NoNewPrivileges,
		// ApparmorProfile
		// SelinuxLabel
	}

	oci.Root = specs.Root{
		Path:     "rootfs",
		Readonly: image.Readonly,
	}

	oci.Hostname = image.Hostname
	oci.Mounts = mountList

	oci.Linux = &specs.Linux{
		// UIDMappings
		// GIDMappings
		Sysctl: image.Sysctl,
		Resources: &specs.LinuxResources{
			// Devices
			DisableOOMKiller: &image.DisableOOMKiller,
			// Memory
			// CPU
			// Pids
			// BlockIO
			// HugepageLimits
			// Network
		},
		CgroupsPath: image.CgroupsPath,
		Namespaces:  namespaces,
		// Devices
		// Seccomp
		RootfsPropagation: image.RootfsPropagation,
		// MaskedPaths
		// ReadonlyPaths
		// MountLabel
		// IntelRdt
	}

	return json.MarshalIndent(oci, "", "    ")
}

func filesystem(m *Moby) (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	defer tw.Close()

	log.Infof("Add files:")
	for _, f := range m.Files {
		log.Infof("  %s", f.Path)
		if f.Path == "" {
			return buf, errors.New("Did not specify path for file")
		}
		if f.Contents == "" {
			return buf, errors.New("Contents of file not specified")
		}
		// we need all the leading directories
		parts := strings.Split(path.Dir(f.Path), "/")
		root := ""
		for _, p := range parts {
			if p == "." || p == "/" {
				continue
			}
			if root == "" {
				root = p
			} else {
				root = root + "/" + p
			}
			hdr := &tar.Header{
				Name:     root,
				Typeflag: tar.TypeDir,
				Mode:     0700,
			}
			err := tw.WriteHeader(hdr)
			if err != nil {
				return buf, err
			}
		}
		hdr := &tar.Header{
			Name: f.Path,
			Mode: 0600,
			Size: int64(len(f.Contents)),
		}
		err := tw.WriteHeader(hdr)
		if err != nil {
			return buf, err
		}
		_, err = tw.Write([]byte(f.Contents))
		if err != nil {
			return buf, err
		}
	}
	return buf, nil
}
<|endoftext|>


<commit_before>package main

import (
	"dynamics"
	"time"
)

func main() {
	start := time.Date(2016, 3, 14, 9, 31, 0, 0, time.UTC)
	end := time.Date(2016, 10, 13, 0, 0, 0, 0, time.UTC)
	sc := dynamics.NewEmptySC("test", 100)
	// OEs
	//earth := dynamics.Earth.HelioOrbit(start)
	//mars := dynamics.Mars.HelioOrbit(start)
	//a := 0.5 * (earth.GetApoapsis() + mars.GetApoapsis())
	a := 1.5 * dynamics.Earth.Radius
	e := 1e-7
	i := 1e-7
	Ω := 90.0
	ω := 45.0
	ν := 20.5
	mss := dynamics.NewMission(sc, dynamics.NewOrbitFromOE(a, e, i, Ω, ω, ν, dynamics.Earth), start, end, false, dynamics.ExportConfig{Filename: "Inc", OE: true, Cosmo: true, Timestamp: false})
	mss.Propagate()
}
<commit_msg>But around Mars, it's wrong...<commit_after>package main

import (
	"dynamics"
	"time"
)

func main() {
	start := time.Now().UTC()
	end := start.Add(time.Duration(5*30.5*24) * time.Hour)
	sc := dynamics.NewEmptySC("test", 100)
	// OEs
	//earth := dynamics.Earth.HelioOrbit(start)
	//mars := dynamics.Mars.HelioOrbit(start)
	//a := 0.5 * (earth.GetApoapsis() + mars.GetApoapsis())
	a := 1.5 * dynamics.Mars.Radius
	e := 1e-7
	i := 1e-7
	Ω := 90.0
	ω := 45.0
	ν := 20.5
	mss := dynamics.NewMission(sc, dynamics.NewOrbitFromOE(a, e, i, Ω, ω, ν, dynamics.Mars), start, end, false, dynamics.ExportConfig{Filename: "Inc", OE: true, Cosmo: true, Timestamp: false})
	mss.Propagate()
}
<|endoftext|>


<commit_before>// ----------------------------------------------------------------------------
//
//     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
//
// ----------------------------------------------------------------------------
//
//     This file is automatically generated by Magic Modules and manual
//     changes will be clobbered when the file is regenerated.
//
//     Please read more about how to change this file in
//     .github/CONTRIBUTING.md.
//
// ----------------------------------------------------------------------------

package google

import "reflect"

func GetBillingBudgetCaiObject(d TerraformResourceData, config *Config) ([]Asset, error) {
	name, err := assetName(d, config, "//billing.googleapis.com/{{name}}")
	if err != nil {
		return []Asset{}, err
	}
	if obj, err := GetBillingBudgetApiObject(d, config); err == nil {
		return []Asset{{
			Name: name,
			Type: "billing.googleapis.com/Budget",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/billing/v1/rest",
				DiscoveryName:        "Budget",
				Data:                 obj,
			},
		}}, nil
	} else {
		return []Asset{}, err
	}
}

func GetBillingBudgetApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	obj := make(map[string]interface{})
	displayNameProp, err := expandBillingBudgetDisplayName(d.Get("display_name"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("display_name"); !isEmptyValue(reflect.ValueOf(displayNameProp)) && (ok || !reflect.DeepEqual(v, displayNameProp)) {
		obj["displayName"] = displayNameProp
	}
	budgetFilterProp, err := expandBillingBudgetBudgetFilter(d.Get("budget_filter"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("budget_filter"); !isEmptyValue(reflect.ValueOf(budgetFilterProp)) && (ok || !reflect.DeepEqual(v, budgetFilterProp)) {
		obj["budgetFilter"] = budgetFilterProp
	}
	amountProp, err := expandBillingBudgetAmount(d.Get("amount"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("amount"); !isEmptyValue(reflect.ValueOf(amountProp)) && (ok || !reflect.DeepEqual(v, amountProp)) {
		obj["amount"] = amountProp
	}
	thresholdRulesProp, err := expandBillingBudgetThresholdRules(d.Get("threshold_rules"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("threshold_rules"); !isEmptyValue(reflect.ValueOf(thresholdRulesProp)) && (ok || !reflect.DeepEqual(v, thresholdRulesProp)) {
		obj["thresholdRules"] = thresholdRulesProp
	}
	notificationsRuleProp, err := expandBillingBudgetAllUpdatesRule(d.Get("all_updates_rule"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("all_updates_rule"); !isEmptyValue(reflect.ValueOf(notificationsRuleProp)) && (ok || !reflect.DeepEqual(v, notificationsRuleProp)) {
		obj["notificationsRule"] = notificationsRuleProp
	}

	return obj, nil
}

func expandBillingBudgetDisplayName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilter(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedProjects, err := expandBillingBudgetBudgetFilterProjects(original["projects"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedProjects); val.IsValid() && !isEmptyValue(val) {
		transformed["projects"] = transformedProjects
	}

	transformedCreditTypesTreatment, err := expandBillingBudgetBudgetFilterCreditTypesTreatment(original["credit_types_treatment"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedCreditTypesTreatment); val.IsValid() && !isEmptyValue(val) {
		transformed["creditTypesTreatment"] = transformedCreditTypesTreatment
	}

	transformedServices, err := expandBillingBudgetBudgetFilterServices(original["services"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedServices); val.IsValid() && !isEmptyValue(val) {
		transformed["services"] = transformedServices
	}

	transformedCreditTypes, err := expandBillingBudgetBudgetFilterCreditTypes(original["credit_types"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedCreditTypes); val.IsValid() && !isEmptyValue(val) {
		transformed["creditTypes"] = transformedCreditTypes
	}

	transformedSubaccounts, err := expandBillingBudgetBudgetFilterSubaccounts(original["subaccounts"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSubaccounts); val.IsValid() && !isEmptyValue(val) {
		transformed["subaccounts"] = transformedSubaccounts
	}

	transformedLabels, err := expandBillingBudgetBudgetFilterLabels(original["labels"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedLabels); val.IsValid() && !isEmptyValue(val) {
		transformed["labels"] = transformedLabels
	}

	return transformed, nil
}

func expandBillingBudgetBudgetFilterProjects(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterCreditTypesTreatment(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterServices(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterCreditTypes(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterSubaccounts(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) {
	if v == nil {
		return map[string]string{}, nil
	}
	m := make(map[string]string)
	for k, val := range v.(map[string]interface{}) {
		m[k] = val.(string)
	}
	return m, nil
}

func expandBillingBudgetAmount(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedSpecifiedAmount, err := expandBillingBudgetAmountSpecifiedAmount(original["specified_amount"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSpecifiedAmount); val.IsValid() && !isEmptyValue(val) {
		transformed["specifiedAmount"] = transformedSpecifiedAmount
	}

	transformedLastPeriodAmount, err := expandBillingBudgetAmountLastPeriodAmount(original["last_period_amount"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedLastPeriodAmount); val.IsValid() && !isEmptyValue(val) {
		transformed["lastPeriodAmount"] = transformedLastPeriodAmount
	}

	return transformed, nil
}

func expandBillingBudgetAmountSpecifiedAmount(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedCurrencyCode, err := expandBillingBudgetAmountSpecifiedAmountCurrencyCode(original["currency_code"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedCurrencyCode); val.IsValid() && !isEmptyValue(val) {
		transformed["currencyCode"] = transformedCurrencyCode
	}

	transformedUnits, err := expandBillingBudgetAmountSpecifiedAmountUnits(original["units"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedUnits); val.IsValid() && !isEmptyValue(val) {
		transformed["units"] = transformedUnits
	}

	transformedNanos, err := expandBillingBudgetAmountSpecifiedAmountNanos(original["nanos"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedNanos); val.IsValid() && !isEmptyValue(val) {
		transformed["nanos"] = transformedNanos
	}

	return transformed, nil
}

func expandBillingBudgetAmountSpecifiedAmountCurrencyCode(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAmountSpecifiedAmountUnits(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAmountSpecifiedAmountNanos(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAmountLastPeriodAmount(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	if v == nil || !v.(bool) {
		return nil, nil
	}

	return struct{}{}, nil
}

func expandBillingBudgetThresholdRules(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	req := make([]interface{}, 0, len(l))
	for _, raw := range l {
		if raw == nil {
			continue
		}
		original := raw.(map[string]interface{})
		transformed := make(map[string]interface{})

		transformedThresholdPercent, err := expandBillingBudgetThresholdRulesThresholdPercent(original["threshold_percent"], d, config)
		if err != nil {
			return nil, err
		} else {
			transformed["thresholdPercent"] = transformedThresholdPercent
		}

		transformedSpendBasis, err := expandBillingBudgetThresholdRulesSpendBasis(original["spend_basis"], d, config)
		if err != nil {
			return nil, err
		} else if val := reflect.ValueOf(transformedSpendBasis); val.IsValid() && !isEmptyValue(val) {
			transformed["spendBasis"] = transformedSpendBasis
		}

		req = append(req, transformed)
	}
	return req, nil
}

func expandBillingBudgetThresholdRulesThresholdPercent(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetThresholdRulesSpendBasis(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRule(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedPubsubTopic, err := expandBillingBudgetAllUpdatesRulePubsubTopic(original["pubsub_topic"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedPubsubTopic); val.IsValid() && !isEmptyValue(val) {
		transformed["pubsubTopic"] = transformedPubsubTopic
	}

	transformedSchemaVersion, err := expandBillingBudgetAllUpdatesRuleSchemaVersion(original["schema_version"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSchemaVersion); val.IsValid() && !isEmptyValue(val) {
		transformed["schemaVersion"] = transformedSchemaVersion
	}

	transformedMonitoringNotificationChannels, err := expandBillingBudgetAllUpdatesRuleMonitoringNotificationChannels(original["monitoring_notification_channels"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedMonitoringNotificationChannels); val.IsValid() && !isEmptyValue(val) {
		transformed["monitoringNotificationChannels"] = transformedMonitoringNotificationChannels
	}

	transformedDisableDefaultIamRecipients, err := expandBillingBudgetAllUpdatesRuleDisableDefaultIamRecipients(original["disable_default_iam_recipients"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedDisableDefaultIamRecipients); val.IsValid() && !isEmptyValue(val) {
		transformed["disableDefaultIamRecipients"] = transformedDisableDefaultIamRecipients
	}

	return transformed, nil
}

func expandBillingBudgetAllUpdatesRulePubsubTopic(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRuleSchemaVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRuleMonitoringNotificationChannels(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRuleDisableDefaultIamRecipients(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}
<commit_msg>Made deletion of all budget filter projects be recognized (#4731) (#692)<commit_after>// ----------------------------------------------------------------------------
//
//     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
//
// ----------------------------------------------------------------------------
//
//     This file is automatically generated by Magic Modules and manual
//     changes will be clobbered when the file is regenerated.
//
//     Please read more about how to change this file in
//     .github/CONTRIBUTING.md.
//
// ----------------------------------------------------------------------------

package google

import "reflect"

func GetBillingBudgetCaiObject(d TerraformResourceData, config *Config) ([]Asset, error) {
	name, err := assetName(d, config, "//billing.googleapis.com/billingAccounts/{{billing_account}}/budgets/{{name}}")
	if err != nil {
		return []Asset{}, err
	}
	if obj, err := GetBillingBudgetApiObject(d, config); err == nil {
		return []Asset{{
			Name: name,
			Type: "billing.googleapis.com/Budget",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/billing/v1/rest",
				DiscoveryName:        "Budget",
				Data:                 obj,
			},
		}}, nil
	} else {
		return []Asset{}, err
	}
}

func GetBillingBudgetApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	obj := make(map[string]interface{})
	displayNameProp, err := expandBillingBudgetDisplayName(d.Get("display_name"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("display_name"); !isEmptyValue(reflect.ValueOf(displayNameProp)) && (ok || !reflect.DeepEqual(v, displayNameProp)) {
		obj["displayName"] = displayNameProp
	}
	budgetFilterProp, err := expandBillingBudgetBudgetFilter(d.Get("budget_filter"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("budget_filter"); !isEmptyValue(reflect.ValueOf(budgetFilterProp)) && (ok || !reflect.DeepEqual(v, budgetFilterProp)) {
		obj["budgetFilter"] = budgetFilterProp
	}
	amountProp, err := expandBillingBudgetAmount(d.Get("amount"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("amount"); !isEmptyValue(reflect.ValueOf(amountProp)) && (ok || !reflect.DeepEqual(v, amountProp)) {
		obj["amount"] = amountProp
	}
	thresholdRulesProp, err := expandBillingBudgetThresholdRules(d.Get("threshold_rules"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("threshold_rules"); !isEmptyValue(reflect.ValueOf(thresholdRulesProp)) && (ok || !reflect.DeepEqual(v, thresholdRulesProp)) {
		obj["thresholdRules"] = thresholdRulesProp
	}
	notificationsRuleProp, err := expandBillingBudgetAllUpdatesRule(d.Get("all_updates_rule"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("all_updates_rule"); !isEmptyValue(reflect.ValueOf(notificationsRuleProp)) && (ok || !reflect.DeepEqual(v, notificationsRuleProp)) {
		obj["notificationsRule"] = notificationsRuleProp
	}

	return obj, nil
}

func expandBillingBudgetDisplayName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilter(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedProjects, err := expandBillingBudgetBudgetFilterProjects(original["projects"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedProjects); val.IsValid() && !isEmptyValue(val) {
		transformed["projects"] = transformedProjects
	}

	transformedCreditTypesTreatment, err := expandBillingBudgetBudgetFilterCreditTypesTreatment(original["credit_types_treatment"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedCreditTypesTreatment); val.IsValid() && !isEmptyValue(val) {
		transformed["creditTypesTreatment"] = transformedCreditTypesTreatment
	}

	transformedServices, err := expandBillingBudgetBudgetFilterServices(original["services"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedServices); val.IsValid() && !isEmptyValue(val) {
		transformed["services"] = transformedServices
	}

	transformedCreditTypes, err := expandBillingBudgetBudgetFilterCreditTypes(original["credit_types"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedCreditTypes); val.IsValid() && !isEmptyValue(val) {
		transformed["creditTypes"] = transformedCreditTypes
	}

	transformedSubaccounts, err := expandBillingBudgetBudgetFilterSubaccounts(original["subaccounts"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSubaccounts); val.IsValid() && !isEmptyValue(val) {
		transformed["subaccounts"] = transformedSubaccounts
	}

	transformedLabels, err := expandBillingBudgetBudgetFilterLabels(original["labels"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedLabels); val.IsValid() && !isEmptyValue(val) {
		transformed["labels"] = transformedLabels
	}

	return transformed, nil
}

func expandBillingBudgetBudgetFilterProjects(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterCreditTypesTreatment(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterServices(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterCreditTypes(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterSubaccounts(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetBudgetFilterLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) {
	if v == nil {
		return map[string]string{}, nil
	}
	m := make(map[string]string)
	for k, val := range v.(map[string]interface{}) {
		m[k] = val.(string)
	}
	return m, nil
}

func expandBillingBudgetAmount(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedSpecifiedAmount, err := expandBillingBudgetAmountSpecifiedAmount(original["specified_amount"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSpecifiedAmount); val.IsValid() && !isEmptyValue(val) {
		transformed["specifiedAmount"] = transformedSpecifiedAmount
	}

	transformedLastPeriodAmount, err := expandBillingBudgetAmountLastPeriodAmount(original["last_period_amount"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedLastPeriodAmount); val.IsValid() && !isEmptyValue(val) {
		transformed["lastPeriodAmount"] = transformedLastPeriodAmount
	}

	return transformed, nil
}

func expandBillingBudgetAmountSpecifiedAmount(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedCurrencyCode, err := expandBillingBudgetAmountSpecifiedAmountCurrencyCode(original["currency_code"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedCurrencyCode); val.IsValid() && !isEmptyValue(val) {
		transformed["currencyCode"] = transformedCurrencyCode
	}

	transformedUnits, err := expandBillingBudgetAmountSpecifiedAmountUnits(original["units"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedUnits); val.IsValid() && !isEmptyValue(val) {
		transformed["units"] = transformedUnits
	}

	transformedNanos, err := expandBillingBudgetAmountSpecifiedAmountNanos(original["nanos"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedNanos); val.IsValid() && !isEmptyValue(val) {
		transformed["nanos"] = transformedNanos
	}

	return transformed, nil
}

func expandBillingBudgetAmountSpecifiedAmountCurrencyCode(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAmountSpecifiedAmountUnits(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAmountSpecifiedAmountNanos(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAmountLastPeriodAmount(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	if v == nil || !v.(bool) {
		return nil, nil
	}

	return struct{}{}, nil
}

func expandBillingBudgetThresholdRules(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	req := make([]interface{}, 0, len(l))
	for _, raw := range l {
		if raw == nil {
			continue
		}
		original := raw.(map[string]interface{})
		transformed := make(map[string]interface{})

		transformedThresholdPercent, err := expandBillingBudgetThresholdRulesThresholdPercent(original["threshold_percent"], d, config)
		if err != nil {
			return nil, err
		} else {
			transformed["thresholdPercent"] = transformedThresholdPercent
		}

		transformedSpendBasis, err := expandBillingBudgetThresholdRulesSpendBasis(original["spend_basis"], d, config)
		if err != nil {
			return nil, err
		} else if val := reflect.ValueOf(transformedSpendBasis); val.IsValid() && !isEmptyValue(val) {
			transformed["spendBasis"] = transformedSpendBasis
		}

		req = append(req, transformed)
	}
	return req, nil
}

func expandBillingBudgetThresholdRulesThresholdPercent(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetThresholdRulesSpendBasis(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRule(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedPubsubTopic, err := expandBillingBudgetAllUpdatesRulePubsubTopic(original["pubsub_topic"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedPubsubTopic); val.IsValid() && !isEmptyValue(val) {
		transformed["pubsubTopic"] = transformedPubsubTopic
	}

	transformedSchemaVersion, err := expandBillingBudgetAllUpdatesRuleSchemaVersion(original["schema_version"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSchemaVersion); val.IsValid() && !isEmptyValue(val) {
		transformed["schemaVersion"] = transformedSchemaVersion
	}

	transformedMonitoringNotificationChannels, err := expandBillingBudgetAllUpdatesRuleMonitoringNotificationChannels(original["monitoring_notification_channels"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedMonitoringNotificationChannels); val.IsValid() && !isEmptyValue(val) {
		transformed["monitoringNotificationChannels"] = transformedMonitoringNotificationChannels
	}

	transformedDisableDefaultIamRecipients, err := expandBillingBudgetAllUpdatesRuleDisableDefaultIamRecipients(original["disable_default_iam_recipients"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedDisableDefaultIamRecipients); val.IsValid() && !isEmptyValue(val) {
		transformed["disableDefaultIamRecipients"] = transformedDisableDefaultIamRecipients
	}

	return transformed, nil
}

func expandBillingBudgetAllUpdatesRulePubsubTopic(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRuleSchemaVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRuleMonitoringNotificationChannels(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandBillingBudgetAllUpdatesRuleDisableDefaultIamRecipients(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"net"
	"time"
)

func main() {
	listener, err := net.Listen("tcp", "localhost:8000")
	if err != nil {
		log.Fatal(err)
	}
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Print(err)
			continue
		}
		handleConn(conn)
	}
}

func handleConn(c net.Conn) {
	defer c.Close()
	for {
		_, err := io.WriteString(c, time.Now().Format("15:04:04\n"))
		if err != nil {
			return
		}
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>[8.1] clock with goroutine<commit_after>package main

import (
	"io"
	"log"
	"net"
	"time"
)

func main() {
	// port := flag.Int("port", 8000, "clock port")
	listener, err := net.Listen("tcp", "localhost:8000")
	if err != nil {
		log.Fatal(err)
	}
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Print(err)
			continue
		}
		go handleConn(conn)
	}
}

func handleConn(c net.Conn) {
	defer c.Close()
	for {
		_, err := io.WriteString(c, time.Now().Format("15:04:04\n"))
		if err != nil {
			return
		}
		time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os/exec"
	"sort"
	"strings"
)

func imports(name string) []string {
	binary, err := exec.LookPath("go")
	if err != nil {
		log.Fatal(err)
	}
	out, err := exec.Command(binary, "list", "-json", name).Output()
	if err != nil {
		log.Fatal(err)
	}
	properties := struct{ Imports []string }{}
	if err := json.Unmarshal(out, &properties); err != nil {
		log.Fatal(err)
	}
	return properties.Imports
}

func walk(seen map[string]bool, name string) {
	ok := seen[name]
	if ok {
		return
	}
	seen[name] = true
	for _, pkg := range imports(name) {
		walk(seen, pkg)
	}
}

func main() {
	flag.Parse()
	packages := flag.Args()
	_ = packages
	deps := make(map[string]bool)
	walk(deps, packages[0])
	out := make([]string, 0, len(deps))
	for k := range deps {
		out = append(out, k)
	}
	out = sort.StringSlice(out)
	fmt.Println(strings.Join(out, "\n"))
}
<commit_msg>[10.4] small improvements<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os/exec"
)

var binaryPath string

func init() {
	var err error
	binaryPath, err = exec.LookPath("go")
	if err != nil {
		log.Fatal(err)
	}
}

type properties struct {
	Name       string
	ImportPath string
	Deps       []string
}

func getProperties(name string) (*properties, error) {
	out, err := exec.Command(binaryPath, "list", "-json", name).Output()
	if err != nil {
		return nil, err
	}
	var p properties
	if err := json.Unmarshal(out, &p); err != nil {
		return nil, err
	}
	return &p, nil
}

func main() {
	flag.Parse()
	packages := flag.Args()
	if len(packages) <= 0 {
		log.Fatal("please, specify package name")
	}
	dependencies := make(map[string]int)
	for _, p := range packages {
		prop, err := getProperties(p)
		if err != nil {
			log.Fatal(err)
		}
		for _, d := range prop.Deps {
			dependencies[d]++
		}
	}
	fmt.Println(dependencies)
}
<|endoftext|>


<commit_before>package operations

import (
	"encoding/json"

	"github.com/omise/omise-go"
	"github.com/omise/omise-go/internal"
)

// Example:
//
//	charges, list := &omise.ChargeList{}, &ListCharges{
//		List{
//			Limit: 100,
//			From: time.Now().Add(-1 * time.Hour),
//		},
//	}
//	if e := client.Do(charges, list); e != nil {
//		panic(e)
//	}
//
//	fmt.Println("# of charges made in the last hour:", len(charges.Data))
//
type ListCharges struct {
	List
}

func (req *ListCharges) Describe() *internal.Description {
	return &internal.Description{
		Endpoint: internal.API,
		Method:   "GET",
		Path:     "/charges",
	}
}

// Note that because bool defaults to false in GO, we use DontCapture instead of Capture
// here so it matches with Omise's REST API default capture=true.
//
// Example:
//
//	charge, create := &omise.Charge{}, &CreateCharge{
//		Amount:      204842,
//		Currency:    "thb",
//		Description: "initial charge.",
//		Card:        token.ID,
//	}
//	if e := client.Do(charge, create); e != nil {
//		panic(e)
//	}
//
//	fmt.Println("created charge:", charge.ID)
//
type CreateCharge struct {
	Customer    string             `json:"customer,omitempty"`
	Card        string             `json:"card,omitempty"`
	Source      string             `json:"source,omitempty"`
	Amount      int64              `json:"amount"`
	Currency    string             `json:"currency"`
	Offsite     omise.OffsiteTypes `json:"offsite,omitempty"`
	Description string             `json:"description,omitempty"`
	DontCapture bool               `json:"-"` // inverse, since `capture` defaults to true
	ReturnURI   string             `json:"return_uri,omitempty"`

	Capture *bool `json:"capture,omitempty"`
}

type optionalFieldCreateCharge CreateCharge

func (req *CreateCharge) MarshalJSON() ([]byte, error) {
	req.SetOptionalField()
	return json.Marshal((*optionalFieldCreateCharge)(req))
}

func (req *CreateCharge) SetOptionalField() {
	if req.DontCapture {
		req.Capture = new(bool)
	}
}

func (req *CreateCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "POST",
		Path:        "/charges",
		ContentType: "application/json",
	}
}

// Example:
//
//	charge, update := &omise.Charge{}, &UpdateCharge{
//		ChargeID:    "chrg_456",
//		Description: "updated charge.",
//	}
//	if e := client.Do(charge, update); e != nil {
//		panic(e)
//	}
//
//	fmt.Printf("updated charge: %#v\n", charge)
//
type UpdateCharge struct {
	ChargeID    string `json:"-"`
	Description string `json:"description"`
}

func (req *UpdateCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "PATCH",
		Path:        "/charges/" + req.ChargeID,
		ContentType: "application/json",
	}
}

// Example:
//
//	charge := &omise.Charge{ID: "chrg_323"}
//	if e := client.Do(charge, &RetrieveCharge{charge.ID}); e != nil {
//		panic(e)
//	}
//
//	fmt.Printf("charge #chrg_323: %#v\n", charge)
//
type RetrieveCharge struct {
	ChargeID string `json:"-"`
}

func (req *RetrieveCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "GET",
		Path:        "/charges/" + req.ChargeID,
		ContentType: "application/json",
	}
}

// If you have created a charge and passed capture=false you'll have an authorized only
// charge that you can capture at a later time. You can hold it for as long as permitted
// by the issuing bank. This delay may vary between cards from 1 to 30 days.
//
// Example:
//
//	charge := &omise.Charge{ID: "chrg_1234"}
//	if e := client.Do(charge, &CaptureCharge{charge.ID}); e != nil {
//		panic(e)
//	}
//
//	fmt.Println("captured:", charge.Captured)
//
type CaptureCharge struct {
	ChargeID string `json:"-"`
}

func (req *CaptureCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "POST",
		Path:        "/charges/" + req.ChargeID + "/capture",
		ContentType: "application/json",
	}
}

// If you have created a charge and passed capture=false you'll have an authorized only
// charge that can be reversed, releasing held money, at a later time without incurring a
// refund fee.
type ReverseCharge struct {
	ChargeID string `json:"-"`
}

func (req *ReverseCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "POST",
		Path:        "/charges/" + req.ChargeID + "/reverse",
		ContentType: "application/json",
	}
}
<commit_msg>CreateCharge use custom struct in MarshalJSON<commit_after>package operations

import (
	"encoding/json"

	"github.com/omise/omise-go"
	"github.com/omise/omise-go/internal"
)

// Example:
//
//	charges, list := &omise.ChargeList{}, &ListCharges{
//		List{
//			Limit: 100,
//			From: time.Now().Add(-1 * time.Hour),
//		},
//	}
//	if e := client.Do(charges, list); e != nil {
//		panic(e)
//	}
//
//	fmt.Println("# of charges made in the last hour:", len(charges.Data))
//
type ListCharges struct {
	List
}

func (req *ListCharges) Describe() *internal.Description {
	return &internal.Description{
		Endpoint: internal.API,
		Method:   "GET",
		Path:     "/charges",
	}
}

// Note that because bool defaults to false in GO, we use DontCapture instead of Capture
// here so it matches with Omise's REST API default capture=true.
//
// Example:
//
//	charge, create := &omise.Charge{}, &CreateCharge{
//		Amount:      204842,
//		Currency:    "thb",
//		Description: "initial charge.",
//		Card:        token.ID,
//	}
//	if e := client.Do(charge, create); e != nil {
//		panic(e)
//	}
//
//	fmt.Println("created charge:", charge.ID)
//
type CreateCharge struct {
	Customer    string             `json:"customer,omitempty"`
	Card        string             `json:"card,omitempty"`
	Source      string             `json:"source,omitempty"`
	Amount      int64              `json:"amount"`
	Currency    string             `json:"currency"`
	Offsite     omise.OffsiteTypes `json:"offsite,omitempty"`
	Description string             `json:"description,omitempty"`
	DontCapture bool               `json:"-"` // inverse, since `capture` defaults to true
	ReturnURI   string             `json:"return_uri,omitempty"`
}

func (req *CreateCharge) MarshalJSON() ([]byte, error) {
	type Alias CreateCharge
	params := struct {
		*Alias
		Capture *bool `json:"capture,omitempty"`
	}{
		Alias: (*Alias)(req),
	}
	if params.DontCapture {
		params.Capture = new(bool)
	}
	return json.Marshal(params)
}

func (req *CreateCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "POST",
		Path:        "/charges",
		ContentType: "application/json",
	}
}

// Example:
//
//	charge, update := &omise.Charge{}, &UpdateCharge{
//		ChargeID:    "chrg_456",
//		Description: "updated charge.",
//	}
//	if e := client.Do(charge, update); e != nil {
//		panic(e)
//	}
//
//	fmt.Printf("updated charge: %#v\n", charge)
//
type UpdateCharge struct {
	ChargeID    string `json:"-"`
	Description string `json:"description"`
}

func (req *UpdateCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "PATCH",
		Path:        "/charges/" + req.ChargeID,
		ContentType: "application/json",
	}
}

// Example:
//
//	charge := &omise.Charge{ID: "chrg_323"}
//	if e := client.Do(charge, &RetrieveCharge{charge.ID}); e != nil {
//		panic(e)
//	}
//
//	fmt.Printf("charge #chrg_323: %#v\n", charge)
//
type RetrieveCharge struct {
	ChargeID string `json:"-"`
}

func (req *RetrieveCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "GET",
		Path:        "/charges/" + req.ChargeID,
		ContentType: "application/json",
	}
}

// If you have created a charge and passed capture=false you'll have an authorized only
// charge that you can capture at a later time. You can hold it for as long as permitted
// by the issuing bank. This delay may vary between cards from 1 to 30 days.
//
// Example:
//
//	charge := &omise.Charge{ID: "chrg_1234"}
//	if e := client.Do(charge, &CaptureCharge{charge.ID}); e != nil {
//		panic(e)
//	}
//
//	fmt.Println("captured:", charge.Captured)
//
type CaptureCharge struct {
	ChargeID string `json:"-"`
}

func (req *CaptureCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "POST",
		Path:        "/charges/" + req.ChargeID + "/capture",
		ContentType: "application/json",
	}
}

// If you have created a charge and passed capture=false you'll have an authorized only
// charge that can be reversed, releasing held money, at a later time without incurring a
// refund fee.
type ReverseCharge struct {
	ChargeID string `json:"-"`
}

func (req *ReverseCharge) Describe() *internal.Description {
	return &internal.Description{
		Endpoint:    internal.API,
		Method:      "POST",
		Path:        "/charges/" + req.ChargeID + "/reverse",
		ContentType: "application/json",
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"net"
	"time"

	"fmt"

	"github.com/snagles/docker-registry-manager/models/client"
	"github.com/snagles/docker-registry-manager/utilities"

	"code.cloudfoundry.org/bytefmt"
)

// Registries contains a map of all active registries identified by their name
var Registries map[string]*registry

func init() {
	Registries = map[string]*registry{}
}

func refresh() {
	for {
		utils.Log.Debug("Start refreshing")
		for name, registry := range Registries {
			utils.Log.Debug("Refreshing " + name)
			registry.Refresh()
			utils.Log.Debug("Refreshed " + name)
		}
		utils.Log.Debug("Refreshing is done")
		time.Sleep(45 * time.Second)
	}
}

func AddRegistry(scheme, host string, port int) error {
	r, err := NewRegistry(scheme, host, port)
	if err != nil {
		return err
	}

	err = client.HealthCheck(r.URI())
	if err != nil {
		return err
	}
	r.Refresh()
	go refresh()

	return nil
}

// Create a new registry from URI parts
func NewRegistry(scheme, host string, port int) (registry, error) {
	var err error
	r := registry{
		Scheme:  scheme,
		Host:    host,
		Port:    port,
		Version: 2,
	}

	// Lookup the ip for the passed host
	ip, err := net.LookupHost(host)
	if err != nil {
		utils.Log.Error(err)
		return registry{}, err
	}
	if len(ip) > 0 {
		r.IP = ip[0]
	}

	return r, nil
}

// Registry contains all identifying information for communicating with a registry
type registry struct {
	Host         string
	IP           string
	Scheme       string
	Port         int
	Version      int
	Repositories map[string]*Repository
	Status       int
}

// URI returns the full url path for communicating with this registry
func (r *registry) URI() string {
	return fmt.Sprintf("%s://%s:%d/v%d", r.Scheme, r.Host, r.Port, r.Version)
}

func (r *registry) TagCount() int {
	var count int
	for _, repo := range r.Repositories {
		count += len(repo.Tags)
	}
	return count
}

func (r *registry) DiskSize() string {
	registryLayerMap := make(map[string]bool, 0)
	var size int64

	for _, repo := range r.Repositories {
		for _, tag := range repo.Tags {
			for _, layer := range tag.Image.FsLayers {
				if _, ok := registryLayerMap[layer.BlobSum]; !ok {
					registryLayerMap[layer.BlobSum] = true
					size += layer.Size
				}
			}
		}
	}
	return bytefmt.ByteSize(uint64(size))
}

func (r registry) Refresh() {

	// Get the list of repositories on this registry
	repoList, _ := client.GetRepositories(r.URI())

	// Get the repository information
	r.Repositories = map[string]*Repository{}
	for _, repoName := range repoList {

		// Build a repository object
		repo := Repository{Name: repoName}

		tagList, _ := client.GetTags(r.URI(), repoName)
		repo.Tags = make(map[string]*Tag, 0)
		for _, tagName := range tagList {
			tag := Tag{Name: tagName}
			img, _ := client.GetImage(r.URI(), repoName, tagName)
			tag.Image = Image{img}

			// Add the tag to the repository
			repo.Tags[tagName] = &tag
		}
		r.Repositories[repoName] = &repo
	}
	Registries[r.Host] = &r
}

type Repository struct {
	Name string
	Tags map[string]*Tag
}

func (r *Repository) LastModified() time.Time {
	var lastModified time.Time
	for _, tag := range r.Tags {
		for _, history := range tag.Image.History {
			if history.V1Compatibility.Created.After(lastModified) {
				lastModified = history.V1Compatibility.Created
			}
		}
	}
	return lastModified
}

func (r *Repository) LastModifiedTimeAgo() string {
	lastModified := r.LastModified()
	return utils.TimeAgo(lastModified)
}

type Tag struct {
	Image

	ID   string
	Name string
}

func (t *Tag) LastModified() time.Time {
	var lastModified time.Time
	for _, history := range t.Image.History {
		if history.V1Compatibility.Created.After(lastModified) {
			lastModified = history.V1Compatibility.Created
		}
	}
	return lastModified
}

func (t *Tag) LastModifiedTimeAgo() string {
	lastModified := t.LastModified()
	return utils.TimeAgo(lastModified)
}

func (t *Tag) Size() string {
	var size int64
	for _, layer := range t.Image.FsLayers {
		size += layer.Size
	}
	return bytefmt.ByteSize(uint64(size))
}

func (t *Tag) LayerCount() int {
	return len(t.Image.FsLayers)
}

type Image struct {
	client.Image
}

func (i *Image) LastModified() time.Time {
	var lastModified time.Time
	for _, history := range i.History {
		if history.V1Compatibility.Created.After(lastModified) {
			lastModified = history.V1Compatibility.Created
		}
	}
	return lastModified
}

func (i *Image) LastModifiedTimeAgo() string {
	lastModified := i.LastModified()
	return utils.TimeAgo(lastModified)
}

func (i *Image) Size() string {
	var size int64
	for _, layer := range i.FsLayers {
		size += layer.Size
	}
	return bytefmt.ByteSize(uint64(size))
}

func (i *Image) LayerCount() int {
	return len(i.FsLayers)
}
<commit_msg>Cosmetic changes<commit_after>package registry

import (
	"net"
	"time"

	"fmt"

	"github.com/snagles/docker-registry-manager/models/client"
	"github.com/snagles/docker-registry-manager/utilities"

	"code.cloudfoundry.org/bytefmt"
)

// Registries contains a map of all active registries identified by their name
var Registries map[string]*registry

func init() {
	Registries = map[string]*registry{}
}

func refresh() {
	for {
		utils.Log.Debug("Start refreshing")
		for name, registry := range Registries {
			utils.Log.Debug("Refreshing " + name)
			registry.Refresh()
			utils.Log.Debug("Refreshed " + name)
		}
		utils.Log.Debug("Refreshing is done")
		time.Sleep(45 * time.Second)
	}
}

func AddRegistry(scheme, host string, port int) error {
	r, err := NewRegistry(scheme, host, port)
	if err != nil {
		return err
	}

	err = client.HealthCheck(r.URI())
	if err != nil {
		return err
	}
	r.Refresh()
	go refresh()

	return nil
}

// Create a new registry from URI parts
func NewRegistry(scheme, host string, port int) (registry, error) {
	var err error
	r := registry{
		Scheme:  scheme,
		Host:    host,
		Port:    port,
		Version: 2,
	}

	// Lookup the ip for the passed host
	ip, err := net.LookupHost(host)
	if err != nil {
		utils.Log.Error(err)
		return registry{}, err
	}
	if len(ip) > 0 {
		r.IP = ip[0]
	}

	return r, nil
}

// Registry contains all identifying information for communicating with a registry
type registry struct {
	Host         string
	IP           string
	Scheme       string
	Port         int
	Version      int
	Repositories map[string]*Repository
	Status       int
}

// URI returns the full url path for communicating with this registry
func (r *registry) URI() string {
	return fmt.Sprintf("%s://%s:%d/v%d", r.Scheme, r.Host, r.Port, r.Version)
}

func (r *registry) TagCount() int {
	var count int
	for _, repo := range r.Repositories {
		count += len(repo.Tags)
	}
	return count
}

func (r *registry) DiskSize() string {
	registryLayerMap := map[string]bool{}
	var size int64

	for _, repo := range r.Repositories {
		for _, tag := range repo.Tags {
			for _, layer := range tag.Image.FsLayers {
				if _, ok := registryLayerMap[layer.BlobSum]; !ok {
					registryLayerMap[layer.BlobSum] = true
					size += layer.Size
				}
			}
		}
	}
	return bytefmt.ByteSize(uint64(size))
}

func (r registry) Refresh() {

	// Get the list of repositories on this registry
	repoList, _ := client.GetRepositories(r.URI())

	// Get the repository information
	r.Repositories = map[string]*Repository{}
	for _, repoName := range repoList {

		// Build a repository object
		repo := Repository{Name: repoName}

		tagList, _ := client.GetTags(r.URI(), repoName)
		repo.Tags = map[string]*Tag{}
		for _, tagName := range tagList {
			tag := Tag{Name: tagName}
			img, _ := client.GetImage(r.URI(), repoName, tagName)
			tag.Image = Image{img}

			// Add the tag to the repository
			repo.Tags[tagName] = &tag
		}
		r.Repositories[repoName] = &repo
	}
	Registries[r.Host] = &r
}

type Repository struct {
	Name string
	Tags map[string]*Tag
}

func (r *Repository) LastModified() time.Time {
	var lastModified time.Time
	for _, tag := range r.Tags {
		for _, history := range tag.Image.History {
			if history.V1Compatibility.Created.After(lastModified) {
				lastModified = history.V1Compatibility.Created
			}
		}
	}
	return lastModified
}

func (r *Repository) LastModifiedTimeAgo() string {
	lastModified := r.LastModified()
	return utils.TimeAgo(lastModified)
}

type Tag struct {
	Image

	ID   string
	Name string
}

func (t *Tag) LastModified() time.Time {
	var lastModified time.Time
	for _, history := range t.Image.History {
		if history.V1Compatibility.Created.After(lastModified) {
			lastModified = history.V1Compatibility.Created
		}
	}
	return lastModified
}

func (t *Tag) LastModifiedTimeAgo() string {
	lastModified := t.LastModified()
	return utils.TimeAgo(lastModified)
}

func (t *Tag) Size() string {
	var size int64
	for _, layer := range t.Image.FsLayers {
		size += layer.Size
	}
	return bytefmt.ByteSize(uint64(size))
}

func (t *Tag) LayerCount() int {
	return len(t.Image.FsLayers)
}

type Image struct {
	client.Image
}

func (i *Image) LastModified() time.Time {
	var lastModified time.Time
	for _, history := range i.History {
		if history.V1Compatibility.Created.After(lastModified) {
			lastModified = history.V1Compatibility.Created
		}
	}
	return lastModified
}

func (i *Image) LastModifiedTimeAgo() string {
	lastModified := i.LastModified()
	return utils.TimeAgo(lastModified)
}

func (i *Image) Size() string {
	var size int64
	for _, layer := range i.FsLayers {
		size += layer.Size
	}
	return bytefmt.ByteSize(uint64(size))
}

func (i *Image) LayerCount() int {
	return len(i.FsLayers)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2020 The Knative Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package config

import (
	"fmt"

	"github.com/kelseyhightower/envconfig"
)

type Env struct {
	DataPlaneConfigMapNamespace string `required:"true" split_words:"true"`
	DataPlaneConfigMapName      string `required:"true" split_words:"true"`
	GeneralConfigMapName        string `required:"true" split_words:"true"`
	IngressName                 string `required:"true" split_words:"true"`
	IngressPodPort              string `required:"false" split_words:"true"`
	SystemNamespace             string `required:"true" split_words:"true"`
	DataPlaneConfigFormat       string `required:"true" split_words:"true"`
	DefaultBackoffDelayMs       uint64 `required:"false" split_words:"true"`
}

// ValidationOption represents a function to validate the Env configurations.
type ValidationOption func(env Env) error

func GetEnvConfig(prefix string, validations ...ValidationOption) (*Env, error) {
	env := &Env{}

	if err := envconfig.Process(prefix, env); err != nil {
		return nil, fmt.Errorf("failed to process env config: %w", err)
	}

	for _, v := range validations {
		if err := v(*env); err != nil {
			return nil, fmt.Errorf("env config validation failed: %w", err)
		}
	}

	return env, nil
}

func (c *Env) DataPlaneConfigMapAsString() string {
	return fmt.Sprintf("%s/%s", c.DataPlaneConfigMapNamespace, c.DataPlaneConfigMapName)
}
<commit_msg>Document env struct (#2332)<commit_after>/*
 * Copyright 2020 The Knative Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package config

import (
	"fmt"

	"github.com/kelseyhightower/envconfig"
)

type Env struct {
	// DataPlaneConfigMapNamespace is the namespace of the configmap that holds the contract between the control plane
	// and the data plane. Might be ignored in some cases such as the configmap is expected to be in the resource
	// namespace
	DataPlaneConfigMapNamespace string `required:"true" split_words:"true"`

	// DataPlaneConfigMapName is the name of the configmap that holds the contract between the control plane
	// and the data plane.
	DataPlaneConfigMapName string `required:"true" split_words:"true"` // example: kafka-broker-triggers

	// GeneralConfigMapName is the name of the configmap that holds configuration that affects the control plane
	// and the data plane. For example, Kafka bootstrap server information could be in here for broker configuration.
	GeneralConfigMapName string `required:"true" split_words:"true"` // example: kafka-broker-config

	IngressName           string `required:"true" split_words:"true"` // example: kafka-broker-ingress
	IngressPodPort        string `required:"false" split_words:"true"`
	SystemNamespace       string `required:"true" split_words:"true"`
	DataPlaneConfigFormat string `required:"true" split_words:"true"`
	DefaultBackoffDelayMs uint64 `required:"false" split_words:"true"`
}

// ValidationOption represents a function to validate the Env configurations.
type ValidationOption func(env Env) error

func GetEnvConfig(prefix string, validations ...ValidationOption) (*Env, error) {
	env := &Env{}

	if err := envconfig.Process(prefix, env); err != nil {
		return nil, fmt.Errorf("failed to process env config: %w", err)
	}

	for _, v := range validations {
		if err := v(*env); err != nil {
			return nil, fmt.Errorf("env config validation failed: %w", err)
		}
	}

	return env, nil
}

func (c *Env) DataPlaneConfigMapAsString() string {
	return fmt.Sprintf("%s/%s", c.DataPlaneConfigMapNamespace, c.DataPlaneConfigMapName)
}
<|endoftext|>


<commit_before>/**
 * Copyright 2015 @ z3q.net.
 * name : download.go
 * author : jarryliu
 * date : 2015-12-31 12:23
 * description :
 * history :
 */
package domain

import (
	"bufio"
	"fmt"
	"github.com/jsix/gof"
	"go2o/core/variable"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"
)

func getFileName(disposition string, guessExt string) string {
	ext := guessExt
	if len(disposition) > 0 && strings.Index(disposition, "filename=") != -1 {
		ext = disposition[strings.Index(disposition, ".")+1:]
	}
	return fmt.Sprintf("%d_%d.%s", time.Now().Unix(),
		100+rand.Intn(899), ext)
}

// 下载远程资源并返回本地地址
func DownloadToLocal(url string, savePath string, ext string) string {
	var req *http.Request
	var err error
	req, err = http.NewRequest("GET", url, nil)
	if err == nil {
		req.Header.Set("User_Agent", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5")
		if rsp, err := http.DefaultClient.Do(req); err == nil {

			fileName := getFileName(rsp.Header.Get("Content-Disposition"), ext)
			if _, err := os.Stat(savePath); os.IsNotExist(err) {
				os.MkdirAll(savePath, os.ModePerm)
			}
			var filePath string = savePath + fileName

			src := rsp.Body
			defer src.Close()

			fi, err := os.OpenFile(filePath,
				os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
				os.ModePerm)

			if err == nil {
				defer fi.Close()
				buf := bufio.NewWriter(fi)
				bufSize := 100
				buffer := make([]byte, bufSize)
				var n int
				var l int
				for {
					if n, err = src.Read(buffer); err == io.EOF {
						break
					}

					if n != bufSize {
						buf.Write(buffer[:n])
					} else {
						buf.Write(buffer)
					}

					l += n
				}
				buf.Flush()

				return filePath[2:] // 去掉"./"
			}
		}
	}

	if err != nil {
		log.Println("[ Download]- ", url, "\n  ", err.Error())
	}

	return url
}
<commit_msg>download<commit_after>/**
 * Copyright 2015 @ z3q.net.
 * name : download.go
 * author : jarryliu
 * date : 2015-12-31 12:23
 * description :
 * history :
 */
package domain

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"
)

func getFileName(disposition string, guessExt string) string {
	ext := guessExt
	if len(disposition) > 0 && strings.Index(disposition, "filename=") != -1 {
		ext = disposition[strings.Index(disposition, ".")+1:]
	}
	return fmt.Sprintf("%d_%d.%s", time.Now().Unix(),
		100+rand.Intn(899), ext)
}

// 下载远程资源并返回本地地址
func DownloadToLocal(url string, savePath string, ext string) string {
	var req *http.Request
	var err error
	req, err = http.NewRequest("GET", url, nil)
	if err == nil {
		req.Header.Set("User_Agent", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5")
		if rsp, err := http.DefaultClient.Do(req); err == nil {

			fileName := getFileName(rsp.Header.Get("Content-Disposition"), ext)
			if _, err := os.Stat(savePath); os.IsNotExist(err) {
				os.MkdirAll(savePath, os.ModePerm)
			}
			var filePath string = savePath + fileName

			src := rsp.Body
			defer src.Close()

			fi, err := os.OpenFile(filePath,
				os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
				os.ModePerm)

			if err == nil {
				defer fi.Close()
				buf := bufio.NewWriter(fi)
				bufSize := 100
				buffer := make([]byte, bufSize)
				var n int
				var l int
				for {
					if n, err = src.Read(buffer); err == io.EOF {
						break
					}

					if n != bufSize {
						buf.Write(buffer[:n])
					} else {
						buf.Write(buffer)
					}

					l += n
				}
				buf.Flush()

				return filePath[2:] // 去掉"./"
			}
		}
	}

	if err != nil {
		log.Println("[ Download]- ", url, "\n  ", err.Error())
	}

	return url
}
<|endoftext|>


<commit_before>package shard

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/unigraph/rdb"
)

var ShardNameFn = func(i uint) string { return fmt.Sprintf("%03d", i) }

type Shard struct {
	dbs []*rdb.DB
}

func Open(opts *rdb.Options, name string, shardsNum uint) (*Shard, error) {
	if err := checkValid(name, shardsNum); err != nil {
		return nil, err
	}
	s := &Shard{}
	for i := uint(0); i < shardsNum; i++ {
		sName := filepath.Join(name, ShardNameFn(i))
		db, err := rdb.OpenDb(opts, sName)
		if err != nil {
			s.Close()
			return nil, err
		}
		s.dbs = append(s.dbs, db)
	}
	return s, nil
}

func OpenForReadOnly(opts *rdb.Options, name string, shardsNum uint, errorIfLogFileExist bool) (*Shard, error) {
	if err := checkValid(name, shardsNum); err != nil {
		return nil, err
	}
	s := &Shard{}
	for i := uint(0); i < shardsNum; i++ {
		sName := filepath.Join(name, ShardNameFn(i))
		db, err := rdb.OpenDbForReadOnly(opts, sName, errorIfLogFileExist)
		if err != nil {
			s.Close()
			return nil, err
		}
		s.dbs = append(s.dbs, db)
	}
	return s, nil
}

type errors []error

func (e errors) Error() string {
	res := ""
	for _, err := range e {
		res += err.Error()
	}
	return res
}

func (s *Shard) Flush(opts *rdb.FlushOptions) error {
	wg := sync.WaitGroup{}
	wg.Add(len(s.dbs))
	err := errors(nil)
	l := sync.RWMutex{}
	for _, db := range s.dbs {
		go func(db *rdb.DB) {
			if e := db.Flush(opts); e != nil {
				l.Lock()
				err = append(err, e)
				l.Unlock()
			}
			wg.Done()
		}(db)
	}
	wg.Wait()
	if len(err) == 0 {
		return nil
	}
	return err
}

func (s *Shard) CompactRange(r rdb.Range) {
	wg := sync.WaitGroup{}
	wg.Add(len(s.dbs))
	for _, db := range s.dbs {
		go func(db *rdb.DB) {
			db.CompactRange(r)
			wg.Done()
		}(db)
	}
	wg.Wait()
}

func (s *Shard) DBs() []*rdb.DB {
	return append([]*rdb.DB(nil), s.dbs...)
}

func (s *Shard) Close() {
	wg := sync.WaitGroup{}
	wg.Add(len(s.dbs))
	for _, db := range s.dbs {
		go func(db *rdb.DB) {
			db.Close()
			wg.Done()
		}(db)
	}
	wg.Wait()
}

func GetShardNum(name string) uint {
	if files, err := ioutil.ReadDir(name); os.IsNotExist(err) {
		return 0
	} else {
		shards := map[string]bool{}
		for _, file := range files {
			if matched, _ := regexp.MatchString(`\d{3}`, file.Name()); matched {
				shards[file.Name()] = true
			}
		}
		return uint(len(shards))
	}
}

func checkValid(name string, shardsNum uint) error {
	if shardsNum == 0 || shardsNum > 999 {
		return fmt.Errorf("Number of shards has to be bigger than 0 and lower than 1000")
	}
	files, err := ioutil.ReadDir(name)
	if os.IsNotExist(err) { // does not exists, let's create empty
		return os.Mkdir(name, 0700)
	} else if err != nil { // some other error related to ReadDir
		return err
	} else { // exists, let's check the content
		shards := map[string]bool{}
		for _, file := range files {
			if matched, _ := regexp.MatchString(`\d{3}`, file.Name()); matched {
				shards[file.Name()] = true
			}
		}
		if len(shards) != 0 {
			if uint(len(shards)) != shardsNum {
				return fmt.Errorf("Wrong number of shards provided (%v)", len(shards))
			}
			for i := uint(0); i < shardsNum; i++ {
				sName := ShardNameFn(i)
				if !shards[sName] {
					return fmt.Errorf("Wrong number of shards provided (%v)", len(shards))
				}
			}
		}
	}
	return nil
}
<commit_msg>update<commit_after>package shard

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/unigraph/rdb"
)

var ShardNameFn = func(i uint) string { return fmt.Sprintf("%03d", i) }

type Shard struct {
	dbs []*rdb.DB
}

func Open(opts *rdb.Options, name string, shardsNum uint) (*Shard, error) {
	if err := checkValid(name, shardsNum); err != nil {
		return nil, err
	}
	s := &Shard{}
	for i := uint(0); i < shardsNum; i++ {
		sName := filepath.Join(name, ShardNameFn(i))
		db, err := rdb.OpenDb(opts, sName)
		if err != nil {
			s.Close()
			return nil, err
		}
		s.dbs = append(s.dbs, db)
	}
	return s, nil
}

func OpenForReadOnly(opts *rdb.Options, name string, shardsNum uint, errorIfLogFileExist bool) (*Shard, error) {
	if err := checkValid(name, shardsNum); err != nil {
		return nil, err
	}
	s := &Shard{}
	for i := uint(0); i < shardsNum; i++ {
		sName := filepath.Join(name, ShardNameFn(i))
		db, err := rdb.OpenDbForReadOnly(opts, sName, errorIfLogFileExist)
		if err != nil {
			s.Close()
			return nil, err
		}
		s.dbs = append(s.dbs, db)
	}
	return s, nil
}

type errors []error

func (e errors) Error() string {
	res := ""
	for _, err := range e {
		res += err.Error()
	}
	return res
}

func (s *Shard) Flush(opts *rdb.FlushOptions) error {
	wg := sync.WaitGroup{}
	wg.Add(len(s.dbs))
	err := errors(nil)
	l := sync.RWMutex{}
	for _, db := range s.dbs {
		go func(db *rdb.DB) {
			if e := db.Flush(opts); e != nil {
				l.Lock()
				err = append(err, e)
				l.Unlock()
			}
			wg.Done()
		}(db)
	}
	wg.Wait()
	if len(err) == 0 {
		return nil
	}
	return err
}

func (s *Shard) CompactRange(r rdb.Range) {
	wg := sync.WaitGroup{}
	wg.Add(len(s.dbs))
	for _, db := range s.dbs {
		go func(db *rdb.DB) {
			db.CompactRange(r)
			wg.Done()
		}(db)
	}
	wg.Wait()
}

func (s *Shard) DBs() []*rdb.DB {
	return append([]*rdb.DB(nil), s.dbs...)
}

func (s *Shard) Close() {
	wg := sync.WaitGroup{}
	wg.Add(len(s.dbs))
	for _, db := range s.dbs {
		go func(db *rdb.DB) {
			db.Close()
			wg.Done()
		}(db)
	}
	wg.Wait()
}

func GetShardNum(name string) uint {
	if files, err := ioutil.ReadDir(name); os.IsNotExist(err) {
		return 0
	} else {
		shards := map[string]bool{}
		for _, file := range files {
			shards[file.Name()] = true
		}
		i := 0
		for shards[ShardNameFn(uint(i))] {
			i++
		}
		if len(shards) != i {
			return 0
		}
		return uint(i)
	}
}

func checkValid(name string, shardsNum uint) error {
	if shardsNum == 0 || shardsNum > 999 {
		return fmt.Errorf("Number of shards has to be bigger than 0 and lower than 1000")
	}
	files, err := ioutil.ReadDir(name)
	if os.IsNotExist(err) { // does not exists, let's create empty
		return os.Mkdir(name, 0700)
	} else if err != nil { // some other error related to ReadDir
		return err
	} else { // exists, let's check the content
		shards := map[string]bool{}
		for _, file := range files {
			if matched, _ := regexp.MatchString(`\d{3}`, file.Name()); matched {
				shards[file.Name()] = true
			}
		}
		if len(shards) != 0 {
			if uint(len(shards)) != shardsNum {
				return fmt.Errorf("Wrong number of shards provided (%v)", len(shards))
			}
			for i := uint(0); i < shardsNum; i++ {
				sName := ShardNameFn(i)
				if !shards[sName] {
					return fmt.Errorf("Wrong number of shards provided (%v)", len(shards))
				}
			}
		}
	}
	return nil
}
<|endoftext|>