<commit_before>package dockerclient

import "time"

type ContainerConfig struct {
	Hostname        string
	Domainname      string
	User            string
	Memory          int64
	MemorySwap      int64
	CpuShares       int64
	Cpuset          string
	AttachStdin     bool
	AttachStdout    bool
	AttachStderr    bool
	PortSpecs       []string
	ExposedPorts    map[string]struct{}
	Tty             bool
	OpenStdin       bool
	StdinOnce       bool
	Env             []string
	Cmd             []string
	Image           string
	Labels          map[string]string
	Volumes         map[string]struct{}
	WorkingDir      string
	Entrypoint      []string
	NetworkDisabled bool
	OnBuild         []string

	// This is used only by the create command
	HostConfig HostConfig
}

type HostConfig struct {
	Binds           []string
	ContainerIDFile string
	LxcConf         []map[string]string
	Privileged      bool
	PortBindings    map[string][]PortBinding
	Links           []string
	PublishAllPorts bool
	Dns             []string
	DnsSearch       []string
	VolumesFrom     []string
	SecurityOpt     []string
	NetworkMode     string
	RestartPolicy   RestartPolicy
}

type ExecConfig struct {
	AttachStdin  bool
	AttachStdout bool
	AttachStderr bool
	Tty          bool
	Cmd          []string
	Container    string
	Detach       bool
}

type LogOptions struct {
	Follow     bool
	Stdout     bool
	Stderr     bool
	Timestamps bool
	Tail       int64
}

type RestartPolicy struct {
	Name              string
	MaximumRetryCount int64
}

type PortBinding struct {
	HostIp   string
	HostPort string
}

type ContainerInfo struct {
	Id      string
	Created string
	Path    string
	Name    string
	Args    []string
	ExecIDs []string
	Config  *ContainerConfig
	State   struct {
		Running    bool
		Paused     bool
		Restarting bool
		Pid        int
		ExitCode   int
		StartedAt  time.Time
		FinishedAt time.Time
		Ghost      bool
	}
	Image           string
	NetworkSettings struct {
		IPAddress   string `json:"IpAddress"`
		IPPrefixLen int    `json:"IpPrefixLen"`
		Gateway     string
		Bridge      string
		Ports       map[string][]PortBinding
	}
	SysInitPath    string
	ResolvConfPath string
	Volumes        map[string]string
	HostConfig     *HostConfig
}

type ContainerChanges struct {
	Path string
	Kind int
}

type Port struct {
	IP          string
	PrivatePort int
	PublicPort  int
	Type        string
}

type Container struct {
	Id         string
	Names      []string
	Image      string
	Command    string
	Created    int64
	Status     string
	Ports      []Port
	SizeRw     int64
	SizeRootFs int64
	Labels     map[string]string
}

type Event struct {
	Id     string
	Status string
	From   string
	Time   int64
}

type Version struct {
	Version   string
	GitCommit string
	GoVersion string
}

type RespContainersCreate struct {
	Id       string
	Warnings []string
}

type Image struct {
	Created     int64
	Id          string
	ParentId    string
	RepoTags    []string
	Size        int64
	VirtualSize int64
}

type Info struct {
	ID              string
	Containers      int64
	Driver          string
	DriverStatus    [][]string
	ExecutionDriver string
	Images          int64
	KernelVersion   string
	OperatingSystem string
	NCPU            int64
	MemTotal        int64
	Name            string
	Labels          []string
}

type ImageDelete struct {
	Deleted  string
	Untagged string
}

// The following are types for the API stats endpoint
type ThrottlingData struct {
	// Number of periods with throttling active
	Periods uint64 `json:"periods"`
	// Number of periods when the container hit its throttling limit.
	ThrottledPeriods uint64 `json:"throttled_periods"`
	// Aggregate time the container was throttled for in nanoseconds.
	ThrottledTime uint64 `json:"throttled_time"`
}

type CpuUsage struct {
	// Total CPU time consumed.
	// Units: nanoseconds.
	TotalUsage uint64 `json:"total_usage"`
	// Total CPU time consumed per core.
	// Units: nanoseconds.
	PercpuUsage []uint64 `json:"percpu_usage"`
	// Time spent by tasks of the cgroup in kernel mode.
	// Units: nanoseconds.
	UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
	// Time spent by tasks of the cgroup in user mode.
	// Units: nanoseconds.
	UsageInUsermode uint64 `json:"usage_in_usermode"`
}

type CpuStats struct {
	CpuUsage       CpuUsage       `json:"cpu_usage"`
	SystemUsage    uint64         `json:"system_cpu_usage"`
	ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
}

type NetworkStats struct {
	RxBytes   uint64 `json:"rx_bytes"`
	RxPackets uint64 `json:"rx_packets"`
	RxErrors  uint64 `json:"rx_errors"`
	RxDropped uint64 `json:"rx_dropped"`
	TxBytes   uint64 `json:"tx_bytes"`
	TxPackets uint64 `json:"tx_packets"`
	TxErrors  uint64 `json:"tx_errors"`
	TxDropped uint64 `json:"tx_dropped"`
}

type MemoryStats struct {
	Usage    uint64            `json:"usage"`
	MaxUsage uint64            `json:"max_usage"`
	Stats    map[string]uint64 `json:"stats"`
	Failcnt  uint64            `json:"failcnt"`
	Limit    uint64            `json:"limit"`
}

type BlkioStatEntry struct {
	Major uint64 `json:"major"`
	Minor uint64 `json:"minor"`
	Op    string `json:"op"`
	Value uint64 `json:"value"`
}

type BlkioStats struct {
	// number of bytes tranferred to and from the block device
	IoServiceBytesRecursive []BlkioStatEntry `json:"io_service_bytes_recursive"`
	IoServicedRecursive     []BlkioStatEntry `json:"io_serviced_recursive"`
	IoQueuedRecursive       []BlkioStatEntry `json:"io_queue_recursive"`
	IoServiceTimeRecursive  []BlkioStatEntry `json:"io_service_time_recursive"`
	IoWaitTimeRecursive     []BlkioStatEntry `json:"io_wait_time_recursive"`
	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive"`
	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive"`
	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive"`
}

type Stats struct {
	Read         time.Time    `json:"read"`
	NetworkStats NetworkStats `json:"network,omitempty"`
	CpuStats     CpuStats     `json:"cpu_stats,omitempty"`
	MemoryStats  MemoryStats  `json:"memory_stats,omitempty"`
	BlkioStats   BlkioStats   `json:"blkio_stats,omitempty"`
}
<commit_msg>Add LogConfig and Ulimits(new in docker 1.6).<commit_after>package dockerclient

import "time"

type ContainerConfig struct {
	Hostname        string
	Domainname      string
	User            string
	Memory          int64
	MemorySwap      int64
	CpuShares       int64
	Cpuset          string
	AttachStdin     bool
	AttachStdout    bool
	AttachStderr    bool
	PortSpecs       []string
	ExposedPorts    map[string]struct{}
	Tty             bool
	OpenStdin       bool
	StdinOnce       bool
	Env             []string
	Cmd             []string
	Image           string
	Labels          map[string]string
	Volumes         map[string]struct{}
	WorkingDir      string
	Entrypoint      []string
	NetworkDisabled bool
	OnBuild         []string

	// This is used only by the create command
	HostConfig HostConfig
}

type HostConfig struct {
	Binds           []string
	ContainerIDFile string
	LxcConf         []map[string]string
	Privileged      bool
	PortBindings    map[string][]PortBinding
	Links           []string
	PublishAllPorts bool
	Dns             []string
	DnsSearch       []string
	VolumesFrom     []string
	SecurityOpt     []string
	NetworkMode     string
	RestartPolicy   RestartPolicy
	Ulimits         []Ulimit
	LogConfig       LogConfig
}

type ExecConfig struct {
	AttachStdin  bool
	AttachStdout bool
	AttachStderr bool
	Tty          bool
	Cmd          []string
	Container    string
	Detach       bool
}

type LogOptions struct {
	Follow     bool
	Stdout     bool
	Stderr     bool
	Timestamps bool
	Tail       int64
}

type RestartPolicy struct {
	Name              string
	MaximumRetryCount int64
}

type PortBinding struct {
	HostIp   string
	HostPort string
}

type ContainerInfo struct {
	Id      string
	Created string
	Path    string
	Name    string
	Args    []string
	ExecIDs []string
	Config  *ContainerConfig
	State   struct {
		Running    bool
		Paused     bool
		Restarting bool
		Pid        int
		ExitCode   int
		StartedAt  time.Time
		FinishedAt time.Time
		Ghost      bool
	}
	Image           string
	NetworkSettings struct {
		IPAddress   string `json:"IpAddress"`
		IPPrefixLen int    `json:"IpPrefixLen"`
		Gateway     string
		Bridge      string
		Ports       map[string][]PortBinding
	}
	SysInitPath    string
	ResolvConfPath string
	Volumes        map[string]string
	HostConfig     *HostConfig
}

type ContainerChanges struct {
	Path string
	Kind int
}

type Port struct {
	IP          string
	PrivatePort int
	PublicPort  int
	Type        string
}

type Container struct {
	Id         string
	Names      []string
	Image      string
	Command    string
	Created    int64
	Status     string
	Ports      []Port
	SizeRw     int64
	SizeRootFs int64
	Labels     map[string]string
}

type Event struct {
	Id     string
	Status string
	From   string
	Time   int64
}

type Version struct {
	Version   string
	GitCommit string
	GoVersion string
}

type RespContainersCreate struct {
	Id       string
	Warnings []string
}

type Image struct {
	Created     int64
	Id          string
	ParentId    string
	RepoTags    []string
	Size        int64
	VirtualSize int64
}

type Info struct {
	ID              string
	Containers      int64
	Driver          string
	DriverStatus    [][]string
	ExecutionDriver string
	Images          int64
	KernelVersion   string
	OperatingSystem string
	NCPU            int64
	MemTotal        int64
	Name            string
	Labels          []string
}

type ImageDelete struct {
	Deleted  string
	Untagged string
}

// The following are types for the API stats endpoint
type ThrottlingData struct {
	// Number of periods with throttling active
	Periods uint64 `json:"periods"`
	// Number of periods when the container hit its throttling limit.
	ThrottledPeriods uint64 `json:"throttled_periods"`
	// Aggregate time the container was throttled for in nanoseconds.
	ThrottledTime uint64 `json:"throttled_time"`
}

type CpuUsage struct {
	// Total CPU time consumed.
	// Units: nanoseconds.
	TotalUsage uint64 `json:"total_usage"`
	// Total CPU time consumed per core.
	// Units: nanoseconds.
	PercpuUsage []uint64 `json:"percpu_usage"`
	// Time spent by tasks of the cgroup in kernel mode.
	// Units: nanoseconds.
	UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
	// Time spent by tasks of the cgroup in user mode.
	// Units: nanoseconds.
	UsageInUsermode uint64 `json:"usage_in_usermode"`
}

type CpuStats struct {
	CpuUsage       CpuUsage       `json:"cpu_usage"`
	SystemUsage    uint64         `json:"system_cpu_usage"`
	ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
}

type NetworkStats struct {
	RxBytes   uint64 `json:"rx_bytes"`
	RxPackets uint64 `json:"rx_packets"`
	RxErrors  uint64 `json:"rx_errors"`
	RxDropped uint64 `json:"rx_dropped"`
	TxBytes   uint64 `json:"tx_bytes"`
	TxPackets uint64 `json:"tx_packets"`
	TxErrors  uint64 `json:"tx_errors"`
	TxDropped uint64 `json:"tx_dropped"`
}

type MemoryStats struct {
	Usage    uint64            `json:"usage"`
	MaxUsage uint64            `json:"max_usage"`
	Stats    map[string]uint64 `json:"stats"`
	Failcnt  uint64            `json:"failcnt"`
	Limit    uint64            `json:"limit"`
}

type BlkioStatEntry struct {
	Major uint64 `json:"major"`
	Minor uint64 `json:"minor"`
	Op    string `json:"op"`
	Value uint64 `json:"value"`
}

type BlkioStats struct {
	// number of bytes tranferred to and from the block device
	IoServiceBytesRecursive []BlkioStatEntry `json:"io_service_bytes_recursive"`
	IoServicedRecursive     []BlkioStatEntry `json:"io_serviced_recursive"`
	IoQueuedRecursive       []BlkioStatEntry `json:"io_queue_recursive"`
	IoServiceTimeRecursive  []BlkioStatEntry `json:"io_service_time_recursive"`
	IoWaitTimeRecursive     []BlkioStatEntry `json:"io_wait_time_recursive"`
	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive"`
	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive"`
	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive"`
}

type Stats struct {
	Read         time.Time    `json:"read"`
	NetworkStats NetworkStats `json:"network,omitempty"`
	CpuStats     CpuStats     `json:"cpu_stats,omitempty"`
	MemoryStats  MemoryStats  `json:"memory_stats,omitempty"`
	BlkioStats   BlkioStats   `json:"blkio_stats,omitempty"`
}

type Ulimit struct {
	Name string `json:"name"`
	Soft uint64 `json:"soft"`
	Hard uint64 `json:"hard"`
}

type LogConfig struct {
	Type   string            `json:"type"`
	Config map[string]string `json:"config"`
}
<|endoftext|>


<commit_before>package sched

import (
    "io"
    "log"
    "encoding/json"
)

type Client struct {
    sched *Sched
    conn Conn
}

func NewClient(sched *Sched, conn Conn) (client *Client) {
    client = new(Client)
    client.conn = conn
    client.sched = sched
    return
}

func (client *Client) Handle() {
    var payload []byte
    var err error
    var conn = client.conn
    defer conn.Close()
    for {
        payload, err = conn.Receive()
        if err != nil {
            if err != io.EOF {
                log.Printf("WorkerError: %s\n", err.Error())
            }
            return
        }

        switch payload[0] {
        case SUBMIT_JOB:
            err = client.HandleSubmitJob(payload[2:])
            break
        case STATUS:
            err = client.HandleStatus()
            break
        case PING:
            err = conn.Send(PackCmd(PONG))
            break
        case DROP_FUNC:
            err = client.HandleDropFunc(payload[2:])
            break
        default:
            err = conn.Send(PackCmd(UNKNOWN))
            break
        }
        if err != nil {
            if err != io.EOF {
                log.Printf("WorkerError: %s\n", err.Error())
            }
            return
        }
    }
}

func (client *Client) HandleSubmitJob(payload []byte) (err error) {
    var job Job
    var e error
    var conn = client.conn
    var sched = client.sched
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    e = json.Unmarshal(payload, &job)
    if e != nil {
        err = conn.Send([]byte(e.Error()))
        return
    }
    is_new := true
    changed := false
    job.Status = JOB_STATUS_READY
    oldJob, e := sched.driver.GetOne(job.Func, job.Name)
    if e == nil && oldJob.Id > 0 {
        job.Id = oldJob.Id
        if oldJob.Status == JOB_STATUS_PROC {
            sched.DecrStatProc(oldJob)
            changed = true
        }
        is_new = false
    }
    e = sched.driver.Save(&job)
    if e != nil {
        err = conn.Send([]byte(e.Error()))
        return
    }

    if is_new {
        sched.IncrStatJob(job)
    }
    if is_new || changed {
        sched.pushJobPQ(job)
    }
    sched.Notify()
    err = conn.Send([]byte("ok"))
    return
}

func (client *Client) HandleStatus() (err error) {
    var conn = client.conn
    var sched = client.sched
    data, _ := json.Marshal(sched.Funcs)
    err = conn.Send(data)
    return
}

func (client *Client) HandleDropFunc(payload []byte) (err error) {
    Func := string(payload)
    stat, ok := client.sched.Funcs[Func]
    sched := client.sched
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if ok && stat.Worker == 0 {
        iter := sched.driver.NewIterator(payload)
        deleteJob := make([]int64, 0)
        for {
            if !iter.Next() {
                break
            }
            job := iter.Value()
            deleteJob = append(deleteJob, job.Id)
        }
        iter.Close()
        for _, jobId := range deleteJob {
            sched.driver.Delete(jobId)
        }
        delete(client.sched.Funcs, Func)
        delete(client.sched.jobPQ, Func)
    }
    err = client.conn.Send([]byte("ok"))
    return
}
<commit_msg>Add client panic recover<commit_after>package sched

import (
    "io"
    "log"
    "encoding/json"
)

type Client struct {
    sched *Sched
    conn Conn
}

func NewClient(sched *Sched, conn Conn) (client *Client) {
    client = new(Client)
    client.conn = conn
    client.sched = sched
    return
}

func (client *Client) Handle() {
    var payload []byte
    var err error
    var conn = client.conn
    defer func() {
        if x := recover(); x != nil {
            log.Printf("[Client] painc: %v\n", x)
        }
    } ()
    defer conn.Close()
    for {
        payload, err = conn.Receive()
        if err != nil {
            if err != io.EOF {
                log.Printf("ClientError: %s\n", err.Error())
            }
            return
        }

        switch payload[0] {
        case SUBMIT_JOB:
            err = client.HandleSubmitJob(payload[2:])
            break
        case STATUS:
            err = client.HandleStatus()
            break
        case PING:
            err = conn.Send(PackCmd(PONG))
            break
        case DROP_FUNC:
            err = client.HandleDropFunc(payload[2:])
            break
        default:
            err = conn.Send(PackCmd(UNKNOWN))
            break
        }
        if err != nil {
            if err != io.EOF {
                log.Printf("ClientError: %s\n", err.Error())
            }
            return
        }
    }
}

func (client *Client) HandleSubmitJob(payload []byte) (err error) {
    var job Job
    var e error
    var conn = client.conn
    var sched = client.sched
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    e = json.Unmarshal(payload, &job)
    if e != nil {
        err = conn.Send([]byte(e.Error()))
        return
    }
    is_new := true
    changed := false
    job.Status = JOB_STATUS_READY
    oldJob, e := sched.driver.GetOne(job.Func, job.Name)
    if e == nil && oldJob.Id > 0 {
        job.Id = oldJob.Id
        if oldJob.Status == JOB_STATUS_PROC {
            sched.DecrStatProc(oldJob)
            changed = true
        }
        is_new = false
    }
    e = sched.driver.Save(&job)
    if e != nil {
        err = conn.Send([]byte(e.Error()))
        return
    }

    if is_new {
        sched.IncrStatJob(job)
    }
    if is_new || changed {
        sched.pushJobPQ(job)
    }
    sched.Notify()
    err = conn.Send([]byte("ok"))
    return
}

func (client *Client) HandleStatus() (err error) {
    var conn = client.conn
    var sched = client.sched
    data, _ := json.Marshal(sched.Funcs)
    err = conn.Send(data)
    return
}

func (client *Client) HandleDropFunc(payload []byte) (err error) {
    Func := string(payload)
    stat, ok := client.sched.Funcs[Func]
    sched := client.sched
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if ok && stat.Worker == 0 {
        iter := sched.driver.NewIterator(payload)
        deleteJob := make([]int64, 0)
        for {
            if !iter.Next() {
                break
            }
            job := iter.Value()
            deleteJob = append(deleteJob, job.Id)
        }
        iter.Close()
        for _, jobId := range deleteJob {
            sched.driver.Delete(jobId)
        }
        delete(client.sched.Funcs, Func)
        delete(client.sched.jobPQ, Func)
    }
    err = client.conn.Send([]byte("ok"))
    return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
)

var (
	BuildOS    = flag.String("os", "", "OS to target: darwin, freebsd, linux, windows")
	BuildArch  = flag.String("arch", "", "Arch to target: 386, amd64")
	BuildAll   = flag.Bool("all", false, "Builds all architectures")
	ShowHelp   = flag.Bool("help", false, "Shows help")
	matrixKeys = map[string]string{
		"darwin":  "Mac",
		"freebsd": "FreeBSD",
		"linux":   "Linux",
		"windows": "Windows",
		"amd64":   "AMD64",
	}
)

func mainBuild() {
	if *ShowHelp {
		fmt.Println("usage: script/bootstrap [-os] [-arch] [-all]")
		flag.PrintDefaults()
		return
	}

	buildMatrix := make(map[string]Release)

	errored := false

	if *BuildAll {
		for _, buildos := range []string{"darwin", "freebsd", "linux", "windows"} {
			for _, buildarch := range []string{"386", "amd64"} {
				if err := build(buildos, buildarch, buildMatrix); err != nil {
					errored = true
				}
			}
		}
	} else {
		if err := build(*BuildOS, *BuildArch, buildMatrix); err != nil {
			log.Fatalln(err)
		}
		return // skip build matrix stuff
	}

	if errored {
		os.Exit(1)
	}

	by, err := json.Marshal(buildMatrix)
	if err != nil {
		log.Fatalln("Error encoding build matrix to json:", err)
	}

	file, err := os.Create("bin/releases/build_matrix.json")
	if err != nil {
		log.Fatalln("Error creating build_matrix.json:", err)
	}

	written, err := file.Write(by)
	file.Close()

	if err != nil {
		log.Fatalln("Error writing build_matrix.json", err)
	}

	if jsonSize := len(by); written != jsonSize {
		log.Fatalf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
	}
}

func build(buildos, buildarch string, buildMatrix map[string]Release) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := "git-lfs-" + lfs.Version
	dir := "bin"

	if addenv {
		fmt.Printf("Building for %s/%s\n", buildos, buildarch)
		dir = filepath.Join(dir, "releases", buildos+"-"+buildarch, name)
	}

	if err := buildCommand(dir, buildos, buildarch); err != nil {
		return err
	}

	if addenv {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}

		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}
	}

	return nil
}

func buildCommand(dir, buildos, buildarch string) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0

	bin := filepath.Join(dir, "git-lfs")

	if buildos == "windows" {
		bin = bin + ".exe"
	}

	cmd := exec.Command("go", "build", "-o", bin, ".")
	if addenv {
		cmd.Env = []string{
			"GOOS=" + buildos,
			"GOARCH=" + buildarch,
			"GOPATH=" + os.Getenv("GOPATH"),
			"GOROOT=" + os.Getenv("GOROOT"),
		}
	}

	output, err := cmd.CombinedOutput()
	if len(output) > 0 {
		fmt.Println(string(output))
	}
	return err
}

func setupInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	if buildos == "windows" {
		return winInstaller(buildos, buildarch, dir, buildMatrix)
	} else {
		return unixInstaller(buildos, buildarch, dir, buildMatrix)
	}
}

func unixInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	fullInstallPath := filepath.Join(dir, "install.sh")
	cmd := exec.Command("cp", "script/install.sh.example", fullInstallPath)
	if err := logAndRun(cmd); err != nil {
		return err
	}

	if err := os.Chmod(fullInstallPath, 0755); err != nil {
		return err
	}

	name := zipName(buildos, buildarch) + ".tar.gz"

	addToMatrix(buildMatrix, buildos, buildarch, name)

	cmd = exec.Command("tar", "czf", "../"+name, filepath.Base(dir))
	cmd.Dir = filepath.Dir(dir)
	return logAndRun(cmd)
}

func addToMatrix(buildMatrix map[string]Release, buildos, buildarch, name string) {
	buildMatrix[fmt.Sprintf("%s-%s", buildos, buildarch)] = Release{
		Label:    releaseLabel(buildos, buildarch),
		Filename: name,
	}
}

func winInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	cmd := exec.Command("cp", "script/install.bat.example", filepath.Join(dir, "install.bat"))
	if err := logAndRun(cmd); err != nil {
		return err
	}

	installerPath := filepath.Dir(filepath.Dir(dir))

	name := zipName(buildos, buildarch) + ".zip"
	full := filepath.Join(installerPath, name)
	matches, err := filepath.Glob(dir + "/*")
	if err != nil {
		return err
	}

	addToMatrix(buildMatrix, buildos, buildarch, name)

	args := make([]string, len(matches)+2)
	args[0] = "-j" // junk the zip paths
	args[1] = full
	copy(args[2:], matches)

	cmd = exec.Command("zip", args...)
	return logAndRun(cmd)
}

func logAndRun(cmd *exec.Cmd) error {
	fmt.Printf(" - %s\n", strings.Join(cmd.Args, " "))
	if len(cmd.Dir) > 0 {
		fmt.Printf("   - in %s\n", cmd.Dir)
	}

	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	return err
}

func zipName(os, arch string) string {
	return fmt.Sprintf("git-lfs-%s-%s-%s", os, arch, lfs.Version)
}

func releaseLabel(buildos, buildarch string) string {
	return fmt.Sprintf("%s %s", key(buildos), key(buildarch))
}

func key(k string) string {
	if s, ok := matrixKeys[k]; ok {
		return s
	}
	return k
}
<commit_msg>ラララララ ラー ララララー<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
)

var (
	BuildOS    = flag.String("os", "", "OS to target: darwin, freebsd, linux, windows")
	BuildArch  = flag.String("arch", "", "Arch to target: 386, amd64")
	BuildAll   = flag.Bool("all", false, "Builds all architectures")
	ShowHelp   = flag.Bool("help", false, "Shows help")
	matrixKeys = map[string]string{
		"darwin":  "Mac",
		"freebsd": "FreeBSD",
		"linux":   "Linux",
		"windows": "Windows",
		"amd64":   "AMD64",
	}
	LdFlag string
)

func mainBuild() {
	if *ShowHelp {
		fmt.Println("usage: script/bootstrap [-os] [-arch] [-all]")
		flag.PrintDefaults()
		return
	}

	cmd, err := exec.Command("git", "rev-parse", "--short", "HEAD").Output()
	if err != nil {
		panic(err)
	}

	if len(cmd) > 0 {
		arg := strings.TrimSpace("-X github.com/github/git-lfs/lfs.GitCommit " + string(cmd))
		LdFlag = fmt.Sprintf("-ldflags=%q", arg)
	}

	buildMatrix := make(map[string]Release)
	errored := false

	if *BuildAll {
		for _, buildos := range []string{"darwin", "freebsd", "linux", "windows"} {
			for _, buildarch := range []string{"386", "amd64"} {
				if err := build(buildos, buildarch, buildMatrix); err != nil {
					errored = true
				}
			}
		}
	} else {
		if err := build(*BuildOS, *BuildArch, buildMatrix); err != nil {
			log.Fatalln(err)
		}
		return // skip build matrix stuff
	}

	if errored {
		os.Exit(1)
	}

	by, err := json.Marshal(buildMatrix)
	if err != nil {
		log.Fatalln("Error encoding build matrix to json:", err)
	}

	file, err := os.Create("bin/releases/build_matrix.json")
	if err != nil {
		log.Fatalln("Error creating build_matrix.json:", err)
	}

	written, err := file.Write(by)
	file.Close()

	if err != nil {
		log.Fatalln("Error writing build_matrix.json", err)
	}

	if jsonSize := len(by); written != jsonSize {
		log.Fatalf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
	}
}

func build(buildos, buildarch string, buildMatrix map[string]Release) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := "git-lfs-" + lfs.Version
	dir := "bin"

	if addenv {
		fmt.Printf("Building for %s/%s\n", buildos, buildarch)
		dir = filepath.Join(dir, "releases", buildos+"-"+buildarch, name)
	}

	if err := buildCommand(dir, buildos, buildarch); err != nil {
		return err
	}

	if addenv {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}

		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
		if err != nil {
			log.Println("Error setting up installer:\n", err.Error())
			return err
		}
	}

	return nil
}

func buildCommand(dir, buildos, buildarch string) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0

	bin := filepath.Join(dir, "git-lfs")

	if buildos == "windows" {
		bin = bin + ".exe"
	}

	args := make([]string, 1, 5)
	args[0] = "build"
	if len(LdFlag) > 0 {
		args = append(args, LdFlag)
	}
	args = append(args, "-o", bin, ".")

	cmd := exec.Command("go", args...)
	if addenv {
		cmd.Env = []string{
			"GOOS=" + buildos,
			"GOARCH=" + buildarch,
			"GOPATH=" + os.Getenv("GOPATH"),
			"GOROOT=" + os.Getenv("GOROOT"),
		}
	}

	output, err := cmd.CombinedOutput()
	if len(output) > 0 {
		fmt.Println(string(output))
	}
	return err
}

func setupInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	if buildos == "windows" {
		return winInstaller(buildos, buildarch, dir, buildMatrix)
	} else {
		return unixInstaller(buildos, buildarch, dir, buildMatrix)
	}
}

func unixInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	fullInstallPath := filepath.Join(dir, "install.sh")
	cmd := exec.Command("cp", "script/install.sh.example", fullInstallPath)
	if err := logAndRun(cmd); err != nil {
		return err
	}

	if err := os.Chmod(fullInstallPath, 0755); err != nil {
		return err
	}

	name := zipName(buildos, buildarch) + ".tar.gz"

	addToMatrix(buildMatrix, buildos, buildarch, name)

	cmd = exec.Command("tar", "czf", "../"+name, filepath.Base(dir))
	cmd.Dir = filepath.Dir(dir)
	return logAndRun(cmd)
}

func addToMatrix(buildMatrix map[string]Release, buildos, buildarch, name string) {
	buildMatrix[fmt.Sprintf("%s-%s", buildos, buildarch)] = Release{
		Label:    releaseLabel(buildos, buildarch),
		Filename: name,
	}
}

func winInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	cmd := exec.Command("cp", "script/install.bat.example", filepath.Join(dir, "install.bat"))
	if err := logAndRun(cmd); err != nil {
		return err
	}

	installerPath := filepath.Dir(filepath.Dir(dir))

	name := zipName(buildos, buildarch) + ".zip"
	full := filepath.Join(installerPath, name)
	matches, err := filepath.Glob(dir + "/*")
	if err != nil {
		return err
	}

	addToMatrix(buildMatrix, buildos, buildarch, name)

	args := make([]string, len(matches)+2)
	args[0] = "-j" // junk the zip paths
	args[1] = full
	copy(args[2:], matches)

	cmd = exec.Command("zip", args...)
	return logAndRun(cmd)
}

func logAndRun(cmd *exec.Cmd) error {
	fmt.Printf(" - %s\n", strings.Join(cmd.Args, " "))
	if len(cmd.Dir) > 0 {
		fmt.Printf("   - in %s\n", cmd.Dir)
	}

	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	return err
}

func zipName(os, arch string) string {
	return fmt.Sprintf("git-lfs-%s-%s-%s", os, arch, lfs.Version)
}

func releaseLabel(buildos, buildarch string) string {
	return fmt.Sprintf("%s %s", key(buildos), key(buildarch))
}

func key(k string) string {
	if s, ok := matrixKeys[k]; ok {
		return s
	}
	return k
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"github.com/syhlion/go-common"
	"strings"
	"time"
)

type AppData struct {
	db *sql.DB
}

func NewAppData(db *sql.DB) *AppData {
	return &AppData{db}
}

func (d *AppData) IsExist(app_key string) bool {
	sql := "SELECT EXISTS(SELECT 1  FROM  `appdata` WHERE `app_key`= $1)"
	var result int
	err := d.db.QueryRow(sql, app_key).Scan(&result)
	if err != nil {
		log.Debug(app_key, " ", err)
		return false
	}

	if result == 0 {
		log.Debug(app_key, " no exist")
		return false
	}
	return true

}

func (d *AppData) Delete(app_key string) (err error) {
	sql := "DELETE FROM `appdata` where app_key = ?"

	tx, err := d.db.Begin()
	if err != nil {
		log.Debug(err)
		return
	}
	stmt, err := tx.Prepare(sql)
	if err != nil {
		log.Debug(app_key, " ", err)
		return
	}

	_, err = stmt.Exec(app_key)
	if err != nil {
		log.Debug(app_key, " ", err)
		return
	}
	err = tx.Commit()
	if err != nil {
		log.Debug(app_key, " ", err)
		return
	}
	return

}

func (d *AppData) GetAll() (r []AppDataResult, err error) {

	sql := "SELECT * FROM `appdata`"
	rows, err := d.db.Query(sql)
	if err != nil {
		log.Debug(err)
		return
	}
	var apps AppDataResult
	for rows.Next() {
		err = rows.Scan(&apps.AppName, &apps.RequestIP, &apps.AppKey, &apps.Timestamp, &apps.Date)
		if err != nil {
			log.Debug(err)
			return
		}
		r = append(r, apps)
	}
	return

}

func (d *AppData) Register(app_name string, auth_account string, auth_password string, request_ip string) (app_key string, err error) {
	cmd := "INSERT INTO appdata(app_name,auth_account,auth_password,request_ip,app_key,timestamp,date) VALUES (?,?,?,?,?,?,?)"
	tx, err := d.db.Begin()
	if err != nil {
		log.Debug(err)
		return
	}
	stmt, err := tx.Prepare(cmd)
	if err != nil {
		log.Debug(app_name, " ", request_ip, " ", err)
		return
	}
	date := time.Now().Format("2006/01/02 15:04:05")

	seeds := []string{app_name, auth_account, auth_password, request_ip, common.TimeToString(), date}
	seed := strings.Join(seeds, ",")
	app_key = common.EncodeMd5(seed)

	log.Info(app_key)
	_, err = stmt.Exec(app_name, auth_account, auth_password, request_ip, app_key, common.Time(), date)
	if err != nil {
		log.Debug(app_name, " ", request_ip, " ", err)
		return
	}
	err = tx.Commit()
	if err != nil {
		log.Debug(app_name, " ", request_ip, " ", err)
		return
	}
	defer stmt.Close()
	return
}
<commit_msg>appdata get-all 加入新欄位<commit_after>package main

import (
	"database/sql"
	"github.com/syhlion/go-common"
	"strings"
	"time"
)

type AppData struct {
	db *sql.DB
}

func NewAppData(db *sql.DB) *AppData {
	return &AppData{db}
}

func (d *AppData) IsExist(app_key string) bool {
	sql := "SELECT EXISTS(SELECT 1  FROM  `appdata` WHERE `app_key`= $1)"
	var result int
	err := d.db.QueryRow(sql, app_key).Scan(&result)
	if err != nil {
		log.Debug(app_key, " ", err)
		return false
	}

	if result == 0 {
		log.Debug(app_key, " no exist")
		return false
	}
	return true

}

func (d *AppData) Delete(app_key string) (err error) {
	sql := "DELETE FROM `appdata` where app_key = ?"

	tx, err := d.db.Begin()
	if err != nil {
		log.Debug(err)
		return
	}
	stmt, err := tx.Prepare(sql)
	if err != nil {
		log.Debug(app_key, " ", err)
		return
	}

	_, err = stmt.Exec(app_key)
	if err != nil {
		log.Debug(app_key, " ", err)
		return
	}
	err = tx.Commit()
	if err != nil {
		log.Debug(app_key, " ", err)
		return
	}
	return

}

func (d *AppData) GetAll() (r []AppDataResult, err error) {

	sql := "SELECT * FROM `appdata`"
	rows, err := d.db.Query(sql)
	if err != nil {
		log.Debug(err)
		return
	}
	var apps AppDataResult
	for rows.Next() {
		err = rows.Scan(&apps.AppName, &apps.AuthAccount, &apps.AuthPassword, &apps.RequestIP, &apps.AppKey, &apps.Timestamp, &apps.Date)
		if err != nil {
			log.Debug(err)
			return
		}
		r = append(r, apps)
	}
	return

}

func (d *AppData) Register(app_name string, auth_account string, auth_password string, request_ip string) (app_key string, err error) {
	cmd := "INSERT INTO appdata(app_name,auth_account,auth_password,request_ip,app_key,timestamp,date) VALUES (?,?,?,?,?,?,?)"
	tx, err := d.db.Begin()
	if err != nil {
		log.Debug(err)
		return
	}
	stmt, err := tx.Prepare(cmd)
	if err != nil {
		log.Debug(app_name, " ", request_ip, " ", err)
		return
	}
	date := time.Now().Format("2006/01/02 15:04:05")

	seeds := []string{app_name, auth_account, auth_password, request_ip, common.TimeToString(), date}
	seed := strings.Join(seeds, ",")
	app_key = common.EncodeMd5(seed)

	log.Info(app_key)
	_, err = stmt.Exec(app_name, auth_account, auth_password, request_ip, app_key, common.Time(), date)
	if err != nil {
		log.Debug(app_name, " ", request_ip, " ", err)
		return
	}
	err = tx.Commit()
	if err != nil {
		log.Debug(app_name, " ", request_ip, " ", err)
		return
	}
	defer stmt.Close()
	return
}
<|endoftext|>


<commit_before>package bitmask

type BitMask []byte

func New(bits int) BitMask {
	numCells := bits >> 3
	if bits&7 != 0 {
		numCells++
	}
	return make(BitMask, numCells)
}

func (b BitMask) Get(bit int) bool {
	pos := bit >> 3
	if pos > len(b) {
		return false
	}
	return b[bit>>3]&byte(1<<byte(bit&7)) != 0
}

func (b BitMask) Set(bit int, d bool) {
	pos := bit >> 3
	if pos > len(b) {
		return
	}
	shift := byte(1 << byte(bit&7))
	if d {
		b[pos] |= shift
	} else {
		b[pos] &^= shift
	}
}
<commit_msg>added comments<commit_after>// Package bitmask implements a simple bitmask type
package bitmask

// BitMask is the main type. It is completely safe to cast from an existing byte
// slice or to create a BitMask with make.
type BitMask []byte

// New creates a new BitMask with enough storage to hold at least the required
// number of bits.
func New(bits int) BitMask {
	numCells := bits >> 3
	if bits&7 != 0 {
		numCells++
	}
	return make(BitMask, numCells)
}

// Get will retrieve the bool stored at given bit position.
func (b BitMask) Get(bit int) bool {
	pos := bit >> 3
	if pos > len(b) {
		return false
	}
	return b[bit>>3]&byte(1<<byte(bit&7)) != 0
}

// Set will set the given bool at the given position.
func (b BitMask) Set(bit int, d bool) {
	pos := bit >> 3
	if pos > len(b) {
		return
	}
	shift := byte(1 << byte(bit&7))
	if d {
		b[pos] |= shift
	} else {
		b[pos] &^= shift
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ui

import (
	"encoding/json"
	"net/http"
	"path"
	"strconv"
	"time"

	"github.com/square/metrics/inspect"
	"github.com/square/metrics/log"
	_ "github.com/square/metrics/main/static" // ensure that the static files are included.
	"github.com/square/metrics/query"
)

var failedMessage []byte

func init() {
	var err error
	failedMessage, err = json.MarshalIndent(response{Success: false, Message: "Failed to encode the result message."}, "", "  ")
	if err != nil {
		panic(err.Error())
	}
}

// tokenHandler exposes all the tokens available in the system for the autocomplete.
type tokenHandler struct {
	hook    Hook
	context query.ExecutionContext
}

type queryHandler struct {
	hook    Hook
	context query.ExecutionContext
}

// generic response functions
// --------------------------
func commonResponse(writer http.ResponseWriter) {
	writer.Header().Set("Content-Type", "application/json")
}
func errorResponse(writer http.ResponseWriter, code int, err error) {
	commonResponse(writer)
	writer.WriteHeader(code)
	encoded, err := json.MarshalIndent(response{Success: false, Message: err.Error()}, "", "  ")
	if err != nil {
		writer.WriteHeader(http.StatusInternalServerError)
		writer.Write(failedMessage)
		return
	}
	writer.Write(encoded)
}

func bodyResponse(writer http.ResponseWriter, response response) {
	commonResponse(writer)
	response.Success = true
	encoded, err := json.MarshalIndent(response, "", "  ")
	if err != nil {
		writer.WriteHeader(http.StatusInternalServerError)
		writer.Write(failedMessage)
		return
	}
	writer.Write(encoded)
}

// parsing functions
// -----------------

type queryForm struct {
	input   string // query to execute.
	profile bool   // if true, then profile information will be exposed to the user.
}

func parseBool(input string, defaultValue bool) bool {
	value, err := strconv.ParseBool(input)
	if err != nil {
		return defaultValue
	}
	return value
}

func parseQueryForm(request *http.Request) (form queryForm) {
	form.input = request.Form.Get("query")
	form.profile = parseBool(request.Form.Get("profile"), false)
	return
}

func convertProfile(profiler *inspect.Profiler) []profileJSON {
	profiles := profiler.All()
	result := make([]profileJSON, len(profiles))
	for i, p := range profiles {
		result[i] = profileJSON{
			Name:   p.Name(),
			Start:  p.Start().UnixNano() / int64(time.Millisecond),
			Finish: p.Finish().UnixNano() / int64(time.Millisecond),
		}
	}
	return result
}

func (h tokenHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	body := make(map[string]interface{}) // map to array-like types.
	// extract out all the possible tokens
	// 1. keywords
	// 2. functions
	// 3. identifiers
	body["functions"] = h.context.Registry.All()
	metrics, err := h.context.API.GetAllMetrics()
	if err != nil {
		errorResponse(writer, http.StatusInternalServerError, err)
		return
	} else {
		body["metrics"] = metrics
	}
	response := response{
		Body: body,
	}
	bodyResponse(writer, response)
}

func (q queryHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	err := request.ParseForm()
	if err != nil {
		errorResponse(writer, http.StatusBadRequest, err)
		return
	}
	parsedForm := parseQueryForm(request)
	log.Infof("INPUT: %+v\n", parsedForm)

	cmd, err := query.Parse(parsedForm.input)
	if err != nil {
		errorResponse(writer, http.StatusBadRequest, err)
		return
	}

	cmd, profiler := query.NewProfilingCommand(cmd)
	result, err := cmd.Execute(q.context)
	if err != nil {
		errorResponse(writer, http.StatusInternalServerError, err)
		return
	}
	response := response{
		Body: result,
		Name: cmd.Name(),
	}
	if parsedForm.profile {
		response.Profile = convertProfile(profiler)
	}
	bodyResponse(writer, response)
	if q.hook.OnQuery != nil {
		q.hook.OnQuery <- profiler
	}
}

type staticHandler struct {
	Directory  string
	StaticPath string
}

func (h staticHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	res := path.Join(h.Directory, request.URL.Path[len(h.StaticPath):])
	log.Infof("url.path=%s, resource=%s\n", request.URL.Path, res)
	http.ServeFile(writer, request, res)
}

type singleStaticHandler struct {
	Path string
}

func (h singleStaticHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	http.ServeFile(writer, request, h.Path)
}

func NewMux(config Config, context query.ExecutionContext, hook Hook) *http.ServeMux {
	// Wrap the given API and Backend in their Profiling counterparts.
	httpMux := http.NewServeMux()
	httpMux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
		http.Redirect(writer, request, "/ui", http.StatusTemporaryRedirect)
	})
	httpMux.Handle("/ui", singleStaticHandler{path.Join(config.StaticDir, "index.html")})
	httpMux.Handle("/embed", singleStaticHandler{path.Join(config.StaticDir, "embed.html")})
	httpMux.Handle("/query", queryHandler{
		context: context,
		hook:    hook,
	})
	httpMux.Handle("/token", tokenHandler{
		context: context,
		hook:    hook,
	})
	staticPath := "/static/"
	httpMux.Handle(staticPath, staticHandler{StaticPath: staticPath, Directory: config.StaticDir})
	return httpMux
}
<commit_msg>Default to non-indented JSON, allow supplying pretty=true<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ui

import (
	"encoding/json"
	"net/http"
	"path"
	"strconv"
	"time"

	"github.com/square/metrics/inspect"
	"github.com/square/metrics/log"
	_ "github.com/square/metrics/main/static" // ensure that the static files are included.
	"github.com/square/metrics/query"
)

var failedMessage []byte

func init() {
	var err error
	failedMessage, err = json.MarshalIndent(response{Success: false, Message: "Failed to encode the result message."}, "", "  ")
	if err != nil {
		panic(err.Error())
	}
}

// tokenHandler exposes all the tokens available in the system for the autocomplete.
type tokenHandler struct {
	hook    Hook
	context query.ExecutionContext
}

type queryHandler struct {
	hook    Hook
	context query.ExecutionContext
}

// generic response functions
// --------------------------
func commonResponse(writer http.ResponseWriter) {
	writer.Header().Set("Content-Type", "application/json")
}
func errorResponse(writer http.ResponseWriter, code int, err error) {
	commonResponse(writer)
	writer.WriteHeader(code)
	encoded, err := json.MarshalIndent(response{Success: false, Message: err.Error()}, "", "  ")
	if err != nil {
		writer.WriteHeader(http.StatusInternalServerError)
		writer.Write(failedMessage)
		return
	}
	writer.Write(encoded)
}

func bodyResponse(writer http.ResponseWriter, request *http.Request, response response) {
	// Make sure the query params have been parsed
	err := request.ParseForm()
	if err != nil {
		errorResponse(writer, http.StatusBadRequest, err)
		return
	}
	pretty := parseBool(request.Form.Get("pretty"), false)

	commonResponse(writer)
	response.Success = true
	var encoded []byte
	if pretty {
		encoded, err = json.MarshalIndent(response, "", "  ")
	} else {
		encoded, err = json.Marshal(response)
	}
	if err != nil {
		writer.WriteHeader(http.StatusInternalServerError)
		writer.Write(failedMessage)
		return
	}
	writer.Write(encoded)
}

// parsing functions
// -----------------

type queryForm struct {
	input   string // query to execute.
	profile bool   // if true, then profile information will be exposed to the user.
}

func parseBool(input string, defaultValue bool) bool {
	value, err := strconv.ParseBool(input)
	if err != nil {
		return defaultValue
	}
	return value
}

func parseQueryForm(request *http.Request) (form queryForm) {
	form.input = request.Form.Get("query")
	form.profile = parseBool(request.Form.Get("profile"), false)
	return
}

func convertProfile(profiler *inspect.Profiler) []profileJSON {
	profiles := profiler.All()
	result := make([]profileJSON, len(profiles))
	for i, p := range profiles {
		result[i] = profileJSON{
			Name:   p.Name(),
			Start:  p.Start().UnixNano() / int64(time.Millisecond),
			Finish: p.Finish().UnixNano() / int64(time.Millisecond),
		}
	}
	return result
}

func (h tokenHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	body := make(map[string]interface{}) // map to array-like types.
	// extract out all the possible tokens
	// 1. keywords
	// 2. functions
	// 3. identifiers
	body["functions"] = h.context.Registry.All()
	metrics, err := h.context.API.GetAllMetrics()
	if err != nil {
		errorResponse(writer, http.StatusInternalServerError, err)
		return
	}

	body["metrics"] = metrics
	response := response{
		Body: body,
	}
	bodyResponse(writer, request, response)
}

func (q queryHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	err := request.ParseForm()
	if err != nil {
		errorResponse(writer, http.StatusBadRequest, err)
		return
	}
	parsedForm := parseQueryForm(request)
	log.Infof("INPUT: %+v\n", parsedForm)

	cmd, err := query.Parse(parsedForm.input)
	if err != nil {
		errorResponse(writer, http.StatusBadRequest, err)
		return
	}

	cmd, profiler := query.NewProfilingCommand(cmd)
	result, err := cmd.Execute(q.context)
	if err != nil {
		errorResponse(writer, http.StatusInternalServerError, err)
		return
	}
	response := response{
		Body: result,
		Name: cmd.Name(),
	}
	if parsedForm.profile {
		response.Profile = convertProfile(profiler)
	}
	bodyResponse(writer, request, response)
	if q.hook.OnQuery != nil {
		q.hook.OnQuery <- profiler
	}
}

type staticHandler struct {
	Directory  string
	StaticPath string
}

func (h staticHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	res := path.Join(h.Directory, request.URL.Path[len(h.StaticPath):])
	log.Infof("url.path=%s, resource=%s\n", request.URL.Path, res)
	http.ServeFile(writer, request, res)
}

type singleStaticHandler struct {
	Path string
}

func (h singleStaticHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	http.ServeFile(writer, request, h.Path)
}

func NewMux(config Config, context query.ExecutionContext, hook Hook) *http.ServeMux {
	// Wrap the given API and Backend in their Profiling counterparts.
	httpMux := http.NewServeMux()
	httpMux.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
		http.Redirect(writer, request, "/ui", http.StatusTemporaryRedirect)
	})
	httpMux.Handle("/ui", singleStaticHandler{path.Join(config.StaticDir, "index.html")})
	httpMux.Handle("/embed", singleStaticHandler{path.Join(config.StaticDir, "embed.html")})
	httpMux.Handle("/query", queryHandler{
		context: context,
		hook:    hook,
	})
	httpMux.Handle("/token", tokenHandler{
		context: context,
		hook:    hook,
	})
	staticPath := "/static/"
	httpMux.Handle(staticPath, staticHandler{StaticPath: staticPath, Directory: config.StaticDir})
	return httpMux
}
<|endoftext|>


<commit_before>package comb

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestMany(t *testing.T) {
	for _, str := range []string{"", "  "} {
		s := NewState(str)
		result, err := s.Many(s.Char(' '))()

		t.Logf("%#v", result)

		assert.NotEqual(t, result, nil)
		assert.Equal(t, err, nil)
	}
}

func TestManyFail(t *testing.T) {
	for _, str := range []string{"="} {
		s := NewState(str)
		result, err := s.Exhaust(s.Many(func() (interface{}, error) {
			x, err := s.String("=")()

			if err != nil {
				return nil, err
			}

			if x.(string) == "=" {
				return nil, fmt.Errorf("Invalid word")
			}

			return x, nil
		}))()

		t.Logf("%#v", result)

		assert.Equal(t, result, nil)
		assert.NotEqual(t, err, nil)
	}
}

func testMany1Space(str string) (interface{}, error) {
	s := NewState(str)
	return s.Many1(s.Char(' '))()
}

func TestMany1(t *testing.T) {
	result, err := testMany1Space(" ")

	t.Logf("%#v", result)

	assert.NotEqual(t, result, nil)
	assert.Equal(t, err, nil)
}

func TestXFailMany1(t *testing.T) {
	result, err := testMany1Space("")

	t.Log(err)

	assert.Equal(t, result, nil)
	assert.NotEqual(t, err, nil)
}

func TestMany1Nest(t *testing.T) {
	s := NewState("    ")
	result, err := s.Many1(s.Many1(s.Char(' ')))()

	t.Logf("%#v", result)

	assert.NotEqual(t, result, nil)
	assert.Equal(t, err, nil)
}

func testOr(str string) (interface{}, error) {
	s := NewState(str)
	return s.Or(s.Char('a'), s.Char('b'))()
}

func TestOr(t *testing.T) {
	for _, str := range []string{"a", "b"} {
		result, err := testOr(str)

		t.Logf("%#v", result)

		assert.NotEqual(t, result, nil)
		assert.Equal(t, err, nil)
	}
}

func TestXFailOr(t *testing.T) {
	result, err := testOr("c")

	t.Log(err)

	assert.Equal(t, result, nil)
	assert.NotEqual(t, err, nil)
}

func TestMaybeSuccess(t *testing.T) {
	s := NewState("foo")
	result, err := s.Maybe(s.String("foo"))()

	t.Log(result)

	assert.Equal(t, "foo", result)
	assert.Equal(t, nil, err)
}

func TestMaybeFailure(t *testing.T) {
	s := NewState("bar")
	result, err := s.Maybe(s.String("foo"))()

	t.Log(result)

	assert.Equal(t, nil, result)
	assert.Equal(t, nil, err)
}

func TestExhaustWithErroneousParser(t *testing.T) {
	s := NewState("")
	_, err := s.Exhaust(s.String("foo"))()
	assert.NotEqual(t, err, nil)
}

func TestStringify(t *testing.T) {
	str := "foo"
	s := NewState(str)
	result, err := s.Exhaust(s.Stringify(s.And(s.String(str))))()
	assert.Equal(t, str, result.(string))
	assert.Equal(t, err, nil)
}

func TestStringifyFail(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Fail()
		}
	}()

	stringify(42)
}
<commit_msg>Test replace combinator<commit_after>package comb

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestMany(t *testing.T) {
	for _, str := range []string{"", "  "} {
		s := NewState(str)
		result, err := s.Many(s.Char(' '))()

		t.Logf("%#v", result)

		assert.NotEqual(t, result, nil)
		assert.Equal(t, err, nil)
	}
}

func TestManyFail(t *testing.T) {
	for _, str := range []string{"="} {
		s := NewState(str)
		result, err := s.Exhaust(s.Many(func() (interface{}, error) {
			x, err := s.String("=")()

			if err != nil {
				return nil, err
			}

			if x.(string) == "=" {
				return nil, fmt.Errorf("Invalid word")
			}

			return x, nil
		}))()

		t.Logf("%#v", result)

		assert.Equal(t, result, nil)
		assert.NotEqual(t, err, nil)
	}
}

func testMany1Space(str string) (interface{}, error) {
	s := NewState(str)
	return s.Many1(s.Char(' '))()
}

func TestMany1(t *testing.T) {
	result, err := testMany1Space(" ")

	t.Logf("%#v", result)

	assert.NotEqual(t, result, nil)
	assert.Equal(t, err, nil)
}

func TestXFailMany1(t *testing.T) {
	result, err := testMany1Space("")

	t.Log(err)

	assert.Equal(t, result, nil)
	assert.NotEqual(t, err, nil)
}

func TestMany1Nest(t *testing.T) {
	s := NewState("    ")
	result, err := s.Many1(s.Many1(s.Char(' ')))()

	t.Logf("%#v", result)

	assert.NotEqual(t, result, nil)
	assert.Equal(t, err, nil)
}

func testOr(str string) (interface{}, error) {
	s := NewState(str)
	return s.Or(s.Char('a'), s.Char('b'))()
}

func TestOr(t *testing.T) {
	for _, str := range []string{"a", "b"} {
		result, err := testOr(str)

		t.Logf("%#v", result)

		assert.NotEqual(t, result, nil)
		assert.Equal(t, err, nil)
	}
}

func TestXFailOr(t *testing.T) {
	result, err := testOr("c")

	t.Log(err)

	assert.Equal(t, result, nil)
	assert.NotEqual(t, err, nil)
}

func TestMaybeSuccess(t *testing.T) {
	s := NewState("foo")
	result, err := s.Maybe(s.String("foo"))()

	t.Log(result)

	assert.Equal(t, "foo", result)
	assert.Equal(t, nil, err)
}

func TestMaybeFailure(t *testing.T) {
	s := NewState("bar")
	result, err := s.Maybe(s.String("foo"))()

	t.Log(result)

	assert.Equal(t, nil, result)
	assert.Equal(t, nil, err)
}

func TestExhaustWithErroneousParser(t *testing.T) {
	s := NewState("")
	_, err := s.Exhaust(s.String("foo"))()
	assert.NotEqual(t, err, nil)
}

func TestReplace(t *testing.T) {
	s := NewState("foo")
	x, err := s.Exhaust(s.Replace("bar", s.String("foo")))()
	assert.Equal(t, "bar", x)
	assert.Equal(t, err, nil)
}

func TestStringify(t *testing.T) {
	str := "foo"
	s := NewState(str)
	x, err := s.Exhaust(s.Stringify(s.And(s.String(str))))()
	assert.Equal(t, str, x.(string))
	assert.Equal(t, err, nil)
}

func TestStringifyFail(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Fail()
		}
	}()

	stringify(42)
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"github.com/pkg/errors"
	"github.com/viant/toolbox"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"net/http"
	"strings"
	"path/filepath"
	"time"
	"bytes"
)

//PasswordCredential represents a password based credential
type PasswordCredential struct {
	Username string
	Password string
}

//httpStorageService represents basic http storage service (only limited listing and full download are supported)
type httpStorageService struct {
	Credential *PasswordCredential
}

func newHttpClient() (*http.Client, error) {
	return toolbox.NewHttpClient(&toolbox.HttpOptions{Key:"MaxIdleConns", Value:0})
}

func (s *httpStorageService) addCredentialToURLIfNeeded(URL string) string {
	if s.Credential == nil || s.Credential.Password == "" || s.Credential.Username == "" {
		return URL
	}
	prasedURL, err := url.Parse(URL)
	if err != nil {
		return URL
	}
	if prasedURL.User != nil {
		return URL
	}
	return strings.Replace(URL, "://", fmt.Sprintf("://%v:%v@", s.Credential.Username, s.Credential.Password), 1)
}

type hRef struct {
	URL   string
	Value string
}

func extractLinks(body string) []*hRef {
	var result = make([]*hRef, 0)
	var linkContents = strings.Split(string(body), "href=\"")
	for i := 1; i < len(linkContents); i++ {
		var linkContent = linkContents[i]
		linkEndPosition := strings.Index(linkContent, "\"")
		if linkEndPosition == -1 {
			continue
		}
		var content = ""
		contentStartPosition := strings.Index(linkContent, ">")
		if contentStartPosition != 1 {
			content = string(linkContent[contentStartPosition+1:])
			contentEndPosition := strings.Index(content, "<")
			if contentEndPosition != -1 {
				content = string(content[:contentEndPosition])
			}
		}

		link := &hRef{
			URL:   string(linkContent[:linkEndPosition]),
			Value: strings.Trim(content, " \t\r\n"),
		}
		result = append(result, link)

	}
	return result
}

//List returns a list of object for supplied url
func (s *httpStorageService) List(URL string) ([]Object, error) {
	listURL := s.addCredentialToURLIfNeeded(URL)
	client, err := newHttpClient()
	if err != nil {
		return nil, err
	}
	response, err := client.Get(listURL)

	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	now := time.Now()
	contentType := response.Header.Get("Content-Type")
	var result = make([]Object, 0)

	if response.Status != "200 OK" {
		return nil, fmt.Errorf("Invalid response code: %v", response.Status)
	}

	isGitUrl := strings.Contains(URL, "github.com")
	if strings.Contains(contentType, "text/html") {

		links := extractLinks(string(body))

		if isGitUrl {

			for _, link := range links {
				if ! ((strings.Contains(link.URL, "/blob/") || strings.Contains(link.URL, "/tree/")) && strings.HasSuffix(link.URL, link.Value)) {
					continue
				}

				linkType := StorageObjectContentType
				if strings.Contains(link.URL, "/tree/") {
					linkType = StorageObjectFolderType
				}

				baseURL := toolbox.URLBase(URL)
				objectURL := toolbox.URLPathJoin(baseURL, link.URL)

				if linkType == StorageObjectContentType {
					objectURL = strings.Replace(objectURL, "/blob/", "/", 1)
					objectURL = strings.Replace(objectURL, "github.com", "raw.githubusercontent.com", 1)
				}
				storageObject := newHttpFileObject(objectURL, linkType, nil, &now, 0)
				result = append(result, storageObject)
			}

		} else {
			for _, link := range links {
				if link.URL == "" || strings.Contains(link.URL, ":") || strings.HasPrefix(link.URL, "#") || strings.HasPrefix(link.URL, "?") || strings.HasPrefix(link.URL, ".") || strings.HasPrefix(link.URL, "/") {
					continue
				}
				linkType := StorageObjectContentType
				if strings.HasSuffix(link.URL, "/") {
					linkType = StorageObjectFolderType
				}
				objectURL := toolbox.URLPathJoin(URL, link.URL)
				storageObject := newHttpFileObject(objectURL, linkType, nil, &now, 0)
				result = append(result, storageObject)
			}
		}
	}

	if strings.Contains(string(body), ">..<") {
		return result, err
	}
	storageObject := newHttpFileObject(URL, StorageObjectContentType, nil, &now, response.ContentLength)
	result = append(result, storageObject)
	return result, err
}

//Exists returns true if resource exists
func (s *httpStorageService) Exists(URL string) (bool, error) {
	client, err := newHttpClient()
	if err != nil {
		return false, err
	}
	response, err := client.Get(URL)
	if err != nil {
		return false, err
	}
	return response.StatusCode == 200, nil
}

//Object returns a Object for supplied url
func (s *httpStorageService) StorageObject(URL string) (Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("Resource not found: %v", URL)
	}

	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *httpStorageService) Download(object Object) (io.Reader, error) {
	client, err := newHttpClient()
	if err != nil {
		return nil, err
	}
	response, err := client.Get(s.addCredentialToURLIfNeeded(object.URL()))
	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	return bytes.NewReader(content), err
}

//Upload uploads provided reader content for supplied url.
func (s *httpStorageService) Upload(URL string, reader io.Reader) error {
	return errors.New("unsupported")
}

func (s *httpStorageService) Register(schema string, service Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *httpStorageService) Delete(object Object) error {
	fileName, err := toolbox.FileFromURL(object.URL())
	if err != nil {
		return err
	}
	return os.Remove(fileName)
}

func NewHttpStorageService(credential *PasswordCredential) Service {
	return &httpStorageService{
		Credential: credential,
	}
}

type httpStorageObject struct {
	*AbstractObject
}

func (o *httpStorageObject) Unwrap(target interface{}) error {
	return fmt.Errorf("unsuported target %T", target)
}

func newHttpFileObject(url string, objectType int, source interface{}, lastModified *time.Time, size int64) Object {
	abstract := NewAbstractStorageObject(url, source, objectType, lastModified, size)
	result := &httpStorageObject{
		AbstractObject: abstract,
	}
	result.AbstractObject.Object = result
	return result
}

const HttpProviderScheme = "http"
const HttpsProviderScheme = "https"

func init() {
	NewStorageProvider().Registry[HttpsProviderScheme] = serviceProvider
	NewStorageProvider().Registry[HttpProviderScheme] = serviceProvider

}

func serviceProvider(credentialFile string) (Service, error) {

	if credentialFile == "" {
		return NewHttpStorageService(nil), nil
	}

	if !strings.HasPrefix(credentialFile, "/") {
		dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
		credentialFile = path.Join(dir, credentialFile)
	}
	config := &PasswordCredential{}
	err := toolbox.LoadConfigFromUrl("file://"+credentialFile, config)
	if err != nil {
		return nil, err
	}
	return NewHttpStorageService(config), nil
}
<commit_msg>updated credential dep<commit_after>package storage

import (
	"fmt"
	"github.com/pkg/errors"
	"github.com/viant/toolbox"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"net/http"
	"strings"
	"path/filepath"
	"time"
	"bytes"
	"github.com/viant/toolbox/cred"
)

//httpStorageService represents basic http storage service (only limited listing and full download are supported)
type httpStorageService struct {
	Credential *cred.Config
}

func newHttpClient() (*http.Client, error) {
	return toolbox.NewHttpClient(&toolbox.HttpOptions{Key:"MaxIdleConns", Value:0})
}

func (s *httpStorageService) addCredentialToURLIfNeeded(URL string) string {
	if s.Credential == nil || s.Credential.Password == "" || s.Credential.Username == "" {
		return URL
	}
	prasedURL, err := url.Parse(URL)
	if err != nil {
		return URL
	}
	if prasedURL.User != nil {
		return URL
	}
	return strings.Replace(URL, "://", fmt.Sprintf("://%v:%v@", s.Credential.Username, s.Credential.Password), 1)
}

type hRef struct {
	URL   string
	Value string
}

func extractLinks(body string) []*hRef {
	var result = make([]*hRef, 0)
	var linkContents = strings.Split(string(body), "href=\"")
	for i := 1; i < len(linkContents); i++ {
		var linkContent = linkContents[i]
		linkEndPosition := strings.Index(linkContent, "\"")
		if linkEndPosition == -1 {
			continue
		}
		var content = ""
		contentStartPosition := strings.Index(linkContent, ">")
		if contentStartPosition != 1 {
			content = string(linkContent[contentStartPosition+1:])
			contentEndPosition := strings.Index(content, "<")
			if contentEndPosition != -1 {
				content = string(content[:contentEndPosition])
			}
		}

		link := &hRef{
			URL:   string(linkContent[:linkEndPosition]),
			Value: strings.Trim(content, " \t\r\n"),
		}
		result = append(result, link)

	}
	return result
}

//List returns a list of object for supplied url
func (s *httpStorageService) List(URL string) ([]Object, error) {
	listURL := s.addCredentialToURLIfNeeded(URL)
	client, err := newHttpClient()
	if err != nil {
		return nil, err
	}
	response, err := client.Get(listURL)

	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	now := time.Now()
	contentType := response.Header.Get("Content-Type")
	var result = make([]Object, 0)

	if response.Status != "200 OK" {
		return nil, fmt.Errorf("Invalid response code: %v", response.Status)
	}

	isGitUrl := strings.Contains(URL, "github.com")
	if strings.Contains(contentType, "text/html") {

		links := extractLinks(string(body))

		if isGitUrl {

			for _, link := range links {
				if ! ((strings.Contains(link.URL, "/blob/") || strings.Contains(link.URL, "/tree/")) && strings.HasSuffix(link.URL, link.Value)) {
					continue
				}

				linkType := StorageObjectContentType
				if strings.Contains(link.URL, "/tree/") {
					linkType = StorageObjectFolderType
				}

				baseURL := toolbox.URLBase(URL)
				objectURL := toolbox.URLPathJoin(baseURL, link.URL)

				if linkType == StorageObjectContentType {
					objectURL = strings.Replace(objectURL, "/blob/", "/", 1)
					objectURL = strings.Replace(objectURL, "github.com", "raw.githubusercontent.com", 1)
				}
				storageObject := newHttpFileObject(objectURL, linkType, nil, &now, 0)
				result = append(result, storageObject)
			}

		} else {
			for _, link := range links {
				if link.URL == "" || strings.Contains(link.URL, ":") || strings.HasPrefix(link.URL, "#") || strings.HasPrefix(link.URL, "?") || strings.HasPrefix(link.URL, ".") || strings.HasPrefix(link.URL, "/") {
					continue
				}
				linkType := StorageObjectContentType
				if strings.HasSuffix(link.URL, "/") {
					linkType = StorageObjectFolderType
				}
				objectURL := toolbox.URLPathJoin(URL, link.URL)
				storageObject := newHttpFileObject(objectURL, linkType, nil, &now, 0)
				result = append(result, storageObject)
			}
		}
	}

	if strings.Contains(string(body), ">..<") {
		return result, err
	}
	storageObject := newHttpFileObject(URL, StorageObjectContentType, nil, &now, response.ContentLength)
	result = append(result, storageObject)
	return result, err
}

//Exists returns true if resource exists
func (s *httpStorageService) Exists(URL string) (bool, error) {
	client, err := newHttpClient()
	if err != nil {
		return false, err
	}
	response, err := client.Get(URL)
	if err != nil {
		return false, err
	}
	return response.StatusCode == 200, nil
}

//Object returns a Object for supplied url
func (s *httpStorageService) StorageObject(URL string) (Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("Resource not found: %v", URL)
	}

	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *httpStorageService) Download(object Object) (io.Reader, error) {
	client, err := newHttpClient()
	if err != nil {
		return nil, err
	}
	response, err := client.Get(s.addCredentialToURLIfNeeded(object.URL()))
	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	return bytes.NewReader(content), err
}

//Upload uploads provided reader content for supplied url.
func (s *httpStorageService) Upload(URL string, reader io.Reader) error {
	return errors.New("unsupported")
}

func (s *httpStorageService) Register(schema string, service Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *httpStorageService) Delete(object Object) error {
	fileName, err := toolbox.FileFromURL(object.URL())
	if err != nil {
		return err
	}
	return os.Remove(fileName)
}

func NewHttpStorageService(credential *cred.Config) Service {
	return &httpStorageService{
		Credential: credential,
	}
}

type httpStorageObject struct {
	*AbstractObject
}

func (o *httpStorageObject) Unwrap(target interface{}) error {
	return fmt.Errorf("unsuported target %T", target)
}

func newHttpFileObject(url string, objectType int, source interface{}, lastModified *time.Time, size int64) Object {
	abstract := NewAbstractStorageObject(url, source, objectType, lastModified, size)
	result := &httpStorageObject{
		AbstractObject: abstract,
	}
	result.AbstractObject.Object = result
	return result
}

const HttpProviderScheme = "http"
const HttpsProviderScheme = "https"

func init() {
	NewStorageProvider().Registry[HttpsProviderScheme] = serviceProvider
	NewStorageProvider().Registry[HttpProviderScheme] = serviceProvider

}

func serviceProvider(credentialFile string) (Service, error) {

	if credentialFile == "" {
		return NewHttpStorageService(nil), nil
	}

	if !strings.HasPrefix(credentialFile, "/") {
		dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
		credentialFile = path.Join(dir, credentialFile)
	}
	config, err :=cred.NewConfig(credentialFile)
	if err != nil {
		return nil, err
	}
	return NewHttpStorageService(config), nil
}
<|endoftext|>


<commit_before>package redis

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/go-redis/redis/v8"
	perrors "github.com/pkg/errors"
	"github.com/projecteru2/core/log"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
)

var (
	// ErrMaxRetryExceeded indicates redis transaction failed after all the retries
	ErrMaxRetryExceeded = errors.New("[Redis transaction] Max retry exceeded")
	// ErrAlreadyExists indicates the key already exists when do redis SETNX
	ErrAlreadyExists = errors.New("[Redis setnx] Already exists")
	// ErrBadCmdType indicates command type is not correct
	// e.g. SET should be StringCmd
	ErrBadCmdType = errors.New("[Redis cmd] Bad cmd type")
	// ErrKeyNotExitsts indicates no key found
	// When do update, we need to ensure the key exists, just like the behavior of etcd client
	ErrKeyNotExitsts = errors.New("[Redis exists] Key not exists")

	_cache = utils.NewEngineCache(12*time.Hour, 10*time.Minute)
)

const (
	// storage key pattern
	podInfoKey       = "/pod/info/%s" // /pod/info/{podname}
	serviceStatusKey = "/services/%s" // /service/{ipv4:port}

	nodeInfoKey      = "/node/%s"              // /node/{nodename}
	nodePodKey       = "/node/%s:pod/%s"       // /node/{podname}:pod/{nodename}
	nodeCaKey        = "/node/%s:ca"           // /node/{nodename}:ca
	nodeCertKey      = "/node/%s:cert"         // /node/{nodename}:cert
	nodeKeyKey       = "/node/%s:key"          // /node/{nodename}:key
	nodeStatusPrefix = "/status:node/"         // /status:node/{nodename} -> node status key
	nodeWorkloadsKey = "/node/%s:workloads/%s" // /node/{nodename}:workloads/{workloadID}

	workloadInfoKey          = "/workloads/%s" // /workloads/{workloadID}
	workloadDeployPrefix     = "/deploy"       // /deploy/{appname}/{entrypoint}/{nodename}/{workloadID}
	workloadStatusPrefix     = "/status"       // /status/{appname}/{entrypoint}/{nodename}/{workloadID} value -> something by agent
	workloadProcessingPrefix = "/processing"   // /processing/{appname}/{entrypoint}/{nodename}/{opsIdent} value -> count

	// keyspace notification prefix pattern
	keyNotifyPrefix = "__keyspace@%d__:%s"

	// key event action
	actionExpire  = "expire"
	actionExpired = "expired"
	actionSet     = "set"
	actionDel     = "del"
)

// go-redis doesn't export its proto.Error type,
// we have to check the content in this error
func isRedisNoKeyError(e error) bool {
	return e != nil && strings.Contains(e.Error(), "redis: nil")
}

// Rediaron is a store implemented by redis
type Rediaron struct {
	cli    *redis.Client
	config types.Config
	db     int
}

// New creates a new Rediaron instance from config
// Only redis address and db is used
// db is used to separate data, by default db 0 will be used
func New(config types.Config, embeddedStorage bool) (*Rediaron, error) {
	cli := redis.NewClient(&redis.Options{
		Addr: config.Redis.Addr,
		DB:   config.Redis.DB,
	})

	return &Rediaron{
		cli:    cli,
		config: config,
		db:     config.Redis.DB,
	}, nil
}

// KNotifyMessage is received when using KNotify
type KNotifyMessage struct {
	Key    string
	Action string
}

// KNotify is like `watch` in etcd
// knotify comes from inotify, when a key is changed, notification will be published
func (r *Rediaron) KNotify(ctx context.Context, pattern string) chan *KNotifyMessage {
	ch := make(chan *KNotifyMessage)
	go func() {
		defer close(ch)

		prefix := fmt.Sprintf(keyNotifyPrefix, r.db, "")
		channel := fmt.Sprintf(keyNotifyPrefix, r.db, pattern)
		pubsub := r.cli.PSubscribe(ctx, channel)
		subC := pubsub.Channel()

		for {
			select {
			case <-ctx.Done():
				pubsub.Close()
				return
			case v := <-subC:
				if v == nil {
					log.Warnf(ctx, "[KNotify] channel already closed, knotify returns")
					return
				}
				ch <- &KNotifyMessage{
					Key:    strings.TrimPrefix(v.Channel, prefix),
					Action: strings.ToLower(v.Payload),
				}
			}
		}
	}()
	return ch
}

// GetOne is a wrapper
func (r *Rediaron) GetOne(ctx context.Context, key string) (string, error) {
	value, err := r.cli.Get(ctx, key).Result()
	if isRedisNoKeyError(err) {
		return "", perrors.WithMessage(err, fmt.Sprintf("Key not found: %s", key))
	}
	return value, err
}

// GetMulti is a wrapper
func (r *Rediaron) GetMulti(ctx context.Context, keys []string) (map[string]string, error) {
	data := map[string]string{}
	fetch := func(pipe redis.Pipeliner) error {
		for _, k := range keys {
			_, err := pipe.Get(ctx, k).Result()
			if isRedisNoKeyError(err) {
				return perrors.WithMessage(err, fmt.Sprintf("Key not found: %s", k))
			}
			if err != nil {
				return err
			}
		}
		return nil
	}
	cmders, err := r.cli.Pipelined(ctx, fetch)
	for _, cmd := range cmders {
		c, ok := cmd.(*redis.StringCmd)
		if !ok {
			return nil, ErrBadCmdType
		}

		args := c.Args()
		if len(args) != 2 {
			return nil, ErrBadCmdType
		}

		key, ok := args[1].(string)
		if !ok {
			return nil, ErrBadCmdType
		}

		data[key] = c.Val()
	}
	return data, err
}

// BatchUpdate is wrapper to adapt etcd batch update
func (r *Rediaron) BatchUpdate(ctx context.Context, data map[string]string) error {
	keys := []string{}
	for k := range data {
		keys = append(keys, k)
	}

	// check existence of keys
	// FIXME: no transaction ensured
	e, err := r.cli.Exists(ctx, keys...).Result()
	if err != nil {
		return err
	}
	if int(e) != len(keys) {
		return ErrKeyNotExitsts
	}

	update := func(pipe redis.Pipeliner) error {
		for key, value := range data {
			_, err := r.cli.Set(ctx, key, value, 0).Result()
			if err != nil {
				return err
			}
		}

		return nil
	}
	_, err = r.cli.TxPipelined(ctx, update)
	return err
}

// BatchCreate is wrapper to adapt etcd batch create
func (r *Rediaron) BatchCreate(ctx context.Context, data map[string]string) error {
	create := func(pipe redis.Pipeliner) error {
		for key, value := range data {
			created, err := r.cli.SetNX(ctx, key, value, 0).Result()
			if !created {
				return ErrAlreadyExists
			}
			if err != nil {
				return err
			}
		}
		return nil
	}
	_, err := r.cli.TxPipelined(ctx, create)
	return err
}

// BatchDelete is wrapper to adapt etcd batch delete
func (r *Rediaron) BatchDelete(ctx context.Context, keys []string) error {
	del := func(pipe redis.Pipeliner) error {
		for _, key := range keys {
			_, err := pipe.Del(ctx, key).Result()
			if err != nil {
				return err
			}
		}
		return nil
	}
	_, err := r.cli.TxPipelined(ctx, del)
	return err
}

// BindStatus is wrapper to adapt etcd bind status
func (r *Rediaron) BindStatus(ctx context.Context, entityKey, statusKey, statusValue string, ttl int64) error {
	count, err := r.cli.Exists(ctx, entityKey).Result()
	if err != nil {
		return err
	}
	// doesn't exist, returns nil, does nothing
	// to behave just like etcd
	if count != 1 {
		return nil
	}

	_, err = r.cli.Set(ctx, statusKey, statusValue, time.Duration(ttl)*time.Second).Result()
	return err
}

// TerminateEmbededStorage terminates embedded store
// in order to implement Store interface
// we can't use embedded redis, it doesn't support keyspace notification
// never call this except running unittests
func (r *Rediaron) TerminateEmbededStorage() {
	_ = r.cli.Close()
}
<commit_msg>should check cmd.Err when using multi<commit_after>package redis

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/go-redis/redis/v8"
	perrors "github.com/pkg/errors"
	"github.com/projecteru2/core/log"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
)

var (
	// ErrMaxRetryExceeded indicates redis transaction failed after all the retries
	ErrMaxRetryExceeded = errors.New("[Redis transaction] Max retry exceeded")
	// ErrAlreadyExists indicates the key already exists when do redis SETNX
	ErrAlreadyExists = errors.New("[Redis setnx] Already exists")
	// ErrBadCmdType indicates command type is not correct
	// e.g. SET should be StringCmd
	ErrBadCmdType = errors.New("[Redis cmd] Bad cmd type")
	// ErrKeyNotExitsts indicates no key found
	// When do update, we need to ensure the key exists, just like the behavior of etcd client
	ErrKeyNotExitsts = errors.New("[Redis exists] Key not exists")

	_cache = utils.NewEngineCache(12*time.Hour, 10*time.Minute)
)

const (
	// storage key pattern
	podInfoKey       = "/pod/info/%s" // /pod/info/{podname}
	serviceStatusKey = "/services/%s" // /service/{ipv4:port}

	nodeInfoKey      = "/node/%s"              // /node/{nodename}
	nodePodKey       = "/node/%s:pod/%s"       // /node/{podname}:pod/{nodename}
	nodeCaKey        = "/node/%s:ca"           // /node/{nodename}:ca
	nodeCertKey      = "/node/%s:cert"         // /node/{nodename}:cert
	nodeKeyKey       = "/node/%s:key"          // /node/{nodename}:key
	nodeStatusPrefix = "/status:node/"         // /status:node/{nodename} -> node status key
	nodeWorkloadsKey = "/node/%s:workloads/%s" // /node/{nodename}:workloads/{workloadID}

	workloadInfoKey          = "/workloads/%s" // /workloads/{workloadID}
	workloadDeployPrefix     = "/deploy"       // /deploy/{appname}/{entrypoint}/{nodename}/{workloadID}
	workloadStatusPrefix     = "/status"       // /status/{appname}/{entrypoint}/{nodename}/{workloadID} value -> something by agent
	workloadProcessingPrefix = "/processing"   // /processing/{appname}/{entrypoint}/{nodename}/{opsIdent} value -> count

	// keyspace notification prefix pattern
	keyNotifyPrefix = "__keyspace@%d__:%s"

	// key event action
	actionExpire  = "expire"
	actionExpired = "expired"
	actionSet     = "set"
	actionDel     = "del"
)

// go-redis doesn't export its proto.Error type,
// we have to check the content in this error
func isRedisNoKeyError(e error) bool {
	return e != nil && strings.Contains(e.Error(), "redis: nil")
}

// Rediaron is a store implemented by redis
type Rediaron struct {
	cli    *redis.Client
	config types.Config
	db     int
}

// New creates a new Rediaron instance from config
// Only redis address and db is used
// db is used to separate data, by default db 0 will be used
func New(config types.Config, embeddedStorage bool) (*Rediaron, error) {
	cli := redis.NewClient(&redis.Options{
		Addr: config.Redis.Addr,
		DB:   config.Redis.DB,
	})

	return &Rediaron{
		cli:    cli,
		config: config,
		db:     config.Redis.DB,
	}, nil
}

// KNotifyMessage is received when using KNotify
type KNotifyMessage struct {
	Key    string
	Action string
}

// KNotify is like `watch` in etcd
// knotify comes from inotify, when a key is changed, notification will be published
func (r *Rediaron) KNotify(ctx context.Context, pattern string) chan *KNotifyMessage {
	ch := make(chan *KNotifyMessage)
	go func() {
		defer close(ch)

		prefix := fmt.Sprintf(keyNotifyPrefix, r.db, "")
		channel := fmt.Sprintf(keyNotifyPrefix, r.db, pattern)
		pubsub := r.cli.PSubscribe(ctx, channel)
		subC := pubsub.Channel()

		for {
			select {
			case <-ctx.Done():
				pubsub.Close()
				return
			case v := <-subC:
				if v == nil {
					log.Warnf(ctx, "[KNotify] channel already closed, knotify returns")
					return
				}
				ch <- &KNotifyMessage{
					Key:    strings.TrimPrefix(v.Channel, prefix),
					Action: strings.ToLower(v.Payload),
				}
			}
		}
	}()
	return ch
}

// GetOne is a wrapper
func (r *Rediaron) GetOne(ctx context.Context, key string) (string, error) {
	value, err := r.cli.Get(ctx, key).Result()
	if isRedisNoKeyError(err) {
		return "", perrors.WithMessage(err, fmt.Sprintf("Key not found: %s", key))
	}
	return value, err
}

// GetMulti is a wrapper
func (r *Rediaron) GetMulti(ctx context.Context, keys []string) (map[string]string, error) {
	data := map[string]string{}
	fetch := func(pipe redis.Pipeliner) error {
		for _, k := range keys {
			_, err := pipe.Get(ctx, k).Result()
			if err != nil {
				return err
			}
		}
		return nil
	}
	cmders, err := r.cli.Pipelined(ctx, fetch)
	for _, cmd := range cmders {
		c, ok := cmd.(*redis.StringCmd)
		if !ok {
			return nil, ErrBadCmdType
		}

		args := c.Args()
		if len(args) != 2 {
			return nil, ErrBadCmdType
		}

		key, ok := args[1].(string)
		if !ok {
			return nil, ErrBadCmdType
		}

		if isRedisNoKeyError(c.Err()) {
			return nil, perrors.WithMessage(err, fmt.Sprintf("Key not found: %s", key))
		}

		data[key] = c.Val()
	}
	return data, err
}

// BatchUpdate is wrapper to adapt etcd batch update
func (r *Rediaron) BatchUpdate(ctx context.Context, data map[string]string) error {
	keys := []string{}
	for k := range data {
		keys = append(keys, k)
	}

	// check existence of keys
	// FIXME: no transaction ensured
	e, err := r.cli.Exists(ctx, keys...).Result()
	if err != nil {
		return err
	}
	if int(e) != len(keys) {
		return ErrKeyNotExitsts
	}

	update := func(pipe redis.Pipeliner) error {
		for key, value := range data {
			_, err := r.cli.Set(ctx, key, value, 0).Result()
			if err != nil {
				return err
			}
		}

		return nil
	}
	_, err = r.cli.TxPipelined(ctx, update)
	return err
}

// BatchCreate is wrapper to adapt etcd batch create
func (r *Rediaron) BatchCreate(ctx context.Context, data map[string]string) error {
	create := func(pipe redis.Pipeliner) error {
		for key, value := range data {
			created, err := r.cli.SetNX(ctx, key, value, 0).Result()
			if !created {
				return ErrAlreadyExists
			}
			if err != nil {
				return err
			}
		}
		return nil
	}
	_, err := r.cli.TxPipelined(ctx, create)
	return err
}

// BatchDelete is wrapper to adapt etcd batch delete
func (r *Rediaron) BatchDelete(ctx context.Context, keys []string) error {
	del := func(pipe redis.Pipeliner) error {
		for _, key := range keys {
			_, err := pipe.Del(ctx, key).Result()
			if err != nil {
				return err
			}
		}
		return nil
	}
	_, err := r.cli.TxPipelined(ctx, del)
	return err
}

// BindStatus is wrapper to adapt etcd bind status
func (r *Rediaron) BindStatus(ctx context.Context, entityKey, statusKey, statusValue string, ttl int64) error {
	count, err := r.cli.Exists(ctx, entityKey).Result()
	if err != nil {
		return err
	}
	// doesn't exist, returns nil, does nothing
	// to behave just like etcd
	if count != 1 {
		return nil
	}

	_, err = r.cli.Set(ctx, statusKey, statusValue, time.Duration(ttl)*time.Second).Result()
	return err
}

// TerminateEmbededStorage terminates embedded store
// in order to implement Store interface
// we can't use embedded redis, it doesn't support keyspace notification
// never call this except running unittests
func (r *Rediaron) TerminateEmbededStorage() {
	_ = r.cli.Close()
}
<|endoftext|>