<commit_before>package readline

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

type AutoCompleter interface {
	// Readline will pass the whole line and current offset to it
	// Completer need to pass all the candidates, and how long they shared the same characters in line
	// Example:
	//   [go, git, git-shell, grep]
	//   Do("g", 1) => ["o", "it", "it-shell", "rep"], 1
	//   Do("gi", 2) => ["t", "t-shell"], 2
	//   Do("git", 3) => ["", "-shell"], 3
	Do(line []rune, pos int) (newLine [][]rune, length int)
}

type TabCompleter struct{}

func (t *TabCompleter) Do([]rune, int) ([][]rune, int) {
	return [][]rune{[]rune("\t")}, 0
}

type opCompleter struct {
	w     io.Writer
	op    *Operation
	width int

	inCompleteMode  bool
	inSelectMode    bool
	candidate       [][]rune
	candidateSource []rune
	candidateOff    int
	candidateChoise int
	candidateColNum int
}

func newOpCompleter(w io.Writer, op *Operation, width int) *opCompleter {
	return &opCompleter{
		w:     w,
		op:    op,
		width: width,
	}
}

func (o *opCompleter) doSelect() {
	if len(o.candidate) == 1 {
		o.op.buf.WriteRunes(o.candidate[0])
		o.ExitCompleteMode(false)
		return
	}
	o.nextCandidate(1)
	o.CompleteRefresh()
}

func (o *opCompleter) nextCandidate(i int) {
	o.candidateChoise += i
	o.candidateChoise = o.candidateChoise % len(o.candidate)
	if o.candidateChoise < 0 {
		o.candidateChoise = len(o.candidate) + o.candidateChoise
	}
}

func (o *opCompleter) OnComplete() bool {
	if o.width == 0 {
		return false
	}
	if o.IsInCompleteSelectMode() {
		o.doSelect()
		return true
	}

	buf := o.op.buf
	rs := buf.Runes()

	if o.IsInCompleteMode() && o.candidateSource != nil && runes.Equal(rs, o.candidateSource) {
		o.EnterCompleteSelectMode()
		o.doSelect()
		return true
	}

	o.ExitCompleteSelectMode()
	o.candidateSource = rs
	newLines, offset := o.op.cfg.AutoComplete.Do(rs, buf.idx)
	if len(newLines) == 0 {
		o.ExitCompleteMode(false)
		return true
	}

	// only Aggregate candidates in non-complete mode
	if !o.IsInCompleteMode() {
		if len(newLines) == 1 {
			buf.WriteRunes(newLines[0])
			o.ExitCompleteMode(false)
			return true
		}

		same, size := runes.Aggregate(newLines)
		if size > 0 {
			buf.WriteRunes(same)
			o.ExitCompleteMode(false)
			return true
		}
	}

	o.EnterCompleteMode(offset, newLines)
	return true
}

func (o *opCompleter) IsInCompleteSelectMode() bool {
	return o.inSelectMode
}

func (o *opCompleter) IsInCompleteMode() bool {
	return o.inCompleteMode
}

func (o *opCompleter) HandleCompleteSelect(r rune) bool {
	next := true
	switch r {
	case CharEnter, CharCtrlJ:
		next = false
		o.op.buf.WriteRunes(o.op.candidate[o.op.candidateChoise])
		o.ExitCompleteMode(false)
	case CharLineStart:
		num := o.candidateChoise % o.candidateColNum
		o.nextCandidate(-num)
	case CharLineEnd:
		num := o.candidateColNum - o.candidateChoise%o.candidateColNum - 1
		o.candidateChoise += num
		if o.candidateChoise >= len(o.candidate) {
			o.candidateChoise = len(o.candidate) - 1
		}
	case CharBackspace:
		o.ExitCompleteSelectMode()
		next = false
	case CharTab, CharForward:
		o.doSelect()
	case CharBell, CharInterrupt:
		o.ExitCompleteMode(true)
		next = false
	case CharNext:
		tmpChoise := o.candidateChoise + o.candidateColNum
		if tmpChoise >= o.getMatrixSize() {
			tmpChoise -= o.getMatrixSize()
		} else if tmpChoise >= len(o.candidate) {
			tmpChoise += o.candidateColNum
			tmpChoise -= o.getMatrixSize()
		}
		o.candidateChoise = tmpChoise
	case CharBackward:
		o.nextCandidate(-1)
	case CharPrev:
		tmpChoise := o.candidateChoise - o.candidateColNum
		if tmpChoise < 0 {
			tmpChoise += o.getMatrixSize()
			if tmpChoise >= len(o.candidate) {
				tmpChoise -= o.candidateColNum
			}
		}
		o.candidateChoise = tmpChoise
	default:
		next = false
		o.ExitCompleteSelectMode()
	}
	if next {
		o.CompleteRefresh()
		return true
	}
	return false
}

func (o *opCompleter) getMatrixSize() int {
	line := len(o.candidate) / o.candidateColNum
	if len(o.candidate)%o.candidateColNum != 0 {
		line++
	}
	return line * o.candidateColNum
}

func (o *opCompleter) OnWidthChange(newWidth int) {
	o.width = newWidth
}

func (o *opCompleter) CompleteRefresh() {
	if !o.inCompleteMode {
		return
	}
	lineCnt := o.op.buf.CursorLineCount()
	colWidth := 0
	for _, c := range o.candidate {
		w := runes.WidthAll(c)
		if w > colWidth {
			colWidth = w
		}
	}
	colWidth += o.candidateOff + 1
	same := o.op.buf.RuneSlice(-o.candidateOff)

	// -1 to avoid reach the end of line
	width := o.width - 1
	colNum := width / colWidth
	if colNum != 0 {
		colWidth += (width - (colWidth * colNum)) / colNum
	}

	o.candidateColNum = colNum
	buf := bufio.NewWriter(o.w)
	buf.Write(bytes.Repeat([]byte("\n"), lineCnt))

	colIdx := 0
	lines := 1
	buf.WriteString("\033[J")
	for idx, c := range o.candidate {
		inSelect := idx == o.candidateChoise && o.IsInCompleteSelectMode()
		if inSelect {
			buf.WriteString("\033[30;47m")
		}
		buf.WriteString(string(same))
		buf.WriteString(string(c))
		buf.Write(bytes.Repeat([]byte(" "), colWidth-len(c)-len(same)))

		if inSelect {
			buf.WriteString("\033[0m")
		}

		colIdx++
		if colIdx == colNum {
			buf.WriteString("\n")
			lines++
			colIdx = 0
		}
	}

	// move back
	fmt.Fprintf(buf, "\033[%dA\r", lineCnt-1+lines)
	fmt.Fprintf(buf, "\033[%dC", o.op.buf.idx+o.op.buf.PromptLen())
	buf.Flush()
}

func (o *opCompleter) aggCandidate(candidate [][]rune) int {
	offset := 0
	for i := 0; i < len(candidate[0]); i++ {
		for j := 0; j < len(candidate)-1; j++ {
			if i > len(candidate[j]) {
				goto aggregate
			}
			if candidate[j][i] != candidate[j+1][i] {
				goto aggregate
			}
		}
		offset = i
	}
aggregate:
	return offset
}

func (o *opCompleter) EnterCompleteSelectMode() {
	o.inSelectMode = true
	o.candidateChoise = -1
	o.CompleteRefresh()
}

func (o *opCompleter) EnterCompleteMode(offset int, candidate [][]rune) {
	o.inCompleteMode = true
	o.candidate = candidate
	o.candidateOff = offset
	o.CompleteRefresh()
}

func (o *opCompleter) ExitCompleteSelectMode() {
	o.inSelectMode = false
	o.candidate = nil
	o.candidateChoise = -1
	o.candidateOff = -1
	o.candidateSource = nil
}

func (o *opCompleter) ExitCompleteMode(revent bool) {
	o.inCompleteMode = false
	o.ExitCompleteSelectMode()
}
<commit_msg>Modify the display width for Chinese characters and so on (#145)<commit_after>package readline

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

type AutoCompleter interface {
	// Readline will pass the whole line and current offset to it
	// Completer need to pass all the candidates, and how long they shared the same characters in line
	// Example:
	//   [go, git, git-shell, grep]
	//   Do("g", 1) => ["o", "it", "it-shell", "rep"], 1
	//   Do("gi", 2) => ["t", "t-shell"], 2
	//   Do("git", 3) => ["", "-shell"], 3
	Do(line []rune, pos int) (newLine [][]rune, length int)
}

type TabCompleter struct{}

func (t *TabCompleter) Do([]rune, int) ([][]rune, int) {
	return [][]rune{[]rune("\t")}, 0
}

type opCompleter struct {
	w     io.Writer
	op    *Operation
	width int

	inCompleteMode  bool
	inSelectMode    bool
	candidate       [][]rune
	candidateSource []rune
	candidateOff    int
	candidateChoise int
	candidateColNum int
}

func newOpCompleter(w io.Writer, op *Operation, width int) *opCompleter {
	return &opCompleter{
		w:     w,
		op:    op,
		width: width,
	}
}

func (o *opCompleter) doSelect() {
	if len(o.candidate) == 1 {
		o.op.buf.WriteRunes(o.candidate[0])
		o.ExitCompleteMode(false)
		return
	}
	o.nextCandidate(1)
	o.CompleteRefresh()
}

func (o *opCompleter) nextCandidate(i int) {
	o.candidateChoise += i
	o.candidateChoise = o.candidateChoise % len(o.candidate)
	if o.candidateChoise < 0 {
		o.candidateChoise = len(o.candidate) + o.candidateChoise
	}
}

func (o *opCompleter) OnComplete() bool {
	if o.width == 0 {
		return false
	}
	if o.IsInCompleteSelectMode() {
		o.doSelect()
		return true
	}

	buf := o.op.buf
	rs := buf.Runes()

	if o.IsInCompleteMode() && o.candidateSource != nil && runes.Equal(rs, o.candidateSource) {
		o.EnterCompleteSelectMode()
		o.doSelect()
		return true
	}

	o.ExitCompleteSelectMode()
	o.candidateSource = rs
	newLines, offset := o.op.cfg.AutoComplete.Do(rs, buf.idx)
	if len(newLines) == 0 {
		o.ExitCompleteMode(false)
		return true
	}

	// only Aggregate candidates in non-complete mode
	if !o.IsInCompleteMode() {
		if len(newLines) == 1 {
			buf.WriteRunes(newLines[0])
			o.ExitCompleteMode(false)
			return true
		}

		same, size := runes.Aggregate(newLines)
		if size > 0 {
			buf.WriteRunes(same)
			o.ExitCompleteMode(false)
			return true
		}
	}

	o.EnterCompleteMode(offset, newLines)
	return true
}

func (o *opCompleter) IsInCompleteSelectMode() bool {
	return o.inSelectMode
}

func (o *opCompleter) IsInCompleteMode() bool {
	return o.inCompleteMode
}

func (o *opCompleter) HandleCompleteSelect(r rune) bool {
	next := true
	switch r {
	case CharEnter, CharCtrlJ:
		next = false
		o.op.buf.WriteRunes(o.op.candidate[o.op.candidateChoise])
		o.ExitCompleteMode(false)
	case CharLineStart:
		num := o.candidateChoise % o.candidateColNum
		o.nextCandidate(-num)
	case CharLineEnd:
		num := o.candidateColNum - o.candidateChoise%o.candidateColNum - 1
		o.candidateChoise += num
		if o.candidateChoise >= len(o.candidate) {
			o.candidateChoise = len(o.candidate) - 1
		}
	case CharBackspace:
		o.ExitCompleteSelectMode()
		next = false
	case CharTab, CharForward:
		o.doSelect()
	case CharBell, CharInterrupt:
		o.ExitCompleteMode(true)
		next = false
	case CharNext:
		tmpChoise := o.candidateChoise + o.candidateColNum
		if tmpChoise >= o.getMatrixSize() {
			tmpChoise -= o.getMatrixSize()
		} else if tmpChoise >= len(o.candidate) {
			tmpChoise += o.candidateColNum
			tmpChoise -= o.getMatrixSize()
		}
		o.candidateChoise = tmpChoise
	case CharBackward:
		o.nextCandidate(-1)
	case CharPrev:
		tmpChoise := o.candidateChoise - o.candidateColNum
		if tmpChoise < 0 {
			tmpChoise += o.getMatrixSize()
			if tmpChoise >= len(o.candidate) {
				tmpChoise -= o.candidateColNum
			}
		}
		o.candidateChoise = tmpChoise
	default:
		next = false
		o.ExitCompleteSelectMode()
	}
	if next {
		o.CompleteRefresh()
		return true
	}
	return false
}

func (o *opCompleter) getMatrixSize() int {
	line := len(o.candidate) / o.candidateColNum
	if len(o.candidate)%o.candidateColNum != 0 {
		line++
	}
	return line * o.candidateColNum
}

func (o *opCompleter) OnWidthChange(newWidth int) {
	o.width = newWidth
}

func (o *opCompleter) CompleteRefresh() {
	if !o.inCompleteMode {
		return
	}
	lineCnt := o.op.buf.CursorLineCount()
	colWidth := 0
	for _, c := range o.candidate {
		w := runes.WidthAll(c)
		if w > colWidth {
			colWidth = w
		}
	}
	colWidth += o.candidateOff + 1
	same := o.op.buf.RuneSlice(-o.candidateOff)

	// -1 to avoid reach the end of line
	width := o.width - 1
	colNum := width / colWidth
	if colNum != 0 {
		colWidth += (width - (colWidth * colNum)) / colNum
	}

	o.candidateColNum = colNum
	buf := bufio.NewWriter(o.w)
	buf.Write(bytes.Repeat([]byte("\n"), lineCnt))

	colIdx := 0
	lines := 1
	buf.WriteString("\033[J")
	for idx, c := range o.candidate {
		inSelect := idx == o.candidateChoise && o.IsInCompleteSelectMode()
		if inSelect {
			buf.WriteString("\033[30;47m")
		}
		buf.WriteString(string(same))
		buf.WriteString(string(c))
		buf.Write(bytes.Repeat([]byte(" "), colWidth-runes.WidthAll(c)-runes.WidthAll(same)))

		if inSelect {
			buf.WriteString("\033[0m")
		}

		colIdx++
		if colIdx == colNum {
			buf.WriteString("\n")
			lines++
			colIdx = 0
		}
	}

	// move back
	fmt.Fprintf(buf, "\033[%dA\r", lineCnt-1+lines)
	fmt.Fprintf(buf, "\033[%dC", o.op.buf.idx+o.op.buf.PromptLen())
	buf.Flush()
}

func (o *opCompleter) aggCandidate(candidate [][]rune) int {
	offset := 0
	for i := 0; i < len(candidate[0]); i++ {
		for j := 0; j < len(candidate)-1; j++ {
			if i > len(candidate[j]) {
				goto aggregate
			}
			if candidate[j][i] != candidate[j+1][i] {
				goto aggregate
			}
		}
		offset = i
	}
aggregate:
	return offset
}

func (o *opCompleter) EnterCompleteSelectMode() {
	o.inSelectMode = true
	o.candidateChoise = -1
	o.CompleteRefresh()
}

func (o *opCompleter) EnterCompleteMode(offset int, candidate [][]rune) {
	o.inCompleteMode = true
	o.candidate = candidate
	o.candidateOff = offset
	o.CompleteRefresh()
}

func (o *opCompleter) ExitCompleteSelectMode() {
	o.inSelectMode = false
	o.candidate = nil
	o.candidateChoise = -1
	o.candidateOff = -1
	o.candidateSource = nil
}

func (o *opCompleter) ExitCompleteMode(revent bool) {
	o.inCompleteMode = false
	o.ExitCompleteSelectMode()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"time"
)

var (
	localUpTimeout = flag.Duration("local-up-timeout", 2*time.Minute, "(local only) Time limit between 'local-up-cluster.sh' and a response from the Kubernetes API.")
)

type localCluster struct {
	tempDir    string
	kubeConfig string
}

var _ deployer = localCluster{}

func newLocalCluster() *localCluster {
	tempDir, err := ioutil.TempDir("", "kubetest-local")
	if err != nil {
		log.Fatal("unable to create temp directory")
	}
	err = os.Chmod(tempDir, 0755)
	if err != nil {
		log.Fatal("unable to change temp directory permissions")
	}
	return &localCluster{
		tempDir: tempDir,
	}
}

func (n localCluster) getScript(scriptPath string) (string, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	path := filepath.Join(cwd, scriptPath)
	if _, err := os.Stat(path); err == nil {
		return path, nil
	}
	return "", fmt.Errorf("unable to find script %v in directory %v", scriptPath, cwd)
}

func (n localCluster) Up() error {
	script, err := n.getScript("hack/local-up-cluster.sh")
	if err != nil {
		return err
	}
	cmd := exec.Command(script)
	cmd.Env = os.Environ()
	cmd.Env = append(cmd.Env, "ENABLE_DAEMON=true")
	cmd.Env = append(cmd.Env, fmt.Sprintf("LOG_DIR=%s", n.tempDir))
	err = control.FinishRunning(cmd)
	if err != nil {
		return err
	}
	n.kubeConfig = "/var/run/kubernetes/admin.kubeconfig"
	_, err = os.Stat(n.kubeConfig)
	return err
}

func (n localCluster) IsUp() error {
	if n.kubeConfig != "" {
		err := os.Setenv("KUBECONFIG", n.kubeConfig)
		if err != nil {
			log.Fatal("unable to set KUBECONFIG environment variable")
		}
	}
	stop := time.Now().Add(*localUpTimeout)
	for {
		script, err := n.getScript("cluster/kubectl.sh")
		if err != nil {
			return err
		}
		nodes, err := kubectlGetNodes(script)
		if err != nil {
			return err
		}
		readyNodes := countReadyNodes(nodes)
		if readyNodes > 0 {
			return nil
		}
		if time.Now().After(stop) {
			break
		} else {
			time.Sleep(5 * time.Second)
		}
	}
	return errors.New("local-up-cluster.sh is not ready")
}

func (n localCluster) DumpClusterLogs(localPath, gcsPath string) error {
	cmd := exec.Command("sudo", "cp", "-r", n.tempDir, localPath)
	return control.FinishRunning(cmd)
}

func (n localCluster) TestSetup() error {
	return nil
}

func (n localCluster) Down() error {
	err := control.FinishRunning(exec.Command("bash", "-c", "docker rm -f $(docker ps -a -q)"))
	if err != nil {
		log.Printf("unable to cleanup containers in docker: %v", err)
	}
	err = control.FinishRunning(exec.Command("pkill", "hyperkube"))
	if err != nil {
		log.Printf("unable to kill hyperkube processes: %v", err)
	}
	err = control.FinishRunning(exec.Command("pkill", "etcd"))
	if err != nil {
		log.Printf("unable to kill etcd: %v", err)
	}
	return nil
}

func (n localCluster) GetClusterCreated(gcpProject string) (time.Time, error) {
	return time.Time{}, errors.New("GetClusterCreated not implemented in localCluster")
}
<commit_msg>Tweak env vars for local-up-cluster for CI<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"time"
)

var (
	localUpTimeout = flag.Duration("local-up-timeout", 2*time.Minute, "(local only) Time limit between 'local-up-cluster.sh' and a response from the Kubernetes API.")
)

type localCluster struct {
	tempDir    string
	kubeConfig string
}

var _ deployer = localCluster{}

func newLocalCluster() *localCluster {
	tempDir, err := ioutil.TempDir("", "kubetest-local")
	if err != nil {
		log.Fatal("unable to create temp directory")
	}
	err = os.Chmod(tempDir, 0755)
	if err != nil {
		log.Fatal("unable to change temp directory permissions")
	}
	return &localCluster{
		tempDir: tempDir,
	}
}

func (n localCluster) getScript(scriptPath string) (string, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	path := filepath.Join(cwd, scriptPath)
	if _, err := os.Stat(path); err == nil {
		return path, nil
	}
	return "", fmt.Errorf("unable to find script %v in directory %v", scriptPath, cwd)
}

func (n localCluster) Up() error {
	script, err := n.getScript("hack/local-up-cluster.sh")
	if err != nil {
		return err
	}

	cmd := exec.Command(script)
	cmd.Env = os.Environ()
	cmd.Env = append(cmd.Env, "ENABLE_DAEMON=true")
	cmd.Env = append(cmd.Env, fmt.Sprintf("LOG_DIR=%s", n.tempDir))

	// when we are running in a DIND scenario, we should use the ip address of
	// the docker0 network interface, This ensures that when the pods come up
	// the health checks (for example for kubedns) succeed. If there is no
	// docker0, just use the defaults in local-up-cluster.sh
	dockerIp := ""
	docker0, err := net.InterfaceByName("docker0")
	if err == nil {
		addresses, err := docker0.Addrs()
		if err == nil {
			for _, address := range addresses {
				if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
					if ipnet.IP.To4() != nil {
						dockerIp = ipnet.IP.String()
						break
					}
				}
			}
		} else {
			log.Printf("unable to get addresses from docker0 interface : %v", err)
		}
	} else {
		log.Printf("unable to find docker0 interface : %v", err)
	}
	if dockerIp != "" {
		log.Printf("using %v for API_HOST_IP, HOSTNAME_OVERRIDE, KUBELET_HOST", dockerIp)
		cmd.Env = append(cmd.Env, fmt.Sprintf("API_HOST_IP=%s", dockerIp))
		cmd.Env = append(cmd.Env, fmt.Sprintf("HOSTNAME_OVERRIDE=%s", dockerIp))
		cmd.Env = append(cmd.Env, fmt.Sprintf("KUBELET_HOST=%s", dockerIp))
	} else {
		log.Println("using local-up-cluster.sh's defaults for API_HOST_IP, HOSTNAME_OVERRIDE, KUBELET_HOST")
	}

	err = control.FinishRunning(cmd)
	if err != nil {
		return err
	}
	n.kubeConfig = "/var/run/kubernetes/admin.kubeconfig"
	_, err = os.Stat(n.kubeConfig)
	return err
}

func (n localCluster) IsUp() error {
	if n.kubeConfig != "" {
		if err := os.Setenv("KUBECONFIG", n.kubeConfig); err != nil {
			return err
		}
	}
	if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil {
		return err
	}
	if err := os.Setenv("KUBERNETES_PROVIDER", "local"); err != nil {
		return err
	}

	stop := time.Now().Add(*localUpTimeout)
	for {
		script, err := n.getScript("cluster/kubectl.sh")
		if err != nil {
			return err
		}
		nodes, err := kubectlGetNodes(script)
		if err != nil {
			return err
		}
		readyNodes := countReadyNodes(nodes)
		if readyNodes > 0 {
			return nil
		}
		if time.Now().After(stop) {
			break
		} else {
			time.Sleep(5 * time.Second)
		}
	}
	return errors.New("local-up-cluster.sh is not ready")
}

func (n localCluster) DumpClusterLogs(localPath, gcsPath string) error {
	cmd := exec.Command("sudo", "cp", "-r", n.tempDir, localPath)
	return control.FinishRunning(cmd)
}

func (n localCluster) TestSetup() error {
	return nil
}

func (n localCluster) Down() error {
	err := control.FinishRunning(exec.Command("bash", "-c", "docker rm -f $(docker ps -a -q)"))
	if err != nil {
		log.Printf("unable to cleanup containers in docker: %v", err)
	}
	err = control.FinishRunning(exec.Command("pkill", "hyperkube"))
	if err != nil {
		log.Printf("unable to kill hyperkube processes: %v", err)
	}
	err = control.FinishRunning(exec.Command("pkill", "etcd"))
	if err != nil {
		log.Printf("unable to kill etcd: %v", err)
	}
	return nil
}

func (n localCluster) GetClusterCreated(gcpProject string) (time.Time, error) {
	return time.Time{}, errors.New("GetClusterCreated not implemented in localCluster")
}
<|endoftext|>


<commit_before>package multi

import (
	"io"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/gotgo/fw/me"
)

func download(url, filename, folder string, timeout time.Duration) (*FileDownloadOutput, error) {
	output := &FileDownloadOutput{}

	//create file first, so we know we're able to save to disk
	fp := path.Join(folder, filename)
	file, err := os.Create(fp)
	if err != nil {
		return nil, me.Err(err, "create file")
	}
	defer file.Close()

	client := http.Client{
		Timeout: timeout,
	}
	resp, err := client.Get(url)
	if err != nil {
		return nil, me.Err(err, "failed to get URL ")
	}
	defer resp.Body.Close()
	//output.ContentType = resp.ContentType
	//d.Track.Duration("download", started)
	//d.Track.Size("download", resp.ContentLength)

	//save to disk
	if size, err := io.Copy(file, resp.Body); err != nil {
		return nil, me.Err(err, "failed to save downloaded file")
	} else if size == 0 {
		return nil, me.NewErr("downloaded file size was zero on copy")
	} else if size > 0 {
		output.Size = size
		//	d.Track.Size("saved", size)
	}
	file.Sync()
	file.Close()
	output.Path = fp
	return output, nil
}

const defaultTimeout = time.Second * 30

type FileDownloadTask struct {
	Folder  string
	Timeout time.Duration
}

func (d *FileDownloadTask) Run(input interface{}) (interface{}, error) {
	in, ok := input.(*FileDownloadInput)
	if !ok {
		panic("unexpected type")
	}
	timeout := d.Timeout
	if timeout <= 0 {
		timeout = defaultTimeout
	}
	return download(in.Url, in.Filename, d.Folder, timeout)
}

func (d *FileDownloadTask) Name() string {
	return "fileDownload"
}

type FileDownloadInput struct {
	Url      string
	Filename string
}

type FileDownloadOutput struct {
	Path        string
	Size        int64
	ContentType string
}
<commit_msg>sleep<commit_after>package multi

import (
	"io"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/gotgo/fw/me"
)

func download(url, filename, folder string, timeout time.Duration) (*FileDownloadOutput, error) {
	output := &FileDownloadOutput{}

	//create file first, so we know we're able to save to disk
	fp := path.Join(folder, filename)
	file, err := os.Create(fp)
	if err != nil {
		return nil, me.Err(err, "create file")
	}
	defer file.Close()

	client := http.Client{
		Timeout: timeout,
	}
	resp, err := client.Get(url)
	if err != nil {
		return nil, me.Err(err, "failed to get URL ")
	}
	defer resp.Body.Close()
	//output.ContentType = resp.ContentType
	//d.Track.Duration("download", started)
	//d.Track.Size("download", resp.ContentLength)

	//save to disk
	if size, err := io.Copy(file, resp.Body); err != nil {
		return nil, me.Err(err, "failed to save downloaded file")
	} else if size == 0 {
		return nil, me.NewErr("downloaded file size was zero on copy")
	} else if size > 0 {
		output.Size = size
		//	d.Track.Size("saved", size)
	}
	file.Sync()
	file.Close()
	output.Path = fp
	time.Sleep(time.Second)
	return output, nil
}

const defaultTimeout = time.Second * 30

type FileDownloadTask struct {
	Folder  string
	Timeout time.Duration
}

func (d *FileDownloadTask) Run(input interface{}) (interface{}, error) {
	in, ok := input.(*FileDownloadInput)
	if !ok {
		panic("unexpected type")
	}
	timeout := d.Timeout
	if timeout <= 0 {
		timeout = defaultTimeout
	}
	return download(in.Url, in.Filename, d.Folder, timeout)
}

func (d *FileDownloadTask) Name() string {
	return "fileDownload"
}

type FileDownloadInput struct {
	Url      string
	Filename string
}

type FileDownloadOutput struct {
	Path        string
	Size        int64
	ContentType string
}
<|endoftext|>


<commit_before>// Copyright 2016 e-Xpert Solutions SA. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package diff implements diff functions to compare objects.
package diff

import (
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// A Diff represents the changes between two structures.
type Diff map[string]interface{}

// HasChange report whether the Diff contains some changes.
func (d Diff) HasChange() bool {
	return len(d) > 0
}

// PrettyJSON serializes the Diff into JSON in a indented (hence human readable)
// way.
func (d Diff) PrettyJSON() []byte {
	bs, err := json.MarshalIndent(d, "", "   ")
	if err != nil {
		return []byte{}
	}
	return bs
}

// JSON serializes the Diff into JSON in a compact format (no indentation).
func (d Diff) JSON() []byte {
	bs, err := json.Marshal(d)
	if err != nil {
		return []byte{}
	}
	return bs
}

type ChangeType string

// Possible values for a ChangeType.
const (
	AddType ChangeType = "ADD" // addition
	DelType ChangeType = "DEL" // deletion
	ModType ChangeType = "MOD" // modification
)

type Change struct {
	OldVal interface{} `json:"old_value,omitempty"`
	NewVal interface{} `json:"new_value,omitempty"`
	Type   ChangeType  `json:"type,omitempty"`
}

func Compute(x, y interface{}, recursive bool) (Diff, error) {
	vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
	tx, ty := vx.Type(), vy.Type()

	if !tx.AssignableTo(ty) {
		return nil, errors.New("input objects do not share the same type")
	}

	// since x and y share the same type, there is no need to check them both
	if vx.Kind() != reflect.Struct {
		return nil, errors.New("input values are not struct")
	}

	xNumFields := vx.NumField()

	delta := make(Diff)

	for i := 0; i < xNumFields; i++ {
		fx := vx.Field(i)
		typ := tx.Field(i)
		if !isExported(typ.Name) { // skip non-exported fields
			continue
		}
		fy := vy.FieldByName(typ.Name)

		if d := handleValue(fx, fy); d != nil {
			delta[typ.Name] = d
		}
	}

	return delta, nil
}

func handleValue(fx, fy reflect.Value) interface{} {
	switch fx.Kind() {

	case reflect.Struct:
		return handleStruct(fx, fy)
	case reflect.Array, reflect.Slice:
		return handleSlice(fx, fy)
	case reflect.Map:
		return nil

	case reflect.Ptr:
		if fx.IsNil() {
			if fy.IsNil() {
				return nil
			}
			return Change{OldVal: nil, NewVal: fy.Elem().Interface()}
		} else if fy.IsNil() {
			return Change{OldVal: fx.Elem().Interface(), NewVal: nil, Type: ModType}
		}
		return handleValue(fx.Elem(), fy.Elem())

	case reflect.Interface, reflect.Func, reflect.Chan, reflect.Invalid, reflect.UnsafePointer, reflect.Complex64, reflect.Complex128:
		// TODO(gilliek): support complex numbers
		return nil

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ix, iy := fx.Int(), fy.Int()
		if ix != iy {
			return Change{OldVal: ix, NewVal: iy, Type: ModType}
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		uix, uiy := fx.Uint(), fy.Uint()
		if uix != uiy {
			return Change{OldVal: uix, NewVal: uiy, Type: ModType}
		}
	case reflect.Float32, reflect.Float64:
		flx, fly := fx.Float(), fy.Float()
		if flx-fly < 0.000001 {
			return Change{OldVal: flx, NewVal: fly, Type: ModType}
		}
	case reflect.String:
		sx, sy := fx.String(), fy.String()
		if sx != sy {
			return Change{OldVal: sx, NewVal: sy, Type: ModType}
		}
	}

	return nil
}

func handleStruct(fx, fy reflect.Value) interface{} {
	if isFullyNonExportedStruct(fx) {
		if !isEqual(fx, fy) {
			return Change{OldVal: fx.Interface(), NewVal: fy.Interface()}
		}
		return nil
	}

	delta := make(Diff)
	numFields := fx.NumField()
	for i := 0; i < numFields; i++ {
		newFx := fx.Field(i)
		typ := fx.Type().Field(i)
		if !isExported(typ.Name) { // skip non-exported fields
			continue
		}
		newFy := fy.FieldByName(typ.Name)

		if d := handleValue(newFx, newFy); d != nil {
			delta[typ.Name] = d
		}
	}
	if len(delta) > 0 {
		return delta
	}
	return nil
}

func handleSlice(fx, fy reflect.Value) interface{} {
	xLen, yLen := fx.Len(), fy.Len()

	changes := make(map[string]Change)
	if xLen == 0 {
		if yLen == 0 {
			return nil
		}
		for i := 0; i < yLen; i++ {
			changes[strconv.Itoa(i)] = Change{NewVal: fy.Index(i).Interface(), Type: AddType}
		}
	} else if yLen == 0 {
		for i := 0; i < xLen; i++ {
			changes[strconv.Itoa(i)] = Change{OldVal: fx.Index(i).Interface(), Type: DelType}
		}
	} else {
		var maxLen int
		if xLen > yLen {
			maxLen = yLen
			for i := yLen; i < xLen; i++ {
				changes[strconv.Itoa(i)] = Change{OldVal: fx.Index(i).Interface(), Type: DelType}
			}
		} else if xLen < yLen {
			maxLen = xLen
			for i := xLen; i < yLen; i++ {
				changes[strconv.Itoa(i)] = Change{NewVal: fy.Index(i).Interface(), Type: AddType}
			}
		}
		for i := 0; i < maxLen; i++ {
			if d := handleValue(fx.Index(i), fy.Index(i)); d != nil {
				changes[strconv.Itoa(i)] = Change{NewVal: d, Type: ModType}
			}
		}
	}
	if len(changes) > 0 {
		return changes
	}
	return nil
}

func isExported(fieldName string) bool {
	if fieldName == "" {
		return false
	}
	firstLetter := string(fieldName[0])
	return firstLetter != strings.ToLower(firstLetter)
}

func isFullyNonExportedStruct(s reflect.Value) bool {
	if s.Kind() != reflect.Struct {
		return false
	}
	numFields := s.NumField()
	for i := 0; i < numFields; i++ {
		typ := s.Type().Field(i)
		if isExported(typ.Name) {
			return false
		}
	}
	return true
}

func isEqual(x, y reflect.Value) bool {
	return fmt.Sprint(x.Interface()) == fmt.Sprint(y.Interface())
}
<commit_msg>diff: fix missing length for equal size slices<commit_after>// Copyright 2016 e-Xpert Solutions SA. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package diff implements diff functions to compare objects.
package diff

import (
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// A Diff represents the changes between two structures.
type Diff map[string]interface{}

// HasChange report whether the Diff contains some changes.
func (d Diff) HasChange() bool {
	return len(d) > 0
}

// PrettyJSON serializes the Diff into JSON in a indented (hence human readable)
// way.
func (d Diff) PrettyJSON() []byte {
	bs, err := json.MarshalIndent(d, "", "   ")
	if err != nil {
		return []byte{}
	}
	return bs
}

// JSON serializes the Diff into JSON in a compact format (no indentation).
func (d Diff) JSON() []byte {
	bs, err := json.Marshal(d)
	if err != nil {
		return []byte{}
	}
	return bs
}

type ChangeType string

// Possible values for a ChangeType.
const (
	AddType ChangeType = "ADD" // addition
	DelType ChangeType = "DEL" // deletion
	ModType ChangeType = "MOD" // modification
)

type Change struct {
	OldVal interface{} `json:"old_value,omitempty"`
	NewVal interface{} `json:"new_value,omitempty"`
	Type   ChangeType  `json:"type,omitempty"`
}

func Compute(x, y interface{}, recursive bool) (Diff, error) {
	vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
	tx, ty := vx.Type(), vy.Type()

	if !tx.AssignableTo(ty) {
		return nil, errors.New("input objects do not share the same type")
	}

	// since x and y share the same type, there is no need to check them both
	if vx.Kind() != reflect.Struct {
		return nil, errors.New("input values are not struct")
	}

	xNumFields := vx.NumField()

	delta := make(Diff)

	for i := 0; i < xNumFields; i++ {
		fx := vx.Field(i)
		typ := tx.Field(i)
		if !isExported(typ.Name) { // skip non-exported fields
			continue
		}
		fy := vy.FieldByName(typ.Name)

		if d := handleValue(fx, fy); d != nil {
			delta[typ.Name] = d
		}
	}

	return delta, nil
}

func handleValue(fx, fy reflect.Value) interface{} {
	switch fx.Kind() {

	case reflect.Struct:
		return handleStruct(fx, fy)
	case reflect.Array, reflect.Slice:
		return handleSlice(fx, fy)
	case reflect.Map:
		return nil

	case reflect.Ptr:
		if fx.IsNil() {
			if fy.IsNil() {
				return nil
			}
			return Change{OldVal: nil, NewVal: fy.Elem().Interface()}
		} else if fy.IsNil() {
			return Change{OldVal: fx.Elem().Interface(), NewVal: nil, Type: ModType}
		}
		return handleValue(fx.Elem(), fy.Elem())

	case reflect.Interface, reflect.Func, reflect.Chan, reflect.Invalid, reflect.UnsafePointer, reflect.Complex64, reflect.Complex128:
		// TODO(gilliek): support complex numbers
		return nil

	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		ix, iy := fx.Int(), fy.Int()
		if ix != iy {
			return Change{OldVal: ix, NewVal: iy, Type: ModType}
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		uix, uiy := fx.Uint(), fy.Uint()
		if uix != uiy {
			return Change{OldVal: uix, NewVal: uiy, Type: ModType}
		}
	case reflect.Float32, reflect.Float64:
		flx, fly := fx.Float(), fy.Float()
		if flx-fly < 0.000001 {
			return Change{OldVal: flx, NewVal: fly, Type: ModType}
		}
	case reflect.String:
		sx, sy := fx.String(), fy.String()
		if sx != sy {
			return Change{OldVal: sx, NewVal: sy, Type: ModType}
		}
	}

	return nil
}

func handleStruct(fx, fy reflect.Value) interface{} {
	if isFullyNonExportedStruct(fx) {
		if !isEqual(fx, fy) {
			return Change{OldVal: fx.Interface(), NewVal: fy.Interface()}
		}
		return nil
	}

	delta := make(Diff)
	numFields := fx.NumField()
	for i := 0; i < numFields; i++ {
		newFx := fx.Field(i)
		typ := fx.Type().Field(i)
		if !isExported(typ.Name) { // skip non-exported fields
			continue
		}
		newFy := fy.FieldByName(typ.Name)

		if d := handleValue(newFx, newFy); d != nil {
			delta[typ.Name] = d
		}
	}
	if len(delta) > 0 {
		return delta
	}
	return nil
}

func handleSlice(fx, fy reflect.Value) interface{} {
	xLen, yLen := fx.Len(), fy.Len()

	changes := make(map[string]Change)
	if xLen == 0 {
		if yLen == 0 {
			return nil
		}
		for i := 0; i < yLen; i++ {
			changes[strconv.Itoa(i)] = Change{NewVal: fy.Index(i).Interface(), Type: AddType}
		}
	} else if yLen == 0 {
		for i := 0; i < xLen; i++ {
			changes[strconv.Itoa(i)] = Change{OldVal: fx.Index(i).Interface(), Type: DelType}
		}
	} else {
		var maxLen int
		if xLen > yLen {
			maxLen = yLen
			for i := yLen; i < xLen; i++ {
				changes[strconv.Itoa(i)] = Change{OldVal: fx.Index(i).Interface(), Type: DelType}
			}
		} else if xLen < yLen {
			maxLen = xLen
			for i := xLen; i < yLen; i++ {
				changes[strconv.Itoa(i)] = Change{NewVal: fy.Index(i).Interface(), Type: AddType}
			}
		} else {
			maxLen = xLen
		}
		for i := 0; i < maxLen; i++ {
			if d := handleValue(fx.Index(i), fy.Index(i)); d != nil {
				changes[strconv.Itoa(i)] = Change{NewVal: d, Type: ModType}
			}
		}
	}
	if len(changes) > 0 {
		return changes
	}
	return nil
}

func isExported(fieldName string) bool {
	if fieldName == "" {
		return false
	}
	firstLetter := string(fieldName[0])
	return firstLetter != strings.ToLower(firstLetter)
}

func isFullyNonExportedStruct(s reflect.Value) bool {
	if s.Kind() != reflect.Struct {
		return false
	}
	numFields := s.NumField()
	for i := 0; i < numFields; i++ {
		typ := s.Type().Field(i)
		if isExported(typ.Name) {
			return false
		}
	}
	return true
}

func isEqual(x, y reflect.Value) bool {
	return fmt.Sprint(x.Interface()) == fmt.Sprint(y.Interface())
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"container/list"
	"fmt"
	"os"

	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// A type that manages read and read/write leases for anonymous temporary files.
//
// Safe for concurrent access. Must be created with NewFileLeaser.
type FileLeaser struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	dir   string
	limit int64

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A lock that guards the mutable state in this struct, which must not be
	// held for any blocking operation.
	//
	// Lock ordering
	// -------------
	//
	// Define our strict partial order < as follows:
	//
	//  1. For any read/write lease W, W < leaser.
	//  2. For any read lease R, R < leaser.
	//  3. For any read/write lease W and read lease R, W < R.
	//
	// In other words: read/write before read before leaser, and never hold two
	// locks from the same category together.
	mu syncutil.InvariantMutex

	// The current estimated total size of outstanding read/write leases. This is
	// only an estimate because we can't synchronize its update with a call to
	// the wrapped file to e.g. write or truncate.
	readWriteOutstanding int64

	// All outstanding read leases, ordered by recency of use.
	//
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	readLeases list.List

	// The sum of all outstanding read lease sizes.
	//
	// INVARIANT: Equal to the sum over readLeases sizes.
	// INVARIANT: 0 <= readOutstanding
	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	readOutstanding int64

	// Index of read leases by pointer.
	//
	// INVARIANT: Is an index of exactly the elements of readLeases
	readLeasesIndex map[*readLease]*list.Element
}

// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in bytes below the
// given limit. If dir is empty, the system default will be used.
//
// Usage may exceed the given limit if there are read/write leases whose total
// size exceeds the limit, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitBytes int64) (fl *FileLeaser) {
	fl = &FileLeaser{
		dir:   dir,
		limit: limitBytes,
	}

	fl.mu = syncutil.NewInvariantMutex(fl.checkInvariants)

	return
}

// Create a new anonymous file, and return a read/write lease for it. The
// read/write lease will pin resources until rwl.Downgrade is called. It need
// not be called if the process is exiting.
func (fl *FileLeaser) NewFile() (rwl ReadWriteLease, err error) {
	// Create an anonymous file.
	f, err := fsutil.AnonymousFile(fl.dir)
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Wrap a lease around it.
	rwl = newReadWriteLease(fl, f)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func maxInt64(a int64, b int64) int64 {
	if a > b {
		return a
	}

	return b
}

// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) checkInvariants() {
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		if rl.revoked() {
			panic("Found revoked read lease")
		}
	}

	// INVARIANT: Equal to the sum over readLeases sizes.
	var sum int64
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		sum += rl.Size()
	}

	if fl.readOutstanding != sum {
		panic(fmt.Sprintf(
			"readOutstanding mismatch: %v vs. %v",
			fl.readOutstanding,
			sum))
	}

	// INVARIANT: 0 <= readOutstanding
	if !(0 <= fl.readOutstanding) {
		panic(fmt.Sprintf("Unexpected readOutstanding: %v", fl.readOutstanding))
	}

	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	if !(fl.readOutstanding <= maxInt64(0, fl.limit-fl.readWriteOutstanding)) {
		panic(fmt.Sprintf(
			"Unexpected readOutstanding: %v. limit: %v, readWriteOutstanding: %v",
			fl.readOutstanding,
			fl.limit,
			fl.readWriteOutstanding))
	}

	// INVARIANT: Is an index of exactly the elements of readLeases
	if len(fl.readLeasesIndex) != fl.readLeases.Len() {
		panic(fmt.Sprintf(
			"readLeasesIndex length mismatch: %v vs. %v",
			len(fl.readLeasesIndex),
			fl.readLeases.Len()))
	}

	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		if fl.readLeasesIndex[e.Value.(*readLease)] != e {
			panic("Mismatch in readLeasesIndex")
		}
	}
}

// Add the supplied delta to the leaser's view of outstanding read/write lease
// bytes, then revoke read leases until we're under limit or we run out of
// leases to revoke.
//
// Called by readWriteLease while holding its lock.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) addReadWriteByteDelta(delta int64) {
	// TODO(jacobsa): When evicting, repeatedly:
	// 1. Find least recently used read lease.
	// 2. Drop leaser lock.
	// 3. Acquire read lease lock.
	// 4. Reacquire leaser lock.
	// 5. If under limit now, drop both locks and return.
	// 6. If lease already evicted, drop its lock and go to #1.
	// 7. Evict lease, drop both locks. If still above limit, start over.
	panic("TODO")
}

// Downgrade the supplied read/write lease, given its current size and the
// underlying file.
//
// Called by readWriteLease with its lock held.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) downgrade(
	rwl *readWriteLease,
	size int64,
	file *os.File) (rl ReadLease) {
	rl = newReadLease(size, fl, file)

	// TODO(jacobsa): Update fl's state, too. Don't forget to take the lock.

	return
}

// Upgrade the supplied read lease, given its size and the underlying file.
//
// Called by readLease with its lock held.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) upgrade(
	rl *readLease,
	size int64,
	file *os.File) (rwl ReadWriteLease) {
	// TODO(jacobsa): Remove read lease from the map, our size counter, etc. Then
	// drop the lock.
	panic("TODO")

	// TODO(jacobsa): This should take a size parameter, telling the read/write
	// lease that we already know its initial size.
	rwl = newReadWriteLease(fl, file)

	return
}

// Called by the read lease when the user wants to manually revoke it.
//
// LOCKS_REQUIRED(rl)
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) revokeVoluntarily(rl *readLease) {
	// TODO(jacobsa): Acquire file leaser lock, update file leaser state, call
	// rl.destroy. Later we can factor all but acquiring the lock out into a
	// separate helper that is shared by the revoke-for-capacity logic, which
	// will already have the lock.
	panic("TODO")
}
<commit_msg>FileLeaser.downgrade<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

import (
	"container/list"
	"fmt"
	"os"

	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// A type that manages read and read/write leases for anonymous temporary files.
//
// Safe for concurrent access. Must be created with NewFileLeaser.
type FileLeaser struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	dir   string
	limit int64

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A lock that guards the mutable state in this struct, which must not be
	// held for any blocking operation.
	//
	// Lock ordering
	// -------------
	//
	// Define our strict partial order < as follows:
	//
	//  1. For any read/write lease W, W < leaser.
	//  2. For any read lease R, R < leaser.
	//  3. For any read/write lease W and read lease R, W < R.
	//
	// In other words: read/write before read before leaser, and never hold two
	// locks from the same category together.
	mu syncutil.InvariantMutex

	// The current estimated total size of outstanding read/write leases. This is
	// only an estimate because we can't synchronize its update with a call to
	// the wrapped file to e.g. write or truncate.
	readWriteOutstanding int64

	// All outstanding read leases, ordered by recency of use.
	//
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	readLeases list.List

	// The sum of all outstanding read lease sizes.
	//
	// INVARIANT: Equal to the sum over readLeases sizes.
	// INVARIANT: 0 <= readOutstanding
	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	readOutstanding int64

	// Index of read leases by pointer.
	//
	// INVARIANT: Is an index of exactly the elements of readLeases
	readLeasesIndex map[*readLease]*list.Element
}

// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in bytes below the
// given limit. If dir is empty, the system default will be used.
//
// Usage may exceed the given limit if there are read/write leases whose total
// size exceeds the limit, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitBytes int64) (fl *FileLeaser) {
	fl = &FileLeaser{
		dir:   dir,
		limit: limitBytes,
	}

	fl.mu = syncutil.NewInvariantMutex(fl.checkInvariants)

	return
}

// Create a new anonymous file, and return a read/write lease for it. The
// read/write lease will pin resources until rwl.Downgrade is called. It need
// not be called if the process is exiting.
func (fl *FileLeaser) NewFile() (rwl ReadWriteLease, err error) {
	// Create an anonymous file.
	f, err := fsutil.AnonymousFile(fl.dir)
	if err != nil {
		err = fmt.Errorf("AnonymousFile: %v", err)
		return
	}

	// Wrap a lease around it.
	rwl = newReadWriteLease(fl, f)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func maxInt64(a int64, b int64) int64 {
	if a > b {
		return a
	}

	return b
}

// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) checkInvariants() {
	// INVARIANT: Each element is of type *readLease
	// INVARIANT: No element has been revoked.
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		if rl.revoked() {
			panic("Found revoked read lease")
		}
	}

	// INVARIANT: Equal to the sum over readLeases sizes.
	var sum int64
	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		rl := e.Value.(*readLease)
		sum += rl.Size()
	}

	if fl.readOutstanding != sum {
		panic(fmt.Sprintf(
			"readOutstanding mismatch: %v vs. %v",
			fl.readOutstanding,
			sum))
	}

	// INVARIANT: 0 <= readOutstanding
	if !(0 <= fl.readOutstanding) {
		panic(fmt.Sprintf("Unexpected readOutstanding: %v", fl.readOutstanding))
	}

	// INVARIANT: readOutstanding <= max(0, limit - readWriteOutstanding)
	if !(fl.readOutstanding <= maxInt64(0, fl.limit-fl.readWriteOutstanding)) {
		panic(fmt.Sprintf(
			"Unexpected readOutstanding: %v. limit: %v, readWriteOutstanding: %v",
			fl.readOutstanding,
			fl.limit,
			fl.readWriteOutstanding))
	}

	// INVARIANT: Is an index of exactly the elements of readLeases
	if len(fl.readLeasesIndex) != fl.readLeases.Len() {
		panic(fmt.Sprintf(
			"readLeasesIndex length mismatch: %v vs. %v",
			len(fl.readLeasesIndex),
			fl.readLeases.Len()))
	}

	for e := fl.readLeases.Front(); e != nil; e = e.Next() {
		if fl.readLeasesIndex[e.Value.(*readLease)] != e {
			panic("Mismatch in readLeasesIndex")
		}
	}
}

// Add the supplied delta to the leaser's view of outstanding read/write lease
// bytes, then revoke read leases until we're under limit or we run out of
// leases to revoke.
//
// Called by readWriteLease while holding its lock.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) addReadWriteByteDelta(delta int64) {
	// TODO(jacobsa): When evicting, repeatedly:
	// 1. Find least recently used read lease.
	// 2. Drop leaser lock.
	// 3. Acquire read lease lock.
	// 4. Reacquire leaser lock.
	// 5. If under limit now, drop both locks and return.
	// 6. If lease already evicted, drop its lock and go to #1.
	// 7. Evict lease, drop both locks. If still above limit, start over.
	panic("TODO")
}

// Revoke read leases until we're under limit or we run out of things to revoke.
//
// LOCKS_REQUIRED(fl.mu)
func (fl *FileLeaser) evict() {
	panic("TODO")
}

// Downgrade the supplied read/write lease, given its current size and the
// underlying file.
//
// Called by readWriteLease with its lock held.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) downgrade(
	rwl *readWriteLease,
	size int64,
	file *os.File) (rl ReadLease) {
	// Create the read lease.
	rlTyped := newReadLease(size, fl, file)
	rl = rlTyped

	// Update the leaser's state, noting the new read lease and that the
	// read/write lease has gone away.
	fl.mu.Lock()
	defer fl.mu.Unlock()

	fl.readWriteOutstanding -= size
	fl.readOutstanding += size

	e := fl.readLeases.PushFront(rl)
	fl.readLeasesIndex[rlTyped] = e

	// Ensure that we're not now over capacity.
	fl.evict()

	return
}

// Upgrade the supplied read lease, given its size and the underlying file.
//
// Called by readLease with its lock held.
//
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) upgrade(
	rl *readLease,
	size int64,
	file *os.File) (rwl ReadWriteLease) {
	// TODO(jacobsa): Remove read lease from the map, our size counter, etc. Then
	// drop the lock.
	panic("TODO")

	// TODO(jacobsa): This should take a size parameter, telling the read/write
	// lease that we already know its initial size.
	rwl = newReadWriteLease(fl, file)

	return
}

// Called by the read lease when the user wants to manually revoke it.
//
// LOCKS_REQUIRED(rl)
// LOCKS_EXCLUDED(fl.mu)
func (fl *FileLeaser) revokeVoluntarily(rl *readLease) {
	// TODO(jacobsa): Acquire file leaser lock, update file leaser state, call
	// rl.destroy. Later we can factor all but acquiring the lock out into a
	// separate helper that is shared by the revoke-for-capacity logic, which
	// will already have the lock.
	panic("TODO")
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

// A type that manages read and read/write leases for anonymous temporary files.
//
// Safe for concurrent access. Must be created with NewFileLeaser.
type FileLeaser struct {
}

// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in bytes below the
// given limit. If dir is empty, the system default will be used.
//
// Usage may exceed the given limit if there are read/write leases whose total
// size exceeds the limit, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitBytes int64) (fl *FileLeaser) {
	panic("TODO")
}

// Create a new anonymous file, and return a read/write lease for it. The
// read/write lease will pin resources until rwl.Downgrade is called. It need
// not be called if the process is exiting.
func (fl *FileLeaser) New() (rwl ReadWriteLease) {
	panic("TODO")
}
<commit_msg>NewFileLeaser<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lease

// A type that manages read and read/write leases for anonymous temporary files.
//
// Safe for concurrent access. Must be created with NewFileLeaser.
type FileLeaser struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	dir   string
	limit int64
}

// Create a new file leaser that uses the supplied directory for temporary
// files (before unlinking them) and attempts to keep usage in bytes below the
// given limit. If dir is empty, the system default will be used.
//
// Usage may exceed the given limit if there are read/write leases whose total
// size exceeds the limit, since such leases cannot be revoked.
func NewFileLeaser(
	dir string,
	limitBytes int64) (fl *FileLeaser) {
	fl = &FileLeaser{
		dir:   dir,
		limit: limitBytes,
	}

	return
}

// Create a new anonymous file, and return a read/write lease for it. The
// read/write lease will pin resources until rwl.Downgrade is called. It need
// not be called if the process is exiting.
func (fl *FileLeaser) New() (rwl ReadWriteLease) {
	panic("TODO")
}
<|endoftext|>


<commit_before>// +build windows

package main

import (
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/sys/windows/svc"

	"github.com/StackExchange/wmi"
	"github.com/martinlindhe/wmi_exporter/collector"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"gopkg.in/alecthomas/kingpin.v2"
)

// WmiCollector implements the prometheus.Collector interface.
type WmiCollector struct {
	collectors map[string]collector.Collector
}

const (
	defaultCollectors            = "cpu,cs,logical_disk,net,os,service,system,textfile"
	defaultCollectorsPlaceholder = "[defaults]"
	serviceName                  = "wmi_exporter"
)

var (
	scrapeDurationDesc = prometheus.NewDesc(
		prometheus.BuildFQName(collector.Namespace, "exporter", "collector_duration_seconds"),
		"wmi_exporter: Duration of a collection.",
		[]string{"collector"},
		nil,
	)
	scrapeSuccessDesc = prometheus.NewDesc(
		prometheus.BuildFQName(collector.Namespace, "exporter", "collector_success"),
		"wmi_exporter: Whether the collector was successful.",
		[]string{"collector"},
		nil,
	)

	// This can be removed when client_golang exposes this on Windows
	// (See https://github.com/prometheus/client_golang/issues/376)
	startTime     = float64(time.Now().Unix())
	startTimeDesc = prometheus.NewDesc(
		"process_start_time_seconds",
		"Start time of the process since unix epoch in seconds.",
		nil,
		nil,
	)
)

// Describe sends all the descriptors of the collectors included to
// the provided channel.
func (coll WmiCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- scrapeDurationDesc
	ch <- scrapeSuccessDesc
}

// Collect sends the collected metrics from each of the collectors to
// prometheus. Collect could be called several times concurrently
// and thus its run is protected by a single mutex.
func (coll WmiCollector) Collect(ch chan<- prometheus.Metric) {
	scrapeContext, err := collector.PrepareScrapeContext()
	if err != nil {
		ch <- prometheus.NewInvalidMetric(scrapeSuccessDesc, fmt.Errorf("failed to prepare scrape: %v", err))
		return
	}

	wg := sync.WaitGroup{}
	wg.Add(len(coll.collectors))
	for name, c := range coll.collectors {
		go func(name string, c collector.Collector) {
			execute(name, c, scrapeContext, ch)
			wg.Done()
		}(name, c)
	}

	ch <- prometheus.MustNewConstMetric(
		startTimeDesc,
		prometheus.CounterValue,
		startTime,
	)
	wg.Wait()
}

func filterAvailableCollectors(collectors string) string {
	var availableCollectors []string
	for _, c := range strings.Split(collectors, ",") {
		_, ok := collector.Factories[c]
		if ok {
			availableCollectors = append(availableCollectors, c)
		}
	}
	return strings.Join(availableCollectors, ",")
}

func execute(name string, c collector.Collector, ctx *collector.ScrapeContext, ch chan<- prometheus.Metric) {
	begin := time.Now()
	err := c.Collect(ctx, ch)
	duration := time.Since(begin)
	var success float64

	if err != nil {
		log.Errorf("collector %s failed after %fs: %s", name, duration.Seconds(), err)
		success = 0
	} else {
		log.Debugf("collector %s succeeded after %fs.", name, duration.Seconds())
		success = 1
	}
	ch <- prometheus.MustNewConstMetric(
		scrapeDurationDesc,
		prometheus.GaugeValue,
		duration.Seconds(),
		name,
	)
	ch <- prometheus.MustNewConstMetric(
		scrapeSuccessDesc,
		prometheus.GaugeValue,
		success,
		name,
	)
}

func expandEnabledCollectors(enabled string) []string {
	expanded := strings.Replace(enabled, defaultCollectorsPlaceholder, defaultCollectors, -1)
	separated := strings.Split(expanded, ",")
	unique := map[string]bool{}
	for _, s := range separated {
		if s != "" {
			unique[s] = true
		}
	}
	result := make([]string, 0, len(unique))
	for s := range unique {
		result = append(result, s)
	}
	return result
}

func loadCollectors(list string) (map[string]collector.Collector, error) {
	collectors := map[string]collector.Collector{}
	enabled := expandEnabledCollectors(list)

	for _, name := range enabled {
		fn, ok := collector.Factories[name]
		if !ok {
			return nil, fmt.Errorf("collector '%s' not available", name)
		}
		c, err := fn()
		if err != nil {
			return nil, err
		}
		collectors[name] = c
	}
	return collectors, nil
}

func init() {
	prometheus.MustRegister(version.NewCollector("wmi_exporter"))
}

func initWbem() {
	// This initialization prevents a memory leak on WMF 5+. See
	// https://github.com/martinlindhe/wmi_exporter/issues/77 and linked issues
	// for details.
	log.Debugf("Initializing SWbemServices")
	s, err := wmi.InitializeSWbemServices(wmi.DefaultClient)
	if err != nil {
		log.Fatal(err)
	}
	wmi.DefaultClient.AllowMissingFields = true
	wmi.DefaultClient.SWbemServicesClient = s
}

func main() {
	var (
		listenAddress = kingpin.Flag(
			"telemetry.addr",
			"host:port for WMI exporter.",
		).Default(":9182").String()
		metricsPath = kingpin.Flag(
			"telemetry.path",
			"URL path for surfacing collected metrics.",
		).Default("/metrics").String()
		enabledCollectors = kingpin.Flag(
			"collectors.enabled",
			"Comma-separated list of collectors to use. Use '[defaults]' as a placeholder for all the collectors enabled by default.").
			Default(filterAvailableCollectors(defaultCollectors)).String()
		printCollectors = kingpin.Flag(
			"collectors.print",
			"If true, print available collectors and exit.",
		).Bool()
	)

	log.AddFlags(kingpin.CommandLine)
	kingpin.Version(version.Print("wmi_exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()

	if *printCollectors {
		collectorNames := make(sort.StringSlice, 0, len(collector.Factories))
		for n := range collector.Factories {
			collectorNames = append(collectorNames, n)
		}
		collectorNames.Sort()
		fmt.Printf("Available collectors:\n")
		for _, n := range collectorNames {
			fmt.Printf(" - %s\n", n)
		}
		return
	}

	initWbem()

	isInteractive, err := svc.IsAnInteractiveSession()
	if err != nil {
		log.Fatal(err)
	}

	stopCh := make(chan bool)
	if !isInteractive {
		go func() {
			err = svc.Run(serviceName, &wmiExporterService{stopCh: stopCh})
			if err != nil {
				log.Errorf("Failed to start service: %v", err)
			}
		}()
	}

	collectors, err := loadCollectors(*enabledCollectors)
	if err != nil {
		log.Fatalf("Couldn't load collectors: %s", err)
	}

	log.Infof("Enabled collectors: %v", strings.Join(keys(collectors), ", "))

	nodeCollector := WmiCollector{collectors: collectors}
	prometheus.MustRegister(nodeCollector)

	http.Handle(*metricsPath, promhttp.Handler())
	http.HandleFunc("/health", healthCheck)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Infoln("Starting WMI exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	go func() {
		log.Infoln("Starting server on", *listenAddress)
		log.Fatalf("cannot start WMI exporter: %s", http.ListenAndServe(*listenAddress, nil))
	}()

	for {
		if <-stopCh {
			log.Info("Shutting down WMI exporter")
			break
		}
	}
}

func healthCheck(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_, err := fmt.Fprintln(w, `{"status":"ok"}`)
	if err != nil {
		log.Debugf("Failed to write to stream: %v", err)
	}
}

func keys(m map[string]collector.Collector) []string {
	ret := make([]string, 0, len(m))
	for key := range m {
		ret = append(ret, key)
	}
	return ret
}

type wmiExporterService struct {
	stopCh chan<- bool
}

func (s *wmiExporterService) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
	changes <- svc.Status{State: svc.StartPending}
	changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
loop:
	for {
		select {
		case c := <-r:
			switch c.Cmd {
			case svc.Interrogate:
				changes <- c.CurrentStatus
			case svc.Stop, svc.Shutdown:
				s.stopCh <- true
				break loop
			default:
				log.Error(fmt.Sprintf("unexpected control request #%d", c))
			}
		}
	}
	changes <- svc.Status{State: svc.StopPending}
	return
}
<commit_msg>Abort scrapes after configurable timeout<commit_after>// +build windows

package main

import (
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/sys/windows/svc"

	"github.com/StackExchange/wmi"
	"github.com/martinlindhe/wmi_exporter/collector"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"gopkg.in/alecthomas/kingpin.v2"
)

// WmiCollector implements the prometheus.Collector interface.
type WmiCollector struct {
	maxScrapeDuration time.Duration
	collectors        map[string]collector.Collector
}

const (
	defaultCollectors            = "cpu,cs,logical_disk,net,os,service,system,textfile"
	defaultCollectorsPlaceholder = "[defaults]"
	serviceName                  = "wmi_exporter"
)

var (
	scrapeDurationDesc = prometheus.NewDesc(
		prometheus.BuildFQName(collector.Namespace, "exporter", "collector_duration_seconds"),
		"wmi_exporter: Duration of a collection.",
		[]string{"collector"},
		nil,
	)
	scrapeSuccessDesc = prometheus.NewDesc(
		prometheus.BuildFQName(collector.Namespace, "exporter", "collector_success"),
		"wmi_exporter: Whether the collector was successful.",
		[]string{"collector"},
		nil,
	)
	scrapeTimeoutDesc = prometheus.NewDesc(
		prometheus.BuildFQName(collector.Namespace, "exporter", "collector_timeout"),
		"wmi_exporter: Whether the collector timed out.",
		[]string{"collector"},
		nil,
	)

	// This can be removed when client_golang exposes this on Windows
	// (See https://github.com/prometheus/client_golang/issues/376)
	startTime     = float64(time.Now().Unix())
	startTimeDesc = prometheus.NewDesc(
		"process_start_time_seconds",
		"Start time of the process since unix epoch in seconds.",
		nil,
		nil,
	)
)

// Describe sends all the descriptors of the collectors included to
// the provided channel.
func (coll WmiCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- scrapeDurationDesc
	ch <- scrapeSuccessDesc
}

// Collect sends the collected metrics from each of the collectors to
// prometheus.
func (coll WmiCollector) Collect(ch chan<- prometheus.Metric) {
	scrapeContext, err := collector.PrepareScrapeContext()
	if err != nil {
		ch <- prometheus.NewInvalidMetric(scrapeSuccessDesc, fmt.Errorf("failed to prepare scrape: %v", err))
		return
	}

	remainingCollectors := make(map[string]bool)
	for name := range coll.collectors {
		remainingCollectors[name] = true
	}

	metricsBuffer := make(chan prometheus.Metric)
	allDone := make(chan struct{})
	stopped := false
	go func() {
		for {
			select {
			case m := <-metricsBuffer:
				if !stopped {
					ch <- m
				}
			case <-allDone:
				return
			}
		}
	}()

	wg := sync.WaitGroup{}
	wg.Add(len(coll.collectors))
	go func() {
		wg.Wait()
		close(allDone)
		close(metricsBuffer)
	}()

	for name, c := range coll.collectors {
		go func(name string, c collector.Collector) {
			execute(name, c, scrapeContext, metricsBuffer)
			wg.Done()
			delete(remainingCollectors, name)
		}(name, c)
	}

	ch <- prometheus.MustNewConstMetric(
		startTimeDesc,
		prometheus.CounterValue,
		startTime,
	)

	select {
	case <-allDone:
		stopped = true
		return
	case <-time.After(coll.maxScrapeDuration):
		stopped = true
		remainingCollectorNames := make([]string, 0, len(remainingCollectors))
		for rc := range remainingCollectors {
			remainingCollectorNames = append(remainingCollectorNames, rc)
		}
		log.Warn("Collection timed out, still waiting for ", remainingCollectorNames)
		for name := range remainingCollectors {
			ch <- prometheus.MustNewConstMetric(
				scrapeSuccessDesc,
				prometheus.GaugeValue,
				0.0,
				name,
			)
			ch <- prometheus.MustNewConstMetric(
				scrapeTimeoutDesc,
				prometheus.GaugeValue,
				1.0,
				name,
			)
		}
	}
}

func filterAvailableCollectors(collectors string) string {
	var availableCollectors []string
	for _, c := range strings.Split(collectors, ",") {
		_, ok := collector.Factories[c]
		if ok {
			availableCollectors = append(availableCollectors, c)
		}
	}
	return strings.Join(availableCollectors, ",")
}

func execute(name string, c collector.Collector, ctx *collector.ScrapeContext, ch chan<- prometheus.Metric) {
	begin := time.Now()
	err := c.Collect(ctx, ch)
	duration := time.Since(begin)
	var success float64

	if err != nil {
		log.Errorf("collector %s failed after %fs: %s", name, duration.Seconds(), err)
		success = 0
	} else {
		log.Debugf("collector %s succeeded after %fs.", name, duration.Seconds())
		success = 1
	}
	ch <- prometheus.MustNewConstMetric(
		scrapeDurationDesc,
		prometheus.GaugeValue,
		duration.Seconds(),
		name,
	)
	ch <- prometheus.MustNewConstMetric(
		scrapeSuccessDesc,
		prometheus.GaugeValue,
		success,
		name,
	)
	ch <- prometheus.MustNewConstMetric(
		scrapeTimeoutDesc,
		prometheus.GaugeValue,
		0.0,
		name,
	)
}

func expandEnabledCollectors(enabled string) []string {
	expanded := strings.Replace(enabled, defaultCollectorsPlaceholder, defaultCollectors, -1)
	separated := strings.Split(expanded, ",")
	unique := map[string]bool{}
	for _, s := range separated {
		if s != "" {
			unique[s] = true
		}
	}
	result := make([]string, 0, len(unique))
	for s := range unique {
		result = append(result, s)
	}
	return result
}

func loadCollectors(list string) (map[string]collector.Collector, error) {
	collectors := map[string]collector.Collector{}
	enabled := expandEnabledCollectors(list)

	for _, name := range enabled {
		fn, ok := collector.Factories[name]
		if !ok {
			return nil, fmt.Errorf("collector '%s' not available", name)
		}
		c, err := fn()
		if err != nil {
			return nil, err
		}
		collectors[name] = c
	}
	return collectors, nil
}

func init() {
	prometheus.MustRegister(version.NewCollector("wmi_exporter"))
}

func initWbem() {
	// This initialization prevents a memory leak on WMF 5+. See
	// https://github.com/martinlindhe/wmi_exporter/issues/77 and linked issues
	// for details.
	log.Debugf("Initializing SWbemServices")
	s, err := wmi.InitializeSWbemServices(wmi.DefaultClient)
	if err != nil {
		log.Fatal(err)
	}
	wmi.DefaultClient.AllowMissingFields = true
	wmi.DefaultClient.SWbemServicesClient = s
}

func main() {
	var (
		listenAddress = kingpin.Flag(
			"telemetry.addr",
			"host:port for WMI exporter.",
		).Default(":9182").String()
		metricsPath = kingpin.Flag(
			"telemetry.path",
			"URL path for surfacing collected metrics.",
		).Default("/metrics").String()
		enabledCollectors = kingpin.Flag(
			"collectors.enabled",
			"Comma-separated list of collectors to use. Use '[defaults]' as a placeholder for all the collectors enabled by default.").
			Default(filterAvailableCollectors(defaultCollectors)).String()
		printCollectors = kingpin.Flag(
			"collectors.print",
			"If true, print available collectors and exit.",
		).Bool()
		maxScrapeDuration = kingpin.Flag(
			"scrape.max-duration",
			"Time after which collectors are aborted during a scrape",
		).Default("30s").Duration()
	)

	log.AddFlags(kingpin.CommandLine)
	kingpin.Version(version.Print("wmi_exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()

	if *printCollectors {
		collectorNames := make(sort.StringSlice, 0, len(collector.Factories))
		for n := range collector.Factories {
			collectorNames = append(collectorNames, n)
		}
		collectorNames.Sort()
		fmt.Printf("Available collectors:\n")
		for _, n := range collectorNames {
			fmt.Printf(" - %s\n", n)
		}
		return
	}

	initWbem()

	isInteractive, err := svc.IsAnInteractiveSession()
	if err != nil {
		log.Fatal(err)
	}

	stopCh := make(chan bool)
	if !isInteractive {
		go func() {
			err = svc.Run(serviceName, &wmiExporterService{stopCh: stopCh})
			if err != nil {
				log.Errorf("Failed to start service: %v", err)
			}
		}()
	}

	collectors, err := loadCollectors(*enabledCollectors)
	if err != nil {
		log.Fatalf("Couldn't load collectors: %s", err)
	}

	log.Infof("Enabled collectors: %v", strings.Join(keys(collectors), ", "))

	exporter := WmiCollector{
		collectors:        collectors,
		maxScrapeDuration: *maxScrapeDuration,
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, promhttp.Handler())
	http.HandleFunc("/health", healthCheck)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Infoln("Starting WMI exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	go func() {
		log.Infoln("Starting server on", *listenAddress)
		log.Fatalf("cannot start WMI exporter: %s", http.ListenAndServe(*listenAddress, nil))
	}()

	for {
		if <-stopCh {
			log.Info("Shutting down WMI exporter")
			break
		}
	}
}

func healthCheck(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_, err := fmt.Fprintln(w, `{"status":"ok"}`)
	if err != nil {
		log.Debugf("Failed to write to stream: %v", err)
	}
}

func keys(m map[string]collector.Collector) []string {
	ret := make([]string, 0, len(m))
	for key := range m {
		ret = append(ret, key)
	}
	return ret
}

type wmiExporterService struct {
	stopCh chan<- bool
}

func (s *wmiExporterService) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
	changes <- svc.Status{State: svc.StartPending}
	changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
loop:
	for {
		select {
		case c := <-r:
			switch c.Cmd {
			case svc.Interrogate:
				changes <- c.CurrentStatus
			case svc.Stop, svc.Shutdown:
				s.stopCh <- true
				break loop
			default:
				log.Error(fmt.Sprintf("unexpected control request #%d", c))
			}
		}
	}
	changes <- svc.Status{State: svc.StopPending}
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements Json<->Lisp conversions using frames.

package golisp

import (
	"encoding/json"
	"fmt"
	"math"
	"reflect"
	"strings"
)

func JsonToLispWithFrames(json interface{}) (result *Data) {
	if json == nil {
		return
	}

	rt := reflect.TypeOf(json)
	rtKind := rt.Kind()

	if rtKind == reflect.Map && reflect.Type.Key(rt).Kind() == reflect.String {
		mapValue := reflect.ValueOf(json)
		m := FrameMap{}
		m.Data = make(FrameMapData, mapValue.Len())
		for _, key := range mapValue.MapKeys() {
			val := mapValue.MapIndex(key)
			value := JsonToLispWithFrames(val.Interface())
			m.Data[fmt.Sprintf("%s:", key.Interface().(string))] = value
		}
		return FrameWithValue(&m)
	}

	if rtKind == reflect.Array || rtKind == reflect.Slice {
		arrayValues := reflect.ValueOf(json)
		var ary *Data
		for i := 0; i < arrayValues.Len(); i++ {
			val := arrayValues.Index(i).Interface()
			value := JsonToLispWithFrames(val)
			ary = Cons(value, ary)
		}
		return Reverse(ary)
	}

	// handle conversion for all numeric primitives
	float64Value, ok := json.(float64)
	if ok {
		if math.Trunc(float64Value) == float64Value {
			return IntegerWithValue(int64(float64Value))
		} else {
			return FloatWithValue(float32(float64Value))
		}
	}

	float32Value, ok := json.(float32)
	if ok {
		if math.Trunc(float64(float32Value)) == float64(float32Value) {
			return IntegerWithValue(int64(float32Value))
		} else {
			return FloatWithValue(float32Value)
		}
	}

	intValue, ok := json.(int)
	if ok {
		return IntegerWithValue(int64(intValue))
	}

	int8Value, ok := json.(int8)
	if ok {
		return IntegerWithValue(int64(int8Value))
	}

	int16Value, ok := json.(int16)
	if ok {
		return IntegerWithValue(int64(int16Value))
	}

	int32Value, ok := json.(int32)
	if ok {
		return IntegerWithValue(int64(int32Value))
	}

	int64Value, ok := json.(int64)
	if ok {
		return IntegerWithValue(int64Value)
	}

	uintValue, ok := json.(uint)
	if ok {
		return IntegerWithValue(int64(uintValue))
	}

	uint8Value, ok := json.(uint8)
	if ok {
		return IntegerWithValue(int64(uint8Value))
	}

	uint16Value, ok := json.(uint16)
	if ok {
		return IntegerWithValue(int64(uint16Value))
	}

	uint32Value, ok := json.(uint32)
	if ok {
		return IntegerWithValue(int64(uint32Value))
	}

	uint64Value, ok := json.(uint64)
	if ok {
		return IntegerWithValue(int64(uint64Value))
	}

	strValue, ok := json.(string)
	if ok {
		return StringWithValue(strValue)
	}

	boolValue, ok := json.(bool)
	if ok {
		return BooleanWithValue(boolValue)
	}

	return
}

func JsonStringToLispWithFrames(jsonData string) (result *Data) {
	b := []byte(jsonData)
	var data interface{}
	err := json.Unmarshal(b, &data)
	if err != nil {
		fmt.Printf("Returning empty frame because of badly formed json: '%s'\n --> %v\n", jsonData, err)
		m := FrameMap{}
		m.Data = make(FrameMapData, 0)
		return FrameWithValue(&m)
	}
	return JsonToLispWithFrames(data)
}

func LispWithFramesToJson(d *Data) (result interface{}) {
	if d == nil {
		return ""
	}

	if IntegerP(d) {
		return IntegerValue(d)
	}

	if FloatP(d) {
		return FloatValue(d)
	}

	if StringP(d) || SymbolP(d) {
		return StringValue(d)
	}

	if BooleanP(d) {
		return BooleanValue(d)
	}

	if PairP(d) {
		ary := make([]interface{}, 0, Length(d))
		for c := d; NotNilP(c); c = Cdr(c) {
			ary = append(ary, LispWithFramesToJson(Car(c)))
		}
		return ary
	}

	if ObjectP(d) && ObjectType(d) == "[]byte" {
		ary := make([]interface{}, 0, Length(d))
		for _, b := range *(*[]byte)(ObjectValue(d)) {
			ary = append(ary, float64(b))
		}
		return ary
	}

	if FrameP(d) {
		dict := make(map[string]interface{}, Length(d))
		frame := FrameValue(d)
		frame.Mutex.RLock()
		for k, v := range frame.Data {
			if !FunctionP(v) {
				dict[strings.TrimRight(k, ":")] = LispWithFramesToJson(v)
			}
		}
		frame.Mutex.RUnlock()
		return dict
	}

	return ""
}

func LispWithFramesToJsonString(d *Data) (result string) {
	temp := LispWithFramesToJson(d)
	j, err := json.Marshal(temp)
	if err == nil {
		return string(j)
	} else {
		return ""
	}
}
<commit_msg>further simplify json conversion<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements Json<->Lisp conversions using frames.

package golisp

import (
	"encoding/json"
	"fmt"
	"math"
	"reflect"
	"strings"
)

func JsonToLispWithFrames(json interface{}) *Data {
	if json == nil {
		return nil
	}

	rt := reflect.TypeOf(json)
	rv := reflect.ValueOf(json)
	rtKind := rt.Kind()

	// maps with string keys get converted to frames
	if rtKind == reflect.Map && reflect.Type.Key(rt).Kind() == reflect.String {
		m := &FrameMap{}
		m.Data = make(FrameMapData, rv.Len())
		for _, key := range rv.MapKeys() {
			val := rv.MapIndex(key)
			value := JsonToLispWithFrames(val.Interface())
			m.Data[fmt.Sprintf("%s:", key.Interface().(string))] = value
		}
		return FrameWithValue(m)
	}

	// slices and arrays get converted to lists
	if rtKind == reflect.Array || rtKind == reflect.Slice {
		var ary *Data
		for i := 0; i < rv.Len(); i++ {
			val := rv.Index(i).Interface()
			value := JsonToLispWithFrames(val)
			ary = Cons(value, ary)
		}
		return Reverse(ary)
	}

	// handle conversion for all primitives
	switch rtKind {
	case
		reflect.Int,
		reflect.Int8,
		reflect.Int16,
		reflect.Int32,
		reflect.Int64,
		reflect.Uint,
		reflect.Uint8,
		reflect.Uint16,
		reflect.Uint32,
		reflect.Uint64,
		reflect.Uintptr:
		var intValue int64
		intValue = reflect.ValueOf(json).Convert(reflect.TypeOf(intValue)).Int()
		return IntegerWithValue(intValue)
	case reflect.Float32, reflect.Float64:
		var floatValue float64
		floatValue = reflect.ValueOf(json).Convert(reflect.TypeOf(floatValue)).Float()
		if math.Trunc(floatValue) == floatValue {
			return IntegerWithValue(int64(floatValue))
		} else {
			return FloatWithValue(float32(floatValue))
		}
	case reflect.String:
		return StringWithValue(rv.String())
	case reflect.Bool:
		return BooleanWithValue(rv.Bool())
	}

	return nil
}

func JsonStringToLispWithFrames(jsonData string) (result *Data) {
	b := []byte(jsonData)
	var data interface{}
	err := json.Unmarshal(b, &data)
	if err != nil {
		fmt.Printf("Returning empty frame because of badly formed json: '%s'\n --> %v\n", jsonData, err)
		m := FrameMap{}
		m.Data = make(FrameMapData, 0)
		return FrameWithValue(&m)
	}
	return JsonToLispWithFrames(data)
}

func LispWithFramesToJson(d *Data) (result interface{}) {
	if d == nil {
		return ""
	}

	if IntegerP(d) {
		return IntegerValue(d)
	}

	if FloatP(d) {
		return FloatValue(d)
	}

	if StringP(d) || SymbolP(d) {
		return StringValue(d)
	}

	if BooleanP(d) {
		return BooleanValue(d)
	}

	if PairP(d) {
		ary := make([]interface{}, 0, Length(d))
		for c := d; NotNilP(c); c = Cdr(c) {
			ary = append(ary, LispWithFramesToJson(Car(c)))
		}
		return ary
	}

	if ObjectP(d) && ObjectType(d) == "[]byte" {
		ary := make([]interface{}, 0, Length(d))
		for _, b := range *(*[]byte)(ObjectValue(d)) {
			ary = append(ary, float64(b))
		}
		return ary
	}

	if FrameP(d) {
		dict := make(map[string]interface{}, Length(d))
		frame := FrameValue(d)
		frame.Mutex.RLock()
		for k, v := range frame.Data {
			if !FunctionP(v) {
				dict[strings.TrimRight(k, ":")] = LispWithFramesToJson(v)
			}
		}
		frame.Mutex.RUnlock()
		return dict
	}

	return ""
}

func LispWithFramesToJsonString(d *Data) (result string) {
	temp := LispWithFramesToJson(d)
	j, err := json.Marshal(temp)
	if err == nil {
		return string(j)
	} else {
		return ""
	}
}
<|endoftext|>


<commit_before><commit_msg>fix: modify http state<commit_after><|endoftext|>


<commit_before>// Package fastsql is a library which extends Go's standard database/sql library.  It provides performance that's easy to take advantage of.
//
// Even better, the fastsql.DB object embeds the standard sql.DB object meaning access to all the standard database/sql library functionality is preserved.  It also means that integrating fastsql into existing codebases is a breeze.
//
// Additional functionality inclues:
//
// 1. Easy, readable, and performant batch insert queries using the BatchInsert method.
// 2. Automatic creation and re-use of prepared statements.
// 3. A convenient holder for manually used prepared statements.
package fastsql

import (
	"database/sql"
	"strings"
	"sync"
)

// DB is a database handle that embeds the standard library's sql.DB struct.
//
//This means the fastsql.DB struct has, and allows, access to all of the standard library functionality while also providng a superset of functionality such as batch operations, autmatically created prepared statmeents, and more.
type DB struct {
	*sql.DB
	PreparedStatements map[string]*sql.Stmt
	prepstmts          map[string]*sql.Stmt
	driverName         string
	flushInterval      uint
	batchInserts       map[string]*insert
}

type insert struct {
	bindParams []interface{}
	insertCtr  uint
	queryPart1 string
	queryPart2 string
	values     string
}

func newInsert() *insert {
	return &insert{
		bindParams: make([]interface{}, 0),
		values:     " VALUES",
	}
}

// Close is the same a sql.Close, but first closes any opened prepared statements.
func (d *DB) Close() error {
	var (
		wg sync.WaitGroup
	)

	wg.Add(1)
	go func(wg *sync.WaitGroup) {
		defer wg.Done()

		for _, stmt := range d.PreparedStatements {
			_ = stmt.Close()
		}
	}(&wg)

	wg.Add(1)
	go func(wg *sync.WaitGroup) {
		defer wg.Done()

		for _, stmt := range d.prepstmts {
			_ = stmt.Close()
		}
	}(&wg)

	wg.Wait()
	return d.DB.Close()
}

// Open is the same as sql.Open, but returns an *fastsql.DB instead.
func Open(driverName, dataSourceName string, flushInterval uint) (*DB, error) {
	var (
		err error
		dbh *sql.DB
	)

	if dbh, err = sql.Open(driverName, dataSourceName); err != nil {
		return nil, err
	}

	return &DB{
		DB:                 dbh,
		PreparedStatements: make(map[string]*sql.Stmt),
		prepstmts:          make(map[string]*sql.Stmt),
		driverName:         driverName,
		flushInterval:      flushInterval,
		batchInserts:       make(map[string]*insert),
	}, err
}

// BatchInsert takes a singlular INSERT query and converts it to a batch-insert query for the caller.  A batch-insert is ran every time BatchInsert is called a multiple of flushInterval times.
func (d *DB) BatchInsert(query string, params ...interface{}) (err error) {
	if _, ok := d.batchInserts[query]; !ok {
		d.batchInserts[query] = newInsert()
	} //if

	// Only split out query the first time Insert is called
	if d.batchInserts[query].queryPart1 == "" {
		d.batchInserts[query].splitQuery(query)
	}

	d.batchInserts[query].insertCtr++

	// Build VALUES seciton of query and add to parameter slice
	d.batchInserts[query].values += d.batchInserts[query].queryPart2
	d.batchInserts[query].bindParams = append(d.batchInserts[query].bindParams, params...)

	// If the batch interval has been hit, execute a batch insert
	if d.batchInserts[query].insertCtr >= d.flushInterval {
		err = d.flushInsert(d.batchInserts[query])
	} //if

	return err
}

// flushInsert performs the acutal batch-insert query.
func (d *DB) flushInsert(in *insert) (err error) {
	var (
		query string = in.queryPart1 + in.values[:len(in.values)-1]
	)

	// Prepare query
	if _, ok := d.prepstmts[query]; !ok {
		if stmt, err := d.DB.Prepare(query); err == nil {
			d.prepstmts[query] = stmt
		} else {
			return err
		}
	}

	// Executate batch insert
	if _, err = d.prepstmts[query].Exec(in.bindParams...); err != nil {
		return err
	} //if

	// Reset vars
	in.values = " VALUES"
	in.bindParams = make([]interface{}, 0)
	in.insertCtr = 0

	return err
}

func (d *DB) setDB(dbh *sql.DB) (err error) {
	if err = dbh.Ping(); err != nil {
		return err
	}

	d.DB = dbh
	return nil
}

func (in *insert) splitQuery(query string) {
	var (
		ndxParens, ndxValues int
	)

	// Normalize and split query
	query = strings.ToLower(query)
	ndxValues = strings.LastIndex(query, "values")
	ndxParens = strings.LastIndex(query, ")")

	// Save the first and second parts of the query separately for easier building later
	in.queryPart1 = strings.TrimSpace(query[:ndxValues])
	in.queryPart2 = query[ndxValues+6:ndxParens+1] + ","
}
<commit_msg>Rearranged code so that funcs are below their types<commit_after>// Package fastsql is a library which extends Go's standard database/sql library.  It provides performance that's easy to take advantage of.
//
// Even better, the fastsql.DB object embeds the standard sql.DB object meaning access to all the standard database/sql library functionality is preserved.  It also means that integrating fastsql into existing codebases is a breeze.
//
// Additional functionality inclues:
//
// 1. Easy, readable, and performant batch insert queries using the BatchInsert method.
// 2. Automatic creation and re-use of prepared statements.
// 3. A convenient holder for manually used prepared statements.
package fastsql

import (
	"database/sql"
	"strings"
	"sync"
)

// DB is a database handle that embeds the standard library's sql.DB struct.
//
//This means the fastsql.DB struct has, and allows, access to all of the standard library functionality while also providng a superset of functionality such as batch operations, autmatically created prepared statmeents, and more.
type DB struct {
	*sql.DB
	PreparedStatements map[string]*sql.Stmt
	prepstmts          map[string]*sql.Stmt
	driverName         string
	flushInterval      uint
	batchInserts       map[string]*insert
}

// Close is the same a sql.Close, but first closes any opened prepared statements.
func (d *DB) Close() error {
	var (
		wg sync.WaitGroup
	)

	wg.Add(1)
	go func(wg *sync.WaitGroup) {
		defer wg.Done()

		for _, stmt := range d.PreparedStatements {
			_ = stmt.Close()
		}
	}(&wg)

	wg.Add(1)
	go func(wg *sync.WaitGroup) {
		defer wg.Done()

		for _, stmt := range d.prepstmts {
			_ = stmt.Close()
		}
	}(&wg)

	wg.Wait()
	return d.DB.Close()
}

// Open is the same as sql.Open, but returns an *fastsql.DB instead.
func Open(driverName, dataSourceName string, flushInterval uint) (*DB, error) {
	var (
		err error
		dbh *sql.DB
	)

	if dbh, err = sql.Open(driverName, dataSourceName); err != nil {
		return nil, err
	}

	return &DB{
		DB:                 dbh,
		PreparedStatements: make(map[string]*sql.Stmt),
		prepstmts:          make(map[string]*sql.Stmt),
		driverName:         driverName,
		flushInterval:      flushInterval,
		batchInserts:       make(map[string]*insert),
	}, err
}

// BatchInsert takes a singlular INSERT query and converts it to a batch-insert query for the caller.  A batch-insert is ran every time BatchInsert is called a multiple of flushInterval times.
func (d *DB) BatchInsert(query string, params ...interface{}) (err error) {
	if _, ok := d.batchInserts[query]; !ok {
		d.batchInserts[query] = newInsert()
	} //if

	// Only split out query the first time Insert is called
	if d.batchInserts[query].queryPart1 == "" {
		d.batchInserts[query].splitQuery(query)
	}

	d.batchInserts[query].insertCtr++

	// Build VALUES seciton of query and add to parameter slice
	d.batchInserts[query].values += d.batchInserts[query].queryPart2
	d.batchInserts[query].bindParams = append(d.batchInserts[query].bindParams, params...)

	// If the batch interval has been hit, execute a batch insert
	if d.batchInserts[query].insertCtr >= d.flushInterval {
		err = d.flushInsert(d.batchInserts[query])
	} //if

	return err
}

// flushInsert performs the acutal batch-insert query.
func (d *DB) flushInsert(in *insert) (err error) {
	var (
		query string = in.queryPart1 + in.values[:len(in.values)-1]
	)

	// Prepare query
	if _, ok := d.prepstmts[query]; !ok {
		if stmt, err := d.DB.Prepare(query); err == nil {
			d.prepstmts[query] = stmt
		} else {
			return err
		}
	}

	// Executate batch insert
	if _, err = d.prepstmts[query].Exec(in.bindParams...); err != nil {
		return err
	} //if

	// Reset vars
	in.values = " VALUES"
	in.bindParams = make([]interface{}, 0)
	in.insertCtr = 0

	return err
}

func (d *DB) setDB(dbh *sql.DB) (err error) {
	if err = dbh.Ping(); err != nil {
		return err
	}

	d.DB = dbh
	return nil
}

type insert struct {
	bindParams []interface{}
	insertCtr  uint
	queryPart1 string
	queryPart2 string
	values     string
}

func newInsert() *insert {
	return &insert{
		bindParams: make([]interface{}, 0),
		values:     " VALUES",
	}
}

func (in *insert) splitQuery(query string) {
	var (
		ndxParens, ndxValues int
	)

	// Normalize and split query
	query = strings.ToLower(query)
	ndxValues = strings.LastIndex(query, "values")
	ndxParens = strings.LastIndex(query, ")")

	// Save the first and second parts of the query separately for easier building later
	in.queryPart1 = strings.TrimSpace(query[:ndxValues])
	in.queryPart2 = query[ndxValues+6:ndxParens+1] + ","
}
<|endoftext|>


<commit_before>package main

import "github.com/lightningnetwork/lnd/lnwire"

// globalFeatures feature vector which affects HTLCs and thus are also
// advertised to other nodes.
var globalFeatures = lnwire.NewFeatureVector([]lnwire.Feature{})

// localFeatures is an feature vector which represent the features which
// only affect the protocol between these two nodes.
var localFeatures = lnwire.NewFeatureVector([]lnwire.Feature{
	{
		Name: "new-ping-and-funding",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "node-ann-feature-addr-swap",
		Flag: lnwire.RequiredFlag,
	},
})
<commit_msg>features: add temporary feature bit for dynamic fees<commit_after>package main

import "github.com/lightningnetwork/lnd/lnwire"

// globalFeatures feature vector which affects HTLCs and thus are also
// advertised to other nodes.
var globalFeatures = lnwire.NewFeatureVector([]lnwire.Feature{})

// localFeatures is an feature vector which represent the features which
// only affect the protocol between these two nodes.
var localFeatures = lnwire.NewFeatureVector([]lnwire.Feature{
	{
		Name: "new-ping-and-funding",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "node-ann-feature-addr-swap",
		Flag: lnwire.RequiredFlag,
	},
	{
		Name: "dynamic-fees",
		Flag: lnwire.RequiredFlag,
	},
})
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"log"
)

func NewContactQuery(page int, perPage int) *ContactQuery {
	if page < 1 {
		page = 1
	}
	if perPage < 1 {
		perPage = 1
	}

	return &ContactQuery{
		limit:      perPage,
		offset:     perPage * (page - 1),
		collection: NewContactList(perPage),
	}
}

type ContactQuery struct {
	limit      int
	offset     int
	collection ContactList
	conn       *sql.DB
}

func (cq *ContactQuery) All() []*Contact {
	if !cq.fillUsers() {
		return NewContactList(0).Items()
	}

	if err := cq.fillDependentData(); err != nil {
		log.Print(err)
	}

	return cq.collection.Items()
}

func (cq *ContactQuery) fillUsers() (ok bool) {
	var err error

	defer func() {
		if err != nil {
			log.Print(err)
		}

		if cq.collection.Any() {
			ok = true
		}
	}()

	ps, err := cq.selectUsersStmt()
	if err != nil {
		return
	}
	defer ps.Close()

	rows, err := ps.Query(cq.limit, cq.offset)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		contact := NewContact()
		rows.Scan(
			&contact.Id,
			&contact.Email,
			&contact.FirstName,
			&contact.LastName,
			&contact.MiddleName,
			&contact.DateOfBirth,
			&contact.Sex,
		)

		cq.collection.Append(contact)
	}

	return
}

func (cq *ContactQuery) fillDependentData() (err error) {
	ps, err := cq.selectDependentDataStmt()
	if err != nil {
		return
	}
	defer ps.Close()

	rows, err := ps.Query(cq.collection.Ids())
	if err != nil {
		return
	}
	defer rows.Close()

	var userId sql.NullInt64

	current := cq.collection.Next()

	if current == nil {
		return errors.New("Empty collection")
	}

	for rows.Next() {

		profile := NewProfile()
		subject := NewSubject()
		rows.Scan(
			&profile.Id,
			&profile.Type,
			&userId,
			&profile.School.Id,
			&profile.School.Name,
			&profile.School.Guid,
			&profile.ClassUnit.Id,
			&profile.ClassUnit.Name,
			&profile.ClassUnit.EnlistedOn,
			&profile.ClassUnit.LeftOn,
			&subject.Id,
			&subject.Name,
		)

		for current.Id != userId {
			if next := cq.collection.Next(); next != nil {
				current = next
			} else {
				break
			}
		}

		if current.Id != userId {
			continue
		}

		if lastPr := current.LastProfile(); lastPr == nil {
			current.Profiles = append(current.Profiles, profile)
		} else if lastPr.Id != profile.Id {
			current.Profiles = append(current.Profiles, profile)
		}

		if subject.Id.Valid {
			current.LastProfile().Subjects = append(
				current.LastProfile().Subjects,
				subject,
			)
		}
	}

	return
}

func (cq *ContactQuery) selectUsersStmt() (*sql.Stmt, error) {
	db, err := DBConn()
	if err != nil {
		return nil, err
	}

	return db.Prepare(`
		select	id,
		      	email,
		      	first_name,
		      	last_name,
		      	middle_name,
		      	date_of_birth,
		      	sex
		  from users
		  where deleted_at is null
		  order by id
		  limit $1
		  offset $2`)
}

func (cq *ContactQuery) selectDependentDataStmt() (*sql.Stmt, error) {
	db, err := DBConn()
	if err != nil {
		return nil, err
	}

	return db.Prepare(`
		select	p.id,
		      	p.type,
		      	p.user_id,
		      	p.school_id,
		      	s.short_name,
		      	s.guid,
		      	p.class_unit_id,
		      	cu.name,
		      	p.enlisted_on,
		      	p.left_on,
		      	c.subject_id,
		      	sb.name
		  from profiles p
		  left outer join schools s
		    on s.id = p.school_id
		    and s.deleted_at is null
		  left outer join class_units cu
		    on cu.id = p.class_unit_id
		    and cu.deleted_at is null
		  left outer join competences c
		    on c.profile_id = p.id
		  left outer join subjects sb
		    on c.subject_id = sb.id
		  where p.deleted_at is null
		    and p.user_id = any($1::integer[])
		  order by p.user_id, p.id`)
}
<commit_msg>[kami][ContactQuery] Refactor<commit_after>package main

import (
	"errors"
	"log"

	"github.com/jmoiron/sqlx"
)

const (
	USERS_QUERY = `
		select	id,
		      	email,
		      	first_name,
		      	last_name,
		      	middle_name,
		      	date_of_birth,
		      	sex
		  from users
		  where deleted_at is null
		  order by id
		  limit ?
		  offset ?;`

	PROFILES_QUERY = `
		select	p.id,
		      	p.type,
		      	p.user_id,
		      	p.school_id,
		      	s.short_name,
		      	s.guid,
		      	p.class_unit_id,
		      	cu.name,
		      	p.enlisted_on,
		      	p.left_on,
		      	c.subject_id,
		      	sb.name
		  from profiles p
		  left outer join schools s
		    on s.id = p.school_id
		    and s.deleted_at is null
		  left outer join class_units cu
		    on cu.id = p.class_unit_id
		    and cu.deleted_at is null
		  left outer join competences c
		    on c.profile_id = p.id
		  left outer join subjects sb
		    on c.subject_id = sb.id
		  where p.deleted_at is null
		    and p.user_id in (?)
		  order by p.user_id, p.id;`
)

func NewContactQuery(page int, perPage int) *ContactQuery {
	if page < 1 {
		page = 1
	}
	if perPage < 1 {
		perPage = 1
	}

	return &ContactQuery{
		limit:      perPage,
		offset:     perPage * (page - 1),
		collection: NewContactList(perPage),
	}
}

type ContactQuery struct {
	limit      int
	offset     int
	collection ContactList
}

func (cq *ContactQuery) All() []*Contact {
	if !cq.fillUsers() {
		return NewContactList(0).Items
	}

	if err := cq.fillDependentData(); err != nil {
		log.Print(err)
	}

	return cq.collection.Items
}

func (cq *ContactQuery) fillUsers() (ok bool) {
	var err error

	defer func() {
		if err != nil {
			log.Print(err)
		}

		if cq.collection.Any() {
			ok = true
		}
	}()

	db, err := DBConn()
	if err != nil {
		return
	}

	query := db.Rebind(USERS_QUERY)

	rows, err := db.Queryx(query, cq.limit, cq.offset)
	if err != nil {
		return
	}
	defer rows.Close()

	var contact *Contact
	for rows.Next() {
		contact = NewContact()
		rows.Scan(
			&contact.Id,
			&contact.Email,
			&contact.FirstName,
			&contact.LastName,
			&contact.MiddleName,
			&contact.DateOfBirth,
			&contact.Sex,
		)
		cq.collection.Items = append(cq.collection.Items, contact)
	}

	return
}

func (cq *ContactQuery) fillDependentData() (err error) {
	db, err := DBConn()
	if err != nil {
		return
	}

	query, args, err := sqlx.In(PROFILES_QUERY, cq.collection.Ids())
	if err != nil {
		return
	}

	query = db.Rebind(query)

	rows, err := db.Queryx(query, args...)
	if err != nil {
		return
	}
	defer rows.Close()

	current := cq.collection.Next()

	if current == nil {
		return errors.New("Empty collection")
	}

	var (
		profile   *Profile
		classUnit *ClassUnit
		school    *School
		subject   *Subject
	)

	for rows.Next() {
		profile = NewProfile()
		classUnit = NewClassUnit()
		school = NewSchool()
		subject = NewSubject()

		rows.Scan(
			&profile.Id,
			&profile.Type,
			&profile.UserId,
			&school.Id,
			&school.Name,
			&school.Guid,
			&classUnit.Id,
			&classUnit.Name,
			&classUnit.EnlistedOn,
			&classUnit.LeftOn,
			&subject.Id,
			&subject.Name,
		)

		for *current.Id != *profile.UserId {
			if next := cq.collection.Next(); next != nil {
				current = next
			} else {
				break
			}
		}

		if *current.Id != *profile.UserId {
			continue
		}

		if classUnit.Id != nil {
			profile.ClassUnit = classUnit
		}

		if school.Id != nil {
			profile.School = school
		}

		if lastPr := current.LastProfile(); lastPr == nil {
			current.Profiles = append(current.Profiles, profile)
		} else if *lastPr.Id != *profile.Id {
			current.Profiles = append(current.Profiles, profile)
		}

		if subject.Id != nil {
			current.LastProfile().Subjects = append(
				current.LastProfile().Subjects,
				subject,
			)
		}
	}

	return
}
<|endoftext|>


<commit_before>package facter

import (
	"github.com/zstyblik/go-facter/lib/formatter"
)

type Facter struct {
	facts     map[string]interface{}
	formatter Formatter
}

type FacterConfig struct {
	Formatter Formatter
}

type Formatter interface {
	Print(map[string]interface{}) error
}

func New(userConf *FacterConfig) *Facter {
	var conf *FacterConfig
	if userConf != nil {
		conf = conf
	} else {
		conf = &FacterConfig{
			Formatter: formatter.NewFormatter(),
		}
	}
	f := &Facter{
		facts:     make(map[string]interface{}),
		formatter: conf.Formatter,
	}
	return f
}

func (f *Facter) Add(k string, v interface{}) {
	f.facts[k] = v
}

func (f *Facter) Print() {
	f.formatter.Print(f.facts)
}
<commit_msg>Fix null pointer in facter.New()<commit_after>package facter

import (
	"github.com/zstyblik/go-facter/lib/formatter"
)

type Facter struct {
	facts     map[string]interface{}
	formatter Formatter
}

type FacterConfig struct {
	Formatter Formatter
}

type Formatter interface {
	Print(map[string]interface{}) error
}

func New(userConf *FacterConfig) *Facter {
	var conf *FacterConfig
	if userConf != nil {
		conf = userConf
	} else {
		conf = &FacterConfig{
			Formatter: formatter.NewFormatter(),
		}
	}
	f := &Facter{
		facts:     make(map[string]interface{}),
		formatter: conf.Formatter,
	}
	return f
}

func (f *Facter) Add(k string, v interface{}) {
	f.facts[k] = v
}

func (f *Facter) Print() {
	f.formatter.Print(f.facts)
}
<|endoftext|>