<commit_before>package classic

import (
// "fmt"
)

var jjnextStates = []int{
	37, 39, 40, 17, 18, 20, 42, 45, 31, 46, 43, 22, 23, 25, 26, 24,
	25, 26, 45, 31, 46, 44, 47, 35, 22, 28, 29, 27, 27, 30, 30, 0,
	1, 2, 4, 5,
}

var jjstrLiteralImages = map[int]string{
	0: "", 11: "\u0053", 12: "\055",
	14: "\050", 15: "\051", 16: "\072", 17: "\052", 18: "\136",
	25: "\133", 26: "\173", 28: "\124\117", 29: "\135", 30: "\175",
}

type TokenManager struct {
	curLexState     int
	defaultLexState int
	jjnewStateCnt   int
	jjround         int
	jjmatchedPos    int
	jjmatchedKind   int

	input_stream CharStream
	jjrounds     []int
	jjstateSet   []int
	curChar      rune
}

func newTokenManager(stream CharStream) *TokenManager {
	return &TokenManager{
		curLexState:     2,
		defaultLexState: 2,
		input_stream:    stream,
		jjrounds:        make([]int, 49),
		jjstateSet:      make([]int, 98),
	}
}

// L41

func (tm *TokenManager) jjMoveStringLiteralDfa0_2() int {
	switch tm.curChar {
	case 40:
		panic("not implemented yet")
	case 41:
		panic("not implemented yet")
	case 42:
		panic("not implemented yet")
	case 43:
		panic("not implemented yet")
	case 45:
		panic("not implemented yet")
	case 58:
		panic("not implemented yet")
	case 91:
		panic("not implemented yet")
	case 94:
		panic("not implemented yet")
	case 123:
		panic("not implemented yet")
	default:
		return tm.jjMoveNfa_2(0, 0)
	}
}

// L87

func (tm *TokenManager) jjMoveNfa_2(startState, curPos int) int {
	startsAt := 0
	tm.jjnewStateCnt = 49
	i := 1
	tm.jjstateSet[0] = startState
	kind := 0x7fffffff
	for {
		if tm.jjround++; tm.jjround == 0x7fffffff {
			tm.reInitRounds()
		}
		if tm.curChar < 64 {
			l := int64(1 << uint(tm.curChar))
			for {
				i--
				switch tm.jjstateSet[i] {
				case 49, 33:
					if (0xfbff7cf8ffffd9ff & uint64(l)) != 0 {
						if kind > 23 {
							kind = 23
						}
						tm.jjCheckNAddTwoStates(33, 34)
					}
				case 0:
					if (0xfbff54f8ffffd9ff & uint64(l)) != 0 {
						if kind > 23 {
							kind = 23
						}
						tm.jjCheckNAddTwoStates(33, 34)
					} else if (0x100002600 & l) != 0 {
						panic("not implemented yet")
					} else if (0x280200000000 & l) != 0 {
						panic("not implemented yet")
					} else if tm.curChar == 47 {
						panic("not implemented yet")
					} else if tm.curChar == 34 {
						panic("not implemented yet")
					}
					if (0x7bff50f8ffffd9ff & l) != 0 {
						if kind > 20 {
							kind = 20
						}
						tm.jjCheckNAddStates(6, 10)
					} else if tm.curChar == 42 {
						panic("not implemented yet")
					} else if tm.curChar == 33 {
						panic("not implemented yet")
					}
					if tm.curChar == 38 {
						panic("not implemented yet")
					}

				case 4:
					panic("not implemented yet")
				case 5:
					panic("not implemented yet")
				case 13:
					panic("not implemented yet")
				case 14:
					panic("not implemented yet")
				case 15:
					panic("not implemented yet")
				case 16:
					panic("not implemented yet")
				case 17:
					panic("not implemented yet")
				case 19:
					panic("not implemented yet")
				case 20:
					panic("not implemented yet")
				case 22:
					panic("not implemented yet")
				case 23:
					panic("not implemented yet")
				case 24:
					panic("not implemented yet")
				case 25:
					panic("not implemented yet")
				case 27:
					panic("not implemented yet")
				case 28:
					panic("not implemented yet")
				case 30:
					panic("not implemented yet")
				case 31:
					if tm.curChar == 42 && kind > 22 {
						kind = 22
					}
				case 32:
					panic("not implemented yet")
				case 35:
					panic("not implemented yet")
				case 36, 38:
					panic("not implemented yet")
				case 37:
					panic("not implemented yet")
				case 40:
					panic("not implemented yet")
				case 41:
					panic("not implemented yet")
				case 42:
					if (0x7bff78f8ffffd9ff & l) != 0 {
						if kind > 20 {
							kind = 20
						}
						tm.jjCheckNAddTwoStates(42, 43)
					}
				case 44:
					panic("not implemented yet")
				case 45:
					if (0x7bff78f8ffffd9ff & l) != 0 {
						tm.jjCheckNAddStates(18, 20)
					}
				case 47:
					panic("not implemented yet")
				}
				if i == startsAt {
					break
				}
			}
		} else if tm.curChar < 128 {
			panic("not implemented yet")
		} else {
			panic("not implemented yet")
		}
		if kind != 0x7fffffff {
			tm.jjmatchedKind = kind
			tm.jjmatchedPos = curPos
			kind = 0x7fffffff
		}
		curPos++
		i = tm.jjnewStateCnt
		tm.jjnewStateCnt = startsAt
		startsAt = 49 - tm.jjnewStateCnt
		if i == startsAt {
			return curPos
		}
		var err error
		if tm.curChar, err = tm.input_stream.readChar(); err != nil {
			return curPos
		}
	}
	panic("should not be here")
}

func (tm *TokenManager) ReInit(stream CharStream) {
	tm.jjmatchedPos = 0
	tm.jjnewStateCnt = 0
	tm.curLexState = tm.defaultLexState
	tm.input_stream = stream
	tm.reInitRounds()
}

func (tm *TokenManager) reInitRounds() {
	tm.jjround = 0x80000001
	for i := 48; i >= 0; i-- {
		tm.jjrounds[i] = 0x80000000
	}
}

// L1027

func (tm *TokenManager) jjFillToken() *Token {
	var curTokenImage string
	if im, ok := jjstrLiteralImages[tm.jjmatchedKind]; ok {
		curTokenImage = im
	} else {
		curTokenImage = tm.input_stream.image()
	}
	beginLine := tm.input_stream.beginLine()
	beginColumn := tm.input_stream.beginColumn()
	endLine := tm.input_stream.endLine()
	endColumn := tm.input_stream.endColumn()
	t := newToken(tm.jjmatchedKind, curTokenImage)

	t.beginLine = beginLine
	t.endLine = endLine
	t.beginColumn = beginColumn
	t.endColumn = endColumn
	return t
}

func (tm *TokenManager) nextToken() (matchedToken *Token) {
	curPos := 0
	var err error
	var eof = false
	for !eof {
		if tm.curChar, err = tm.input_stream.beginToken(); err != nil {
			tm.jjmatchedKind = 0
			matchedToken = tm.jjFillToken()
			return
		}

		switch tm.curLexState {
		case 0:
			panic("not implemented yet")
		case 1:
			panic("not implemented yet")
		case 2:
			tm.jjmatchedKind = 0x7fffffff
			tm.jjmatchedPos = 0
			curPos = tm.jjMoveStringLiteralDfa0_2()
		}

		if tm.jjmatchedKind != 0x7fffffff {
			panic("not implemented yet")
		}
		error_line := tm.input_stream.endLine()
		error_column := tm.input_stream.endColumn()
		var error_after string
		var eofSeen = false
		if _, err = tm.input_stream.readChar(); err == nil {
			tm.input_stream.backup(1)
			tm.input_stream.backup(1)
			if curPos > 1 {
				error_after = tm.input_stream.image()
			}
		} else {
			eofSeen = true
			if curPos > 1 {
				error_after = tm.input_stream.image()
			}
			if tm.curChar == '\n' || tm.curChar == '\r' {
				error_line++
				error_column = 0
			} else {
				error_column++
			}
		}
		panic(newTokenMgrError(eofSeen, tm.curLexState, error_line,
			error_column, error_after, tm.curChar, LEXICAL_ERROR))
	}
	panic("should not be here")
}

// L1137
func (tm *TokenManager) jjCheckNAdd(state int) {
	if tm.jjrounds[state] != tm.jjround {
		tm.jjstateSet[tm.jjnewStateCnt] = state
		tm.jjnewStateCnt++
		tm.jjrounds[state] = tm.jjround
	}
}

// L1151

func (tm *TokenManager) jjCheckNAddTwoStates(state1, state2 int) {
	tm.jjCheckNAdd(state1)
	tm.jjCheckNAdd(state2)
}

func (tm *TokenManager) jjCheckNAddStates(start, end int) {
	assert(start < end)
	assert(start >= 0)
	assert(end <= len(jjnextStates))
	for {
		tm.jjCheckNAdd(jjnextStates[start])
		start++
		if start >= end {
			break
		}
	}
}

func assert(ok bool) {
	if !ok {
		panic("assert fail")
	}
}
<commit_msg>implement TOkenManager.jjMoveNfa_2()<commit_after>package classic

import (
// "fmt"
)

var jjnextStates = []int{
	37, 39, 40, 17, 18, 20, 42, 45, 31, 46, 43, 22, 23, 25, 26, 24,
	25, 26, 45, 31, 46, 44, 47, 35, 22, 28, 29, 27, 27, 30, 30, 0,
	1, 2, 4, 5,
}

var jjstrLiteralImages = map[int]string{
	0: "", 11: "\u0053", 12: "\055",
	14: "\050", 15: "\051", 16: "\072", 17: "\052", 18: "\136",
	25: "\133", 26: "\173", 28: "\124\117", 29: "\135", 30: "\175",
}

type TokenManager struct {
	curLexState     int
	defaultLexState int
	jjnewStateCnt   int
	jjround         int
	jjmatchedPos    int
	jjmatchedKind   int

	input_stream CharStream
	jjrounds     []int
	jjstateSet   []int
	curChar      rune
}

func newTokenManager(stream CharStream) *TokenManager {
	return &TokenManager{
		curLexState:     2,
		defaultLexState: 2,
		input_stream:    stream,
		jjrounds:        make([]int, 49),
		jjstateSet:      make([]int, 98),
	}
}

// L41

func (tm *TokenManager) jjMoveStringLiteralDfa0_2() int {
	switch tm.curChar {
	case 40:
		panic("not implemented yet")
	case 41:
		panic("not implemented yet")
	case 42:
		panic("not implemented yet")
	case 43:
		panic("not implemented yet")
	case 45:
		panic("not implemented yet")
	case 58:
		panic("not implemented yet")
	case 91:
		panic("not implemented yet")
	case 94:
		panic("not implemented yet")
	case 123:
		panic("not implemented yet")
	default:
		return tm.jjMoveNfa_2(0, 0)
	}
}

// L87

func (tm *TokenManager) jjMoveNfa_2(startState, curPos int) int {
	startsAt := 0
	tm.jjnewStateCnt = 49
	i := 1
	tm.jjstateSet[0] = startState
	kind := 0x7fffffff
	for {
		if tm.jjround++; tm.jjround == 0x7fffffff {
			tm.reInitRounds()
		}
		if tm.curChar < 64 {
			l := int64(1 << uint(tm.curChar))
			for {
				i--
				switch tm.jjstateSet[i] {
				case 49, 33:
					if (0xfbff7cf8ffffd9ff & uint64(l)) != 0 {
						if kind > 23 {
							kind = 23
						}
						tm.jjCheckNAddTwoStates(33, 34)
					}
				case 0:
					if (0xfbff54f8ffffd9ff & uint64(l)) != 0 {
						if kind > 23 {
							kind = 23
						}
						tm.jjCheckNAddTwoStates(33, 34)
					} else if (0x100002600 & l) != 0 {
						panic("not implemented yet")
					} else if (0x280200000000 & l) != 0 {
						panic("not implemented yet")
					} else if tm.curChar == 47 {
						panic("not implemented yet")
					} else if tm.curChar == 34 {
						panic("not implemented yet")
					}
					if (0x7bff50f8ffffd9ff & l) != 0 {
						if kind > 20 {
							kind = 20
						}
						tm.jjCheckNAddStates(6, 10)
					} else if tm.curChar == 42 {
						panic("not implemented yet")
					} else if tm.curChar == 33 {
						panic("not implemented yet")
					}
					if tm.curChar == 38 {
						panic("not implemented yet")
					}

				case 4:
					panic("not implemented yet")
				case 5:
					panic("not implemented yet")
				case 13:
					panic("not implemented yet")
				case 14:
					panic("not implemented yet")
				case 15:
					panic("not implemented yet")
				case 16:
					panic("not implemented yet")
				case 17:
					panic("not implemented yet")
				case 19:
					panic("not implemented yet")
				case 20:
					panic("not implemented yet")
				case 22:
					panic("not implemented yet")
				case 23:
					panic("not implemented yet")
				case 24:
					panic("not implemented yet")
				case 25:
					panic("not implemented yet")
				case 27:
					panic("not implemented yet")
				case 28:
					panic("not implemented yet")
				case 30:
					panic("not implemented yet")
				case 31:
					if tm.curChar == 42 && kind > 22 {
						kind = 22
					}
				case 32:
					panic("not implemented yet")
				case 35:
					panic("not implemented yet")
				case 36, 38:
					panic("not implemented yet")
				case 37:
					panic("not implemented yet")
				case 40:
					panic("not implemented yet")
				case 41:
					panic("not implemented yet")
				case 42:
					if (0x7bff78f8ffffd9ff & l) != 0 {
						if kind > 20 {
							kind = 20
						}
						tm.jjCheckNAddTwoStates(42, 43)
					}
				case 44:
					panic("not implemented yet")
				case 45:
					if (0x7bff78f8ffffd9ff & l) != 0 {
						tm.jjCheckNAddStates(18, 20)
					}
				case 47:
					panic("not implemented yet")
				}
				if i == startsAt {
					break
				}
			}
		} else if tm.curChar < 128 {
			panic("not implemented yet")
		} else {
			hiByte := int(tm.curChar >> 8)
			i1 := hiByte >> 6
			l1 := int64(1 << (uint64(hiByte) & 077))
			i2 := int((tm.curChar & 0xff) >> 6)
			l2 := int64(1 << uint64(tm.curChar&077))
			for {
				i--
				switch tm.jjstateSet[i] {
				case 49, 33:
					panic("not implemented yet")
				case 0:
					if jjCanMove_0(hiByte, i1, i2, l1, l2) {
						if kind > 7 {
							kind = 7
						}
					}
					if jjCanMove_2(hiByte, i1, i2, l1, l2) {
						if kind > 23 {
							kind = 23
						}
						tm.jjCheckNAddTwoStates(33, 34)
					}
					if jjCanMove_2(hiByte, i1, i2, l1, l2) {
						if kind > 20 {
							kind = 20
						}
						tm.jjCheckNAddStates(6, 10)
					}
				case 15:
					panic("not implemented yet")
				case 17, 19:
					panic("not implemented yet")
				case 25:
					panic("not implemented yet")
				case 27:
					panic("not implemented yet")
				case 28:
					panic("not implemented yet")
				case 30:
					panic("not implemented yet")
				case 32:
					panic("not implemented yet")
				case 35:
					panic("not implemented yet")
				case 37:
					panic("not implemented yet")
				case 41:
					panic("not implemented yet")
				case 42:
					panic("not implemented yet")
				case 44:
					panic("not implemented yet")
				case 45:
					panic("not implemented yet")
				case 47:
					panic("not implemented yet")
				}
				if i == startsAt {
					break
				}
			}
		}
		if kind != 0x7fffffff {
			tm.jjmatchedKind = kind
			tm.jjmatchedPos = curPos
			kind = 0x7fffffff
		}
		curPos++
		i = tm.jjnewStateCnt
		tm.jjnewStateCnt = startsAt
		startsAt = 49 - tm.jjnewStateCnt
		if i == startsAt {
			return curPos
		}
		var err error
		if tm.curChar, err = tm.input_stream.readChar(); err != nil {
			return curPos
		}
	}
	panic("should not be here")
}

func jjCanMove_0(hiByte, i1, i2 int, l1, l2 int64) bool {
	panic("not implemented yet")
}

func jjCanMove_2(hiByte, i1, i2 int, l1, l2 int64) bool {
	panic("not implemented yet")
}

func (tm *TokenManager) ReInit(stream CharStream) {
	tm.jjmatchedPos = 0
	tm.jjnewStateCnt = 0
	tm.curLexState = tm.defaultLexState
	tm.input_stream = stream
	tm.reInitRounds()
}

func (tm *TokenManager) reInitRounds() {
	tm.jjround = 0x80000001
	for i := 48; i >= 0; i-- {
		tm.jjrounds[i] = 0x80000000
	}
}

// L1027

func (tm *TokenManager) jjFillToken() *Token {
	var curTokenImage string
	if im, ok := jjstrLiteralImages[tm.jjmatchedKind]; ok {
		curTokenImage = im
	} else {
		curTokenImage = tm.input_stream.image()
	}
	beginLine := tm.input_stream.beginLine()
	beginColumn := tm.input_stream.beginColumn()
	endLine := tm.input_stream.endLine()
	endColumn := tm.input_stream.endColumn()
	t := newToken(tm.jjmatchedKind, curTokenImage)

	t.beginLine = beginLine
	t.endLine = endLine
	t.beginColumn = beginColumn
	t.endColumn = endColumn
	return t
}

func (tm *TokenManager) nextToken() (matchedToken *Token) {
	curPos := 0
	var err error
	var eof = false
	for !eof {
		if tm.curChar, err = tm.input_stream.beginToken(); err != nil {
			tm.jjmatchedKind = 0
			matchedToken = tm.jjFillToken()
			return
		}

		switch tm.curLexState {
		case 0:
			panic("not implemented yet")
		case 1:
			panic("not implemented yet")
		case 2:
			tm.jjmatchedKind = 0x7fffffff
			tm.jjmatchedPos = 0
			curPos = tm.jjMoveStringLiteralDfa0_2()
		}

		if tm.jjmatchedKind != 0x7fffffff {
			panic("not implemented yet")
		}
		error_line := tm.input_stream.endLine()
		error_column := tm.input_stream.endColumn()
		var error_after string
		var eofSeen = false
		if _, err = tm.input_stream.readChar(); err == nil {
			tm.input_stream.backup(1)
			tm.input_stream.backup(1)
			if curPos > 1 {
				error_after = tm.input_stream.image()
			}
		} else {
			eofSeen = true
			if curPos > 1 {
				error_after = tm.input_stream.image()
			}
			if tm.curChar == '\n' || tm.curChar == '\r' {
				error_line++
				error_column = 0
			} else {
				error_column++
			}
		}
		panic(newTokenMgrError(eofSeen, tm.curLexState, error_line,
			error_column, error_after, tm.curChar, LEXICAL_ERROR))
	}
	panic("should not be here")
}

// L1137
func (tm *TokenManager) jjCheckNAdd(state int) {
	if tm.jjrounds[state] != tm.jjround {
		tm.jjstateSet[tm.jjnewStateCnt] = state
		tm.jjnewStateCnt++
		tm.jjrounds[state] = tm.jjround
	}
}

// L1151

func (tm *TokenManager) jjCheckNAddTwoStates(state1, state2 int) {
	tm.jjCheckNAdd(state1)
	tm.jjCheckNAdd(state2)
}

func (tm *TokenManager) jjCheckNAddStates(start, end int) {
	assert(start < end)
	assert(start >= 0)
	assert(end <= len(jjnextStates))
	for {
		tm.jjCheckNAdd(jjnextStates[start])
		start++
		if start >= end {
			break
		}
	}
}

func assert(ok bool) {
	if !ok {
		panic("assert fail")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"

	. "github.com/clearcontainers/tests"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

func withWorkload(workload string, expectedExitCode int) TableEntry {
	return Entry(fmt.Sprintf("with '%v' as workload", workload), workload, expectedExitCode)
}

var _ = Describe("run", func() {
	var (
		args []string
		id   string
	)

	BeforeEach(func() {
		id = RandID(30)
		args = []string{"run", "--rm", "--name", id, Image, "sh", "-c"}
	})

	AfterEach(func() {
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
	})

	DescribeTable("container with docker",
		func(workload string, expectedExitCode int) {
			args = append(args, workload)

			command := NewCommand(Docker, args...)
			Expect(command).NotTo(BeNil())

			_, _, exitCode := command.Run()

			Expect(expectedExitCode).To(Equal(exitCode))
		},
		withWorkload("true", 0),
		withWorkload("false", 1),
		withWorkload("exit 0", 0),
		withWorkload("exit 1", 1),
		withWorkload("exit 15", 15),
		withWorkload("exit 123", 123),
	)
})

var _ = Describe("run", func() {
	var (
		args []string
		id   string
	)

	BeforeEach(func() {
		id = RandID(30)
		args = []string{"run", "--name", id}
	})

	AfterEach(func() {
		Expect(RemoveDockerContainer(id)).To(BeTrue())
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
	})

	DescribeTable("container with docker",
		func(options, expectedStatus string) {
			args = append(args, options, Image, "sh")

			command := NewCommand(Docker, args...)
			Expect(command).NotTo(BeNil())

			_, _, exitCode := command.Run()

			Expect(exitCode).To(BeZero())

			Expect(StatusDockerContainer(id)).To(Equal(expectedStatus))

			Expect(ExistDockerContainer(id)).To(BeTrue())
		},
		Entry("in background and interactive", "-di", "Up"),
		Entry("in background, interactive and with a tty", "-dit", "Up"),
	)
})

// creates a new disk file using 'dd' command, returns the path to disk file and
// its loop device representation
func createLoopDevice() (string, string, error) {
	f, err := ioutil.TempFile("", "dd")
	if err != nil {
		return "", "", err
	}
	defer f.Close()

	// create disk file
	ddArgs := []string{"if=/dev/zero", fmt.Sprintf("of=%s", f.Name()), "count=1", "bs=5M"}
	ddCmd := NewCommand("dd", ddArgs...)
	if _, stderr, exitCode := ddCmd.Run(); exitCode != 0 {
		return "", "", fmt.Errorf("%s", stderr)
	}

	// partitioning disk file
	fdiskArgs := []string{"-c", fmt.Sprintf(`printf "g\nn\n\n\n\nw\n" | fdisk %s`, f.Name())}
	fdiskCmd := NewCommand("bash", fdiskArgs...)
	if _, stderr, exitCode := fdiskCmd.Run(); exitCode != 0 {
		return "", "", fmt.Errorf("%s", stderr)
	}

	// create loop device
	partxArgs := []string{"-av", f.Name()}
	partxCmd := NewCommand("partx", partxArgs...)
	stdout, stderr, exitCode := partxCmd.Run()
	if exitCode != 0 {
		return "", "", fmt.Errorf("%s", stderr)
	}
	partxRegexp := regexp.MustCompile("/dev/loop[[:digit:]]+")
	loopFile := partxRegexp.Find([]byte(stdout))
	if len(loopFile) == 0 {
		return "", "", fmt.Errorf("failed to create loop device")
	}

	return f.Name(), string(loopFile), nil
}

func deleteLoopDevice(loopFile string) error {
	partxArgs := []string{"-d", loopFile}
	partxCmd := NewCommand("partx", partxArgs...)
	_, stderr, exitCode := partxCmd.Run()
	if exitCode != 0 {
		return fmt.Errorf("%s", stderr)
	}

	return nil
}

var _ = Describe("run", func() {
	var (
		err        error
		diskFile   string
		loopFile   string
		dockerArgs []string
		id         string
	)

	BeforeEach(func() {
		if os.Getuid() != 0 {
			Skip("only root user can create loop devices")
		}
		id = RandID(30)
		diskFile, loopFile, err = createLoopDevice()
		Expect(err).ToNot(HaveOccurred())
		dockerArgs = []string{"--rm", "--name", id, "--device", loopFile,
			Image, "stat", loopFile}
	})

	AfterEach(func() {
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
		err = deleteLoopDevice(loopFile)
		Expect(err).ToNot(HaveOccurred())
		err = os.Remove(diskFile)
		Expect(err).ToNot(HaveOccurred())
	})

	Context("hot plug a block device", func() {
		It("should be attached", func() {
			//FIXME
			//Skip("Issue: https://github.com/clearcontainers/runtime/issues/789")
			_, _, exitCode := DockerRun(dockerArgs...)
			Expect(exitCode).To(BeZero())
		})
	})
})
<commit_msg>integration: Disable hotplug block device<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"

	. "github.com/clearcontainers/tests"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
)

func withWorkload(workload string, expectedExitCode int) TableEntry {
	return Entry(fmt.Sprintf("with '%v' as workload", workload), workload, expectedExitCode)
}

var _ = Describe("run", func() {
	var (
		args []string
		id   string
	)

	BeforeEach(func() {
		id = RandID(30)
		args = []string{"run", "--rm", "--name", id, Image, "sh", "-c"}
	})

	AfterEach(func() {
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
	})

	DescribeTable("container with docker",
		func(workload string, expectedExitCode int) {
			args = append(args, workload)

			command := NewCommand(Docker, args...)
			Expect(command).NotTo(BeNil())

			_, _, exitCode := command.Run()

			Expect(expectedExitCode).To(Equal(exitCode))
		},
		withWorkload("true", 0),
		withWorkload("false", 1),
		withWorkload("exit 0", 0),
		withWorkload("exit 1", 1),
		withWorkload("exit 15", 15),
		withWorkload("exit 123", 123),
	)
})

var _ = Describe("run", func() {
	var (
		args []string
		id   string
	)

	BeforeEach(func() {
		id = RandID(30)
		args = []string{"run", "--name", id}
	})

	AfterEach(func() {
		Expect(RemoveDockerContainer(id)).To(BeTrue())
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
	})

	DescribeTable("container with docker",
		func(options, expectedStatus string) {
			args = append(args, options, Image, "sh")

			command := NewCommand(Docker, args...)
			Expect(command).NotTo(BeNil())

			_, _, exitCode := command.Run()

			Expect(exitCode).To(BeZero())

			Expect(StatusDockerContainer(id)).To(Equal(expectedStatus))

			Expect(ExistDockerContainer(id)).To(BeTrue())
		},
		Entry("in background and interactive", "-di", "Up"),
		Entry("in background, interactive and with a tty", "-dit", "Up"),
	)
})

// creates a new disk file using 'dd' command, returns the path to disk file and
// its loop device representation
func createLoopDevice() (string, string, error) {
	f, err := ioutil.TempFile("", "dd")
	if err != nil {
		return "", "", err
	}
	defer f.Close()

	// create disk file
	ddArgs := []string{"if=/dev/zero", fmt.Sprintf("of=%s", f.Name()), "count=1", "bs=5M"}
	ddCmd := NewCommand("dd", ddArgs...)
	if _, stderr, exitCode := ddCmd.Run(); exitCode != 0 {
		return "", "", fmt.Errorf("%s", stderr)
	}

	// partitioning disk file
	fdiskArgs := []string{"-c", fmt.Sprintf(`printf "g\nn\n\n\n\nw\n" | fdisk %s`, f.Name())}
	fdiskCmd := NewCommand("bash", fdiskArgs...)
	if _, stderr, exitCode := fdiskCmd.Run(); exitCode != 0 {
		return "", "", fmt.Errorf("%s", stderr)
	}

	// create loop device
	partxArgs := []string{"-av", f.Name()}
	partxCmd := NewCommand("partx", partxArgs...)
	stdout, stderr, exitCode := partxCmd.Run()
	if exitCode != 0 {
		return "", "", fmt.Errorf("%s", stderr)
	}
	partxRegexp := regexp.MustCompile("/dev/loop[[:digit:]]+")
	loopFile := partxRegexp.Find([]byte(stdout))
	if len(loopFile) == 0 {
		return "", "", fmt.Errorf("failed to create loop device")
	}

	return f.Name(), string(loopFile), nil
}

func deleteLoopDevice(loopFile string) error {
	partxArgs := []string{"-d", loopFile}
	partxCmd := NewCommand("partx", partxArgs...)
	_, stderr, exitCode := partxCmd.Run()
	if exitCode != 0 {
		return fmt.Errorf("%s", stderr)
	}

	return nil
}

var _ = Describe("run", func() {
	var (
		err        error
		diskFile   string
		loopFile   string
		dockerArgs []string
		id         string
	)

	BeforeEach(func() {
		Skip("Issue: https://github.com/clearcontainers/tests/issues/728")
		if os.Getuid() != 0 {
			Skip("only root user can create loop devices")
		}
		id = RandID(30)
		diskFile, loopFile, err = createLoopDevice()
		Expect(err).ToNot(HaveOccurred())
		dockerArgs = []string{"--rm", "--name", id, "--device", loopFile,
			Image, "stat", loopFile}
	})

	AfterEach(func() {
		Skip("Issue: https://github.com/clearcontainers/tests/issues/728")
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
		err = deleteLoopDevice(loopFile)
		Expect(err).ToNot(HaveOccurred())
		err = os.Remove(diskFile)
		Expect(err).ToNot(HaveOccurred())
	})

	Context("hot plug a block device", func() {
		It("should be attached", func() {
			Skip("Issue: https://github.com/clearcontainers/tests/issues/728")
			_, _, exitCode := DockerRun(dockerArgs...)
			Expect(exitCode).To(BeZero())
		})
	})
})
<|endoftext|>


<commit_before>/*
 * Copyright 2019 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package cache implements caches to be used in gRPC.
package cache

import (
	"sync"
	"time"
)

type cacheEntry struct {
	item     interface{}
	callback func()
	timer    *time.Timer
	// deleted is set to true in Remove() when the call to timer.Stop() fails.
	// This can happen when the timer in the cache entry fires around the same
	// time that timer.stop() is called in Remove().
	deleted bool
}

// TimeoutCache is a cache with items to be deleted after a timeout.
type TimeoutCache struct {
	mu      sync.Mutex
	timeout time.Duration
	cache   map[interface{}]*cacheEntry
}

// NewTimeoutCache creates a TimeoutCache with the given timeout.
func NewTimeoutCache(timeout time.Duration) *TimeoutCache {
	return &TimeoutCache{
		timeout: timeout,
		cache:   make(map[interface{}]*cacheEntry),
	}
}

// Add adds an item to the cache, with the specified callback to be called when
// the item is removed from the cache upon timeout. If the item is removed from
// the cache using a call to Remove before the timeout expires, the callback
// will not be called.
//
// If the Add was successful, it returns (newly added item, true). If there is
// an existing entry for the specified key, the cache entry is not be updated
// with the specified item and it returns (existing item, false).
func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{}, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if e, ok := c.cache[key]; ok {
		return e.item, false
	}

	entry := &cacheEntry{
		item:     item,
		callback: callback,
	}
	entry.timer = time.AfterFunc(c.timeout, func() {
		c.mu.Lock()
		if entry.deleted {
			c.mu.Unlock()
			// Abort the delete since this has been taken care of in Remove().
			return
		}
		delete(c.cache, key)
		c.mu.Unlock()
		entry.callback()
	})
	c.cache[key] = entry
	return item, true
}

// Remove the item with the key from the cache.
//
// If the specified key exists in the cache, it returns (item associated with
// key, true) and the callback associated with the item is guaranteed to be not
// called. If the given key is not found in the cache, it returns (nil, false)
func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	entry, ok := c.removeInternal(key, false)
	if !ok {
		return nil, false
	}
	return entry.item, true
}

// removeInternal removes and returns the item with key.
//
// caller must hold c.mu.
func (c *TimeoutCache) removeInternal(key interface{}, runCallback bool) (*cacheEntry, bool) {
	entry, ok := c.cache[key]
	if !ok {
		return nil, false
	}
	delete(c.cache, key)
	if !entry.timer.Stop() {
		// If stop was not successful, the timer has fired (this can only happen
		// in a race). But the deleting function is blocked on c.mu because the
		// mutex was held by the caller of this function.
		//
		// Set deleted to true to abort the deleting function. When the lock is
		// released, the delete function will acquire the lock, check the value
		// of deleted and return.
		entry.deleted = true
	}
	if runCallback {
		entry.callback()
	}
	return entry, true
}

// Clear removes all entries, and runs the callbacks if runCallback is true.
func (c *TimeoutCache) Clear(runCallback bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	for key := range c.cache {
		c.removeInternal(key, runCallback)
	}
}
<commit_msg>cache: callback without cache's mutex (#3603)<commit_after>/*
 * Copyright 2019 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package cache implements caches to be used in gRPC.
package cache

import (
	"sync"
	"time"
)

type cacheEntry struct {
	item interface{}
	// Note that to avoid deadlocks (potentially caused by lock ordering),
	// callback can only be called without holding cache's mutex.
	callback func()
	timer    *time.Timer
	// deleted is set to true in Remove() when the call to timer.Stop() fails.
	// This can happen when the timer in the cache entry fires around the same
	// time that timer.stop() is called in Remove().
	deleted bool
}

// TimeoutCache is a cache with items to be deleted after a timeout.
type TimeoutCache struct {
	mu      sync.Mutex
	timeout time.Duration
	cache   map[interface{}]*cacheEntry
}

// NewTimeoutCache creates a TimeoutCache with the given timeout.
func NewTimeoutCache(timeout time.Duration) *TimeoutCache {
	return &TimeoutCache{
		timeout: timeout,
		cache:   make(map[interface{}]*cacheEntry),
	}
}

// Add adds an item to the cache, with the specified callback to be called when
// the item is removed from the cache upon timeout. If the item is removed from
// the cache using a call to Remove before the timeout expires, the callback
// will not be called.
//
// If the Add was successful, it returns (newly added item, true). If there is
// an existing entry for the specified key, the cache entry is not be updated
// with the specified item and it returns (existing item, false).
func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{}, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if e, ok := c.cache[key]; ok {
		return e.item, false
	}

	entry := &cacheEntry{
		item:     item,
		callback: callback,
	}
	entry.timer = time.AfterFunc(c.timeout, func() {
		c.mu.Lock()
		if entry.deleted {
			c.mu.Unlock()
			// Abort the delete since this has been taken care of in Remove().
			return
		}
		delete(c.cache, key)
		c.mu.Unlock()
		entry.callback()
	})
	c.cache[key] = entry
	return item, true
}

// Remove the item with the key from the cache.
//
// If the specified key exists in the cache, it returns (item associated with
// key, true) and the callback associated with the item is guaranteed to be not
// called. If the given key is not found in the cache, it returns (nil, false)
func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	entry, ok := c.removeInternal(key)
	if !ok {
		return nil, false
	}
	return entry.item, true
}

// removeInternal removes and returns the item with key.
//
// caller must hold c.mu.
func (c *TimeoutCache) removeInternal(key interface{}) (*cacheEntry, bool) {
	entry, ok := c.cache[key]
	if !ok {
		return nil, false
	}
	delete(c.cache, key)
	if !entry.timer.Stop() {
		// If stop was not successful, the timer has fired (this can only happen
		// in a race). But the deleting function is blocked on c.mu because the
		// mutex was held by the caller of this function.
		//
		// Set deleted to true to abort the deleting function. When the lock is
		// released, the delete function will acquire the lock, check the value
		// of deleted and return.
		entry.deleted = true
	}
	return entry, true
}

// Clear removes all entries, and runs the callbacks if runCallback is true.
func (c *TimeoutCache) Clear(runCallback bool) {
	var entries []*cacheEntry
	c.mu.Lock()
	for key := range c.cache {
		if e, ok := c.removeInternal(key); ok {
			entries = append(entries, e)
		}
	}
	c.mu.Unlock()

	if !runCallback {
		return
	}

	// removeInternal removes entries from cache, and also stops the timer, so
	// the callback is guaranteed to be not called. If runCallback is true,
	// manual execute all callbacks.
	for _, entry := range entries {
		entry.callback()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package exporter

import (
	"context"
	"io/ioutil"
	"math"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/metrics/datum"
	"github.com/google/mtail/internal/testutil"
)

var handleJSONTests = []struct {
	name     string
	metrics  []*metrics.Metric
	expected string
}{
	{
		"empty",
		[]*metrics.Metric{},
		"[]",
	},
	{
		"single",
		[]*metrics.Metric{
			{
				Name:        "foo",
				Program:     "test",
				Kind:        metrics.Counter,
				LabelValues: []*metrics.LabelValue{{Labels: []string{}, Value: datum.MakeInt(1, time.Unix(0, 0))}},
			},
		},
		`[
  {
    "Name": "foo",
    "Program": "test",
    "Kind": 1,
    "Type": 0,
    "LabelValues": [
      {
        "Value": {
          "Value": 1,
          "Time": 0
        }
      }
    ]
  }
]`,
	},
	{
		"dimensioned",
		[]*metrics.Metric{
			{
				Name:        "foo",
				Program:     "test",
				Kind:        metrics.Counter,
				Keys:        []string{"a", "b"},
				LabelValues: []*metrics.LabelValue{{Labels: []string{"1", "2"}, Value: datum.MakeInt(1, time.Unix(0, 0))}},
			},
		},
		`[
  {
    "Name": "foo",
    "Program": "test",
    "Kind": 1,
    "Type": 0,
    "Keys": [
      "a",
      "b"
    ],
    "LabelValues": [
      {
        "Labels": [
          "1",
          "2"
        ],
        "Value": {
          "Value": 1,
          "Time": 0
        }
      }
    ]
  }
]`,
	},
	{
		"histogram",
		[]*metrics.Metric{
			{
				Name:        "foo",
				Program:     "test",
				Kind:        metrics.Histogram,
				Keys:        []string{"a", "b"},
				LabelValues: []*metrics.LabelValue{{Labels: []string{"1", "2"}, Value: datum.MakeInt(1, time.Unix(0, 0))}},
				Buckets:     []datum.Range{datum.Range{Min: 0, Max: math.Inf(1)}},
			},
		},
		`[
  {
    "Name": "foo",
    "Program": "test",
    "Kind": 5,
    "Type": 0,
    "Keys": [
      "a",
      "b"
    ],
    "LabelValues": [
      {
        "Labels": [
          "1",
          "2"
        ],
        "Value": {
          "Value": 1,
          "Time": 0
        }
      }
    ],
    "Buckets": [
      {
        "Min": "0",
        "Max": "+Inf"
      }
    ]
  }
]`,
	},
}

func TestHandleJSON(t *testing.T) {
	for _, tc := range handleJSONTests {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			var wg sync.WaitGroup
			ms := metrics.NewStore()
			for _, metric := range tc.metrics {
				testutil.FatalIfErr(t, ms.Add(metric))
			}
			e, err := New(ctx, &wg, ms, Hostname("gunstar"))
			testutil.FatalIfErr(t, err)
			response := httptest.NewRecorder()
			e.HandleJSON(response, &http.Request{})
			if response.Code != 200 {
				t.Errorf("response code not 200: %d", response.Code)
			}
			b, err := ioutil.ReadAll(response.Body)
			if err != nil {
				t.Errorf("failed to read response: %s", err)
			}
			testutil.ExpectNoDiff(t, tc.expected, string(b), testutil.IgnoreUnexported(sync.RWMutex{}))
			cancel()
			wg.Wait()
		})
	}
}
<commit_msg>Reformat files with gofmt -s.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package exporter

import (
	"context"
	"io/ioutil"
	"math"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/metrics/datum"
	"github.com/google/mtail/internal/testutil"
)

var handleJSONTests = []struct {
	name     string
	metrics  []*metrics.Metric
	expected string
}{
	{
		"empty",
		[]*metrics.Metric{},
		"[]",
	},
	{
		"single",
		[]*metrics.Metric{
			{
				Name:        "foo",
				Program:     "test",
				Kind:        metrics.Counter,
				LabelValues: []*metrics.LabelValue{{Labels: []string{}, Value: datum.MakeInt(1, time.Unix(0, 0))}},
			},
		},
		`[
  {
    "Name": "foo",
    "Program": "test",
    "Kind": 1,
    "Type": 0,
    "LabelValues": [
      {
        "Value": {
          "Value": 1,
          "Time": 0
        }
      }
    ]
  }
]`,
	},
	{
		"dimensioned",
		[]*metrics.Metric{
			{
				Name:        "foo",
				Program:     "test",
				Kind:        metrics.Counter,
				Keys:        []string{"a", "b"},
				LabelValues: []*metrics.LabelValue{{Labels: []string{"1", "2"}, Value: datum.MakeInt(1, time.Unix(0, 0))}},
			},
		},
		`[
  {
    "Name": "foo",
    "Program": "test",
    "Kind": 1,
    "Type": 0,
    "Keys": [
      "a",
      "b"
    ],
    "LabelValues": [
      {
        "Labels": [
          "1",
          "2"
        ],
        "Value": {
          "Value": 1,
          "Time": 0
        }
      }
    ]
  }
]`,
	},
	{
		"histogram",
		[]*metrics.Metric{
			{
				Name:        "foo",
				Program:     "test",
				Kind:        metrics.Histogram,
				Keys:        []string{"a", "b"},
				LabelValues: []*metrics.LabelValue{{Labels: []string{"1", "2"}, Value: datum.MakeInt(1, time.Unix(0, 0))}},
				Buckets:     []datum.Range{{Min: 0, Max: math.Inf(1)}},
			},
		},
		`[
  {
    "Name": "foo",
    "Program": "test",
    "Kind": 5,
    "Type": 0,
    "Keys": [
      "a",
      "b"
    ],
    "LabelValues": [
      {
        "Labels": [
          "1",
          "2"
        ],
        "Value": {
          "Value": 1,
          "Time": 0
        }
      }
    ],
    "Buckets": [
      {
        "Min": "0",
        "Max": "+Inf"
      }
    ]
  }
]`,
	},
}

func TestHandleJSON(t *testing.T) {
	for _, tc := range handleJSONTests {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			ctx, cancel := context.WithCancel(context.Background())
			var wg sync.WaitGroup
			ms := metrics.NewStore()
			for _, metric := range tc.metrics {
				testutil.FatalIfErr(t, ms.Add(metric))
			}
			e, err := New(ctx, &wg, ms, Hostname("gunstar"))
			testutil.FatalIfErr(t, err)
			response := httptest.NewRecorder()
			e.HandleJSON(response, &http.Request{})
			if response.Code != 200 {
				t.Errorf("response code not 200: %d", response.Code)
			}
			b, err := ioutil.ReadAll(response.Body)
			if err != nil {
				t.Errorf("failed to read response: %s", err)
			}
			testutil.ExpectNoDiff(t, tc.expected, string(b), testutil.IgnoreUnexported(sync.RWMutex{}))
			cancel()
			wg.Wait()
		})
	}
}
<|endoftext|>


<commit_before>package fusefrontend

// This file forwards file encryption operations to cryptfs

import (
	"path/filepath"

	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// isFiltered - check if plaintext "path" should be forbidden
//
// Prevents name clashes with internal files when file names are not encrypted
func (fs *FS) isFiltered(path string) bool {
	if !fs.args.PlaintextNames {
		return false
	}
	// gocryptfs.conf in the root directory is forbidden
	if path == configfile.ConfDefaultName {
		tlog.Info.Printf("The name /%s is reserved when -plaintextnames is used\n",
			configfile.ConfDefaultName)
		return true
	}
	// Note: gocryptfs.diriv is NOT forbidden because diriv and plaintextnames
	// are exclusive
	return false
}

// GetBackingPath - get the absolute encrypted path of the backing file
// from the relative plaintext path "relPath"
func (fs *FS) getBackingPath(relPath string) (string, error) {
	cPath, err := fs.encryptPath(relPath)
	if err != nil {
		return "", err
	}
	cAbsPath := filepath.Join(fs.args.Cipherdir, cPath)
	tlog.Debug.Printf("getBackingPath: %s + %s -> %s", fs.args.Cipherdir, relPath, cAbsPath)
	return cAbsPath, nil
}

// openBackingDir opens the parent ciphertext directory of plaintext path
// "relPath" and returns the dirfd and the encrypted basename.
// The caller should then use Openat(dirfd, cName, ...) and friends.
// openBackingDir is secure against symlink races.
func (fs *FS) openBackingDir(relPath string) (int, string, error) {
	cRelPath, err := fs.encryptPath(relPath)
	if err != nil {
		return -1, "", err
	}
	// Open parent dir
	dirfd, err := syscallcompat.OpenDirNofollow(fs.args.Cipherdir, filepath.Dir(cRelPath))
	if err != nil {
		return -1, "", err
	}
	return dirfd, filepath.Base(cRelPath), nil
}

// encryptPath - encrypt relative plaintext path
func (fs *FS) encryptPath(plainPath string) (string, error) {
	if fs.args.PlaintextNames {
		return plainPath, nil
	}
	fs.dirIVLock.RLock()
	cPath, err := fs.nameTransform.EncryptPathDirIV(plainPath, fs.args.Cipherdir)
	tlog.Debug.Printf("encryptPath '%s' -> '%s' (err: %v)", plainPath, cPath, err)
	fs.dirIVLock.RUnlock()
	return cPath, err
}
<commit_msg>fusefrontend: add named parameters to openBackingDir<commit_after>package fusefrontend

// This file forwards file encryption operations to cryptfs

import (
	"path/filepath"

	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// isFiltered - check if plaintext "path" should be forbidden
//
// Prevents name clashes with internal files when file names are not encrypted
func (fs *FS) isFiltered(path string) bool {
	if !fs.args.PlaintextNames {
		return false
	}
	// gocryptfs.conf in the root directory is forbidden
	if path == configfile.ConfDefaultName {
		tlog.Info.Printf("The name /%s is reserved when -plaintextnames is used\n",
			configfile.ConfDefaultName)
		return true
	}
	// Note: gocryptfs.diriv is NOT forbidden because diriv and plaintextnames
	// are exclusive
	return false
}

// GetBackingPath - get the absolute encrypted path of the backing file
// from the relative plaintext path "relPath"
func (fs *FS) getBackingPath(relPath string) (string, error) {
	cPath, err := fs.encryptPath(relPath)
	if err != nil {
		return "", err
	}
	cAbsPath := filepath.Join(fs.args.Cipherdir, cPath)
	tlog.Debug.Printf("getBackingPath: %s + %s -> %s", fs.args.Cipherdir, relPath, cAbsPath)
	return cAbsPath, nil
}

// openBackingDir opens the parent ciphertext directory of plaintext path
// "relPath" and returns the dirfd and the encrypted basename.
// The caller should then use Openat(dirfd, cName, ...) and friends.
// openBackingDir is secure against symlink races.
func (fs *FS) openBackingDir(relPath string) (dirfd int, cName string, err error) {
	cRelPath, err := fs.encryptPath(relPath)
	if err != nil {
		return -1, "", err
	}
	// Open parent dir
	dirfd, err = syscallcompat.OpenDirNofollow(fs.args.Cipherdir, filepath.Dir(cRelPath))
	if err != nil {
		return -1, "", err
	}
	cName = filepath.Base(cRelPath)
	return dirfd, cName, nil
}

// encryptPath - encrypt relative plaintext path
func (fs *FS) encryptPath(plainPath string) (string, error) {
	if fs.args.PlaintextNames {
		return plainPath, nil
	}
	fs.dirIVLock.RLock()
	cPath, err := fs.nameTransform.EncryptPathDirIV(plainPath, fs.args.Cipherdir)
	tlog.Debug.Printf("encryptPath '%s' -> '%s' (err: %v)", plainPath, cPath, err)
	fs.dirIVLock.RUnlock()
	return cPath, err
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package diff

import (
	"reflect"
	"testing"
)

func TestDiff(t *testing.T) {
	for _, tt := range []struct {
		a, b       []string
		lines      []*Op
		operations []*Op
	}{
		{
			a: []string{"A", "B", "C", "A", "B", "B", "A"},
			b: []string{"C", "B", "A", "B", "A", "C"},
			operations: []*Op{
				&Op{Kind: Delete, I1: 0, I2: 1, J1: 0},
				&Op{Kind: Delete, I1: 1, I2: 2, J1: 0},
				&Op{Kind: Insert, Content: []string{"B"}, I1: 3, I2: 3, J1: 1},
				&Op{Kind: Delete, I1: 5, I2: 6, J1: 4},
				&Op{Kind: Insert, Content: []string{"C"}, I1: 7, I2: 7, J1: 5},
			},
		},
		{
			a: []string{"A", "B"},
			b: []string{"A", "C", ""},
			operations: []*Op{
				&Op{Kind: Delete, I1: 1, I2: 2, J1: 1},
				&Op{Kind: Insert, Content: []string{"C"}, I1: 2, I2: 2, J1: 1},
				&Op{Kind: Insert, Content: []string{""}, I1: 2, I2: 2, J1: 2},
			},
		},
	} {
		ops := Operations(tt.a, tt.b)
		if len(ops) != len(tt.operations) {
			t.Fatalf("expected %v operations, got %v", len(tt.operations), len(ops))
		}
		for i, got := range ops {
			want := tt.operations[i]
			if !reflect.DeepEqual(want, got) {
				t.Errorf("expected %v, got %v", want, got)
			}
		}
		b := ApplyEdits(tt.a, tt.operations)
		for i, want := range tt.b {
			got := b[i]
			if got != want {
				t.Errorf("expected %v got %v", want, got)
			}
		}
	}
}
<commit_msg>internal/lsp: change diff tests to take strings not string arrays to make them more realistic<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package diff

import (
	"reflect"
	"strings"
	"testing"
)

func TestDiff(t *testing.T) {
	for _, test := range []struct {
		a, b       string
		lines      []*Op
		operations []*Op
	}{
		{
			a: "A\nB\nC\nA\nB\nB\nA\n",
			b: "C\nB\nA\nB\nA\nC\n",
			operations: []*Op{
				&Op{Kind: Delete, I1: 0, I2: 1, J1: 0},
				&Op{Kind: Delete, I1: 1, I2: 2, J1: 0},
				&Op{Kind: Insert, Content: []string{"B\n"}, I1: 3, I2: 3, J1: 1},
				&Op{Kind: Delete, I1: 5, I2: 6, J1: 4},
				&Op{Kind: Insert, Content: []string{"C\n"}, I1: 7, I2: 7, J1: 5},
			},
		},
		{
			a: "A\nB\n",
			b: "A\nC\n\n",
			operations: []*Op{
				&Op{Kind: Delete, I1: 1, I2: 2, J1: 1},
				&Op{Kind: Insert, Content: []string{"C\n"}, I1: 2, I2: 2, J1: 1},
				&Op{Kind: Insert, Content: []string{"\n"}, I1: 2, I2: 2, J1: 2},
			},
		},
	} {
		a := strings.SplitAfter(test.a, "\n")
		b := strings.SplitAfter(test.b, "\n")
		ops := Operations(a, b)
		if len(ops) != len(test.operations) {
			t.Fatalf("expected %v operations, got %v", len(test.operations), len(ops))
		}
		for i, got := range ops {
			want := test.operations[i]
			if !reflect.DeepEqual(want, got) {
				t.Errorf("expected %v, got %v", want, got)
			}
		}
		applied := ApplyEdits(a, test.operations)
		for i, want := range applied {
			got := b[i]
			if got != want {
				t.Errorf("expected %v got %v", want, got)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package metrics

import (
	"testing"
	"time"

	"github.com/google/mtail/internal/metrics/datum"
)

func TestMatchingKind(t *testing.T) {
	s := NewStore()
	m1 := NewMetric("foo", "prog", Counter, Int)
	err := s.Add(m1)
	if err != nil {
		t.Fatalf("should be nil: %s", err)
	}
	m2 := NewMetric("foo", "prog1", Gauge, Int)
	err = s.Add(m2)
	if err == nil {
		t.Fatal("should be err")
	}
}

func TestDuplicateMetric(t *testing.T) {
	expectedMetrics := 0
	s := NewStore()
	_ = s.Add(NewMetric("foo", "prog", Counter, Int, "user", "host"))
	_ = s.Add(NewMetric("foo", "prog", Counter, Int))
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should not add duplicate metric. Store: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog", Counter, Float))
	t.Logf("Store: %v", s)
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should add metric of a different type: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog", Counter, Int, "user", "host", "zone", "domain"))
	t.Logf("Store: %v", s)
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should not add duplicate metric, but replace the old one. Store: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog1", Counter, Int))
	t.Logf("Store: %v", s)
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should add metric with a different prog: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog1", Counter, Float))
	t.Logf("Store: %v", s)
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should add metric of a different type: %v", s)
	}
}

/* A program can add a metric with the same name and
   of different type.
   Prometheus behavior in this case is undefined.
   @see https://github.com/google/mtail/issues/130
*/
func TestAddMetricDifferentType(t *testing.T) {
	expected := 2
	s := NewStore()
	err := s.Add(NewMetric("foo", "prog", Counter, Int))
	if err != nil {
		t.Fatalf("should be nil: %s", err)
	}
	// Duplicate metric of different type from *the same program
	err = s.Add(NewMetric("foo", "prog", Counter, Float))
	if err != nil {
		t.Fatalf("should add a new metric to the store: %s. Store: %v", err, s.Metrics)
	}
	if len(s.Metrics["foo"]) != expected {
		t.Fatalf("should have %d metrics of different Type: %v", expected, s.Metrics)
	}

	// Duplicate metric of different type from a different program
	err = s.Add(NewMetric("foo", "prog1", Counter, Float))
	expected++
	if err != nil {
		t.Fatalf("should add a new metric to the store: %s. Store: %v", err, s.Metrics)
	}
	if len(s.Metrics["foo"]) != expected {
		t.Fatalf("should have %d metrics of different Type: %v", expected, s.Metrics)
	}
}

func TestExpireMetric(t *testing.T) {
	s := NewStore()
	m := NewMetric("foo", "prog", Counter, Int, "a", "b", "c")
	s.Add(m)
	d, err := m.GetDatum("1", "2", "3")
	if err != nil {
		t.Error(err)
	}
	datum.SetInt(d, 1, time.Now().Add(-time.Hour))
	lv := m.FindLabelValueOrNil([]string{"1", "2", "3"})
	if lv == nil {
		t.Errorf("couldn't find lv")
	}
	lv.Expiry = time.Minute
	d, err = m.GetDatum("4", "5", "6")
	if err != nil {
		t.Error(err)
	}
	datum.SetInt(d, 1, time.Now().Add(-time.Hour))
	lv = m.FindLabelValueOrNil([]string{"4", "5", "6"})
	if lv == nil {
		t.Errorf("couldn't find lv")
	}

	s.Expire()
	lv = m.FindLabelValueOrNil([]string{"1", "2", "3"})
	if lv != nil {
		t.Errorf("lv not expired: %#v", lv)
		t.Logf("Store: %#v", s)
	}
	lv = m.FindLabelValueOrNil([]string{"4", "5", "6"})
	if lv == nil {
		t.Errorf("lv expired")
		t.Logf("Store: %#v", s)
	}
}
<commit_msg>Change test logs to glogs.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package metrics

import (
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/metrics/datum"
)

func TestMatchingKind(t *testing.T) {
	s := NewStore()
	m1 := NewMetric("foo", "prog", Counter, Int)
	err := s.Add(m1)
	if err != nil {
		t.Fatalf("should be nil: %s", err)
	}
	m2 := NewMetric("foo", "prog1", Gauge, Int)
	err = s.Add(m2)
	if err == nil {
		t.Fatal("should be err")
	}
}

func TestDuplicateMetric(t *testing.T) {
	expectedMetrics := 0
	s := NewStore()
	_ = s.Add(NewMetric("foo", "prog", Counter, Int, "user", "host"))
	_ = s.Add(NewMetric("foo", "prog", Counter, Int))
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should not add duplicate metric. Store: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog", Counter, Float))
	glog.Infof("Store: %v", s)
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should add metric of a different type: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog", Counter, Int, "user", "host", "zone", "domain"))
	glog.Infof("Store: %v", s)
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should not add duplicate metric, but replace the old one. Store: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog1", Counter, Int))
	glog.Infof("Store: %v", s)
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should add metric with a different prog: %v", s)
	}

	_ = s.Add(NewMetric("foo", "prog1", Counter, Float))
	glog.Infof("Store: %v", s)
	expectedMetrics++
	if len(s.Metrics["foo"]) != expectedMetrics {
		t.Fatalf("should add metric of a different type: %v", s)
	}
}

/* A program can add a metric with the same name and
   of different type.
   Prometheus behavior in this case is undefined.
   @see https://github.com/google/mtail/issues/130
*/
func TestAddMetricDifferentType(t *testing.T) {
	expected := 2
	s := NewStore()
	err := s.Add(NewMetric("foo", "prog", Counter, Int))
	if err != nil {
		t.Fatalf("should be nil: %s", err)
	}
	// Duplicate metric of different type from *the same program
	err = s.Add(NewMetric("foo", "prog", Counter, Float))
	if err != nil {
		t.Fatalf("should add a new metric to the store: %s. Store: %v", err, s.Metrics)
	}
	if len(s.Metrics["foo"]) != expected {
		t.Fatalf("should have %d metrics of different Type: %v", expected, s.Metrics)
	}

	// Duplicate metric of different type from a different program
	err = s.Add(NewMetric("foo", "prog1", Counter, Float))
	expected++
	if err != nil {
		t.Fatalf("should add a new metric to the store: %s. Store: %v", err, s.Metrics)
	}
	if len(s.Metrics["foo"]) != expected {
		t.Fatalf("should have %d metrics of different Type: %v", expected, s.Metrics)
	}
}

func TestExpireMetric(t *testing.T) {
	s := NewStore()
	m := NewMetric("foo", "prog", Counter, Int, "a", "b", "c")
	s.Add(m)
	d, err := m.GetDatum("1", "2", "3")
	if err != nil {
		t.Error(err)
	}
	datum.SetInt(d, 1, time.Now().Add(-time.Hour))
	lv := m.FindLabelValueOrNil([]string{"1", "2", "3"})
	if lv == nil {
		t.Errorf("couldn't find lv")
	}
	lv.Expiry = time.Minute
	d, err = m.GetDatum("4", "5", "6")
	if err != nil {
		t.Error(err)
	}
	datum.SetInt(d, 1, time.Now().Add(-time.Hour))
	lv = m.FindLabelValueOrNil([]string{"4", "5", "6"})
	if lv == nil {
		t.Errorf("couldn't find lv")
	}

	s.Expire()
	lv = m.FindLabelValueOrNil([]string{"1", "2", "3"})
	if lv != nil {
		t.Errorf("lv not expired: %#v", lv)
		t.Logf("Store: %#v", s)
	}
	lv = m.FindLabelValueOrNil([]string{"4", "5", "6"})
	if lv == nil {
		t.Errorf("lv expired")
		t.Logf("Store: %#v", s)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package reverse

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// startReverse manages the actual reverse connection to the Circonus broker
func (c *Connection) startReverse() error {
	for {
		conn, cerr := c.connect()
		if cerr != nil {
			if cerr.fatal {
				c.logger.Error().Err(cerr.err).Msg("connecting to broker")
				return cerr.err
			}
			c.logger.Warn().Err(cerr.err).Msg("retrying")
			continue
		}

		if c.shutdown() {
			return nil
		}

		done := make(chan interface{})
		commandReader := c.newCommandReader(done, conn)
		commandProcessor := c.newCommandProcessor(done, commandReader)
		for result := range commandProcessor {
			if c.shutdown() {
				close(done)
				conn.Close()
				return nil
			}
			if result.ignore {
				// c.logger.Debug().Err(result.err).Int("timeouts", c.commTimeouts).Msg("ignored")
				continue
			}
			if result.err != nil {
				if result.reset {
					c.logger.Warn().Err(result.err).Int("timeouts", c.commTimeouts).Msg("resetting connection")
					close(done)
					break
				} else if result.fatal {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("fatal error, exiting")
					conn.Close()
					close(done)
					return result.err
				} else {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("unhandled error state...")
					continue
				}
			}

			// send metrics to broker
			if err := c.sendMetricData(conn, result.channelID, result.metrics); err != nil {
				c.logger.Warn().Err(err).Msg("sending metric data, resetting connection")
				close(done)
				break
			}

			c.resetConnectionAttempts()
		}

		conn.Close()
		if c.shutdown() {
			return nil
		}

	}
}

// connect to broker w/tls and send initial introduction
// NOTE: all reverse connections require tls
func (c *Connection) connect() (*tls.Conn, *connError) {
	c.Lock()
	if c.connAttempts > 0 {
		if c.maxConnRetry != -1 && c.connAttempts >= c.maxConnRetry {
			c.logger.Fatal().Int("max_attempts", c.maxConnRetry).Int("attempts", c.connAttempts).Msg("max broker connection attempts reached, exiting")
		}

		c.logger.Info().
			Str("delay", c.delay.String()).
			Int("attempt", c.connAttempts).
			Msg("connect retry")

		time.Sleep(c.delay)
		c.delay = c.getNextDelay(c.delay)

		// Under normal circumstances the configuration for reverse is
		// non-volatile. There are, however, some situations where the
		// configuration must be rebuilt. (e.g. ip of broker changed,
		// check changed to use a different broker, broker certificate
		// changes, etc.) The majority of configuration based errors are
		// fatal, no attempt is made to resolve.
		if c.connAttempts%c.configRetryLimit == 0 {
			c.logger.Info().Int("attempts", c.connAttempts).Msg("reconfig triggered")
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("refreshing check")
			if err := c.check.RefreshCheckConfig(); err != nil {
				c.logger.Fatal().Err(err).Msg("unable to refresh check configuration")
				// return nil, &connError{fatal: true, err: errors.Wrap(err, "refreshing check configuration")}
			}
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("setting reverse config")
			rc, err := c.check.GetReverseConfig()
			if err != nil {
				c.logger.Fatal().Err(err).Msg("unable to reconfigure reverse connection after check refresh")
				// return nil, &connError{fatal: true, err: errors.Wrap(err, "reconfiguring reverse connection")}
			}
			if rc == nil {
				c.logger.Fatal().Msg("invalid reverse configuration (nil)")
				// return nil, &connError{fatal: true, err: errors.Wrap(err, "invalid reverse configuration (nil)")}
			}
			c.revConfig = *rc
			c.logger = log.With().Str("pkg", "reverse").Str("cid", viper.GetString(config.KeyCheckBundleID)).Logger()
			c.logger.Info().
				Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).
				Str("rev_host", c.revConfig.ReverseURL.Hostname()).
				Str("rev_port", c.revConfig.ReverseURL.Port()).
				Str("rev_path", c.revConfig.ReverseURL.Path).
				Str("agent", c.agentAddress).
				Msg("reverse configuration")
		}
	}
	c.Unlock()

	revHost := c.revConfig.ReverseURL.Host
	c.logger.Debug().Str("host", revHost).Msg("connecting")
	c.Lock()
	c.connAttempts++
	c.Unlock()
	dialer := &net.Dialer{Timeout: c.dialerTimeout}
	conn, err := tls.DialWithDialer(dialer, "tcp", c.revConfig.BrokerAddr.String(), c.revConfig.TLSConfig)
	if err != nil {
		return nil, &connError{fatal: false, err: errors.Wrapf(err, "connecting to %s", revHost)}
	}
	c.logger.Info().Str("host", revHost).Msg("connected")

	conn.SetDeadline(time.Now().Add(c.commTimeout))
	introReq := "REVERSE " + c.revConfig.ReverseURL.Path
	if c.revConfig.ReverseURL.Fragment != "" {
		introReq += "#" + c.revConfig.ReverseURL.Fragment // reverse secret is placed here when reverse url is parsed
	}
	c.logger.Debug().Msg(fmt.Sprintf("sending intro '%s'", introReq))
	if _, err := fmt.Fprintf(conn, "%s HTTP/1.1\r\n\r\n", introReq); err != nil {
		if err != nil {
			c.logger.Error().Err(err).Msg("sending intro")
			return nil, &connError{fatal: false, err: errors.Wrapf(err, "unable to write intro to %s", revHost)}
		}
	}

	c.Lock()
	// reset timeouts after successful (re)connection
	c.commTimeouts = 0
	c.Unlock()

	return conn, nil
}

// getNextDelay for failed connection attempts
func (c *Connection) getNextDelay(currDelay time.Duration) time.Duration {
	if currDelay == c.maxDelay {
		return currDelay
	}

	delay := currDelay

	if delay < c.maxDelay {
		drift := rand.Intn(c.maxDelayStep-c.minDelayStep) + c.minDelayStep
		delay += time.Duration(drift) * time.Second
	}

	if delay > c.maxDelay {
		delay = c.maxDelay
	}

	return delay
}

// resetConnectionAttempts on successful send/receive
func (c *Connection) resetConnectionAttempts() {
	c.Lock()
	if c.connAttempts > 0 {
		c.delay = 1 * time.Second
		c.connAttempts = 0
	}
	c.Unlock()
}

// Error returns string representation of a connError
func (e *connError) Error() string {
	return e.err.Error()
}
<commit_msg>upd: centralize fatal connection error handling in startReverse method<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package reverse

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net"
	"time"

	"github.com/circonus-labs/circonus-agent/internal/config"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// startReverse manages the actual reverse connection to the Circonus broker
func (c *Connection) startReverse() error {
	for {
		conn, cerr := c.connect()
		if cerr != nil {
			if cerr.fatal {
				c.logger.Fatal().Err(cerr.err).Msg("unable to establish reverse connection to broker")
			}
			c.logger.Warn().Err(cerr.err).Msg("retrying")
			continue
		}

		if c.shutdown() {
			return nil
		}

		done := make(chan interface{})
		commandReader := c.newCommandReader(done, conn)
		commandProcessor := c.newCommandProcessor(done, commandReader)
		for result := range commandProcessor {
			if c.shutdown() {
				close(done)
				conn.Close()
				return nil
			}
			if result.ignore {
				// c.logger.Debug().Err(result.err).Int("timeouts", c.commTimeouts).Msg("ignored")
				continue
			}
			if result.err != nil {
				if result.reset {
					c.logger.Warn().Err(result.err).Int("timeouts", c.commTimeouts).Msg("resetting connection")
					close(done)
					break
				} else if result.fatal {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("fatal error, exiting")
					conn.Close()
					close(done)
					return result.err
				} else {
					c.logger.Error().Err(result.err).Interface("result", result).Msg("unhandled error state...")
					continue
				}
			}

			// send metrics to broker
			if err := c.sendMetricData(conn, result.channelID, result.metrics); err != nil {
				c.logger.Warn().Err(err).Msg("sending metric data, resetting connection")
				close(done)
				break
			}

			c.resetConnectionAttempts()
		}

		conn.Close()
		if c.shutdown() {
			return nil
		}

	}
}

// connect to broker w/tls and send initial introduction
// NOTE: all reverse connections require tls
func (c *Connection) connect() (*tls.Conn, *connError) {
	c.Lock()
	if c.connAttempts > 0 {
		if c.maxConnRetry != -1 && c.connAttempts >= c.maxConnRetry {
			// c.logger.Fatal().Int("max_attempts", c.maxConnRetry).Int("attempts", c.connAttempts).Msg("max broker connection attempts reached, exiting")
			return nil, &connError{fatal: true, err: errors.Errorf("max broker connection attempts reached (%d of %d)", c.connAttempts, c.maxConnRetry)}
		}

		c.logger.Info().
			Str("delay", c.delay.String()).
			Int("attempt", c.connAttempts).
			Msg("connect retry")

		time.Sleep(c.delay)
		c.delay = c.getNextDelay(c.delay)

		// Under normal circumstances the configuration for reverse is
		// non-volatile. There are, however, some situations where the
		// configuration must be rebuilt. (e.g. ip of broker changed,
		// check changed to use a different broker, broker certificate
		// changes, etc.) The majority of configuration based errors are
		// fatal, no attempt is made to resolve.
		if c.connAttempts%c.configRetryLimit == 0 {
			c.logger.Info().Int("attempts", c.connAttempts).Msg("reconfig triggered")
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("refreshing check")
			if err := c.check.RefreshCheckConfig(); err != nil {
				// c.logger.Fatal().Err(err).Msg("unable to refresh check configuration")
				return nil, &connError{fatal: true, err: errors.Wrap(err, "refreshing check configuration")}
			}
			c.logger.Debug().Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).Msg("setting reverse config")
			rc, err := c.check.GetReverseConfig()
			if err != nil {
				// c.logger.Fatal().Err(err).Msg("unable to reconfigure reverse connection after check refresh")
				return nil, &connError{fatal: true, err: errors.Wrap(err, "reconfiguring reverse connection")}
			}
			if rc == nil {
				// c.logger.Fatal().Msg("invalid reverse configuration (nil)")
				return nil, &connError{fatal: true, err: errors.Wrap(err, "invalid reverse configuration (nil)")}
			}
			c.revConfig = *rc
			c.logger = log.With().Str("pkg", "reverse").Str("cid", viper.GetString(config.KeyCheckBundleID)).Logger()
			c.logger.Info().
				Str("check_bundle", viper.GetString(config.KeyCheckBundleID)).
				Str("rev_host", c.revConfig.ReverseURL.Hostname()).
				Str("rev_port", c.revConfig.ReverseURL.Port()).
				Str("rev_path", c.revConfig.ReverseURL.Path).
				Str("agent", c.agentAddress).
				Msg("reverse configuration")
		}
	}
	c.Unlock()

	revHost := c.revConfig.ReverseURL.Host
	c.logger.Debug().Str("host", revHost).Msg("connecting")
	c.Lock()
	c.connAttempts++
	c.Unlock()
	dialer := &net.Dialer{Timeout: c.dialerTimeout}
	conn, err := tls.DialWithDialer(dialer, "tcp", c.revConfig.BrokerAddr.String(), c.revConfig.TLSConfig)
	if err != nil {
		return nil, &connError{fatal: false, err: errors.Wrapf(err, "connecting to %s", revHost)}
	}
	c.logger.Info().Str("host", revHost).Msg("connected")

	conn.SetDeadline(time.Now().Add(c.commTimeout))
	introReq := "REVERSE " + c.revConfig.ReverseURL.Path
	if c.revConfig.ReverseURL.Fragment != "" {
		introReq += "#" + c.revConfig.ReverseURL.Fragment // reverse secret is placed here when reverse url is parsed
	}
	c.logger.Debug().Msg(fmt.Sprintf("sending intro '%s'", introReq))
	if _, err := fmt.Fprintf(conn, "%s HTTP/1.1\r\n\r\n", introReq); err != nil {
		if err != nil {
			c.logger.Error().Err(err).Msg("sending intro")
			return nil, &connError{fatal: false, err: errors.Wrapf(err, "unable to write intro to %s", revHost)}
		}
	}

	c.Lock()
	// reset timeouts after successful (re)connection
	c.commTimeouts = 0
	c.Unlock()

	return conn, nil
}

// getNextDelay for failed connection attempts
func (c *Connection) getNextDelay(currDelay time.Duration) time.Duration {
	if currDelay == c.maxDelay {
		return currDelay
	}

	delay := currDelay

	if delay < c.maxDelay {
		drift := rand.Intn(c.maxDelayStep-c.minDelayStep) + c.minDelayStep
		delay += time.Duration(drift) * time.Second
	}

	if delay > c.maxDelay {
		delay = c.maxDelay
	}

	return delay
}

// resetConnectionAttempts on successful send/receive
func (c *Connection) resetConnectionAttempts() {
	c.Lock()
	if c.connAttempts > 0 {
		c.delay = 1 * time.Second
		c.connAttempts = 0
	}
	c.Unlock()
}

// Error returns string representation of a connError
func (e *connError) Error() string {
	return e.err.Error()
}
<|endoftext|>


<commit_before>package testutil

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
)

var (
	jenkinsHost = "http://localhost:8001/jenkins"
	netrcFile   = filepath.Join(os.Getenv("HOME"), ".netrc")
)

// requireEnv makes sure that the given environment variables are set.
func requireEnv(names []string) error {
	for _, name := range names {
		if os.Getenv(name) == "" {
			return fmt.Errorf("environment variable %q is not set", name)
		}
	}
	return nil
}

// vanadiumPresubmitPoll polls vanadium projects for new patchsets for
// which to run presubmit tests.
func vanadiumPresubmitPoll(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	root, err := util.VanadiumRoot()
	if err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Use the "presubmit query" command to poll for new changes.
	logfile := filepath.Join(root, ".presubmit_log")
	args := []string{}
	if ctx.Verbose() {
		args = append(args, "-v")
	}
	args = append(args,
		"-host", jenkinsHost,
		"-netrc", netrcFile,
		"query",
		"-log_file", logfile,
		"-manifest", "public",
	)
	if err := ctx.Run().Command("presubmit", args...); err != nil {
		return nil, err
	}

	return &TestResult{Status: TestPassed}, nil
}

// vanadiumPresubmitTest runs presubmit tests for a given project specified
// in TEST environment variable.
func vanadiumPresubmitTest(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	if err := requireEnv([]string{"BUILD_NUMBER", "REFS", "PROJECTS", "TEST", "WORKSPACE"}); err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Use the "presubmit test" command to run the presubmit test.
	args := []string{}
	if ctx.Verbose() {
		args = append(args, "-v")
	}
	test := os.Getenv("TEST")
	args = append(args,
		"-host", jenkinsHost,
		"-netrc", netrcFile,
		"test",
		"-build_number", os.Getenv("BUILD_NUMBER"),
		"-manifest", "public",
		"-projects", os.Getenv("PROJECTS"),
		"-refs", os.Getenv("REFS"),
		"-test", test,
	)
	if err := ctx.Run().Command("presubmit", args...); err != nil {
		return nil, err
	}

	// Remove any test result files that are empty.
	testResultFiles, err := findTestResultFiles(ctx, test)
	if err != nil {
		return nil, err
	}
	for _, file := range testResultFiles {
		fileInfo, err := os.Stat(file)
		if err != nil {
			return nil, err
		}
		if fileInfo.Size() == 0 {
			if err := ctx.Run().RemoveAll(file); err != nil {
				return nil, err
			}
		}
	}

	return &TestResult{Status: TestPassed}, nil
}

// vanadiumPresubmitResult runs "presubmit result" command to process and post test resutls.
func vanadiumPresubmitResult(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	if err := requireEnv([]string{"BUILD_NUMBER", "REFS", "PROJECTS", "WORKSPACE"}); err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Run "presubmit result".
	args := []string{}
	if ctx.Verbose() {
		args = append(args, "-v")
	}
	args = append(args,
		"-host", jenkinsHost,
		"-netrc", netrcFile,
		"result",
		"-build_number", os.Getenv("BUILD_NUMBER"),
		"-refs", os.Getenv("REFS"),
		"-projects", os.Getenv("PROJECTS"),
	)
	if err := ctx.Run().Command("presubmit", args...); err != nil {
		return nil, err
	}

	return &TestResult{Status: TestPassed}, nil
}
<commit_msg>TBR: presubmit: use "tools" manifest in presubmit jobs.<commit_after>package testutil

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
)

var (
	jenkinsHost = "http://localhost:8001/jenkins"
	netrcFile   = filepath.Join(os.Getenv("HOME"), ".netrc")
)

// requireEnv makes sure that the given environment variables are set.
func requireEnv(names []string) error {
	for _, name := range names {
		if os.Getenv(name) == "" {
			return fmt.Errorf("environment variable %q is not set", name)
		}
	}
	return nil
}

// vanadiumPresubmitPoll polls vanadium projects for new patchsets for
// which to run presubmit tests.
func vanadiumPresubmitPoll(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	root, err := util.VanadiumRoot()
	if err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Use the "presubmit query" command to poll for new changes.
	logfile := filepath.Join(root, ".presubmit_log")
	args := []string{}
	if ctx.Verbose() {
		args = append(args, "-v")
	}
	args = append(args,
		"-host", jenkinsHost,
		"-netrc", netrcFile,
		"query",
		"-log_file", logfile,
		"-manifest", "tools",
	)
	if err := ctx.Run().Command("presubmit", args...); err != nil {
		return nil, err
	}

	return &TestResult{Status: TestPassed}, nil
}

// vanadiumPresubmitTest runs presubmit tests for a given project specified
// in TEST environment variable.
func vanadiumPresubmitTest(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	if err := requireEnv([]string{"BUILD_NUMBER", "REFS", "PROJECTS", "TEST", "WORKSPACE"}); err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Use the "presubmit test" command to run the presubmit test.
	args := []string{}
	if ctx.Verbose() {
		args = append(args, "-v")
	}
	test := os.Getenv("TEST")
	args = append(args,
		"-host", jenkinsHost,
		"-netrc", netrcFile,
		"test",
		"-build_number", os.Getenv("BUILD_NUMBER"),
		"-manifest", "tools",
		"-projects", os.Getenv("PROJECTS"),
		"-refs", os.Getenv("REFS"),
		"-test", test,
	)
	if err := ctx.Run().Command("presubmit", args...); err != nil {
		return nil, err
	}

	// Remove any test result files that are empty.
	testResultFiles, err := findTestResultFiles(ctx, test)
	if err != nil {
		return nil, err
	}
	for _, file := range testResultFiles {
		fileInfo, err := os.Stat(file)
		if err != nil {
			return nil, err
		}
		if fileInfo.Size() == 0 {
			if err := ctx.Run().RemoveAll(file); err != nil {
				return nil, err
			}
		}
	}

	return &TestResult{Status: TestPassed}, nil
}

// vanadiumPresubmitResult runs "presubmit result" command to process and post test resutls.
func vanadiumPresubmitResult(ctx *tool.Context, testName string, _ ...TestOpt) (_ *TestResult, e error) {
	if err := requireEnv([]string{"BUILD_NUMBER", "REFS", "PROJECTS", "WORKSPACE"}); err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	// Run "presubmit result".
	args := []string{}
	if ctx.Verbose() {
		args = append(args, "-v")
	}
	args = append(args,
		"-host", jenkinsHost,
		"-netrc", netrcFile,
		"result",
		"-build_number", os.Getenv("BUILD_NUMBER"),
		"-refs", os.Getenv("REFS"),
		"-projects", os.Getenv("PROJECTS"),
	)
	if err := ctx.Run().Command("presubmit", args...); err != nil {
		return nil, err
	}

	return &TestResult{Status: TestPassed}, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"

	"github.com/keybase/client/go/chat/globals"
	"github.com/keybase/client/go/chat/utils"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/chat1"
	"github.com/keybase/client/go/protocol/gregor1"
	"golang.org/x/net/context"
)

type delhTracker struct {
	globals.Contextified
	utils.DebugLabeler
}

type delhTrackerEntry struct {
	StorageVersion int `codec:"v"`
	// The latest DeleteHistory upto value that has been applied locally
	MaxDeleteHistoryUpto chat1.MessageID `codec:"ldhu"`
	// The most ancient message that a new DeleteHistory could affect
	MinDeletableMessage chat1.MessageID `codec:"em"`
}

const delhTrackerDiskVersion = 2

func newDelhTracker(g *globals.Context) *delhTracker {
	return &delhTracker{
		Contextified: globals.NewContextified(g),
		DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "DelhTracker", false),
	}
}

func (t *delhTracker) makeDbKey(convID chat1.ConversationID, uid gregor1.UID) libkb.DbKey {
	return libkb.DbKey{
		Typ: libkb.DBChatBlocks,
		Key: fmt.Sprintf("delh:%s:%s", uid, convID),
	}
}

func (t *delhTracker) getEntry(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID) (delhTrackerEntry, Error) {

	var blank delhTrackerEntry
	var res delhTrackerEntry

	dbKey := t.makeDbKey(convID, uid)
	raw, found, err := t.G().LocalChatDb.GetRaw(dbKey)
	if err != nil {
		return res, NewInternalError(ctx, t.DebugLabeler, "GetRaw error: %s", err.Error())
	}
	if !found {
		return res, MissError{}
	}

	err = decode(raw, &res)
	if err != nil {
		return blank, NewInternalError(ctx, t.DebugLabeler, "decode error: %s", err.Error())
	}
	switch res.StorageVersion {
	case delhTrackerDiskVersion:
		return res, nil
	default:
		// ignore other versions
		return blank, MissError{}
	}
}

func (t *delhTracker) setEntry(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, entry delhTrackerEntry) Error {

	entry.StorageVersion = delhTrackerDiskVersion
	data, err := encode(entry)
	if err != nil {
		return NewInternalError(ctx, t.DebugLabeler, "encode error: %s", err.Error())
	}

	dbKey := t.makeDbKey(convID, uid)
	err = t.G().LocalChatDb.PutRaw(dbKey, data)
	if err != nil {
		return NewInternalError(ctx, t.DebugLabeler, "PutRaw error: %s", err.Error())
	}
	t.Debug(ctx, "delhTracker.setEntry(%v, %+v)", convID, entry)
	return nil
}

func (t *delhTracker) setMaxDeleteHistoryUpto(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, msgid chat1.MessageID) Error {

	// No need to use transaction here since the Storage class takes lock.

	entry, err := t.getEntry(ctx, convID, uid)
	switch err.(type) {
	case nil:
	case MissError:
	default:
		return err
	}
	entry.MaxDeleteHistoryUpto = msgid
	return t.setEntry(ctx, convID, uid, entry)
}

func (t *delhTracker) setMinDeletableMessage(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, msgid chat1.MessageID) Error {

	entry, err := t.getEntry(ctx, convID, uid)
	switch err.(type) {
	case nil:
	case MissError:
	default:
		return err
	}
	entry.MinDeletableMessage = msgid
	return t.setEntry(ctx, convID, uid, entry)
}

// Set both values to msgid
func (t *delhTracker) setDeletedUpto(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, msgid chat1.MessageID) Error {

	entry, err := t.getEntry(ctx, convID, uid)
	switch err.(type) {
	case nil:
	case MissError:
	default:
		return err
	}
	entry.MaxDeleteHistoryUpto = msgid
	entry.MinDeletableMessage = msgid
	return t.setEntry(ctx, convID, uid, entry)
}
<commit_msg>break delh cache<commit_after>package storage

import (
	"fmt"

	"github.com/keybase/client/go/chat/globals"
	"github.com/keybase/client/go/chat/utils"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/chat1"
	"github.com/keybase/client/go/protocol/gregor1"
	"golang.org/x/net/context"
)

type delhTracker struct {
	globals.Contextified
	utils.DebugLabeler
}

type delhTrackerEntry struct {
	StorageVersion int `codec:"v"`
	// The latest DeleteHistory upto value that has been applied locally
	MaxDeleteHistoryUpto chat1.MessageID `codec:"ldhu"`
	// The most ancient message that a new DeleteHistory could affect
	MinDeletableMessage chat1.MessageID `codec:"em"`
}

const delhTrackerDiskVersion = 3

func newDelhTracker(g *globals.Context) *delhTracker {
	return &delhTracker{
		Contextified: globals.NewContextified(g),
		DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "DelhTracker", false),
	}
}

func (t *delhTracker) makeDbKey(convID chat1.ConversationID, uid gregor1.UID) libkb.DbKey {
	return libkb.DbKey{
		Typ: libkb.DBChatBlocks,
		Key: fmt.Sprintf("delh:%s:%s", uid, convID),
	}
}

func (t *delhTracker) getEntry(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID) (delhTrackerEntry, Error) {

	var blank delhTrackerEntry
	var res delhTrackerEntry

	dbKey := t.makeDbKey(convID, uid)
	raw, found, err := t.G().LocalChatDb.GetRaw(dbKey)
	if err != nil {
		return res, NewInternalError(ctx, t.DebugLabeler, "GetRaw error: %s", err.Error())
	}
	if !found {
		return res, MissError{}
	}

	err = decode(raw, &res)
	if err != nil {
		return blank, NewInternalError(ctx, t.DebugLabeler, "decode error: %s", err.Error())
	}
	switch res.StorageVersion {
	case delhTrackerDiskVersion:
		return res, nil
	default:
		// ignore other versions
		return blank, MissError{}
	}
}

func (t *delhTracker) setEntry(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, entry delhTrackerEntry) Error {

	entry.StorageVersion = delhTrackerDiskVersion
	data, err := encode(entry)
	if err != nil {
		return NewInternalError(ctx, t.DebugLabeler, "encode error: %s", err.Error())
	}

	dbKey := t.makeDbKey(convID, uid)
	err = t.G().LocalChatDb.PutRaw(dbKey, data)
	if err != nil {
		return NewInternalError(ctx, t.DebugLabeler, "PutRaw error: %s", err.Error())
	}
	t.Debug(ctx, "delhTracker.setEntry(%v, %+v)", convID, entry)
	return nil
}

func (t *delhTracker) setMaxDeleteHistoryUpto(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, msgid chat1.MessageID) Error {

	// No need to use transaction here since the Storage class takes lock.

	entry, err := t.getEntry(ctx, convID, uid)
	switch err.(type) {
	case nil:
	case MissError:
	default:
		return err
	}
	entry.MaxDeleteHistoryUpto = msgid
	return t.setEntry(ctx, convID, uid, entry)
}

func (t *delhTracker) setMinDeletableMessage(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, msgid chat1.MessageID) Error {

	entry, err := t.getEntry(ctx, convID, uid)
	switch err.(type) {
	case nil:
	case MissError:
	default:
		return err
	}
	entry.MinDeletableMessage = msgid
	return t.setEntry(ctx, convID, uid, entry)
}

// Set both values to msgid
func (t *delhTracker) setDeletedUpto(ctx context.Context,
	convID chat1.ConversationID, uid gregor1.UID, msgid chat1.MessageID) Error {

	entry, err := t.getEntry(ctx, convID, uid)
	switch err.(type) {
	case nil:
	case MissError:
	default:
		return err
	}
	entry.MaxDeleteHistoryUpto = msgid
	entry.MinDeletableMessage = msgid
	return t.setEntry(ctx, convID, uid, entry)
}
<|endoftext|>


<commit_before>package hadoopconf

import (
	"archive/zip"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

type HadoopConf struct {
	multiSourceConf
	CoreSite   *ConfWithDefault
	HdfsSite   *ConfWithDefault
	MapredSite *ConfWithDefault
	YarnSite   *ConfWithDefault
}

type HadoopDefaultConf struct {
	CoreSite   ConfSourcer
	HdfsSite   ConfSourcer
	MapredSite ConfSourcer
	YarnSite   ConfSourcer
}

func (c *HadoopConf) Save(backup bool) error {
	for _, conf := range []*ConfWithDefault{c.CoreSite, c.HdfsSite, c.MapredSite, c.YarnSite} {
		if err := conf.Conf.(*FileConfiguration).Save(backup); err != nil {
			return err
		}
	}
	return nil
}

func FromConf(coreSite *ConfWithDefault, hdfsSite *ConfWithDefault,
	mapredSite *ConfWithDefault, yarnSite *ConfWithDefault) *HadoopConf {
	confs := []ConfSourcer{coreSite, hdfsSite}
	if yarnSite != nil {
		confs = append(confs, yarnSite)
	}
	if mapredSite != nil {
		confs = append(confs, mapredSite)
	}
	return &HadoopConf{confs, coreSite, hdfsSite, mapredSite, yarnSite}
}

func anyRegexpMatch(s string, res []*regexp.Regexp) bool {
	for _, re := range res {
		if re.MatchString(s) {
			return true
		}
	}
	return false
}

func ConfsFromJar(jar string, files ...string) ([]ConfSourcer, error) {
	rv := make([]ConfSourcer, len(files))
	r, err := zip.OpenReader(jar)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	m := make(map[string]int)
	for i, name := range files {
		m[name] = i
	}
	for _, f := range r.File {
		if ix, ok := m[f.Name]; ok {
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			b, err := ioutil.ReadAll(r)
			if err != nil {
				return nil, err
			}
			if rv[ix], err = NewGeneratedConfFromBytes(Source{filepath.Join(jar, f.Name), FileFromJar}, b); err != nil {
				return nil, err
			}
		}
	}
	return rv, nil
}

func ConfFromJar(jar string, file string) (ConfSourcer, error) {
	confs, err := ConfsFromJar(jar, file)
	if err != nil {
		return nil, err
	}
	if confs[0] == nil {
		return nil, errors.New("Cannot find " + file + " in " + jar)
	}
	return confs[0], nil
}

func globRegexp(basedirs []string, res []*regexp.Regexp) (string, error) {
	for _, glob := range basedirs {
		dirs, err := filepath.Glob(glob)
		if err != nil {
			return "", err
		}
		for _, dir := range dirs {
			jars, err := ioutil.ReadDir(dir)
			if err != nil {
				return "", err
			}
			for _, jar := range jars {
				if anyRegexpMatch(jar.Name(), res) {
					return filepath.Join(dir, jar.Name()), nil
				}
			}
		}
	}
	return "", errors.New(fmt.Sprintln("in", basedirs, "jar was not found", res))
}

func getDefault(filename string, res []*regexp.Regexp, basedirs ...string) (ConfSourcer, error) {
	jar, err := globRegexp(basedirs, res)
	if err != nil {
		return nil, err
	}
	return ConfFromJar(jar, filename)
}

func getCoreDefault(basedirs ...string) (ConfSourcer, error) {
	hadoopCommonRegexp := []*regexp.Regexp{
		regexp.MustCompile(`hadoop-(common|core)-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`),
	}
	jar, err := globRegexp(basedirs, hadoopCommonRegexp)
	if err != nil {
		return nil, err
	}
	return ConfFromJar(jar, "core-default.xml")
}

func getConf(confName string, globs ...string) (*FileConfiguration, error) {
	for _, glob := range globs {
		dirs, err := filepath.Glob(glob)
		if err != nil {
			return nil, err
		}
		for _, dir := range dirs {
			conf, err := NewFileConfiguration(filepath.Join(dir, confName))
			if os.IsNotExist(err) {
				continue
			}
			if err != nil {
				return nil, err
			}
			return conf, nil
		}
	}
	return nil, errors.New(fmt.Sprintln("cannot find file", confName, "in any of", globs))
}

type returnPanic struct {
	err error
}

func re(pats ...string) []*regexp.Regexp {
	res := []*regexp.Regexp{}
	for _, pat := range pats {
		res = append(res, regexp.MustCompile(pat))
	}
	return res
}

func Jars(basedir string) (*HadoopDefaultConf, error) {
	coreDefault, err := getDefault("core-default.xml", re(`hadoop-(common|core)-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`, "hadoop-common.jar"), basedir,
		filepath.Join(basedir, "share/hadoop/common"),
		"/usr/lib/hadoop",
		"/share/hadoop/common")
	if err != nil {
		return nil, err
	}
	hdfsDefault, err := getDefault("hdfs-default.xml", re(`hadoop-(hdfs|core)-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`), basedir,
		filepath.Join(basedir, "share/hadoop/hdfs"),
		filepath.Join(basedir, "hadoop-hdfs"),
		"/share/hadoop/hdfs",
		"/usr/lib/hadoop-hdfs")
	if err != nil {
		return nil, err
	}
	mapredDefault, err := getDefault("mapred-default.xml", re(`hadoop-(mapreduce-client-)?core-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`), basedir,
		filepath.Join(basedir, "hadoop-0.20-mapreduce"),
		filepath.Join(basedir, "hadoop-mapreduce"),
		filepath.Join(basedir, "share/hadoop/mapreduce"),
		"/share/hadoop/mapreduce",
		"/usr/lib/hadoop-0.20-mapreduce",
		"/usr/lib/hadoop-mapreduce")
	if mapredDefault == nil {
		fmt.Println("got", err)
	}
	yarnDefault, _ := getDefault("yarn-default.xml", re(`hadoop-yarn-common-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`), basedir,
		filepath.Join(basedir, "share/hadoop/yarn"),
		filepath.Join(basedir, "hadoop-yarn"),
		"/usr/lib/hadoop-yarn",
		"/share/hadoop/yarn")
	return &HadoopDefaultConf{
		CoreSite:   coreDefault,
		HdfsSite:   hdfsDefault,
		MapredSite: mapredDefault,
		YarnSite:   yarnDefault,
	}, nil
}

func New(basedir string, defaultConf *HadoopDefaultConf) (conf *HadoopConf, err error) {
	j := func(s string) string {
		return filepath.Join(basedir, s)
	}
	coreSite, err := getConf("core-site.xml", j("etc/hadoop"), j("conf"), basedir)
	if err != nil {
		return nil, err
	}
	if _, err := os.Stat(coreSite.Path); err != nil {
		return nil, err
	}
	hdfsSite, err := getConf("hdfs-site.xml", j("etc/hadoop"), j("conf"), basedir)
	if err != nil {
		return nil, err
	}
	mapredSite, _ := getConf("mapred-site.xml", j("etc/hadoop"), j("conf"), basedir)
	yarnSite, _ := getConf("yarn-site.xml", j("etc/hadoop"), j("conf"), basedir)
	return FromConf(&ConfWithDefault{Default: defaultConf.CoreSite, Conf: coreSite},
		&ConfWithDefault{Default: defaultConf.HdfsSite, Conf: hdfsSite},
		&ConfWithDefault{Default: defaultConf.MapredSite, Conf: mapredSite},
		&ConfWithDefault{Default: defaultConf.YarnSite, Conf: yarnSite},
	), nil
}
<commit_msg>add missing configs to hdfs-default.xml<commit_after>package hadoopconf

import (
	"archive/zip"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

type HadoopConf struct {
	multiSourceConf
	CoreSite   *ConfWithDefault
	HdfsSite   *ConfWithDefault
	MapredSite *ConfWithDefault
	YarnSite   *ConfWithDefault
}

type HadoopDefaultConf struct {
	CoreSite   ConfSourcer
	HdfsSite   ConfSourcer
	MapredSite ConfSourcer
	YarnSite   ConfSourcer
}

func (c *HadoopConf) Save(backup bool) error {
	for _, conf := range []*ConfWithDefault{c.CoreSite, c.HdfsSite, c.MapredSite, c.YarnSite} {
		if err := conf.Conf.(*FileConfiguration).Save(backup); err != nil {
			return err
		}
	}
	return nil
}

func FromConf(coreSite *ConfWithDefault, hdfsSite *ConfWithDefault,
	mapredSite *ConfWithDefault, yarnSite *ConfWithDefault) *HadoopConf {
	confs := []ConfSourcer{coreSite, hdfsSite}
	if yarnSite != nil {
		confs = append(confs, yarnSite)
	}
	if mapredSite != nil {
		confs = append(confs, mapredSite)
	}
	return &HadoopConf{confs, coreSite, hdfsSite, mapredSite, yarnSite}
}

func anyRegexpMatch(s string, res []*regexp.Regexp) bool {
	for _, re := range res {
		if re.MatchString(s) {
			return true
		}
	}
	return false
}

func ConfsFromJar(jar string, files ...string) ([]ConfSourcer, error) {
	rv := make([]ConfSourcer, len(files))
	r, err := zip.OpenReader(jar)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	m := make(map[string]int)
	for i, name := range files {
		m[name] = i
	}
	for _, f := range r.File {
		if ix, ok := m[f.Name]; ok {
			r, err := f.Open()
			if err != nil {
				return nil, err
			}
			b, err := ioutil.ReadAll(r)
			if err != nil {
				return nil, err
			}
			if rv[ix], err = NewGeneratedConfFromBytes(Source{filepath.Join(jar, f.Name), FileFromJar}, b); err != nil {
				return nil, err
			}
		}
	}
	return rv, nil
}

func ConfFromJar(jar string, file string) (ConfSourcer, error) {
	confs, err := ConfsFromJar(jar, file)
	if err != nil {
		return nil, err
	}
	if confs[0] == nil {
		return nil, errors.New("Cannot find " + file + " in " + jar)
	}
	return confs[0], nil
}

func globRegexp(basedirs []string, res []*regexp.Regexp) (string, error) {
	for _, glob := range basedirs {
		dirs, err := filepath.Glob(glob)
		if err != nil {
			return "", err
		}
		for _, dir := range dirs {
			jars, err := ioutil.ReadDir(dir)
			if err != nil {
				return "", err
			}
			for _, jar := range jars {
				if anyRegexpMatch(jar.Name(), res) {
					return filepath.Join(dir, jar.Name()), nil
				}
			}
		}
	}
	return "", errors.New(fmt.Sprintln("in", basedirs, "jar was not found", res))
}

func getDefault(filename string, res []*regexp.Regexp, basedirs ...string) (ConfSourcer, error) {
	jar, err := globRegexp(basedirs, res)
	if err != nil {
		return nil, err
	}
	return ConfFromJar(jar, filename)
}

func getCoreDefault(basedirs ...string) (ConfSourcer, error) {
	hadoopCommonRegexp := []*regexp.Regexp{
		regexp.MustCompile(`hadoop-(common|core)-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`),
	}
	jar, err := globRegexp(basedirs, hadoopCommonRegexp)
	if err != nil {
		return nil, err
	}
	return ConfFromJar(jar, "core-default.xml")
}

func getConf(confName string, globs ...string) (*FileConfiguration, error) {
	for _, glob := range globs {
		dirs, err := filepath.Glob(glob)
		if err != nil {
			return nil, err
		}
		for _, dir := range dirs {
			conf, err := NewFileConfiguration(filepath.Join(dir, confName))
			if os.IsNotExist(err) {
				continue
			}
			if err != nil {
				return nil, err
			}
			return conf, nil
		}
	}
	return nil, errors.New(fmt.Sprintln("cannot find file", confName, "in any of", globs))
}

type returnPanic struct {
	err error
}

func re(pats ...string) []*regexp.Regexp {
	res := []*regexp.Regexp{}
	for _, pat := range pats {
		res = append(res, regexp.MustCompile(pat))
	}
	return res
}

func Jars(basedir string) (*HadoopDefaultConf, error) {
	coreDefault, err := getDefault("core-default.xml", re(`hadoop-(common|core)-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`, "hadoop-common.jar"), basedir,
		filepath.Join(basedir, "share/hadoop/common"),
		"/usr/lib/hadoop",
		"/share/hadoop/common")
	if err != nil {
		return nil, err
	}
	hdfsDefault, err := getDefault("hdfs-default.xml", re(`hadoop-(hdfs|core)-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`), basedir,
		filepath.Join(basedir, "share/hadoop/hdfs"),
		filepath.Join(basedir, "hadoop-hdfs"),
		"/share/hadoop/hdfs",
		"/usr/lib/hadoop-hdfs")
	if err != nil {
		return nil, err
	}
	// dfs.*.{kerberos,https}.principal dfs.*.keytab.file does not appear in hdfs-default.xml for some reason
	for _, role := range []string {"namenode", "namenode.secondary", "datanode"} {
		hdfsDefault.Set("dfs." + role + ".keytab.file", "")
		hdfsDefault.Set("dfs." + role + ".kerberos.principal", "")
		hdfsDefault.Set("dfs." + role + ".https.principal", "")
	}
	mapredDefault, err := getDefault("mapred-default.xml", re(`hadoop-(mapreduce-client-)?core-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`), basedir,
		filepath.Join(basedir, "hadoop-0.20-mapreduce"),
		filepath.Join(basedir, "hadoop-mapreduce"),
		filepath.Join(basedir, "share/hadoop/mapreduce"),
		"/share/hadoop/mapreduce",
		"/usr/lib/hadoop-0.20-mapreduce",
		"/usr/lib/hadoop-mapreduce")
	if mapredDefault == nil {
		fmt.Println("got", err)
	}
	yarnDefault, _ := getDefault("yarn-default.xml", re(`hadoop-yarn-common-[0-9.]+-?([a-zA-Z0-9._]+)?\.jar`), basedir,
		filepath.Join(basedir, "share/hadoop/yarn"),
		filepath.Join(basedir, "hadoop-yarn"),
		"/usr/lib/hadoop-yarn",
		"/share/hadoop/yarn")
	return &HadoopDefaultConf{
		CoreSite:   coreDefault,
		HdfsSite:   hdfsDefault,
		MapredSite: mapredDefault,
		YarnSite:   yarnDefault,
	}, nil
}

func New(basedir string, defaultConf *HadoopDefaultConf) (conf *HadoopConf, err error) {
	j := func(s string) string {
		return filepath.Join(basedir, s)
	}
	coreSite, err := getConf("core-site.xml", j("etc/hadoop"), j("conf"), basedir)
	if err != nil {
		return nil, err
	}
	if _, err := os.Stat(coreSite.Path); err != nil {
		return nil, err
	}
	hdfsSite, err := getConf("hdfs-site.xml", j("etc/hadoop"), j("conf"), basedir)
	if err != nil {
		return nil, err
	}
	mapredSite, _ := getConf("mapred-site.xml", j("etc/hadoop"), j("conf"), basedir)
	yarnSite, _ := getConf("yarn-site.xml", j("etc/hadoop"), j("conf"), basedir)
	return FromConf(&ConfWithDefault{Default: defaultConf.CoreSite, Conf: coreSite},
		&ConfWithDefault{Default: defaultConf.HdfsSite, Conf: hdfsSite},
		&ConfWithDefault{Default: defaultConf.MapredSite, Conf: mapredSite},
		&ConfWithDefault{Default: defaultConf.YarnSite, Conf: yarnSite},
	), nil
}
<|endoftext|>


<commit_before>package models

import (
	"strings"
)

// Data holds arbitrary key value.
type Data map[string]interface{}

// GetString returns value as string with given key
func (d *Data) GetString(key string) (string, error) {
	dt, err := d.Get(key)
	if err != nil {
		return "", err
	}

	s, ok := dt.(string)
	if !ok {
		return "", ErrDataInvalidType
	}

	return s, nil
}

// Get returns value with given key
func (d *Data) Get(key string) (interface{}, error) {
	if d == nil {
		return "", ErrDataKeyNotExists
	}

	dt, ok := findPath(strings.Split(key, "."), *d)
	if !ok {
		return "", ErrDataKeyNotExists
	}

	return dt, nil
}

func findPath(path []string, data Data) (interface{}, bool) {
	if len(path) == 0 {
		return nil, false
	}
	key := path[0]
	dt, ok := data[key]
	if !ok {
		return nil, false
	}

	if len(path) == 1 {
		return dt, true
	}

	next, ok := dt.(Data)
	if !ok {
		return nil, false
	}

	return findPath(path[1:], next)
}
<commit_msg>go/models: add GetData method<commit_after>package models

import (
	"strings"
)

// Data holds arbitrary key value.
type Data map[string]interface{}

// GetString returns value as string with given key
func (d *Data) GetString(key string) (string, error) {
	dt, err := d.Get(key)
	if err != nil {
		return "", err
	}

	s, ok := dt.(string)
	if !ok {
		return "", ErrDataInvalidType
	}

	return s, nil
}

// GetData returns value as models.Data with given key
func (d *Data) GetData(key string) (*Data, error) {
	dt, err := d.Get(key)
	if err != nil {
		return nil, err
	}

	s, ok := dt.(*Data)
	if !ok {
		return nil, ErrDataInvalidType
	}

	return s, nil
}

// Get returns value with given key
func (d *Data) Get(key string) (interface{}, error) {
	if d == nil {
		return "", ErrDataKeyNotExists
	}

	dt, ok := findPath(strings.Split(key, "."), *d)
	if !ok {
		return "", ErrDataKeyNotExists
	}

	return dt, nil
}

func findPath(path []string, data Data) (interface{}, bool) {
	if len(path) == 0 {
		return nil, false
	}
	key := path[0]
	dt, ok := data[key]
	if !ok {
		return nil, false
	}

	if len(path) == 1 {
		return dt, true
	}

	next, ok := dt.(Data)
	if !ok {
		return nil, false
	}

	return findPath(path[1:], next)
}
<|endoftext|>