<commit_before>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"fmt"
	"math"
	"testing"

	"github.com/gonum/blas"
	"github.com/gonum/blas/blas64"
)

var inf = math.Inf(1)

func leaksPanic(fn func()) (panicked bool) {
	defer func() {
		r := recover()
		panicked = r != nil
	}()
	Maybe(fn)
	return
}

func panics(fn func()) (panicked bool, message string) {
	defer func() {
		r := recover()
		panicked = r != nil
		message = fmt.Sprint(r)
	}()
	fn()
	return
}

func flatten(f [][]float64) (r, c int, d []float64) {
	r = len(f)
	if r == 0 {
		panic("bad test: no row")
	}
	c = len(f[0])
	d = make([]float64, 0, r*c)
	for _, row := range f {
		if len(row) != c {
			panic("bad test: ragged input")
		}
		d = append(d, row...)
	}
	return r, c, d
}

func unflatten(r, c int, d []float64) [][]float64 {
	m := make([][]float64, r)
	for i := 0; i < r; i++ {
		m[i] = d[i*c : (i+1)*c]
	}
	return m
}

func eye() *Dense {
	return NewDense(3, 3, []float64{
		1, 0, 0,
		0, 1, 0,
		0, 0, 1,
	})
}

func TestCol(t *testing.T) {
	f := func(a Matrix) interface{} {
		_, c := a.Dims()
		ans := make([][]float64, c)
		for j := range ans {
			ans[j] = Col(nil, j, a)
		}
		return ans
	}
	denseComparison := func(a *Dense) interface{} {
		_, c := a.Dims()
		ans := make([][]float64, c)
		for j := range ans {
			ans[j] = Col(nil, j, a)
		}
		return ans
	}
	testOneInputFunc(t, "Col", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
	f = func(a Matrix) interface{} {
		r, c := a.Dims()
		ans := make([][]float64, c)
		for j := range ans {
			ans[j] = make([]float64, r)
			Col(ans[j], j, a)
		}
		return ans
	}
	testOneInputFunc(t, "Col", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
}

func TestRow(t *testing.T) {
	f := func(a Matrix) interface{} {
		r, _ := a.Dims()
		ans := make([][]float64, r)
		for i := range ans {
			ans[i] = Row(nil, i, a)
		}
		return ans
	}
	denseComparison := func(a *Dense) interface{} {
		r, _ := a.Dims()
		ans := make([][]float64, r)
		for i := range ans {
			ans[i] = Row(nil, i, a)
		}
		return ans
	}
	testOneInputFunc(t, "Row", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
	f = func(a Matrix) interface{} {
		r, c := a.Dims()
		ans := make([][]float64, r)
		for i := range ans {
			ans[i] = make([]float64, c)
			Row(ans[i], i, a)
		}
		return ans
	}
	testOneInputFunc(t, "Row", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
}

func TestDot(t *testing.T) {
	f := func(a, b Matrix) interface{} {
		return Dot(a, b)
	}
	denseComparison := func(a, b *Dense) interface{} {
		return Dot(a, b)
	}
	testTwoInputFunc(t, "Dot", f, denseComparison, sameAnswerFloatApprox, legalTypesAll, legalSizeSameRectangular)
}

func TestEqual(t *testing.T) {
	f := func(a, b Matrix) interface{} {
		return Equal(a, b)
	}
	denseComparison := func(a, b *Dense) interface{} {
		return Equal(a, b)
	}
	testTwoInputFunc(t, "Equal", f, denseComparison, sameAnswerBool, legalTypesAll, isAnySize2)
}

func TestMax(t *testing.T) {
	// A direct test of Max with *Dense arguments is in TestNewDense.
	f := func(a Matrix) interface{} {
		return Max(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Max(a)
	}
	testOneInputFunc(t, "Max", f, denseComparison, sameAnswerFloat, isAnyType, isAnySize)
}

func TestMin(t *testing.T) {
	// A direct test of Min with *Dense arguments is in TestNewDense.
	f := func(a Matrix) interface{} {
		return Min(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Min(a)
	}
	testOneInputFunc(t, "Min", f, denseComparison, sameAnswerFloat, isAnyType, isAnySize)
}

func TestMaybe(t *testing.T) {
	for i, test := range []struct {
		fn     func()
		panics bool
	}{
		{
			func() {},
			false,
		},
		{
			func() { panic("panic") },
			true,
		},
		{
			func() { panic(Error{"panic"}) },
			false,
		},
	} {
		if panicked := leaksPanic(test.fn); panicked != test.panics {
			t.Errorf("unexpected panic state for test %d: got: panicked=%t want panicked=%t",
				i, panicked, test.panics)
		}
	}
}

func TestNorm(t *testing.T) {
	for i, test := range []struct {
		a    [][]float64
		ord  float64
		norm float64
	}{
		{
			a:    [][]float64{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}},
			ord:  1,
			norm: 30,
		},
		{
			a:    [][]float64{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}},
			ord:  2,
			norm: 25.495097567963924,
		},
		{
			a:    [][]float64{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}},
			ord:  inf,
			norm: 33,
		},
		{
			a:    [][]float64{{1, -2, -2}, {-4, 5, 6}},
			ord:  1,
			norm: 8,
		},
		{
			a:    [][]float64{{1, -2, -2}, {-4, 5, 6}},
			ord:  inf,
			norm: 15,
		},
	} {
		a := NewDense(flatten(test.a))
		if math.Abs(Norm(a, test.ord)-test.norm) > 1e-14 {
			t.Errorf("Mismatch test %d: %v norm = %f", i, test.a, test.norm)
		}
	}

	f := func(a Matrix) interface{} {
		return Norm(a, 1)
	}
	denseComparison := func(a *Dense) interface{} {
		return Norm(a, 1)
	}
	testOneInputFunc(t, "Norm_1", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)

	f = func(a Matrix) interface{} {
		return Norm(a, 2)
	}
	denseComparison = func(a *Dense) interface{} {
		return Norm(a, 2)
	}
	testOneInputFunc(t, "Norm_2", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)

	f = func(a Matrix) interface{} {
		return Norm(a, math.Inf(1))
	}
	denseComparison = func(a *Dense) interface{} {
		return Norm(a, math.Inf(1))
	}
	testOneInputFunc(t, "Norm_inf", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)
}

func TestNormZero(t *testing.T) {
	for _, a := range []Matrix{
		&Dense{},
		&SymDense{mat: blas64.Symmetric{Uplo: blas.Upper}},
		&TriDense{mat: blas64.Triangular{Uplo: blas.Upper, Diag: blas.NonUnit}},
		&Vector{},
	} {
		for _, norm := range []float64{1, 2, math.Inf(1)} {
			panicked, message := panics(func() { Norm(a, norm) })
			if panicked {
				t.Errorf("unexpected panic for Norm(&%T{}, %v): %v", a, norm, message)
			}
		}
	}
}

func TestSum(t *testing.T) {
	f := func(a Matrix) interface{} {
		return Sum(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Sum(a)
	}
	testOneInputFunc(t, "Sum", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)
}

func TestTrace(t *testing.T) {
	for _, test := range []struct {
		a     *Dense
		trace float64
	}{
		{
			a:     NewDense(3, 3, []float64{1, 2, 3, 4, 5, 6, 7, 8, 9}),
			trace: 15,
		},
	} {
		trace := Trace(test.a)
		if trace != test.trace {
			t.Errorf("Trace mismatch. Want %v, got %v", test.trace, trace)
		}
	}
	f := func(a Matrix) interface{} {
		return Trace(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Trace(a)
	}
	testOneInputFunc(t, "Trace", f, denseComparison, sameAnswerFloat, isAnyType, isSquare)
}
<commit_msg>Check result of Norm for zero-sized matrices and vectors<commit_after>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"fmt"
	"math"
	"testing"

	"github.com/gonum/blas"
	"github.com/gonum/blas/blas64"
)

var inf = math.Inf(1)

func leaksPanic(fn func()) (panicked bool) {
	defer func() {
		r := recover()
		panicked = r != nil
	}()
	Maybe(fn)
	return
}

func panics(fn func()) (panicked bool, message string) {
	defer func() {
		r := recover()
		panicked = r != nil
		message = fmt.Sprint(r)
	}()
	fn()
	return
}

func flatten(f [][]float64) (r, c int, d []float64) {
	r = len(f)
	if r == 0 {
		panic("bad test: no row")
	}
	c = len(f[0])
	d = make([]float64, 0, r*c)
	for _, row := range f {
		if len(row) != c {
			panic("bad test: ragged input")
		}
		d = append(d, row...)
	}
	return r, c, d
}

func unflatten(r, c int, d []float64) [][]float64 {
	m := make([][]float64, r)
	for i := 0; i < r; i++ {
		m[i] = d[i*c : (i+1)*c]
	}
	return m
}

func eye() *Dense {
	return NewDense(3, 3, []float64{
		1, 0, 0,
		0, 1, 0,
		0, 0, 1,
	})
}

func TestCol(t *testing.T) {
	f := func(a Matrix) interface{} {
		_, c := a.Dims()
		ans := make([][]float64, c)
		for j := range ans {
			ans[j] = Col(nil, j, a)
		}
		return ans
	}
	denseComparison := func(a *Dense) interface{} {
		_, c := a.Dims()
		ans := make([][]float64, c)
		for j := range ans {
			ans[j] = Col(nil, j, a)
		}
		return ans
	}
	testOneInputFunc(t, "Col", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
	f = func(a Matrix) interface{} {
		r, c := a.Dims()
		ans := make([][]float64, c)
		for j := range ans {
			ans[j] = make([]float64, r)
			Col(ans[j], j, a)
		}
		return ans
	}
	testOneInputFunc(t, "Col", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
}

func TestRow(t *testing.T) {
	f := func(a Matrix) interface{} {
		r, _ := a.Dims()
		ans := make([][]float64, r)
		for i := range ans {
			ans[i] = Row(nil, i, a)
		}
		return ans
	}
	denseComparison := func(a *Dense) interface{} {
		r, _ := a.Dims()
		ans := make([][]float64, r)
		for i := range ans {
			ans[i] = Row(nil, i, a)
		}
		return ans
	}
	testOneInputFunc(t, "Row", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
	f = func(a Matrix) interface{} {
		r, c := a.Dims()
		ans := make([][]float64, r)
		for i := range ans {
			ans[i] = make([]float64, c)
			Row(ans[i], i, a)
		}
		return ans
	}
	testOneInputFunc(t, "Row", f, denseComparison, sameAnswerF64SliceOfSlice, isAnyType, isAnySize)
}

func TestDot(t *testing.T) {
	f := func(a, b Matrix) interface{} {
		return Dot(a, b)
	}
	denseComparison := func(a, b *Dense) interface{} {
		return Dot(a, b)
	}
	testTwoInputFunc(t, "Dot", f, denseComparison, sameAnswerFloatApprox, legalTypesAll, legalSizeSameRectangular)
}

func TestEqual(t *testing.T) {
	f := func(a, b Matrix) interface{} {
		return Equal(a, b)
	}
	denseComparison := func(a, b *Dense) interface{} {
		return Equal(a, b)
	}
	testTwoInputFunc(t, "Equal", f, denseComparison, sameAnswerBool, legalTypesAll, isAnySize2)
}

func TestMax(t *testing.T) {
	// A direct test of Max with *Dense arguments is in TestNewDense.
	f := func(a Matrix) interface{} {
		return Max(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Max(a)
	}
	testOneInputFunc(t, "Max", f, denseComparison, sameAnswerFloat, isAnyType, isAnySize)
}

func TestMin(t *testing.T) {
	// A direct test of Min with *Dense arguments is in TestNewDense.
	f := func(a Matrix) interface{} {
		return Min(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Min(a)
	}
	testOneInputFunc(t, "Min", f, denseComparison, sameAnswerFloat, isAnyType, isAnySize)
}

func TestMaybe(t *testing.T) {
	for i, test := range []struct {
		fn     func()
		panics bool
	}{
		{
			func() {},
			false,
		},
		{
			func() { panic("panic") },
			true,
		},
		{
			func() { panic(Error{"panic"}) },
			false,
		},
	} {
		if panicked := leaksPanic(test.fn); panicked != test.panics {
			t.Errorf("unexpected panic state for test %d: got: panicked=%t want panicked=%t",
				i, panicked, test.panics)
		}
	}
}

func TestNorm(t *testing.T) {
	for i, test := range []struct {
		a    [][]float64
		ord  float64
		norm float64
	}{
		{
			a:    [][]float64{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}},
			ord:  1,
			norm: 30,
		},
		{
			a:    [][]float64{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}},
			ord:  2,
			norm: 25.495097567963924,
		},
		{
			a:    [][]float64{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}},
			ord:  inf,
			norm: 33,
		},
		{
			a:    [][]float64{{1, -2, -2}, {-4, 5, 6}},
			ord:  1,
			norm: 8,
		},
		{
			a:    [][]float64{{1, -2, -2}, {-4, 5, 6}},
			ord:  inf,
			norm: 15,
		},
	} {
		a := NewDense(flatten(test.a))
		if math.Abs(Norm(a, test.ord)-test.norm) > 1e-14 {
			t.Errorf("Mismatch test %d: %v norm = %f", i, test.a, test.norm)
		}
	}

	f := func(a Matrix) interface{} {
		return Norm(a, 1)
	}
	denseComparison := func(a *Dense) interface{} {
		return Norm(a, 1)
	}
	testOneInputFunc(t, "Norm_1", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)

	f = func(a Matrix) interface{} {
		return Norm(a, 2)
	}
	denseComparison = func(a *Dense) interface{} {
		return Norm(a, 2)
	}
	testOneInputFunc(t, "Norm_2", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)

	f = func(a Matrix) interface{} {
		return Norm(a, math.Inf(1))
	}
	denseComparison = func(a *Dense) interface{} {
		return Norm(a, math.Inf(1))
	}
	testOneInputFunc(t, "Norm_inf", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)
}

func TestNormZero(t *testing.T) {
	for _, a := range []Matrix{
		&Dense{},
		&SymDense{mat: blas64.Symmetric{Uplo: blas.Upper}},
		&TriDense{mat: blas64.Triangular{Uplo: blas.Upper, Diag: blas.NonUnit}},
		&Vector{},
	} {
		want := 0.0
		for _, norm := range []float64{1, 2, math.Inf(1)} {
			var got float64
			panicked, message := panics(func() { got = Norm(a, norm) })
			if panicked {
				t.Errorf("unexpected panic for Norm(&%T{}, %v): %v", a, norm, message)
			}
			if got != want {
				t.Errorf("unexpected result for Norm(&%T{}, %v). Want %v, got %v", a, norm, want, got)
			}
		}
	}
}

func TestSum(t *testing.T) {
	f := func(a Matrix) interface{} {
		return Sum(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Sum(a)
	}
	testOneInputFunc(t, "Sum", f, denseComparison, sameAnswerFloatApprox, isAnyType, isAnySize)
}

func TestTrace(t *testing.T) {
	for _, test := range []struct {
		a     *Dense
		trace float64
	}{
		{
			a:     NewDense(3, 3, []float64{1, 2, 3, 4, 5, 6, 7, 8, 9}),
			trace: 15,
		},
	} {
		trace := Trace(test.a)
		if trace != test.trace {
			t.Errorf("Trace mismatch. Want %v, got %v", test.trace, trace)
		}
	}
	f := func(a Matrix) interface{} {
		return Trace(a)
	}
	denseComparison := func(a *Dense) interface{} {
		return Trace(a)
	}
	testOneInputFunc(t, "Trace", f, denseComparison, sameAnswerFloat, isAnyType, isSquare)
}
<|endoftext|>


<commit_before>package matchers

import (
	"bytes"
	"encoding/binary"
)

var (
	TypeDoc  = newType("doc", "application/msword")
	TypeDocx = newType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
	TypeXls  = newType("xls", "application/vnd.ms-excel")
	TypeXlsx = newType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
	TypePpt  = newType("ppt", "application/vnd.ms-powerpoint")
	TypePptx = newType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation")
)

var Document = Map{
	TypeDoc:  Doc,
	TypeDocx: Docx,
	TypeXls:  Xls,
	TypeXlsx: Xlsx,
	TypePpt:  Ppt,
	TypePptx: Pptx,
}

type docType int

const (
	TYPE_DOC docType = iota
	TYPE_DOCX
	TYPE_XLS
	TYPE_XLSX
	TYPE_PPT
	TYPE_PPTX
	TYPE_OOXML
)

func Doc(buf []byte) bool {
	return len(buf) > 7 &&
		buf[0] == 0xD0 && buf[1] == 0xCF &&
		buf[2] == 0x11 && buf[3] == 0xE0 &&
		buf[4] == 0xA1 && buf[5] == 0xB1 &&
		buf[6] == 0x1A && buf[7] == 0xE1
}

func Docx(buf []byte) bool {
	typ, ok := msooxml(buf)
	return ok && typ == TYPE_DOCX
}

func Xls(buf []byte) bool {
	return len(buf) > 7 &&
		buf[0] == 0xD0 && buf[1] == 0xCF &&
		buf[2] == 0x11 && buf[3] == 0xE0 &&
		buf[4] == 0xA1 && buf[5] == 0xB1 &&
		buf[6] == 0x1A && buf[7] == 0xE1
}

func Xlsx(buf []byte) bool {
	typ, ok := msooxml(buf)
	return ok && typ == TYPE_XLSX
}

func Ppt(buf []byte) bool {
	return len(buf) > 7 &&
		buf[0] == 0xD0 && buf[1] == 0xCF &&
		buf[2] == 0x11 && buf[3] == 0xE0 &&
		buf[4] == 0xA1 && buf[5] == 0xB1 &&
		buf[6] == 0x1A && buf[7] == 0xE1
}

func Pptx(buf []byte) bool {
	typ, ok := msooxml(buf)
	return ok && typ == TYPE_PPTX
}

func msooxml(buf []byte) (typ docType, found bool) {
	signature := []byte{'P', 'K', 0x03, 0x04}

	// start by checking for ZIP local file header signature
	if ok := compareBytes(buf, signature, 0); !ok {
		return
	}

	// make sure the first file is correct
	if v, ok := checkMSOoml(buf, 0x1E); ok {
		return v, ok
	}

	if !compareBytes(buf, []byte("[Content_Types].xml"), 0x1E) && !compareBytes(buf, []byte("_rels/.rels"), 0x1E) {
		return
	}

	// skip to the second local file header
	// since some documents include a 520-byte extra field following the file
	// header, we need to scan for the next header
	startOffset := int(binary.LittleEndian.Uint32(buf[18:22]) + 49)
	idx := search(buf, startOffset, 6000)
	if idx == -1 {
		return
	}

	// now skip to the *third* local file header; again, we need to scan due to a
	// 520-byte extra field following the file header
	startOffset += idx + 4 + 26
	idx = search(buf, startOffset, 6000)
	if idx == -1 {
		return
	}

	// and check the subdirectory name to determine which type of OOXML
	// file we have.  Correct the mimetype with the registered ones:
	// http://technet.microsoft.com/en-us/library/cc179224.aspx
	startOffset += idx + 4 + 26
	if typ, ok := checkMSOoml(buf, startOffset); ok {
		return typ, ok
	}

	// OpenOffice/Libreoffice orders ZIP entry differently, so check the 4th file
	startOffset += 26
	idx = search(buf, startOffset, 6000)
	if idx == -1 {
		return TYPE_OOXML, true
	}

	startOffset += idx + 4 + 26
	if typ, ok := checkMSOoml(buf, startOffset); ok {
		return typ, ok
	} else {
		return TYPE_OOXML, true
	}
}

func compareBytes(slice, subSlice []byte, startOffset int) bool {
	sl := len(subSlice)

	if startOffset+sl > len(slice) {
		return false
	}

	s := slice[startOffset : startOffset+sl]
	for i := range s {
		if subSlice[i] != s[i] {
			return false
		}
	}

	return true
}

func checkMSOoml(buf []byte, offset int) (typ docType, ok bool) {
	ok = true

	switch {
	case compareBytes(buf, []byte("word/"), offset):
		typ = TYPE_DOCX
	case compareBytes(buf, []byte("ppt/"), offset):
		typ = TYPE_PPTX
	case compareBytes(buf, []byte("xl/"), offset):
		typ = TYPE_XLSX
	default:
		ok = false
	}

	return
}

func search(buf []byte, start, rangeNum int) int {
	length := len(buf)
	end := start + rangeNum
	signature := []byte{'P', 'K', 0x03, 0x04}

	if end > length {
		end = length
	}

	if start >= end {
		return -1
	}

	return bytes.Index(buf[start:end], signature)
}
<commit_msg>fix msooxml<commit_after>package matchers

import (
	"bytes"
	"encoding/binary"
)

var (
	TypeDoc  = newType("doc", "application/msword")
	TypeDocx = newType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
	TypeXls  = newType("xls", "application/vnd.ms-excel")
	TypeXlsx = newType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
	TypePpt  = newType("ppt", "application/vnd.ms-powerpoint")
	TypePptx = newType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation")
)

var Document = Map{
	TypeDoc:  Doc,
	TypeDocx: Docx,
	TypeXls:  Xls,
	TypeXlsx: Xlsx,
	TypePpt:  Ppt,
	TypePptx: Pptx,
}

type docType int

const (
	TYPE_DOC docType = iota
	TYPE_DOCX
	TYPE_XLS
	TYPE_XLSX
	TYPE_PPT
	TYPE_PPTX
	TYPE_OOXML
)

func Doc(buf []byte) bool {
	return len(buf) > 7 &&
		buf[0] == 0xD0 && buf[1] == 0xCF &&
		buf[2] == 0x11 && buf[3] == 0xE0 &&
		buf[4] == 0xA1 && buf[5] == 0xB1 &&
		buf[6] == 0x1A && buf[7] == 0xE1
}

func Docx(buf []byte) bool {
	typ, ok := msooxml(buf)
	return ok && typ == TYPE_DOCX
}

func Xls(buf []byte) bool {
	return len(buf) > 7 &&
		buf[0] == 0xD0 && buf[1] == 0xCF &&
		buf[2] == 0x11 && buf[3] == 0xE0 &&
		buf[4] == 0xA1 && buf[5] == 0xB1 &&
		buf[6] == 0x1A && buf[7] == 0xE1
}

func Xlsx(buf []byte) bool {
	typ, ok := msooxml(buf)
	return ok && typ == TYPE_XLSX
}

func Ppt(buf []byte) bool {
	return len(buf) > 7 &&
		buf[0] == 0xD0 && buf[1] == 0xCF &&
		buf[2] == 0x11 && buf[3] == 0xE0 &&
		buf[4] == 0xA1 && buf[5] == 0xB1 &&
		buf[6] == 0x1A && buf[7] == 0xE1
}

func Pptx(buf []byte) bool {
	typ, ok := msooxml(buf)
	return ok && typ == TYPE_PPTX
}

func msooxml(buf []byte) (typ docType, found bool) {
	signature := []byte{'P', 'K', 0x03, 0x04}

	// start by checking for ZIP local file header signature
	if ok := compareBytes(buf, signature, 0); !ok {
		return
	}

	// make sure the first file is correct
	if v, ok := checkMSOoml(buf, 0x1E); ok {
		return v, ok
	}

	if !compareBytes(buf, []byte("[Content_Types].xml"), 0x1E) &&
		!compareBytes(buf, []byte("_rels/.rels"), 0x1E) &&
		!compareBytes(buf, []byte("docProps"), 0x1E) {
		return
	}

	// skip to the second local file header
	// since some documents include a 520-byte extra field following the file
	// header, we need to scan for the next header
	startOffset := int(binary.LittleEndian.Uint32(buf[18:22]) + 49)
	idx := search(buf, startOffset, 6000)
	if idx == -1 {
		return
	}

	// now skip to the *third* local file header; again, we need to scan due to a
	// 520-byte extra field following the file header
	startOffset += idx + 4 + 26
	idx = search(buf, startOffset, 6000)
	if idx == -1 {
		return
	}

	// and check the subdirectory name to determine which type of OOXML
	// file we have.  Correct the mimetype with the registered ones:
	// http://technet.microsoft.com/en-us/library/cc179224.aspx
	startOffset += idx + 4 + 26
	if typ, ok := checkMSOoml(buf, startOffset); ok {
		return typ, ok
	}

	// OpenOffice/Libreoffice orders ZIP entry differently, so check the 4th file
	startOffset += 26
	idx = search(buf, startOffset, 6000)
	if idx == -1 {
		return TYPE_OOXML, true
	}

	startOffset += idx + 4 + 26
	if typ, ok := checkMSOoml(buf, startOffset); ok {
		return typ, ok
	} else {
		return TYPE_OOXML, true
	}
}

func compareBytes(slice, subSlice []byte, startOffset int) bool {
	sl := len(subSlice)

	if startOffset+sl > len(slice) {
		return false
	}

	s := slice[startOffset : startOffset+sl]
	for i := range s {
		if subSlice[i] != s[i] {
			return false
		}
	}

	return true
}

func checkMSOoml(buf []byte, offset int) (typ docType, ok bool) {
	ok = true

	switch {
	case compareBytes(buf, []byte("word/"), offset):
		typ = TYPE_DOCX
	case compareBytes(buf, []byte("ppt/"), offset):
		typ = TYPE_PPTX
	case compareBytes(buf, []byte("xl/"), offset):
		typ = TYPE_XLSX
	default:
		ok = false
	}

	return
}

func search(buf []byte, start, rangeNum int) int {
	length := len(buf)
	end := start + rangeNum
	signature := []byte{'P', 'K', 0x03, 0x04}

	if end > length {
		end = length
	}

	if start >= end {
		return -1
	}

	return bytes.Index(buf[start:end], signature)
}
<|endoftext|>


<commit_before>package matchers

import (
	"github.com/h2non/filetype/types"
)

// Internal shortcut to NewType
var newType = types.NewType

// Matcher function interface as type alias
type Matcher func([]byte) bool

// Type interface to store pairs of type with its matcher function
type Map map[types.Type]Matcher

// Type specific matcher function interface
type TypeMatcher func([]byte) types.Type

// Store registered file type matchers
var Matchers = make(map[types.Type]TypeMatcher)
var MatcherKeys []types.Type

// Create and register a new type matcher function
func NewMatcher(kind types.Type, fn Matcher) TypeMatcher {
	matcher := func(buf []byte) types.Type {
		if fn(buf) {
			return kind
		}
		return types.Unknown
	}

	Matchers[kind] = matcher
	MatcherKeys = append(MatcherKeys, kind)
	return matcher
}

func register(matchers ...Map) {
	MatcherKeys = MatcherKeys[:0]
	for _, m := range matchers {
		for kind, matcher := range m {
			NewMatcher(kind, matcher)
		}
	}
}

func init() {
	// Arguments order is intentional
	register(Image, Video, Audio, Font, Document, Archive)
}
<commit_msg>bugfix: reverse order of matcher key list so user registered matchers appear first<commit_after>package matchers

import (
	"github.com/h2non/filetype/types"
)

// Internal shortcut to NewType
var newType = types.NewType

// Matcher function interface as type alias
type Matcher func([]byte) bool

// Type interface to store pairs of type with its matcher function
type Map map[types.Type]Matcher

// Type specific matcher function interface
type TypeMatcher func([]byte) types.Type

// Store registered file type matchers
var Matchers = make(map[types.Type]TypeMatcher)
var MatcherKeys []types.Type

// Create and register a new type matcher function
func NewMatcher(kind types.Type, fn Matcher) TypeMatcher {
	matcher := func(buf []byte) types.Type {
		if fn(buf) {
			return kind
		}
		return types.Unknown
	}

	Matchers[kind] = matcher
	// prepend here so any user defined matchers get added first
	MatcherKeys = append([]types.Type{kind}, MatcherKeys...)
	return matcher
}

func register(matchers ...Map) {
	MatcherKeys = MatcherKeys[:0]
	for _, m := range matchers {
		for kind, matcher := range m {
			NewMatcher(kind, matcher)
		}
	}
}

func init() {
	// Arguments order is intentional
	// Archive files will be checked last due to prepend above in func NewMatcher
	register(Archive, Document, Font, Audio, Video, Image)
}
<|endoftext|>


<commit_before>package conf
import (
	"encoding/xml"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"path"
	"math"
)

type XConfig struct {
	XMLName    xml.Name    `xml:"config"`
	Server     XServer     `xml:"server"`
	Users      []XUser     `xml:"user"`
	Commands   []XCommands `xml:"commands"`
	Files      []XFiles    `xml:"files"`
	Databases  []XDatabase `xml:"database"`
}

type XServer struct {
	Listen  string      `xml:"listen"`
	Auth    XAuth       `xml:"auth"`
}

type XAuth struct {
	Enabled       bool     `xml:"enabled,attr"`
	MaxTimeDelta  uint32   `xml:"maxTimeDelta"`
}

type XUser struct {
	Name      string           `xml:"id,attr"`
	Hosts     []string         `xml:"host"`
	Key       string           `xml:"key"`
	Files     []XUserFiles     `xml:"files"`
	Commands  []XUserCommands  `xml:"commands"`
	Databases []XUserDatabases `xml:"databases"`
}

type XCommands struct {
	Name     string      `xml:"id,attr"`
	Commands []XCommand  `xml:"command"`
}

type XCommand struct {
	Name         string  `xml:"id,attr"`
	Lang         string	 `xml:"lang,attr"`
	Code         string  `xml:"code"`
	Timeout      uint32  `xml:"timeout,attr"`
	User         string  `xml:"runas,attr"`
	Lock         XLock   `xml:"lock"`
}

type XDatabase struct {
	Name    string    `xml:"id,attr"`
	Driver  string    `xml:"driver,attr"`
	Dsn     string    `xml:"dsn,attr"`
	Queries []XQuery  `xml:"query"`
}

type XQuery struct {
	Name    string `xml:"id,attr"`
	Sql     string `xml:",chardata"`
}

type XLock struct {
	Name     string  `xml:"id,attr"`
	Timeout  uint    `xml:"timeout,attr"`
	Wait     bool    `xml:"wait,attr"`
}

type XFiles struct {
	Name   string       `xml:"id,attr"`
	Dirs   []XDir       `xml:"dir"`
}

type XDir struct {
	Name      string    `xml:"id,attr"`
	Root      string    `xml:"root"`
	Allows    []string  `xml:"allow"`
	Patterns  []string  `xml:"pattern"`
}

type XUserFiles struct {
	Name   string   `xml:"id,attr"`
}

type XUserCommands struct {
	Name   string   `xml:"id,attr"`
}

type XUserDatabases struct {
	Name   string   `xml:"id,attr"`
}

func XConfigFromData(data []byte) (*XConfig, error) {
	ret := XConfig{}
	err := xml.Unmarshal(data, &ret)
	if err != nil {
		return nil, err
	}
	return &ret, nil
}

func XConfigFromReader(reader io.Reader) (*XConfig, error) {
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	return XConfigFromData(data)
}

func XConfigFromFile(path string) (*XConfig, error) {
	reader, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	return XConfigFromReader(reader)
}

func (conf *XConfig) ToConfig() *Config {
	ret := Config{}
	conf.IntoConfig(&ret)
	return &ret
}

func (conf *XConfig) IntoConfig(ret *Config) {
	if ret.Server.Listen == "" {
		ret.Server = Server{
			Listen: conf.Server.Listen,
		}
		ret.Auth = Auth {
			Enabled:      conf.Server.Auth.Enabled,
			MaxTimeDelta: conf.Server.Auth.MaxTimeDelta,
		}
	}

	ret.Files = make(map[string]*Files)
	for _, file := range(conf.Files) {
		fname := file.Name
		ret.Files[fname] = &Files{
			Dirs: make(map[string]*Dir),
		}
		for _, dir := range(file.Dirs) {
			dname := dir.Name
			dir := &Dir{
				Root: path.Clean(strings.TrimSpace(dir.Root)),
				Allows: make([]string, 0, 4),
				Patterns: make([]string, 0, 4),
			}
			for _, method := range(dir.Allows) {
				dir.Allows = append(dir.Allows, strings.ToUpper(strings.TrimSpace(method)))
			}
			for _, pattern := range(dir.Patterns) {
				dir.Patterns = append(dir.Patterns, strings.TrimSpace(pattern))
			}
			ret.Files[fname].Dirs[dname] = dir
		}
	}
	ret.Commands = make(map[string]*Commands)
	for _, commands := range(conf.Commands) {
		csname := commands.Name
		ret.Commands[csname] = &Commands{
			Commands: make(map[string]*Command),
		}
		for _, command := range(commands.Commands) {
			cname := command.Name
			if command.Timeout == 0 {
				command.Timeout = math.MaxUint32
			}
			if command.Lock.Timeout == 0 {
				command.Lock.Timeout = math.MaxUint32
			}
			ret.Commands[csname].Commands[cname] = &Command{
				Code: strings.TrimSpace(command.Code),
				Lang: command.Lang,
				User: command.User,
				Timeout: command.Timeout,
				Lock: Lock {
					Name: strings.TrimSpace(command.Lock.Name),
					Timeout: command.Lock.Timeout,
					Wait: command.Lock.Wait,
				},
			}
		}
	}
	ret.Databases = make(map[string]*Database)
	for _, database := range(conf.Databases) {
		dname := database.Name
		ret.Databases[dname] = &Database{
			Dsn: database.Dsn,
			Driver: database.Driver,
			Queries: make(map[string]*Query),
		}
		for _, query := range(database.Queries) {
			ret.Databases[dname].Queries[query.Name] = &Query{ Sql: query.Sql }
		}
	}
	ret.Users = make(map[string]*User)
	for _, user := range(conf.Users) {
		uname := user.Name
		u := &User{
			Key: strings.TrimSpace(user.Key),
			Hosts: make([]string, len(user.Hosts)),
		}
		for j := range(user.Hosts) {
			u.Hosts[j] = strings.TrimSpace(user.Hosts[j])
		}
		u.Allows = make(map[string][]string)
		u.Allows["commands"] = make([]string, 0, 2)
		u.Allows["files"] = make([]string, 0, 2)
		u.Allows["databases"] = make([]string, 0, 2)
		for _, command := range(user.Commands) {
			u.Allows["commands"] = append(u.Allows["commands"], command.Name)
		}
		for _, file := range(user.Files) {
			u.Allows["files"] = append(u.Allows["files"], file.Name)
		}
		for _, database := range(user.Databases) {
			u.Allows["databases"] = append(u.Allows["databases"], database.Name)
		}
		ret.Users[uname] = u
	}
}

<commit_msg>fix file config<commit_after>package conf
import (
	"encoding/xml"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"path"
	"math"
)

type XConfig struct {
	XMLName    xml.Name    `xml:"config"`
	Server     XServer     `xml:"server"`
	Users      []XUser     `xml:"user"`
	Commands   []XCommands `xml:"commands"`
	Files      []XFiles    `xml:"files"`
	Databases  []XDatabase `xml:"database"`
}

type XServer struct {
	Listen  string      `xml:"listen"`
	Auth    XAuth       `xml:"auth"`
}

type XAuth struct {
	Enabled       bool     `xml:"enabled,attr"`
	MaxTimeDelta  uint32   `xml:"maxTimeDelta"`
}

type XUser struct {
	Name      string           `xml:"id,attr"`
	Hosts     []string         `xml:"host"`
	Key       string           `xml:"key"`
	Files     []XUserFiles     `xml:"files"`
	Commands  []XUserCommands  `xml:"commands"`
	Databases []XUserDatabases `xml:"databases"`
}

type XCommands struct {
	Name     string      `xml:"id,attr"`
	Commands []XCommand  `xml:"command"`
}

type XCommand struct {
	Name         string  `xml:"id,attr"`
	Lang         string	 `xml:"lang,attr"`
	Code         string  `xml:"code"`
	Timeout      uint32  `xml:"timeout,attr"`
	User         string  `xml:"runas,attr"`
	Lock         XLock   `xml:"lock"`
}

type XDatabase struct {
	Name    string    `xml:"id,attr"`
	Driver  string    `xml:"driver,attr"`
	Dsn     string    `xml:"dsn,attr"`
	Queries []XQuery  `xml:"query"`
}

type XQuery struct {
	Name    string `xml:"id,attr"`
	Sql     string `xml:",chardata"`
}

type XLock struct {
	Name     string  `xml:"id,attr"`
	Timeout  uint    `xml:"timeout,attr"`
	Wait     bool    `xml:"wait,attr"`
}

type XFiles struct {
	Name   string       `xml:"id,attr"`
	Dirs   []XDir       `xml:"dir"`
}

type XDir struct {
	Name      string    `xml:"id,attr"`
	Root      string    `xml:"root"`
	Allows    []string  `xml:"allow"`
	Patterns  []string  `xml:"pattern"`
}

type XUserFiles struct {
	Name   string   `xml:"id,attr"`
}

type XUserCommands struct {
	Name   string   `xml:"id,attr"`
}

type XUserDatabases struct {
	Name   string   `xml:"id,attr"`
}

func XConfigFromData(data []byte) (*XConfig, error) {
	ret := XConfig{}
	err := xml.Unmarshal(data, &ret)
	if err != nil {
		return nil, err
	}
	return &ret, nil
}

func XConfigFromReader(reader io.Reader) (*XConfig, error) {
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	return XConfigFromData(data)
}

func XConfigFromFile(path string) (*XConfig, error) {
	reader, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	return XConfigFromReader(reader)
}

func (conf *XConfig) ToConfig() *Config {
	ret := Config{}
	conf.IntoConfig(&ret)
	return &ret
}

func (conf *XConfig) IntoConfig(ret *Config) {
	if ret.Server.Listen == "" {
		ret.Server = Server{
			Listen: conf.Server.Listen,
		}
		ret.Auth = Auth {
			Enabled:      conf.Server.Auth.Enabled,
			MaxTimeDelta: conf.Server.Auth.MaxTimeDelta,
		}
	}

	ret.Files = make(map[string]*Files)
	for _, file := range(conf.Files) {
		fname := file.Name
		ret.Files[fname] = &Files{
			Dirs: make(map[string]*Dir),
		}
		for _, xdir := range(file.Dirs) {
			dname := xdir.Name
			dir := &Dir{
				Root: path.Clean(strings.TrimSpace(xdir.Root)),
				Allows: make([]string, 0, 4),
				Patterns: make([]string, 0, 4),
			}
			for _, method := range(xdir.Allows) {
				dir.Allows = append(dir.Allows, strings.ToUpper(strings.TrimSpace(method)))
			}
			for _, pattern := range(xdir.Patterns) {
				dir.Patterns = append(dir.Patterns, strings.TrimSpace(pattern))
			}
			ret.Files[fname].Dirs[dname] = dir
		}
	}
	ret.Commands = make(map[string]*Commands)
	for _, commands := range(conf.Commands) {
		csname := commands.Name
		ret.Commands[csname] = &Commands{
			Commands: make(map[string]*Command),
		}
		for _, command := range(commands.Commands) {
			cname := command.Name
			if command.Timeout == 0 {
				command.Timeout = math.MaxUint32
			}
			if command.Lock.Timeout == 0 {
				command.Lock.Timeout = math.MaxUint32
			}
			ret.Commands[csname].Commands[cname] = &Command{
				Code: strings.TrimSpace(command.Code),
				Lang: command.Lang,
				User: command.User,
				Timeout: command.Timeout,
				Lock: Lock {
					Name: strings.TrimSpace(command.Lock.Name),
					Timeout: command.Lock.Timeout,
					Wait: command.Lock.Wait,
				},
			}
		}
	}
	ret.Databases = make(map[string]*Database)
	for _, database := range(conf.Databases) {
		dname := database.Name
		ret.Databases[dname] = &Database{
			Dsn: database.Dsn,
			Driver: database.Driver,
			Queries: make(map[string]*Query),
		}
		for _, query := range(database.Queries) {
			ret.Databases[dname].Queries[query.Name] = &Query{ Sql: query.Sql }
		}
	}
	ret.Users = make(map[string]*User)
	for _, user := range(conf.Users) {
		uname := user.Name
		u := &User{
			Key: strings.TrimSpace(user.Key),
			Hosts: make([]string, len(user.Hosts)),
		}
		for j := range(user.Hosts) {
			u.Hosts[j] = strings.TrimSpace(user.Hosts[j])
		}
		u.Allows = make(map[string][]string)
		u.Allows["commands"] = make([]string, 0, 2)
		u.Allows["files"] = make([]string, 0, 2)
		u.Allows["databases"] = make([]string, 0, 2)
		for _, command := range(user.Commands) {
			u.Allows["commands"] = append(u.Allows["commands"], command.Name)
		}
		for _, file := range(user.Files) {
			u.Allows["files"] = append(u.Allows["files"], file.Name)
		}
		for _, database := range(user.Databases) {
			u.Allows["databases"] = append(u.Allows["databases"], database.Name)
		}
		ret.Users[uname] = u
	}
}

<|endoftext|>


<commit_before>package proxy

import (
	"context"
	"log"
	"net"
	"os"
	"testing"

	"github.com/hashicorp/consul/agent"
	agConnect "github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/connect"
	"github.com/hashicorp/consul/lib/freeport"
	"github.com/hashicorp/consul/testutil/retry"
	"github.com/stretchr/testify/require"
)

func TestProxy_public(t *testing.T) {
	t.Parallel()

	require := require.New(t)
	ports := freeport.GetT(t, 1)

	a := agent.NewTestAgent(t.Name(), "")
	defer a.Shutdown()
	client := a.Client()

	// Register the service so we can get a leaf cert
	_, err := client.Catalog().Register(&api.CatalogRegistration{
		Datacenter: "dc1",
		Node:       "local",
		Address:    "127.0.0.1",
		Service: &api.AgentService{
			Service: "echo",
		},
	}, nil)
	require.NoError(err)

	// Start the backend service that is being proxied
	testApp := NewTestTCPServer(t)
	defer testApp.Close()

	// Start the proxy
	p, err := New(client, NewStaticConfigWatcher(&Config{
		ProxiedServiceName: "echo",
		PublicListener: PublicListenerConfig{
			BindAddress:         "127.0.0.1",
			BindPort:            ports[0],
			LocalServiceAddress: testApp.Addr().String(),
		},
	}), testLogger(t))
	require.NoError(err)
	defer p.Close()
	go p.Serve()

	// Create a test connection to the proxy. We retry here a few times
	// since this is dependent on the agent actually starting up and setting
	// up the CA.
	var conn net.Conn
	svc, err := connect.NewService("echo", client)
	require.NoError(err)
	retry.Run(t, func(r *retry.R) {
		conn, err = svc.Dial(context.Background(), &connect.StaticResolver{
			Addr:    TestLocalAddr(ports[0]),
			CertURI: agConnect.TestSpiffeIDService(t, "echo"),
		})
		if err != nil {
			r.Fatalf("err: %s", err)
		}
	})

	// Connection works, test it is the right one
	TestEchoConn(t, conn, "")
}

func testLogger(t *testing.T) *log.Logger {
	return log.New(os.Stderr, "", log.LstdFlags)
}
<commit_msg>Fixed unstable test TestProxy_public (#4587)<commit_after>package proxy

import (
	"context"
	"log"
	"net"
	"os"
	"testing"

	"github.com/hashicorp/consul/testrpc"

	"github.com/hashicorp/consul/agent"
	agConnect "github.com/hashicorp/consul/agent/connect"
	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/connect"
	"github.com/hashicorp/consul/lib/freeport"
	"github.com/hashicorp/consul/testutil/retry"
	"github.com/stretchr/testify/require"
)

func TestProxy_public(t *testing.T) {
	t.Parallel()

	require := require.New(t)
	ports := freeport.GetT(t, 1)

	a := agent.NewTestAgent(t.Name(), "")
	defer a.Shutdown()
	client := a.Client()
	testrpc.WaitForLeader(t, a.RPC, "dc1")

	// Register the service so we can get a leaf cert
	_, err := client.Catalog().Register(&api.CatalogRegistration{
		Datacenter: "dc1",
		Node:       "local",
		Address:    "127.0.0.1",
		Service: &api.AgentService{
			Service: "echo",
		},
	}, nil)
	require.NoError(err)

	// Start the backend service that is being proxied
	testApp := NewTestTCPServer(t)
	defer testApp.Close()

	// Start the proxy
	p, err := New(client, NewStaticConfigWatcher(&Config{
		ProxiedServiceName: "echo",
		PublicListener: PublicListenerConfig{
			BindAddress:         "127.0.0.1",
			BindPort:            ports[0],
			LocalServiceAddress: testApp.Addr().String(),
		},
	}), testLogger(t))
	require.NoError(err)
	defer p.Close()
	go p.Serve()

	// Create a test connection to the proxy. We retry here a few times
	// since this is dependent on the agent actually starting up and setting
	// up the CA.
	var conn net.Conn
	svc, err := connect.NewService("echo", client)
	require.NoError(err)
	retry.Run(t, func(r *retry.R) {
		conn, err = svc.Dial(context.Background(), &connect.StaticResolver{
			Addr:    TestLocalAddr(ports[0]),
			CertURI: agConnect.TestSpiffeIDService(t, "echo"),
		})
		if err != nil {
			r.Fatalf("err: %s", err)
		}
	})

	// Connection works, test it is the right one
	TestEchoConn(t, conn, "")
}

func testLogger(t *testing.T) *log.Logger {
	return log.New(os.Stderr, "", log.LstdFlags)
}
<|endoftext|>


<commit_before>package connector

import (
	"crypto/tls"
	"crypto/x509"

	"fmt"

	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/coreos/dex/pkg/log"
	"github.com/coreos/go-oidc/oidc"

	"gopkg.in/ldap.v2"
)

const (
	LDAPConnectorType         = "ldap"
	LDAPLoginPageTemplateName = "ldap-login.html"
)

func init() {
	RegisterConnectorConfigType(LDAPConnectorType, func() ConnectorConfig { return &LDAPConnectorConfig{} })
}

type LDAPConnectorConfig struct {
	ID                   string        `json:"id"`
	ServerHost           string        `json:"serverHost"`
	ServerPort           uint16        `json:"serverPort"`
	Timeout              time.Duration `json:"timeout"`
	UseTLS               bool          `json:"useTLS"`
	UseSSL               bool          `json:"useSSL"`
	CertFile             string        `json:"certFile"`
	KeyFile              string        `json:"keyFile"`
	CaFile               string        `json:"caFile"`
	SkipCertVerification bool          `json:"skipCertVerification"`
	BaseDN               string        `json:"baseDN"`
	NameAttribute        string        `json:"nameAttribute"`
	EmailAttribute       string        `json:"emailAttribute"`
	SearchBeforeAuth     bool          `json:"searchBeforeAuth"`
	SearchFilter         string        `json:"searchFilter"`
	SearchScope          string        `json:"searchScope"`
	SearchBindDN         string        `json:"searchBindDN"`
	SearchBindPw         string        `json:"searchBindPw"`
	BindTemplate         string        `json:"bindTemplate"`
	TrustedEmailProvider bool          `json:"trustedEmailProvider"`
}

func (cfg *LDAPConnectorConfig) ConnectorID() string {
	return cfg.ID
}

func (cfg *LDAPConnectorConfig) ConnectorType() string {
	return LDAPConnectorType
}

type LDAPConnector struct {
	id                   string
	idp                  *LDAPIdentityProvider
	namespace            url.URL
	trustedEmailProvider bool
	loginFunc            oidc.LoginFunc
	loginTpl             *template.Template
}

func (cfg *LDAPConnectorConfig) Connector(ns url.URL, lf oidc.LoginFunc, tpls *template.Template) (Connector, error) {
	ns.Path = path.Join(ns.Path, httpPathCallback)
	tpl := tpls.Lookup(LDAPLoginPageTemplateName)
	if tpl == nil {
		return nil, fmt.Errorf("unable to find necessary HTML template")
	}

	if cfg.UseTLS && cfg.UseSSL {
		return nil, fmt.Errorf("Invalid configuration. useTLS and useSSL are mutual exclusive.")
	}

	if len(cfg.CertFile) > 0 && len(cfg.KeyFile) == 0 {
		return nil, fmt.Errorf("Invalid configuration. Both certFile and keyFile must be specified.")
	}

	var nameAttribute, emailAttribute, bindTemplate string
	if len(cfg.NameAttribute) > 0 {
		nameAttribute = cfg.NameAttribute
	} else {
		nameAttribute = "cn"
	}

	if len(cfg.EmailAttribute) > 0 {
		emailAttribute = cfg.EmailAttribute
	} else {
		emailAttribute = "mail"
	}

	if len(cfg.BindTemplate) > 0 {
		if cfg.SearchBeforeAuth {
			log.Warningf("bindTemplate not used when searchBeforeAuth specified.")
		}
		bindTemplate = cfg.BindTemplate
	} else {
		bindTemplate = "uid=%u,%b"
	}

	var searchScope int
	if len(cfg.SearchScope) > 0 {
		switch {
		case strings.EqualFold(cfg.SearchScope, "BASE"):
			searchScope = ldap.ScopeBaseObject
		case strings.EqualFold(cfg.SearchScope, "ONE"):
			searchScope = ldap.ScopeSingleLevel
		case strings.EqualFold(cfg.SearchScope, "SUB"):
			searchScope = ldap.ScopeWholeSubtree
		default:
			return nil, fmt.Errorf("Invalid value for searchScope: '%v'. Must be one of 'base', 'one' or 'sub'.", cfg.SearchScope)
		}
	} else {
		searchScope = ldap.ScopeSingleLevel
	}

	if cfg.Timeout != 0 {
		ldap.DefaultTimeout = cfg.Timeout * time.Millisecond
	}

	tlsConfig := &tls.Config{
		ServerName:         cfg.ServerHost,
		InsecureSkipVerify: cfg.SkipCertVerification,
	}

	if (cfg.UseTLS || cfg.UseSSL) && len(cfg.CaFile) > 0 {
		buf, err := ioutil.ReadFile(cfg.CaFile)
		if err != nil {
			return nil, err
		}

		rootCertPool := x509.NewCertPool()
		ok := rootCertPool.AppendCertsFromPEM(buf)
		if ok {
			tlsConfig.RootCAs = rootCertPool
		} else {
			return nil, fmt.Errorf("%v: Unable to parse certificate data.", cfg.CaFile)
		}
	}

	if (cfg.UseTLS || cfg.UseSSL) && len(cfg.CertFile) > 0 && len(cfg.KeyFile) > 0 {
		cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
		if err != nil {
			return nil, err
		}
		tlsConfig.Certificates = []tls.Certificate{cert}
	}

	idp := &LDAPIdentityProvider{
		serverHost:       cfg.ServerHost,
		serverPort:       cfg.ServerPort,
		useTLS:           cfg.UseTLS,
		useSSL:           cfg.UseSSL,
		baseDN:           cfg.BaseDN,
		nameAttribute:    nameAttribute,
		emailAttribute:   emailAttribute,
		searchBeforeAuth: cfg.SearchBeforeAuth,
		searchFilter:     cfg.SearchFilter,
		searchScope:      searchScope,
		searchBindDN:     cfg.SearchBindDN,
		searchBindPw:     cfg.SearchBindPw,
		bindTemplate:     bindTemplate,
		tlsConfig:        tlsConfig,
	}

	idpc := &LDAPConnector{
		id:                   cfg.ID,
		idp:                  idp,
		namespace:            ns,
		trustedEmailProvider: cfg.TrustedEmailProvider,
		loginFunc:            lf,
		loginTpl:             tpl,
	}

	return idpc, nil
}

func (c *LDAPConnector) ID() string {
	return c.id
}

func (c *LDAPConnector) Healthy() error {
	ldapConn, err := c.idp.LDAPConnect()
	if err == nil {
		ldapConn.Close()
	}
	return err
}

func (c *LDAPConnector) LoginURL(sessionKey, prompt string) (string, error) {
	q := url.Values{}
	q.Set("session_key", sessionKey)
	q.Set("prompt", prompt)
	enc := q.Encode()

	return path.Join(c.namespace.Path, "login") + "?" + enc, nil
}

func (c *LDAPConnector) Register(mux *http.ServeMux, errorURL url.URL) {
	route := path.Join(c.namespace.Path, "login")
	mux.Handle(route, handleLoginFunc(c.loginFunc, c.loginTpl, c.idp, route, errorURL))
}

func (c *LDAPConnector) Sync() chan struct{} {
	return make(chan struct{})
}

func (c *LDAPConnector) TrustedEmailProvider() bool {
	return c.trustedEmailProvider
}

type LDAPIdentityProvider struct {
	serverHost       string
	serverPort       uint16
	useTLS           bool
	useSSL           bool
	baseDN           string
	nameAttribute    string
	emailAttribute   string
	searchBeforeAuth bool
	searchFilter     string
	searchScope      int
	searchBindDN     string
	searchBindPw     string
	bindTemplate     string
	tlsConfig        *tls.Config
}

func (m *LDAPIdentityProvider) LDAPConnect() (*ldap.Conn, error) {
	var err error
	var ldapConn *ldap.Conn

	log.Debugf("LDAPConnect()")
	if m.useSSL {
		ldapConn, err = ldap.DialTLS("tcp", fmt.Sprintf("%s:%d", m.serverHost, m.serverPort), m.tlsConfig)
		if err != nil {
			return nil, err
		}
	} else {
		ldapConn, err = ldap.Dial("tcp", fmt.Sprintf("%s:%d", m.serverHost, m.serverPort))
		if err != nil {
			return nil, err
		}
		if m.useTLS {
			err = ldapConn.StartTLS(m.tlsConfig)
			if err != nil {
				return nil, err
			}
		}
	}

	return ldapConn, err
}

func (m *LDAPIdentityProvider) ParseString(template, username string) string {
	result := template
	result = strings.Replace(result, "%u", username, -1)
	result = strings.Replace(result, "%b", m.baseDN, -1)

	return result
}

func (m *LDAPIdentityProvider) Identity(username, password string) (*oidc.Identity, error) {
	var err error
	var bindDN, ldapUid, ldapName, ldapEmail string
	var ldapConn *ldap.Conn

	ldapConn, err = m.LDAPConnect()
	if err != nil {
		return nil, err
	}
	defer ldapConn.Close()

	if m.searchBeforeAuth {
		err = ldapConn.Bind(m.searchBindDN, m.searchBindPw)
		if err != nil {
			return nil, err
		}

		filter := m.ParseString(m.searchFilter, username)

		attributes := []string{
			m.nameAttribute,
			m.emailAttribute,
		}

		s := ldap.NewSearchRequest(m.baseDN, m.searchScope, ldap.NeverDerefAliases, 0, 0, false, filter, attributes, nil)

		sr, err := ldapConn.Search(s)
		if err != nil {
			return nil, err
		}
		if len(sr.Entries) == 0 {
			err = fmt.Errorf("Search returned no match. filter='%v' base='%v'", filter, m.baseDN)
			return nil, err
		}

		bindDN = sr.Entries[0].DN
		ldapName = sr.Entries[0].GetAttributeValue(m.nameAttribute)
		ldapEmail = sr.Entries[0].GetAttributeValue(m.emailAttribute)

		// drop to anonymous bind, prepare for bind as user
		err = ldapConn.Bind("", "")
		if err != nil {
			// unsupported or disallowed, reconnect
			log.Warningf("Re-connecting to LDAP Server after failure to bind anonymously: %v", err)
			ldapConn.Close()
			ldapConn, err = m.LDAPConnect()
			if err != nil {
				return nil, err
			}
		}
	} else {
		bindDN = m.ParseString(m.bindTemplate, username)
	}

	// authenticate user
	err = ldapConn.Bind(bindDN, password)
	if err != nil {
		return nil, err
	}

	ldapUid = bindDN

	return &oidc.Identity{
		ID:    ldapUid,
		Name:  ldapName,
		Email: ldapEmail,
	}, nil
}
<commit_msg>Make constants for default values, simplify logic<commit_after>package connector

import (
	"crypto/tls"
	"crypto/x509"

	"fmt"

	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/coreos/dex/pkg/log"
	"github.com/coreos/go-oidc/oidc"

	"gopkg.in/ldap.v2"
)

const (
	LDAPConnectorType         = "ldap"
	LDAPLoginPageTemplateName = "ldap-login.html"
)

func init() {
	RegisterConnectorConfigType(LDAPConnectorType, func() ConnectorConfig { return &LDAPConnectorConfig{} })
}

type LDAPConnectorConfig struct {
	ID                   string        `json:"id"`
	ServerHost           string        `json:"serverHost"`
	ServerPort           uint16        `json:"serverPort"`
	Timeout              time.Duration `json:"timeout"`
	UseTLS               bool          `json:"useTLS"`
	UseSSL               bool          `json:"useSSL"`
	CertFile             string        `json:"certFile"`
	KeyFile              string        `json:"keyFile"`
	CaFile               string        `json:"caFile"`
	SkipCertVerification bool          `json:"skipCertVerification"`
	BaseDN               string        `json:"baseDN"`
	NameAttribute        string        `json:"nameAttribute"`
	EmailAttribute       string        `json:"emailAttribute"`
	SearchBeforeAuth     bool          `json:"searchBeforeAuth"`
	SearchFilter         string        `json:"searchFilter"`
	SearchScope          string        `json:"searchScope"`
	SearchBindDN         string        `json:"searchBindDN"`
	SearchBindPw         string        `json:"searchBindPw"`
	BindTemplate         string        `json:"bindTemplate"`
	TrustedEmailProvider bool          `json:"trustedEmailProvider"`
}

func (cfg *LDAPConnectorConfig) ConnectorID() string {
	return cfg.ID
}

func (cfg *LDAPConnectorConfig) ConnectorType() string {
	return LDAPConnectorType
}

type LDAPConnector struct {
	id                   string
	idp                  *LDAPIdentityProvider
	namespace            url.URL
	trustedEmailProvider bool
	loginFunc            oidc.LoginFunc
	loginTpl             *template.Template
}

func (cfg *LDAPConnectorConfig) Connector(ns url.URL, lf oidc.LoginFunc, tpls *template.Template) (Connector, error) {
	ns.Path = path.Join(ns.Path, httpPathCallback)
	tpl := tpls.Lookup(LDAPLoginPageTemplateName)
	if tpl == nil {
		return nil, fmt.Errorf("unable to find necessary HTML template")
	}

	// defaults
	const defaultNameAttribute = "cn"
	const defaultEmailAttribute = "mail"
	const defaultBindTemplate = "uid=%u,%b"
	const defaultSearchScope = ldap.ScopeWholeSubtree

	if cfg.UseTLS && cfg.UseSSL {
		return nil, fmt.Errorf("Invalid configuration. useTLS and useSSL are mutual exclusive.")
	}

	if len(cfg.CertFile) > 0 && len(cfg.KeyFile) == 0 {
		return nil, fmt.Errorf("Invalid configuration. Both certFile and keyFile must be specified.")
	}

	nameAttribute := defaultNameAttribute
	if len(cfg.NameAttribute) > 0 {
		nameAttribute = cfg.NameAttribute
	}

	emailAttribute := defaultEmailAttribute
	if len(cfg.EmailAttribute) > 0 {
		emailAttribute = cfg.EmailAttribute
	}

	bindTemplate := defaultBindTemplate
	if len(cfg.BindTemplate) > 0 {
		if cfg.SearchBeforeAuth {
			log.Warningf("bindTemplate not used when searchBeforeAuth specified.")
		}
		bindTemplate = cfg.BindTemplate
	}

	searchScope := defaultSearchScope
	if len(cfg.SearchScope) > 0 {
		switch {
		case strings.EqualFold(cfg.SearchScope, "BASE"):
			searchScope = ldap.ScopeBaseObject
		case strings.EqualFold(cfg.SearchScope, "ONE"):
			searchScope = ldap.ScopeSingleLevel
		case strings.EqualFold(cfg.SearchScope, "SUB"):
			searchScope = ldap.ScopeWholeSubtree
		default:
			return nil, fmt.Errorf("Invalid value for searchScope: '%v'. Must be one of 'base', 'one' or 'sub'.", cfg.SearchScope)
		}
	}

	if cfg.Timeout != 0 {
		ldap.DefaultTimeout = cfg.Timeout * time.Millisecond
	}

	tlsConfig := &tls.Config{
		ServerName:         cfg.ServerHost,
		InsecureSkipVerify: cfg.SkipCertVerification,
	}

	if (cfg.UseTLS || cfg.UseSSL) && len(cfg.CaFile) > 0 {
		buf, err := ioutil.ReadFile(cfg.CaFile)
		if err != nil {
			return nil, err
		}

		rootCertPool := x509.NewCertPool()
		ok := rootCertPool.AppendCertsFromPEM(buf)
		if ok {
			tlsConfig.RootCAs = rootCertPool
		} else {
			return nil, fmt.Errorf("%v: Unable to parse certificate data.", cfg.CaFile)
		}
	}

	if (cfg.UseTLS || cfg.UseSSL) && len(cfg.CertFile) > 0 && len(cfg.KeyFile) > 0 {
		cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
		if err != nil {
			return nil, err
		}
		tlsConfig.Certificates = []tls.Certificate{cert}
	}

	idp := &LDAPIdentityProvider{
		serverHost:       cfg.ServerHost,
		serverPort:       cfg.ServerPort,
		useTLS:           cfg.UseTLS,
		useSSL:           cfg.UseSSL,
		baseDN:           cfg.BaseDN,
		nameAttribute:    nameAttribute,
		emailAttribute:   emailAttribute,
		searchBeforeAuth: cfg.SearchBeforeAuth,
		searchFilter:     cfg.SearchFilter,
		searchScope:      searchScope,
		searchBindDN:     cfg.SearchBindDN,
		searchBindPw:     cfg.SearchBindPw,
		bindTemplate:     bindTemplate,
		tlsConfig:        tlsConfig,
	}

	idpc := &LDAPConnector{
		id:                   cfg.ID,
		idp:                  idp,
		namespace:            ns,
		trustedEmailProvider: cfg.TrustedEmailProvider,
		loginFunc:            lf,
		loginTpl:             tpl,
	}

	return idpc, nil
}

func (c *LDAPConnector) ID() string {
	return c.id
}

func (c *LDAPConnector) Healthy() error {
	ldapConn, err := c.idp.LDAPConnect()
	if err == nil {
		ldapConn.Close()
	}
	return err
}

func (c *LDAPConnector) LoginURL(sessionKey, prompt string) (string, error) {
	q := url.Values{}
	q.Set("session_key", sessionKey)
	q.Set("prompt", prompt)
	enc := q.Encode()

	return path.Join(c.namespace.Path, "login") + "?" + enc, nil
}

func (c *LDAPConnector) Register(mux *http.ServeMux, errorURL url.URL) {
	route := path.Join(c.namespace.Path, "login")
	mux.Handle(route, handleLoginFunc(c.loginFunc, c.loginTpl, c.idp, route, errorURL))
}

func (c *LDAPConnector) Sync() chan struct{} {
	return make(chan struct{})
}

func (c *LDAPConnector) TrustedEmailProvider() bool {
	return c.trustedEmailProvider
}

type LDAPIdentityProvider struct {
	serverHost       string
	serverPort       uint16
	useTLS           bool
	useSSL           bool
	baseDN           string
	nameAttribute    string
	emailAttribute   string
	searchBeforeAuth bool
	searchFilter     string
	searchScope      int
	searchBindDN     string
	searchBindPw     string
	bindTemplate     string
	tlsConfig        *tls.Config
}

func (m *LDAPIdentityProvider) LDAPConnect() (*ldap.Conn, error) {
	var err error
	var ldapConn *ldap.Conn

	log.Debugf("LDAPConnect()")
	if m.useSSL {
		ldapConn, err = ldap.DialTLS("tcp", fmt.Sprintf("%s:%d", m.serverHost, m.serverPort), m.tlsConfig)
		if err != nil {
			return nil, err
		}
	} else {
		ldapConn, err = ldap.Dial("tcp", fmt.Sprintf("%s:%d", m.serverHost, m.serverPort))
		if err != nil {
			return nil, err
		}
		if m.useTLS {
			err = ldapConn.StartTLS(m.tlsConfig)
			if err != nil {
				return nil, err
			}
		}
	}

	return ldapConn, err
}

func (m *LDAPIdentityProvider) ParseString(template, username string) string {
	result := template
	result = strings.Replace(result, "%u", username, -1)
	result = strings.Replace(result, "%b", m.baseDN, -1)

	return result
}

func (m *LDAPIdentityProvider) Identity(username, password string) (*oidc.Identity, error) {
	var err error
	var bindDN, ldapUid, ldapName, ldapEmail string
	var ldapConn *ldap.Conn

	ldapConn, err = m.LDAPConnect()
	if err != nil {
		return nil, err
	}
	defer ldapConn.Close()

	if m.searchBeforeAuth {
		err = ldapConn.Bind(m.searchBindDN, m.searchBindPw)
		if err != nil {
			return nil, err
		}

		filter := m.ParseString(m.searchFilter, username)

		attributes := []string{
			m.nameAttribute,
			m.emailAttribute,
		}

		s := ldap.NewSearchRequest(m.baseDN, m.searchScope, ldap.NeverDerefAliases, 0, 0, false, filter, attributes, nil)

		sr, err := ldapConn.Search(s)
		if err != nil {
			return nil, err
		}
		if len(sr.Entries) == 0 {
			err = fmt.Errorf("Search returned no match. filter='%v' base='%v'", filter, m.baseDN)
			return nil, err
		}

		bindDN = sr.Entries[0].DN
		ldapName = sr.Entries[0].GetAttributeValue(m.nameAttribute)
		ldapEmail = sr.Entries[0].GetAttributeValue(m.emailAttribute)

		// drop to anonymous bind, prepare for bind as user
		err = ldapConn.Bind("", "")
		if err != nil {
			// unsupported or disallowed, reconnect
			log.Warningf("Re-connecting to LDAP Server after failure to bind anonymously: %v", err)
			ldapConn.Close()
			ldapConn, err = m.LDAPConnect()
			if err != nil {
				return nil, err
			}
		}
	} else {
		bindDN = m.ParseString(m.bindTemplate, username)
	}

	// authenticate user
	err = ldapConn.Bind(bindDN, password)
	if err != nil {
		return nil, err
	}

	ldapUid = bindDN

	return &oidc.Identity{
		ID:    ldapUid,
		Name:  ldapName,
		Email: ldapEmail,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"flag"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"github.com/docker/libcontainer/cgroups/systemd"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/container/libcontainer"
	"github.com/google/cadvisor/info"
)

var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint")

type dockerFactory struct {
	machineInfoFactory info.MachineInfoFactory

	// Whether this system is using systemd.
	useSystemd bool

	client *docker.Client
}

func (self *dockerFactory) String() string {
	return "docker"
}

func (self *dockerFactory) NewContainerHandler(name string) (handler container.ContainerHandler, err error) {
	client, err := docker.NewClient(*ArgDockerEndpoint)
	if err != nil {
		return
	}
	handler, err = newDockerContainerHandler(
		client,
		name,
		self.machineInfoFactory,
		self.useSystemd,
	)
	return
}

// Docker handles all containers under /docker
// TODO(vishh): Change the CanHandle interface to be able to return errors.
func (self *dockerFactory) CanHandle(name string) bool {
	// In systemd systems the containers are: /docker-{ID}
	if self.useSystemd {
		if !strings.HasPrefix(name, "/docker-") {
			return false
		}
	} else if name == "/" {
		return false
	} else if name == "/docker" {
		// We need the docker driver to handle /docker. Otherwise the aggregation at the API level will break.
		return true
	} else if !strings.HasPrefix(name, "/docker/") {
		return false
	}
	// Check if the container is known to docker and it is active.
	_, id, err := libcontainer.SplitName(name)
	if err != nil {
		return false
	}
	ctnr, err := self.client.InspectContainer(id)
	// We assume that if Inspect fails then the container is not known to docker.
	// TODO(vishh): Detect lxc containers and avoid handling them.
	if err != nil || !ctnr.State.Running {
		return false
	}

	return true
}

func parseDockerVersion(full_version_string string) ([]int, error) {
	version_regexp_string := "(\\d+)\\.(\\d+)\\.(\\d+)"
	version_re := regexp.MustCompile(version_regexp_string)
	matches := version_re.FindAllStringSubmatch(full_version_string, -1)
	if len(matches) != 1 {
		return nil, fmt.Errorf("Version string \"%v\" doesn't match expected regular expression: \"%v\"", full_version_string, version_regexp_string)
	}
	version_string_array := matches[0][1:]
	version_array := make([]int, 3)
	for index, version_string := range version_string_array {
		version, err := strconv.Atoi(version_string)
		if err != nil {
			return nil, fmt.Errorf("Error while parsing \"%v\" in \"%v\"", version_string, full_version_string)
		}
		version_array[index] = version
	}
	return version_array, nil
}

// Register root container before running this function!
func Register(factory info.MachineInfoFactory) error {
	client, err := docker.NewClient(*ArgDockerEndpoint)
	if err != nil {
		return fmt.Errorf("unable to communicate with docker daemon: %v", err)
	}
	if version, err := client.Version(); err != nil {
		return fmt.Errorf("unable to communicate with docker daemon: %v", err)
	} else {
		expected_version := []int{0, 11, 1}
		version_string := version.Get("Version")
		version, err := parseDockerVersion(version_string)
		if err != nil {
			return fmt.Errorf("Couldn't parse docker version: %v", err)
		}
		for index, number := range version {
			if number > expected_version[index] {
				break
			} else if number < expected_version[index] {
				return fmt.Errorf("cAdvisor requires docker version above %v but we have found version %v reported as \"%v\"", expected_version, version, version_string)
			}
		}
	}
	f := &dockerFactory{
		machineInfoFactory: factory,
		useSystemd:         systemd.UseSystemd(),
		client:             client,
	}
	log.Printf("Registering Docker factory")
	container.RegisterContainerHandlerFactory(f)
	return nil
}
<commit_msg>Fix Docker container slice in systemd systems.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	"flag"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"

	"github.com/docker/libcontainer/cgroups/systemd"
	"github.com/fsouza/go-dockerclient"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/container/libcontainer"
	"github.com/google/cadvisor/info"
)

var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "docker endpoint")

type dockerFactory struct {
	machineInfoFactory info.MachineInfoFactory

	// Whether this system is using systemd.
	useSystemd bool

	client *docker.Client
}

func (self *dockerFactory) String() string {
	return "docker"
}

func (self *dockerFactory) NewContainerHandler(name string) (handler container.ContainerHandler, err error) {
	client, err := docker.NewClient(*ArgDockerEndpoint)
	if err != nil {
		return
	}
	handler, err = newDockerContainerHandler(
		client,
		name,
		self.machineInfoFactory,
		self.useSystemd,
	)
	return
}

// Docker handles all containers under /docker
// TODO(vishh): Change the CanHandle interface to be able to return errors.
func (self *dockerFactory) CanHandle(name string) bool {
	// In systemd systems the containers are: /system.slice/docker-{ID}
	if self.useSystemd {
		if !strings.HasPrefix(name, "/system.slice/docker-") {
			return false
		}
	} else if name == "/" {
		return false
	} else if name == "/docker" {
		// We need the docker driver to handle /docker. Otherwise the aggregation at the API level will break.
		return true
	} else if !strings.HasPrefix(name, "/docker/") {
		return false
	}
	// Check if the container is known to docker and it is active.
	_, id, err := libcontainer.SplitName(name)
	if err != nil {
		return false
	}
	ctnr, err := self.client.InspectContainer(id)
	// We assume that if Inspect fails then the container is not known to docker.
	// TODO(vishh): Detect lxc containers and avoid handling them.
	if err != nil || !ctnr.State.Running {
		return false
	}

	return true
}

func parseDockerVersion(full_version_string string) ([]int, error) {
	version_regexp_string := "(\\d+)\\.(\\d+)\\.(\\d+)"
	version_re := regexp.MustCompile(version_regexp_string)
	matches := version_re.FindAllStringSubmatch(full_version_string, -1)
	if len(matches) != 1 {
		return nil, fmt.Errorf("Version string \"%v\" doesn't match expected regular expression: \"%v\"", full_version_string, version_regexp_string)
	}
	version_string_array := matches[0][1:]
	version_array := make([]int, 3)
	for index, version_string := range version_string_array {
		version, err := strconv.Atoi(version_string)
		if err != nil {
			return nil, fmt.Errorf("Error while parsing \"%v\" in \"%v\"", version_string, full_version_string)
		}
		version_array[index] = version
	}
	return version_array, nil
}

// Register root container before running this function!
func Register(factory info.MachineInfoFactory) error {
	client, err := docker.NewClient(*ArgDockerEndpoint)
	if err != nil {
		return fmt.Errorf("unable to communicate with docker daemon: %v", err)
	}
	if version, err := client.Version(); err != nil {
		return fmt.Errorf("unable to communicate with docker daemon: %v", err)
	} else {
		expected_version := []int{0, 11, 1}
		version_string := version.Get("Version")
		version, err := parseDockerVersion(version_string)
		if err != nil {
			return fmt.Errorf("Couldn't parse docker version: %v", err)
		}
		for index, number := range version {
			if number > expected_version[index] {
				break
			} else if number < expected_version[index] {
				return fmt.Errorf("cAdvisor requires docker version above %v but we have found version %v reported as \"%v\"", expected_version, version, version_string)
			}
		}
	}
	f := &dockerFactory{
		machineInfoFactory: factory,
		useSystemd:         systemd.UseSystemd(),
		client:             client,
	}
	if f.useSystemd {
		log.Printf("System is using systemd")
	}
	log.Printf("Registering Docker factory")
	container.RegisterContainerHandlerFactory(f)
	return nil
}
<|endoftext|>


<commit_before>package fasthttp

import (
	"net"
	"runtime/debug"
	"sync"
	"time"
)

// workerPool serves incoming connections via a pool of workers
// in FIFO order, i.e. the most recently stopped worker will serve the next
// incoming connection.
type workerPool struct {
	// Function for serving server connections.
	// It must close c before returning.
	WorkerFunc func(c net.Conn) error

	// Maximum number of workers to create.
	MaxWorkersCount int

	// Logger used by workerPool.
	Logger Logger

	lock         sync.Mutex
	workersCount int
	mustStop     bool

	ready []*workerChan

	stopCh chan struct{}
}

type workerChan struct {
	t  time.Time
	ch chan net.Conn
}

func (wp *workerPool) Start() {
	if wp.stopCh != nil {
		panic("BUG: workerPool already started")
	}
	wp.stopCh = make(chan struct{})
	stopCh := wp.stopCh
	go func() {
		for {
			select {
			case <-stopCh:
				return
			default:
				time.Sleep(time.Second)
			}
			wp.clean()
		}
	}()
}

func (wp *workerPool) Stop() {
	if wp.stopCh == nil {
		panic("BUG: workerPool wasn't started")
	}
	close(wp.stopCh)
	wp.stopCh = nil

	// Stop all the workers waiting for incoming connections.
	// Do not wait for busy workers - they will stop after
	// serving the connection and noticing wp.mustStop = true.
	wp.lock.Lock()
	for _, ch := range wp.ready {
		ch.ch <- nil
	}
	wp.ready = nil
	wp.mustStop = true
	wp.lock.Unlock()
}

func (wp *workerPool) clean() {
	// Clean least recently used workers if they didn't serve connections
	// for more than one second.
	wp.lock.Lock()
	chans := wp.ready
	for len(chans) > 1 && time.Since(chans[0].t) > time.Second {
		chans[0].ch <- nil
		copy(chans, chans[1:])
		chans = chans[:len(chans)-1]
		wp.ready = chans
		wp.workersCount--
	}
	wp.lock.Unlock()
}

func (wp *workerPool) Serve(c net.Conn) bool {
	ch := wp.getCh()
	if ch == nil {
		return false
	}
	ch.ch <- c
	return true
}

func (wp *workerPool) getCh() *workerChan {
	var ch *workerChan
	createWorker := false

	wp.lock.Lock()
	chans := wp.ready
	n := len(chans) - 1
	if n < 0 {
		if wp.workersCount < wp.MaxWorkersCount {
			createWorker = true
			wp.workersCount++
		}
	} else {
		ch = chans[n]
		wp.ready = chans[:n]
	}
	wp.lock.Unlock()

	if ch == nil {
		if !createWorker {
			return nil
		}
		vch := workerChanPool.Get()
		if vch == nil {
			vch = &workerChan{
				ch: make(chan net.Conn, 1),
			}
		}
		ch = vch.(*workerChan)
		go func() {
			wp.workerFunc(ch)
			workerChanPool.Put(vch)
		}()
	}
	return ch
}

func (wp *workerPool) release(ch *workerChan) bool {
	ch.t = time.Now()
	wp.lock.Lock()
	if wp.mustStop {
		return false
	}
	wp.ready = append(wp.ready, ch)
	wp.lock.Unlock()
	return true
}

var workerChanPool sync.Pool

func (wp *workerPool) workerFunc(ch *workerChan) {
	defer func() {
		if r := recover(); r != nil {
			wp.Logger.Printf("panic: %s\nStack trace:\n%s", r, debug.Stack())
		}
	}()

	var c net.Conn
	var err error
	for c = range ch.ch {
		if c == nil {
			break
		}
		if err = wp.WorkerFunc(c); err != nil {
			wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err)
		}
		c = nil

		if !wp.release(ch) {
			break
		}
	}
}
<commit_msg>Added missing unlock when stopping worker pool<commit_after>package fasthttp

import (
	"net"
	"runtime/debug"
	"sync"
	"time"
)

// workerPool serves incoming connections via a pool of workers
// in FIFO order, i.e. the most recently stopped worker will serve the next
// incoming connection.
type workerPool struct {
	// Function for serving server connections.
	// It must close c before returning.
	WorkerFunc func(c net.Conn) error

	// Maximum number of workers to create.
	MaxWorkersCount int

	// Logger used by workerPool.
	Logger Logger

	lock         sync.Mutex
	workersCount int
	mustStop     bool

	ready []*workerChan

	stopCh chan struct{}
}

type workerChan struct {
	t  time.Time
	ch chan net.Conn
}

func (wp *workerPool) Start() {
	if wp.stopCh != nil {
		panic("BUG: workerPool already started")
	}
	wp.stopCh = make(chan struct{})
	stopCh := wp.stopCh
	go func() {
		for {
			select {
			case <-stopCh:
				return
			default:
				time.Sleep(time.Second)
			}
			wp.clean()
		}
	}()
}

func (wp *workerPool) Stop() {
	if wp.stopCh == nil {
		panic("BUG: workerPool wasn't started")
	}
	close(wp.stopCh)
	wp.stopCh = nil

	// Stop all the workers waiting for incoming connections.
	// Do not wait for busy workers - they will stop after
	// serving the connection and noticing wp.mustStop = true.
	wp.lock.Lock()
	for _, ch := range wp.ready {
		ch.ch <- nil
	}
	wp.ready = nil
	wp.mustStop = true
	wp.lock.Unlock()
}

func (wp *workerPool) clean() {
	// Clean least recently used workers if they didn't serve connections
	// for more than one second.
	wp.lock.Lock()
	chans := wp.ready
	for len(chans) > 1 && time.Since(chans[0].t) > time.Second {
		chans[0].ch <- nil
		copy(chans, chans[1:])
		chans = chans[:len(chans)-1]
		wp.ready = chans
		wp.workersCount--
	}
	wp.lock.Unlock()
}

func (wp *workerPool) Serve(c net.Conn) bool {
	ch := wp.getCh()
	if ch == nil {
		return false
	}
	ch.ch <- c
	return true
}

func (wp *workerPool) getCh() *workerChan {
	var ch *workerChan
	createWorker := false

	wp.lock.Lock()
	chans := wp.ready
	n := len(chans) - 1
	if n < 0 {
		if wp.workersCount < wp.MaxWorkersCount {
			createWorker = true
			wp.workersCount++
		}
	} else {
		ch = chans[n]
		wp.ready = chans[:n]
	}
	wp.lock.Unlock()

	if ch == nil {
		if !createWorker {
			return nil
		}
		vch := workerChanPool.Get()
		if vch == nil {
			vch = &workerChan{
				ch: make(chan net.Conn, 1),
			}
		}
		ch = vch.(*workerChan)
		go func() {
			wp.workerFunc(ch)
			workerChanPool.Put(vch)
		}()
	}
	return ch
}

func (wp *workerPool) release(ch *workerChan) bool {
	ch.t = time.Now()
	wp.lock.Lock()
	if wp.mustStop {
		wp.lock.Unlock()
		return false
	}
	wp.ready = append(wp.ready, ch)
	wp.lock.Unlock()
	return true
}

var workerChanPool sync.Pool

func (wp *workerPool) workerFunc(ch *workerChan) {
	defer func() {
		if r := recover(); r != nil {
			wp.Logger.Printf("panic: %s\nStack trace:\n%s", r, debug.Stack())
		}
	}()

	var c net.Conn
	var err error
	for c = range ch.ch {
		if c == nil {
			break
		}
		if err = wp.WorkerFunc(c); err != nil {
			wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err)
		}
		c = nil

		if !wp.release(ch) {
			break
		}
	}
}
<|endoftext|>


<commit_before>package parser

import (
  "goprotobuf.googlecode.com/hg/proto"
  ir "tritium/proto"
  "io/ioutil"
  . "tritium/tokenizer" // was meant to be in this package
  "path"
  //"fmt"
)

type Parser struct {
  *Tokenizer
  FileName string
  DirName string
  FullPath string
  Lookahead *Token
  counter int
}

func (p *Parser) gensym() string {
  return p.FullPath + string(p.counter)
}

func (p *Parser) peek() *Token {
  return p.Lookahead
}

func (p *Parser) pop() *Token {
  val := p.Lookahead
  p.Lookahead = p.Tokenizer.Pop()
  return val
}

func MakeParser(fullpath string) *Parser {
  src, _ := ioutil.ReadFile(fullpath)
  d, f := path.Split(fullpath)
  p := &Parser {
    Tokenizer: MakeTokenizer(src),
    FileName: f,
    DirName: d,
    FullPath: fullpath,
    Lookahead: nil,
    counter: 0,
  }
  p.pop()
  return p
}

// function (p *Parser) script() tritium.ScriptObject {
//   switch p.peek().Lexeme {
//

func (p *Parser) Parse() *ir.ScriptObject {
  script := new(ir.ScriptObject)
  script.Name = proto.String(p.FullPath)

  stmts := make([]*ir.Instruction, 0)
  //defs := make([]*ir.Function, 0)

  switch p.peek().Lexeme {
  case FUNC:
    // to do
  default:
    for p.peek().Lexeme != EOF {
      stmts = append(stmts, p.statement())
    }
    if len(stmts) == 0 {
      stmts = nil
    }
    script.Root = &ir.Instruction {
      Type: ir.NewInstruction_InstructionType(ir.Instruction_BLOCK),
      Children: stmts,
    }
  }
  return script
}

func (p *Parser) statement() *ir.Instruction {
  node := new(ir.Instruction)
  switch p.peek().Lexeme {
  case IMPORT:
    token := p.pop()
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_IMPORT)
    node.Value = proto.String(path.Join(p.DirName, token.Value))
  default:
    node = p.expression()
  }
  return node
}

func (p *Parser) expression() *ir.Instruction {
  node := new(ir.Instruction)
  switch p.peek().Lexeme {
  case STRING, REGEXP, POS:
    node = p.literal()
  case READ:
    node = p.read()
  case ID:
    node = p.call()
  case GVAR, LVAR:
    node = p.variable()
  default:
    panic("malformed expression")
  }
  return node
}

func (p *Parser) literal() *ir.Instruction {
  node := new(ir.Instruction)
  token := p.pop()
  switch token.Lexeme {
  case STRING:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    node.Value = proto.String(token.Value)
  case REGEXP:
    pattern := new(ir.Instruction)
    options := new(ir.Instruction)
    po := make([]*ir.Instruction, 2)
    pattern.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    pattern.Value = proto.String(token.Value)
    options.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    options.Value = proto.String(token.ExtraValue)
    po[0] = pattern
    po[1] = options
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    node.Value = proto.String("regexp")
    node.Arguments = po
  case POS:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_POSITION)
    node.Value = proto.String(token.Value)
  }
  return node
}

func (p *Parser) read() *ir.Instruction {
  node := new(ir.Instruction)
  p.pop() // pop the "read" keyword
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for read")
  }
  p.pop() // pop the lparen
  if p.peek().Lexeme != STRING {
    panic("read requires a literal string argument")
  }
  readPath := p.pop().Value
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in read")
  }
  p.pop() // pop the rparen
  contents, _ := ioutil.ReadFile(path.Join(p.DirName, readPath))
  node.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
  node.Value = proto.String(string(contents))
  return node
}

func (p *Parser) call() *ir.Instruction {
  node := new(ir.Instruction)
  funcName := p.pop().Value // grab the function name
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for call to " + funcName)
  }
  p.pop() // pop the lparen
  ords, kwds := p.arguments() // gather the arguments
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in call to " + funcName)
  }
  p.pop() // pop the rparen
  var block []*ir.Instruction
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }

  // Expand keyword args
  if kwds != nil {
    kwdToGensym := make(map[string]string, len(kwds))
    outer := make([]*ir.Instruction, 0)
    for k, v:= range kwds {
      temp := new(ir.Instruction)
      temp.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      temp.Value = proto.String("var")
      temp.Arguments = make([]*ir.Instruction, 2)
      tempName := p.gensym()
      temp.Arguments[0] = &ir.Instruction {
        Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
        Value: proto.String(tempName),
      }
      temp.Arguments[1] = v
      outer = append(outer, temp)
      kwdToGensym[k] = tempName
    }
    inner := make([]*ir.Instruction, 0)
    for k, _ := range kwds {
      getter := new(ir.Instruction)
      getter.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      getter.Value = proto.String("var")
      getter.Arguments = make([]*ir.Instruction, 1)
      getter.Arguments[0] = &ir.Instruction {
        Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
        Value: proto.String(kwdToGensym[k]),
      }
      setter := new(ir.Instruction)
      setter.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      setter.Value = proto.String("set")
      setter.Arguments = make([]*ir.Instruction, 2)
      setter.Arguments[0] = &ir.Instruction {
        Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
        Value: proto.String(k),
      }
      setter.Arguments[1] = getter
      inner = append(inner, setter)
    }

    theCall := new(ir.Instruction)
    theCall.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    theCall.Value = proto.String(funcName)
    theCall.Arguments = ords

    if block == nil {
      block = inner
    } else {
      for _, v := range block {
        inner = append(inner, v)
      }
    }

    theCall.Children = inner
    outer = append(outer, theCall)

    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_BLOCK)
    node.Children = outer

  } else {
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    node.Value = proto.String(funcName)
    node.Arguments = ords
    node.Children = block
  }
  return node
}

func (p *Parser) arguments() (ords []*ir.Instruction, kwds map[string]*ir.Instruction) {
  ords, kwds = make([]*ir.Instruction, 0), make(map[string]*ir.Instruction, 0)
  if p.peek().Lexeme != RPAREN {
    if p.peek().Lexeme == KWD {
      k := p.pop().Value
      kwds[k] = p.expression()
    } else {
      ords = append(ords, p.expression())
    }
  }
  for p.peek().Lexeme != RPAREN {
    if p.peek().Lexeme == COMMA {
      p.pop()
    } else {
      panic("arguments must be separated by commas")
    }
    if p.peek().Lexeme == KWD {
      k := p.pop().Value
      kwds[k] = p.expression()
    } else {
      ords = append(ords, p.expression())
    }
  }
  if len(ords) == 0 {
    ords = nil
  }
  if len(kwds) == 0 {
    kwds = nil
  }
  return ords, kwds
}

func (p *Parser) variable() *ir.Instruction {
  node := new(ir.Instruction)
  switch p.peek().Lexeme {
  case LVAR:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_LOCAL_VAR)
    node.Value = proto.String(p.pop().Value)
    if p.peek().Lexeme == EQUAL {
      p.pop() // pop the equal sign
      node.Arguments = make([]*ir.Instruction, 1)
      node.Arguments[0] = p.expression()
    }
  case GVAR:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    node.Value = proto.String("var")
    name := p.pop().Value
    value := new(ir.Instruction)
    if p.peek().Lexeme == EQUAL {
      p.pop() // pop the equal sign
      value = p.expression()
    }
    if value == nil {
      node.Arguments = make([]*ir.Instruction, 1)
    } else {
      node.Arguments = make([]*ir.Instruction, 2)
      node.Arguments[1] = value
    }
    node.Arguments[0] = &ir.Instruction {
      Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
      Value: proto.String(name),
    }
  }
  if p.peek().Lexeme == LBRACE {
    node.Children = p.block()
  }
  return node
}

func (p *Parser) block() []*ir.Instruction {
  stmts := make([]*ir.Instruction, 0)
  p.pop() // pop the lbrace
  for p.peek().Lexeme != RBRACE {
    stmts = append(stmts, p.statement())
  }
  p.pop() // pop the rbrace
  if len(stmts) == 0 {
    stmts = nil
  }
  return stmts
}

<commit_msg>Fixing a bug in global variable parsing.<commit_after>package parser

import (
  "goprotobuf.googlecode.com/hg/proto"
  ir "tritium/proto"
  "io/ioutil"
  . "tritium/tokenizer" // was meant to be in this package
  "path"
  //"fmt"
)

type Parser struct {
  *Tokenizer
  FileName string
  DirName string
  FullPath string
  Lookahead *Token
  counter int
  header bool
}

func (p *Parser) gensym() string {
  return p.FullPath + string(p.counter)
}

func (p *Parser) peek() *Token {
  return p.Lookahead
}

func (p *Parser) pop() *Token {
  val := p.Lookahead
  p.Lookahead = p.Tokenizer.Pop()
  return val
}

func MakeParser(fullpath string) *Parser {
  src, _ := ioutil.ReadFile(fullpath)
  d, f := path.Split(fullpath)
  p := &Parser {
    Tokenizer: MakeTokenizer(src),
    FileName: f,
    DirName: d,
    FullPath: fullpath,
    Lookahead: nil,
    counter: 0,
    header: false,
  }
  p.pop()
  return p
}

func (p *Parser) Parse() *ir.ScriptObject {
  script := new(ir.ScriptObject)
  script.Name = proto.String(p.FullPath)

  stmts := make([]*ir.Instruction, 0)
  // defs := make([]*ir.Function, 0)

  switch p.peek().Lexeme {
  // case FUNC:
  //   for p.peek().Lexeme != EOF {
  //     defs = append(defs, p.definition())
  //   }
  //   if len(defs) == 0 {
  //     defs = nil
  //   }
  //   script.Functions = defs
  // }
  default:
    for p.peek().Lexeme != EOF {
      stmts = append(stmts, p.statement())
    }
    if len(stmts) == 0 {
      stmts = nil
    }
    script.Root = &ir.Instruction {
      Type: ir.NewInstruction_InstructionType(ir.Instruction_BLOCK),
      Children: stmts,
    }
  }
  return script
}

func (p *Parser) statement() *ir.Instruction {
  node := new(ir.Instruction)
  switch p.peek().Lexeme {
  case IMPORT:
    token := p.pop() // pop the "@import" keyword
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_IMPORT)
    node.Value = proto.String(path.Join(p.DirName, token.Value))
  default:
    node = p.expression()
  }
  return node
}

func (p *Parser) expression() *ir.Instruction {
  node := new(ir.Instruction)
  switch p.peek().Lexeme {
  case STRING, REGEXP, POS:
    node = p.literal()
  case READ:
    node = p.read()
  case ID:
    node = p.call()
  case GVAR, LVAR:
    node = p.variable()
  default:
    panic("malformed expression")
  }
  return node
}

func (p *Parser) literal() *ir.Instruction {
  node := new(ir.Instruction)
  token := p.pop()
  switch token.Lexeme {
  case STRING:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    node.Value = proto.String(token.Value)
  case REGEXP:
    pattern := new(ir.Instruction)
    options := new(ir.Instruction)
    po := make([]*ir.Instruction, 2)
    pattern.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    pattern.Value = proto.String(token.Value)
    options.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    options.Value = proto.String(token.ExtraValue)
    po[0] = pattern
    po[1] = options
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    node.Value = proto.String("regexp")
    node.Arguments = po
  case POS:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_POSITION)
    node.Value = proto.String(token.Value)
  }
  return node
}

func (p *Parser) read() *ir.Instruction {
  node := new(ir.Instruction)
  p.pop() // pop the "read" keyword
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for read")
  }
  p.pop() // pop the lparen
  if p.peek().Lexeme != STRING {
    panic("read requires a literal string argument")
  }
  readPath := p.pop().Value
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in read")
  }
  p.pop() // pop the rparen
  contents, _ := ioutil.ReadFile(path.Join(p.DirName, readPath))
  node.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
  node.Value = proto.String(string(contents))
  return node
}

func (p *Parser) call() *ir.Instruction {
  node := new(ir.Instruction)
  funcName := p.pop().Value // grab the function name
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for call to " + funcName)
  }
  p.pop() // pop the lparen
  ords, kwds := p.arguments() // gather the arguments
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in call to " + funcName)
  }
  p.pop() // pop the rparen
  var block []*ir.Instruction
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }

  // Expand keyword args
  if kwds != nil {
    kwdToGensym := make(map[string]string, len(kwds))
    outer := make([]*ir.Instruction, 0)
    for k, v:= range kwds {
      temp := new(ir.Instruction)
      temp.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      temp.Value = proto.String("var")
      temp.Arguments = make([]*ir.Instruction, 2)
      tempName := p.gensym()
      temp.Arguments[0] = &ir.Instruction {
        Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
        Value: proto.String(tempName),
      }
      temp.Arguments[1] = v
      outer = append(outer, temp)
      kwdToGensym[k] = tempName
    }
    inner := make([]*ir.Instruction, 0)
    for k, _ := range kwds {
      getter := new(ir.Instruction)
      getter.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      getter.Value = proto.String("var")
      getter.Arguments = make([]*ir.Instruction, 1)
      getter.Arguments[0] = &ir.Instruction {
        Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
        Value: proto.String(kwdToGensym[k]),
      }
      setter := new(ir.Instruction)
      setter.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      setter.Value = proto.String("set")
      setter.Arguments = make([]*ir.Instruction, 2)
      setter.Arguments[0] = &ir.Instruction {
        Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
        Value: proto.String(k),
      }
      setter.Arguments[1] = getter
      inner = append(inner, setter)
    }

    theCall := new(ir.Instruction)
    theCall.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    theCall.Value = proto.String(funcName)
    theCall.Arguments = ords

    if block == nil {
      block = inner
    } else {
      for _, v := range block {
        inner = append(inner, v)
      }
    }

    theCall.Children = inner
    outer = append(outer, theCall)

    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_BLOCK)
    node.Children = outer

  } else {
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    node.Value = proto.String(funcName)
    node.Arguments = ords
    node.Children = block
  }
  return node
}

func (p *Parser) arguments() (ords []*ir.Instruction, kwds map[string]*ir.Instruction) {
  ords, kwds = make([]*ir.Instruction, 0), make(map[string]*ir.Instruction, 0)
  if p.peek().Lexeme != RPAREN {
    if p.peek().Lexeme == KWD {
      k := p.pop().Value
      kwds[k] = p.expression()
    } else {
      ords = append(ords, p.expression())
    }
  }
  for p.peek().Lexeme != RPAREN {
    if p.peek().Lexeme == COMMA {
      p.pop()
    } else {
      panic("arguments must be separated by commas")
    }
    if p.peek().Lexeme == KWD {
      k := p.pop().Value
      kwds[k] = p.expression()
    } else {
      ords = append(ords, p.expression())
    }
  }
  if len(ords) == 0 {
    ords = nil
  }
  if len(kwds) == 0 {
    kwds = nil
  }
  return ords, kwds
}

func (p *Parser) variable() *ir.Instruction {
  node := new(ir.Instruction)
  switch p.peek().Lexeme {
  case LVAR:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_LOCAL_VAR)
    node.Value = proto.String(p.pop().Value)
    if p.peek().Lexeme == EQUAL {
      p.pop() // pop the equal sign
      node.Arguments = make([]*ir.Instruction, 1)
      node.Arguments[0] = p.expression()
    }
  case GVAR:
    node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    node.Value = proto.String("var")
    name := p.pop().Value
    value := new(ir.Instruction)
    if p.peek().Lexeme == EQUAL {
      p.pop() // pop the equal sign
      value = p.expression()
    } else {
      value = nil
    }
    if value == nil {
      node.Arguments = make([]*ir.Instruction, 1)
    } else {
      node.Arguments = make([]*ir.Instruction, 2)
      node.Arguments[1] = value
    }
    node.Arguments[0] = &ir.Instruction {
      Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
      Value: proto.String(name),
    }
  }
  if p.peek().Lexeme == LBRACE {
    node.Children = p.block()
  }
  return node
}

func (p *Parser) block() []*ir.Instruction {
  stmts := make([]*ir.Instruction, 0)
  p.pop() // pop the lbrace
  for p.peek().Lexeme != RBRACE {
    stmts = append(stmts, p.statement())
  }
  p.pop() // pop the rbrace
  if len(stmts) == 0 {
    stmts = nil
  }
  return stmts
}

// func (p *Parser) definition() *ir.Function {
//
//   p.pop() // pop the "@func" keyword
//   opensIn := ""
//   if p.peek().Lexeme == TYPE {
//     opensIn = p.pop().Value
//   }
//
// }

<|endoftext|>


<commit_before><commit_msg>Adding required package name to fix compilation<commit_after>package stackutil

<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cgotest

// Issue 8428.  Cgo inconsistently translated zero size arrays.

/*
struct issue8428one {
	char b;
	char rest[];
};

struct issue8428two {
	void *p;
	char b;
	char rest[0];
};

struct issue8428three {
	char w[1][2][3][0];
	char x[2][3][0][1];
	char y[3][0][1][2];
	char z[0][1][2][3];
};
*/
import "C"

import "unsafe"

var _ = C.struct_issue8428one{
	b:    C.char(0),
	rest: [0]C.char{},
}

var _ = C.struct_issue8428two{
	p:    unsafe.Pointer(nil),
	b:    C.char(0),
	rest: [0]C.char{},
}

var _ = C.struct_issue8428three{
	w: [1][2][3][0]C.char{},
	x: [2][3][0][1]C.char{},
	y: [3][0][1][2]C.char{},
	z: [0][1][2][3]C.char{},
}
<commit_msg>misc/cgo/test: disable issue 8428 regress test on darwin<commit_after>// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This test fails on older versions of OS X because they use older buggy
// versions of Clang that emit ambiguous DWARF info.  See issue 8611.
// +build !darwin

package cgotest

// Issue 8428.  Cgo inconsistently translated zero size arrays.

/*
struct issue8428one {
	char b;
	char rest[];
};

struct issue8428two {
	void *p;
	char b;
	char rest[0];
};

struct issue8428three {
	char w[1][2][3][0];
	char x[2][3][0][1];
	char y[3][0][1][2];
	char z[0][1][2][3];
};
*/
import "C"

import "unsafe"

var _ = C.struct_issue8428one{
	b:    C.char(0),
	rest: [0]C.char{},
}

var _ = C.struct_issue8428two{
	p:    unsafe.Pointer(nil),
	b:    C.char(0),
	rest: [0]C.char{},
}

var _ = C.struct_issue8428three{
	w: [1][2][3][0]C.char{},
	x: [2][3][0][1]C.char{},
	y: [3][0][1][2]C.char{},
	z: [0][1][2][3]C.char{},
}
<|endoftext|>


<commit_before>// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nginxreceiver

import (
	"context"
	"net/http"
	"time"

	"github.com/nginxinc/nginx-prometheus-exporter/client"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/consumer/pdata"
	"go.opentelemetry.io/collector/consumer/simple"
	"go.uber.org/zap"

	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/nginxreceiver/internal/metadata"
)

type nginxScraper struct {
	httpClient *http.Client
	client     *client.NginxClient

	logger *zap.Logger
	cfg    *Config
}

func newNginxScraper(
	logger *zap.Logger,
	cfg *Config,
) *nginxScraper {
	return &nginxScraper{
		logger: logger,
		cfg:    cfg,
	}
}

func (r *nginxScraper) start(_ context.Context, host component.Host) error {
	httpClient, err := r.cfg.ToClient(host.GetExtensions())
	if err != nil {
		return err
	}
	r.httpClient = httpClient

	return nil
}

func (r *nginxScraper) scrape(context.Context) (pdata.ResourceMetricsSlice, error) {
	// Init client in scrape method in case there are transient errors in the
	// constructor.
	if r.client == nil {
		var err error
		r.client, err = client.NewNginxClient(r.httpClient, r.cfg.HTTPClientSettings.Endpoint)
		if err != nil {
			r.client = nil
			return pdata.ResourceMetricsSlice{}, err
		}
	}

	metrics := simple.Metrics{
		Metrics:                    pdata.NewMetrics(),
		Timestamp:                  time.Now(),
		MetricFactoriesByName:      metadata.M.FactoriesByName(),
		InstrumentationLibraryName: "otelcol/nginx",
	}

	stats, err := r.client.GetStubStats()
	if err != nil {
		r.logger.Error("Failed to fetch nginx stats", zap.Error(err))
		return pdata.ResourceMetricsSlice{}, err
	}

	metrics.AddSumDataPoint(metadata.M.NginxRequests.Name(), stats.Requests)
	metrics.AddSumDataPoint(metadata.M.NginxConnectionsAccepted.Name(), stats.Connections.Accepted)
	metrics.AddSumDataPoint(metadata.M.NginxConnectionsHandled.Name(), stats.Connections.Handled)

	metrics.WithLabels(map[string]string{metadata.L.State: metadata.LabelState.Active}).AddGaugeDataPoint(metadata.M.NginxConnectionsCurrent.Name(), stats.Connections.Active)
	metrics.WithLabels(map[string]string{metadata.L.State: metadata.LabelState.Reading}).AddGaugeDataPoint(metadata.M.NginxConnectionsCurrent.Name(), stats.Connections.Reading)
	metrics.WithLabels(map[string]string{metadata.L.State: metadata.LabelState.Writing}).AddGaugeDataPoint(metadata.M.NginxConnectionsCurrent.Name(), stats.Connections.Writing)
	metrics.WithLabels(map[string]string{metadata.L.State: metadata.LabelState.Waiting}).AddGaugeDataPoint(metadata.M.NginxConnectionsCurrent.Name(), stats.Connections.Waiting)

	return metrics.Metrics.ResourceMetrics(), nil
}
<commit_msg>Remove usage of simple package in nginx receiver (#3801)<commit_after>// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nginxreceiver

import (
	"context"
	"net/http"
	"time"

	"github.com/nginxinc/nginx-prometheus-exporter/client"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/consumer/pdata"
	"go.uber.org/zap"

	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/nginxreceiver/internal/metadata"
)

type nginxScraper struct {
	httpClient *http.Client
	client     *client.NginxClient

	logger *zap.Logger
	cfg    *Config
}

func newNginxScraper(
	logger *zap.Logger,
	cfg *Config,
) *nginxScraper {
	return &nginxScraper{
		logger: logger,
		cfg:    cfg,
	}
}

func (r *nginxScraper) start(_ context.Context, host component.Host) error {
	httpClient, err := r.cfg.ToClient(host.GetExtensions())
	if err != nil {
		return err
	}
	r.httpClient = httpClient

	return nil
}

func (r *nginxScraper) scrape(context.Context) (pdata.ResourceMetricsSlice, error) {
	// Init client in scrape method in case there are transient errors in the
	// constructor.
	if r.client == nil {
		var err error
		r.client, err = client.NewNginxClient(r.httpClient, r.cfg.HTTPClientSettings.Endpoint)
		if err != nil {
			r.client = nil
			return pdata.ResourceMetricsSlice{}, err
		}
	}

	stats, err := r.client.GetStubStats()
	if err != nil {
		r.logger.Error("Failed to fetch nginx stats", zap.Error(err))
		return pdata.ResourceMetricsSlice{}, err
	}

	now := pdata.TimestampFromTime(time.Now())
	metrics := pdata.NewMetrics()
	ilm := metrics.ResourceMetrics().AppendEmpty().InstrumentationLibraryMetrics().AppendEmpty()
	ilm.InstrumentationLibrary().SetName("otelcol/nginx")

	addSum(ilm.Metrics(), metadata.M.NginxRequests.Init, now, stats.Requests)
	addSum(ilm.Metrics(), metadata.M.NginxConnectionsAccepted.Init, now, stats.Connections.Accepted)
	addSum(ilm.Metrics(), metadata.M.NginxConnectionsHandled.Init, now, stats.Connections.Handled)

	currConnMetric := ilm.Metrics().AppendEmpty()
	metadata.M.NginxConnectionsCurrent.Init(currConnMetric)
	dps := currConnMetric.IntGauge().DataPoints()
	addCurrentConnectionDataPoint(dps, metadata.LabelState.Active, now, stats.Connections.Active)
	addCurrentConnectionDataPoint(dps, metadata.LabelState.Reading, now, stats.Connections.Reading)
	addCurrentConnectionDataPoint(dps, metadata.LabelState.Writing, now, stats.Connections.Writing)
	addCurrentConnectionDataPoint(dps, metadata.LabelState.Waiting, now, stats.Connections.Waiting)

	return metrics.ResourceMetrics(), nil
}

func addSum(metrics pdata.MetricSlice, initFunc func(pdata.Metric), now pdata.Timestamp, value int64) {
	metric := metrics.AppendEmpty()
	initFunc(metric)
	dp := metric.IntSum().DataPoints().AppendEmpty()
	dp.SetTimestamp(now)
	dp.SetValue(value)
}

func addCurrentConnectionDataPoint(dps pdata.IntDataPointSlice, stateValue string, now pdata.Timestamp, value int64) {
	dp := dps.AppendEmpty()
	dp.LabelsMap().Upsert(metadata.L.State, stateValue)
	dp.SetTimestamp(now)
	dp.SetValue(value)
}
<|endoftext|>