<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"fmt"

	. "github.com/araddon/gou"
)

var (
	_ = DEBUG
)

// A bool (and/or) clause
type BoolClause string

// Filter clause is either a boolClause or FilterOp
type FilterClause interface {
	String() string
}

// A wrapper to allow for custom serialization
type FilterWrap struct {
	boolClause string
	filters    []interface{}
}

func NewFilterWrap() *FilterWrap {
	return &FilterWrap{filters: make([]interface{}, 0), boolClause: "and"}
}

func (f *FilterWrap) String() string {
	return fmt.Sprintf(`fopv: %d:%v`, len(f.filters), f.filters)
}

// Bool sets the type of boolean filter to use.
// Accepted values are "and" and "or".
func (f *FilterWrap) Bool(s string) {
	f.boolClause = s
}

// Custom marshalling to support the query dsl
func (f *FilterWrap) addFilters(fl []interface{}) {
	if len(fl) > 1 {
		fc := fl[0]
		switch fc.(type) {
		case BoolClause, string:
			f.boolClause = fc.(string)
			fl = fl[1:]
		}
	}
	f.filters = append(f.filters, fl...)
}

// Custom marshalling to support the query dsl
func (f *FilterWrap) MarshalJSON() ([]byte, error) {
	var root interface{}
	if len(f.filters) > 1 {
		root = map[string]interface{}{f.boolClause: f.filters}
	} else if len(f.filters) == 1 {
		root = f.filters[0]
	}
	return json.Marshal(root)
}

/*
	"filter": {
		"range": {
		  "@timestamp": {
		    "from": "2012-12-29T16:52:48+00:00",
		    "to": "2012-12-29T17:52:48+00:00"
		  }
		}
	}
	"filter": {
	    "missing": {
	        "field": "repository.name"
	    }
	}

	"filter" : {
	    "terms" : {
	        "user" : ["kimchy", "elasticsearch"],
	        "execution" : "bool",
	        "_cache": true
	    }
	}

	"filter" : {
	    "term" : { "user" : "kimchy"}
	}

	"filter" : {
	    "and" : [
	        {
	            "range" : {
	                "postDate" : {
	                    "from" : "2010-03-01",
	                    "to" : "2010-04-01"
	                }
	            }
	        },
	        {
	            "prefix" : { "name.second" : "ba" }
	        }
	    ]
	}

*/

// HasChildFilterOp represents a has_child filter.
type HasChildFilterOp struct {
	Type        string `json:"type"`
	MinChildren int    `json:"min_children,omitempty"`
	MaxChildren int    `json:"max_children,omitempty"`

	filters *FilterWrap
}

// Filter adds the provided filters to the has_child condition.
func (hc *HasChildFilterOp) Filter(fs ...interface{}) *HasChildFilterOp {
	if hc.filters == nil {
		hc.filters = NewFilterWrap()
	}
	hc.filters.addFilters(fs)
	return hc
}

// HasChild creates a has_child filter.
func HasChild(docType string, min, max int) *FilterOp {
	return &FilterOp{
		HasChild: &HasChildFilterOp{
			Type:        docType,
			MinChildren: min,
			MaxChildren: max,
		},
	}
}

// HasParentFilterOp represents a has_parent filter.
type HasParentFilterOp struct {
	Type string `json:"type"`

	filters *FilterWrap
}

// Filter adds the provided filters to the has_parent condition.
func (hp *HasParentFilterOp) Filter(fs ...interface{}) *HasParentFilterOp {
	if hp.filters == nil {
		hp.filters = NewFilterWrap()
	}
	hp.filters.addFilters(fs)
	return hp
}

// HasParent creates a has_parent filter.
func HasParent(docType string) *FilterOp {
	return &FilterOp{
		HasParent: &HasParentFilterOp{
			Type: docType,
		},
	}
}

// Filter Operation
//
//   Filter().Term("user","kimchy")
//
//   // we use variadics to allow n arguments, first is the "field" rest are values
//   Filter().Terms("user", "kimchy", "elasticsearch")
//
//   Filter().Exists("repository.name")
//
func Filter() *FilterOp {
	return &FilterOp{}
}

func CompoundFilter(fl ...interface{}) *FilterWrap {
	FilterVal := NewFilterWrap()
	FilterVal.addFilters(fl)
	return FilterVal
}

// BoolFilterOp.
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html
type BoolFilterOp struct {
	MinShouldMatch int                                 `json:"minimum_should_match,omitempty"`
	Boost          float64                             `json:"boost,omitempty"`
	Should         []map[string]map[string]interface{} `json:"should,omitempty"`
	Must           []map[string]map[string]interface{} `json:"must,omitempty"`
}

// AddShould adds a new Should term criterion on the BoolFilterOp.
func (b *BoolFilterOp) AddShould(term string, val interface{}) {
	m := make(map[string]map[string]interface{})
	m["term"] = map[string]interface{}{term: val}
	b.Should = append(b.Should, m)
}

// AddMust adds a new Must term criterion on the BoolFilterOp.
func (b *BoolFilterOp) AddMust(term string, val interface{}) {
	m := make(map[string]map[string]interface{})
	m["term"] = map[string]interface{}{term: val}
	b.Must = append(b.Must, m)
}

type FilterOp struct {
	curField   string
	TermsMap   map[string][]interface{}          `json:"terms,omitempty"`
	Range      map[string]map[string]interface{} `json:"range,omitempty"`
	Exist      map[string]string                 `json:"exists,omitempty"`
	MissingVal map[string]string                 `json:"missing,omitempty"`
	Bool       *BoolFilterOp                     `json:"bool,omitempty"`
	HasChild   *HasChildFilterOp                 `json:"has_child,omitempty"`
	HasParent  *HasParentFilterOp                `json:"has_parent,omitempty"`
}

// Bool creates a bool filter.
func Bool(minMatch int, boost float64) *FilterOp {
	return &FilterOp{Bool: &BoolFilterOp{
		MinShouldMatch: minMatch,
		Boost:          boost,
	}}
}

// A range is a special type of Filter operation
//
//    Range().Exists("repository.name")
func Range() *FilterOp {
	return &FilterOp{Range: make(map[string]map[string]interface{})}
}

func (f *FilterOp) Field(fld string) *FilterOp {
	f.curField = fld
	if _, ok := f.Range[fld]; !ok {
		m := make(map[string]interface{})
		f.Range[fld] = m
	}
	return f
}

// Filter Terms
//
//   Filter().Terms("user","kimchy")
//
//   // we use variadics to allow n arguments, first is the "field" rest are values
//   Filter().Terms("user", "kimchy", "elasticsearch")
//
func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
	if len(f.TermsMap) == 0 {
		f.TermsMap = make(map[string][]interface{})
	}
	for _, val := range values {
		f.TermsMap[field] = append(f.TermsMap[field], val)
	}

	return f
}
func (f *FilterOp) From(from string) *FilterOp {
	f.Range[f.curField]["from"] = from
	return f
}
func (f *FilterOp) To(to string) *FilterOp {
	f.Range[f.curField]["to"] = to
	return f
}
func (f *FilterOp) Gt(gt interface{}) *FilterOp {
	f.Range[f.curField]["gt"] = gt
	return f
}
func (f *FilterOp) Lt(lt interface{}) *FilterOp {
	f.Range[f.curField]["lt"] = lt
	return f
}
func (f *FilterOp) Gte(gte interface{}) *FilterOp {
	f.Range[f.curField]["gte"] = gte
	return f
}
func (f *FilterOp) Lte(lte interface{}) *FilterOp {
	f.Range[f.curField]["lte"] = lte
	return f
}
func (f *FilterOp) Exists(name string) *FilterOp {
	f.Exist = map[string]string{"field": name}
	return f
}
func (f *FilterOp) Missing(name string) *FilterOp {
	f.MissingVal = map[string]string{"field": name}
	return f
}

// Add another Filterop, "combines" two filter ops into one
func (f *FilterOp) Add(fop *FilterOp) *FilterOp {
	// TODO, this is invalid, refactor
	if len(fop.Exist) > 0 {
		f.Exist = fop.Exist
	}
	if len(fop.MissingVal) > 0 {
		f.MissingVal = fop.MissingVal
	}
	if len(fop.Range) > 0 {
		f.Range = fop.Range
	}
	return f
}
<commit_msg>lib/searchfilter: actually marshal the filter<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"fmt"

	. "github.com/araddon/gou"
)

var (
	_ = DEBUG
)

// A bool (and/or) clause
type BoolClause string

// Filter clause is either a boolClause or FilterOp
type FilterClause interface {
	String() string
}

// A wrapper to allow for custom serialization
type FilterWrap struct {
	boolClause string
	filters    []interface{}
}

func NewFilterWrap() *FilterWrap {
	return &FilterWrap{filters: make([]interface{}, 0), boolClause: "and"}
}

func (f *FilterWrap) String() string {
	return fmt.Sprintf(`fopv: %d:%v`, len(f.filters), f.filters)
}

// Bool sets the type of boolean filter to use.
// Accepted values are "and" and "or".
func (f *FilterWrap) Bool(s string) {
	f.boolClause = s
}

// Custom marshalling to support the query dsl
func (f *FilterWrap) addFilters(fl []interface{}) {
	if len(fl) > 1 {
		fc := fl[0]
		switch fc.(type) {
		case BoolClause, string:
			f.boolClause = fc.(string)
			fl = fl[1:]
		}
	}
	f.filters = append(f.filters, fl...)
}

// Custom marshalling to support the query dsl
func (f *FilterWrap) MarshalJSON() ([]byte, error) {
	var root interface{}
	if len(f.filters) > 1 {
		root = map[string]interface{}{f.boolClause: f.filters}
	} else if len(f.filters) == 1 {
		root = f.filters[0]
	}
	return json.Marshal(root)
}

/*
	"filter": {
		"range": {
		  "@timestamp": {
		    "from": "2012-12-29T16:52:48+00:00",
		    "to": "2012-12-29T17:52:48+00:00"
		  }
		}
	}
	"filter": {
	    "missing": {
	        "field": "repository.name"
	    }
	}

	"filter" : {
	    "terms" : {
	        "user" : ["kimchy", "elasticsearch"],
	        "execution" : "bool",
	        "_cache": true
	    }
	}

	"filter" : {
	    "term" : { "user" : "kimchy"}
	}

	"filter" : {
	    "and" : [
	        {
	            "range" : {
	                "postDate" : {
	                    "from" : "2010-03-01",
	                    "to" : "2010-04-01"
	                }
	            }
	        },
	        {
	            "prefix" : { "name.second" : "ba" }
	        }
	    ]
	}

*/

// HasChildFilterOp represents a has_child filter.
type HasChildFilterOp struct {
	Type        string      `json:"type"`
	MinChildren int         `json:"min_children,omitempty"`
	MaxChildren int         `json:"max_children,omitempty"`
	Filters     *FilterWrap `json:"filter,omitempty"`
}

// Filter adds the provided filters to the has_child condition.
func (hc *HasChildFilterOp) Filter(fs ...interface{}) *HasChildFilterOp {
	if hc.Filters == nil {
		hc.Filters = NewFilterWrap()
	}
	hc.Filters.addFilters(fs)
	return hc
}

// HasChild creates a has_child filter.
func HasChild(docType string, min, max int) *FilterOp {
	return &FilterOp{
		HasChild: &HasChildFilterOp{
			Type:        docType,
			MinChildren: min,
			MaxChildren: max,
		},
	}
}

// HasParentFilterOp represents a has_parent filter.
type HasParentFilterOp struct {
	Type    string      `json:"type"`
	Filters *FilterWrap `json:"filter,omitempty"`
}

// Filter adds the provided filters to the has_parent condition.
func (hp *HasParentFilterOp) Filter(fs ...interface{}) *HasParentFilterOp {
	if hp.Filters == nil {
		hp.Filters = NewFilterWrap()
	}
	hp.Filters.addFilters(fs)
	return hp
}

// HasParent creates a has_parent filter.
func HasParent(docType string) *FilterOp {
	return &FilterOp{
		HasParent: &HasParentFilterOp{
			Type: docType,
		},
	}
}

// Filter Operation
//
//   Filter().Term("user","kimchy")
//
//   // we use variadics to allow n arguments, first is the "field" rest are values
//   Filter().Terms("user", "kimchy", "elasticsearch")
//
//   Filter().Exists("repository.name")
//
func Filter() *FilterOp {
	return &FilterOp{}
}

func CompoundFilter(fl ...interface{}) *FilterWrap {
	FilterVal := NewFilterWrap()
	FilterVal.addFilters(fl)
	return FilterVal
}

// BoolFilterOp.
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html
type BoolFilterOp struct {
	MinShouldMatch int                                 `json:"minimum_should_match,omitempty"`
	Boost          float64                             `json:"boost,omitempty"`
	Should         []map[string]map[string]interface{} `json:"should,omitempty"`
	Must           []map[string]map[string]interface{} `json:"must,omitempty"`
}

// AddShould adds a new Should term criterion on the BoolFilterOp.
func (b *BoolFilterOp) AddShould(term string, val interface{}) {
	m := make(map[string]map[string]interface{})
	m["term"] = map[string]interface{}{term: val}
	b.Should = append(b.Should, m)
}

// AddMust adds a new Must term criterion on the BoolFilterOp.
func (b *BoolFilterOp) AddMust(term string, val interface{}) {
	m := make(map[string]map[string]interface{})
	m["term"] = map[string]interface{}{term: val}
	b.Must = append(b.Must, m)
}

type FilterOp struct {
	curField   string
	TermsMap   map[string][]interface{}          `json:"terms,omitempty"`
	Range      map[string]map[string]interface{} `json:"range,omitempty"`
	Exist      map[string]string                 `json:"exists,omitempty"`
	MissingVal map[string]string                 `json:"missing,omitempty"`
	Bool       *BoolFilterOp                     `json:"bool,omitempty"`
	HasChild   *HasChildFilterOp                 `json:"has_child,omitempty"`
	HasParent  *HasParentFilterOp                `json:"has_parent,omitempty"`
}

// Bool creates a bool filter.
func Bool(minMatch int, boost float64) *FilterOp {
	return &FilterOp{Bool: &BoolFilterOp{
		MinShouldMatch: minMatch,
		Boost:          boost,
	}}
}

// A range is a special type of Filter operation
//
//    Range().Exists("repository.name")
func Range() *FilterOp {
	return &FilterOp{Range: make(map[string]map[string]interface{})}
}

func (f *FilterOp) Field(fld string) *FilterOp {
	f.curField = fld
	if _, ok := f.Range[fld]; !ok {
		m := make(map[string]interface{})
		f.Range[fld] = m
	}
	return f
}

// Filter Terms
//
//   Filter().Terms("user","kimchy")
//
//   // we use variadics to allow n arguments, first is the "field" rest are values
//   Filter().Terms("user", "kimchy", "elasticsearch")
//
func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
	if len(f.TermsMap) == 0 {
		f.TermsMap = make(map[string][]interface{})
	}
	for _, val := range values {
		f.TermsMap[field] = append(f.TermsMap[field], val)
	}

	return f
}
func (f *FilterOp) From(from string) *FilterOp {
	f.Range[f.curField]["from"] = from
	return f
}
func (f *FilterOp) To(to string) *FilterOp {
	f.Range[f.curField]["to"] = to
	return f
}
func (f *FilterOp) Gt(gt interface{}) *FilterOp {
	f.Range[f.curField]["gt"] = gt
	return f
}
func (f *FilterOp) Lt(lt interface{}) *FilterOp {
	f.Range[f.curField]["lt"] = lt
	return f
}
func (f *FilterOp) Gte(gte interface{}) *FilterOp {
	f.Range[f.curField]["gte"] = gte
	return f
}
func (f *FilterOp) Lte(lte interface{}) *FilterOp {
	f.Range[f.curField]["lte"] = lte
	return f
}
func (f *FilterOp) Exists(name string) *FilterOp {
	f.Exist = map[string]string{"field": name}
	return f
}
func (f *FilterOp) Missing(name string) *FilterOp {
	f.MissingVal = map[string]string{"field": name}
	return f
}

// Add another Filterop, "combines" two filter ops into one
func (f *FilterOp) Add(fop *FilterOp) *FilterOp {
	// TODO, this is invalid, refactor
	if len(fop.Exist) > 0 {
		f.Exist = fop.Exist
	}
	if len(fop.MissingVal) > 0 {
		f.MissingVal = fop.MissingVal
	}
	if len(fop.Range) > 0 {
		f.Range = fop.Range
	}
	return f
}
<|endoftext|>


<commit_before>package tinylfu

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestUniformSaturation(t *testing.T) {
	p := newTestPolicy(t)
	for i := 0; i < 20; i++ {
		p.Record(uint64(i))
	}

	checkData(t, p, []uint64{12, 13, 14, 15, 16, 17, 18, 19})
	checkSegment(t, p.window, []uint64{19, 18})
	checkSegment(t, p.probation, []uint64{17, 16, 15, 14, 13, 12})
	checkSegment(t, p.protected, nil)
}

func TestTinyLFU(t *testing.T) {
	p := newTestPolicy(t)

	// Saturate the window and probation segments.
	for i := 0; i < 8; i++ {
		p.Record(uint64(i))
	}

	// Access some probation, but don't evict or demote anything yet.
	for i := 0; i < 4; i++ {
		p.Record(uint64(i))
	}

	checkData(t, p, []uint64{0, 1, 2, 3, 4, 5, 6, 7})
	checkSegment(t, p.window, []uint64{7, 6})
	checkSegment(t, p.probation, []uint64{5, 4})
	checkSegment(t, p.protected, []uint64{3, 2, 1, 0})

	// Refresh something in the protected region and promote something from probation.
	p.Record(2)
	p.Record(5) // Demote 0

	checkData(t, p, []uint64{0, 1, 2, 3, 4, 5, 6, 7})
	checkSegment(t, p.window, []uint64{7, 6})
	checkSegment(t, p.probation, []uint64{0, 4})
	checkSegment(t, p.protected, []uint64{5, 2, 3, 1})

	// Evict a few values.
	for i := 10; i < 13; i++ {
		p.Record(uint64(i))
	}

	checkData(t, p, []uint64{1, 2, 3, 5, 7, 10, 11, 12})
	checkSegment(t, p.window, []uint64{12, 11})
	checkSegment(t, p.probation, []uint64{10, 7})
	checkSegment(t, p.protected, []uint64{5, 2, 3, 1})

	// Finally, promote a window value.
	//f
}

func newTestPolicy(t *testing.T) *Policy {
	// Create a policy with 2 window, 2 probation, 4 protected slots. This is
	// enough to fully exercise most cases without being onerous to validate
	// comprehensivley.
	return New(8, WithSegmentation(.75, .67))
}

// Verify a policy's data map contains the given keys in any order.
func checkData(t *testing.T, p *Policy, values []uint64) {
	t.Helper()
	if !assert.Equal(t, len(values), len(p.data), "data size") {
		return
	}

	for _, v := range values {
		e, ok := p.data[v]
		if assert.True(t, ok, "key %d exists", v) {
			assert.Equal(t, v, e.Value, "entry node matches key")
		}
	}
}

// Verify a segment contains the given values in order.
func checkSegment(t *testing.T, l *list, values []uint64) {
	t.Helper()
	if !assert.Equal(t, len(values), l.Len(), "segment size") {
		return
	}

	node := l.Front()
	for _, v := range values {
		assert.Equal(t, v, node.Value)
		node = node.Next()
	}
}
<commit_msg>Finish primary test which trailed off<commit_after>package tinylfu

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestUniformSaturation(t *testing.T) {
	p := newTestPolicy(t)
	for i := 0; i < 20; i++ {
		p.Record(uint64(i))
	}

	checkData(t, p, []uint64{12, 13, 14, 15, 16, 17, 18, 19})
	checkSegment(t, p.window, []uint64{19, 18})
	checkSegment(t, p.probation, []uint64{17, 16, 15, 14, 13, 12})
	checkSegment(t, p.protected, nil)
}

func TestTinyLFU(t *testing.T) {
	p := newTestPolicy(t)

	// Saturate the window and probation segments.
	for i := 0; i < 8; i++ {
		p.Record(uint64(i))
	}

	// Access some probation, but don't evict or demote anything yet.
	for i := 0; i < 4; i++ {
		p.Record(uint64(i))
	}

	checkData(t, p, []uint64{0, 1, 2, 3, 4, 5, 6, 7})
	checkSegment(t, p.window, []uint64{7, 6})
	checkSegment(t, p.probation, []uint64{5, 4})
	checkSegment(t, p.protected, []uint64{3, 2, 1, 0})

	// Refresh something in the protected region and promote something from probation.
	p.Record(2)
	p.Record(5) // Demote 0

	checkData(t, p, []uint64{0, 1, 2, 3, 4, 5, 6, 7})
	checkSegment(t, p.window, []uint64{7, 6})
	checkSegment(t, p.probation, []uint64{0, 4})
	checkSegment(t, p.protected, []uint64{5, 2, 3, 1})

	// Evict a few values.
	for i := 10; i < 13; i++ {
		p.Record(uint64(i))
	}

	checkData(t, p, []uint64{1, 2, 3, 5, 7, 10, 11, 12})
	checkSegment(t, p.window, []uint64{12, 11})
	checkSegment(t, p.probation, []uint64{10, 7})
	checkSegment(t, p.protected, []uint64{5, 2, 3, 1})

	// Finally, promote a window value.
	p.Record(11)

	checkData(t, p, []uint64{1, 2, 3, 5, 7, 10, 11, 12})
	checkSegment(t, p.window, []uint64{11, 12})
	checkSegment(t, p.probation, []uint64{10, 7})
	checkSegment(t, p.protected, []uint64{5, 2, 3, 1})
}

func newTestPolicy(t *testing.T) *Policy {
	// Create a policy with 2 window, 2 probation, 4 protected slots. This is
	// enough to fully exercise most cases without being onerous to validate
	// comprehensivley.
	return New(8, WithSegmentation(.75, .67))
}

// Verify a policy's data map contains the given keys in any order.
func checkData(t *testing.T, p *Policy, values []uint64) {
	t.Helper()
	if !assert.Equal(t, len(values), len(p.data), "data size") {
		return
	}

	for _, v := range values {
		e, ok := p.data[v]
		if assert.True(t, ok, "key %d exists", v) {
			assert.Equal(t, v, e.Value, "entry node matches key")
		}
	}
}

// Verify a segment contains the given values in order.
func checkSegment(t *testing.T, l *list, values []uint64) {
	t.Helper()
	if !assert.Equal(t, len(values), l.Len(), "segment size") {
		return
	}

	node := l.Front()
	for _, v := range values {
		assert.Equal(t, v, node.Value)
		node = node.Next()
	}
}
<|endoftext|>


<commit_before>// +build !trace

package sqlite3

import "errors"

// Trace... constants identify the possible events causing callback invocation.
// Values are same as the corresponding SQLite Trace Event Codes.
const (
	TraceStmt    = uint32(0x01)
	TraceProfile = uint32(0x02)
	TraceRow     = uint32(0x04)
	TraceClose   = uint32(0x08)
)

type TraceInfo struct {
	// Pack together the shorter fields, to keep the struct smaller.
	// On a 64-bit machine there would be padding
	// between EventCode and ConnHandle; having AutoCommit here is "free":
	EventCode  uint32
	AutoCommit bool
	ConnHandle uintptr

	// Usually filled, unless EventCode = TraceClose = SQLITE_TRACE_CLOSE:
	// identifier for a prepared statement:
	StmtHandle uintptr

	// Two strings filled when EventCode = TraceStmt = SQLITE_TRACE_STMT:
	// (1) either the unexpanded SQL text of the prepared statement, or
	//     an SQL comment that indicates the invocation of a trigger;
	// (2) expanded SQL, if requested and if (1) is not an SQL comment.
	StmtOrTrigger string
	ExpandedSQL   string // only if requested (TraceConfig.WantExpandedSQL = true)

	// filled when EventCode = TraceProfile = SQLITE_TRACE_PROFILE:
	// estimated number of nanoseconds that the prepared statement took to run:
	RunTimeNanosec int64
}

type TraceUserCallback func(TraceInfo) int

type TraceConfig struct {
	Callback        TraceUserCallback
	EventMask       uint
	WantExpandedSQL bool
}

// RegisterAggregator register the aggregator.
func (c *SQLiteConn) RegisterAggregator(name string, impl interface{}, pure bool) error {
	return errors.New("This feature is not implemented")
}

func (c *SQLiteConn) SetTrace(requested *TraceConfig) error {
	return errors.New("This feature is not implemented")
}
<commit_msg>remove fallbak for trace<commit_after><|endoftext|>


<commit_before>package web

import (
	"net/http"
	"net/url"
	"time"

	"github.com/pkg/errors"
	"github.com/yuuki/diamondb/lib/env"
	"github.com/yuuki/diamondb/lib/log"
	"github.com/yuuki/diamondb/lib/query"
	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/timeparser"
)

const (
	// DayTime is one day period.
	DayTime = time.Duration(24*60*60) * time.Second
)

// PingHandler returns a HTTP handler for the endpoint to ping storage.
func PingHandler(env *env.Env) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := env.Fetcher.Ping(); err != nil {
			unavaliableError(w, errors.Cause(err).Error())
			return
		}
		ok(w, "PONG")
		return
	})
}

// RenderHandler returns a HTTP handler for the endpoint to read data.
func RenderHandler(env *env.Env) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		until := time.Now().Round(time.Second)
		from := until.Add(-DayTime)

		if v := r.FormValue("from"); v != "" {
			t, err := timeparser.ParseAtTime(url.QueryEscape(v))
			if err != nil {
				log.Printf("%+v", err) // Print stack trace by pkg/errors
				badRequest(w, errors.Cause(err).Error())
				return
			}
			from = t
		}
		if v := r.FormValue("until"); v != "" {
			t, err := timeparser.ParseAtTime(url.QueryEscape(v))
			if err != nil {
				log.Printf("%+v", err) // Print stack trace by pkg/errors
				badRequest(w, errors.Cause(err).Error())
				return
			}
			until = t
		}
		log.Debugf("from:%d until:%d", from.Unix(), until.Unix())

		targets := r.Form["target"]
		if len(targets) < 1 {
			badRequest(w, "no targets requested")
			return
		}

		seriesResps := series.SeriesSlice{}
		for _, target := range targets {
			seriesSlice, err := query.EvalTarget(env.Fetcher, target, from, until)
			if err != nil {
				log.Printf("%+v", err) // Print stack trace by pkg/errors
				switch err.(type) {
				case *query.ParserError, *query.UnsupportedFunctionError:
					badRequest(w, errors.Cause(err).Error())
				default:
					serverError(w, errors.Cause(err).Error())
				}
				return
			}
			seriesResps = append(seriesResps, seriesSlice...)
		}
		renderJSON(w, http.StatusOK, seriesResps)
	})
}
<commit_msg>Fix bug that error code is always 500<commit_after>package web

import (
	"net/http"
	"net/url"
	"time"

	"github.com/pkg/errors"
	"github.com/yuuki/diamondb/lib/env"
	"github.com/yuuki/diamondb/lib/log"
	"github.com/yuuki/diamondb/lib/query"
	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/timeparser"
)

const (
	// DayTime is one day period.
	DayTime = time.Duration(24*60*60) * time.Second
)

// PingHandler returns a HTTP handler for the endpoint to ping storage.
func PingHandler(env *env.Env) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := env.Fetcher.Ping(); err != nil {
			unavaliableError(w, errors.Cause(err).Error())
			return
		}
		ok(w, "PONG")
		return
	})
}

// RenderHandler returns a HTTP handler for the endpoint to read data.
func RenderHandler(env *env.Env) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		until := time.Now().Round(time.Second)
		from := until.Add(-DayTime)

		if v := r.FormValue("from"); v != "" {
			t, err := timeparser.ParseAtTime(url.QueryEscape(v))
			if err != nil {
				log.Printf("%+v", err) // Print stack trace by pkg/errors
				badRequest(w, errors.Cause(err).Error())
				return
			}
			from = t
		}
		if v := r.FormValue("until"); v != "" {
			t, err := timeparser.ParseAtTime(url.QueryEscape(v))
			if err != nil {
				log.Printf("%+v", err) // Print stack trace by pkg/errors
				badRequest(w, errors.Cause(err).Error())
				return
			}
			until = t
		}
		log.Debugf("from:%d until:%d", from.Unix(), until.Unix())

		targets := r.Form["target"]
		if len(targets) < 1 {
			badRequest(w, "no targets requested")
			return
		}

		seriesResps := series.SeriesSlice{}
		for _, target := range targets {
			seriesSlice, err := query.EvalTarget(env.Fetcher, target, from, until)
			if err != nil {
				log.Printf("%+v", err) // Print stack trace by pkg/errors
				switch errors.Cause(err).(type) {
				case *query.ParserError, *query.UnsupportedFunctionError:
					badRequest(w, errors.Cause(err).Error())
				default:
					serverError(w, errors.Cause(err).Error())
				}
				return
			}
			seriesResps = append(seriesResps, seriesSlice...)
		}
		renderJSON(w, http.StatusOK, seriesResps)
	})
}
<|endoftext|>


<commit_before>package logging

import (
	"errors"
	"testing"
)

func TestStatus(t *testing.T) {
	j := Job{}

	if j.Status() != "0" {
		t.Error("Wrong status", j.Status())
	}

	task := Task{
		Return: errors.New("foo"),
	}
	j.Tasks = append(j.Tasks, task)

	if j.Status() == "0" {
		t.Error("Wrong status", j.Status())
	}
}

func TestSuccess(t *testing.T) {
	j := Job{}

	if j.Success() == false {
		t.Error("Returned error for successful build")
	}

	task := Task{
		Return: errors.New("foo"),
	}
	j.Tasks = append(j.Tasks, task)

	if j.Success() == true {
		t.Error("Returned no error for failed build")
	}
}
<commit_msg>add tests for new task methods<commit_after>package logging

import (
	"errors"
	"testing"
)

func TestStatus(t *testing.T) {
	j := Job{}

	if j.Status() != "0" {
		t.Error("Wrong status", j.Status())
	}

	task := Task{
		Return: errors.New("foo"),
	}
	j.Tasks = append(j.Tasks, task)

	if j.Status() == "0" {
		t.Error("Wrong status", j.Status())
	}
}

func TestSuccess(t *testing.T) {
	j := Job{}

	if j.Success() == false {
		t.Error("Returned error for successful build")
	}

	task := Task{
		Return: errors.New("foo"),
	}
	j.Tasks = append(j.Tasks, task)

	if j.Success() == true {
		t.Error("Returned no error for failed build")
	}
}

func TestAddTask(t *testing.T) {
	j := Job{}
	task := Task{}

	j.AddTask(task)

	if len(j.Tasks) != 1 {
		t.Error("Wrong length of task list", len(j.Tasks))
	}
}

func TestTaskStatus(t *testing.T) {
	task := Task{}

	if task.Status() != "success" {
		t.Error("Wrong status", task.Status())
	}

	task.Return = errors.New("foo")

	if task.Status() == "success" {
		t.Error("Wrong status", task.Status())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"

	"github.com/spf13/cobra"

	// Used by cgo
	_ "github.com/lxc/lxd/lxd/include"

	"github.com/lxc/lxd/shared/netutils"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <errno.h>
#include <fcntl.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include "include/macro.h"
#include "include/memory_utils.h"

extern char *advance_arg(bool required);
extern bool change_namespaces(int pidfd, int nsfd, unsigned int flags);
extern int pidfd_nsfd(int pidfd, pid_t pid);

void forkdonetinfo(int pidfd, int ns_fd)
{
	if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNET)) {
		fprintf(stderr, "Failed setns to container network namespace: %s\n", strerror(errno));
		_exit(1);
	}

	// Jump back to Go for the rest
}

static int dosetns_file(char *file, char *nstype)
{
	__do_close int ns_fd = -EBADF;

	ns_fd = open(file, O_RDONLY);
	if (ns_fd < 0) {
		fprintf(stderr, "%m - Failed to open \"%s\"", file);
		return -1;
	}

	if (setns(ns_fd, 0) < 0) {
		fprintf(stderr, "%m - Failed to attach to namespace \"%s\"", file);
		return -1;
	}

	return 0;
}

void forkdonetdetach(char *file) {
	if (dosetns_file(file, "net") < 0) {
		fprintf(stderr, "Failed setns to container network namespace: %s\n", strerror(errno));
		_exit(1);
	}

	// Jump back to Go for the rest
}

void forknet(void)
{
	char *command = NULL;
	char *cur = NULL;
	pid_t pid = 0;

	// Get the subcommand
	command = advance_arg(false);
	if (command == NULL || (strcmp(command, "--help") == 0 || strcmp(command, "--version") == 0 || strcmp(command, "-h") == 0)) {
		return;
	}

	// skip "--"
	advance_arg(true);

	// Get the pid
	cur = advance_arg(false);
	if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) {
		return;
	}

	// Check that we're root
	if (geteuid() != 0) {
		fprintf(stderr, "Error: forknet requires root privileges\n");
		_exit(1);
	}

	// Call the subcommands
	if (strcmp(command, "info") == 0) {
		int ns_fd, pidfd;
		pid = atoi(cur);

		pidfd = atoi(advance_arg(true));
		ns_fd = pidfd_nsfd(pidfd, pid);
		if (ns_fd < 0)
			_exit(1);

		forkdonetinfo(pidfd, ns_fd);
	}

	if (strcmp(command, "detach") == 0)
		forkdonetdetach(cur);
}
*/
import "C"
import "github.com/lxc/lxd/shared"

type cmdForknet struct {
	global *cmdGlobal
}

func (c *cmdForknet) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forknet"
	cmd.Short = "Perform container network operations"
	cmd.Long = `Description:
  Perform container network operations

  This set of internal commands are used for some container network
  operations which require attaching to the container's network namespace.
`
	cmd.Hidden = true

	// pull
	cmdInfo := &cobra.Command{}
	cmdInfo.Use = "info <PID> <PidFd>"
	cmdInfo.Args = cobra.ExactArgs(2)
	cmdInfo.RunE = c.RunInfo
	cmd.AddCommand(cmdInfo)

	// detach
	cmdDetach := &cobra.Command{}
	cmdDetach.Use = "detach <netns file> <LXD PID> <ifname> <hostname>"
	cmdDetach.Args = cobra.ExactArgs(4)
	cmdDetach.RunE = c.RunDetach
	cmd.AddCommand(cmdDetach)

	return cmd
}

func (c *cmdForknet) RunInfo(cmd *cobra.Command, args []string) error {
	networks, err := netutils.NetnsGetifaddrs(-1)
	if err != nil {
		return err
	}

	buf, err := json.Marshal(networks)
	if err != nil {
		return err
	}

	fmt.Printf("%s\n", buf)

	return nil
}

func (c *cmdForknet) RunDetach(cmd *cobra.Command, args []string) error {
	lxdPID := args[1]
	ifName := args[2]
	hostName := args[3]

	if lxdPID == "" {
		return fmt.Errorf("LXD PID argument is required")
	}

	if ifName == "" {
		return fmt.Errorf("ifname argument is required")
	}

	if hostName == "" {
		return fmt.Errorf("hostname argument is required")
	}

	// Remove all IP addresses from interface before moving to parent netns.
	// This is to avoid any container address config leaking into host.
	_, err := shared.RunCommand("ip", "address", "flush", "dev", ifName)
	if err != nil {
		return err
	}

	// Rename the interface, set it down, and move into parent netns.
	_, err = shared.RunCommand("ip", "link", "set", ifName, "down", "name", hostName, "netns", lxdPID)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd: Replace ip command with ip package<commit_after>package main

import (
	"encoding/json"
	"fmt"

	"github.com/spf13/cobra"

	// Used by cgo
	_ "github.com/lxc/lxd/lxd/include"

	"github.com/lxc/lxd/lxd/ip"
	"github.com/lxc/lxd/shared/netutils"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <errno.h>
#include <fcntl.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include "include/macro.h"
#include "include/memory_utils.h"

extern char *advance_arg(bool required);
extern bool change_namespaces(int pidfd, int nsfd, unsigned int flags);
extern int pidfd_nsfd(int pidfd, pid_t pid);

void forkdonetinfo(int pidfd, int ns_fd)
{
	if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNET)) {
		fprintf(stderr, "Failed setns to container network namespace: %s\n", strerror(errno));
		_exit(1);
	}

	// Jump back to Go for the rest
}

static int dosetns_file(char *file, char *nstype)
{
	__do_close int ns_fd = -EBADF;

	ns_fd = open(file, O_RDONLY);
	if (ns_fd < 0) {
		fprintf(stderr, "%m - Failed to open \"%s\"", file);
		return -1;
	}

	if (setns(ns_fd, 0) < 0) {
		fprintf(stderr, "%m - Failed to attach to namespace \"%s\"", file);
		return -1;
	}

	return 0;
}

void forkdonetdetach(char *file) {
	if (dosetns_file(file, "net") < 0) {
		fprintf(stderr, "Failed setns to container network namespace: %s\n", strerror(errno));
		_exit(1);
	}

	// Jump back to Go for the rest
}

void forknet(void)
{
	char *command = NULL;
	char *cur = NULL;
	pid_t pid = 0;

	// Get the subcommand
	command = advance_arg(false);
	if (command == NULL || (strcmp(command, "--help") == 0 || strcmp(command, "--version") == 0 || strcmp(command, "-h") == 0)) {
		return;
	}

	// skip "--"
	advance_arg(true);

	// Get the pid
	cur = advance_arg(false);
	if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) {
		return;
	}

	// Check that we're root
	if (geteuid() != 0) {
		fprintf(stderr, "Error: forknet requires root privileges\n");
		_exit(1);
	}

	// Call the subcommands
	if (strcmp(command, "info") == 0) {
		int ns_fd, pidfd;
		pid = atoi(cur);

		pidfd = atoi(advance_arg(true));
		ns_fd = pidfd_nsfd(pidfd, pid);
		if (ns_fd < 0)
			_exit(1);

		forkdonetinfo(pidfd, ns_fd);
	}

	if (strcmp(command, "detach") == 0)
		forkdonetdetach(cur);
}
*/
import "C"

type cmdForknet struct {
	global *cmdGlobal
}

func (c *cmdForknet) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forknet"
	cmd.Short = "Perform container network operations"
	cmd.Long = `Description:
  Perform container network operations

  This set of internal commands are used for some container network
  operations which require attaching to the container's network namespace.
`
	cmd.Hidden = true

	// pull
	cmdInfo := &cobra.Command{}
	cmdInfo.Use = "info <PID> <PidFd>"
	cmdInfo.Args = cobra.ExactArgs(2)
	cmdInfo.RunE = c.RunInfo
	cmd.AddCommand(cmdInfo)

	// detach
	cmdDetach := &cobra.Command{}
	cmdDetach.Use = "detach <netns file> <LXD PID> <ifname> <hostname>"
	cmdDetach.Args = cobra.ExactArgs(4)
	cmdDetach.RunE = c.RunDetach
	cmd.AddCommand(cmdDetach)

	return cmd
}

func (c *cmdForknet) RunInfo(cmd *cobra.Command, args []string) error {
	networks, err := netutils.NetnsGetifaddrs(-1)
	if err != nil {
		return err
	}

	buf, err := json.Marshal(networks)
	if err != nil {
		return err
	}

	fmt.Printf("%s\n", buf)

	return nil
}

func (c *cmdForknet) RunDetach(cmd *cobra.Command, args []string) error {
	lxdPID := args[1]
	ifName := args[2]
	hostName := args[3]

	if lxdPID == "" {
		return fmt.Errorf("LXD PID argument is required")
	}

	if ifName == "" {
		return fmt.Errorf("ifname argument is required")
	}

	if hostName == "" {
		return fmt.Errorf("hostname argument is required")
	}

	// Remove all IP addresses from interface before moving to parent netns.
	// This is to avoid any container address config leaking into host.
	addr := &ip.Addr{
		DevName: ifName,
	}
	err := addr.Flush()
	if err != nil {
		return err
	}

	// Rename the interface, set it down, and move into parent netns.
	link := &ip.Link{Name: ifName}
	err = link.SetDown()
	if err != nil {
		return err
	}

	err = link.SetName(hostName)
	if err != nil {
		return err
	}

	link = &ip.Link{Name: hostName}
	err = link.SetNetns(lxdPID)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package homedir

import (
	"os"
	"runtime"
)

// HomeDir returns the home directory for the current user
func HomeDir() string {
	if runtime.GOOS == "windows" {

		// First prefer the HOME environmental variable
		if home := os.Getenv("HOME"); len(home) > 0 {
			if _, err := os.Stat(home); err == nil {
				return home
			}
		}
		if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 {
			homeDir := homeDrive + homePath
			if _, err := os.Stat(homeDir); err == nil {
				return homeDir
			}
		}
		if userProfile := os.Getenv("USERPROFILE"); len(userProfile) > 0 {
			if _, err := os.Stat(userProfile); err == nil {
				return userProfile
			}
		}
	}
	return os.Getenv("HOME")
}
<commit_msg>Improve windows home directory selection<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package homedir

import (
	"os"
	"path/filepath"
	"runtime"
)

// HomeDir returns the home directory for the current user.
// On Windows:
// 1. the first of %HOME%, %HOMEDRIVE%%HOMEPATH%, %USERPROFILE% containing a `.kube\config` file is returned.
// 2. if none of those locations contain a `.kube\config` file, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% that exists and is writeable is returned.
// 3. if none of those locations are writeable, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% that exists is returned.
// 4. if none of those locations exists, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% that is set is returned.
func HomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOME")
		homeDriveHomePath := ""
		if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 {
			homeDriveHomePath = homeDrive + homePath
		}
		userProfile := os.Getenv("USERPROFILE")

		// Return first of %HOME%, %HOMEDRIVE%/%HOMEPATH%, %USERPROFILE% that contains a `.kube\config` file.
		// %HOMEDRIVE%/%HOMEPATH% is preferred over %USERPROFILE% for backwards-compatibility.
		for _, p := range []string{home, homeDriveHomePath, userProfile} {
			if len(p) == 0 {
				continue
			}
			if _, err := os.Stat(filepath.Join(p, ".kube", "config")); err != nil {
				continue
			}
			return p
		}

		firstSetPath := ""
		firstExistingPath := ""

		// Prefer %USERPROFILE% over %HOMEDRIVE%/%HOMEPATH% for compatibility with other auth-writing tools
		for _, p := range []string{home, userProfile, homeDriveHomePath} {
			if len(p) == 0 {
				continue
			}
			if len(firstSetPath) == 0 {
				// remember the first path that is set
				firstSetPath = p
			}
			info, err := os.Stat(p)
			if err != nil {
				continue
			}
			if len(firstExistingPath) == 0 {
				// remember the first path that exists
				firstExistingPath = p
			}
			if info.IsDir() && info.Mode().Perm()&(1<<(uint(7))) != 0 {
				// return first path that is writeable
				return p
			}
		}

		// If none are writeable, return first location that exists
		if len(firstExistingPath) > 0 {
			return firstExistingPath
		}

		// If none exist, return first location that is set
		if len(firstSetPath) > 0 {
			return firstSetPath
		}

		// We've got nothing
		return ""
	}
	return os.Getenv("HOME")
}
<|endoftext|>


<commit_before>package lzma

import (
	"io"

	"github.com/uli-go/xz/xlog"
)

const maxConsecutiveEmptyReads = 100

type bWriter struct {
	io.Writer
	a []byte
}

func newByteWriter(w io.Writer) io.ByteWriter {
	if b, ok := w.(io.ByteWriter); ok {
		return b
	}
	return &bWriter{w, make([]byte, 1)}
}

func (b *bWriter) WriteByte(c byte) error {
	b.a[0] = c
	n, err := b.Write(b.a)
	switch {
	case n > 1:
		panic("n > 1 for writing a single byte")
	case n == 1:
		return nil
	case err == nil:
		panic("no error for n == 0")
	}
	return err
}

type bReader struct {
	io.Reader
	a []byte
}

func newByteReader(r io.Reader) io.ByteReader {
	if b, ok := r.(io.ByteReader); ok {
		return b
	}
	return &bReader{r, make([]byte, 1)}
}

func (b bReader) ReadByte() (byte, error) {
	n, err := b.Read(b.a)
	switch {
	case n > 1:
		panic("n < 1 for reading a single byte")
	case n == 1:
		return b.a[0], nil
	}
	return 0, err
}

// rangeEncoder implements range encoding of single bits. The low value can
// overflow therefore we need uint64. The cache value is used to handle
// overflows.
type rangeEncoder struct {
	w         io.ByteWriter
	range_    uint32
	low       uint64
	cacheSize int64
	cache     byte
	// for debugging
	bitCounter int
}

// newRangeEncoder creates a new range encoder.
func newRangeEncoder(w io.Writer) *rangeEncoder {
	return &rangeEncoder{
		w:         newByteWriter(w),
		range_:    0xffffffff,
		cacheSize: 1}
}

var encBitCounter int

// DirectEncodeBit encodes the least-significant bit of b with probability 1/2.
func (e *rangeEncoder) DirectEncodeBit(b uint32) error {
	e.bitCounter++
	e.range_ >>= 1
	e.low += uint64(e.range_) & (0 - (uint64(b) & 1))
	if err := e.normalize(); err != nil {
		return err
	}

	xlog.Printf(debug, "D %3d %0x08x %d\n", e.bitCounter, e.range_, b)
	return nil
}

// EncodeBit encodes the least significant bit of b. The p value will be
// updated by the function depending on the bit encoded.
func (e *rangeEncoder) EncodeBit(b uint32, p *prob) error {
	e.bitCounter++
	bound := p.bound(e.range_)
	if b&1 == 0 {
		e.range_ = bound
		p.inc()
	} else {
		e.low += uint64(bound)
		e.range_ -= bound
		p.dec()
	}
	if err := e.normalize(); err != nil {
		return err
	}

	xlog.Printf(debug, "B %3d 0x%08x 0x%03x %d\n", e.bitCounter, e.range_,
		*p, b)
	return nil
}

// Flush writes a complete copy of the low value.
func (e *rangeEncoder) Close() error {
	for i := 0; i < 5; i++ {
		if err := e.shiftLow(); err != nil {
			return err
		}
	}
	return nil
}

// newRangeDecoder initializes a range decoder. It reads five bytes from the
// reader and therefore may return an error.
func newRangeDecoder(r io.Reader) (d *rangeDecoder, err error) {
	d = &rangeDecoder{r: newByteReader(r)}
	err = d.init()
	return
}

// possiblyAtEnd checks whether the decoder may be at the end of the stream.
func (d *rangeDecoder) possiblyAtEnd() bool {
	return d.code == 0
}

var bitCounter int

// DirectDecodeBit decodes a bit with probability 1/2. The return value b will
// contain the bit at the least-significant position. All other bits will be
// zero.
func (d *rangeDecoder) DirectDecodeBit() (b uint32, err error) {
	d.bitCounter++
	d.range_ >>= 1
	d.code -= d.range_
	t := 0 - (d.code >> 31)
	d.code += d.range_ & t

	// d.code will stay less then d.range_

	if err = d.normalize(); err != nil {
		return 0, err
	}

	b = (t + 1) & 1

	xlog.Printf(debug, "D %3d 0x%08x %d\n", d.bitCounter, d.range_, b)
	return b, nil
}

// decodeBit decodes a single bit. The bit will be returned at the
// least-significant position. All other bits will be zero. The probability
// value will be updated.
func (d *rangeDecoder) DecodeBit(p *prob) (b uint32, err error) {
	d.bitCounter++
	bound := p.bound(d.range_)
	if d.code < bound {
		d.range_ = bound
		p.inc()
		b = 0
	} else {
		d.code -= bound
		d.range_ -= bound
		p.dec()
		b = 1
	}

	// d.code will stay less then d.range_

	if err = d.normalize(); err != nil {
		return 0, err
	}

	xlog.Printf(debug, "B %3d 0x%08x 0x%03x %d\n", d.bitCounter, d.range_,
		*p, b)
	return b, nil
}

// shiftLow() shifts the low value for 8 bit. The shifted byte is written into
// the byte writer. The cache value is used to handle overflows.
func (e *rangeEncoder) shiftLow() error {
	if uint32(e.low) < 0xff000000 || (e.low>>32) != 0 {
		tmp := e.cache
		for {
			err := e.w.WriteByte(tmp + byte(e.low>>32))
			if err != nil {
				return err
			}
			tmp = 0xff
			e.cacheSize--
			if e.cacheSize <= 0 {
				if e.cacheSize < 0 {
					return newError("negative e.cacheSize")
				}
				break
			}
		}
		e.cache = byte(uint32(e.low) >> 24)
	}
	e.cacheSize++
	e.low = uint64(uint32(e.low) << 8)
	return nil
}

// normalize handles shifts of range_ and low.
func (e *rangeEncoder) normalize() error {
	const top = 1 << 24
	if e.range_ >= top {
		return nil
	}
	e.range_ <<= 8
	return e.shiftLow()
}

// rangeDecoder decodes single bits of the range encoding stream.
type rangeDecoder struct {
	r      io.ByteReader
	range_ uint32
	code   uint32
	// for debugging
	bitCounter int
}

// init initializes the range decoder, by reading from the byte reader.
func (d *rangeDecoder) init() error {
	d.range_ = 0xffffffff
	d.code = 0

	b, err := d.r.ReadByte()
	if err != nil {
		return err
	}
	if b != 0 {
		return newError("first byte not zero")
	}

	for i := 0; i < 4; i++ {
		if err = d.updateCode(); err != nil {
			return err
		}
	}

	if d.code >= d.range_ {
		return newError("newRangeDecoder: d.code >= d.range_")
	}

	return nil
}

// updateCode reads a new byte into the code.
func (d *rangeDecoder) updateCode() error {
	b, err := d.r.ReadByte()
	if err != nil {
		return err
	}
	d.code = (d.code << 8) | uint32(b)
	return nil
}

// normalize the top value and update the code value.
func (d *rangeDecoder) normalize() error {
	// assume d.code < d.range_
	const top = 1 << 24
	if d.range_ < top {
		d.range_ <<= 8
		// d.code < d.range_ will be maintained
		if err := d.updateCode(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>lzma: the constant maxConsecutiveEmptyReads is not required<commit_after>package lzma

import (
	"io"

	"github.com/uli-go/xz/xlog"
)

type bWriter struct {
	io.Writer
	a []byte
}

func newByteWriter(w io.Writer) io.ByteWriter {
	if b, ok := w.(io.ByteWriter); ok {
		return b
	}
	return &bWriter{w, make([]byte, 1)}
}

func (b *bWriter) WriteByte(c byte) error {
	b.a[0] = c
	n, err := b.Write(b.a)
	switch {
	case n > 1:
		panic("n > 1 for writing a single byte")
	case n == 1:
		return nil
	case err == nil:
		panic("no error for n == 0")
	}
	return err
}

type bReader struct {
	io.Reader
	a []byte
}

func newByteReader(r io.Reader) io.ByteReader {
	if b, ok := r.(io.ByteReader); ok {
		return b
	}
	return &bReader{r, make([]byte, 1)}
}

func (b bReader) ReadByte() (byte, error) {
	n, err := b.Read(b.a)
	switch {
	case n > 1:
		panic("n < 1 for reading a single byte")
	case n == 1:
		return b.a[0], nil
	}
	return 0, err
}

// rangeEncoder implements range encoding of single bits. The low value can
// overflow therefore we need uint64. The cache value is used to handle
// overflows.
type rangeEncoder struct {
	w         io.ByteWriter
	range_    uint32
	low       uint64
	cacheSize int64
	cache     byte
	// for debugging
	bitCounter int
}

// newRangeEncoder creates a new range encoder.
func newRangeEncoder(w io.Writer) *rangeEncoder {
	return &rangeEncoder{
		w:         newByteWriter(w),
		range_:    0xffffffff,
		cacheSize: 1}
}

var encBitCounter int

// DirectEncodeBit encodes the least-significant bit of b with probability 1/2.
func (e *rangeEncoder) DirectEncodeBit(b uint32) error {
	e.bitCounter++
	e.range_ >>= 1
	e.low += uint64(e.range_) & (0 - (uint64(b) & 1))
	if err := e.normalize(); err != nil {
		return err
	}

	xlog.Printf(debug, "D %3d %0x08x %d\n", e.bitCounter, e.range_, b)
	return nil
}

// EncodeBit encodes the least significant bit of b. The p value will be
// updated by the function depending on the bit encoded.
func (e *rangeEncoder) EncodeBit(b uint32, p *prob) error {
	e.bitCounter++
	bound := p.bound(e.range_)
	if b&1 == 0 {
		e.range_ = bound
		p.inc()
	} else {
		e.low += uint64(bound)
		e.range_ -= bound
		p.dec()
	}
	if err := e.normalize(); err != nil {
		return err
	}

	xlog.Printf(debug, "B %3d 0x%08x 0x%03x %d\n", e.bitCounter, e.range_,
		*p, b)
	return nil
}

// Flush writes a complete copy of the low value.
func (e *rangeEncoder) Close() error {
	for i := 0; i < 5; i++ {
		if err := e.shiftLow(); err != nil {
			return err
		}
	}
	return nil
}

// newRangeDecoder initializes a range decoder. It reads five bytes from the
// reader and therefore may return an error.
func newRangeDecoder(r io.Reader) (d *rangeDecoder, err error) {
	d = &rangeDecoder{r: newByteReader(r)}
	err = d.init()
	return
}

// possiblyAtEnd checks whether the decoder may be at the end of the stream.
func (d *rangeDecoder) possiblyAtEnd() bool {
	return d.code == 0
}

var bitCounter int

// DirectDecodeBit decodes a bit with probability 1/2. The return value b will
// contain the bit at the least-significant position. All other bits will be
// zero.
func (d *rangeDecoder) DirectDecodeBit() (b uint32, err error) {
	d.bitCounter++
	d.range_ >>= 1
	d.code -= d.range_
	t := 0 - (d.code >> 31)
	d.code += d.range_ & t

	// d.code will stay less then d.range_

	if err = d.normalize(); err != nil {
		return 0, err
	}

	b = (t + 1) & 1

	xlog.Printf(debug, "D %3d 0x%08x %d\n", d.bitCounter, d.range_, b)
	return b, nil
}

// decodeBit decodes a single bit. The bit will be returned at the
// least-significant position. All other bits will be zero. The probability
// value will be updated.
func (d *rangeDecoder) DecodeBit(p *prob) (b uint32, err error) {
	d.bitCounter++
	bound := p.bound(d.range_)
	if d.code < bound {
		d.range_ = bound
		p.inc()
		b = 0
	} else {
		d.code -= bound
		d.range_ -= bound
		p.dec()
		b = 1
	}

	// d.code will stay less then d.range_

	if err = d.normalize(); err != nil {
		return 0, err
	}

	xlog.Printf(debug, "B %3d 0x%08x 0x%03x %d\n", d.bitCounter, d.range_,
		*p, b)
	return b, nil
}

// shiftLow() shifts the low value for 8 bit. The shifted byte is written into
// the byte writer. The cache value is used to handle overflows.
func (e *rangeEncoder) shiftLow() error {
	if uint32(e.low) < 0xff000000 || (e.low>>32) != 0 {
		tmp := e.cache
		for {
			err := e.w.WriteByte(tmp + byte(e.low>>32))
			if err != nil {
				return err
			}
			tmp = 0xff
			e.cacheSize--
			if e.cacheSize <= 0 {
				if e.cacheSize < 0 {
					return newError("negative e.cacheSize")
				}
				break
			}
		}
		e.cache = byte(uint32(e.low) >> 24)
	}
	e.cacheSize++
	e.low = uint64(uint32(e.low) << 8)
	return nil
}

// normalize handles shifts of range_ and low.
func (e *rangeEncoder) normalize() error {
	const top = 1 << 24
	if e.range_ >= top {
		return nil
	}
	e.range_ <<= 8
	return e.shiftLow()
}

// rangeDecoder decodes single bits of the range encoding stream.
type rangeDecoder struct {
	r      io.ByteReader
	range_ uint32
	code   uint32
	// for debugging
	bitCounter int
}

// init initializes the range decoder, by reading from the byte reader.
func (d *rangeDecoder) init() error {
	d.range_ = 0xffffffff
	d.code = 0

	b, err := d.r.ReadByte()
	if err != nil {
		return err
	}
	if b != 0 {
		return newError("first byte not zero")
	}

	for i := 0; i < 4; i++ {
		if err = d.updateCode(); err != nil {
			return err
		}
	}

	if d.code >= d.range_ {
		return newError("newRangeDecoder: d.code >= d.range_")
	}

	return nil
}

// updateCode reads a new byte into the code.
func (d *rangeDecoder) updateCode() error {
	b, err := d.r.ReadByte()
	if err != nil {
		return err
	}
	d.code = (d.code << 8) | uint32(b)
	return nil
}

// normalize the top value and update the code value.
func (d *rangeDecoder) normalize() error {
	// assume d.code < d.range_
	const top = 1 << 24
	if d.range_ < top {
		d.range_ <<= 8
		// d.code < d.range_ will be maintained
		if err := d.updateCode(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package httputil

import (
	"net/http"
)

// writer.go implements a http.ResponseWriter so it can be used to generate http access log
// it is copied from go.ice https://github.com/dyweb/go.ice/blob/v0.0.1/ice/transport/http/writer.go

var (
	_ http.ResponseWriter = (*TrackedWriter)(nil)
	_ http.Flusher        = (*TrackedWriter)(nil)
	_ http.Pusher         = (*TrackedWriter)(nil)
)

// NOTE: CloseNotifier is deprecated in favor of context
//var _ http.CloseNotifier = (*TrackedWriter)(nil)

// TrackedWriter keeps track of status code and bytes written so it can be used by logger.
// It proxies all the interfaces except Hijacker, since it is not supported by HTTP/2.
// Most methods comments are copied from net/http
// It is based on https://github.com/gorilla/handlers but put all interface into one struct
type TrackedWriter struct {
	w           http.ResponseWriter
	status      int
	size        int
	writeCalled int
}

// NewTrackedWriter set the underlying writer based on argument,
// It returns a value instead of pointer so it can be allocated on stack.
// TODO: add benchmark to prove it ...
func NewTrackedWriter(w http.ResponseWriter) TrackedWriter {
	return TrackedWriter{w: w, status: 200}
}

// Status return the tracked status code, returns 0 if WriteHeader has not been called
func (tracker *TrackedWriter) Status() int {
	return tracker.status
}

// Size return number of bytes written through Write, returns 0 if Write has not been called
func (tracker *TrackedWriter) Size() int {
	return tracker.size
}

// Header returns the header map of the underlying ResponseWriter
//
// Changing the header map after a call to WriteHeader (or
// Write) has no effect unless the modified headers are
// trailers.
func (tracker *TrackedWriter) Header() http.Header {
	return tracker.w.Header()
}

// Write keeps track of bytes written of the underlying ResponseWriter
//
// Write writes the data to the connection as part of an HTTP reply.
//
// If WriteHeader has not yet been called, Write calls
// WriteHeader(http.StatusOK) before writing the data. If the Header
// does not contain a Content-Type line, Write adds a Content-Type set
// to the result of passing the initial 512 bytes of written data to
// DetectContentType.
func (tracker *TrackedWriter) Write(b []byte) (int, error) {
	tracker.writeCalled++
	size, err := tracker.w.Write(b)
	tracker.size += size
	return size, err
}

// WriteHeader keep track of status code and call the underlying ResponseWriter
//
// WriteHeader sends an HTTP response header with status code.
// If WriteHeader is not called explicitly, the first call to Write
// will trigger an implicit WriteHeader(http.StatusOK).
// Thus explicit calls to WriteHeader are mainly used to
// send error codes.
func (tracker *TrackedWriter) WriteHeader(status int) {
	tracker.status = status
	tracker.w.WriteHeader(status)
}

// Flush calls Flush on underlying ResponseWriter if it implemented http.Flusher
//
// Flusher interface is implemented by ResponseWriters that allow
// an HTTP handler to flush buffered data to the client.
// The default HTTP/1.x and HTTP/2 ResponseWriter implementations
// support Flusher
func (tracker *TrackedWriter) Flush() {
	if f, ok := tracker.w.(http.Flusher); ok {
		f.Flush()
	}
}

// Push returns http.ErrNotSupported if underlying ResponseWriter does not implement http.Pusher
//
// Push initiates an HTTP/2 server push, returns ErrNotSupported if the client has disabled push or if push
// is not supported on the underlying connection.
func (tracker *TrackedWriter) Push(target string, opts *http.PushOptions) error {
	if p, ok := tracker.w.(http.Pusher); ok {
		return p.Push(target, opts)
	}
	return http.ErrNotSupported
}
<commit_msg>[httputil] Supprt http.Hijacker interface in writer<commit_after>package httputil

import (
	"bufio"
	"net"
	"net/http"
)

// writer.go implements a http.ResponseWriter so it can be used to generate http access log
// it is copied from go.ice https://github.com/dyweb/go.ice/blob/v0.0.1/ice/transport/http/writer.go

var (
	_ http.ResponseWriter = (*TrackedWriter)(nil)
	_ http.Flusher        = (*TrackedWriter)(nil)
	_ http.Pusher         = (*TrackedWriter)(nil)
	_ http.Hijacker       = (*TrackedWriter)(nil)
)

// NOTE: CloseNotifier is deprecated in favor of context
//var _ http.CloseNotifier = (*TrackedWriter)(nil)

// TrackedWriter keeps track of status code and bytes written so it can be used by logger.
// It proxies all the interfaces except Hijacker, since it is not supported by HTTP/2.
// Most methods comments are copied from net/http
// It is based on https://github.com/gorilla/handlers but put all interface into one struct
type TrackedWriter struct {
	w           http.ResponseWriter
	status      int
	size        int
	writeCalled int
}

// NewTrackedWriter set the underlying writer based on argument,
// It returns a value instead of pointer so it can be allocated on stack.
// TODO: add benchmark to prove it ...
func NewTrackedWriter(w http.ResponseWriter) TrackedWriter {
	return TrackedWriter{w: w, status: 200}
}

// Status return the tracked status code, returns 0 if WriteHeader has not been called
func (tracker *TrackedWriter) Status() int {
	return tracker.status
}

// Size return number of bytes written through Write, returns 0 if Write has not been called
func (tracker *TrackedWriter) Size() int {
	return tracker.size
}

// Header returns the header map of the underlying ResponseWriter
//
// Changing the header map after a call to WriteHeader (or
// Write) has no effect unless the modified headers are
// trailers.
func (tracker *TrackedWriter) Header() http.Header {
	return tracker.w.Header()
}

// Write keeps track of bytes written of the underlying ResponseWriter
//
// Write writes the data to the connection as part of an HTTP reply.
//
// If WriteHeader has not yet been called, Write calls
// WriteHeader(http.StatusOK) before writing the data. If the Header
// does not contain a Content-Type line, Write adds a Content-Type set
// to the result of passing the initial 512 bytes of written data to
// DetectContentType.
func (tracker *TrackedWriter) Write(b []byte) (int, error) {
	tracker.writeCalled++
	size, err := tracker.w.Write(b)
	tracker.size += size
	return size, err
}

// WriteHeader keep track of status code and call the underlying ResponseWriter
//
// WriteHeader sends an HTTP response header with status code.
// If WriteHeader is not called explicitly, the first call to Write
// will trigger an implicit WriteHeader(http.StatusOK).
// Thus explicit calls to WriteHeader are mainly used to
// send error codes.
func (tracker *TrackedWriter) WriteHeader(status int) {
	tracker.status = status
	tracker.w.WriteHeader(status)
}

// Flush calls Flush on underlying ResponseWriter if it implemented http.Flusher
//
// Flusher interface is implemented by ResponseWriters that allow
// an HTTP handler to flush buffered data to the client.
// The default HTTP/1.x and HTTP/2 ResponseWriter implementations
// support Flusher
func (tracker *TrackedWriter) Flush() {
	if f, ok := tracker.w.(http.Flusher); ok {
		f.Flush()
	}
}

// Push returns http.ErrNotSupported if underlying ResponseWriter does not implement http.Pusher
//
// Push initiates an HTTP/2 server push, returns ErrNotSupported if the client has disabled push or if push
// is not supported on the underlying connection.
func (tracker *TrackedWriter) Push(target string, opts *http.PushOptions) error {
	if p, ok := tracker.w.(http.Pusher); ok {
		return p.Push(target, opts)
	}
	return http.ErrNotSupported
}

// Hijack implements http.Hijacker, which is used by websocket etc.
// It returns http.ErrNotSupported with nil pointer if the underlying writer does not support it
// NOTE: HTTP/1.x supports it but HTTP/2 does not
func (tracker *TrackedWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	if h, ok := tracker.w.(http.Hijacker); ok {
		return h.Hijack()
	}
	return nil, nil, http.ErrNotSupported
}
<|endoftext|>


<commit_before>// Copyright 2014 The roc Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rocserv

import (
	"flag"
	"fmt"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/gin-gonic/gin"
	"github.com/julienschmidt/httprouter"
	"github.com/shawnfeng/sutil/slog"
	"github.com/shawnfeng/sutil/slog/statlog"
	"github.com/shawnfeng/sutil/smetric"
	"github.com/shawnfeng/sutil/trace"
	"reflect"
	"sync"
)

const (
	PROCESSOR_HTTP   = "http"
	PROCESSOR_THRIFT = "thrift"
	PROCESSOR_GRPC   = "gprc"
	PROCESSOR_GIN    = "gin"
)

var service = NewService()

type Service struct {
	sbase ServBase

	mutex   sync.Mutex
	servers map[string]interface{}
}

func NewService() *Service {
	return &Service{
		servers: make(map[string]interface{}),
	}
}

type cmdArgs struct {
	logMaxSize    int
	logMaxBackups int
	servLoc       string
	logDir        string
	sessKey       string
	group         string
}

func (m *Service) parseFlag() (*cmdArgs, error) {
	var serv, logDir, skey, group string
	var logMaxSize, logMaxBackups int
	flag.IntVar(&logMaxSize, "logmaxsize", 0, "logMaxSize is the maximum size in megabytes of the log file")
	flag.IntVar(&logMaxBackups, "logmaxbackups", 0, "logmaxbackups is the maximum number of old log files to retain")
	flag.StringVar(&serv, "serv", "", "servic name")
	flag.StringVar(&logDir, "logdir", "", "serice log dir")
	flag.StringVar(&skey, "skey", "", "service session key")
	flag.StringVar(&group, "group", "", "service group")

	flag.Parse()

	if len(serv) == 0 {
		return nil, fmt.Errorf("serv args need!")
	}

	if len(skey) == 0 {
		return nil, fmt.Errorf("skey args need!")
	}

	return &cmdArgs{
		logMaxSize:    logMaxSize,
		logMaxBackups: logMaxBackups,
		servLoc:       serv,
		logDir:        logDir,
		sessKey:       skey,
		group:         group,
	}, nil

}

func (m *Service) loadDriver(sb ServBase, procs map[string]Processor) (map[string]*ServInfo, error) {
	fun := "Service.loadDriver -->"

	infos := make(map[string]*ServInfo)

	for n, p := range procs {
		addr, driver := p.Driver()
		if driver == nil {
			slog.Infof("%s processor:%s no driver", fun, n)
			continue
		}

		slog.Infof("%s processor:%s type:%s addr:%s", fun, n, reflect.TypeOf(driver), addr)

		switch d := driver.(type) {
		case *httprouter.Router:
			sa, err := powerHttp(addr, d)
			if err != nil {
				return nil, err
			}

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_HTTP,
				Addr: sa,
			}

		case thrift.TProcessor:
			sa, err := powerThrift(addr, d)
			if err != nil {
				return nil, err
			}

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_THRIFT,
				Addr: sa,
			}
		case *GrpcServer:
			sa, err := powerGrpc(addr, d)
			if err != nil {
				return nil, err
			}

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_GRPC,
				Addr: sa,
			}
		case *gin.Engine:
			sa, serv, err := powerGin(addr, d)
			if err != nil {
				return nil, err
			}

			m.addServer(n, serv)

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_GIN,
				Addr: sa,
			}
		default:
			return nil, fmt.Errorf("processor:%s driver not recognition", n)

		}
	}

	return infos, nil
}

func (m *Service) addServer(processor string, server interface{}) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	m.servers[processor] = server
}

func (m *Service) reloadRouter(processor string, driver interface{}) error {
	//fun := "Service.reloadRouter -->"

	m.mutex.Lock()
	defer m.mutex.Unlock()
	server, ok := m.servers[processor]
	if !ok {
		return fmt.Errorf("processor:%s driver not recognition", processor)
	}

	return reloadRouter(processor, server, driver)
}

func (m *Service) Serve(confEtcd configEtcd, initfn func(ServBase) error, procs map[string]Processor) error {
	fun := "Service.Serve -->"

	args, err := m.parseFlag()
	if err != nil {
		slog.Panicf("%s parse arg err:%s", fun, err)
		return err
	}

	return m.Init(confEtcd, args, initfn, procs)
}

func (m *Service) initLog(sb *ServBaseV2, args *cmdArgs) error {
	fun := "Service.initLog -->"

	logDir := args.logDir
	var logConfig struct {
		Log struct {
			Level string
			Dir   string
		}
	}
	logConfig.Log.Level = "INFO"

	err := sb.ServConfig(&logConfig)
	if err != nil {
		slog.Errorf("%s serv config err:%s", fun, err)
		return err
	}

	var logdir string
	if len(logConfig.Log.Dir) > 0 {
		logdir = fmt.Sprintf("%s/%s", logConfig.Log.Dir, sb.Copyname())
	}

	if len(logDir) > 0 {
		logdir = fmt.Sprintf("%s/%s", logDir, sb.Copyname())
	}

	if logDir == "console" {
		logdir = ""
	}

	slog.Infof("%s init log dir:%s name:%s level:%s", fun, logdir, args.servLoc, logConfig.Log.Level)

	slog.Init(logdir, "serv.log", logConfig.Log.Level)
	statlog.Init(logdir, "stat.log", args.servLoc)
	return nil
}

func (m *Service) Init(confEtcd configEtcd, args *cmdArgs, initfn func(ServBase) error, procs map[string]Processor) error {
	fun := "Service.Init -->"

	servLoc := args.servLoc
	sessKey := args.sessKey

	sb, err := NewServBaseV2(confEtcd, servLoc, sessKey)
	if err != nil {
		slog.Panicf("%s init servbase loc:%s key:%s err:%s", fun, servLoc, sessKey, err)
		return err
	}

	m.initLog(sb, args)
	defer slog.Sync()
	defer statlog.Sync()

	err = initfn(sb)
	if err != nil {
		slog.Panicf("%s callInitFunc err:%s", fun, err)
		return err
	}

	err = m.initProcessor(sb, procs)
	if err != nil {
		slog.Panicf("%s initProcessor err:%s", fun, err)
		return err
	}

	sb.SetGroup(args.group)

	m.initTracer(servLoc)
	m.initBackdoork(sb)
	m.initMetric(sb)

	var pause chan bool
	pause <- true

	return nil
}

func (m *Service) initProcessor(sb *ServBaseV2, procs map[string]Processor) error {
	fun := "Service.initProcessor -->"

	for n, p := range procs {
		if len(n) == 0 {
			slog.Errorf("%s processor name empty", fun)
			return fmt.Errorf("processor name empty")
		}

		if n[0] == '_' {
			slog.Errorf("%s processor name can not prefix '_'", fun)
			return fmt.Errorf("processor name can not prefix '_'")
		}

		if p == nil {
			slog.Errorf("%s processor:%s is nil", fun, n)
			return fmt.Errorf("processor:%s is nil", n)
		} else {
			err := p.Init()
			if err != nil {
				slog.Errorf("%s processor:%s init err:%s", fun, err)
				return fmt.Errorf("processor:%s init err:%s", n, err)
			}
		}
	}

	infos, err := m.loadDriver(sb, procs)
	if err != nil {
		slog.Errorf("%s load driver err:%s", fun, err)
		return err
	}

	err = sb.RegisterService(infos)
	if err != nil {
		slog.Errorf("%s regist service err:%s", fun, err)
		return err
	}

	return nil
}

func (m *Service) initTracer(servLoc string) error {
	fun := "Service.initTracer -->"

	err := trace.InitDefaultTracer(servLoc)
	if err != nil {
		slog.Errorf("%s init tracer fail:%v", fun, err)
	}

	return err
}

func (m *Service) initBackdoork(sb *ServBaseV2) error {
	fun := "Service.initBackdoork -->"

	backdoor := &backDoorHttp{}
	err := backdoor.Init()
	if err != nil {
		slog.Errorf("%s init backdoor err:%s", fun, err)
		return err
	}

	binfos, err := m.loadDriver(sb, map[string]Processor{"_PROC_BACKDOOR": backdoor})
	if err == nil {
		err = sb.RegisterBackDoor(binfos)
		if err != nil {
			slog.Errorf("%s regist backdoor err:%s", fun, err)
		}

	} else {
		slog.Warnf("%s load backdoor driver err:%s", fun, err)
	}

	return err
}

func (m *Service) initMetric(sb *ServBaseV2) error {
	fun := "Service.initMetric -->"

	metrics := smetric.NewMetricsprocessor()
	err := metrics.Init()
	if err != nil {
		slog.Warnf("%s init metrics err:%s", fun, err)
	}

	minfos, err := m.loadDriver(sb, map[string]Processor{"_PROC_METRICS": metrics})
	if err == nil {
		err = sb.RegisterMetrics(minfos)
		if err != nil {
			slog.Warnf("%s regist backdoor err:%s", fun, err)
		}

	} else {
		slog.Warnf("%s load metrics driver err:%s", fun, err)
	}
	return err
}

func ReloadRouter(processor string, driver interface{}) error {
	return service.reloadRouter(processor, driver)
}

func Serve(etcds []string, baseLoc string, initfn func(ServBase) error, procs map[string]Processor) error {
	return service.Serve(configEtcd{etcds, baseLoc}, initfn, procs)
}

func Init(etcds []string, baseLoc string, servLoc, servKey, logDir string, initfn func(ServBase) error, procs map[string]Processor) error {
	args := &cmdArgs{
		logMaxSize:    0,
		logMaxBackups: 0,
		servLoc:       servLoc,
		logDir:        logDir,
		sessKey:       servKey,
	}
	return service.Init(configEtcd{etcds, baseLoc}, args, initfn, procs)
}

func GetServBase() ServBase {
	return service.sbase
}

func GetServName() (servName string) {
	if service.sbase != nil {
		servName = service.sbase.Servname()
	}
	return
}
func GetServId() (servId int) {
	if service.sbase != nil {
		servId = service.sbase.Servid()
	}
	return
}

func Test(etcds []string, baseLoc string, initfn func(ServBase) error) error {
	args := &cmdArgs{
		logMaxSize:    0,
		logMaxBackups: 0,
		servLoc:       "test/test",
		sessKey:       "test",
		logDir:        "console",
	}
	return service.Init(configEtcd{etcds, baseLoc}, args, initfn, nil)
}
<commit_msg>fix init tracer order<commit_after>// Copyright 2014 The roc Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rocserv

import (
	"flag"
	"fmt"
	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/gin-gonic/gin"
	"github.com/julienschmidt/httprouter"
	"github.com/shawnfeng/sutil/slog"
	"github.com/shawnfeng/sutil/slog/statlog"
	"github.com/shawnfeng/sutil/smetric"
	"github.com/shawnfeng/sutil/trace"
	"reflect"
	"sync"
)

const (
	PROCESSOR_HTTP   = "http"
	PROCESSOR_THRIFT = "thrift"
	PROCESSOR_GRPC   = "gprc"
	PROCESSOR_GIN    = "gin"
)

var service = NewService()

type Service struct {
	sbase ServBase

	mutex   sync.Mutex
	servers map[string]interface{}
}

func NewService() *Service {
	return &Service{
		servers: make(map[string]interface{}),
	}
}

type cmdArgs struct {
	logMaxSize    int
	logMaxBackups int
	servLoc       string
	logDir        string
	sessKey       string
	group         string
}

func (m *Service) parseFlag() (*cmdArgs, error) {
	var serv, logDir, skey, group string
	var logMaxSize, logMaxBackups int
	flag.IntVar(&logMaxSize, "logmaxsize", 0, "logMaxSize is the maximum size in megabytes of the log file")
	flag.IntVar(&logMaxBackups, "logmaxbackups", 0, "logmaxbackups is the maximum number of old log files to retain")
	flag.StringVar(&serv, "serv", "", "servic name")
	flag.StringVar(&logDir, "logdir", "", "serice log dir")
	flag.StringVar(&skey, "skey", "", "service session key")
	flag.StringVar(&group, "group", "", "service group")

	flag.Parse()

	if len(serv) == 0 {
		return nil, fmt.Errorf("serv args need!")
	}

	if len(skey) == 0 {
		return nil, fmt.Errorf("skey args need!")
	}

	return &cmdArgs{
		logMaxSize:    logMaxSize,
		logMaxBackups: logMaxBackups,
		servLoc:       serv,
		logDir:        logDir,
		sessKey:       skey,
		group:         group,
	}, nil

}

func (m *Service) loadDriver(sb ServBase, procs map[string]Processor) (map[string]*ServInfo, error) {
	fun := "Service.loadDriver -->"

	infos := make(map[string]*ServInfo)

	for n, p := range procs {
		addr, driver := p.Driver()
		if driver == nil {
			slog.Infof("%s processor:%s no driver", fun, n)
			continue
		}

		slog.Infof("%s processor:%s type:%s addr:%s", fun, n, reflect.TypeOf(driver), addr)

		switch d := driver.(type) {
		case *httprouter.Router:
			sa, err := powerHttp(addr, d)
			if err != nil {
				return nil, err
			}

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_HTTP,
				Addr: sa,
			}

		case thrift.TProcessor:
			sa, err := powerThrift(addr, d)
			if err != nil {
				return nil, err
			}

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_THRIFT,
				Addr: sa,
			}
		case *GrpcServer:
			sa, err := powerGrpc(addr, d)
			if err != nil {
				return nil, err
			}

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_GRPC,
				Addr: sa,
			}
		case *gin.Engine:
			sa, serv, err := powerGin(addr, d)
			if err != nil {
				return nil, err
			}

			m.addServer(n, serv)

			slog.Infof("%s load ok processor:%s serv addr:%s", fun, n, sa)
			infos[n] = &ServInfo{
				Type: PROCESSOR_GIN,
				Addr: sa,
			}
		default:
			return nil, fmt.Errorf("processor:%s driver not recognition", n)

		}
	}

	return infos, nil
}

func (m *Service) addServer(processor string, server interface{}) {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	m.servers[processor] = server
}

func (m *Service) reloadRouter(processor string, driver interface{}) error {
	//fun := "Service.reloadRouter -->"

	m.mutex.Lock()
	defer m.mutex.Unlock()
	server, ok := m.servers[processor]
	if !ok {
		return fmt.Errorf("processor:%s driver not recognition", processor)
	}

	return reloadRouter(processor, server, driver)
}

func (m *Service) Serve(confEtcd configEtcd, initfn func(ServBase) error, procs map[string]Processor) error {
	fun := "Service.Serve -->"

	args, err := m.parseFlag()
	if err != nil {
		slog.Panicf("%s parse arg err:%s", fun, err)
		return err
	}

	return m.Init(confEtcd, args, initfn, procs)
}

func (m *Service) initLog(sb *ServBaseV2, args *cmdArgs) error {
	fun := "Service.initLog -->"

	logDir := args.logDir
	var logConfig struct {
		Log struct {
			Level string
			Dir   string
		}
	}
	logConfig.Log.Level = "INFO"

	err := sb.ServConfig(&logConfig)
	if err != nil {
		slog.Errorf("%s serv config err:%s", fun, err)
		return err
	}

	var logdir string
	if len(logConfig.Log.Dir) > 0 {
		logdir = fmt.Sprintf("%s/%s", logConfig.Log.Dir, sb.Copyname())
	}

	if len(logDir) > 0 {
		logdir = fmt.Sprintf("%s/%s", logDir, sb.Copyname())
	}

	if logDir == "console" {
		logdir = ""
	}

	slog.Infof("%s init log dir:%s name:%s level:%s", fun, logdir, args.servLoc, logConfig.Log.Level)

	slog.Init(logdir, "serv.log", logConfig.Log.Level)
	statlog.Init(logdir, "stat.log", args.servLoc)
	return nil
}

func (m *Service) Init(confEtcd configEtcd, args *cmdArgs, initfn func(ServBase) error, procs map[string]Processor) error {
	fun := "Service.Init -->"

	servLoc := args.servLoc
	sessKey := args.sessKey

	sb, err := NewServBaseV2(confEtcd, servLoc, sessKey)
	if err != nil {
		slog.Panicf("%s init servbase loc:%s key:%s err:%s", fun, servLoc, sessKey, err)
		return err
	}

	m.initLog(sb, args)
	defer slog.Sync()
	defer statlog.Sync()

	err = initfn(sb)
	if err != nil {
		slog.Panicf("%s callInitFunc err:%s", fun, err)
		return err
	}

	// NOTE: processor 在初始化 trace middleware 前需要保证 opentracing.GlobalTracer() 初始化完毕
	m.initTracer(servLoc)

	err = m.initProcessor(sb, procs)
	if err != nil {
		slog.Panicf("%s initProcessor err:%s", fun, err)
		return err
	}

	sb.SetGroup(args.group)

	m.initBackdoork(sb)
	m.initMetric(sb)

	var pause chan bool
	pause <- true

	return nil
}

func (m *Service) initProcessor(sb *ServBaseV2, procs map[string]Processor) error {
	fun := "Service.initProcessor -->"

	for n, p := range procs {
		if len(n) == 0 {
			slog.Errorf("%s processor name empty", fun)
			return fmt.Errorf("processor name empty")
		}

		if n[0] == '_' {
			slog.Errorf("%s processor name can not prefix '_'", fun)
			return fmt.Errorf("processor name can not prefix '_'")
		}

		if p == nil {
			slog.Errorf("%s processor:%s is nil", fun, n)
			return fmt.Errorf("processor:%s is nil", n)
		} else {
			err := p.Init()
			if err != nil {
				slog.Errorf("%s processor:%s init err:%s", fun, err)
				return fmt.Errorf("processor:%s init err:%s", n, err)
			}
		}
	}

	infos, err := m.loadDriver(sb, procs)
	if err != nil {
		slog.Errorf("%s load driver err:%s", fun, err)
		return err
	}

	err = sb.RegisterService(infos)
	if err != nil {
		slog.Errorf("%s regist service err:%s", fun, err)
		return err
	}

	return nil
}

func (m *Service) initTracer(servLoc string) error {
	fun := "Service.initTracer -->"

	err := trace.InitDefaultTracer(servLoc)
	if err != nil {
		slog.Errorf("%s init tracer fail:%v", fun, err)
	}

	return err
}

func (m *Service) initBackdoork(sb *ServBaseV2) error {
	fun := "Service.initBackdoork -->"

	backdoor := &backDoorHttp{}
	err := backdoor.Init()
	if err != nil {
		slog.Errorf("%s init backdoor err:%s", fun, err)
		return err
	}

	binfos, err := m.loadDriver(sb, map[string]Processor{"_PROC_BACKDOOR": backdoor})
	if err == nil {
		err = sb.RegisterBackDoor(binfos)
		if err != nil {
			slog.Errorf("%s regist backdoor err:%s", fun, err)
		}

	} else {
		slog.Warnf("%s load backdoor driver err:%s", fun, err)
	}

	return err
}

func (m *Service) initMetric(sb *ServBaseV2) error {
	fun := "Service.initMetric -->"

	metrics := smetric.NewMetricsprocessor()
	err := metrics.Init()
	if err != nil {
		slog.Warnf("%s init metrics err:%s", fun, err)
	}

	minfos, err := m.loadDriver(sb, map[string]Processor{"_PROC_METRICS": metrics})
	if err == nil {
		err = sb.RegisterMetrics(minfos)
		if err != nil {
			slog.Warnf("%s regist backdoor err:%s", fun, err)
		}

	} else {
		slog.Warnf("%s load metrics driver err:%s", fun, err)
	}
	return err
}

func ReloadRouter(processor string, driver interface{}) error {
	return service.reloadRouter(processor, driver)
}

func Serve(etcds []string, baseLoc string, initfn func(ServBase) error, procs map[string]Processor) error {
	return service.Serve(configEtcd{etcds, baseLoc}, initfn, procs)
}

func Init(etcds []string, baseLoc string, servLoc, servKey, logDir string, initfn func(ServBase) error, procs map[string]Processor) error {
	args := &cmdArgs{
		logMaxSize:    0,
		logMaxBackups: 0,
		servLoc:       servLoc,
		logDir:        logDir,
		sessKey:       servKey,
	}
	return service.Init(configEtcd{etcds, baseLoc}, args, initfn, procs)
}

func GetServBase() ServBase {
	return service.sbase
}

func GetServName() (servName string) {
	if service.sbase != nil {
		servName = service.sbase.Servname()
	}
	return
}
func GetServId() (servId int) {
	if service.sbase != nil {
		servId = service.sbase.Servid()
	}
	return
}

func Test(etcds []string, baseLoc string, initfn func(ServBase) error) error {
	args := &cmdArgs{
		logMaxSize:    0,
		logMaxBackups: 0,
		servLoc:       "test/test",
		sessKey:       "test",
		logDir:        "console",
	}
	return service.Init(configEtcd{etcds, baseLoc}, args, initfn, nil)
}
<|endoftext|>


<commit_before>// Package urlutil contains utility functions
// related to URLs.
package urlutil

import (
	"net/url"
	"strings"
)

// isAbs returns true of the URL is absolute.
// This function considers protocol-relative
// URLs to be absolute
func isAbs(u *url.URL) bool {
	return u.IsAbs() || strings.HasPrefix(u.String(), "//")
}

// SameHost returns true iff both URLs point
// to the same host. It works for both absolute
// and relative URLs.
func SameHost(url1, url2 string) bool {
	if url1 == "" || url2 == "" {
		return true
	}
	u1, err := url.Parse(url1)
	if err != nil {
		return false
	}
	if !isAbs(u1) {
		return true
	}
	u2, err := url.Parse(url2)
	if err != nil {
		return false
	}
	if !isAbs(u2) {
		return true
	}
	return u1.Host == u2.Host
}

// Join returns the result of joining the base URL
// with the rel URL. If either base or rel are not
// valid URLs, an error will be returned.
func Join(base string, rel string) (string, error) {
	b, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	r, err := url.Parse(rel)
	if err != nil {
		return "", err
	}
	return b.ResolveReference(r).String(), nil
}
<commit_msg>Add IsURL<commit_after>// Package urlutil contains utility functions
// related to URLs.
package urlutil

import (
	"net/url"
	"strings"
)

// isAbs returns true of the URL is absolute.
// This function considers protocol-relative
// URLs to be absolute
func isAbs(u *url.URL) bool {
	return u.IsAbs() || strings.HasPrefix(u.String(), "//")
}

// SameHost returns true iff both URLs point
// to the same host. It works for both absolute
// and relative URLs.
func SameHost(url1, url2 string) bool {
	if url1 == "" || url2 == "" {
		return true
	}
	u1, err := url.Parse(url1)
	if err != nil {
		return false
	}
	if !isAbs(u1) {
		return true
	}
	u2, err := url.Parse(url2)
	if err != nil {
		return false
	}
	if !isAbs(u2) {
		return true
	}
	return u1.Host == u2.Host
}

// Join returns the result of joining the base URL
// with the rel URL. If either base or rel are not
// valid URLs, an error will be returned.
func Join(base string, rel string) (string, error) {
	b, err := url.Parse(base)
	if err != nil {
		return "", err
	}
	r, err := url.Parse(rel)
	if err != nil {
		return "", err
	}
	return b.ResolveReference(r).String(), nil
}

// IsURL returns true iff s looks like a URL.
func IsURL(s string) bool {
	return strings.Contains(s, "://") || strings.HasPrefix(s, "//")
}
<|endoftext|>