<commit_before>package just

import (
	"errors"
	"mime/multipart"
	"net/url"
	"reflect"
	"strconv"
	"time"
)

func marshalUrlValues(ptr interface{}) ([]byte, error) {
	t, v := reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem()
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return nil, errors.New("Array and slice by root element not supported, only structure!")
	}
	values := make(url.Values)
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		if structFieldKind == reflect.Slice {
			for i := 0; i < structField.Len(); i++ {
				values.Add(inputFieldName, structField.Index(i).String())
			}
		} else {
			values.Set(inputFieldName, structField.String())
		}
	}
	return []byte(values.Encode()), nil
}

// mapForm - form mapping (+ multipart files support)
func mapForm(values map[string][]string, files map[string][]*multipart.FileHeader, ptr interface{}) error {
	t, v := reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem()
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return errors.New("Array and slice by root element not supported, only structure!")
	}
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		inputValue, existsValue := values[inputFieldName]
		if !existsValue {
			// Check files
			if files != nil && len(files) > 0 {
				if list, ok := files[inputFieldName]; ok {
					if numFiles := len(list); numFiles > 0 {
						if structFieldKind == reflect.Slice {
							if structField.Type().Elem().Name() == "FileHeader" {
								slice := reflect.MakeSlice(structField.Type(), numFiles, numFiles)
								for i := 0; i < numFiles; i++ {
									slice.Index(i).Set(reflect.ValueOf(list[i]))
								}
								v.Field(i).Set(slice)
							}
						} else if typeField.Type.Name() == "FileHeader" {
							structField.Set(reflect.ValueOf(list[0]))
						}
					}
				}
			}
			continue
		}
		if numElements := len(inputValue); numElements > 0 {
			if structFieldKind == reflect.Slice {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElements, numElements)
				for i := 0; i < numElements; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				v.Field(i).Set(slice)
			} else {
				if _, isTime := structField.Interface().(time.Time); isTime {
					if err := setTimeField(inputValue[0], typeField, structField); err != nil {
						return err
					}
					continue
				}
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int:
		return setIntField(val, 0, structField)
	case reflect.Int8:
		return setIntField(val, 8, structField)
	case reflect.Int16:
		return setIntField(val, 16, structField)
	case reflect.Int32:
		return setIntField(val, 32, structField)
	case reflect.Int64:
		return setIntField(val, 64, structField)
	case reflect.Uint:
		return setUintField(val, 0, structField)
	case reflect.Uint8:
		return setUintField(val, 8, structField)
	case reflect.Uint16:
		return setUintField(val, 16, structField)
	case reflect.Uint32:
		return setUintField(val, 32, structField)
	case reflect.Uint64:
		return setUintField(val, 64, structField)
	case reflect.Bool:
		return setBoolField(val, structField)
	case reflect.Float32:
		return setFloatField(val, 32, structField)
	case reflect.Float64:
		return setFloatField(val, 64, structField)
	case reflect.String:
		structField.SetString(val)
	default:
		return errors.New("Unknown type")
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return nil
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}

func setTimeField(val string, structField reflect.StructField, value reflect.Value) error {
	timeFormat := structField.Tag.Get("time_format")
	if timeFormat == "" {
		return errors.New("Blank time format")
	}
	if val == "" {
		value.Set(reflect.ValueOf(time.Time{}))
		return nil
	}
	l := time.Local
	if isUTC, _ := strconv.ParseBool(structField.Tag.Get("time_utc")); isUTC {
		l = time.UTC
	}
	t, err := time.ParseInLocation(timeFormat, val, l)
	if err != nil {
		return err
	}
	value.Set(reflect.ValueOf(t))
	return nil
}
<commit_msg>- Add support tree structures<commit_after>package just

import (
	"errors"
	"mime/multipart"
	"net/url"
	"reflect"
	"strconv"
	"time"
)

func marshalUrlValues(ptr interface{}) ([]byte, error) {
	t, v := reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem()
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return nil, errors.New("Array and slice by root element not supported, only structure!")
	}
	values := make(url.Values)
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		if structFieldKind == reflect.Slice {
			for i := 0; i < structField.Len(); i++ {
				values.Add(inputFieldName, structField.Index(i).String())
			}
		} else {
			values.Set(inputFieldName, structField.String())
		}
	}
	return []byte(values.Encode()), nil
}

// mapForm - form mapping (+ multipart files support)
func mapForm(values map[string][]string, files map[string][]*multipart.FileHeader, ptr interface{}) error {
	return recursiveTreeMapForm(values, files, reflect.TypeOf(ptr).Elem(), reflect.ValueOf(ptr).Elem())
}

func recursiveTreeMapForm(values map[string][]string, files map[string][]*multipart.FileHeader, t reflect.Type, v reflect.Value) error {
	if t.Kind() == reflect.Array || t.Kind() == reflect.Slice {
		return errors.New("Array and slice by root element not supported, only structure!")
	}
	for i := 0; i < t.NumField(); i++ {
		typeField, structField := t.Field(i), v.Field(i)
		if !structField.CanSet() {
			if typeField.Type.Kind() == reflect.Struct {
				if err := recursiveTreeMapForm(values, files, typeField.Type, structField); err != nil {
					return err
				}
			}
			continue
		}
		structFieldKind, inputFieldName := structField.Kind(), typeField.Tag.Get("form")
		if len(inputFieldName) < 1 {
			inputFieldName = typeField.Name
		}
		inputValue, existsValue := values[inputFieldName]
		if !existsValue {
			// Check files
			if files != nil && len(files) > 0 {
				if list, ok := files[inputFieldName]; ok {
					if numFiles := len(list); numFiles > 0 {
						if structFieldKind == reflect.Slice {
							if structField.Type().Elem().Name() == "FileHeader" {
								slice := reflect.MakeSlice(structField.Type(), numFiles, numFiles)
								for i := 0; i < numFiles; i++ {
									slice.Index(i).Set(reflect.ValueOf(list[i]))
								}
								v.Field(i).Set(slice)
							}
						} else if typeField.Type.Name() == "FileHeader" {
							structField.Set(reflect.ValueOf(list[0]))
						}
					}
				}
			}
			continue
		}
		if numElements := len(inputValue); numElements > 0 {
			if structFieldKind == reflect.Slice {
				sliceOf := structField.Type().Elem().Kind()
				slice := reflect.MakeSlice(structField.Type(), numElements, numElements)
				for i := 0; i < numElements; i++ {
					if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
						return err
					}
				}
				v.Field(i).Set(slice)
			} else {
				if _, isTime := structField.Interface().(time.Time); isTime {
					if err := setTimeField(inputValue[0], typeField, structField); err != nil {
						return err
					}
					continue
				}
				if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int:
		return setIntField(val, 0, structField)
	case reflect.Int8:
		return setIntField(val, 8, structField)
	case reflect.Int16:
		return setIntField(val, 16, structField)
	case reflect.Int32:
		return setIntField(val, 32, structField)
	case reflect.Int64:
		return setIntField(val, 64, structField)
	case reflect.Uint:
		return setUintField(val, 0, structField)
	case reflect.Uint8:
		return setUintField(val, 8, structField)
	case reflect.Uint16:
		return setUintField(val, 16, structField)
	case reflect.Uint32:
		return setUintField(val, 32, structField)
	case reflect.Uint64:
		return setUintField(val, 64, structField)
	case reflect.Bool:
		return setBoolField(val, structField)
	case reflect.Float32:
		return setFloatField(val, 32, structField)
	case reflect.Float64:
		return setFloatField(val, 64, structField)
	case reflect.String:
		structField.SetString(val)
	default:
		return errors.New("Unknown type")
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return nil
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}

func setTimeField(val string, structField reflect.StructField, value reflect.Value) error {
	timeFormat := structField.Tag.Get("time_format")
	if timeFormat == "" {
		return errors.New("Blank time format")
	}
	if val == "" {
		value.Set(reflect.ValueOf(time.Time{}))
		return nil
	}
	l := time.Local
	if isUTC, _ := strconv.ParseBool(structField.Tag.Get("time_utc")); isUTC {
		l = time.UTC
	}
	t, err := time.ParseInLocation(timeFormat, val, l)
	if err != nil {
		return err
	}
	value.Set(reflect.ValueOf(t))
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"time"

	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

type stats struct {
	Total   int
	Last30d int
	Last7d  int
	Last24h int
	Lines   int
	Time    time.Time
}

func (s stats) FormattedTime() string {
	return s.Time.In(time.UTC).Format(time.RFC850)
}

type tokeiLanguageReport struct {
	Blanks   int `json:"blanks"`
	Code     int `json:"code"`
	Comments int `json:"comments"`
	Stats    []struct {
		Blanks   int    `json:"blanks"`
		Code     int    `json:"code"`
		Comments int    `json:"comments"`
		Lines    int    `json:"lines"`
		Name     string `json:"name"`
	} `json:"stats"`
	Lines int `json:"lines"`
}

type TokeiReport struct {
	CSS        tokeiLanguageReport `json:"Css"`
	Dockerfile tokeiLanguageReport `json:"Dockerfile"`
	Go         tokeiLanguageReport `json:"Go"`
	Hex        tokeiLanguageReport `json:"Hex"`
	HTML       tokeiLanguageReport `json:"Html"`
	Makefile   tokeiLanguageReport `json:"Makefile"`
	Markdown   tokeiLanguageReport `json:"Markdown"`
	Sh         tokeiLanguageReport `json:"Sh"`
	Text       tokeiLanguageReport `json:"Text"`
	Toml       tokeiLanguageReport `json:"Toml"`
	Yaml       tokeiLanguageReport `json:"Yaml"`
}

func getTokeiReport(path string) (*TokeiReport, error) {
	cmd := exec.Command("tokei", "-e", "vendor/", path, "-o", "json")
	buf := new(bytes.Buffer)
	cmd.Stdout = buf
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	decoder := json.NewDecoder(buf)
	r := new(TokeiReport)
	if err := decoder.Decode(r); err != nil {
		return nil, err
	}
	return r, nil
}

func getStats(fetch bool) (*stats, error) {
	n := time.Now()
	last30Days := n.AddDate(0, 0, -30)
	last7Days := n.AddDate(0, 0, -7)
	last24h := n.AddDate(0, 0, -1)
	var total, last30DaysTotal, last7DaysTotal, last24hTotal, lines int
	for _, name := range []string{
		"stun", "turn", "sdp", "web", "stund", "tech-status", "ice", "rtc", "gortcd",
		"ansible-role-nginx", "ansible-go", "api",
	} {
		p := "/tmp/gortc-analyze/" + name
		r, err := git.PlainClone(p, false, &git.CloneOptions{
			URL: "https://github.com/gortc/" + name,
		})
		if err == git.ErrRepositoryAlreadyExists {
			r, err = git.PlainOpen(p)
			if err != nil {
				return nil, err
			}
			w, err := r.Worktree()
			if err != nil {
				return nil, err
			}
			if fetch {
				err = w.Pull(&git.PullOptions{
					Force:      true,
					RemoteName: "origin",
				})
				log.Println("pull", name, err)
				if err == git.NoErrAlreadyUpToDate {
					err = nil
				}
			}
		}
		if err != nil {
			return nil, err
		}
		rep, err := getTokeiReport(p)
		if err != nil {
			return nil, err
		}
		lines += rep.Go.Lines + rep.Yaml.Lines + rep.Dockerfile.Lines
		ref, err := r.Head()
		if err != nil {
			return nil, err
		}
		fmt.Println(name, "head", ref)
		b, err := r.Log(&git.LogOptions{
			From: ref.Hash(),
		})
		var commits int
		countAuthors := map[string]struct{}{
			"ar@cydev.ru":     {},
			"ernado@ya.ru":    {},
			"mail@backkem.me": {},
			"ar@gortc.io":     {},
		}
		if err = b.ForEach(func(commit *object.Commit) error {
			if _, ok := countAuthors[commit.Author.Email]; !ok {
				return nil
			}
			commits++
			if commit.Author.When.After(last30Days) {
				last30DaysTotal++
			}
			if commit.Author.When.After(last7Days) {
				last7DaysTotal++
			}
			if commit.Author.When.After(last24h) {
				last24hTotal++
			}
			return nil
		}); err != nil {
			return nil, err
		}
		total += commits
	}
	return &stats{
		Time:    time.Now(),
		Total:   total,
		Last30d: last30DaysTotal,
		Last7d:  last7DaysTotal,
		Last24h: last24hTotal,
		Lines:   lines,
	}, nil
}
<commit_msg>stats: add docs to tracked repos<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"time"

	"gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

type stats struct {
	Total   int
	Last30d int
	Last7d  int
	Last24h int
	Lines   int
	Time    time.Time
}

func (s stats) FormattedTime() string {
	return s.Time.In(time.UTC).Format(time.RFC850)
}

type tokeiLanguageReport struct {
	Blanks   int `json:"blanks"`
	Code     int `json:"code"`
	Comments int `json:"comments"`
	Stats    []struct {
		Blanks   int    `json:"blanks"`
		Code     int    `json:"code"`
		Comments int    `json:"comments"`
		Lines    int    `json:"lines"`
		Name     string `json:"name"`
	} `json:"stats"`
	Lines int `json:"lines"`
}

type TokeiReport struct {
	CSS        tokeiLanguageReport `json:"Css"`
	Dockerfile tokeiLanguageReport `json:"Dockerfile"`
	Go         tokeiLanguageReport `json:"Go"`
	Hex        tokeiLanguageReport `json:"Hex"`
	HTML       tokeiLanguageReport `json:"Html"`
	Makefile   tokeiLanguageReport `json:"Makefile"`
	Markdown   tokeiLanguageReport `json:"Markdown"`
	Sh         tokeiLanguageReport `json:"Sh"`
	Text       tokeiLanguageReport `json:"Text"`
	Toml       tokeiLanguageReport `json:"Toml"`
	Yaml       tokeiLanguageReport `json:"Yaml"`
}

func getTokeiReport(path string) (*TokeiReport, error) {
	cmd := exec.Command("tokei", "-e", "vendor/", path, "-o", "json")
	buf := new(bytes.Buffer)
	cmd.Stdout = buf
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	decoder := json.NewDecoder(buf)
	r := new(TokeiReport)
	if err := decoder.Decode(r); err != nil {
		return nil, err
	}
	return r, nil
}

func getStats(fetch bool) (*stats, error) {
	n := time.Now()
	last30Days := n.AddDate(0, 0, -30)
	last7Days := n.AddDate(0, 0, -7)
	last24h := n.AddDate(0, 0, -1)
	var total, last30DaysTotal, last7DaysTotal, last24hTotal, lines int
	for _, name := range []string{
		"stun", "turn", "sdp", "web", "stund", "tech-status", "ice", "rtc", "gortcd",
		"ansible-role-nginx", "ansible-go", "api", "docs",
	} {
		p := "/tmp/gortc-analyze/" + name
		r, err := git.PlainClone(p, false, &git.CloneOptions{
			URL: "https://github.com/gortc/" + name,
		})
		if err == git.ErrRepositoryAlreadyExists {
			r, err = git.PlainOpen(p)
			if err != nil {
				return nil, err
			}
			w, err := r.Worktree()
			if err != nil {
				return nil, err
			}
			if fetch {
				err = w.Pull(&git.PullOptions{
					Force:      true,
					RemoteName: "origin",
				})
				log.Println("pull", name, err)
				if err == git.NoErrAlreadyUpToDate {
					err = nil
				}
			}
		}
		if err != nil {
			return nil, err
		}
		rep, err := getTokeiReport(p)
		if err != nil {
			return nil, err
		}
		lines += rep.Go.Lines + rep.Yaml.Lines + rep.Dockerfile.Lines
		ref, err := r.Head()
		if err != nil {
			return nil, err
		}
		fmt.Println(name, "head", ref)
		b, err := r.Log(&git.LogOptions{
			From: ref.Hash(),
		})
		var commits int
		countAuthors := map[string]struct{}{
			"ar@cydev.ru":     {},
			"ernado@ya.ru":    {},
			"mail@backkem.me": {},
			"ar@gortc.io":     {},
		}
		if err = b.ForEach(func(commit *object.Commit) error {
			if _, ok := countAuthors[commit.Author.Email]; !ok {
				return nil
			}
			commits++
			if commit.Author.When.After(last30Days) {
				last30DaysTotal++
			}
			if commit.Author.When.After(last7Days) {
				last7DaysTotal++
			}
			if commit.Author.When.After(last24h) {
				last24hTotal++
			}
			return nil
		}); err != nil {
			return nil, err
		}
		total += commits
	}
	return &stats{
		Time:    time.Now(),
		Total:   total,
		Last30d: last30DaysTotal,
		Last7d:  last7DaysTotal,
		Last24h: last24hTotal,
		Lines:   lines,
	}, nil
}
<|endoftext|>


<commit_before>package boomer

import (
	"time"
)

type RequestStats struct {
	Entries              map[string]*StatsEntry
	Errors               map[string]*StatsError
	NumRequests          int64
	NumFailures          int64
	MaxRequests          int64
	LastRequestTimestamp int64
	StartTime            int64
}

func (this *RequestStats) get(name string, method string) (entry *StatsEntry) {
	entry, ok := this.Entries[name+method]
	if !ok {
		entry = &StatsEntry{
			Stats:         this,
			Name:          name,
			Method:        method,
			NumReqsPerSec: make(map[int64]int64),
			ResponseTimes: make(map[float64]int64),
		}
		entry.reset()
		this.Entries[name+method] = entry
	}
	return this.Entries[name+method]
}

func (this *RequestStats) clearAll() {
	this.NumRequests = 0
	this.NumFailures = 0
	this.Entries = make(map[string]*StatsEntry)
	this.Errors = make(map[string]*StatsError)
	this.MaxRequests = 0
	this.LastRequestTimestamp = 0
	this.StartTime = 0
}

type StatsEntry struct {
	Stats                *RequestStats
	Name                 string
	Method               string
	NumRequests          int64
	NumFailures          int64
	TotalResponseTime    float64
	MinResponseTime      float64
	MaxResponseTime      float64
	NumReqsPerSec        map[int64]int64
	ResponseTimes        map[float64]int64
	TotalContentLength   int64
	StartTime            int64
	LastRequestTimestamp int64
}

func (this *StatsEntry) reset() {
	this.StartTime = int64(time.Now().Unix())
	this.NumRequests = 0
	this.NumFailures = 0
	this.TotalResponseTime = 0
	this.ResponseTimes = make(map[float64]int64)
	this.MinResponseTime = 0
	this.MaxResponseTime = 0
	this.LastRequestTimestamp = int64(time.Now().Unix())
	this.NumReqsPerSec = make(map[int64]int64)
	this.TotalContentLength = 0
}

func (this *StatsEntry) log(responseTime float64, contentLength int64) {
	this.Stats.NumRequests += 1
	this.NumRequests += 1

	this.logTimeOfRequest()
	this.logResponseTime(responseTime)

	this.TotalContentLength += contentLength
}

func (this *StatsEntry) logTimeOfRequest() {
	now := int64(time.Now().Unix())

	_, ok := this.NumReqsPerSec[now]
	if !ok {
		this.NumReqsPerSec[now] = 0
	} else {
		this.NumReqsPerSec[now] += 1
	}

	this.LastRequestTimestamp = now
	this.Stats.LastRequestTimestamp = now
}

func (this *StatsEntry) logResponseTime(responseTime float64) {
	this.TotalResponseTime += responseTime

	if this.MinResponseTime == 0 {
		this.MinResponseTime = responseTime
	}

	if responseTime < this.MinResponseTime {
		this.MinResponseTime = responseTime
	}

	if responseTime > this.MaxResponseTime {
		this.MaxResponseTime = responseTime
	}

	roundedResponseTime := float64(0)

	if responseTime < 100 {
		roundedResponseTime = responseTime
	} else if responseTime < 1000 {
		roundedResponseTime = float64(Round(responseTime, .5, -1))
	} else if responseTime < 10000 {
		roundedResponseTime = float64(Round(responseTime, .5, -2))
	} else {
		roundedResponseTime = float64(Round(responseTime, .5, -3))
	}

	_, ok := this.ResponseTimes[roundedResponseTime]
	if !ok {
		this.ResponseTimes[roundedResponseTime] = 0
	} else {
		this.ResponseTimes[roundedResponseTime] += 1
	}

}

func (this *StatsEntry) logError(err string) {
	this.NumFailures += 1
	this.Stats.NumFailures += 1
	key := MD5(this.Method, this.Name, err)
	entry, ok := this.Stats.Errors[key]
	if !ok {
		entry = &StatsError{
			Name:   this.Name,
			Method: this.Method,
			Error:  err,
		}
		this.Stats.Errors[key] = entry
	}
	entry.occured()
}

func (this *StatsEntry) serialize() map[string]interface{} {
	result := make(map[string]interface{})
	result["name"] = this.Name
	result["method"] = this.Method
	result["last_request_timestamp"] = this.LastRequestTimestamp
	result["start_time"] = this.StartTime
	result["num_requests"] = this.NumRequests
	result["num_failures"] = this.NumFailures
	result["total_response_time"] = this.TotalResponseTime
	result["max_response_time"] = this.MaxResponseTime
	result["min_response_time"] = this.MinResponseTime
	result["total_content_length"] = this.TotalContentLength
	result["response_times"] = this.ResponseTimes
	result["num_reqs_per_sec"] = this.NumReqsPerSec
	return result
}

func (this *StatsEntry) getStrippedReport() map[string]interface{} {
	report := this.serialize()
	this.reset()
	return report
}

type StatsError struct {
	Name       string
	Method     string
	Error      string
	Occurences int64
}

func (this *StatsError) occured() {
	this.Occurences += 1
}

func (this *StatsError) toMap() map[string]interface{} {
	m := make(map[string]interface{})
	m["method"] = this.Method
	m["name"] = this.Name
	m["error"] = this.Error
	m["occurences"] = this.Occurences
	return m
}

func reportToMasterHandler(data *map[string]interface{}) {
	entries := make([]interface{}, 0, len(stats.Entries))
	for _, v := range stats.Entries {
		if !(v.NumRequests == 0 && v.NumFailures == 0) {
			entries = append(entries, v.getStrippedReport())
		}
	}

	errors := make(map[string]map[string]interface{})
	for k, v := range stats.Errors {
		errors[k] = v.toMap()
	}

	(*data)["stats"] = entries
	(*data)["errors"] = errors
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
}

type RequestSuccess struct {
	requestType    string
	name           string
	responseTime   float64
	responseLength int64
}

type RequestFailure struct {
	requestType  string
	name         string
	responseTime float64
	error        string
}

var stats = new(RequestStats)
var RequestSuccessChannel = make(chan *RequestSuccess, 100)
var RequestFailureChannel = make(chan *RequestFailure, 100)
var ClearStatsChannel = make(chan bool)

func init() {
	Events.Subscribe("boomer:report_to_master", reportToMasterHandler)
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
	go func() {
		for {
			select {
			case m := <-RequestSuccessChannel:
				stats.get(m.name, m.requestType).log(m.responseTime, m.responseLength)
			case n := <-RequestFailureChannel:
				stats.get(n.name, n.requestType).logError(n.error)
			case <-ClearStatsChannel:
				stats.clearAll()
			}
		}
	}()
}
<commit_msg>trying to fix an unkown panic.<commit_after>package boomer

import (
	"time"
)

type RequestStats struct {
	Entries              map[string]*StatsEntry
	Errors               map[string]*StatsError
	NumRequests          int64
	NumFailures          int64
	MaxRequests          int64
	LastRequestTimestamp int64
	StartTime            int64
}

func (this *RequestStats) get(name string, method string) (entry *StatsEntry) {
	entry, ok := this.Entries[name+method]
	if !ok {
		entry = &StatsEntry{
			Stats:         this,
			Name:          name,
			Method:        method,
			NumReqsPerSec: make(map[int64]int64),
			ResponseTimes: make(map[float64]int64),
		}
		entry.reset()
		this.Entries[name+method] = entry
	}
	return this.Entries[name+method]
}

func (this *RequestStats) clearAll() {
	this.NumRequests = 0
	this.NumFailures = 0
	this.Entries = make(map[string]*StatsEntry)
	this.Errors = make(map[string]*StatsError)
	this.MaxRequests = 0
	this.LastRequestTimestamp = 0
	this.StartTime = 0
}

type StatsEntry struct {
	Stats                *RequestStats
	Name                 string
	Method               string
	NumRequests          int64
	NumFailures          int64
	TotalResponseTime    float64
	MinResponseTime      float64
	MaxResponseTime      float64
	NumReqsPerSec        map[int64]int64
	ResponseTimes        map[float64]int64
	TotalContentLength   int64
	StartTime            int64
	LastRequestTimestamp int64
}

func (this *StatsEntry) reset() {
	this.StartTime = int64(time.Now().Unix())
	this.NumRequests = 0
	this.NumFailures = 0
	this.TotalResponseTime = 0
	this.ResponseTimes = make(map[float64]int64)
	this.MinResponseTime = 0
	this.MaxResponseTime = 0
	this.LastRequestTimestamp = int64(time.Now().Unix())
	this.NumReqsPerSec = make(map[int64]int64)
	this.TotalContentLength = 0
}

func (this *StatsEntry) log(responseTime float64, contentLength int64) {

	this.NumRequests += 1

	this.logTimeOfRequest()
	this.logResponseTime(responseTime)

	this.TotalContentLength += contentLength

}

func (this *StatsEntry) logTimeOfRequest() {

	now := int64(time.Now().Unix())

	_, ok := this.NumReqsPerSec[now]
	if !ok {
		this.NumReqsPerSec[now] = 0
	} else {
		this.NumReqsPerSec[now] += 1
	}

	this.LastRequestTimestamp = now

}

func (this *StatsEntry) logResponseTime(responseTime float64) {
	this.TotalResponseTime += responseTime

	if this.MinResponseTime == 0 {
		this.MinResponseTime = responseTime
	}

	if responseTime < this.MinResponseTime {
		this.MinResponseTime = responseTime
	}

	if responseTime > this.MaxResponseTime {
		this.MaxResponseTime = responseTime
	}

	roundedResponseTime := float64(0)

	if responseTime < 100 {
		roundedResponseTime = responseTime
	} else if responseTime < 1000 {
		roundedResponseTime = float64(Round(responseTime, .5, -1))
	} else if responseTime < 10000 {
		roundedResponseTime = float64(Round(responseTime, .5, -2))
	} else {
		roundedResponseTime = float64(Round(responseTime, .5, -3))
	}

	_, ok := this.ResponseTimes[roundedResponseTime]
	if !ok {
		this.ResponseTimes[roundedResponseTime] = 0
	} else {
		this.ResponseTimes[roundedResponseTime] += 1
	}

}

func (this *StatsEntry) logError(err string) {
	this.NumFailures += 1
	key := MD5(this.Method, this.Name, err)
	entry, ok := this.Stats.Errors[key]
	if !ok {
		entry = &StatsError{
			Name:   this.Name,
			Method: this.Method,
			Error:  err,
		}
		this.Stats.Errors[key] = entry
	}
	entry.occured()
}

func (this *StatsEntry) serialize() map[string]interface{} {
	result := make(map[string]interface{})
	result["name"] = this.Name
	result["method"] = this.Method
	result["last_request_timestamp"] = this.LastRequestTimestamp
	result["start_time"] = this.StartTime
	result["num_requests"] = this.NumRequests
	result["num_failures"] = this.NumFailures
	result["total_response_time"] = this.TotalResponseTime
	result["max_response_time"] = this.MaxResponseTime
	result["min_response_time"] = this.MinResponseTime
	result["total_content_length"] = this.TotalContentLength
	result["response_times"] = this.ResponseTimes
	result["num_reqs_per_sec"] = this.NumReqsPerSec
	return result
}

func (this *StatsEntry) getStrippedReport() map[string]interface{} {
	report := this.serialize()
	this.reset()
	return report
}

type StatsError struct {
	Name       string
	Method     string
	Error      string
	Occurences int64
}

func (this *StatsError) occured() {
	this.Occurences += 1
}

func (this *StatsError) toMap() map[string]interface{} {
	m := make(map[string]interface{})
	m["method"] = this.Method
	m["name"] = this.Name
	m["error"] = this.Error
	m["occurences"] = this.Occurences
	return m
}

func reportToMasterHandler(data *map[string]interface{}) {
	entries := make([]interface{}, 0, len(stats.Entries))
	for _, v := range stats.Entries {
		if !(v.NumRequests == 0 && v.NumFailures == 0) {
			entries = append(entries, v.getStrippedReport())
		}
	}

	errors := make(map[string]map[string]interface{})
	for k, v := range stats.Errors {
		errors[k] = v.toMap()
	}

	(*data)["stats"] = entries
	(*data)["errors"] = errors
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
}

type RequestSuccess struct {
	requestType    string
	name           string
	responseTime   float64
	responseLength int64
}

type RequestFailure struct {
	requestType  string
	name         string
	responseTime float64
	error        string
}

var stats = new(RequestStats)
var RequestSuccessChannel = make(chan *RequestSuccess, 100)
var RequestFailureChannel = make(chan *RequestFailure, 100)
var ClearStatsChannel = make(chan bool)

func init() {
	Events.Subscribe("boomer:report_to_master", reportToMasterHandler)
	stats.Entries = make(map[string]*StatsEntry)
	stats.Errors = make(map[string]*StatsError)
	go func() {
		for {
			select {
			case m := <-RequestSuccessChannel:
				stats.get(m.name, m.requestType).log(m.responseTime, m.responseLength)
			case n := <-RequestFailureChannel:
				stats.get(n.name, n.requestType).logError(n.error)
			case <-ClearStatsChannel:
				stats.clearAll()
			}
		}
	}()
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2013-2015 SRS(simple-rtmp-server)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package app

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/simple-rtmp-server/go-srs/core"
	"io"
	"io/ioutil"
	"os"
)

// the scope for reload.
const (
	ReloadWorkers = iota
	ReloadLog
)

// the reload handler,
// the client which care about the reload event,
// must implements this interface and then register itself
// to the config.
type ReloadHandler interface {
	// when reload the global scopes,
	// for example, the workers, listen and log.
	// @param scope defined in const ReloadXXX.
	// @param cc the current loaded config, GsConfig.
	// @param pc the previous old config.
	OnReloadGlobal(scope int, cc, pc *Config) error
}

// the reader support c++-style comment,
//      block: /* comments */
//      line: // comments
type Reader struct {
	attention int // attention please, maybe comments.
	f         funcReader
	r         io.Reader
}

func NewReader(r io.Reader) io.Reader {
	return &Reader{r: r}
}

// read from r into p, return actual n bytes, the next handler and err indicates error.
type funcReader func(r io.Reader, p []byte) (n int, next funcReader, err error)

// interface io.Reader
func (v *Reader) Read(p []byte) (n int, err error) {
	var lineReader funcReader
	var blockReader funcReader
	var contentReader funcReader

	lineReader = funcReader(func(r io.Reader, p []byte) (n int, next funcReader, err error) {
		b := make([]byte, 1)
		if n, err = io.ReadAtLeast(r, b, 1); err != nil {
			return
		}

		// skip any util \n
		if b[0] != '\n' {
			return 0, lineReader, nil
		}

		return 0, contentReader, nil
	})

	blockReader = funcReader(func(r io.Reader, p []byte) (n int, next funcReader, err error) {
		if len(p) < v.attention+1 {
			return 0, nil, nil
		}

		// read one byte more.
		b := make([]byte, 1)
		if n, err = io.ReadAtLeast(r, b, 1); err != nil {
			// when EOF, ok for content or content reader,
			// but invalid for block reader.
			if err == io.EOF {
				return 0, nil, errors.New("block comments should not EOF")
			}
			return
		}

		// skip any util */
		if b[0] != '/' && b[0] != '*' {
			return 0, blockReader, nil
		}

		// attention
		if b[0] == '*' {
			v.attention = 1
			return 0, blockReader, nil
		}

		// eof comments.
		if v.attention != 0 && b[0] == '/' {
			v.attention = 0
			return 0, contentReader, nil
		}

		panic(fmt.Sprintf("impossible, attention=%v, b=%v", v.attention, b[0]))
		return
	})

	contentReader = funcReader(func(r io.Reader, p []byte) (n int, next funcReader, err error) {
		if len(p) < v.attention+1 {
			return 0, nil, nil
		}

		// read one byte more.
		b := make([]byte, 1)
		if n, err = io.ReadAtLeast(r, b, 1); err != nil {
			return
		}

		// 2byte push.
		if v.attention != 0 && b[0] != '/' && b[0] != '*' {
			p[0] = '/'
			p[1] = b[0]
			return 2, contentReader, err
		}

		// 1byte push.
		if v.attention == 0 && b[0] != '/' {
			p[0] = b[0]
			return 1, contentReader, err
		}

		// attention
		if v.attention == 0 && b[0] == '/' {
			v.attention = 1
			return 0, contentReader, err
		}

		// line comments.
		if v.attention != 0 && b[0] == '/' {
			v.attention = 0
			return 0, lineReader, err
		}

		// block comments.
		if v.attention != 0 && b[0] == '*' {
			v.attention = 0
			return 0, blockReader, err
		}

		panic("impossible execute path")
		return
	})

	// start using normal byte reader.
	var f funcReader
	if f = v.f; f == nil {
		f = contentReader
	}

	// read util full or no func reader specified.
	for i := 0; f != nil && i < len(p); {
		var ne int
		if ne, f, err = f(v.r, p[i:]); err != nil {
			break
		}

		// apply the consumed bytes.
		n += ne
		i += ne

		// remember the last handler we use.
		if f != nil {
			v.f = f
		}
	}

	return
}

// the config for this application,
// which can load from file in json style,
// and convert to json string.
// @remark user can user the GsConfig object.
type Config struct {
	// the global section.
	Workers int `json:"workers"` // the number of cpus to use

	// the rtmp global section.
	Listen int  `json:"listen"` // the system service RTMP listen port
	Daemon bool `json:"daemon"` // whether enabled the daemon for unix-like os

	// the go section.
	Go struct {
		GcInterval int `json:"gc_interval"` // the gc interval in seconds.
	}

	// the log config.
	Log struct {
		Tank  string `json:"tank"`  // the log tank, file or console
		Level string `json:"level"` // the log level, info/trace/warn/error
		File  string `json:"file"`  // for log tank file, the log file path.
	} `json:"log"`

	conf           string          `json:"-"` // the config file path.
	reloadHandlers []ReloadHandler `json:"-"`
}

// the current global config.
var GsConfig = NewConfig()

func NewConfig() *Config {
	c := &Config{
		reloadHandlers: []ReloadHandler{},
	}

	c.Workers = core.Workers
	c.Listen = core.RtmpListen
	c.Daemon = true
	c.Go.GcInterval = core.GcIntervalSeconds

	c.Log.Tank = "file"
	c.Log.Level = "trace"
	c.Log.File = "gsrs.log"

	return c
}

// loads and validate config from config file.
func (c *Config) Loads(conf string) error {
	c.conf = conf

	// read the whole config to []byte.
	var d *json.Decoder
	if f, err := os.Open(conf); err != nil {
		return err
	} else {
		defer f.Close()
		d = json.NewDecoder(NewReader(f))
	}

	// decode config from stream.
	if err := d.Decode(c); err != nil {
		return err
	}

	// validate the config.
	return c.Validate()
}

// validate the config whether ok.
func (c *Config) Validate() error {
	if c.Log.Level == "info" {
		core.GsWarn.Println("info level hurts performance")
	}

	if c.Workers <= 0 || c.Workers > 64 {
		return errors.New(fmt.Sprintf("workers must in (0, 64], actual is %v", c.Workers))
	}
	if c.Listen <= 0 || c.Listen > 65535 {
		return errors.New(fmt.Sprintf("listen must in (0, 65535], actual is %v", c.Listen))
	}

	if c.Go.GcInterval <= 0 || c.Go.GcInterval > 24*3600 {
		return errors.New(fmt.Sprintf("go gc_interval must in (0, 24*3600], actual is %v", c.Go.GcInterval))
	}

	if c.Log.Level != "info" && c.Log.Level != "trace" && c.Log.Level != "warn" && c.Log.Level != "error" {
		return errors.New(fmt.Sprintf("log.leve must be info/trace/warn/error, actual is %v", c.Log.Level))
	}
	if c.Log.Tank != "console" && c.Log.Tank != "file" {
		return errors.New(fmt.Sprintf("log.tank must be console/file, actual is %v", c.Log.Tank))
	}
	if c.Log.Tank == "file" && len(c.Log.File) == 0 {
		return errors.New("log.file must not be empty for file tank")
	}

	return nil
}

// whether log tank is file
func (c *Config) LogToFile() bool {
	return c.Log.Tank == "file"
}

// get the log tank writer for specified level.
// the param dw is the default writer.
func (c *Config) LogTank(level string, dw io.Writer) io.Writer {
	if c.Log.Level == "info" {
		return dw
	}
	if c.Log.Level == "trace" {
		if level == "info" {
			return ioutil.Discard
		}
		return dw
	}
	if c.Log.Level == "warn" {
		if level == "info" || level == "trace" {
			return ioutil.Discard
		}
		return dw
	}
	if c.Log.Level == "error" {
		if level != "error" {
			return ioutil.Discard
		}
		return dw
	}

	return ioutil.Discard
}

// subscribe the reload event,
// when got reload event, notify all handlers.
func (c *Config) Subscribe(h ReloadHandler) {
	// ignore exists.
	for _, v := range c.reloadHandlers {
		if v == h {
			return
		}
	}

	c.reloadHandlers = append(c.reloadHandlers, h)
}

func (c *Config) Unsubscribe(h ReloadHandler) {
	for i, v := range c.reloadHandlers {
		if v == h {
			c.reloadHandlers = append(c.reloadHandlers[:i], c.reloadHandlers[i+1:]...)
			return
		}
	}
}

func (pc *Config) Reload(cc *Config) (err error) {
	if cc.Workers != pc.Workers {
		for _, h := range cc.reloadHandlers {
			if err = h.OnReloadGlobal(ReloadWorkers, cc, pc); err != nil {
				return
			}
		}
		core.GsTrace.Println("reload apply workers ok")
	} else {
		core.GsInfo.Println("reload ignore workers")
	}

	if cc.Log.File != pc.Log.File || cc.Log.Level != pc.Log.Level || cc.Log.Tank != pc.Log.Tank {
		for _, h := range cc.reloadHandlers {
			if err = h.OnReloadGlobal(ReloadLog, cc, pc); err != nil {
				return
			}
		}
		core.GsTrace.Println("reload apply log ok")
	} else {
		core.GsInfo.Println("reload ignore log")
	}

	return
}
<commit_msg>refine comments for json comments.<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2013-2015 SRS(simple-rtmp-server)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package app

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/simple-rtmp-server/go-srs/core"
	"io"
	"io/ioutil"
	"os"
)

// the scope for reload.
const (
	ReloadWorkers = iota
	ReloadLog
)

// the reload handler,
// the client which care about the reload event,
// must implements this interface and then register itself
// to the config.
type ReloadHandler interface {
	// when reload the global scopes,
	// for example, the workers, listen and log.
	// @param scope defined in const ReloadXXX.
	// @param cc the current loaded config, GsConfig.
	// @param pc the previous old config.
	OnReloadGlobal(scope int, cc, pc *Config) error
}

// the reader support c++-style comment,
//      block: /* comments */
//      line: // comments
type Reader struct {
	attention int // attention please, maybe comments.
	f         funcReader
	r         io.Reader
}

func NewReader(r io.Reader) io.Reader {
	return &Reader{r: r}
}

// read from r into p, return actual n bytes, the next handler and err indicates error.
type funcReader func(r io.Reader, p []byte) (n int, next funcReader, err error)

// interface io.Reader
func (v *Reader) Read(p []byte) (n int, err error) {
	var lineReader funcReader
	var blockReader funcReader
	var contentReader funcReader

	lineReader = funcReader(func(r io.Reader, p []byte) (n int, next funcReader, err error) {
		b := make([]byte, 1)
		if n, err = io.ReadAtLeast(r, b, 1); err != nil {
			return
		}

		// skip any util \n
		if b[0] != '\n' {
			return 0, lineReader, nil
		}

		return 0, contentReader, nil
	})

	blockReader = funcReader(func(r io.Reader, p []byte) (n int, next funcReader, err error) {
		if len(p) < v.attention+1 {
			return 0, nil, nil
		}

		// read one byte more.
		b := make([]byte, 1)
		if n, err = io.ReadAtLeast(r, b, 1); err != nil {
			// when EOF, ok for content or content reader,
			// but invalid for block reader.
			if err == io.EOF {
				return 0, nil, errors.New("block comments should not EOF")
			}
			return
		}

		// skip any util */
		if b[0] != '/' && b[0] != '*' {
			return 0, blockReader, nil
		}

		// attention
		if b[0] == '*' {
			v.attention = 1
			return 0, blockReader, nil
		}

		// eof comments.
		if v.attention != 0 && b[0] == '/' {
			v.attention = 0
			return 0, contentReader, nil
		}

		panic(fmt.Sprintf("invalid block, attention=%v, b=%v", v.attention, b[0]))
		return
	})

	contentReader = funcReader(func(r io.Reader, p []byte) (n int, next funcReader, err error) {
		if len(p) < v.attention+1 {
			return 0, nil, nil
		}

		// read one byte more.
		b := make([]byte, 1)
		if n, err = io.ReadAtLeast(r, b, 1); err != nil {
			return
		}

		// 2byte push.
		if v.attention != 0 && b[0] != '/' && b[0] != '*' {
			p[0] = '/'
			p[1] = b[0]
			return 2, contentReader, err
		}

		// 1byte push.
		if v.attention == 0 && b[0] != '/' {
			p[0] = b[0]
			return 1, contentReader, err
		}

		// attention
		if v.attention == 0 && b[0] == '/' {
			v.attention = 1
			return 0, contentReader, err
		}

		// line comments.
		if v.attention != 0 && b[0] == '/' {
			v.attention = 0
			return 0, lineReader, err
		}

		// block comments.
		if v.attention != 0 && b[0] == '*' {
			v.attention = 0
			return 0, blockReader, err
		}

		panic(fmt.Sprintf("invalid content, attention=%v, b=%v", v.attention, b[0]))
		return
	})

	// start using normal byte reader.
	var f funcReader
	if f = v.f; f == nil {
		f = contentReader
	}

	// read util full or no func reader specified.
	for i := 0; f != nil && i < len(p); {
		var ne int
		if ne, f, err = f(v.r, p[i:]); err != nil {
			break
		}

		// apply the consumed bytes.
		n += ne
		i += ne

		// remember the last handler we use.
		if f != nil {
			v.f = f
		}
	}

	return
}

// the config for this application,
// which can load from file in json style,
// and convert to json string.
// @remark user can user the GsConfig object.
type Config struct {
	// the global section.
	Workers int `json:"workers"` // the number of cpus to use

	// the rtmp global section.
	Listen int  `json:"listen"` // the system service RTMP listen port
	Daemon bool `json:"daemon"` // whether enabled the daemon for unix-like os

	// the go section.
	Go struct {
		GcInterval int `json:"gc_interval"` // the gc interval in seconds.
	}

	// the log config.
	Log struct {
		Tank  string `json:"tank"`  // the log tank, file or console
		Level string `json:"level"` // the log level, info/trace/warn/error
		File  string `json:"file"`  // for log tank file, the log file path.
	} `json:"log"`

	conf           string          `json:"-"` // the config file path.
	reloadHandlers []ReloadHandler `json:"-"`
}

// the current global config.
var GsConfig = NewConfig()

func NewConfig() *Config {
	c := &Config{
		reloadHandlers: []ReloadHandler{},
	}

	c.Workers = core.Workers
	c.Listen = core.RtmpListen
	c.Daemon = true
	c.Go.GcInterval = core.GcIntervalSeconds

	c.Log.Tank = "file"
	c.Log.Level = "trace"
	c.Log.File = "gsrs.log"

	return c
}

// loads and validate config from config file.
func (c *Config) Loads(conf string) error {
	c.conf = conf

	// read the whole config to []byte.
	var d *json.Decoder
	if f, err := os.Open(conf); err != nil {
		return err
	} else {
		defer f.Close()
		d = json.NewDecoder(NewReader(f))
	}

	// decode config from stream.
	if err := d.Decode(c); err != nil {
		return err
	}

	// validate the config.
	return c.Validate()
}

// validate the config whether ok.
func (c *Config) Validate() error {
	if c.Log.Level == "info" {
		core.GsWarn.Println("info level hurts performance")
	}

	if c.Workers <= 0 || c.Workers > 64 {
		return errors.New(fmt.Sprintf("workers must in (0, 64], actual is %v", c.Workers))
	}
	if c.Listen <= 0 || c.Listen > 65535 {
		return errors.New(fmt.Sprintf("listen must in (0, 65535], actual is %v", c.Listen))
	}

	if c.Go.GcInterval <= 0 || c.Go.GcInterval > 24*3600 {
		return errors.New(fmt.Sprintf("go gc_interval must in (0, 24*3600], actual is %v", c.Go.GcInterval))
	}

	if c.Log.Level != "info" && c.Log.Level != "trace" && c.Log.Level != "warn" && c.Log.Level != "error" {
		return errors.New(fmt.Sprintf("log.leve must be info/trace/warn/error, actual is %v", c.Log.Level))
	}
	if c.Log.Tank != "console" && c.Log.Tank != "file" {
		return errors.New(fmt.Sprintf("log.tank must be console/file, actual is %v", c.Log.Tank))
	}
	if c.Log.Tank == "file" && len(c.Log.File) == 0 {
		return errors.New("log.file must not be empty for file tank")
	}

	return nil
}

// whether log tank is file
func (c *Config) LogToFile() bool {
	return c.Log.Tank == "file"
}

// get the log tank writer for specified level.
// the param dw is the default writer.
func (c *Config) LogTank(level string, dw io.Writer) io.Writer {
	if c.Log.Level == "info" {
		return dw
	}
	if c.Log.Level == "trace" {
		if level == "info" {
			return ioutil.Discard
		}
		return dw
	}
	if c.Log.Level == "warn" {
		if level == "info" || level == "trace" {
			return ioutil.Discard
		}
		return dw
	}
	if c.Log.Level == "error" {
		if level != "error" {
			return ioutil.Discard
		}
		return dw
	}

	return ioutil.Discard
}

// subscribe the reload event,
// when got reload event, notify all handlers.
func (c *Config) Subscribe(h ReloadHandler) {
	// ignore exists.
	for _, v := range c.reloadHandlers {
		if v == h {
			return
		}
	}

	c.reloadHandlers = append(c.reloadHandlers, h)
}

func (c *Config) Unsubscribe(h ReloadHandler) {
	for i, v := range c.reloadHandlers {
		if v == h {
			c.reloadHandlers = append(c.reloadHandlers[:i], c.reloadHandlers[i+1:]...)
			return
		}
	}
}

func (pc *Config) Reload(cc *Config) (err error) {
	if cc.Workers != pc.Workers {
		for _, h := range cc.reloadHandlers {
			if err = h.OnReloadGlobal(ReloadWorkers, cc, pc); err != nil {
				return
			}
		}
		core.GsTrace.Println("reload apply workers ok")
	} else {
		core.GsInfo.Println("reload ignore workers")
	}

	if cc.Log.File != pc.Log.File || cc.Log.Level != pc.Log.Level || cc.Log.Tank != pc.Log.Tank {
		for _, h := range cc.reloadHandlers {
			if err = h.OnReloadGlobal(ReloadLog, cc, pc); err != nil {
				return
			}
		}
		core.GsTrace.Println("reload apply log ok")
	} else {
		core.GsInfo.Println("reload ignore log")
	}

	return
}
<|endoftext|>


<commit_before>package memkv

import (
	"path/filepath"
	"sort"
	"sync"
)

type Store struct {
	sync.RWMutex
	m map[string]Node
}

func New() Store {
	return Store{m: make(map[string]Node)}
}

func (s Store) Get(key string) (Node, bool) {
	s.RLock()
	n, ok := s.m[key]
	s.RUnlock()
	return n, ok
}

func (s Store) Glob(pattern string) (Nodes, error) {
	ns := make(Nodes, 0)
	s.RLock()
	defer s.RUnlock()
	for _, n := range s.m {
		m, err := filepath.Match(pattern, n.Key)
		if err != nil {
			return nil, err
		}
		if m {
			ns = append(ns, n)
		}
	}
	sort.Sort(ns)
	return ns, nil
}

func (s Store) Set(key string, value string) {
	s.Lock()
	s.m[key] = Node{key, value}
	s.Unlock()
}
<commit_msg>document store.go src file<commit_after>// Copyright 2014 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by a BSD-style
// license found in the LICENSE file.

// Package memkv implements an in-memory key/value store.
package memkv

import (
	"path/filepath"
	"sort"
	"sync"
)

// A Store represents an in-memory key-value store safe for
// concurrent access.
type Store struct {
	sync.RWMutex
	m map[string]Node
}

// New creates and initializes a new Store.
func New() Store {
	return Store{m: make(map[string]Node)}
}

// Get gets the Node associated with key. If there are no Nodes
// associated with key, Get returns Node{}, false.
func (s Store) Get(key string) (Node, bool) {
	s.RLock()
	n, ok := s.m[key]
	s.RUnlock()
	return n, ok
}

// Glob returns a memkv.Node for all nodes with keys matching pattern.
// The syntax of patterns is the same as in filepath.Match.
func (s Store) Glob(pattern string) (Nodes, error) {
	ns := make(Nodes, 0)
	s.RLock()
	defer s.RUnlock()
	for _, n := range s.m {
		m, err := filepath.Match(pattern, n.Key)
		if err != nil {
			return nil, err
		}
		if m {
			ns = append(ns, n)
		}
	}
	sort.Sort(ns)
	return ns, nil
}

// Set sets the node entry associated with key to value. It replaces
// any existing values associated with key.
func (s Store) Set(key string, value string) {
	s.Lock()
	s.m[key] = Node{key, value}
	s.Unlock()
}
<|endoftext|>


<commit_before>package http

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"sync"

	"github.com/mcuadros/dockership/config"

	"code.google.com/p/goauth2/oauth"
	"github.com/golang/oauth2"
	"github.com/google/go-github/github"
	"github.com/gorilla/sessions"
)

const (
	CODE_REDIRECT = 302
	KEY_TOKEN     = "oauth2_token"
)

type User struct {
	Fullname string
	Avatar   string
}

type OAuth struct {
	PathLogin    string // Path to handle OAuth 2.0 logins.
	PathLogout   string // Path to handle OAuth 2.0 logouts.
	PathCallback string // Path to handle callback from OAuth 2.0 backend
	PathError    string // Path to handle error cases.
	OAuthOptions *oauth2.Options
	Config       *config.Config
	users        map[string]*User
	store        sessions.Store
	sync.Mutex
}

func NewOAuth(config *config.Config) *OAuth {
	authUrl := "https://github.com/login/oauth/authorize"
	tokenUrl := "https://github.com/login/oauth/access_token"

	options, err := oauth2.New(
		oauth2.Client(config.HTTP.GithubID, config.HTTP.GithubSecret),
		oauth2.RedirectURL(config.HTTP.GithubRedirectURL),
		oauth2.Scope("read:org"),
		oauth2.Endpoint(authUrl, tokenUrl),
	)

	if err != nil {
		panic(fmt.Sprintf("oauth2: %s", err))
	}

	return &OAuth{
		PathLogin:    "/login",
		PathLogout:   "/logout",
		PathCallback: "/oauth2callback",
		PathError:    "/oauth2error",
		OAuthOptions: options,
		Config:       config,
		users:        make(map[string]*User, 0),
		store:        sessions.NewCookieStore([]byte("cookie-key")),
	}
}

func (o *OAuth) Handler(w http.ResponseWriter, r *http.Request) bool {
	if r.Method == "GET" {
		switch r.URL.Path {
		case o.PathLogin:
			o.HandleLogin(w, r)
			return false
		case o.PathLogout:
			o.HandleLogout(w, r)
			return false
		case o.PathCallback:
			o.HandleCallback(w, r)
			return false
		}
	}

	token := o.getToken(r)
	failed := true
	if token != nil && !token.Expired() {
		if _, err := o.getValidUser(token); err == nil {
			failed = false
		} else {
			w.Write([]byte(err.Error()))
			return false
		}

	}

	if failed {
		next := url.QueryEscape(r.URL.RequestURI())
		http.Redirect(w, r, o.PathLogin+"?next="+next, CODE_REDIRECT)
		return false
	}

	return true
}

func (o *OAuth) HandleCallback(w http.ResponseWriter, r *http.Request) {
	next := extractPath(r.URL.Query().Get("state"))
	code := r.URL.Query().Get("code")

	t, err := o.OAuthOptions.NewTransportFromCode(code)
	if err != nil {
		// Pass the error message, or allow dev to provide its own
		// error handler.
		http.Redirect(w, r, o.PathError, CODE_REDIRECT)
		return
	}

	o.setToken(w, r, t.Token())
	http.Redirect(w, r, next, CODE_REDIRECT)
}

func (o *OAuth) HandleLogin(w http.ResponseWriter, r *http.Request) {
	next := extractPath(r.URL.Query().Get("next"))
	if o.getToken(r) == nil {
		// User is not logged in.
		if next == "" {
			next = "/"
		}
		http.Redirect(w, r, o.OAuthOptions.AuthCodeURL(next, "", ""), CODE_REDIRECT)
		return
	}

	// No need to login, redirect to the next page.
	http.Redirect(w, r, next, CODE_REDIRECT)
}

func (o *OAuth) HandleLogout(w http.ResponseWriter, r *http.Request) {
	next := extractPath(r.URL.Query().Get("next"))
	//s.Delete(KEY_TOKEN)
	http.Redirect(w, r, next, CODE_REDIRECT)
}

func (s *OAuth) getToken(r *http.Request) *oauth2.Token {
	session, _ := s.store.Get(r, KEY_TOKEN)
	if raw, ok := session.Values["token"]; !ok {
		return nil
	} else {
		var tk oauth2.Token
		json.Unmarshal(raw.([]byte), &tk)

		return &tk
	}
}

func (s *OAuth) setToken(w http.ResponseWriter, r *http.Request, t *oauth2.Token) {
	session, _ := s.store.Get(r, KEY_TOKEN)
	val, _ := json.Marshal(t)
	session.Values["token"] = val
	session.Save(r, w)
}

func (o *OAuth) getValidUser(token *oauth2.Token) (*User, error) {
	o.Lock()
	user, ok := o.users[token.AccessToken]
	o.Unlock()

	if ok {
		return user, nil
	}

	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: token.AccessToken},
	}

	c := github.NewClient(t.Client())
	guser, _, err := c.Users.Get("")
	if err != nil {
		return nil, err
	}

	if err := o.isValidUser(c, guser); err != nil {
		return nil, err
	}

	user = &User{}
	if guser != nil && guser.Name != nil {
		user.Fullname = *guser.Name
	} else if guser.Login != nil {
		user.Fullname = *guser.Login
	}

	if guser != nil && guser.AvatarURL != nil {
		user.Avatar = *guser.AvatarURL
	}

	o.Lock()
	o.users[token.AccessToken] = user
	o.Unlock()

	return user, nil
}

func (o *OAuth) isValidUser(c *github.Client, u *github.User) error {
	if err := o.validateGithubOrganization(c, u); err != nil {
		return err
	}

	if err := o.validateGithubUser(c, u); err != nil {
		return err
	}

	return nil
}

func (o *OAuth) validateGithubOrganization(c *github.Client, u *github.User) error {
	org := o.Config.HTTP.GithubOrganization
	if org == "" {
		return nil
	}

	m, _, err := c.Organizations.IsMember(org, *u.Login)
	if err != nil {
		return err
	}

	if !m {
		return errors.New(fmt.Sprintf(
			"User %q should be member of %q", *u.Login, org,
		))
	}

	return nil
}

func (o *OAuth) validateGithubUser(c *github.Client, u *github.User) error {
	if len(o.Config.HTTP.GithubUsers) == 0 {
		return nil
	}

	for _, user := range o.Config.HTTP.GithubUsers {
		if user == *u.Login {
			return nil
		}
	}

	return errors.New(fmt.Sprintf(
		"User %q not allowed, not in the access list.", *u.Login,
	))
}

func extractPath(next string) string {
	n, err := url.Parse(next)
	if err != nil {
		return "/"
	}
	return n.Path
}
<commit_msg>http: ouath2 fixed client<commit_after>package http

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"sync"

	"github.com/mcuadros/dockership/config"

	"code.google.com/p/goauth2/oauth"
	"github.com/google/go-github/github"
	"github.com/gorilla/sessions"
	"golang.org/x/oauth2"
)

const (
	CODE_REDIRECT = 302
	KEY_TOKEN     = "oauth2_token"
)

type User struct {
	Fullname string
	Avatar   string
}

type OAuth struct {
	PathLogin    string // Path to handle OAuth 2.0 logins.
	PathLogout   string // Path to handle OAuth 2.0 logouts.
	PathCallback string // Path to handle callback from OAuth 2.0 backend
	PathError    string // Path to handle error cases.
	OAuthConfig  *oauth2.Config
	Config       *config.Config
	users        map[string]*User
	store        sessions.Store
	sync.Mutex
}

func NewOAuth(config *config.Config) *OAuth {
	authUrl := "https://github.com/login/oauth/authorize"
	tokenUrl := "https://github.com/login/oauth/access_token"

	return &OAuth{
		PathLogin:    "/login",
		PathLogout:   "/logout",
		PathCallback: "/oauth2callback",
		PathError:    "/oauth2error",
		OAuthConfig: &oauth2.Config{
			ClientID:     config.HTTP.GithubID,
			ClientSecret: config.HTTP.GithubSecret,
			Scopes:       []string{"read:org"},
			Endpoint: oauth2.Endpoint{
				AuthURL:  authUrl,
				TokenURL: tokenUrl,
			},
		},
		Config: config,
		users:  make(map[string]*User, 0),
		store:  sessions.NewCookieStore([]byte("cookie-key")),
	}
}

func (o *OAuth) Handler(w http.ResponseWriter, r *http.Request) bool {
	if r.Method == "GET" {
		switch r.URL.Path {
		case o.PathLogin:
			o.HandleLogin(w, r)
			return false
		case o.PathLogout:
			o.HandleLogout(w, r)
			return false
		case o.PathCallback:
			o.HandleCallback(w, r)
			return false
		}
	}

	token := o.getToken(r)
	failed := true
	if token != nil && token.Valid() {
		if _, err := o.getValidUser(token); err == nil {
			failed = false
		} else {
			w.Write([]byte(err.Error()))
			return false
		}

	}

	if failed {
		next := url.QueryEscape(r.URL.RequestURI())
		http.Redirect(w, r, o.PathLogin+"?next="+next, CODE_REDIRECT)
		return false
	}

	return true
}

func (o *OAuth) HandleCallback(w http.ResponseWriter, r *http.Request) {
	next := extractPath(r.URL.Query().Get("state"))
	code := r.URL.Query().Get("code")

	tok, err := o.OAuthConfig.Exchange(oauth2.NoContext, code)
	if err != nil {
		// Pass the error message, or allow dev to provide its own
		// error handler.
		http.Redirect(w, r, o.PathError, CODE_REDIRECT)
		return
	}

	o.setToken(w, r, tok)
	http.Redirect(w, r, next, CODE_REDIRECT)
}

func (s *OAuth) setToken(w http.ResponseWriter, r *http.Request, t *oauth2.Token) {
	session, _ := s.store.Get(r, KEY_TOKEN)
	val, _ := json.Marshal(t)
	session.Values["token"] = val
	session.Save(r, w)
}

func (o *OAuth) HandleLogin(w http.ResponseWriter, r *http.Request) {
	next := extractPath(r.URL.Query().Get("next"))
	if o.getToken(r) == nil {
		// User is not logged in.
		if next == "" {
			next = "/"
		}
		http.Redirect(w, r, o.OAuthConfig.AuthCodeURL(next), CODE_REDIRECT)
		return
	}

	// No need to login, redirect to the next page.
	http.Redirect(w, r, next, CODE_REDIRECT)
}

func (o *OAuth) HandleLogout(w http.ResponseWriter, r *http.Request) {
	next := extractPath(r.URL.Query().Get("next"))
	//s.Delete(KEY_TOKEN)
	http.Redirect(w, r, next, CODE_REDIRECT)
}

func (s *OAuth) getToken(r *http.Request) *oauth2.Token {
	session, _ := s.store.Get(r, KEY_TOKEN)
	if raw, ok := session.Values["token"]; !ok {
		return nil
	} else {
		var tok oauth2.Token
		json.Unmarshal(raw.([]byte), &tok)

		return &tok
	}
}

func (o *OAuth) getValidUser(token *oauth2.Token) (*User, error) {
	o.Lock()
	user, ok := o.users[token.AccessToken]
	o.Unlock()

	if ok {
		return user, nil
	}

	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: token.AccessToken},
	}

	c := github.NewClient(t.Client())
	guser, _, err := c.Users.Get("")
	if err != nil {
		return nil, err
	}

	if err := o.isValidUser(c, guser); err != nil {
		return nil, err
	}

	user = &User{}
	if guser != nil && guser.Name != nil {
		user.Fullname = *guser.Name
	} else if guser.Login != nil {
		user.Fullname = *guser.Login
	}

	if guser != nil && guser.AvatarURL != nil {
		user.Avatar = *guser.AvatarURL
	}

	o.Lock()
	o.users[token.AccessToken] = user
	o.Unlock()

	return user, nil
}

func (o *OAuth) isValidUser(c *github.Client, u *github.User) error {
	if err := o.validateGithubOrganization(c, u); err != nil {
		return err
	}

	if err := o.validateGithubUser(c, u); err != nil {
		return err
	}

	return nil
}

func (o *OAuth) validateGithubOrganization(c *github.Client, u *github.User) error {
	org := o.Config.HTTP.GithubOrganization
	if org == "" {
		return nil
	}

	m, _, err := c.Organizations.IsMember(org, *u.Login)
	if err != nil {
		return err
	}

	if !m {
		return errors.New(fmt.Sprintf(
			"User %q should be member of %q", *u.Login, org,
		))
	}

	return nil
}

func (o *OAuth) validateGithubUser(c *github.Client, u *github.User) error {
	if len(o.Config.HTTP.GithubUsers) == 0 {
		return nil
	}

	for _, user := range o.Config.HTTP.GithubUsers {
		if user == *u.Login {
			return nil
		}
	}

	return errors.New(fmt.Sprintf(
		"User %q not allowed, not in the access list.", *u.Login,
	))
}

func extractPath(next string) string {
	n, err := url.Parse(next)
	if err != nil {
		return "/"
	}
	return n.Path
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.4.0"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	y      = color.New(color.FgHiYellow)
	g      = color.New(color.FgHiGreen)
	r      = color.New(color.FgHiRed)
	b      = color.New(color.FgHiBlue)
	fgR    = color.New(color.FgRed)
	fgY    = color.New(color.FgYellow)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.Netblock.String()]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(color.Error, chr)
		}
	}

	fmt.Fprintln(color.Error)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(color.Error, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(color.Error, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(color.Error, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(color.Error, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(color.Error, ", ")
		}
		num++
	}
	fmt.Fprintln(color.Error)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(color.Error)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}

		fmt.Fprintf(color.Error, "%s%s %s %s\n",
			blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)

			fmt.Fprintf(color.Error, "%s%s %s\n",
				yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(color.Error, " ")
		}
	}
	r.Fprintln(color.Error, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(color.Error, Version)
	pad(rightmost - len(Author))
	y.Fprintln(color.Error, Author)
	pad(rightmost - len(Description))
	y.Fprintf(color.Error, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Source+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if net.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if net.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}
<commit_msg>v3.4.1 release<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.4.1"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	y      = color.New(color.FgHiYellow)
	g      = color.New(color.FgHiGreen)
	r      = color.New(color.FgHiRed)
	b      = color.New(color.FgHiBlue)
	fgR    = color.New(color.FgRed)
	fgY    = color.New(color.FgYellow)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.Netblock.String()]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(color.Error, chr)
		}
	}

	fmt.Fprintln(color.Error)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(color.Error, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(color.Error, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(color.Error, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(color.Error, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(color.Error, ", ")
		}
		num++
	}
	fmt.Fprintln(color.Error)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(color.Error)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}

		fmt.Fprintf(color.Error, "%s%s %s %s\n",
			blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)

			fmt.Fprintf(color.Error, "%s%s %s\n",
				yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(color.Error, " ")
		}
	}
	r.Fprintln(color.Error, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(color.Error, Version)
	pad(rightmost - len(Author))
	y.Fprintln(color.Error, Author)
	pad(rightmost - len(Description))
	y.Fprintf(color.Error, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Source+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if net.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if net.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}
<|endoftext|>


<commit_before>package xorm_ext

import (
	"reflect"

	. "github.com/gogap/xorm_ext/errorcode"
)

type TransactionCommiter interface {
	Transaction(txFunc TXFunc, repos ...interface{}) (err error)
	TransactionUsing(txFunc TXFunc, name string, repos ...interface{}) (err error)
	NoTransaction(txFunc TXFunc, repos ...interface{}) (err error)
	NoTransactionUsing(txFunc TXFunc, name string, repos ...interface{}) (err error)
}

type DBTXCommiter struct {
}

func (p *DBTXCommiter) Transaction(txFunc TXFunc, originRepos ...interface{}) (err error) {
	return p.TransactionUsing(txFunc, REPO_DEFAULT_ENGINE, originRepos...)
}
func (p *DBTXCommiter) TransactionUsing(txFunc TXFunc, name string, originRepos ...interface{}) (err error) {
	reposLen := 0
	if originRepos != nil {
		reposLen = len(originRepos)
	}

	if reposLen < 1 {
		err = ERR_DB_ONE_REPO_AT_LEAST.New()
		return
	}

	newRepos := []interface{}{}
	newDBRepos := []*DBRepo{}

	for _, originRepo := range originRepos {
		iRepo := reflect.Indirect(reflect.ValueOf(originRepo))

		dbRepo := getRepo(originRepo)

		if dbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		vRepo := reflect.ValueOf(iRepo.Interface())
		newRepoV := reflect.New(vRepo.Type())
		if !newRepoV.IsValid() {
			err = ERR_CAN_NOT_REFLACT_NEW_REPO.New()
			return
		}

		newRepoI := newRepoV.Interface()
		newDbRepo := getRepo(newRepoI)

		if newDbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		newDbRepo.engines = dbRepo.engines
		newDbRepo.defaultEngine = dbRepo.defaultEngine
		newRepos = append(newRepos, newRepoI)
		newDBRepos = append(newDBRepos, newDbRepo)
	}

	if e := newDBRepos[0].beginTransaction(name); e != nil {
		return ERR_DB_TX_CANNOT_BEGIN.New()
	}

	if reposLen > 1 {
		for i := 1; i < reposLen; i++ {
			newDBRepos[i].txSession = newDBRepos[0].txSession
			newDBRepos[i].isTransaction = newDBRepos[0].isTransaction
		}
	}

	return newDBRepos[0].commitTransaction(txFunc, newRepos...)
}

func (p *DBTXCommiter) NoTransaction(txFunc TXFunc, originRepos ...interface{}) (err error) {
	return p.NoTransactionUsing(txFunc, REPO_DEFAULT_ENGINE, originRepos...)
}
func (p *DBTXCommiter) NoTransactionUsing(txFunc TXFunc, name string, originRepos ...interface{}) (err error) {
	reposLen := 0
	if originRepos != nil {
		reposLen = len(originRepos)
	}

	if reposLen < 1 {
		err = ERR_DB_ONE_REPO_AT_LEAST.New()
		return
	}

	newRepos := []interface{}{}
	newDBRepos := []*DBRepo{}

	for _, originRepo := range originRepos {
		iRepo := reflect.Indirect(reflect.ValueOf(originRepo))

		dbRepo := getRepo(originRepo)

		if dbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		vRepo := reflect.ValueOf(iRepo.Interface())
		newRepoV := reflect.New(vRepo.Type())
		if !newRepoV.IsValid() {
			err = ERR_CAN_NOT_REFLACT_NEW_REPO.New()
			return
		}

		newRepoI := newRepoV.Interface()
		newDbRepo := getRepo(newRepoI)

		if newDbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		newDBRepos = append(newDBRepos, newDbRepo)

		newDbRepo.engines = dbRepo.engines
		newDbRepo.defaultEngine = dbRepo.defaultEngine
	}

	return newDBRepos[0].commitNoTransaction(txFunc, name, newRepos...)
}

func getRepo(v interface{}) *DBRepo {
	values := []reflect.Value{}
	deepRepo := deepFields(v, reflect.TypeOf(new(DBRepo)), values)
	if deepRepo, ok := deepRepo.(*DBRepo); ok {
		return deepRepo
	}
	return nil
}

func deepFields(iface interface{}, vType reflect.Type, fields []reflect.Value) interface{} {
	ifv := reflect.ValueOf(iface)
	ift := reflect.TypeOf(iface)

	if ift == vType {

		return iface
	}

	if ifv.Kind() == reflect.Ptr {
		ifv = ifv.Elem()
		ift = ifv.Type()
	}

	for i := 0; i < ift.NumField(); i++ {
		v := ifv.Field(i)
		switch v.Kind() {
		case reflect.Struct:
			var deepIns interface{}
			if v.CanAddr() {
				deepIns = deepFields(v.Addr().Interface(), vType, fields)
			} else {
				deepIns = deepFields(v.Interface(), vType, fields)
			}

			if deepIns != nil {
				return deepIns
			}
		}
	}
	return nil
}
<commit_msg>fix issure of repos<commit_after>package xorm_ext

import (
	"reflect"

	. "github.com/gogap/xorm_ext/errorcode"
)

type TransactionCommiter interface {
	Transaction(txFunc TXFunc, repos ...interface{}) (err error)
	TransactionUsing(txFunc TXFunc, name string, repos ...interface{}) (err error)
	NoTransaction(txFunc TXFunc, repos ...interface{}) (err error)
	NoTransactionUsing(txFunc TXFunc, name string, repos ...interface{}) (err error)
}

type DBTXCommiter struct {
}

func (p *DBTXCommiter) Transaction(txFunc TXFunc, originRepos ...interface{}) (err error) {
	return p.TransactionUsing(txFunc, REPO_DEFAULT_ENGINE, originRepos...)
}
func (p *DBTXCommiter) TransactionUsing(txFunc TXFunc, name string, originRepos ...interface{}) (err error) {
	reposLen := 0
	if originRepos != nil {
		reposLen = len(originRepos)
	}

	if reposLen < 1 {
		err = ERR_DB_ONE_REPO_AT_LEAST.New()
		return
	}

	newRepos := []interface{}{}
	newDBRepos := []*DBRepo{}

	for _, originRepo := range originRepos {
		iRepo := reflect.Indirect(reflect.ValueOf(originRepo))

		dbRepo := getRepo(originRepo)

		if dbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		vRepo := reflect.ValueOf(iRepo.Interface())
		newRepoV := reflect.New(vRepo.Type())
		if !newRepoV.IsValid() {
			err = ERR_CAN_NOT_REFLACT_NEW_REPO.New()
			return
		}

		newRepoI := newRepoV.Interface()
		newDbRepo := getRepo(newRepoI)

		if newDbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		newDbRepo.engines = dbRepo.engines
		newDbRepo.defaultEngine = dbRepo.defaultEngine
		newRepos = append(newRepos, newRepoI)
		newDBRepos = append(newDBRepos, newDbRepo)
	}

	if e := newDBRepos[0].beginTransaction(name); e != nil {
		return ERR_DB_TX_CANNOT_BEGIN.New()
	}

	if reposLen > 1 {
		for i := 1; i < reposLen; i++ {
			newDBRepos[i].txSession = newDBRepos[0].txSession
			newDBRepos[i].isTransaction = newDBRepos[0].isTransaction
		}
	}

	return newDBRepos[0].commitTransaction(txFunc, newRepos...)
}

func (p *DBTXCommiter) NoTransaction(txFunc TXFunc, originRepos ...interface{}) (err error) {
	return p.NoTransactionUsing(txFunc, REPO_DEFAULT_ENGINE, originRepos...)
}
func (p *DBTXCommiter) NoTransactionUsing(txFunc TXFunc, name string, originRepos ...interface{}) (err error) {
	reposLen := 0
	if originRepos != nil {
		reposLen = len(originRepos)
	}

	if reposLen < 1 {
		err = ERR_DB_ONE_REPO_AT_LEAST.New()
		return
	}

	newRepos := []interface{}{}
	newDBRepos := []*DBRepo{}

	for _, originRepo := range originRepos {
		iRepo := reflect.Indirect(reflect.ValueOf(originRepo))

		dbRepo := getRepo(originRepo)

		if dbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		vRepo := reflect.ValueOf(iRepo.Interface())
		newRepoV := reflect.New(vRepo.Type())
		if !newRepoV.IsValid() {
			err = ERR_CAN_NOT_REFLACT_NEW_REPO.New()
			return
		}

		newRepoI := newRepoV.Interface()
		newDbRepo := getRepo(newRepoI)

		if newDbRepo == nil {
			err = ERR_STRUCT_NOT_COMBINE_WITH_DBREPO.New()
			return
		}

		newDbRepo.engines = dbRepo.engines
		newDbRepo.defaultEngine = dbRepo.defaultEngine
		newRepos = append(newRepos, newRepoI)
		newDBRepos = append(newDBRepos, newDbRepo)
	}

	return newDBRepos[0].commitNoTransaction(txFunc, name, newRepos...)
}

func getRepo(v interface{}) *DBRepo {
	values := []reflect.Value{}
	deepRepo := deepFields(v, reflect.TypeOf(new(DBRepo)), values)
	if deepRepo, ok := deepRepo.(*DBRepo); ok {
		return deepRepo
	}
	return nil
}

func deepFields(iface interface{}, vType reflect.Type, fields []reflect.Value) interface{} {
	ifv := reflect.ValueOf(iface)
	ift := reflect.TypeOf(iface)

	if ift == vType {

		return iface
	}

	if ifv.Kind() == reflect.Ptr {
		ifv = ifv.Elem()
		ift = ifv.Type()
	}

	for i := 0; i < ift.NumField(); i++ {
		v := ifv.Field(i)
		switch v.Kind() {
		case reflect.Struct:
			var deepIns interface{}
			if v.CanAddr() {
				deepIns = deepFields(v.Addr().Interface(), vType, fields)
			} else {
				deepIns = deepFields(v.Interface(), vType, fields)
			}

			if deepIns != nil {
				return deepIns
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venafi

import (
	"context"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/util/errors"
	vaddon "github.com/jetstack/cert-manager/test/e2e/suite/issuers/venafi/addon"
)

var _ = framework.ConformanceDescribe("Certificates", func() {
	//// unsupportedFeatures is a list of features that are not supported by the
	//// Venafi issuer.
	//var unsupportedFeatures = certificates.NewFeatureSet(
	//	certificates.DurationFeature,
	//	// Due to the current configuration of the test environment, it does not
	//	// support signing certificates that pair with an elliptic curve private
	//	// key or using the same private key multiple times.
	//	certificates.ECDSAFeature,
	//	certificates.ReusePrivateKeyFeature,
	//)
	//
	//provisioner := new(venafiProvisioner)
	//(&certificates.Suite{
	//	Name:                "Venafi Issuer",
	//	CreateIssuerFunc:    provisioner.createIssuer,
	//	DeleteIssuerFunc:    provisioner.delete,
	//	UnsupportedFeatures: unsupportedFeatures,
	//}).Define()

	//(&certificates.Suite{
	//	Name:                "Venafi ClusterIssuer",
	//	CreateIssuerFunc:    provisioner.createClusterIssuer,
	//	DeleteIssuerFunc:    provisioner.delete,
	//	UnsupportedFeatures: unsupportedFeatures,
	//}).Define()
})

type venafiProvisioner struct {
	tpp *vaddon.VenafiTPP
}

func (v *venafiProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	Expect(v.tpp.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision tpp venafi")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Delete(context.TODO(), ref.Name, metav1.DeleteOptions{})
		Expect(err).NotTo(HaveOccurred())
	}
}

func (v *venafiProvisioner) createIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi Issuer")

	v.tpp = &vaddon.VenafiTPP{
		Namespace: f.Namespace.Name,
	}

	err := v.tpp.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.tpp.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.tpp.Details().BuildIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (v *venafiProvisioner) createClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi ClusterIssuer")

	v.tpp = &vaddon.VenafiTPP{
		Namespace: f.Config.Addons.CertManager.ClusterResourceNamespace,
	}

	err := v.tpp.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.tpp.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.tpp.Details().BuildClusterIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}
<commit_msg>Re-add conformance tests<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venafi

import (
	"context"

	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/util/errors"
	vaddon "github.com/jetstack/cert-manager/test/e2e/suite/issuers/venafi/addon"
)

var _ = framework.ConformanceDescribe("Certificates", func() {
	// unsupportedFeatures is a list of features that are not supported by the
	// Venafi issuer.
	var unsupportedFeatures = certificates.NewFeatureSet(
		certificates.DurationFeature,
		// Due to the current configuration of the test environment, it does not
		// support signing certificates that pair with an elliptic curve private
		// key or using the same private key multiple times.
		certificates.ECDSAFeature,
		certificates.ReusePrivateKeyFeature,
	)

	provisioner := new(venafiProvisioner)
	(&certificates.Suite{
		Name:                "Venafi Issuer",
		CreateIssuerFunc:    provisioner.createIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()

	(&certificates.Suite{
		Name:                "Venafi ClusterIssuer",
		CreateIssuerFunc:    provisioner.createClusterIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()
})

type venafiProvisioner struct {
	tpp *vaddon.VenafiTPP
}

func (v *venafiProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	Expect(v.tpp.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision tpp venafi")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Delete(context.TODO(), ref.Name, metav1.DeleteOptions{})
		Expect(err).NotTo(HaveOccurred())
	}
}

func (v *venafiProvisioner) createIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi Issuer")

	v.tpp = &vaddon.VenafiTPP{
		Namespace: f.Namespace.Name,
	}

	err := v.tpp.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.tpp.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.tpp.Details().BuildIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (v *venafiProvisioner) createClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi ClusterIssuer")

	v.tpp = &vaddon.VenafiTPP{
		Namespace: f.Config.Addons.CertManager.ClusterResourceNamespace,
	}

	err := v.tpp.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.tpp.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.tpp.Details().BuildClusterIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}
<|endoftext|>


<commit_before>package blx

const (
	PayloadTypeTransferBlock string = "transfer-block"
	PayloadTypeRequestBlock         = "request-block"
)

const (
	StatusOK = iota
	StatusNotFound
)

type Block struct {
	Key  string            `json:"key"`
	Meta map[string][]byte `json:"meta,omitempty"`
	Data []byte            `json:"data"`
}

type payloadTransferBlock struct {
	Status int    `json: status`
	Nonce  string `json: nonce`
	Block  *Block `json:"block,omitempty"`
}

type payloadTransferRequestBlock struct {
	Nonce    string `json: nonce`
	Key      string `json:"key,omitempty"`
	response chan interface{}
}
<commit_msg>Fix blx json tags<commit_after>package blx

const (
	PayloadTypeTransferBlock string = "transfer-block"
	PayloadTypeRequestBlock         = "request-block"
)

const (
	StatusOK = iota
	StatusNotFound
)

type Block struct {
	Key  string            `json:"key"`
	Meta map[string][]byte `json:"meta,omitempty"`
	Data []byte            `json:"data"`
}

type payloadTransferBlock struct {
	Status int    `json:"status"`
	Nonce  string `json:"nonce"`
	Block  *Block `json:"block,omitempty"`
}

type payloadTransferRequestBlock struct {
	Nonce    string `json:"nonce"`
	Key      string `json:"key,omitempty"`
	response chan interface{}
}
<|endoftext|>


<commit_before>package labmeasure

type BodyStat struct {
	Examined         int
	BothEmpty        int
	DiffbotEmpty     int
	LabEmpty         int
	Qualified        int
	Acceptable       int
	Unacceptable     int
	IncorrectRecords []PBodyRecord
	TotalRecall      float32
	TotalPrecision   float32
	Configuration    Config
}

func (st BodyStat) GetIncorrectRecords() interface{} {
	return st.IncorrectRecords
}

func (st BodyStat) Correct() int {
	return st.Acceptable + st.BothEmpty
}

func (st BodyStat) Incorrect() int {
	return st.Unacceptable + st.DiffbotEmpty + st.LabEmpty
}

func (st BodyStat) Accuracy() float32 {
	return float32(st.Correct()) / float32(st.Examined)
}

func (st BodyStat) AverageRecall() float32 {
	return float32(st.TotalRecall) / float32(st.Examined)
}

func (st BodyStat) AveragePrecision() float32 {
	return float32(st.TotalPrecision) / float32(st.Examined)
}

func (st BodyStat) PrecisionThreshold() float32 {
	return st.Configuration.PrecisionThreshold
}

func (st BodyStat) RecallThreshold() float32 {
	return st.Configuration.RecallThreshold
}
<commit_msg>calculating body average precision and recall on qualified<commit_after>package labmeasure

type BodyStat struct {
	Examined         int
	BothEmpty        int
	DiffbotEmpty     int
	LabEmpty         int
	Qualified        int
	Acceptable       int
	Unacceptable     int
	IncorrectRecords []PBodyRecord
	TotalRecall      float32
	TotalPrecision   float32
	Configuration    Config
}

func (st BodyStat) GetIncorrectRecords() interface{} {
	return st.IncorrectRecords
}

func (st BodyStat) Correct() int {
	return st.Acceptable + st.BothEmpty
}

func (st BodyStat) Incorrect() int {
	return st.Unacceptable + st.DiffbotEmpty + st.LabEmpty
}

func (st BodyStat) Accuracy() float32 {
	return float32(st.Correct()) / float32(st.Examined)
}

func (st BodyStat) AverageRecall() float32 {
	return float32(st.TotalRecall) / float32(st.Qualified)
}

func (st BodyStat) AveragePrecision() float32 {
	return float32(st.TotalPrecision) / float32(st.Qualified)
}

func (st BodyStat) PrecisionThreshold() float32 {
	return st.Configuration.PrecisionThreshold
}

func (st BodyStat) RecallThreshold() float32 {
	return st.Configuration.RecallThreshold
}
<|endoftext|>