<commit_before>package gotiny

import (
	"encoding"
	"encoding/gob"
	"reflect"
	"sync"
	"unsafe"
)

type (
	encEng    func(*Encoder, unsafe.Pointer)  //编码器
	encEngPtr *func(*Encoder, unsafe.Pointer) //编码器指针
)

var (
	rt2encEng = map[reflect.Type]encEngPtr{
		reflect.TypeOf((*bool)(nil)).Elem():           &encBool,
		reflect.TypeOf((*int)(nil)).Elem():            &encInt,
		reflect.TypeOf((*int8)(nil)).Elem():           &encInt8,
		reflect.TypeOf((*int16)(nil)).Elem():          &encInt16,
		reflect.TypeOf((*int32)(nil)).Elem():          &encInt32,
		reflect.TypeOf((*int64)(nil)).Elem():          &encInt64,
		reflect.TypeOf((*uint)(nil)).Elem():           &encUint,
		reflect.TypeOf((*uint8)(nil)).Elem():          &encUint8,
		reflect.TypeOf((*uint16)(nil)).Elem():         &encUint16,
		reflect.TypeOf((*uint32)(nil)).Elem():         &encUint32,
		reflect.TypeOf((*uint64)(nil)).Elem():         &encUint64,
		reflect.TypeOf((*uintptr)(nil)).Elem():        &encUintptr,
		reflect.TypeOf((*unsafe.Pointer)(nil)).Elem(): &encPointer,
		reflect.TypeOf((*float32)(nil)).Elem():        &encFloat32,
		reflect.TypeOf((*float64)(nil)).Elem():        &encFloat64,
		reflect.TypeOf((*complex64)(nil)).Elem():      &encComplex64,
		reflect.TypeOf((*complex128)(nil)).Elem():     &encComplex128,
		reflect.TypeOf((*[]byte)(nil)).Elem():         &encBytes,
		reflect.TypeOf((*string)(nil)).Elem():         &encString,
		reflect.TypeOf((*struct{})(nil)).Elem():       &encignore,
		reflect.TypeOf(nil):                           &encignore,
	}
	eengs = [...]encEng{
		reflect.Invalid:       encignore,
		reflect.Bool:          encBool,
		reflect.Int:           encInt,
		reflect.Int8:          encInt8,
		reflect.Int16:         encInt16,
		reflect.Int32:         encInt32,
		reflect.Int64:         encInt64,
		reflect.Uint:          encUint,
		reflect.Uint8:         encUint8,
		reflect.Uint16:        encUint16,
		reflect.Uint32:        encUint32,
		reflect.Uint64:        encUint64,
		reflect.Uintptr:       encUintptr,
		reflect.UnsafePointer: encPointer,
		reflect.Float32:       encFloat32,
		reflect.Float64:       encFloat64,
		reflect.Complex64:     encComplex64,
		reflect.Complex128:    encComplex128,
		reflect.String:        encString,
	}

	encLock sync.RWMutex

	interTypes []reflect.Type
	interRTMap map[reflect.Type]int = map[reflect.Type]int{}
	interNames []string             = []string{}
)

func getEncEngine(rt reflect.Type) encEng {
	encLock.RLock()
	engptr := rt2encEng[rt]
	encLock.RUnlock()
	if engptr != nil && *engptr != nil {
		return *engptr
	}
	encLock.Lock()
	engptr = buildEncEngine(rt)
	encLock.Unlock()
	return *engptr
}

func buildEncEngine(rt reflect.Type) encEngPtr {
	//TODO  接口类型处理
	engine, has := rt2encEng[rt]
	if has {
		return engine
	}
	engine = new(func(*Encoder, unsafe.Pointer))
	rt2encEng[rt] = engine

	if reflect.PtrTo(rt).Implements(gobType) {
		*engine = func(e *Encoder, p unsafe.Pointer) {
			buf, err := reflect.NewAt(rt, p).Interface().(gob.GobEncoder).GobEncode()
			if err != nil {
				panic(err)
			}
			e.encLength(len(buf))
			e.buf = append(e.buf, buf...)
		}
		return engine
	}

	if reflect.PtrTo(rt).Implements(binType) {
		*engine = func(e *Encoder, p unsafe.Pointer) {
			buf, err := reflect.NewAt(rt, p).Interface().(encoding.BinaryMarshaler).MarshalBinary()
			if err != nil {
				panic(err)
			}
			e.encLength(len(buf))
			e.buf = append(e.buf, buf...)
		}
		return engine
	}

	if reflect.PtrTo(rt).Implements(tinyType) {
		*engine = func(e *Encoder, p unsafe.Pointer) {
			e.buf = reflect.NewAt(rt, p).Interface().(GoTinySerializer).GotinyEncode(e.buf)
		}
		return engine
	}

	kind := rt.Kind()
	switch kind {
	case reflect.Ptr:
		eEng := buildEncEngine(rt.Elem())
		*engine = func(e *Encoder, p unsafe.Pointer) {
			isNotNil := !isNil(p)
			e.encBool(isNotNil)
			if isNotNil {
				(*eEng)(e, *(*unsafe.Pointer)(p))
			}
		}
	case reflect.Array:
		et, l := rt.Elem(), rt.Len()
		eEng := buildEncEngine(et)
		size := et.Size()
		*engine = func(e *Encoder, p unsafe.Pointer) {
			for i := 0; i < l; i++ {
				(*eEng)(e, unsafe.Pointer(uintptr(p)+uintptr(i)*size))
			}
		}
	case reflect.Slice:
		et := rt.Elem()
		eEng, size := buildEncEngine(et), et.Size()
		*engine = func(e *Encoder, p unsafe.Pointer) {
			isNotNil := !isNil(p)
			e.encBool(isNotNil)
			if isNotNil {
				header := (*sliceHeader)(p)
				l := header.len
				e.encLength(l)
				for i := uintptr(0); i < uintptr(l); i++ {
					(*eEng)(e, unsafe.Pointer(uintptr(header.data)+i*size))
				}
			}
		}
	case reflect.Map:
		eKey, eEng := buildEncEngine(rt.Key()), buildEncEngine(rt.Elem())
		*engine = func(e *Encoder, p unsafe.Pointer) {
			isNotNil := !isNil(p)
			e.encBool(isNotNil)
			if isNotNil {
				e.encLength(*(*int)(*(*unsafe.Pointer)(p)))
				v := reflect.NewAt(rt, p).Elem()
				// TODO flag&flagIndir 在编译时确定
				for _, key := range v.MapKeys() {
					val := v.MapIndex(key)
					kv, vv := (*refVal)(unsafe.Pointer(&key)), (*refVal)(unsafe.Pointer(&val))
					kp, vp := kv.ptr, vv.ptr
					if kv.flag&flagIndir == 0 {
						kp = unsafe.Pointer(&kv.ptr)
					}
					if vv.flag&flagIndir == 0 {
						vp = unsafe.Pointer(&vv.ptr)
					}
					(*eKey)(e, kp)
					(*eEng)(e, vp)
				}
			}
		}
	case reflect.Struct:
		nf := rt.NumField()
		engs, offs := make([]encEngPtr, nf), make([]uintptr, nf)
		for i := 0; i < nf; i++ {
			field := rt.Field(i)
			engs[i] = buildEncEngine(field.Type)
			offs[i] = field.Offset
		}
		*engine = func(e *Encoder, p unsafe.Pointer) {
			for i := 0; i < nf; i++ {
				(*engs[i])(e, unsafe.Pointer(uintptr(p)+offs[i]))
			}
		}
	case reflect.Interface:
		if rt.NumMethod() > 0 {
			*engine = func(e *Encoder, p unsafe.Pointer) {
				isNotNil := !isNil(p)
				e.encBool(isNotNil)
				if isNotNil {
					v := reflect.ValueOf(*(*interface {
						M()
					})(p))
					et := v.Type()
					e.encLength(getRTID(et))
					eEng := buildEncEngine(et)
					vv := (*refVal)(unsafe.Pointer(&v))
					vp := vv.ptr
					if vv.flag&flagIndir == 0 {
						vp = unsafe.Pointer(&vv.ptr)
					}
					(*eEng)(e, vp)
				}
			}
		} else {
			*engine = func(e *Encoder, p unsafe.Pointer) {
				isNotNil := !isNil(p)
				e.encBool(isNotNil)
				if isNotNil {
					v := reflect.ValueOf(*(*interface{})(p))
					et := v.Type()
					e.encLength(getRTID(et))
					eEng := buildEncEngine(et)
					vv := (*refVal)(unsafe.Pointer(&v))
					vp := vv.ptr
					if vv.flag&flagIndir == 0 {
						vp = unsafe.Pointer(&vv.ptr)
					}
					(*eEng)(e, vp)
				}
			}
		}
	case reflect.Chan, reflect.Func:
		panic("not support " + rt.String() + " type")
	default:
		*engine = eengs[kind]
	}
	return engine
}

func getRTID(rt reflect.Type) int {
	if id, has := interRTMap[rt]; has {
		return id
	} else {
		id = len(interTypes)
		interTypes = append(interTypes, rt)
		interRTMap[rt] = id
		return id
	}
}

func Register(i interface{}) {
	register(reflect.TypeOf(i))
}

func register(rt reflect.Type) int {
	if id, has := interRTMap[rt]; has {
		return id
	} else {
		name := getName(rt)
		i := len(interNames)
		interNames = append(interNames, "")
		interTypes = append(interTypes, reflect.Type{})
		for i > 0 {
			if interNames[i-1] > name {
				interTypes[i] = interTypes[i-1]
				interNames[i] = interNames[i-1]
				interRTMap[interTypes[i]] = i
			}
			i--
		}
		interNames[i] = name
		interTypes[i] = rt
		interRTMap[rt] = i
		return i
	}
}

func getName(rt reflect.Type) string {

	name := rt.String()

	// But for named types (or pointers to them), qualify with import path (but see inner comment).
	// Dereference one pointer looking for a named type.
	star := ""
	if rt.Name() == "" {
		if rt.Kind() == reflect.Ptr {
			star = "*"
			rt = rt.Elem()
		} else {
			panic("not support no named type " + name)
		}
	}
	if rt.Name() != "" {
		if rt.PkgPath() == "" {
			name = star + rt.Name()
		} else {
			name = star + rt.PkgPath() + "." + rt.Name()
		}
	}
	return name
}
<commit_msg>fix bug<commit_after>package gotiny

import (
	"encoding"
	"encoding/gob"
	"reflect"
	"sync"
	"unsafe"
)

type (
	encEng    func(*Encoder, unsafe.Pointer)  //编码器
	encEngPtr *func(*Encoder, unsafe.Pointer) //编码器指针
)

var (
	rt2encEng = map[reflect.Type]encEngPtr{
		reflect.TypeOf((*bool)(nil)).Elem():           &encBool,
		reflect.TypeOf((*int)(nil)).Elem():            &encInt,
		reflect.TypeOf((*int8)(nil)).Elem():           &encInt8,
		reflect.TypeOf((*int16)(nil)).Elem():          &encInt16,
		reflect.TypeOf((*int32)(nil)).Elem():          &encInt32,
		reflect.TypeOf((*int64)(nil)).Elem():          &encInt64,
		reflect.TypeOf((*uint)(nil)).Elem():           &encUint,
		reflect.TypeOf((*uint8)(nil)).Elem():          &encUint8,
		reflect.TypeOf((*uint16)(nil)).Elem():         &encUint16,
		reflect.TypeOf((*uint32)(nil)).Elem():         &encUint32,
		reflect.TypeOf((*uint64)(nil)).Elem():         &encUint64,
		reflect.TypeOf((*uintptr)(nil)).Elem():        &encUintptr,
		reflect.TypeOf((*unsafe.Pointer)(nil)).Elem(): &encPointer,
		reflect.TypeOf((*float32)(nil)).Elem():        &encFloat32,
		reflect.TypeOf((*float64)(nil)).Elem():        &encFloat64,
		reflect.TypeOf((*complex64)(nil)).Elem():      &encComplex64,
		reflect.TypeOf((*complex128)(nil)).Elem():     &encComplex128,
		reflect.TypeOf((*[]byte)(nil)).Elem():         &encBytes,
		reflect.TypeOf((*string)(nil)).Elem():         &encString,
		reflect.TypeOf((*struct{})(nil)).Elem():       &encignore,
		reflect.TypeOf(nil):                           &encignore,
	}
	eengs = [...]encEng{
		reflect.Invalid:       encignore,
		reflect.Bool:          encBool,
		reflect.Int:           encInt,
		reflect.Int8:          encInt8,
		reflect.Int16:         encInt16,
		reflect.Int32:         encInt32,
		reflect.Int64:         encInt64,
		reflect.Uint:          encUint,
		reflect.Uint8:         encUint8,
		reflect.Uint16:        encUint16,
		reflect.Uint32:        encUint32,
		reflect.Uint64:        encUint64,
		reflect.Uintptr:       encUintptr,
		reflect.UnsafePointer: encPointer,
		reflect.Float32:       encFloat32,
		reflect.Float64:       encFloat64,
		reflect.Complex64:     encComplex64,
		reflect.Complex128:    encComplex128,
		reflect.String:        encString,
	}

	encLock sync.RWMutex

	interTypes []reflect.Type
	interRTMap map[reflect.Type]int = map[reflect.Type]int{}
	interNames []string             = []string{}
)

func getEncEngine(rt reflect.Type) encEng {
	encLock.RLock()
	engptr := rt2encEng[rt]
	encLock.RUnlock()
	if engptr != nil && *engptr != nil {
		return *engptr
	}
	encLock.Lock()
	engptr = buildEncEngine(rt)
	encLock.Unlock()
	return *engptr
}

func buildEncEngine(rt reflect.Type) encEngPtr {
	//TODO  接口类型处理
	engine, has := rt2encEng[rt]
	if has {
		return engine
	}
	engine = new(func(*Encoder, unsafe.Pointer))
	rt2encEng[rt] = engine

	if reflect.PtrTo(rt).Implements(gobType) {
		*engine = func(e *Encoder, p unsafe.Pointer) {
			buf, err := reflect.NewAt(rt, p).Interface().(gob.GobEncoder).GobEncode()
			if err != nil {
				panic(err)
			}
			e.encLength(len(buf))
			e.buf = append(e.buf, buf...)
		}
		return engine
	}

	if reflect.PtrTo(rt).Implements(binType) {
		*engine = func(e *Encoder, p unsafe.Pointer) {
			buf, err := reflect.NewAt(rt, p).Interface().(encoding.BinaryMarshaler).MarshalBinary()
			if err != nil {
				panic(err)
			}
			e.encLength(len(buf))
			e.buf = append(e.buf, buf...)
		}
		return engine
	}

	if reflect.PtrTo(rt).Implements(tinyType) {
		*engine = func(e *Encoder, p unsafe.Pointer) {
			e.buf = reflect.NewAt(rt, p).Interface().(GoTinySerializer).GotinyEncode(e.buf)
		}
		return engine
	}

	kind := rt.Kind()
	switch kind {
	case reflect.Ptr:
		eEng := buildEncEngine(rt.Elem())
		*engine = func(e *Encoder, p unsafe.Pointer) {
			isNotNil := !isNil(p)
			e.encBool(isNotNil)
			if isNotNil {
				(*eEng)(e, *(*unsafe.Pointer)(p))
			}
		}
	case reflect.Array:
		et, l := rt.Elem(), rt.Len()
		eEng := buildEncEngine(et)
		size := et.Size()
		*engine = func(e *Encoder, p unsafe.Pointer) {
			for i := 0; i < l; i++ {
				(*eEng)(e, unsafe.Pointer(uintptr(p)+uintptr(i)*size))
			}
		}
	case reflect.Slice:
		et := rt.Elem()
		eEng, size := buildEncEngine(et), et.Size()
		*engine = func(e *Encoder, p unsafe.Pointer) {
			isNotNil := !isNil(p)
			e.encBool(isNotNil)
			if isNotNil {
				header := (*sliceHeader)(p)
				l := header.len
				e.encLength(l)
				for i := uintptr(0); i < uintptr(l); i++ {
					(*eEng)(e, unsafe.Pointer(uintptr(header.data)+i*size))
				}
			}
		}
	case reflect.Map:
		eKey, eEng := buildEncEngine(rt.Key()), buildEncEngine(rt.Elem())
		*engine = func(e *Encoder, p unsafe.Pointer) {
			isNotNil := !isNil(p)
			e.encBool(isNotNil)
			if isNotNil {
				e.encLength(*(*int)(*(*unsafe.Pointer)(p)))
				v := reflect.NewAt(rt, p).Elem()
				// TODO flag&flagIndir 在编译时确定
				for _, key := range v.MapKeys() {
					val := v.MapIndex(key)
					kv, vv := (*refVal)(unsafe.Pointer(&key)), (*refVal)(unsafe.Pointer(&val))
					kp, vp := kv.ptr, vv.ptr
					if kv.flag&flagIndir == 0 {
						kp = unsafe.Pointer(&kv.ptr)
					}
					if vv.flag&flagIndir == 0 {
						vp = unsafe.Pointer(&vv.ptr)
					}
					(*eKey)(e, kp)
					(*eEng)(e, vp)
				}
			}
		}
	case reflect.Struct:
		nf := rt.NumField()
		engs, offs := make([]encEngPtr, nf), make([]uintptr, nf)
		for i := 0; i < nf; i++ {
			field := rt.Field(i)
			engs[i] = buildEncEngine(field.Type)
			offs[i] = field.Offset
		}
		*engine = func(e *Encoder, p unsafe.Pointer) {
			for i := 0; i < nf; i++ {
				(*engs[i])(e, unsafe.Pointer(uintptr(p)+offs[i]))
			}
		}
	case reflect.Interface:
		if rt.NumMethod() > 0 {
			*engine = func(e *Encoder, p unsafe.Pointer) {
				isNotNil := !isNil(p)
				e.encBool(isNotNil)
				if isNotNil {
					v := reflect.ValueOf(*(*interface {
						M()
					})(p))
					et := v.Type()
					e.encLength(getRTID(et))
					eEng := buildEncEngine(et)
					vv := (*refVal)(unsafe.Pointer(&v))
					vp := vv.ptr
					if vv.flag&flagIndir == 0 {
						vp = unsafe.Pointer(&vv.ptr)
					}
					(*eEng)(e, vp)
				}
			}
		} else {
			*engine = func(e *Encoder, p unsafe.Pointer) {
				isNotNil := !isNil(p)
				e.encBool(isNotNil)
				if isNotNil {
					v := reflect.ValueOf(*(*interface{})(p))
					et := v.Type()
					e.encLength(getRTID(et))
					eEng := buildEncEngine(et)
					vv := (*refVal)(unsafe.Pointer(&v))
					vp := vv.ptr
					if vv.flag&flagIndir == 0 {
						vp = unsafe.Pointer(&vv.ptr)
					}
					(*eEng)(e, vp)
				}
			}
		}
	case reflect.Chan, reflect.Func:
		panic("not support " + rt.String() + " type")
	default:
		*engine = eengs[kind]
	}
	return engine
}

func getRTID(rt reflect.Type) int {
	if id, has := interRTMap[rt]; has {
		return id
	} else {
		id = len(interTypes)
		interTypes = append(interTypes, rt)
		interRTMap[rt] = id
		return id
	}
}

func Register(i interface{}) {
	register(reflect.TypeOf(i))
}

func register(rt reflect.Type) int {
	if id, has := interRTMap[rt]; has {
		return id
	} else {
		name := getName(rt)
		i := len(interNames)
		interNames = append(interNames, "")
		interTypes = append(interTypes, nil)
		for i > 0 {
			if interNames[i-1] > name {
				interTypes[i] = interTypes[i-1]
				interNames[i] = interNames[i-1]
				interRTMap[interTypes[i]] = i
			}
			i--
		}
		interNames[i] = name
		interTypes[i] = rt
		interRTMap[rt] = i
		return i
	}
}

func getName(rt reflect.Type) string {

	name := rt.String()

	// But for named types (or pointers to them), qualify with import path (but see inner comment).
	// Dereference one pointer looking for a named type.
	star := ""
	if rt.Name() == "" {
		if rt.Kind() == reflect.Ptr {
			star = "*"
			rt = rt.Elem()
		} else {
			panic("not support no named type " + name)
		}
	}
	if rt.Name() != "" {
		if rt.PkgPath() == "" {
			name = star + rt.Name()
		} else {
			name = star + rt.PkgPath() + "." + rt.Name()
		}
	}
	return name
}
<|endoftext|>


<commit_before>// +build ignore

// Enumerate unique keys from key/values found in the whois responses.
// To use: go run enumerate.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"

	"github.com/domainr/whois"
	"github.com/domainr/whoistest"
	"github.com/wsxiaoys/terminal/color"
)

var (
	keys = make(map[string]string)
)

func main() {
	flag.Parse()
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	fns, err := whoistest.ResponseFiles()
	if err != nil {
		return err
	}
	for _, fn := range fns {
		res, err := whois.ReadMIMEFile(fn)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading response file %s: %s\n", fn, err)
			continue
		}
		if res.MediaType != "text/plain" {
			continue
		}
		scan(res)
	}

	sorted := make([]string, 0, len(keys))
	for k, _ := range keys {
		sorted = append(sorted, k)
	}
	sort.Strings(sorted)

	color.Printf("\n@{|w}%d unique keys parsed:\n", len(keys))
	for _, k := range sorted {
		color.Printf("@{|c}%- 40s  @{|.}%s\n", k, keys[k])
	}

	return nil
}

var (
	reEmptyLine     = regexp.MustCompile(`^\s*$`)
	reKey           = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*$`)
	reKeyValue      = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*(.*\S)\s*$`)
	reAltKey        = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*$`)
	reAltKeyValue   = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*(.*\S)\s*$`)
	reIndentedValue = regexp.MustCompile(`^      \s+(.*\S)\s*$`)
	deNotice        = `^% .*$`
	jpNotice        = `^\[ .+ \]$`
	krNotice        = `^# .*$`
	updated         = `^>>>.+<<<$`
	reNotice        = regexp.MustCompile(
		deNotice + "|" + jpNotice + "|" + krNotice + "|" + updated)
)

func scan(res *whois.Response) {
	r, err := res.Reader()
	if err != nil {
		return
	}
	line := 0
	s := bufio.NewScanner(r)
	for s.Scan() {
		line++
		color.Printf("@{|.}% 4d  ", line)

		// Get next line
		text := s.Text()

		// Notices and empty lines
		if reEmptyLine.MatchString(text) {
			color.Printf("@{|w}EMPTY\n")
			continue
		}
		if m := reNotice.FindStringSubmatch(text); m != nil {
			color.Printf("@{|w}%- 10s  %s\n", "NOTICE", m[0])
			continue
		}

		// Keys and values
		if m := reAltKeyValue.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%- 40s @{w}%s\n", "ALT_KEY_VALUE", m[1], m[2])
			continue
		}
		if m := reAltKey.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%s\n", "ALT_KEY", m[1])
			continue
		}
		if m := reKeyValue.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%- 40s @{w}%s\n", "KEY_VALUE", m[1], m[2])
			continue
		}
		if m := reKey.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%s\n", "KEY", m[1])
			continue
		}
		if m := reIndentedValue.FindStringSubmatch(text); m != nil {
			color.Printf("@{|w}%- 10s  @{c}%- 40s @{w}%s\n", "INDENTED_VALUE", "", m[1])
			continue
		}

		// Unknown
		color.Printf("@{|.}%- 10s  @{|.}%s\n", "UNKNOWN", text)
	}

	fmt.Printf("\n")
}

func addKey(k, host string) {
	if _, ok := keys[k]; !ok {
		keys[k] = host
	} else if !strings.Contains(keys[k], host) {
		keys[k] = keys[k] + "  " + host
	}
}
<commit_msg>Blacklist certain :-delimited strings as not key/value pairs.<commit_after>// +build ignore

// Enumerate unique keys from key/values found in the whois responses.
// To use: go run enumerate.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"

	"github.com/domainr/whois"
	"github.com/domainr/whoistest"
	"github.com/wsxiaoys/terminal/color"
)

var (
	keys = make(map[string]string)
)

func main() {
	flag.Parse()
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	fns, err := whoistest.ResponseFiles()
	if err != nil {
		return err
	}
	for _, fn := range fns {
		res, err := whois.ReadMIMEFile(fn)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading response file %s: %s\n", fn, err)
			continue
		}
		if res.MediaType != "text/plain" {
			continue
		}
		scan(res)
	}

	sorted := make([]string, 0, len(keys))
	for k, _ := range keys {
		sorted = append(sorted, k)
	}
	sort.Strings(sorted)

	color.Printf("\n@{|w}%d unique keys parsed:\n", len(keys))
	for _, k := range sorted {
		color.Printf("@{|c}%- 40s  @{|.}%s\n", k, keys[k])
	}

	return nil
}

var (
	reEmptyLine   = regexp.MustCompile(`^\s*$`)
	reBareKey     = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*$`)
	reKeyValue    = regexp.MustCompile(`^\s*([^\:]*\S)\s*\:\s*(.*\S)\s*$`)
	reAltKey      = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*$`)
	reAltKeyValue = regexp.MustCompile(`^\s*\[([^\]]+)\]\s*(.*\S)\s*$`)
	reBareValue   = regexp.MustCompile(`^      \s+(.*\S)\s*$`)
	reNotice      = regexp.MustCompile(strings.Join([]string{
		`^% .*$`,            // whois.de
		`^\[ .+ \]$`,        // whois.jprs.jp
		`^# .*$`,            // whois.kr
		`^>>>.+<<<$`,        // Database last updated...
		`^[^\:]+https?\://`, // Line with an URL
		`^NOTE: `,
		`^NOTICE: `,
	}, "|"))
)

func scan(res *whois.Response) {
	r, err := res.Reader()
	if err != nil {
		return
	}
	line := 0
	s := bufio.NewScanner(r)
	for s.Scan() {
		line++
		color.Printf("@{|.}% 4d  ", line)

		// Get next line
		text := s.Text()

		// Notices and empty lines
		if reEmptyLine.MatchString(text) {
			color.Printf("@{|w}EMPTY\n")
			continue
		}
		if m := reNotice.FindStringSubmatch(text); m != nil {
			color.Printf("@{|w}%- 10s  %s\n", "NOTICE", m[0])
			continue
		}

		// Keys and values
		if m := reAltKeyValue.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%- 40s @{w}%s\n", "ALT_KEY_VALUE", m[1], m[2])
			continue
		}
		if m := reAltKey.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%s\n", "ALT_KEY", m[1])
			continue
		}
		if m := reKeyValue.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%- 40s @{w}%s\n", "KEY_VALUE", m[1], m[2])
			continue
		}
		if m := reBareKey.FindStringSubmatch(text); m != nil {
			addKey(m[1], res.Host)
			color.Printf("@{|w}%- 10s  @{c}%s\n", "BARE_KEY", m[1])
			continue
		}
		if m := reBareValue.FindStringSubmatch(text); m != nil {
			color.Printf("@{|w}%- 10s  @{c}%- 40s @{w}%s\n", "BARE_VALUE", "", m[1])
			continue
		}

		// Unknown
		color.Printf("@{|.}%- 10s  @{|.}%s\n", "UNKNOWN", text)
	}

	fmt.Printf("\n")
}

func addKey(k, host string) {
	if _, ok := keys[k]; !ok {
		keys[k] = host
	} else if !strings.Contains(keys[k], host) {
		keys[k] = keys[k] + "  " + host
	}
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2021 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"context"
	"io/ioutil"
	"testing"

	apierrs "k8s.io/apimachinery/pkg/api/errors"

	"github.com/tektoncd/operator/pkg/reconciler/kubernetes/tektonresult"
	"github.com/tektoncd/operator/test/client"
	"github.com/tektoncd/operator/test/resources"
	"github.com/tektoncd/operator/test/utils"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// TestTektonResultDeployment verifies the TektonResult creation, deployment recreation, and TektonResult deletion.
func TestTektonResultDeployment(t *testing.T) {
	clients := client.Setup(t)

	crNames := utils.ResourceNames{
		TektonPipeline:  "pipeline",
		TektonResult:    "result",
		TargetNamespace: "tekton-pipelines",
	}

	utils.CleanupOnInterrupt(func() { utils.TearDownPipeline(clients, crNames.TektonPipeline) })
	defer utils.TearDownPipeline(clients, crNames.TektonPipeline)

	utils.CleanupOnInterrupt(func() { utils.TearDownResult(clients, crNames.TektonResult) })
	defer utils.TearDownResult(clients, crNames.TektonTrigger)

	// Create a TektonPipeline
	if _, err := resources.EnsureTektonPipelineExists(clients.TektonPipeline(), crNames); err != nil {
		t.Fatalf("TektonPipeline %q failed to create: %v", crNames.TektonPipeline, err)
	}

	// Test if TektonPipeline can reach the READY status
	t.Run("create-pipeline", func(t *testing.T) {
		resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames)
	})

	// Before Installing Results, create the required secrets
	t.Run("create-secrets", func(t *testing.T) {
		createSecret(t, clients, crNames.TargetNamespace)
	})

	// Create a TektonResult
	if _, err := resources.EnsureTektonResultExists(clients.TektonResult(), crNames); err != nil {
		t.Fatalf("TektonResult %q failed to create: %v", crNames.TektonResult, err)
	}

	// Test if TektonResult can reach the READY status
	t.Run("create-result", func(t *testing.T) {
		resources.AssertTektonResultCRReadyStatus(t, clients, crNames)
	})

	// Delete the TektonResult CR instance to see if all resources will be removed
	t.Run("delete-result", func(t *testing.T) {
		resources.AssertTektonResultCRReadyStatus(t, clients, crNames)
		resources.TektonResultCRDDelete(t, clients, crNames)
	})

	// Delete the TektonPipeline CR instance to see if all resources will be removed
	t.Run("delete-pipeline", func(t *testing.T) {
		resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames)
		resources.TektonPipelineCRDelete(t, clients, crNames)
	})
}

func createSecret(t *testing.T, clients *utils.Clients, namespace string) {

	key, err := ioutil.ReadFile("./testdata/key.pem")
	if err != nil {
		t.Fatal(err)
	}

	cert, err := ioutil.ReadFile("./testdata/cert.pem")
	if err != nil {
		t.Fatal(err)
	}

	tlsSecret := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name: tektonresult.TlsSecretName,
		},
		Data: map[string][]byte{
			corev1.TLSCertKey:       cert,
			corev1.TLSPrivateKeyKey: key,
		},
		Type: corev1.SecretTypeTLS,
	}

	_, err = clients.KubeClient.CoreV1().Secrets(namespace).Create(context.TODO(), tlsSecret, metav1.CreateOptions{})
	if err != nil && !apierrs.IsAlreadyExists(err) {
		t.Fatal(err)
	}

	dbSecret := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name: tektonresult.DbSecretName,
		},
		Data: map[string][]byte{
			"user":     []byte("root"),
			"password": []byte("test"),
		},
	}

	_, err = clients.KubeClient.CoreV1().Secrets(namespace).Create(context.TODO(), dbSecret, metav1.CreateOptions{})
	if err != nil && !apierrs.IsAlreadyExists(err) {
		t.Fatal(err)
	}
}
<commit_msg>Skip Tekton Result tests for Z and Power platforms<commit_after>// +build e2e

/*
Copyright 2021 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"context"
	"io/ioutil"
	"os"
	"testing"

	apierrs "k8s.io/apimachinery/pkg/api/errors"

	"github.com/tektoncd/operator/pkg/reconciler/kubernetes/tektonresult"
	"github.com/tektoncd/operator/test/client"
	"github.com/tektoncd/operator/test/resources"
	"github.com/tektoncd/operator/test/utils"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// TestTektonResultDeployment verifies the TektonResult creation, deployment recreation, and TektonResult deletion.
func TestTektonResultDeployment(t *testing.T) {
	platform := os.Getenv("PLATFORM")
	if platform == "linux/ppc64le" || platform == "linux/s390x" {
		t.Skipf("Tekton Result is not available for %q", platform)
	}
	clients := client.Setup(t)

	crNames := utils.ResourceNames{
		TektonPipeline:  "pipeline",
		TektonResult:    "result",
		TargetNamespace: "tekton-pipelines",
	}

	utils.CleanupOnInterrupt(func() { utils.TearDownPipeline(clients, crNames.TektonPipeline) })
	defer utils.TearDownPipeline(clients, crNames.TektonPipeline)

	utils.CleanupOnInterrupt(func() { utils.TearDownResult(clients, crNames.TektonResult) })
	defer utils.TearDownResult(clients, crNames.TektonTrigger)

	// Create a TektonPipeline
	if _, err := resources.EnsureTektonPipelineExists(clients.TektonPipeline(), crNames); err != nil {
		t.Fatalf("TektonPipeline %q failed to create: %v", crNames.TektonPipeline, err)
	}

	// Test if TektonPipeline can reach the READY status
	t.Run("create-pipeline", func(t *testing.T) {
		resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames)
	})

	// Before Installing Results, create the required secrets
	t.Run("create-secrets", func(t *testing.T) {
		createSecret(t, clients, crNames.TargetNamespace)
	})

	// Create a TektonResult
	if _, err := resources.EnsureTektonResultExists(clients.TektonResult(), crNames); err != nil {
		t.Fatalf("TektonResult %q failed to create: %v", crNames.TektonResult, err)
	}

	// Test if TektonResult can reach the READY status
	t.Run("create-result", func(t *testing.T) {
		resources.AssertTektonResultCRReadyStatus(t, clients, crNames)
	})

	// Delete the TektonResult CR instance to see if all resources will be removed
	t.Run("delete-result", func(t *testing.T) {
		resources.AssertTektonResultCRReadyStatus(t, clients, crNames)
		resources.TektonResultCRDDelete(t, clients, crNames)
	})

	// Delete the TektonPipeline CR instance to see if all resources will be removed
	t.Run("delete-pipeline", func(t *testing.T) {
		resources.AssertTektonPipelineCRReadyStatus(t, clients, crNames)
		resources.TektonPipelineCRDelete(t, clients, crNames)
	})
}

func createSecret(t *testing.T, clients *utils.Clients, namespace string) {

	key, err := ioutil.ReadFile("./testdata/key.pem")
	if err != nil {
		t.Fatal(err)
	}

	cert, err := ioutil.ReadFile("./testdata/cert.pem")
	if err != nil {
		t.Fatal(err)
	}

	tlsSecret := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name: tektonresult.TlsSecretName,
		},
		Data: map[string][]byte{
			corev1.TLSCertKey:       cert,
			corev1.TLSPrivateKeyKey: key,
		},
		Type: corev1.SecretTypeTLS,
	}

	_, err = clients.KubeClient.CoreV1().Secrets(namespace).Create(context.TODO(), tlsSecret, metav1.CreateOptions{})
	if err != nil && !apierrs.IsAlreadyExists(err) {
		t.Fatal(err)
	}

	dbSecret := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name: tektonresult.DbSecretName,
		},
		Data: map[string][]byte{
			"user":     []byte("root"),
			"password": []byte("test"),
		},
	}

	_, err = clients.KubeClient.CoreV1().Secrets(namespace).Create(context.TODO(), dbSecret, metav1.CreateOptions{})
	if err != nil && !apierrs.IsAlreadyExists(err) {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before><commit_msg>Use crane options for crane.Catalog (#747)<commit_after><|endoftext|>


<commit_before><commit_msg>one change<commit_after><|endoftext|>


<commit_before>/*
Copyright IBM Corp 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t *SimpleChaincode) Init(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	err := stub.PutState("hello_world", []byte(args[0]))
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Invoke isur entry point to invoke a chaincode function
func (t *SimpleChaincode) Invoke(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)

	// Handle different functions
	if function == "init" {
		return t.Init(stub, "init", args)
	} else if function == "write" {
		return t.write(stub, args)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation")
}

// Query is our entry point for queries
func (t *SimpleChaincode) Query(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)

	// Handle different functions
	if function == "read" {                            //read a variable
		return t.read(stub, args)
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query")
}

// write - invoke function to write key/value pair
func (t *SimpleChaincode) write(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var name, value string
	var err error
	fmt.Println("running write()")

	if len(args) != 2 {
		return nil, errors.New("Incorrect number of arguments. Expecting 2. name of the variable and value to set")
	}

	name = args[0]                            //rename for funsies
	value = args[1]
	err = stub.PutState(name, []byte(value))  //write the variable into the chaincode state
	if err != nil {
		return nil, err
	}
	return nil, nil
}

// read - query function to read key/value pair
func (t *SimpleChaincode) read(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var name, jsonResp string
	var err error

	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting name of the var to query")
	}

	name = args[0]
	valAsbytes, err := stub.GetState(name)
	if err != nil {
		jsonResp = "{\"Error\":\"Failed to get state for " + name + "\"}"
		return nil, errors.New(jsonResp)
	}

	return valAsbytes, nil
}
<commit_msg>added structures<commit_after>/*
Copyright IBM Corp 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"fmt"
	"strconv"
	"encoding/json"
	"time"
	"strings"
	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

type Option struct {
	Id int `json:"id"`
	Name string `json:"name"`
}

type Vote struct {
	Token string `json:"token"`
	OptionId int `json:"optionId"`
	ReceiptId string `json:"receiptId"`
}

type Election struct {
	Name string `json:"name"`
	Question string `json:"question"`
	Options []Option  `json:"options"`
	Tokens []string `json:"tokens"`
	Votes []Vote `json:"vote"`
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t *SimpleChaincode) Init(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

//	err := stub.PutState("hello_world", []byte(args[0]))
//	if err != nil {
//		return nil, err
//	}

	return nil, nil
}

// Invoke isur entry point to invoke a chaincode function
func (t *SimpleChaincode) Invoke(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)

	// Handle different functions
	if function == "init" {
		return t.Init(stub, "init", args)
	} else if function == "createElection" {
		return t.createElection(stub, args)
	} else if function == "vote" {
		return t.vote(stub, args)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation")
}

// Query is our entry point for queries
func (t *SimpleChaincode) Query(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)

	// Handle different functions
	// if function == "read" {                            //read a variable
	// 	return t.read(stub, args)
	// }
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query")
}

func (t *SimpleChaincode) createElection(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	return nil, nil
}

func (t *SimpleChaincode) vote(stub *shim.ChaincodeStub, args []string) ([]byte, error) {

	return nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"strings"
)

func (s *SlackAPI) ApiTest() {
	var response interface{}
	s.GetRequest(&response, "api.test")
	s.PrintJson(response)
}

func (s *SlackAPI) AuthTest() {
	var response interface{}
	s.GetRequest(&response, "auth.test", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) ChannelsInfo(channel string) {
	var response interface{}
	s.GetRequest(&response, "channels.info", "token", "channel="+channel)
	s.PrintJson(response)
}

func (s *SlackAPI) ChannelsList() {
	var response interface{}
	s.GetRequest(&response, "channels.list", "token", "exclude_archived=0")
	s.PrintJson(response)
}

func (s *SlackAPI) ChatDeleteVerbose(channel string, timestamp string) {
	response := s.ChatDelete(channel, timestamp)
	s.PrintJson(response)
}

func (s *SlackAPI) ChatPostMessageVerbose(channel string, message string) {
	response := s.ChatPostMessage(channel, message)
	s.PrintJson(response)
}

func (s *SlackAPI) EmojiList() {
	var response interface{}
	s.GetRequest(&response, "emoji.list", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) GroupsInfo(channel string) {
	var response interface{}
	s.GetRequest(&response, "groups.info", "token", "channel="+channel)
	s.PrintJson(response)
}

func (s *SlackAPI) GroupsList() {
	var response interface{}
	s.GetRequest(&response, "groups.list", "token", "exclude_archived=0")
	s.PrintJson(response)
}

func (s *SlackAPI) InstantMessagingCloseVerbose(channel string) {
	response := s.InstantMessagingClose(channel)
	s.PrintJson(response)
}

func (s *SlackAPI) InstantMessagingList() {
	var response interface{}
	s.GetRequest(&response, "im.list", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) InstantMessagingOpenVerbose(userid string) {
	response := s.InstantMessagingOpen(userid)
	s.PrintJson(response)
}

func (s *SlackAPI) MultiPartyInstantMessagingList() {
	var response interface{}
	s.GetRequest(&response, "mpim.list", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) ReactionsList(userid string) {
	var response interface{}

	if userid == "" {
		s.GetRequest(&response,
			"reactions.list",
			"token",
			"full=true",
			"count=100")
	} else {
		s.GetRequest(&response,
			"reactions.list",
			"token",
			"full=true",
			"count=100",
			"user="+userid)
	}
	s.PrintJson(response)
}

func (s *SlackAPI) ReactionsRemove(name string, channel string, timestamp string) {
	var response interface{}
	s.GetRequest(&response,
		"reactions.remove",
		"token",
		"name="+name,
		"channel="+channel,
		"timestamp="+timestamp)
	s.PrintJson(response)
}

func (s *SlackAPI) TeamInfo() {
	var response interface{}
	s.GetRequest(&response, "team.info", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) UsersGetPresence(query string) {
	var response interface{}
	s.GetRequest(&response, "users.getPresence", "token", "user="+query)
	s.PrintJson(response)
}

func (s *SlackAPI) UsersInfo(query string) {
	var response interface{}
	s.GetRequest(&response, "users.info", "token", "user="+query)
	s.PrintJson(response)
}

func (s *SlackAPI) UsersList() {
	var response interface{}
	s.GetRequest(&response, "users.list", "token", "presence=1")
	s.PrintJson(response)
}

func (s *SlackAPI) UsersSearch(query string) {
	if len(query) == 0 {
		s.ReportError(errors.New("empty query is invalid"))
	}

	var response Users
	var matches []User
	s.GetRequest(&response, "users.list", "token", "presence=1")

	for _, user := range response.Members {
		if strings.Contains(user.Name, query) ||
			strings.Contains(user.RealName, query) ||
			strings.Contains(user.Profile.Email, query) {
			matches = append(matches, user)
		}
	}

	s.PrintJson(matches)
}

func (s *SlackAPI) UsersSetActive() {
	var response interface{}
	s.GetRequest(&response, "users.setActive", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) UsersSetPresence(value string) {
	var response interface{}
	s.GetRequest(&response, "users.setPresence", "token", "presence="+value)
	s.PrintJson(response)
}
<commit_msg>Added support for reactions.remove method with files<commit_after>package main

import (
	"errors"
	"strings"
)

func (s *SlackAPI) ApiTest() {
	var response interface{}
	s.GetRequest(&response, "api.test")
	s.PrintJson(response)
}

func (s *SlackAPI) AuthTest() {
	var response interface{}
	s.GetRequest(&response, "auth.test", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) ChannelsInfo(channel string) {
	var response interface{}
	s.GetRequest(&response, "channels.info", "token", "channel="+channel)
	s.PrintJson(response)
}

func (s *SlackAPI) ChannelsList() {
	var response interface{}
	s.GetRequest(&response, "channels.list", "token", "exclude_archived=0")
	s.PrintJson(response)
}

func (s *SlackAPI) ChatDeleteVerbose(channel string, timestamp string) {
	response := s.ChatDelete(channel, timestamp)
	s.PrintJson(response)
}

func (s *SlackAPI) ChatPostMessageVerbose(channel string, message string) {
	response := s.ChatPostMessage(channel, message)
	s.PrintJson(response)
}

func (s *SlackAPI) EmojiList() {
	var response interface{}
	s.GetRequest(&response, "emoji.list", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) GroupsInfo(channel string) {
	var response interface{}
	s.GetRequest(&response, "groups.info", "token", "channel="+channel)
	s.PrintJson(response)
}

func (s *SlackAPI) GroupsList() {
	var response interface{}
	s.GetRequest(&response, "groups.list", "token", "exclude_archived=0")
	s.PrintJson(response)
}

func (s *SlackAPI) InstantMessagingCloseVerbose(channel string) {
	response := s.InstantMessagingClose(channel)
	s.PrintJson(response)
}

func (s *SlackAPI) InstantMessagingList() {
	var response interface{}
	s.GetRequest(&response, "im.list", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) InstantMessagingOpenVerbose(userid string) {
	response := s.InstantMessagingOpen(userid)
	s.PrintJson(response)
}

func (s *SlackAPI) MultiPartyInstantMessagingList() {
	var response interface{}
	s.GetRequest(&response, "mpim.list", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) ReactionsList(userid string) {
	var response interface{}

	if userid == "" {
		s.GetRequest(&response,
			"reactions.list",
			"token",
			"full=true",
			"count=100")
	} else {
		s.GetRequest(&response,
			"reactions.list",
			"token",
			"full=true",
			"count=100",
			"user="+userid)
	}
	s.PrintJson(response)
}

func (s *SlackAPI) ReactionsRemove(name string, channel string, timestamp string) {
	var response interface{}

	// Remove reaction from a file, file comment, or message.
	if channel[0] == 'F' {
		s.GetRequest(&response,
			"reactions.remove",
			"token",
			"name="+name,
			"file="+channel)
	} else {
		s.GetRequest(&response,
			"reactions.remove",
			"token",
			"name="+name,
			"channel="+channel,
			"timestamp="+timestamp)
	}

	s.PrintJson(response)
}

func (s *SlackAPI) TeamInfo() {
	var response interface{}
	s.GetRequest(&response, "team.info", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) UsersGetPresence(query string) {
	var response interface{}
	s.GetRequest(&response, "users.getPresence", "token", "user="+query)
	s.PrintJson(response)
}

func (s *SlackAPI) UsersInfo(query string) {
	var response interface{}
	s.GetRequest(&response, "users.info", "token", "user="+query)
	s.PrintJson(response)
}

func (s *SlackAPI) UsersList() {
	var response interface{}
	s.GetRequest(&response, "users.list", "token", "presence=1")
	s.PrintJson(response)
}

func (s *SlackAPI) UsersSearch(query string) {
	if len(query) == 0 {
		s.ReportError(errors.New("empty query is invalid"))
	}

	var response Users
	var matches []User
	s.GetRequest(&response, "users.list", "token", "presence=1")

	for _, user := range response.Members {
		if strings.Contains(user.Name, query) ||
			strings.Contains(user.RealName, query) ||
			strings.Contains(user.Profile.Email, query) {
			matches = append(matches, user)
		}
	}

	s.PrintJson(matches)
}

func (s *SlackAPI) UsersSetActive() {
	var response interface{}
	s.GetRequest(&response, "users.setActive", "token")
	s.PrintJson(response)
}

func (s *SlackAPI) UsersSetPresence(value string) {
	var response interface{}
	s.GetRequest(&response, "users.setPresence", "token", "presence="+value)
	s.PrintJson(response)
}
<|endoftext|>


<commit_before>package asics_parser

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"
)

var (
	WirteDir = ""
)

var EuropeEcAsics []string
var NoEcAsics []string
var AllData = map[string]ParseData{}

type ParseData struct {
	Locale string
	Header []Link
	Footer []Link
}

type Link struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
	//the value of imgsize represent the type of the image, small | medium | big
	SingleRow bool
	//when the value of SingleRow is false, there will have two link in a row.
	Items []interface{}
}

type ItemLink struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
}

func ParseFile(dir string) map[string]ParseData {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	for _, file := range files {
		if strings.Contains(file.Name(), ".json") {
			openfile, err := os.Open(dir + file.Name())
			if err != nil {
				log.Fatal(err)
			}
			data, err := ioutil.ReadAll(openfile)
			if err != nil {
				log.Fatal(err)
			}
			var parsedate ParseData
			json.Unmarshal(data, &parsedate)
			AllData[parsedate.Locale] = parsedate

		}

	}
	return AllData

}

func ParseECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	parsedata := ParseData{
		Locale: strings.Split(target_url.Path, "/")[2],
	}

	//parse the header
	doc.Find("#main-menu").Children().Not(".mobile").Not("div").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Url:       paser_url(target_url, s.Children().First().AttrOr("href", "#")),
			Name:      s.Children().First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Find("ul").Each(func(i int, s *goquery.Selection) {
			if s.Children().Size() == 2 {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     1,
					SingleRow: false,
				}

				secondlink := Link{
					Title:     s.Find(".empty-nav-item").Last().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find(".empty-nav-item").Last().Text(),
					Index:     2,
					SingleRow: false,
				}

				is_first := true
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {
					// fmt.Println(i)
					// fmt.Println(s.Children().First().Html())
					if s.Children().First().Is("span") {
						if s.Children().First().Text() != "" {
							is_first = false
						}

					} else {
						itemlink := ItemLink{
							Title: s.Children().First().AttrOr("title", ""),
							Name:  s.Children().First().Text(),
							Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
							Index: i + 1,
						}
						if s.Children().First().Children().First().Is("img") {
							itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
						}

						if is_first {
							firstlink.Items = append(firstlink.Items, itemlink)
						} else {
							secondlink.Items = append(secondlink.Items, itemlink)

						}
					}
				})
				link.Items = append(link.Items, firstlink)
				link.Items = append(link.Items, secondlink)

			} else {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     i + 1,
					SingleRow: true,
				}
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {

					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Name:  s.Children().First().Text(),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Index: i + 1,
					}
					if s.Children().First().Children().First().Is("img") {
						itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
					}

					firstlink.Items = append(firstlink.Items, itemlink)

				})
				link.Items = append(link.Items, firstlink)
			}
		})
		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find("footer").Children().Filter(".tiger-clearfix-toggle").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().Filter("h4").First().AttrOr("title", ""),
			Url:       "#",
			Name:      s.Children().Filter("h4").First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Name:  s.Children().First().Text(),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	fmt.Printf("current url is %s \n", url_string)
	file, _ := os.OpenFile(WirteDir+strings.Split(target_url.Path, "/")[2]+".json", os.O_WRONLY|os.O_CREATE, 0666)
	file.Write(jsondata)

}

func ParseNoECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	locale := strings.Split(target_url.Host, ".")[0] + "-" + strings.Split(target_url.Host, ".")[2]
	parsedata := ParseData{
		Locale: locale,
	}

	//parse the header
	doc.Find("header #asicsAreas").Children().Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Children().First().Text(),
			SingleRow: true,
			Url:       "#",
			Index:     i + 1,
		}
		if i != 3 {
			doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsListing").Each(func(ii int, ss *goquery.Selection) {
				secend_link := Link{
					Title:     doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("title", ""),
					Name:      strings.TrimSpace(doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().Text()),
					Url:       doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("href", "#"),
					Img:       paser_url(target_url, doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().First().Children().First().AttrOr("src", "#")),
					ImgSize:   "small",
					Index:     i + 1,
					SingleRow: true,
				}
				ss.Children().Filter("ul").Children().Each(func(iii int, s *goquery.Selection) {
					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Name:  s.Children().First().Text(),
						Index: iii + 1,
					}

					secend_link.Items = append(secend_link.Items, itemlink)
				})

				link.Items = append(link.Items, secend_link)

			})

		}

		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find(".footer #tertiary section").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Text(),
			Url:       "#",
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Name:  s.Children().First().Text(),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	file, _ := os.OpenFile(WirteDir+locale+".json", os.O_WRONLY|os.O_CREATE, 0666)
	fmt.Printf("current url is %s \n", url_string)
	file.Write(jsondata)
}

func paser_url(target_url *url.URL, url string) (return_url string) {

	if matched, err := regexp.Match(`http`, []byte(url)); matched && err == nil {
		return_url = url
	} else {
		return_url = target_url.Scheme + "://" + target_url.Host + url
	}
	return
}
<commit_msg>add Noasics Locale Map<commit_after>package asics_parser

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"
)

var (
	WirteDir       = ""
	NoEcAsicsLocal = map[string]string{"www.asics.fi": "en-fi", "www.asics.pl": "pl-pl",
		"www.asics.pt": "pt-pt", "en.asics.ch": "en-ch", "fr.asics.ch": "fr-ch",
		"de.asics.ch": "de-ch", "www.asics.ru": "ru-ru",
		"www.asics.co.za": "en-za", "www.asics.com.hk": "en-hk", "www.asics.com.cn": "zh-cn",
		"www.asics.com.sg": "en-sg", "www.asicsindia.in": "en-in",
	}
)

var EuropeEcAsics []string
var NoEcAsics []string
var AllData = map[string]ParseData{}

type ParseData struct {
	Locale string
	Header []Link
	Footer []Link
}

type Link struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
	//the value of imgsize represent the type of the image, small | medium | big
	SingleRow bool
	//when the value of SingleRow is false, there will have two link in a row.
	Items []interface{}
}

type ItemLink struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
}

func ParseFile(dir string) map[string]ParseData {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	for _, file := range files {
		if strings.Contains(file.Name(), ".json") {
			openfile, err := os.Open(dir + file.Name())
			if err != nil {
				log.Fatal(err)
			}
			data, err := ioutil.ReadAll(openfile)
			if err != nil {
				log.Fatal(err)
			}
			var parsedate ParseData
			json.Unmarshal(data, &parsedate)
			AllData[parsedate.Locale] = parsedate

		}

	}
	return AllData

}

func ParseECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	parsedata := ParseData{
		Locale: strings.Split(target_url.Path, "/")[2],
	}

	//parse the header
	doc.Find("#main-menu").Children().Not(".mobile").Not("div").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Url:       paser_url(target_url, s.Children().First().AttrOr("href", "#")),
			Name:      s.Children().First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Find("ul").Each(func(i int, s *goquery.Selection) {
			if s.Children().Size() == 2 {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     1,
					SingleRow: false,
				}

				secondlink := Link{
					Title:     s.Find(".empty-nav-item").Last().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find(".empty-nav-item").Last().Text(),
					Index:     2,
					SingleRow: false,
				}

				is_first := true
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {
					// fmt.Println(i)
					// fmt.Println(s.Children().First().Html())
					if s.Children().First().Is("span") {
						if s.Children().First().Text() != "" {
							is_first = false
						}

					} else {
						itemlink := ItemLink{
							Title: s.Children().First().AttrOr("title", ""),
							Name:  s.Children().First().Text(),
							Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
							Index: i + 1,
						}
						if s.Children().First().Children().First().Is("img") {
							itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
						}

						if is_first {
							firstlink.Items = append(firstlink.Items, itemlink)
						} else {
							secondlink.Items = append(secondlink.Items, itemlink)

						}
					}
				})
				link.Items = append(link.Items, firstlink)
				link.Items = append(link.Items, secondlink)

			} else {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     i + 1,
					SingleRow: true,
				}
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {

					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Name:  s.Children().First().Text(),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Index: i + 1,
					}
					if s.Children().First().Children().First().Is("img") {
						itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
					}

					firstlink.Items = append(firstlink.Items, itemlink)

				})
				link.Items = append(link.Items, firstlink)
			}
		})
		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find("footer").Children().Filter(".tiger-clearfix-toggle").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().Filter("h4").First().AttrOr("title", ""),
			Url:       "#",
			Name:      s.Children().Filter("h4").First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Name:  s.Children().First().Text(),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	fmt.Printf("current url is %s  \n", url_string)
	file, _ := os.OpenFile(WirteDir+strings.Split(target_url.Path, "/")[2]+".json", os.O_WRONLY|os.O_CREATE, 0666)
	file.Write(jsondata)

}

func ParseNoECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	locale := NoEcAsicsLocal[target_url.Host]
	parsedata := ParseData{
		Locale: locale,
	}

	//parse the header
	doc.Find("header #asicsAreas").Children().Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Children().First().Text(),
			SingleRow: true,
			Url:       "#",
			Index:     i + 1,
		}
		if i != 3 {
			doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsListing").Each(func(ii int, ss *goquery.Selection) {
				secend_link := Link{
					Title:     doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("title", ""),
					Name:      strings.TrimSpace(doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().Text()),
					Url:       doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("href", "#"),
					Img:       paser_url(target_url, doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().First().Children().First().AttrOr("src", "#")),
					ImgSize:   "small",
					Index:     i + 1,
					SingleRow: true,
				}
				ss.Children().Filter("ul").Children().Each(func(iii int, s *goquery.Selection) {
					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Name:  s.Children().First().Text(),
						Index: iii + 1,
					}

					secend_link.Items = append(secend_link.Items, itemlink)
				})

				link.Items = append(link.Items, secend_link)

			})

		}

		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find(".footer #tertiary section").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Text(),
			Url:       "#",
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Name:  s.Children().First().Text(),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	file, _ := os.OpenFile(WirteDir+locale+".json", os.O_WRONLY|os.O_CREATE, 0666)
	fmt.Printf("current url is %s ", url_string)
	file.Write(jsondata)
}

func paser_url(target_url *url.URL, url string) (return_url string) {

	if matched, err := regexp.Match(`http`, []byte(url)); matched && err == nil {
		return_url = url
	} else {
		return_url = target_url.Scheme + "://" + target_url.Host + url
	}
	return
}
<|endoftext|>


<commit_before>package etcd

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type TlsOpts struct {
	Ca   string `json:"ca"`
	Cert string `json:"cert"`
	Key  string `json:"key"`
}

type Request struct {
	method  string
	url     *url.URL
	timeout time.Duration
	ctx     context.Context
}

type Option func(r *Request)

func fileExists(filename string) bool {
	info, err := os.Stat(filename)
	if os.IsNotExist(err) {
		return false
	}
	return !info.IsDir()
}

func GET() Option {
	return func(r *Request) {
		r.method = "GET"
	}
}

func POST() Option {
	return func(r *Request) {
		r.method = "POST"
	}
}

func PUT() Option {
	return func(r *Request) {
		r.method = "PUT"
	}
}

func DELETE() Option {
	return func(r *Request) {
		r.method = "DELETE"
	}
}

func Wait(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("wait", "true")
		} else {
			q.Del("wait")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Value(value string) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("value", value)
		r.url.RawQuery = q.Encode()
	}
}

func PrevValue(value string) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("prevValue", value)
		r.url.RawQuery = q.Encode()
	}
}

func PrevIndex(index uint64) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("prevIndex", fmt.Sprintf("%d", index))
		r.url.RawQuery = q.Encode()
	}
}

func PrevExist(exists bool) Option {
	return func(r *Request) {
		q := r.url.Query()
		if exists {
			q.Set("prevExist", "true")
		} else {
			q.Set("prevExist", "false")
		}
		r.url.RawQuery = q.Encode()
	}
}

func TTL(ttl time.Duration) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("ttl", fmt.Sprintf("%d", int(ttl.Seconds())))
		r.url.RawQuery = q.Encode()
	}
}

func Refresh(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("refresh", "true")
		} else {
			q.Del("refresh")
		}

		r.url.RawQuery = q.Encode()
	}
}

func WaitIndex(index uint64) Option {
	return func(r *Request) {
		q := r.url.Query()

		if index > 0 {
			q.Set("waitIndex", fmt.Sprintf("%d", index))
		} else {
			q.Del("waitIndex")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Recursive(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("recursive", "true")

			if !strings.HasSuffix(r.url.Path, "/") {
				r.url.Path += "/"
			}
		} else {
			q.Del("recursive")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Sorted(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("sorted", "true")
		} else {
			q.Del("sorted")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Timeout(t time.Duration) Option {
	return func(r *Request) {
		r.timeout = t
	}
}

func Context(ctx context.Context) Option {
	return func(r *Request) {
		r.ctx = ctx
	}
}

type Node struct {
	Key           string     `json:"key"`
	Dir           bool       `json:"dir,omitempty"`
	Value         string     `json:"value"`
	Nodes         []*Node    `json:"nodes"`
	CreatedIndex  uint64     `json:"createdIndex"`
	ModifiedIndex uint64     `json:"modifiedIndex"`
	Expiration    *time.Time `json:"expiration,omitempty"`
	TTL           int64      `json:"ttl,omitempty"`
}

type Response struct {
	Action       string `json:"action"`
	Node         *Node  `json:"node"`
	PrevNode     *Node  `json:"prevNode"`
	ErrorCode    uint64 `json:"errorCode"`
	ErrorMessage string `json:"message"`
	ErrorCause   string `json:"cause"`
	Index        uint64 `json:"-"`
}

type Client struct {
	sync.RWMutex
	endpoints     []string
	endpointIndex int
	transport     *http.Transport
	debug         bool
}

func NewClient(endpoints []string, tlsOpts TlsOpts, debug bool) (*Client, error) {
	// check endpoints parse
	for _, e := range endpoints {
		_, err := url.Parse(e)
		if err != nil {
			return nil, err
		}
	}

	client := &Client{
		endpoints: endpoints,
		debug:     debug,
	}
	if fileExists(tlsOpts.Ca) && fileExists(tlsOpts.Cert) && fileExists(tlsOpts.Key) {
		// Load client cert
		cert, err := tls.LoadX509KeyPair(tlsOpts.Cert, tlsOpts.Key)
		if err != nil {
			log.Fatal(err)
		}

		// Load CA cert
		caCert, err := ioutil.ReadFile(tlsOpts.Ca)
		if err != nil {
			log.Fatal(err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		// Setup HTTPS client
		tlsConfig := &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
		tlsConfig.BuildNameToCertificate()
		client.transport = &http.Transport{TLSClientConfig: tlsConfig}
	}
	return client, nil
}

func (client *Client) Debug(format string, v ...interface{}) {
	if client.debug {
		log.Printf(format, v...)
	}
}

func (client *Client) currentEndpoint() string {
	client.RLock()
	res := client.endpoints[client.endpointIndex]
	client.RUnlock()
	return res
}

func (client *Client) nextEndpoint() string {
	client.Lock()
	client.endpointIndex = (client.endpointIndex + 1) % len(client.endpoints)
	client.Unlock()

	endpoint := client.currentEndpoint()
	client.Debug("new endpoint: %s", endpoint)

	return endpoint
}

func (client *Client) Query(key string, opts ...Option) (*Response, error) {
	endpoint := client.currentEndpoint()

QueryLoop:
	for {
		u, _ := url.Parse(endpoint) // error validated in client contructor
		u.Path = "/v2/keys" + key

		q := &Request{
			method:  "GET",
			url:     u,
			timeout: time.Minute,
			ctx:     context.Background(),
		}

		for _, o := range opts {
			o(q)
		}

		if q.ctx.Err() != nil {
			client.Debug("ctx.Err(): %s", q.ctx.Err().Error())
			return nil, q.ctx.Err()
		}

		client.Debug("%s %s", q.method, q.url.String())

		var httpClient *http.Client
		if client.transport == (&http.Transport{}) {
			httpClient = &http.Client{
				Timeout: q.timeout,
			}
		} else {
			httpClient = &http.Client{
				Timeout:   q.timeout,
				Transport: client.transport,
			}
		}

		// @TODO: check error?
		req, _ := http.NewRequest(q.method, q.url.String(), nil)
		req = req.WithContext(q.ctx)

		res, httpErr := httpClient.Do(req)
		body := []byte{}

		xEtcdIndex := ""

		if httpErr == nil {
			xEtcdIndex = res.Header.Get("X-Etcd-Index")

			body, httpErr = ioutil.ReadAll(res.Body)
			res.Body.Close()
		}

		// check context error
		if q.ctx.Err() != nil {
			client.Debug("context error: %#v", q.ctx.Err())
			return nil, q.ctx.Err()
		}

		if httpErr != nil {
			client.Debug("error: %s", httpErr.Error())
		} else {
			client.Debug("X-Etcd-Index: %s", xEtcdIndex)
			client.Debug("body: %s", string(body))
		}

		// reconnect on timeout, network error, endpoint down, etc
		if httpErr != nil {
			endpoint = client.nextEndpoint()
			time.Sleep(time.Second)
			continue QueryLoop
		}

		r := &Response{}
		err := json.Unmarshal(body, r)
		if err != nil {
			client.Debug("unmarshal error: %s", err.Error())
			return nil, err
		}

		index, _ := strconv.Atoi(xEtcdIndex)
		r.Index = uint64(index)

		client.Debug("response: %#v", r)

		return r, nil
	}
}
<commit_msg>fix transport detect<commit_after>package etcd

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type TlsOpts struct {
	Ca   string `json:"ca"`
	Cert string `json:"cert"`
	Key  string `json:"key"`
}

type Request struct {
	method  string
	url     *url.URL
	timeout time.Duration
	ctx     context.Context
}

type Option func(r *Request)

func fileExists(filename string) bool {
	info, err := os.Stat(filename)
	if os.IsNotExist(err) {
		return false
	}
	return !info.IsDir()
}

func GET() Option {
	return func(r *Request) {
		r.method = "GET"
	}
}

func POST() Option {
	return func(r *Request) {
		r.method = "POST"
	}
}

func PUT() Option {
	return func(r *Request) {
		r.method = "PUT"
	}
}

func DELETE() Option {
	return func(r *Request) {
		r.method = "DELETE"
	}
}

func Wait(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("wait", "true")
		} else {
			q.Del("wait")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Value(value string) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("value", value)
		r.url.RawQuery = q.Encode()
	}
}

func PrevValue(value string) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("prevValue", value)
		r.url.RawQuery = q.Encode()
	}
}

func PrevIndex(index uint64) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("prevIndex", fmt.Sprintf("%d", index))
		r.url.RawQuery = q.Encode()
	}
}

func PrevExist(exists bool) Option {
	return func(r *Request) {
		q := r.url.Query()
		if exists {
			q.Set("prevExist", "true")
		} else {
			q.Set("prevExist", "false")
		}
		r.url.RawQuery = q.Encode()
	}
}

func TTL(ttl time.Duration) Option {
	return func(r *Request) {
		q := r.url.Query()
		q.Set("ttl", fmt.Sprintf("%d", int(ttl.Seconds())))
		r.url.RawQuery = q.Encode()
	}
}

func Refresh(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("refresh", "true")
		} else {
			q.Del("refresh")
		}

		r.url.RawQuery = q.Encode()
	}
}

func WaitIndex(index uint64) Option {
	return func(r *Request) {
		q := r.url.Query()

		if index > 0 {
			q.Set("waitIndex", fmt.Sprintf("%d", index))
		} else {
			q.Del("waitIndex")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Recursive(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("recursive", "true")

			if !strings.HasSuffix(r.url.Path, "/") {
				r.url.Path += "/"
			}
		} else {
			q.Del("recursive")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Sorted(v bool) Option {
	return func(r *Request) {
		q := r.url.Query()

		if v {
			q.Set("sorted", "true")
		} else {
			q.Del("sorted")
		}

		r.url.RawQuery = q.Encode()
	}
}

func Timeout(t time.Duration) Option {
	return func(r *Request) {
		r.timeout = t
	}
}

func Context(ctx context.Context) Option {
	return func(r *Request) {
		r.ctx = ctx
	}
}

type Node struct {
	Key           string     `json:"key"`
	Dir           bool       `json:"dir,omitempty"`
	Value         string     `json:"value"`
	Nodes         []*Node    `json:"nodes"`
	CreatedIndex  uint64     `json:"createdIndex"`
	ModifiedIndex uint64     `json:"modifiedIndex"`
	Expiration    *time.Time `json:"expiration,omitempty"`
	TTL           int64      `json:"ttl,omitempty"`
}

type Response struct {
	Action       string `json:"action"`
	Node         *Node  `json:"node"`
	PrevNode     *Node  `json:"prevNode"`
	ErrorCode    uint64 `json:"errorCode"`
	ErrorMessage string `json:"message"`
	ErrorCause   string `json:"cause"`
	Index        uint64 `json:"-"`
}

type Client struct {
	sync.RWMutex
	endpoints     []string
	endpointIndex int
	transport     http.RoundTripper
	debug         bool
}

func NewClient(endpoints []string, tlsOpts TlsOpts, debug bool) (*Client, error) {
	// check endpoints parse
	for _, e := range endpoints {
		_, err := url.Parse(e)
		if err != nil {
			return nil, err
		}
	}

	client := &Client{
		endpoints: endpoints,
		debug:     debug,
	}
	if tlsOpts.Ca != "" && tlsOpts.Cert != "" && tlsOpts.Key != "" {
		// Load client cert
		cert, err := tls.LoadX509KeyPair(tlsOpts.Cert, tlsOpts.Key)
		if err != nil {
			log.Fatal(err)
		}

		// Load CA cert
		caCert, err := ioutil.ReadFile(tlsOpts.Ca)
		if err != nil {
			log.Fatal(err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		// Setup HTTPS client
		tlsConfig := &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
		tlsConfig.BuildNameToCertificate()
		client.transport = &http.Transport{TLSClientConfig: tlsConfig}
	}
	return client, nil
}

func (client *Client) Debug(format string, v ...interface{}) {
	if client.debug {
		log.Printf(format, v...)
	}
}

func (client *Client) currentEndpoint() string {
	client.RLock()
	res := client.endpoints[client.endpointIndex]
	client.RUnlock()
	return res
}

func (client *Client) nextEndpoint() string {
	client.Lock()
	client.endpointIndex = (client.endpointIndex + 1) % len(client.endpoints)
	client.Unlock()

	endpoint := client.currentEndpoint()
	client.Debug("new endpoint: %s", endpoint)

	return endpoint
}

func (client *Client) Query(key string, opts ...Option) (*Response, error) {
	endpoint := client.currentEndpoint()

QueryLoop:
	for {
		u, _ := url.Parse(endpoint) // error validated in client contructor
		u.Path = "/v2/keys" + key

		q := &Request{
			method:  "GET",
			url:     u,
			timeout: time.Minute,
			ctx:     context.Background(),
		}

		for _, o := range opts {
			o(q)
		}

		if q.ctx.Err() != nil {
			client.Debug("ctx.Err(): %s", q.ctx.Err().Error())
			return nil, q.ctx.Err()
		}

		client.Debug("%s %s", q.method, q.url.String())

		httpClient := &http.Client{
			Timeout:   q.timeout,
			Transport: client.transport,
		}

		// @TODO: check error?
		req, _ := http.NewRequest(q.method, q.url.String(), nil)
		req = req.WithContext(q.ctx)

		res, httpErr := httpClient.Do(req)
		body := []byte{}

		xEtcdIndex := ""

		if httpErr == nil {
			xEtcdIndex = res.Header.Get("X-Etcd-Index")

			body, httpErr = ioutil.ReadAll(res.Body)
			res.Body.Close()
		}

		// check context error
		if q.ctx.Err() != nil {
			client.Debug("context error: %#v", q.ctx.Err())
			return nil, q.ctx.Err()
		}

		if httpErr != nil {
			client.Debug("error: %s", httpErr.Error())
		} else {
			client.Debug("X-Etcd-Index: %s", xEtcdIndex)
			client.Debug("body: %s", string(body))
		}

		// reconnect on timeout, network error, endpoint down, etc
		if httpErr != nil {
			endpoint = client.nextEndpoint()
			time.Sleep(time.Second)
			continue QueryLoop
		}

		r := &Response{}
		err := json.Unmarshal(body, r)
		if err != nil {
			client.Debug("unmarshal error: %s", err.Error())
			return nil, err
		}

		index, _ := strconv.Atoi(xEtcdIndex)
		r.Index = uint64(index)

		client.Debug("response: %#v", r)

		return r, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Netflix Inc
// Author: Colin McIntosh (colin@netflix.com)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gateway provides an easily configurable server that can connect to multiple gNMI targets (devices) and
// relay received messages to various exporters and to downstream gNMI clients.
//
// Targets are configured via a TargetLoader which generate all of the needed configuration for connecting to a target.
// See github.com/openconfig/gnmi/proto/target for details on the Configuration options that are available.
// See the TargetLoader docs for the minimum configuration required to connect to a target.
//
// The gateway only supports TLS so you'll need to generate some keys first if you don't already have them.
// In production you should use properly signed TLS certificates.
//		# Generate private key (server.key)
//		openssl genrsa -out server.key 2048
//		# or
//		openssl ecparam -genkey -name secp384r1 -out server.key
//
//		# Generation of self-signed(x509) public key (server.crt) based on the private key (server.key)
//		openssl req -new -x509 -sha256 -key server.key -out server.crt -days 3650
//
// You'll also need a copy of the latest OpenConfig YANG models if you don't already have it.
//		git clone https://github.com/openconfig/public.git oc-models
//
// Finally, you need to build your target configurations. Copy targets-example.json to targets.json and edit it to
// match the targets you want to connect to.
//		cp targets-example.json targets.json
//		vim targets.json
//
// See the example below or the Main() function in gateway.go for an example of how to start the server.
// If you'd like to just use the built-in loaders and exporters you can configure them more easily from the command line:
// 		go build
//		./gnmi-gateway -EnableServer -EnablePrometheus -OpenConfigDirectory=./oc-models/
package gateway

import (
	"flag"
	"fmt"
	"os"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/configuration"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/connections"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/exporters"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/targets"
	"strings"
	"time"
)

var (
	// Buildtime is set to the current time during the build process by GOLDFLAGS
	Buildtime string
	// Version is set to the current git tag during the build process by GOLDFLAGS
	Version string
)

var (
	enablePrometheus bool
	logCaller        bool
	printVersion     bool
)

// GatewayStartOpts is passed to StartGateway() and is used to set the running configuration
type GatewayStartOpts struct {
	// Loader for targets
	TargetLoader targets.TargetLoader
	// Exporters to run
	Exporters []exporters.Exporter
}

// Main is the entry point for the command-line and it's a good example of how to call StartGateway but
// other than that you probably don't need Main for anything.
func Main() {
	config := configuration.NewDefaultGatewayConfig()
	ParseArgs(config)

	if printVersion {
		fmt.Println(fmt.Sprintf("gnmi-gateway version %s (Built %s)", Version, Buildtime))
		os.Exit(0)
	}

	if true {
		config.Log = config.Log.With().Caller().Logger()
	}

	opts := &GatewayStartOpts{
		TargetLoader: targets.NewJSONFileTargetLoader(config),
	}

	if enablePrometheus {
		opts.Exporters = append(opts.Exporters, exporters.NewPrometheusExporter(config))
	}

	err := StartGateway(config, opts) // run forever (or until an error happens)
	if err != nil {
		config.Log.Error().Err(err).Msgf("Gateway exited with an error: %v", err)
		os.Exit(1)
	}
}

// ParseArgs will parse all of the command-line parameters and configured the associated attributes on the
// GatewayConfig. ParseArgs calls flag.Parse before returning so if you need to add arguments you should make
// any calls to flag before calling ParseArgs.
func ParseArgs(config *configuration.GatewayConfig) {
	// Execution parameters
	flag.BoolVar(&config.EnableServer, "EnableServer", false, "Enable the gNMI server")
	flag.BoolVar(&enablePrometheus, "EnablePrometheus", false, "Enable the Prometheus exporter")
	flag.BoolVar(&logCaller, "LogCaller", false, "Include the file and line number with each log message")
	flag.BoolVar(&printVersion, "version", false, "Print version and exit")

	// Configuration Parameters
	flag.StringVar(&config.OpenConfigDirectory, "OpenConfigDirectory", "", "Directory (required to enable Prometheus exporter)")
	flag.StringVar(&config.TargetJSONFile, "TargetJSONFile", "targets.json", "JSON file containing the target configurations (default: targets.json)")
	flag.DurationVar(&config.TargetJSONFileReloadInterval, "TargetJSONFileReloadInterval", 10*time.Second, "Interval to reload the JSON file containing the target configurations (default: 10s)")
	flag.DurationVar(&config.TargetDialTimeout, "TargetDialTimeout", 10*time.Second, "Dial timeout time (default: 10s)")
	flag.IntVar(&config.TargetLimit, "TargetLimit", 100, "Maximum number of targets that this instance will connect to at once (default: 100)")
	zkHosts := flag.String("ZookeeperHosts", "127.0.0.1:2181", "Comma separated (no spaces) list of zookeeper hosts including port (default: 127.0.0.1:2181)")
	flag.DurationVar(&config.ZookeeperTimeout, "ZookeeperTimeout", 1*time.Second, "Zookeeper timeout time. Minimum is 1 second. Failover time is (ZookeeperTimeout * 2). (default: 1s)")

	flag.Parse()
	config.ZookeeperHosts = strings.Split(*zkHosts, ",")
}

// StartGateway starts up all of the loaders and exporters provided by GatewayStartOpts. This is the
// primary way the server should be started.
func StartGateway(config *configuration.GatewayConfig, opts *GatewayStartOpts) error {
	config.Log.Info().Msg("Starting GNMI Gateway.")
	connMgr, err := connections.NewConnectionManagerDefault(config)
	if err != nil {
		config.Log.Error().Err(err).Msg("Unable to create connection manager.")
		os.Exit(1)
	}
	config.Log.Info().Msg("Starting connection manager.")
	if err := connMgr.Start(); err != nil {
		config.Log.Error().Err(err).Msgf("Unable to start connection manager: %v", err)
		return err
	}

	// channel to listen for errors from child goroutines
	finished := make(chan error, 1)

	go func() {
		err := opts.TargetLoader.Start()
		if err != nil {
			config.Log.Error().Err(err).Msgf("Unable to start target loader %T", opts.TargetLoader)
			finished <- err
		}
		opts.TargetLoader.WatchConfiguration(connMgr.TargetConfigChan())
	}()

	if config.EnableServer {
		config.Log.Info().Msg("Starting gNMI server.")
		go func() {
			if err := StartServer(config, connMgr.Cache()); err != nil {
				config.Log.Error().Err(err).Msg("Unable to start gNMI server.")
				finished <- err
			}
		}()
	}

	for _, exporter := range opts.Exporters {
		go func(exporter exporters.Exporter) {
			err := exporter.Start(connMgr.Cache())
			if err != nil {
				config.Log.Error().Err(err).Msgf("Unable to start exporter %T", exporter)
				finished <- err
			}
			// TODO: Call SetClient here as it's pretty universal.
		}(exporter)
	}

	return <-finished
}
<commit_msg>Revert making command-line variables non-exported.<commit_after>// Copyright 2020 Netflix Inc
// Author: Colin McIntosh (colin@netflix.com)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gateway provides an easily configurable server that can connect to multiple gNMI targets (devices) and
// relay received messages to various exporters and to downstream gNMI clients.
//
// Targets are configured via a TargetLoader which generate all of the needed configuration for connecting to a target.
// See github.com/openconfig/gnmi/proto/target for details on the Configuration options that are available.
// See the TargetLoader docs for the minimum configuration required to connect to a target.
//
// The gateway only supports TLS so you'll need to generate some keys first if you don't already have them.
// In production you should use properly signed TLS certificates.
//		# Generate private key (server.key)
//		openssl genrsa -out server.key 2048
//		# or
//		openssl ecparam -genkey -name secp384r1 -out server.key
//
//		# Generation of self-signed(x509) public key (server.crt) based on the private key (server.key)
//		openssl req -new -x509 -sha256 -key server.key -out server.crt -days 3650
//
// You'll also need a copy of the latest OpenConfig YANG models if you don't already have it.
//		git clone https://github.com/openconfig/public.git oc-models
//
// Finally, you need to build your target configurations. Copy targets-example.json to targets.json and edit it to
// match the targets you want to connect to.
//		cp targets-example.json targets.json
//		vim targets.json
//
// See the example below or the Main() function in gateway.go for an example of how to start the server.
// If you'd like to just use the built-in loaders and exporters you can configure them more easily from the command line:
// 		go build
//		./gnmi-gateway -EnableServer -EnablePrometheus -OpenConfigDirectory=./oc-models/
package gateway

import (
	"flag"
	"fmt"
	"os"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/configuration"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/connections"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/exporters"
	"stash.corp.netflix.com/ocnas/gnmi-gateway/gateway/targets"
	"strings"
	"time"
)

var (
	// Buildtime is set to the current time during the build process by GOLDFLAGS
	Buildtime string
	// Version is set to the current git tag during the build process by GOLDFLAGS
	Version string
)

var (
	EnablePrometheus bool
	LogCaller        bool
	PrintVersion     bool
)

// GatewayStartOpts is passed to StartGateway() and is used to set the running configuration
type GatewayStartOpts struct {
	// Loader for targets
	TargetLoader targets.TargetLoader
	// Exporters to run
	Exporters []exporters.Exporter
}

// Main is the entry point for the command-line and it's a good example of how to call StartGateway but
// other than that you probably don't need Main for anything.
func Main() {
	config := configuration.NewDefaultGatewayConfig()
	ParseArgs(config)

	if PrintVersion {
		fmt.Println(fmt.Sprintf("gnmi-gateway version %s (Built %s)", Version, Buildtime))
		os.Exit(0)
	}

	if true {
		config.Log = config.Log.With().Caller().Logger()
	}

	opts := &GatewayStartOpts{
		TargetLoader: targets.NewJSONFileTargetLoader(config),
	}

	if EnablePrometheus {
		opts.Exporters = append(opts.Exporters, exporters.NewPrometheusExporter(config))
	}

	err := StartGateway(config, opts) // run forever (or until an error happens)
	if err != nil {
		config.Log.Error().Err(err).Msgf("Gateway exited with an error: %v", err)
		os.Exit(1)
	}
}

// ParseArgs will parse all of the command-line parameters and configured the associated attributes on the
// GatewayConfig. ParseArgs calls flag.Parse before returning so if you need to add arguments you should make
// any calls to flag before calling ParseArgs.
func ParseArgs(config *configuration.GatewayConfig) {
	// Execution parameters
	flag.BoolVar(&config.EnableServer, "EnableServer", false, "Enable the gNMI server")
	flag.BoolVar(&EnablePrometheus, "EnablePrometheus", false, "Enable the Prometheus exporter")
	flag.BoolVar(&LogCaller, "LogCaller", false, "Include the file and line number with each log message")
	flag.BoolVar(&PrintVersion, "version", false, "Print version and exit")

	// Configuration Parameters
	flag.StringVar(&config.OpenConfigDirectory, "OpenConfigDirectory", "", "Directory (required to enable Prometheus exporter)")
	flag.StringVar(&config.TargetJSONFile, "TargetJSONFile", "targets.json", "JSON file containing the target configurations (default: targets.json)")
	flag.DurationVar(&config.TargetJSONFileReloadInterval, "TargetJSONFileReloadInterval", 10*time.Second, "Interval to reload the JSON file containing the target configurations (default: 10s)")
	flag.DurationVar(&config.TargetDialTimeout, "TargetDialTimeout", 10*time.Second, "Dial timeout time (default: 10s)")
	flag.IntVar(&config.TargetLimit, "TargetLimit", 100, "Maximum number of targets that this instance will connect to at once (default: 100)")
	zkHosts := flag.String("ZookeeperHosts", "127.0.0.1:2181", "Comma separated (no spaces) list of zookeeper hosts including port (default: 127.0.0.1:2181)")
	flag.DurationVar(&config.ZookeeperTimeout, "ZookeeperTimeout", 1*time.Second, "Zookeeper timeout time. Minimum is 1 second. Failover time is (ZookeeperTimeout * 2). (default: 1s)")

	flag.Parse()
	config.ZookeeperHosts = strings.Split(*zkHosts, ",")
}

// StartGateway starts up all of the loaders and exporters provided by GatewayStartOpts. This is the
// primary way the server should be started.
func StartGateway(config *configuration.GatewayConfig, opts *GatewayStartOpts) error {
	config.Log.Info().Msg("Starting GNMI Gateway.")
	connMgr, err := connections.NewConnectionManagerDefault(config)
	if err != nil {
		config.Log.Error().Err(err).Msg("Unable to create connection manager.")
		os.Exit(1)
	}
	config.Log.Info().Msg("Starting connection manager.")
	if err := connMgr.Start(); err != nil {
		config.Log.Error().Err(err).Msgf("Unable to start connection manager: %v", err)
		return err
	}

	// channel to listen for errors from child goroutines
	finished := make(chan error, 1)

	go func() {
		err := opts.TargetLoader.Start()
		if err != nil {
			config.Log.Error().Err(err).Msgf("Unable to start target loader %T", opts.TargetLoader)
			finished <- err
		}
		opts.TargetLoader.WatchConfiguration(connMgr.TargetConfigChan())
	}()

	if config.EnableServer {
		config.Log.Info().Msg("Starting gNMI server.")
		go func() {
			if err := StartServer(config, connMgr.Cache()); err != nil {
				config.Log.Error().Err(err).Msg("Unable to start gNMI server.")
				finished <- err
			}
		}()
	}

	for _, exporter := range opts.Exporters {
		go func(exporter exporters.Exporter) {
			err := exporter.Start(connMgr.Cache())
			if err != nil {
				config.Log.Error().Err(err).Msgf("Unable to start exporter %T", exporter)
				finished <- err
			}
			// TODO: Call SetClient here as it's pretty universal.
		}(exporter)
	}

	return <-finished
}
<|endoftext|>