<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"encoding/json"

	"github.com/emicklei/proto"
	"github.com/jamesk/apidoc-proto/apidoc"
)

func main() {
	aSpec, err := apidoc.GetSpecFromFile("test_service.json")
	if err != nil {
		panic(err)
	}

	pFile := getProtoFromAPISpec(aSpec)

	//TODO: safe name?
	f, err := os.Create(aSpec.Name + ".proto")
	if err != nil {
		panic(err)
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	proto.NewFormatter(w, "  ").Format(&pFile)
	w.Flush()
}

const (
	syntaxVersion = "proto3"
)

func getProtoFromAPISpec(spec apidoc.Spec) proto.Proto {
	pFile := proto.Proto{}

	pFile.Elements = append(
		pFile.Elements,
		&proto.Syntax{Value: syntaxVersion},
	)

	for _, aEnum := range spec.Enums {
		pEnum := proto.Enum{Name: aEnum.Name}
		for i, aValue := range aEnum.Values {

			pEnum.Elements = append(
				pEnum.Elements,
				&proto.EnumField{Name: aValue.Name, Integer: i},
			)
		}
		pFile.Elements = append(pFile.Elements, &pEnum)
	}

	//Models
	for _, aModel := range spec.Models {
		pMessage := proto.Message{Name: aModel.Name}
		for i, aField := range aModel.Fields {
			field, err := getProtoFieldFromApidoc(aField, i+1) //TODO: how to handle sequence number changes
			if err != nil {
				panic(err)
			}

			pMessage.Elements = append(
				pMessage.Elements,
				field,
			)
		}
		pFile.Elements = append(pFile.Elements, &pMessage)
	}

	//Unions
	for _, aUnion := range spec.Unions {
		pMessage := proto.Message{Name: aUnion.Name}
		oneOff := proto.Oneof{Name: aUnion.Discriminator}
		if len(oneOff.Name) == 0 {
			oneOff.Name = "union"
		}

		for i, aUnionType := range aUnion.UnionTypes {
			pType := getProtoTypeFromBasicApidocType(aUnionType.TypeValue) //TODO: how to handle sequence number changes
			if len(pType) == 0 {
				panic(fmt.Sprintf("Couldn't find a proto type from union type [%v]", aUnionType))
			}

			//TODO: OPINION: I decided to use the type name here, should convert to underscore case
			field := proto.Field{Name: pType, Type: pType, Sequence: i + 1}

			pMessage.Elements = append(
				pMessage.Elements,
				&proto.OneOfField{Field: &field},
			)
		}
		pFile.Elements = append(pFile.Elements, &pMessage)
	}

	//Resources
	var nameReplace ServiceNameReplaceAttribute
	for _, attr := range spec.Attributes {
		if attr.Name == ServiceNameReplaceAttributeName {
			//TODO: inefficient, any actual performance impact?
			d, err := json.Marshal(attr.Value)
			fmt.Println(attr.Value)
			if err != nil {
				panic(err)
			}

			json.Unmarshal(d, &nameReplace)
			fmt.Println(nameReplace)
		}
	}
	service := proto.Service{Name: getSafeServiceName(spec.Name, nameReplace)}
	pFile.Elements = append(pFile.Elements, &service)

	for _, resource := range spec.Resources {
		for _, operation := range resource.Operations {
			rpc := proto.RPC{}

			rpc.Name = getProtoMethodName(resource, operation)
			//TODO: Handle gathering path, query and body params into one argument (or don't bother??)
			service.Elements = append(service.Elements, &rpc)
		}
	}

	return pFile
}

var ProtoIdentifierRegex = regexp.MustCompile(`[A-Za-z_][\w_]*`)

const (
	ServiceNameReplaceAttributeName = "aproto:service-name-replace"
)

/*
{
      "name": "aproto:service-name-replace",
      "value": {
        "regexMaps" : [{"regex": "[^\\w_]", "replace": "_"}]
      }
    }
*/
type ServiceNameReplaceAttribute struct {
	RegexMaps []ServiceNameReplaceAttributeRegexMap `json:"regexMaps"`
}

type ServiceNameReplaceAttributeRegexMap struct {
	Regex   string `json:"regex"`   //The regex to search for
	Replace string `json:"replace"` //The replace string, see regexp.Regex.Expand for details
}

//Gets a safe service name, proto expects /[A-Za-z_][\w_]*/ for service names
func getSafeServiceName(name string, replaceAttribute ServiceNameReplaceAttribute) string {
	replacedName := name
	for _, m := range replaceAttribute.RegexMaps {
		r, err := regexp.Compile(m.Regex)
		if err != nil {
			panic(err)
		}

		fmt.Printf("Replacing, name currently: [%v], regex [%v]\n", replacedName, m.Regex)
		replacedName = r.ReplaceAllString(replacedName, m.Replace)
		fmt.Printf("Done replacing, name currently: [%v]\n", replacedName)
	}

	pName := convertPartsToPascalCase(strings.Split(replacedName, " "))
	if len(ProtoIdentifierRegex.FindString(pName)) != len(pName) {
		panic(fmt.Sprintf(
			"Invalid proto name for service, processed name was [%v], original name was [%v] ([%v] after replacing) but protobuf identifier must follow the regex [%v]",
			pName,
			name,
			replacedName,
			ProtoIdentifierRegex.String(),
		))
	}

	return pName
}

func convertPartsToPascalCase(parts []string) string {
	name := ""
	for _, part := range parts {
		if len(part) == 0 {
			continue
		}

		name += strings.ToUpper(part[:1]) + part[1:]
	}

	return name
}

func getProtoMethodName(resource apidoc.Resource, operation apidoc.Operation) string {
	path := resource.Path
	if len(path) == 0 {
		path = resource.ResourceType
	}

	parts := []string{}
	method := strings.ToLower(operation.Method)
	if len(method) != 0 {
		method = strings.ToUpper(method[:1]) + method[1:]
		parts = append(parts, method)
	}
	parts = append(parts, strings.Split(path, "/")...)
	parts = append(parts, strings.Split(operation.Path, "/")...)

	return convertPathSegmentsToProtoName(parts)
}

func convertPathSegmentsToProtoName(parts []string) string {
	name := ""
	for _, part := range parts {
		if len(part) == 0 {
			continue
		}
		if part[0] == ':' {
			continue
			//TODO: panic("Cannot handle path variables")
		}

		name += strings.ToUpper(part[:1]) + part[1:]
	}

	return name
}

func getProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (proto.Visitee, error) {
	if strings.HasPrefix(aField.FieldType, "map[") {
		return getMapProtoFieldFromApidoc(aField, sequenceNumber)
	}

	return getNormalProtoFieldFromApidoc(aField, sequenceNumber)
}

func getNormalProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (*proto.NormalField, error) {
	pField := proto.NormalField{Field: &proto.Field{}}

	fieldType := aField.FieldType
	if strings.HasPrefix(fieldType, "[") {
		if !strings.HasSuffix(fieldType, "]") {
			return nil, errors.New("Invalid type, starts with a [ but does not end with one")
		}

		pField.Repeated = true
		fieldType = fieldType[1 : len(fieldType)-1]
	}

	pType := getProtoTypeFromBasicApidocType(fieldType)
	if len(pType) == 0 {
		return nil, createUnsupportedError(aField.Name, pType)
	}
	pField.Type = pType

	pField.Sequence = sequenceNumber
	//TODO: Translate names as per proto styles? https://developers.google.com/protocol-buffers/docs/style#message-and-field-names
	pField.Name = aField.Name

	return &pField, nil
}

func getMapProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (*proto.MapField, error) {
	return &proto.MapField{}, createUnsupportedError(aField.Name, "map")
}

func getProtoTypeFromBasicApidocType(basicType string) string {
	switch basicType {
	case "boolean":
		return "bool"
	case "date-iso8601":
		return "string"
	case "date-time-iso8601":
		return "string"
	case "decimal":
		return ""
	case "double":
		return "double"
	case "integer":
		return "int32"
	case "long":
		return "int64"
	case "object":
		return ""
	case "string":
		return "string"
	case "unit":
		return ""
	case "uuid":
		return "string"
	default:
		//Custom type or wrong type, can't tell here
		return basicType
	}
}

type UnsupportedTypeError error

func createUnsupportedError(fieldName string, fieldType string) error {
	return UnsupportedTypeError(errors.New(fmt.Sprintf("Cannot translate field [%s], field type: [%s] is unsupported", fieldName, fieldType)))
}
<commit_msg>Started adding rpc params<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"encoding/json"

	"github.com/emicklei/proto"
	"github.com/jamesk/apidoc-proto/apidoc"
)

func main() {
	aSpec, err := apidoc.GetSpecFromFile("test_service.json")
	if err != nil {
		panic(err)
	}

	pFile := getProtoFromAPISpec(aSpec)

	//TODO: safe name?
	f, err := os.Create(aSpec.Name + ".proto")
	if err != nil {
		panic(err)
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	proto.NewFormatter(w, "  ").Format(&pFile)
	w.Flush()
}

const (
	syntaxVersion = "proto3"
)

func getProtoFromAPISpec(spec apidoc.Spec) proto.Proto {
	pFile := proto.Proto{}

	pFile.Elements = append(
		pFile.Elements,
		&proto.Syntax{Value: syntaxVersion},
	)

	for _, aEnum := range spec.Enums {
		pEnum := proto.Enum{Name: aEnum.Name}
		for i, aValue := range aEnum.Values {

			pEnum.Elements = append(
				pEnum.Elements,
				&proto.EnumField{Name: aValue.Name, Integer: i},
			)
		}
		pFile.Elements = append(pFile.Elements, &pEnum)
	}

	//Models
	for _, aModel := range spec.Models {
		pMessage := proto.Message{Name: aModel.Name}
		for i, aField := range aModel.Fields {
			field, err := getProtoFieldFromApidoc(aField, i+1) //TODO: how to handle sequence number changes
			if err != nil {
				panic(err)
			}

			pMessage.Elements = append(
				pMessage.Elements,
				field,
			)
		}
		pFile.Elements = append(pFile.Elements, &pMessage)
	}

	//Unions
	for _, aUnion := range spec.Unions {
		pMessage := proto.Message{Name: aUnion.Name}
		oneOff := proto.Oneof{Name: aUnion.Discriminator}
		if len(oneOff.Name) == 0 {
			oneOff.Name = "union"
		}

		for i, aUnionType := range aUnion.UnionTypes {
			pType := getProtoTypeFromBasicApidocType(aUnionType.TypeValue) //TODO: how to handle sequence number changes
			if len(pType) == 0 {
				panic(fmt.Sprintf("Couldn't find a proto type from union type [%v]", aUnionType))
			}

			//TODO: OPINION: I decided to use the type name here, should convert to underscore case
			field := proto.Field{Name: pType, Type: pType, Sequence: i + 1}

			pMessage.Elements = append(
				pMessage.Elements,
				&proto.OneOfField{Field: &field},
			)
		}
		pFile.Elements = append(pFile.Elements, &pMessage)
	}

	//Resources
	var nameReplace ServiceNameReplaceAttribute
	for _, attr := range spec.Attributes {
		if attr.Name == ServiceNameReplaceAttributeName {
			//TODO: inefficient, any actual performance impact?
			d, err := json.Marshal(attr.Value)
			if err != nil {
				panic(err)
			}

			json.Unmarshal(d, &nameReplace)
		}
	}
	service := proto.Service{Name: getSafeServiceName(spec.Name, nameReplace)}
	pFile.Elements = append(pFile.Elements, &service)

	for _, resource := range spec.Resources {
		for _, operation := range resource.Operations {
			rpc := proto.RPC{}

			rpc.Name = getProtoMethodName(resource, operation)
			//TODO: Handle path, query and body params?? Attributes to handle behaviour?
			if len(operation.Body.BodyType) != 0 {
				_, isMap, err := getMapType(operation.Body.BodyType)
				if err != nil {
					panic(err)
				}

				if isMap {
					panic("Map bodys is unsupported at the moment")
				}

				arrayType, isArray, err := getArrayType(operation.Body.BodyType)
				if err != nil {
					panic(err)
				}

				rpc.StreamsRequest = isArray
				rpc.RequestType = getProtoTypeFromBasicApidocType(arrayType)
			} else {
				rpc.RequestType = "google.protobuf.Empty"
			}

			service.Elements = append(service.Elements, &rpc)
		}
	}

	return pFile
}

var ProtoIdentifierRegex = regexp.MustCompile(`[A-Za-z_][\w_]*`)

const (
	ServiceNameReplaceAttributeName = "aproto:service-name-replace"
)

/*
{
      "name": "aproto:service-name-replace",
      "value": {
        "regexMaps" : [{"regex": "[^\\w_]", "replace": "_"}]
      }
    }
*/
type ServiceNameReplaceAttribute struct {
	RegexMaps []ServiceNameReplaceAttributeRegexMap `json:"regexMaps"`
}

type ServiceNameReplaceAttributeRegexMap struct {
	Regex   string `json:"regex"`   //The regex to search for
	Replace string `json:"replace"` //The replace string, see regexp.Regex.Expand for details
}

//Gets a safe service name, proto expects /[A-Za-z_][\w_]*/ for service names
func getSafeServiceName(name string, replaceAttribute ServiceNameReplaceAttribute) string {
	replacedName := name
	for _, m := range replaceAttribute.RegexMaps {
		r, err := regexp.Compile(m.Regex)
		if err != nil {
			panic(err)
		}

		fmt.Printf("Replacing, name currently: [%v], regex [%v]\n", replacedName, m.Regex)
		replacedName = r.ReplaceAllString(replacedName, m.Replace)
		fmt.Printf("Done replacing, name currently: [%v]\n", replacedName)
	}

	pName := convertPartsToPascalCase(strings.Split(replacedName, " "))
	if len(ProtoIdentifierRegex.FindString(pName)) != len(pName) {
		panic(fmt.Sprintf(
			"Invalid proto name for service, processed name was [%v], original name was [%v] ([%v] after replacing) but protobuf identifier must follow the regex [%v]",
			pName,
			name,
			replacedName,
			ProtoIdentifierRegex.String(),
		))
	}

	return pName
}

func convertPartsToPascalCase(parts []string) string {
	name := ""
	for _, part := range parts {
		if len(part) == 0 {
			continue
		}

		name += strings.ToUpper(part[:1]) + part[1:]
	}

	return name
}

func getProtoMethodName(resource apidoc.Resource, operation apidoc.Operation) string {
	path := resource.Path
	if len(path) == 0 {
		path = resource.ResourceType
	}

	parts := []string{}
	//TODO: attribute to ignore method?
	method := strings.ToLower(operation.Method)
	if len(method) != 0 {
		method = strings.ToUpper(method[:1]) + method[1:]
		parts = append(parts, method)
	}
	parts = append(parts, strings.Split(path, "/")...)
	parts = append(parts, strings.Split(operation.Path, "/")...)

	return convertPathSegmentsToProtoName(parts)
}

func convertPathSegmentsToProtoName(parts []string) string {
	name := ""
	for _, part := range parts {
		if len(part) == 0 {
			continue
		}
		if part[0] == ':' {
			continue
			//TODO: panic("Cannot handle path variables")
		}

		name += strings.ToUpper(part[:1]) + part[1:]
	}

	return name
}

func getProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (proto.Visitee, error) {
	mapType, isMap, err := getMapType(aField.FieldType)
	if err != nil {
		return nil, err
	}

	if isMap {
		return getMapProtoFieldFromApidoc(aField, mapType, sequenceNumber)
	}
	return getNormalProtoFieldFromApidoc(aField, sequenceNumber)
}

func getMapType(value string) (string, bool, error) {
	mapType := value
	if strings.HasPrefix(mapType, "map[") {
		if !strings.HasSuffix(mapType, "]") {
			return value, false, fmt.Errorf("Invalid type, starts with a [ but does not end with one, type was [%v]", value)
		}

		return mapType[4 : len(mapType)-1], true, nil
	}

	return value, false, nil
}

func getArrayType(value string) (string, bool, error) {
	arrayType := value
	if strings.HasPrefix(arrayType, "[") {
		if !strings.HasSuffix(arrayType, "]") {
			return value, false, fmt.Errorf("Invalid type, starts with a [ but does not end with one, type was [%v]", value)
		}

		return arrayType[1 : len(arrayType)-1], true, nil
	}

	return value, false, nil
}

func getNormalProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (*proto.NormalField, error) {
	pField := proto.NormalField{Field: &proto.Field{}}

	fieldType, isArray, err := getArrayType(aField.FieldType)
	if err != nil {
		return nil, err
	}

	pField.Repeated = isArray
	fieldType = fieldType[1 : len(fieldType)-1]

	pType := getProtoTypeFromBasicApidocType(fieldType)
	if len(pType) == 0 {
		return nil, createUnsupportedError(aField.Name, pType)
	}
	pField.Type = pType

	pField.Sequence = sequenceNumber
	//TODO: Translate names as per proto styles? https://developers.google.com/protocol-buffers/docs/style#message-and-field-names
	pField.Name = aField.Name

	return &pField, nil
}

func getMapProtoFieldFromApidoc(aField apidoc.Field, mapType string, sequenceNumber int) (*proto.MapField, error) {
	return &proto.MapField{}, createUnsupportedError(aField.Name, "map")
}

//Input is a "basic" type i.e. not a map or array
func getProtoTypeFromBasicApidocType(basicType string) string {
	switch basicType {
	case "boolean":
		return "bool"
	case "date-iso8601":
		return "string"
	case "date-time-iso8601":
		return "string"
	case "decimal":
		return ""
	case "double":
		return "double"
	case "integer":
		return "int32"
	case "long":
		return "int64"
	case "object":
		return ""
	case "string":
		return "string"
	case "unit":
		return ""
	case "uuid":
		return "string"
	default:
		//Custom type or wrong type, can't tell here
		return basicType
	}
}

type UnsupportedTypeError error

func createUnsupportedError(fieldName string, fieldType string) error {
	return UnsupportedTypeError(errors.New(fmt.Sprintf("Cannot translate field [%s], field type: [%s] is unsupported", fieldName, fieldType)))
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

const (
	MaxErrors     = 3
	MaxPages      = 1
	MinReputation = 400
	APIKeyPath    = "./_secret/api.key"
	ApiURL        = "https://api.stackexchange.com/2.2/users?page="
	CQuery        = "pagesize=100&order=desc&sort=reputation&site=stackoverflow"
)

type SOUsers struct {
	Items []struct {
		BadgeCounts struct {
			Bronze int `json:"bronze"`
			Silver int `json:"silver"`
			Gold   int `json:"gold"`
		} `json:"badge_counts"`
		AccountID               int    `json:"account_id"`
		IsEmployee              bool   `json:"is_employee"`
		LastModifiedDate        int    `json:"last_modified_date"`
		LastAccessDate          int    `json:"last_access_date"`
		Age                     int    `json:"age,omitempty"`
		ReputationChangeYear    int    `json:"reputation_change_year"`
		ReputationChangeQuarter int    `json:"reputation_change_quarter"`
		ReputationChangeMonth   int    `json:"reputation_change_month"`
		ReputationChangeWeek    int    `json:"reputation_change_week"`
		ReputationChangeDay     int    `json:"reputation_change_day"`
		Reputation              int    `json:"reputation"`
		CreationDate            int    `json:"creation_date"`
		UserType                string `json:"user_type"`
		UserID                  int    `json:"user_id"`
		AcceptRate              int    `json:"accept_rate,omitempty"`
		Location                string `json:"location,omitempty"`
		WebsiteURL              string `json:"website_url,omitempty"`
		Link                    string `json:"link"`
		ProfileImage            string `json:"profile_image"`
		DisplayName             string `json:"display_name"`
	} `json:"items"`
	HasMore        bool `json:"has_more"`
	QuotaMax       int  `json:"quota_max"`
	QuotaRemaining int  `json:"quota_remaining"`
}

var (
	Trace    *log.Logger
	Info     *log.Logger
	Warning  *log.Logger
	Error    *log.Logger
	location = flag.String("location", "spain", "location")
	jsonfile = flag.String("json", "", "json")
	limit    = flag.Int("limit", 20, "max number of records")
)

func Init(
	traceHandle io.Writer,
	infoHandle io.Writer,
	warningHandle io.Writer,
	errorHandle io.Writer) {

	Trace = log.New(traceHandle,
		"TRACE: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Info = log.New(infoHandle,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(warningHandle,
		"WARN: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(errorHandle,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)

}

func Decode(r io.Reader) (users *SOUsers, err error) {

	users = new(SOUsers)
	return users, json.NewDecoder(r).Decode(users)
}

func StreamHTTP(page int, key string) (users *SOUsers, err error) {

	var reader io.ReadCloser

	url := fmt.Sprintf("%s%d&%s%s", ApiURL, page, CQuery, key)
	Trace.Println(url)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		Trace.Println(err)
		return users, err
	}

	req.Header.Set("Accept-Encoding", "gzip")

	response, err := http.DefaultClient.Do(req)
	if err != nil {
		Trace.Println(err)
		return users, err
	}
	defer response.Body.Close()

	switch response.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err = gzip.NewReader(response.Body)
		if err != nil {
			Trace.Println(err)
			return users, err
		}
		defer reader.Close()
	default:
		reader = response.Body
	}
	return Decode(reader)
}

func StreamFile(jsonfile string) (users *SOUsers, err error) {
	reader, err := os.Open(jsonfile)
	defer reader.Close()

	return Decode(reader)
}

func GetUserInfo(users *SOUsers, location *regexp.Regexp, counter *int, limit int) (rep bool) {

	for _, user := range users.Items {
		if user.Reputation < MinReputation {
			return false
		}
		if location.MatchString(user.Location) {
			*counter += 1
			if *counter == 1 {
				fmt.Printf("%4s %-30s %6s %-30s\n", "Rank", "Name", "Rep", "Location")
			}

			if *counter > limit && limit != 0 {
				return false
			}

			fmt.Printf("%4d %-30s %6d %-30s %s\n", *counter, html.UnescapeString(user.DisplayName),
				user.Reputation, html.UnescapeString(user.Location), user.WebsiteURL)

		}
	}
	return true
}

func GetKey() (key string, err error) {
	_, err = os.Stat(APIKeyPath)

	if err != nil {
		return "", fmt.Errorf("Can't find API key: %s", APIKeyPath)
	}

	strkey, err := ioutil.ReadFile(APIKeyPath)
	if err != nil {
		return "", fmt.Errorf("Can't load API key: %s", err)
	}

	return fmt.Sprintf("&key=%s", strings.TrimRight(string(strkey)[:], "\n")), nil
}

func main() {
	flag.Parse()

	Init(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
	Trace.Println("location: ", *location)
	Trace.Println("json: ", *jsonfile)
	Trace.Println("limit: ", *limit)

	re := regexp.MustCompile(fmt.Sprintf("(?i)%s", *location))

	stop := false
	streamErrors := 0
	currentPage := 1
	counter := 0

	var users *SOUsers

	for {
		if *jsonfile == "" {
			key, err := GetKey()
			if err != nil {
				Warning.Println(err)
			}
			users, err = StreamHTTP(currentPage, key)
			if err != nil || len(users.Items) == 0 {

				Warning.Println("Can't stream data.")
				streamErrors += 1
				if streamErrors >= MaxErrors {
					Error.Println("Max retry number reached")
					os.Exit(5)
				}
				continue
			}
		} else {
			var err error
			users, err = StreamFile(*jsonfile)
			if err != nil {
				Error.Println("Can't decode json file.")
				os.Exit(5)
			}
			stop = true
		}

		repLimit := GetUserInfo(users, re, &counter, *limit)
		if !repLimit {
			break
		}

		currentPage += 1
		if (currentPage >= MaxPages && MaxPages != 0) || !users.HasMore || stop {
			break
		}
	}
	Trace.Printf("%d users founded.\n", counter)
}
<commit_msg>user rank JSON struct<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

const (
	MaxErrors     = 3
	MaxPages      = 1
	MinReputation = 400
	APIKeyPath    = "./_secret/api.key"
	ApiURL        = "https://api.stackexchange.com/2.2/users?page="
	CQuery        = "pagesize=100&order=desc&sort=reputation&site=stackoverflow"
)

type SOUsers struct {
	Items []struct {
		BadgeCounts struct {
			Bronze int `json:"bronze"`
			Silver int `json:"silver"`
			Gold   int `json:"gold"`
		} `json:"badge_counts"`
		AccountID               int    `json:"account_id"`
		IsEmployee              bool   `json:"is_employee"`
		LastModifiedDate        int    `json:"last_modified_date"`
		LastAccessDate          int    `json:"last_access_date"`
		Age                     int    `json:"age,omitempty"`
		ReputationChangeYear    int    `json:"reputation_change_year"`
		ReputationChangeQuarter int    `json:"reputation_change_quarter"`
		ReputationChangeMonth   int    `json:"reputation_change_month"`
		ReputationChangeWeek    int    `json:"reputation_change_week"`
		ReputationChangeDay     int    `json:"reputation_change_day"`
		Reputation              int    `json:"reputation"`
		CreationDate            int    `json:"creation_date"`
		UserType                string `json:"user_type"`
		UserID                  int    `json:"user_id"`
		AcceptRate              int    `json:"accept_rate,omitempty"`
		Location                string `json:"location,omitempty"`
		WebsiteURL              string `json:"website_url,omitempty"`
		Link                    string `json:"link"`
		ProfileImage            string `json:"profile_image"`
		DisplayName             string `json:"display_name"`
	} `json:"items"`
	HasMore        bool `json:"has_more"`
	QuotaMax       int  `json:"quota_max"`
	QuotaRemaining int  `json:"quota_remaining"`
}

type SOUserRank struct {
	Rank         int    `json:"rank"`
	AccountID    int    `json:"account_id"`
	DisplayName  string `json:"display_name"`
	Reputation   int    `json:"reputation"`
	Location     string `json:"location,omitempty"`
	WebsiteURL   string `json:"website_url,omitempty"`
	Link         string `json:"link"`
	ProfileImage string `json:"profile_image"`
}

type Ranks []SOUserRank

var (
	Trace    *log.Logger
	Info     *log.Logger
	Warning  *log.Logger
	Error    *log.Logger
	location = flag.String("location", "spain", "location")
	jsonfile = flag.String("json", "", "json")
	limit    = flag.Int("limit", 20, "max number of records")
)

func Init(
	traceHandle io.Writer,
	infoHandle io.Writer,
	warningHandle io.Writer,
	errorHandle io.Writer) {

	Trace = log.New(traceHandle,
		"TRACE: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Info = log.New(infoHandle,
		"INFO: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Warning = log.New(warningHandle,
		"WARN: ",
		log.Ldate|log.Ltime|log.Lshortfile)

	Error = log.New(errorHandle,
		"ERROR: ",
		log.Ldate|log.Ltime|log.Lshortfile)

}

func Decode(r io.Reader) (users *SOUsers, err error) {

	users = new(SOUsers)
	return users, json.NewDecoder(r).Decode(users)
}

func StreamHTTP(page int, key string) (users *SOUsers, err error) {

	var reader io.ReadCloser

	url := fmt.Sprintf("%s%d&%s%s", ApiURL, page, CQuery, key)
	Trace.Println(url)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		Trace.Println(err)
		return users, err
	}

	req.Header.Set("Accept-Encoding", "gzip")

	response, err := http.DefaultClient.Do(req)
	if err != nil {
		Trace.Println(err)
		return users, err
	}
	defer response.Body.Close()

	switch response.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err = gzip.NewReader(response.Body)
		if err != nil {
			Trace.Println(err)
			return users, err
		}
		defer reader.Close()
	default:
		reader = response.Body
	}
	return Decode(reader)
}

func StreamFile(jsonfile string) (users *SOUsers, err error) {
	reader, err := os.Open(jsonfile)
	defer reader.Close()

	return Decode(reader)
}

func GetUserInfo(users *SOUsers, location *regexp.Regexp, counter *int, limit int, ranks *Ranks) (rep bool) {

	for _, user := range users.Items {
		if user.Reputation < MinReputation {
			return false
		}
		if location.MatchString(user.Location) {
			*counter += 1
			if *counter == 1 {
				fmt.Printf("%4s %-30s %6s %-30s\n", "Rank", "Name", "Rep", "Location")
			}

			if *counter > limit && limit != 0 {
				return false
			}
			s := SOUserRank{Rank: *counter,
				AccountID:    user.AccountID,
				DisplayName:  user.DisplayName,
				Reputation:   user.Reputation,
				Location:     user.Location,
				WebsiteURL:   user.WebsiteURL,
				Link:         user.Link,
				ProfileImage: user.ProfileImage}

			*ranks = append(*ranks, s)

			fmt.Printf("%4d %-30s %6d %-30s %s\n", *counter, html.UnescapeString(user.DisplayName),
				user.Reputation, html.UnescapeString(user.Location), user.WebsiteURL)

		}
	}
	return true
}

func GetKey() (key string, err error) {
	_, err = os.Stat(APIKeyPath)

	if err != nil {
		return "", fmt.Errorf("Can't find API key: %s", APIKeyPath)
	}

	strkey, err := ioutil.ReadFile(APIKeyPath)
	if err != nil {
		return "", fmt.Errorf("Can't load API key: %s", err)
	}

	return fmt.Sprintf("&key=%s", strings.TrimRight(string(strkey)[:], "\n")), nil
}

func main() {
	flag.Parse()

	Init(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)
	Trace.Println("location: ", *location)
	Trace.Println("json: ", *jsonfile)
	Trace.Println("limit: ", *limit)

	re := regexp.MustCompile(fmt.Sprintf("(?i)%s", *location))

	stop := false
	streamErrors := 0
	currentPage := 1
	counter := 0

	var users *SOUsers
	var ranks Ranks

	for {
		if *jsonfile == "" {
			key, err := GetKey()
			if err != nil {
				Warning.Println(err)
			}
			users, err = StreamHTTP(currentPage, key)
			if err != nil || len(users.Items) == 0 {

				Warning.Println("Can't stream data.")
				streamErrors += 1
				if streamErrors >= MaxErrors {
					Error.Println("Max retry number reached")
					os.Exit(5)
				}
				continue
			}
		} else {
			var err error
			users, err = StreamFile(*jsonfile)
			if err != nil {
				Error.Println("Can't decode json file.")
				os.Exit(5)
			}
			stop = true
		}

		repLimit := GetUserInfo(users, re, &counter, *limit, &ranks)
		if !repLimit {
			break
		}

		currentPage += 1
		if (currentPage >= MaxPages && MaxPages != 0) || !users.HasMore || stop {
			break
		}
	}
	Trace.Printf("%d users founded.\n", counter)
}
<|endoftext|>


<commit_before>/* httpstress-go is a CLI interface for httpstress library.
Use it for stress testing of HTTP servers with many concurrent connections.

Usage: httpstress-go -c {concurrent} -n {total} {URL list}
e.g. httpstress-go -c 1000 -n 2000 http://localhost http://google.com

{concurrent} defaults to 1, {total} is optional.

Returns 0 if no errors, 1 if some errors (see stdout) and 2 in case of invalid options.

Please note that this utility uses GOMAXPROCS environment variable if it's present.
If not, this defaults to CPU count + 1. */
package main

/* Copyright 2014 Chai Chillum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

import (
	"flag"
	. "fmt"
	. "github.com/chillum/httpstress"
	. "os"
	. "runtime"
)

func main() {
	var conn, max int
	flag.IntVar(&conn, "c", 1, "concurrent connections count")
	flag.IntVar(&max, "n", 0, "total connections (optional)")
	flag.Parse()

	urls := flag.Args()
	if len(urls) < 1 {
		Println("Usage:", Args[0], "<http://url1> [http://url2] ... [http://urlN]")
		Exit(2)
	}

	if Getenv("GOMAXPROCS") == "" {
		GOMAXPROCS(NumCPU() + 1)
	}

	out, err := Test(conn, max, urls)
	if err != nil {
		Println("ERROR:", err)
		Exit(2)
	}
	if len(out) > 0 {
		Println("Test finished. Failed requests:")
		for url, num := range out {
			Print(" ", url, ": ", num, "\n")
		}
		Exit(1)
	} else {
		Println("Test finished. No failed requests.")
	}
}
<commit_msg>whitespace<commit_after>/* httpstress-go is a CLI interface for httpstress library.
Use it for stress testing of HTTP servers with many concurrent connections.

Usage: httpstress-go -c {concurrent} -n {total} {URL list}
e.g. httpstress-go -c 1000 -n 2000 http://localhost http://google.com

{concurrent} defaults to 1, {total} is optional.

Returns 0 if no errors, 1 if some errors (see stdout) and 2 in case of invalid options.

Please note that this utility uses GOMAXPROCS environment variable if it's present.
If not, this defaults to CPU count + 1. */
package main
/* Copyright 2014 Chai Chillum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

import (
	"flag"
	. "fmt"
	. "github.com/chillum/httpstress"
	. "os"
	. "runtime"
)

func main() {
	var conn, max int
	flag.IntVar(&conn, "c", 1, "concurrent connections count")
	flag.IntVar(&max, "n", 0, "total connections (optional)")
	flag.Parse()

	urls := flag.Args()
	if len(urls) < 1 {
		Println("Usage:", Args[0], "<http://url1> [http://url2] ... [http://urlN]")
		Exit(2)
	}

	if Getenv("GOMAXPROCS") == "" {
		GOMAXPROCS(NumCPU() + 1)
	}

	out, err := Test(conn, max, urls)
	if err != nil {
		Println("ERROR:", err)
		Exit(2)
	}
	if len(out) > 0 {
		Println("Test finished. Failed requests:")
		for url, num := range out {
			Print(" ", url, ": ", num, "\n")
		}
		Exit(1)
	} else {
		Println("Test finished. No failed requests.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"net/http"

	"github.com/Tinker-Ware/gh-service/infrastructure"
	"github.com/Tinker-Ware/gh-service/interfaces"
	"github.com/Tinker-Ware/gh-service/usecases"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
)

const defaultPath = "/etc/gh-service.conf"
const apiVersion = "/v1"

// Define configuration flags
var confFilePath = flag.String("conf", defaultPath, "Custom path for configuration file")

func main() {

	flag.Parse()

	config, err := infrastructure.GetConfiguration(*confFilePath)
	if err != nil {
		fmt.Println(err.Error())
		panic("Cannot parse configuration")
	}

	ghrepo, err := interfaces.NewGithubRepository(config.ClientID, config.ClientSecret, config.Scopes)
	if err != nil {
		panic(err.Error())
	}

	ghinteractor := usecases.GHInteractor{
		GithubRepository: ghrepo,
	}

	store := sessions.NewCookieStore([]byte("something-very-secret"))

	handler := interfaces.WebServiceHandler{
		GHInteractor: ghinteractor,
		Sessions:     store,
	}

	r := mux.NewRouter()

	versionSubrouter := r.PathPrefix(apiVersion).Subrouter()

	subrouter := versionSubrouter.PathPrefix("/api/v1/repository/github/").Subrouter()
	subrouter.HandleFunc("/gh_callback", handler.Login)
	subrouter.HandleFunc("/github_oauth_cb", handler.Callback)
	subrouter.Handle("/user/{username}", interfaces.Adapt(http.HandlerFunc(handler.ShowUser), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	subrouter.Handle("/user/{username}/repos", interfaces.Adapt(http.HandlerFunc(handler.ShowRepos), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	subrouter.Handle("/user/{username}/repos", interfaces.Adapt(http.HandlerFunc(handler.CreateRepo), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	subrouter.Handle("/user/{username}/keys", interfaces.Adapt(http.HandlerFunc(handler.CreateRepo), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	subrouter.Handle("/user/{username}/keys", interfaces.Adapt(http.HandlerFunc(handler.CreateKey), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	subrouter.Handle("/user/{username}/keys/{id}", interfaces.Adapt(http.HandlerFunc(handler.ShowKey), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	subrouter.Handle("/user/{username}/{repo}", interfaces.Adapt(http.HandlerFunc(handler.ShowRepo), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	subrouter.Handle("/user/{username}/{repo}/addfile", interfaces.Adapt(http.HandlerFunc(handler.AddFileToRepository), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	subrouter.Handle("/user/{username}/{repo}/addfiles", interfaces.Adapt(http.HandlerFunc(handler.AddMultipleFilesToRepository), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	subrouter.HandleFunc("/user_info", handler.GetCurrentUser).Methods("GET")

	n := negroni.Classic()
	n.UseHandler(r)

	port := bytes.Buffer{}

	port.WriteString(":")
	port.WriteString(config.Port)

	n.Run(port.String())
}
<commit_msg>Remove all the unused routes<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"net/http"

	"github.com/Tinker-Ware/gh-service/infrastructure"
	"github.com/Tinker-Ware/gh-service/interfaces"
	"github.com/Tinker-Ware/gh-service/usecases"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
)

const defaultPath = "/etc/gh-service.conf"
const apiVersion = "/v1"

// Define configuration flags
var confFilePath = flag.String("conf", defaultPath, "Custom path for configuration file")

func main() {

	flag.Parse()

	config, err := infrastructure.GetConfiguration(*confFilePath)
	if err != nil {
		fmt.Println(err.Error())
		panic("Cannot parse configuration")
	}

	ghrepo, err := interfaces.NewGithubRepository(config.ClientID, config.ClientSecret, config.Scopes)
	if err != nil {
		panic(err.Error())
	}

	ghinteractor := usecases.GHInteractor{
		GithubRepository: ghrepo,
	}

	store := sessions.NewCookieStore([]byte("something-very-secret"))

	handler := interfaces.WebServiceHandler{
		GHInteractor: ghinteractor,
		Sessions:     store,
	}

	r := mux.NewRouter()

	versionSubrouter := r.PathPrefix(apiVersion).Subrouter()

	subrouter := versionSubrouter.PathPrefix("/api/v1/repository/github/").Subrouter()
	subrouter.HandleFunc("/gh_callback", handler.Login)
	subrouter.HandleFunc("/github_oauth_cb", handler.Callback)
	subrouter.Handle("/{username}/repos", interfaces.Adapt(http.HandlerFunc(handler.ShowRepos), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	subrouter.Handle("/{username}/{repo}", interfaces.Adapt(http.HandlerFunc(handler.ShowRepo), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	// subrouter.Handle("/user/{username}", interfaces.Adapt(http.HandlerFunc(handler.ShowUser), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	// subrouter.Handle("/user/{username}/repos", interfaces.Adapt(http.HandlerFunc(handler.CreateRepo), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	// subrouter.Handle("/user/{username}/keys", interfaces.Adapt(http.HandlerFunc(handler.CreateRepo), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	// subrouter.Handle("/user/{username}/keys", interfaces.Adapt(http.HandlerFunc(handler.CreateKey), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	// subrouter.Handle("/user/{username}/keys/{id}", interfaces.Adapt(http.HandlerFunc(handler.ShowKey), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("GET")
	// subrouter.Handle("/user/{username}/{repo}/addfile", interfaces.Adapt(http.HandlerFunc(handler.AddFileToRepository), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	// subrouter.Handle("/user/{username}/{repo}/addfiles", interfaces.Adapt(http.HandlerFunc(handler.AddMultipleFilesToRepository), interfaces.Notify(), interfaces.SetToken(ghrepo))).Methods("POST")
	// subrouter.HandleFunc("/user_info", handler.GetCurrentUser).Methods("GET")

	n := negroni.Classic()
	n.UseHandler(r)

	port := bytes.Buffer{}

	port.WriteString(":")
	port.WriteString(config.Port)

	n.Run(port.String())
}
<|endoftext|>


<commit_before>package zipkinproxy

import (
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/apache/thrift/lib/go/thrift"
	"github.com/flachnetz/dd-zipkin-proxy/datadog"
	"github.com/flachnetz/dd-zipkin-proxy/zipkin"
	"github.com/gorilla/handlers"
	"github.com/jessevdk/go-flags"
	"github.com/julienschmidt/httprouter"
	"github.com/openzipkin/zipkin-go-opentracing"
	"github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore"

	. "github.com/flachnetz/go-admin"
)

var log = logrus.WithField("prefix", "main")

func Main(spanConverter datadog.SpanConverterFunc) {
	var opts struct {
		ZipkinReporterUrl string `long:"zipkin-url" description:"Url for zipkin reporting."`
		DatadogReporting  bool   `long:"datadog-reporting" description:"Enable datadog trace reporting with the default url."`
		ListenAddr        string `long:"listen-address" default:":9411" description:"Address to listen for zipkin connections."`
		Verbose           bool   `long:"verbose" description:"Enable verbose debug logging."`
	}

	if _, err := flags.Parse(&opts); err != nil {
		os.Exit(1)
	}

	if opts.Verbose {
		logrus.SetLevel(logrus.DebugLevel)
	}

	var channels []chan<- *zipkincore.Span

	if opts.DatadogReporting {
		// report spans to datadog
		zipkinSpans := make(chan *zipkincore.Span, 512)
		datadogSpans := datadog.ConvertZipkinSpans(zipkinSpans, spanConverter)
		go datadog.ReportSpans(datadogSpans)
		channels = append(channels, zipkinSpans)
	}

	if opts.ZipkinReporterUrl != "" {
		log.Info("Enable zipkin reporting at url ", opts.ZipkinReporterUrl)

		collector, err := zipkintracer.NewHTTPCollector(opts.ZipkinReporterUrl,
			zipkintracer.HTTPBatchSize(5000),
			zipkintracer.HTTPLogger(funcLogger(log.Warn)))

		if err != nil {
			log.Fatal("Could not create zipkin http collector: ", err)
			return
		}

		zipkinSpans := make(chan *zipkincore.Span, 512)
		go zipkin.ReportSpans(collector, zipkinSpans)
		channels = append(channels, zipkinSpans)
	}

	// multiplex channels
	zipkinSpans := make(chan *zipkincore.Span, 16)
	go forwardSpansToChannels(zipkinSpans, channels)

	admin := NewAdminHandler("/admin", "dd-zipkin-proxy",
		WithDefaults(),
		WithForceGC(),
		WithPProfHandlers(),
		WithHeapDump())

	// listen for zipkin messages
	router := httprouter.New()
	registerAdminHandler(router, admin)
	router.POST("/api/v1/spans", handleSpans(zipkinSpans))
	log.Fatal(http.ListenAndServe(opts.ListenAddr, handlers.LoggingHandler(log.Logger.Writer(), router)))
}

func registerAdminHandler(router *httprouter.Router, handler http.Handler) {
	router.Handler("GET", "/", http.RedirectHandler("/admin", http.StatusTemporaryRedirect))

	methods := []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}
	for _, method := range methods {
		router.Handler(method, "/admin", handler)
		router.Handler(method, "/admin/*path", handler)
	}
}

func forwardSpansToChannels(source <-chan *zipkincore.Span, targets []chan<- *zipkincore.Span) {
	dropped := 0
	lastLogMessageTime := time.Now()

	for span := range source {
		for _, target := range targets {
			// send span, do not block if target is full
			select {
			case target <- span:
			default:
				dropped += 1
			}
		}

		if dropped > 0 && time.Now().Sub(lastLogMessageTime) > 1*time.Second {
			log.Warnf("Dropped %d spans because a target buffer was full.", dropped)
			dropped = 0
			lastLogMessageTime = time.Now()
		}
	}
}

func handleSpans(spans chan<- *zipkincore.Span) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		var bodyReader io.Reader = req.Body
		if req.Header.Get("Content-Encoding") == "gzip" {
			var err error
			bodyReader, err = gzip.NewReader(bodyReader)
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
		}

		body, err := ioutil.ReadAll(bodyReader)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		transport := thrift.NewStreamTransportR(bytes.NewReader(body))
		protocol := thrift.NewTBinaryProtocolTransport(transport)

		_, size, err := protocol.ReadListBegin()
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		for idx := 0; idx < size; idx++ {
			var span zipkincore.Span

			if err := span.Read(protocol); err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}

			if span.Name == "watch-config-key-values" {
				continue
			}

			spans <- &span
		}

		if err := protocol.ReadListEnd(); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		w.WriteHeader(http.StatusNoContent)
	}
}

type funcLogger func(keyvals ...interface{})

func (fn funcLogger) Log(keyvals ...interface{}) error {
	fn(keyvals...)
	return nil
}
<commit_msg>Add an admin route to query the latest spans.<commit_after>package zipkinproxy

import (
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/apache/thrift/lib/go/thrift"
	"github.com/flachnetz/dd-zipkin-proxy/datadog"
	"github.com/flachnetz/dd-zipkin-proxy/zipkin"
	"github.com/gorilla/handlers"
	"github.com/jessevdk/go-flags"
	"github.com/julienschmidt/httprouter"
	"github.com/openzipkin/zipkin-go-opentracing"
	"github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore"

	"encoding/json"
	. "github.com/flachnetz/go-admin"
	"sync"
)

var log = logrus.WithField("prefix", "main")

func Main(spanConverter datadog.SpanConverterFunc) {
	var opts struct {
		ZipkinReporterUrl string `long:"zipkin-url" description:"Url for zipkin reporting."`
		DatadogReporting  bool   `long:"datadog-reporting" description:"Enable datadog trace reporting with the default url."`
		ListenAddr        string `long:"listen-address" default:":9411" description:"Address to listen for zipkin connections."`
		Verbose           bool   `long:"verbose" description:"Enable verbose debug logging."`
	}

	if _, err := flags.Parse(&opts); err != nil {
		os.Exit(1)
	}

	if opts.Verbose {
		logrus.SetLevel(logrus.DebugLevel)
	}

	var channels []chan<- *zipkincore.Span

	if opts.DatadogReporting {
		// report spans to datadog
		zipkinSpans := make(chan *zipkincore.Span, 512)
		datadogSpans := datadog.ConvertZipkinSpans(zipkinSpans, spanConverter)
		go datadog.ReportSpans(datadogSpans)
		channels = append(channels, zipkinSpans)
	}

	if opts.ZipkinReporterUrl != "" {
		log.Info("Enable zipkin reporting at url ", opts.ZipkinReporterUrl)

		collector, err := zipkintracer.NewHTTPCollector(opts.ZipkinReporterUrl,
			zipkintracer.HTTPBatchSize(5000),
			zipkintracer.HTTPLogger(funcLogger(log.Warn)))

		if err != nil {
			log.Fatal("Could not create zipkin http collector: ", err)
			return
		}

		zipkinSpans := make(chan *zipkincore.Span, 512)
		go zipkin.ReportSpans(collector, zipkinSpans)
		channels = append(channels, zipkinSpans)
	}

	// a channel to store the last spans that were received
	var buffer *SpansBuffer
	{
		zipkinSpans := make(chan *zipkincore.Span, 512)
		buffer = NewSpansBuffer(2048)
		go buffer.ReadFrom(zipkinSpans)
		channels = append(channels, zipkinSpans)
	}

	// multiplex channels
	zipkinSpans := make(chan *zipkincore.Span, 16)
	go forwardSpansToChannels(zipkinSpans, channels)

	admin := NewAdminHandler("/admin", "dd-zipkin-proxy",
		WithDefaults(),
		WithForceGC(),
		WithPProfHandlers(),
		WithHeapDump(),

		Describe("A buffer of the previous traces (in openzipkin-format) in the order they were received.",
			WithGenericValue("/spans", buffer.ToSlice)))

	// listen for zipkin messages
	router := httprouter.New()
	registerAdminHandler(router, admin)
	router.POST("/api/v1/spans", handleSpans(zipkinSpans))
	log.Fatal(http.ListenAndServe(opts.ListenAddr, handlers.LoggingHandler(log.Logger.Writer(), router)))
}

func registerAdminHandler(router *httprouter.Router, handler http.Handler) {
	router.Handler("GET", "/", http.RedirectHandler("/admin", http.StatusTemporaryRedirect))

	methods := []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}
	for _, method := range methods {
		router.Handler(method, "/admin", handler)
		router.Handler(method, "/admin/*path", handler)
	}
}

func forwardSpansToChannels(source <-chan *zipkincore.Span, targets []chan<- *zipkincore.Span) {
	dropped := 0
	lastLogMessageTime := time.Now()

	for span := range source {
		for _, target := range targets {
			// send span, do not block if target is full
			select {
			case target <- span:
			default:
				dropped += 1
			}
		}

		if dropped > 0 && time.Now().Sub(lastLogMessageTime) > 1*time.Second {
			log.Warnf("Dropped %d spans because a target buffer was full.", dropped)
			dropped = 0
			lastLogMessageTime = time.Now()
		}
	}
}

func handleSpans(spans chan<- *zipkincore.Span) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		var bodyReader io.Reader = req.Body
		if req.Header.Get("Content-Encoding") == "gzip" {
			var err error
			bodyReader, err = gzip.NewReader(bodyReader)
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
		}

		body, err := ioutil.ReadAll(bodyReader)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		// parse with correct mime type
		if req.Header.Get("Content-Type") == "application/json" {
			err = parseSpansWithJSON(spans, body)
		} else {
			err = parseSpansWithThrift(spans, body)
		}

		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		} else {
			w.WriteHeader(http.StatusNoContent)
		}
	}
}

func parseSpansWithJSON(spans chan<- *zipkincore.Span, body []byte) error {
	parsedSpans := []zipkincore.Span{}

	if err := json.Unmarshal(body, &parsedSpans); err != nil {
		return err
	}

	for idx := range parsedSpans {
		spans <- &parsedSpans[idx]
	}

	return nil
}

func parseSpansWithThrift(spans chan<- *zipkincore.Span, body []byte) error {
	transport := thrift.NewStreamTransportR(bytes.NewReader(body))
	protocol := thrift.NewTBinaryProtocolTransport(transport)

	_, size, err := protocol.ReadListBegin()
	if err != nil {
		return err
	}

	for idx := 0; idx < size; idx++ {
		var span zipkincore.Span

		if err := span.Read(protocol); err != nil {
			return err
		}

		spans <- &span
	}

	return protocol.ReadListEnd()
}

type funcLogger func(keyvals ...interface{})

func (fn funcLogger) Log(keyvals ...interface{}) error {
	fn(keyvals...)
	return nil
}

type SpansBuffer struct {
	lock     sync.Mutex
	position uint
	spans    []*zipkincore.Span
}

func NewSpansBuffer(capacity uint) *SpansBuffer {
	spans := make([]*zipkincore.Span, capacity)
	return &SpansBuffer{spans: spans}
}

func (buffer *SpansBuffer) ReadFrom(spans <-chan *zipkincore.Span) {
	for span := range spans {
		buffer.lock.Lock()
		buffer.spans[buffer.position] = span
		buffer.position = (buffer.position + 1) % uint(len(buffer.spans))
		buffer.lock.Unlock()
	}
}

func (buffer *SpansBuffer) ToSlice() []*zipkincore.Span {
	buffer.lock.Lock()
	defer buffer.lock.Unlock()

	var result []*zipkincore.Span
	for _, span := range buffer.spans {
		if span != nil {
			result = append(result, span)
		}
	}

	return result
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"

	"github.com/gin-gonic/gin"
	"github.com/jessevdk/go-flags"
	_ "github.com/lib/pq"
)

const VERSION = "0.5.1"

type Options struct {
	Version  bool   `short:"v" long:"version" description:"Print version"`
	Debug    bool   `short:"d" long:"debug" description:"Enable debugging mode" default:"false"`
	Url      string `long:"url" description:"Database connection string"`
	Host     string `long:"host" description:"Server hostname or IP"`
	Port     int    `long:"port" description:"Server port" default:"5432"`
	User     string `long:"user" description:"Database user"`
	Pass     string `long:"pass" description:"Password for user"`
	DbName   string `long:"db" description:"Database name"`
	Ssl      string `long:"ssl" description:"SSL option"`
	HttpHost string `long:"bind" description:"HTTP server host" default:"localhost"`
	HttpPort uint   `long:"listen" description:"HTTP server listen port" default:"8080"`
	AuthUser string `long:"auth-user" description:"HTTP basic auth user"`
	AuthPass string `long:"auth-pass" description:"HTTP basic auth password"`
	SkipOpen bool   `short:"s" long:"skip-open" description:"Skip browser open on start"`
}

var dbClient *Client
var options Options

func exitWithMessage(message string) {
	fmt.Println("Error:", message)
	os.Exit(1)
}

func initClient() {
	if connectionSettingsBlank(options) {
		return
	}

	client, err := NewClient()
	if err != nil {
		exitWithMessage(err.Error())
	}

	if options.Debug {
		fmt.Println("Server connection string:", client.connectionString)
	}

	fmt.Println("Connecting to server...")
	err = client.Test()
	if err != nil {
		exitWithMessage(err.Error())
	}

	fmt.Println("Checking tables...")
	_, err = client.Tables()
	if err != nil {
		exitWithMessage(err.Error())
	}

	dbClient = client
}

func initOptions() {
	_, err := flags.ParseArgs(&options, os.Args)

	if err != nil {
		os.Exit(1)
	}

	if options.Url == "" {
		options.Url = os.Getenv("DATABASE_URL")
	}

	if options.Version {
		fmt.Printf("pgweb v%s\n", VERSION)
		os.Exit(0)
	}
}

func startServer() {
	router := gin.Default()

	// Enable HTTP basic authentication only if both user and password are set
	if options.AuthUser != "" && options.AuthPass != "" {
		auth := map[string]string{options.AuthUser: options.AuthPass}
		router.Use(gin.BasicAuth(auth))
	}

	setupRoutes(router)

	fmt.Println("Starting server...")
	go func() {
		err := router.Run(fmt.Sprintf("%v:%v", options.HttpHost, options.HttpPort))
		if err != nil {
			fmt.Println("Cant start server:", err)
			os.Exit(1)
		}
	}()
}

func handleSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	<-c
}

func openPage() {
	url := fmt.Sprintf("http://%v:%v", options.HttpHost, options.HttpPort)
	fmt.Println("To view database open", url, "in browser")

	if options.SkipOpen {
		return
	}

	_, err := exec.Command("which", "open").Output()
	if err != nil {
		return
	}

	exec.Command("open", url).Output()
}

func main() {
	initOptions()

	fmt.Println("Pgweb version", VERSION)
	initClient()

	if dbClient != nil {
		defer dbClient.db.Close()
	}

	if !options.Debug {
		gin.SetMode("release")
	}

	if options.Debug {
		startRuntimeProfiler()
	}

	startServer()
	openPage()
	handleSignals()
}
<commit_msg>Version bump: 0.5.2<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"

	"github.com/gin-gonic/gin"
	"github.com/jessevdk/go-flags"
	_ "github.com/lib/pq"
)

const VERSION = "0.5.2"

type Options struct {
	Version  bool   `short:"v" long:"version" description:"Print version"`
	Debug    bool   `short:"d" long:"debug" description:"Enable debugging mode" default:"false"`
	Url      string `long:"url" description:"Database connection string"`
	Host     string `long:"host" description:"Server hostname or IP"`
	Port     int    `long:"port" description:"Server port" default:"5432"`
	User     string `long:"user" description:"Database user"`
	Pass     string `long:"pass" description:"Password for user"`
	DbName   string `long:"db" description:"Database name"`
	Ssl      string `long:"ssl" description:"SSL option"`
	HttpHost string `long:"bind" description:"HTTP server host" default:"localhost"`
	HttpPort uint   `long:"listen" description:"HTTP server listen port" default:"8080"`
	AuthUser string `long:"auth-user" description:"HTTP basic auth user"`
	AuthPass string `long:"auth-pass" description:"HTTP basic auth password"`
	SkipOpen bool   `short:"s" long:"skip-open" description:"Skip browser open on start"`
}

var dbClient *Client
var options Options

func exitWithMessage(message string) {
	fmt.Println("Error:", message)
	os.Exit(1)
}

func initClient() {
	if connectionSettingsBlank(options) {
		return
	}

	client, err := NewClient()
	if err != nil {
		exitWithMessage(err.Error())
	}

	if options.Debug {
		fmt.Println("Server connection string:", client.connectionString)
	}

	fmt.Println("Connecting to server...")
	err = client.Test()
	if err != nil {
		exitWithMessage(err.Error())
	}

	fmt.Println("Checking tables...")
	_, err = client.Tables()
	if err != nil {
		exitWithMessage(err.Error())
	}

	dbClient = client
}

func initOptions() {
	_, err := flags.ParseArgs(&options, os.Args)

	if err != nil {
		os.Exit(1)
	}

	if options.Url == "" {
		options.Url = os.Getenv("DATABASE_URL")
	}

	if options.Version {
		fmt.Printf("pgweb v%s\n", VERSION)
		os.Exit(0)
	}
}

func startServer() {
	router := gin.Default()

	// Enable HTTP basic authentication only if both user and password are set
	if options.AuthUser != "" && options.AuthPass != "" {
		auth := map[string]string{options.AuthUser: options.AuthPass}
		router.Use(gin.BasicAuth(auth))
	}

	setupRoutes(router)

	fmt.Println("Starting server...")
	go func() {
		err := router.Run(fmt.Sprintf("%v:%v", options.HttpHost, options.HttpPort))
		if err != nil {
			fmt.Println("Cant start server:", err)
			os.Exit(1)
		}
	}()
}

func handleSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	<-c
}

func openPage() {
	url := fmt.Sprintf("http://%v:%v", options.HttpHost, options.HttpPort)
	fmt.Println("To view database open", url, "in browser")

	if options.SkipOpen {
		return
	}

	_, err := exec.Command("which", "open").Output()
	if err != nil {
		return
	}

	exec.Command("open", url).Output()
}

func main() {
	initOptions()

	fmt.Println("Pgweb version", VERSION)
	initClient()

	if dbClient != nil {
		defer dbClient.db.Close()
	}

	if !options.Debug {
		gin.SetMode("release")
	}

	if options.Debug {
		startRuntimeProfiler()
	}

	startServer()
	openPage()
	handleSignals()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/choueric/readline"
)

var (
	inst = &readline.Instance{
		Prompt: "\033[32m>>\033[0m ",
	}
	regLen = 32
	binStr string
)

// field range
type fRange struct {
	start int
	end   int
}

// setFieldRangeOfBinStr sets to 1 or clears to 0 with the filed range of binStr
// rStr is a filed range string
func setFieldOfBinStr(rStr string, set bool) {
	r, err := getRange(rStr)
	if err != nil {
		inst.Error(fmt.Sprintf("parse field start index failed, %v\n", err))
		return
	}

	binByte := []byte(binStr)
	c := byte('0')
	if set {
		c = '1'
	}
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = c
	}

	binStr = string(binByte)
}

// updateBinStr converts decimal or heximal format s into register-length
// binary string and update the global variable binStr
// e.g. "0x11" -> "00010001" (regLen = 8)
func updateBinStr(s string) {
	val, err := parseInt(s)
	if err != nil {
		inst.Error(fmt.Sprintf("convert to Int failed: %v\n", err))
		return
	}

	s = strconv.FormatInt(val, 2)
	l := len(s)
	bin := strings.Repeat("0", regLen-l)
	binStr = bin + s
}

// writeFiledOfBinStr changes the value in filed rStr by vStr in binStr
func writeFiledOfBinStr(rStr, vStr string) {
	r, err := getRange(rStr)
	if err != nil {
		inst.Error(fmt.Sprintf("parse field start index failed, %v\n", err))
		return
	}

	val, err := parseInt(vStr)
	if err != nil {
		inst.Error(fmt.Sprintf("convert to Int failed: %v\n", err))
		return
	}

	max := (2 << uint(r.end-r.start)) - 1
	if val < 0 || int(val) > max {
		inst.Error(fmt.Sprint("val is out of range [%d, %d]\n", 0, max))
		return
	}

	s := strconv.FormatInt(val, 2)
	l := len(s)
	sub := strings.Repeat("0", r.end-r.start+1-l)
	sub = sub + s
	subByte := []byte(sub)
	fmt.Println(sub)

	binByte := []byte(binStr)
	j := r.end - r.start
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = subByte[j]
		j--
	}

	binStr = string(binByte)
}

func showTruesOffset(target rune) {
	listOffsets(os.Stdout, binStr, target)
}

// showFieldOfBinStr shows the filed rStr of binStr in three formats.
func showFieldOfBinStr(rStr string) {
	subbin, err := getFieldStr(rStr, binStr)
	if err != nil {
		inst.Error(fmt.Sprintf("getFieldStr failed: %v\n", err))
		return
	}

	outputTriFormat(os.Stdout, subbin)
}

func executeCmdline(input string, data interface{}) (exit bool) {
	exit = false

	cmdline := strings.Fields(input)
	if len(cmdline) == 0 {
		printUsage()
		return
	}

	switch cmdline[0] {
	case "exit", "quit":
		exit = true
	case "help", "h":
		printUsage()
	case "print", "p":
		outputTriFormat(os.Stdout, binStr)
	case "value", "v":
		if len(cmdline) < 2 {
			inst.Error("Needs argument: <field>\n")
			return
		}
		updateBinStr(cmdline[1])
		outputTriFormat(os.Stdout, binStr)
	case "set", "s", "clear", "c":
		if len(cmdline) < 2 {
			inst.Error("Needs argument: <field>\n")
			return
		}
		set := true
		if cmdline[0] == "clear" || cmdline[0] == "c" {
			set = false
		}
		setFieldOfBinStr(cmdline[1], set)
		outputTriFormat(os.Stdout, binStr)
	case "write", "w":
		if len(cmdline) < 3 {
			inst.Error("Needs arguments: <field> <val>\n")
			return
		}
		writeFiledOfBinStr(cmdline[1], cmdline[2])
		outputTriFormat(os.Stdout, binStr)
	case "list", "l":
		target := '1'
		if len(cmdline) == 2 && cmdline[1] == "0" {
			target = '0'
		}
		showTruesOffset(target)
	default:
		showFieldOfBinStr(cmdline[0])
	}

	return
}

func printUsage() {
	inst.Print("Usage:\n")
	inst.Print("  [h]elp          : print this message.\n")
	inst.Print("  [p]rint         : show current value.\n")
	inst.Print("  [v]alue <v>     : change value to <v>.\n")
	inst.Print("  [s]et <f>       : set <f to 1.\n")
	inst.Print("  [c]lear <f>     : clear <f> to 0.\n")
	inst.Print("  [w]rite <f> <v> : write val <v> into field <f>.\n")
	inst.Print("  <f>             : read the value of field <f>.\n")
	inst.Print("  [l]ist [0]      : list all offsets of '1's or '0's.\n")
	inst.Print("  exit, quit      : exit this program.\n")
	inst.Print("  \nTwo formats to represent filed:\n")
	inst.Print("  single bit  : like 1, 3, 0\n")
	inst.Print("  field range : like 0:3, 3:1\n")
}

func main() {
	flag.BoolVar(&inst.Debug, "d", false, "enable debug")
	flag.IntVar(&regLen, "l", 32, "register length.")
	flag.Parse()
	if len(os.Args) == 2 && (os.Args[1] == "help" || os.Args[1] == "-h") {
		flag.Usage()
		return
	}

	inst.Init(os.Stdin, os.Stdout)
	defer inst.Deinit()

	inst.SetExecute(executeCmdline, nil)
	inst.SetCompleter(
		readline.Cmd("print"),
		readline.Cmd("value"),
		readline.Cmd("set"),
		readline.Cmd("clear"),
		readline.Cmd("write"),
		readline.Cmd("list"),
		readline.Cmd("help"),
		readline.Cmd("exit"),
		readline.Cmd("quit"),
	)
	inst.PrintTree(os.Stdout)

	updateBinStr("0")

	readline.InputLoop(inst)
}
<commit_msg>remove print commands tree<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/choueric/readline"
)

var (
	inst = &readline.Instance{
		Prompt: "\033[32m>>\033[0m ",
	}
	regLen = 32
	binStr string
)

// field range
type fRange struct {
	start int
	end   int
}

// setFieldRangeOfBinStr sets to 1 or clears to 0 with the filed range of binStr
// rStr is a filed range string
func setFieldOfBinStr(rStr string, set bool) {
	r, err := getRange(rStr)
	if err != nil {
		inst.Error(fmt.Sprintf("parse field start index failed, %v\n", err))
		return
	}

	binByte := []byte(binStr)
	c := byte('0')
	if set {
		c = '1'
	}
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = c
	}

	binStr = string(binByte)
}

// updateBinStr converts decimal or heximal format s into register-length
// binary string and update the global variable binStr
// e.g. "0x11" -> "00010001" (regLen = 8)
func updateBinStr(s string) {
	val, err := parseInt(s)
	if err != nil {
		inst.Error(fmt.Sprintf("convert to Int failed: %v\n", err))
		return
	}

	s = strconv.FormatInt(val, 2)
	l := len(s)
	bin := strings.Repeat("0", regLen-l)
	binStr = bin + s
}

// writeFiledOfBinStr changes the value in filed rStr by vStr in binStr
func writeFiledOfBinStr(rStr, vStr string) {
	r, err := getRange(rStr)
	if err != nil {
		inst.Error(fmt.Sprintf("parse field start index failed, %v\n", err))
		return
	}

	val, err := parseInt(vStr)
	if err != nil {
		inst.Error(fmt.Sprintf("convert to Int failed: %v\n", err))
		return
	}

	max := (2 << uint(r.end-r.start)) - 1
	if val < 0 || int(val) > max {
		inst.Error(fmt.Sprint("val is out of range [%d, %d]\n", 0, max))
		return
	}

	s := strconv.FormatInt(val, 2)
	l := len(s)
	sub := strings.Repeat("0", r.end-r.start+1-l)
	sub = sub + s
	subByte := []byte(sub)
	fmt.Println(sub)

	binByte := []byte(binStr)
	j := r.end - r.start
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = subByte[j]
		j--
	}

	binStr = string(binByte)
}

func showTruesOffset(target rune) {
	listOffsets(os.Stdout, binStr, target)
}

// showFieldOfBinStr shows the filed rStr of binStr in three formats.
func showFieldOfBinStr(rStr string) {
	subbin, err := getFieldStr(rStr, binStr)
	if err != nil {
		inst.Error(fmt.Sprintf("getFieldStr failed: %v\n", err))
		return
	}

	outputTriFormat(os.Stdout, subbin)
}

func executeCmdline(input string, data interface{}) (exit bool) {
	exit = false

	cmdline := strings.Fields(input)
	if len(cmdline) == 0 {
		printUsage()
		return
	}

	switch cmdline[0] {
	case "exit", "quit":
		exit = true
	case "help", "h":
		printUsage()
	case "print", "p":
		outputTriFormat(os.Stdout, binStr)
	case "value", "v":
		if len(cmdline) < 2 {
			inst.Error("Needs argument: <field>\n")
			return
		}
		updateBinStr(cmdline[1])
		outputTriFormat(os.Stdout, binStr)
	case "set", "s", "clear", "c":
		if len(cmdline) < 2 {
			inst.Error("Needs argument: <field>\n")
			return
		}
		set := true
		if cmdline[0] == "clear" || cmdline[0] == "c" {
			set = false
		}
		setFieldOfBinStr(cmdline[1], set)
		outputTriFormat(os.Stdout, binStr)
	case "write", "w":
		if len(cmdline) < 3 {
			inst.Error("Needs arguments: <field> <val>\n")
			return
		}
		writeFiledOfBinStr(cmdline[1], cmdline[2])
		outputTriFormat(os.Stdout, binStr)
	case "list", "l":
		target := '1'
		if len(cmdline) == 2 && cmdline[1] == "0" {
			target = '0'
		}
		showTruesOffset(target)
	default:
		showFieldOfBinStr(cmdline[0])
	}

	return
}

func printUsage() {
	inst.Print("Usage:\n")
	inst.Print("  [h]elp          : print this message.\n")
	inst.Print("  [p]rint         : show current value.\n")
	inst.Print("  [v]alue <v>     : change value to <v>.\n")
	inst.Print("  [s]et <f>       : set <f to 1.\n")
	inst.Print("  [c]lear <f>     : clear <f> to 0.\n")
	inst.Print("  [w]rite <f> <v> : write val <v> into field <f>.\n")
	inst.Print("  <f>             : read the value of field <f>.\n")
	inst.Print("  [l]ist [0]      : list all offsets of '1's or '0's.\n")
	inst.Print("  exit, quit      : exit this program.\n")
	inst.Print("  \nTwo formats to represent filed:\n")
	inst.Print("  single bit  : like 1, 3, 0\n")
	inst.Print("  field range : like 0:3, 3:1\n")
}

func main() {
	flag.BoolVar(&inst.Debug, "d", false, "enable debug")
	flag.IntVar(&regLen, "l", 32, "register length.")
	flag.Parse()
	if len(os.Args) == 2 && (os.Args[1] == "help" || os.Args[1] == "-h") {
		flag.Usage()
		return
	}

	inst.Init(os.Stdin, os.Stdout)
	defer inst.Deinit()

	inst.SetExecute(executeCmdline, nil)
	inst.SetCompleter(
		readline.Cmd("print"),
		readline.Cmd("value"),
		readline.Cmd("set"),
		readline.Cmd("clear"),
		readline.Cmd("write"),
		readline.Cmd("list"),
		readline.Cmd("help"),
		readline.Cmd("exit"),
		readline.Cmd("quit"),
	)

	updateBinStr("0")

	readline.InputLoop(inst)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type dir struct {
	name string
	path string
	files []string
	subdirs []dir
}

const htmlHeader = `

<title>%s</title>
<link rel="stylesheet" href="//clee.github.io/arborealize/arborealize.css" />
<meta name="generator" content="http://github.com/clee/arborealize/" />

`
const htmlFooter = `
`

func subdirIndex(subdirs []dir, name string) int {
	for i, d := range subdirs {
		if d.name == name {
			return i
		}
	}
	return -1
}

func m(indent int) string {
	return strings.Repeat(" ", indent)
}

func markupFromTree(tree dir, indent int) (ret string) {
	name := tree.name
	if tree.name == "" {
		ret = m(indent) + "
\n" + m(indent + 1) + "
"
		name = "/"
	} else {
		ret = m(indent) + "
\n" + m(indent + 1) + "
"
	}
	id := strings.Replace(tree.path, "/", "_", -1)
	if name == "/" {
		ret += fmt.Sprintf(`<label for="root">%s</label> <input type="checkbox" checked="checked" id="root">`, name) + "\n"
	} else {
		ret += fmt.Sprintf(`<label for="%s">%s</label> <input type="checkbox" id="%s">`, id, name, id) + "\n"
	}

	ret += m(indent + 2) + "
\n"

	for _, s := range tree.subdirs {
		ret += m(indent + 3) + "
\n" + markupFromTree(s, indent + 4) + m(indent + 3) + "
\n"
	}

	for _, f := range tree.files {
		ret += m(indent + 3) + fmt.Sprintf(`<li class="file"><a href="%s%s">%s</a></li>`, tree.path, f, f) + "\n"
	}

	ret += m(indent + 2) + "
\n"

	ret += m(indent + 1) + "
\n" + m(indent) + "
\n"
	return ret
}

func treeFromFiles(files map[string][]string) dir {
	root := dir{name: "", files: files[""], subdirs: []dir{}}

	for key := range files {
		var currentDir *dir = &root
		var newDir *dir
		subdirNames := strings.Split(key, "/")
		for i, d := range subdirNames {
			// Skip empty post-trailing-/ string
			if i == len(subdirNames) - 1 {
				continue
			}
			path := strings.Join(subdirNames[0:i+1], "/") + "/"
			j := subdirIndex(currentDir.subdirs, d)
			if j == -1 {
				newDir = new(dir)
				newDir.name = d
				newDir.path = path
				newDir.files = files[path]
				newDir.subdirs = []dir{}

				currentDir.subdirs = append(currentDir.subdirs, (*newDir))
				j = subdirIndex(currentDir.subdirs, d)
			}
			currentDir = &currentDir.subdirs[j]
		}
	}
	return root
}

func main() {
	var root string
	var err error
	files := make(map[string][]string)

	if root, err = os.Getwd(); err != nil {
		fmt.Printf("can't get the current path\n")
		panic(err)
	}

	flag.StringVar(&root, "root", root, "filesystem root for scan")
	flag.Parse()
	if !strings.HasSuffix(root, "/") {
		root = root + "/"
	}

	filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path = strings.TrimPrefix(path, root)
		dir := strings.TrimSuffix(path, f.Name())
		if f.Mode().IsRegular() {
			files[dir] = append(files[dir], f.Name())
		}
		return nil
	})

	f := treeFromFiles(files)
	html := fmt.Sprintf(htmlHeader, root)
	html += markupFromTree(f, 1)
	html += htmlFooter
	fmt.Printf("%s\n", html)
}
<commit_msg>Make folder icons clickable<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type dir struct {
	name string
	path string
	files []string
	subdirs []dir
}

const htmlHeader = `

<title>%s</title>
<link rel="stylesheet" href="//clee.github.io/arborealize/arborealize.css" />
<meta name="generator" content="http://github.com/clee/arborealize/" />

`
const htmlFooter = `
`

func subdirIndex(subdirs []dir, name string) int {
	for i, d := range subdirs {
		if d.name == name {
			return i
		}
	}
	return -1
}

func m(indent int) string {
	return strings.Repeat(" ", indent)
}

func markupFromTree(tree dir, indent int) (ret string) {
	name := tree.name
	if tree.name == "" {
		ret = m(indent) + "
\n" + m(indent + 1) + "
"
		name = "/"
	} else {
		ret = m(indent) + "
\n" + m(indent + 1) + "
"
	}
	id := strings.Replace(tree.path, "/", "_", -1)
	if name == "/" {
		ret += fmt.Sprintf(`<input type="checkbox" checked="checked" id="root"><label for="root">%s</label>`, name) + "\n"
	} else {
		ret += fmt.Sprintf(`<input type="checkbox" id="%s"><label for="%s">%s</label>`, id, id, name) + "\n"
	}

	ret += m(indent + 2) + "
\n"

	for _, s := range tree.subdirs {
		ret += m(indent + 3) + "
\n" + markupFromTree(s, indent + 4) + m(indent + 3) + "
\n"
	}

	for _, f := range tree.files {
		ret += m(indent + 3) + fmt.Sprintf(`<li class="file"><a href="%s%s">%s</a></li>`, tree.path, f, f) + "\n"
	}

	ret += m(indent + 2) + "
\n"

	ret += m(indent + 1) + "
\n" + m(indent) + "
\n"
	return ret
}

func treeFromFiles(files map[string][]string) dir {
	root := dir{name: "", files: files[""], subdirs: []dir{}}

	for key := range files {
		var currentDir *dir = &root
		var newDir *dir
		subdirNames := strings.Split(key, "/")
		for i, d := range subdirNames {
			// Skip empty post-trailing-/ string
			if i == len(subdirNames) - 1 {
				continue
			}
			path := strings.Join(subdirNames[0:i+1], "/") + "/"
			j := subdirIndex(currentDir.subdirs, d)
			if j == -1 {
				newDir = new(dir)
				newDir.name = d
				newDir.path = path
				newDir.files = files[path]
				newDir.subdirs = []dir{}

				currentDir.subdirs = append(currentDir.subdirs, (*newDir))
				j = subdirIndex(currentDir.subdirs, d)
			}
			currentDir = &currentDir.subdirs[j]
		}
	}
	return root
}

func main() {
	var root string
	var err error
	files := make(map[string][]string)

	if root, err = os.Getwd(); err != nil {
		fmt.Printf("can't get the current path\n")
		panic(err)
	}

	flag.StringVar(&root, "root", root, "filesystem root for scan")
	flag.Parse()
	if !strings.HasSuffix(root, "/") {
		root = root + "/"
	}

	filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path = strings.TrimPrefix(path, root)
		dir := strings.TrimSuffix(path, f.Name())
		if f.Mode().IsRegular() {
			files[dir] = append(files[dir], f.Name())
		}
		return nil
	})

	f := treeFromFiles(files)
	html := fmt.Sprintf(htmlHeader, root)
	html += markupFromTree(f, 1)
	html += htmlFooter
	fmt.Printf("%s\n", html)
}
<|endoftext|>


<commit_before>package main

import (
	"container/ring"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/fatih/color"
	"github.com/lucagrulla/cw/cloudwatch"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	timeFormat = "2006-01-02T15:04:05"
	version    = "3.0.01

	kp = kingpin.New("cw", "The best way to tail AWS Cloudwatch Logs from your terminal.")

	awsProfile = kp.Flag("profile", "The target AWS profile. By default cw will use the default profile defined in the .aws/credentials file.").Short('p').String()
	awsRegion  = kp.Flag("region", "The target AWS region. By default cw will use the default region defined in the .aws/credentials file.").Short('r').String()
	noColor    = kp.Flag("no-color", "Disable coloured output.").Short('c').Default("false").Bool()
	debug      = kp.Flag("debug", "Enable debug logging.").Short('d').Default("false").Hidden().Bool()

	lsCommand = kp.Command("ls", "Show an entity.")

	lsGroups = lsCommand.Command("groups", "Show all groups.")

	lsStreams      = lsCommand.Command("streams", "Show all streams in a given log group.")
	lsLogGroupName = lsStreams.Arg("group", "The group name.").Required().String()

	tailCommand        = kp.Command("tail", "Tail log groups/streams.")
	logGroupStreamName = tailCommand.Arg("groupName[:logStreamPrefix]", "The log group and stream name, with group:prefix syntax."+
		"Stream name can be just the prefix. If no stream name is specified all stream names in the given group will be tailed."+
		"Multiple group/stream tuple can be passed. e.g. cw tail group1:prefix1 group2:prefix2 group3:prefix3.").
		Required().Strings()

	follow          = tailCommand.Flag("follow", "Don't stop when the end of streams is reached, but rather wait for additional data to be appended.").Short('f').Default("false").Bool()
	printTimestamp  = tailCommand.Flag("timestamp", "Print the event timestamp.").Short('t').Default("false").Bool()
	printEventID    = tailCommand.Flag("event-id", "Print the event Id.").Short('i').Default("false").Bool()
	printStreamName = tailCommand.Flag("stream-name", "Print the log stream name this event belongs to.").Short('s').Default("false").Bool()
	printGroupName  = tailCommand.Flag("group-name", "Print the log group name this event belongs to.").Short('n').Default("false").Bool()
	startTime       = tailCommand.Flag("start", "The UTC start time. Passed as either date/time or human-friendly format."+
		" The human-friendly format accepts the number of hours and minutes prior to the present. "+
		"Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m."+
		" If just time is used (format: hh[:mm]) it is expanded to today at the given time."+
		" Full available date/time format: 2017-02-27[T09[:00[:00]].").
		Short('b').Default(time.Now().UTC().Add(-30 * time.Second).Format(timeFormat)).String()
	endTime = tailCommand.Flag("end", "The UTC end time. Passed as either date/time or human-friendly format. "+
		"The human-friendly format accepts the number of hours and minutes prior to the present. "+
		"Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m."+
		"If just time is used (format: hh[:mm]) it is expanded to today at the given time. Full available date/time format: 2017-02-27[T09[:00[:00]].").
		Short('e').Default("").String()
	local = tailCommand.Flag("local", "Treat date and time in Local timezone.").Short('l').Default("false").Bool()
	grep  = tailCommand.Flag("grep", "Pattern to filter logs by. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html for syntax.").
		Short('g').Default("").String()
	grepv = tailCommand.Flag("grepv", "Equivalent of grep --invert-match. Invert match pattern to filter logs by.").Short('v').Default("").String()
)

func timestampToTime(timeStamp *string) (time.Time, error) {
	var zone *time.Location
	if *local {
		zone = time.Local
	} else {
		zone = time.UTC
	}
	if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02", *timeStamp, zone)
		return t, nil
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15", *timeStamp, zone)
		return t, nil
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15:04", *timeStamp, zone)
		return t, nil
	} else if regexp.MustCompile(`^\d{1,2}$`).MatchString(*timeStamp) {
		y, m, d := time.Now().In(zone).Date()
		t, _ := strconv.Atoi(*timeStamp)
		return time.Date(y, m, d, t, 0, 0, 0, zone), nil
	} else if res := regexp.MustCompile(`^(?P<Hour>\d{1,2}):(?P<Minute>\d{2})$`).FindStringSubmatch(*timeStamp); res != nil {
		y, m, d := time.Now().Date()

		t, _ := strconv.Atoi(res[1])
		mm, _ := strconv.Atoi(res[2])

		return time.Date(y, m, d, t, mm, 0, 0, zone), nil
	} else if regexp.MustCompile(`^\d{1,}h$|^\d{1,}m$|^\d{1,}h\d{1,}m$`).MatchString(*timeStamp) {
		d, _ := time.ParseDuration(*timeStamp)

		t := time.Now().In(zone).Add(-d)
		y, m, dd := t.Date()
		return time.Date(y, m, dd, t.Hour(), t.Minute(), 0, 0, zone), nil
	}

	//TODO check even last scenario and if it's not a recognized pattern throw an error
	t, err := time.ParseInLocation("2006-01-02T15:04:05", *timeStamp, zone)
	if err != nil {
		return t, err
	}
	return t, nil
}

type logEvent struct {
	logEvent cloudwatchlogs.FilteredLogEvent
	logGroup string
}

func formatLogMsg(ev logEvent, printTime *bool, printStreamName *bool, printGroupName *bool) string {
	msg := *ev.logEvent.Message
	if *printEventID {
		if *noColor {
			msg = fmt.Sprintf("%s - %s", *ev.logEvent.EventId, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.YellowString(*ev.logEvent.EventId), msg)
		}
	}
	if *printStreamName {
		if *noColor {
			msg = fmt.Sprintf("%s - %s", *ev.logEvent.LogStreamName, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.BlueString(*ev.logEvent.LogStreamName), msg)
		}
	}

	if *printGroupName {
		if *noColor {
			msg = fmt.Sprintf("%s - %s", ev.logGroup, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.CyanString(ev.logGroup), msg)
		}
	}

	if *printTime {
		eventTimestamp := *ev.logEvent.Timestamp / 1000
		ts := time.Unix(eventTimestamp, 0).Format(timeFormat)
		if *noColor {
			msg = fmt.Sprintf("%s - %s", ts, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.GreenString(ts), msg)
		}
	}
	return msg
}

func oldSyntaxWarning() {
	tokenNum := len(*logGroupStreamName)
	if tokenNum > 1 { //a log stream is also specified
		if (*logGroupStreamName)[1] == "*" {
			fmt.Println("WARNING:tail syntax has changed with V3. Please refer to the documentation for the new syntax details: https://www.lucagrulla.com/cw/#v2-to-v3-breaking-changes")
			os.Exit(0)
		}
		if tokenNum > 2 { //group, stream and date
			ts := (*logGroupStreamName)[2]
			if _, err := timestampToTime(&ts); err == nil {
				fmt.Println("WARNING:tail syntax has changed with V3. Please refer to the documentation for the new syntax details: https://www.lucagrulla.com/cw/#v2-to-v3-breaking-changes")
				os.Exit(0)
			}
		}
	}
}

func main() {
	kp.Version(version).Author("Luca Grulla")

	defer newVersionMsg(version, fetchLatestVersion(), *noColor)
	go versionCheckOnSigterm()

	cmd := kingpin.MustParse(kp.Parse(os.Args[1:]))
	c := cloudwatch.New(awsProfile, awsRegion, debug)
	switch cmd {
	case "ls groups":

		for msg := range c.LsGroups() {
			fmt.Println(*msg)
		}
	case "ls streams":
		for msg := range c.LsStreams(lsLogGroupName, nil) {
			fmt.Println(*msg)
		}
	case "tail":
		oldSyntaxWarning()

		st, err := timestampToTime(startTime)
		if err != nil {
			log.Fatalf("can't parse %s as a valid date/time", *startTime)
		}
		var et time.Time
		if *endTime != "" {
			endT, errr := timestampToTime(endTime)
			if errr != nil {
				log.Fatalf("can't parse %s as a valid date/time", *endTime)
			} else {
				et = endT
			}
		}
		out := make(chan *logEvent)

		var wg sync.WaitGroup

		triggerChannels := make([]chan<- time.Time, len(*logGroupStreamName))

		coordinator := &tailCoordinator{}
		for idx, gs := range *logGroupStreamName {
			trigger := make(chan time.Time, 1)
			go func(groupStream string) {
				tokens := strings.Split(groupStream, ":")
				var prefix string
				group := tokens[0]
				if len(tokens) > 1 && tokens[1] != "*" {
					prefix = tokens[1]
				}
				for c := range c.Tail(&group, &prefix, follow, &st, &et, grep, grepv, trigger) {
					out <- &logEvent{logEvent: *c, logGroup: group}
				}
				coordinator.remove(trigger)
				wg.Done()
			}(gs)
			triggerChannels[idx] = trigger
			wg.Add(1)
		}

		coordinator.start(triggerChannels)

		go func() {
			wg.Wait()
			if *debug {
				fmt.Println("closing main channel...")
			}
			close(out)
		}()

		for logEv := range out {
			fmt.Println(formatLogMsg(*logEv, printTimestamp, printStreamName, printGroupName))
		}
	}
}

type tailCoordinator struct {
	targets *ring.Ring
	sync.RWMutex
}

func (f *tailCoordinator) start(targets []chan<- time.Time) {
	f.targets = ring.New(len(targets))
	for i := 0; i < f.targets.Len(); i++ {
		f.targets.Value = targets[i]
		f.targets = f.targets.Next()
	}

	//AWS API accepts 5 reqs/sec for account
	ticker := time.NewTicker(205 * time.Millisecond)
	go func() {
		for range ticker.C {
			if f.targets == nil {
				if *debug {
					fmt.Println("coordinator: ring buffer is empty, exiting scheduler.")
				}
				return
			}
			f.Lock()
			x := f.targets.Value.(chan<- time.Time)
			x <- time.Now()
			f.targets = f.targets.Next()
			f.Unlock()
		}
	}()
}

func (f *tailCoordinator) remove(c chan<- time.Time) {
	f.RLock()
	initialLen := f.targets.Len()
	f.RUnlock()

	var visited int
	f.Lock()
	defer f.Unlock()

	if f.targets.Len() == 1 {
		f.targets = ring.New(0)
		if *debug {
			fmt.Println("coordinator: single node buffer: reset", f.targets.Len())
		}
		return
	}

	for visited = 0; visited < f.targets.Len(); visited++ {
		if f.targets.Value == c {
			targetChan := f.targets.Value.(chan<- time.Time)

			f.targets = f.targets.Prev()
			f.targets.Unlink(1)
			close(targetChan)
			if *debug {
				fmt.Printf("coordinator: channel found and removed at index: %d\n", visited)
			}
			break
		}
		f.targets = f.targets.Next()
	}

	if f.targets.Len() < initialLen {
		for i := 0; i < visited; i++ {
			f.targets = f.targets.Prev()
		}
	}
}
<commit_msg>v3.0.1 prep<commit_after>package main

import (
	"container/ring"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/fatih/color"
	"github.com/lucagrulla/cw/cloudwatch"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	timeFormat = "2006-01-02T15:04:05"
	version    = "3.0.1"

	kp = kingpin.New("cw", "The best way to tail AWS Cloudwatch Logs from your terminal.")

	awsProfile = kp.Flag("profile", "The target AWS profile. By default cw will use the default profile defined in the .aws/credentials file.").Short('p').String()
	awsRegion  = kp.Flag("region", "The target AWS region. By default cw will use the default region defined in the .aws/credentials file.").Short('r').String()
	noColor    = kp.Flag("no-color", "Disable coloured output.").Short('c').Default("false").Bool()
	debug      = kp.Flag("debug", "Enable debug logging.").Short('d').Default("false").Hidden().Bool()

	lsCommand = kp.Command("ls", "Show an entity.")

	lsGroups = lsCommand.Command("groups", "Show all groups.")

	lsStreams      = lsCommand.Command("streams", "Show all streams in a given log group.")
	lsLogGroupName = lsStreams.Arg("group", "The group name.").Required().String()

	tailCommand        = kp.Command("tail", "Tail log groups/streams.")
	logGroupStreamName = tailCommand.Arg("groupName[:logStreamPrefix]", "The log group and stream name, with group:prefix syntax."+
		"Stream name can be just the prefix. If no stream name is specified all stream names in the given group will be tailed."+
		"Multiple group/stream tuple can be passed. e.g. cw tail group1:prefix1 group2:prefix2 group3:prefix3.").
		Required().Strings()

	follow          = tailCommand.Flag("follow", "Don't stop when the end of streams is reached, but rather wait for additional data to be appended.").Short('f').Default("false").Bool()
	printTimestamp  = tailCommand.Flag("timestamp", "Print the event timestamp.").Short('t').Default("false").Bool()
	printEventID    = tailCommand.Flag("event-id", "Print the event Id.").Short('i').Default("false").Bool()
	printStreamName = tailCommand.Flag("stream-name", "Print the log stream name this event belongs to.").Short('s').Default("false").Bool()
	printGroupName  = tailCommand.Flag("group-name", "Print the log group name this event belongs to.").Short('n').Default("false").Bool()
	startTime       = tailCommand.Flag("start", "The UTC start time. Passed as either date/time or human-friendly format."+
		" The human-friendly format accepts the number of hours and minutes prior to the present. "+
		"Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m."+
		" If just time is used (format: hh[:mm]) it is expanded to today at the given time."+
		" Full available date/time format: 2017-02-27[T09[:00[:00]].").
		Short('b').Default(time.Now().UTC().Add(-30 * time.Second).Format(timeFormat)).String()
	endTime = tailCommand.Flag("end", "The UTC end time. Passed as either date/time or human-friendly format. "+
		"The human-friendly format accepts the number of hours and minutes prior to the present. "+
		"Denote hours with 'h' and minutes with 'm' i.e. 80m, 4h30m."+
		"If just time is used (format: hh[:mm]) it is expanded to today at the given time. Full available date/time format: 2017-02-27[T09[:00[:00]].").
		Short('e').Default("").String()
	local = tailCommand.Flag("local", "Treat date and time in Local timezone.").Short('l').Default("false").Bool()
	grep  = tailCommand.Flag("grep", "Pattern to filter logs by. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html for syntax.").
		Short('g').Default("").String()
	grepv = tailCommand.Flag("grepv", "Equivalent of grep --invert-match. Invert match pattern to filter logs by.").Short('v').Default("").String()
)

func timestampToTime(timeStamp *string) (time.Time, error) {
	var zone *time.Location
	if *local {
		zone = time.Local
	} else {
		zone = time.UTC
	}
	if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02", *timeStamp, zone)
		return t, nil
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15", *timeStamp, zone)
		return t, nil
	} else if regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$`).MatchString(*timeStamp) {
		t, _ := time.ParseInLocation("2006-01-02T15:04", *timeStamp, zone)
		return t, nil
	} else if regexp.MustCompile(`^\d{1,2}$`).MatchString(*timeStamp) {
		y, m, d := time.Now().In(zone).Date()
		t, _ := strconv.Atoi(*timeStamp)
		return time.Date(y, m, d, t, 0, 0, 0, zone), nil
	} else if res := regexp.MustCompile(`^(?P<Hour>\d{1,2}):(?P<Minute>\d{2})$`).FindStringSubmatch(*timeStamp); res != nil {
		y, m, d := time.Now().Date()

		t, _ := strconv.Atoi(res[1])
		mm, _ := strconv.Atoi(res[2])

		return time.Date(y, m, d, t, mm, 0, 0, zone), nil
	} else if regexp.MustCompile(`^\d{1,}h$|^\d{1,}m$|^\d{1,}h\d{1,}m$`).MatchString(*timeStamp) {
		d, _ := time.ParseDuration(*timeStamp)

		t := time.Now().In(zone).Add(-d)
		y, m, dd := t.Date()
		return time.Date(y, m, dd, t.Hour(), t.Minute(), 0, 0, zone), nil
	}

	//TODO check even last scenario and if it's not a recognized pattern throw an error
	t, err := time.ParseInLocation("2006-01-02T15:04:05", *timeStamp, zone)
	if err != nil {
		return t, err
	}
	return t, nil
}

type logEvent struct {
	logEvent cloudwatchlogs.FilteredLogEvent
	logGroup string
}

func formatLogMsg(ev logEvent, printTime *bool, printStreamName *bool, printGroupName *bool) string {
	msg := *ev.logEvent.Message
	if *printEventID {
		if *noColor {
			msg = fmt.Sprintf("%s - %s", *ev.logEvent.EventId, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.YellowString(*ev.logEvent.EventId), msg)
		}
	}
	if *printStreamName {
		if *noColor {
			msg = fmt.Sprintf("%s - %s", *ev.logEvent.LogStreamName, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.BlueString(*ev.logEvent.LogStreamName), msg)
		}
	}

	if *printGroupName {
		if *noColor {
			msg = fmt.Sprintf("%s - %s", ev.logGroup, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.CyanString(ev.logGroup), msg)
		}
	}

	if *printTime {
		eventTimestamp := *ev.logEvent.Timestamp / 1000
		ts := time.Unix(eventTimestamp, 0).Format(timeFormat)
		if *noColor {
			msg = fmt.Sprintf("%s - %s", ts, msg)
		} else {
			msg = fmt.Sprintf("%s - %s", color.GreenString(ts), msg)
		}
	}
	return msg
}

func oldSyntaxWarning() {
	tokenNum := len(*logGroupStreamName)
	if tokenNum > 1 { //a log stream is also specified
		if (*logGroupStreamName)[1] == "*" {
			fmt.Println("WARNING:tail syntax has changed with V3. Please refer to the documentation for the new syntax details: https://www.lucagrulla.com/cw/#v2-to-v3-breaking-changes")
			os.Exit(0)
		}
		if tokenNum > 2 { //group, stream and date
			ts := (*logGroupStreamName)[2]
			if _, err := timestampToTime(&ts); err == nil {
				fmt.Println("WARNING:tail syntax has changed with V3. Please refer to the documentation for the new syntax details: https://www.lucagrulla.com/cw/#v2-to-v3-breaking-changes")
				os.Exit(0)
			}
		}
	}
}

func main() {
	kp.Version(version).Author("Luca Grulla")

	defer newVersionMsg(version, fetchLatestVersion(), *noColor)
	go versionCheckOnSigterm()

	cmd := kingpin.MustParse(kp.Parse(os.Args[1:]))
	c := cloudwatch.New(awsProfile, awsRegion, debug)
	switch cmd {
	case "ls groups":

		for msg := range c.LsGroups() {
			fmt.Println(*msg)
		}
	case "ls streams":
		for msg := range c.LsStreams(lsLogGroupName, nil) {
			fmt.Println(*msg)
		}
	case "tail":
		oldSyntaxWarning()

		st, err := timestampToTime(startTime)
		if err != nil {
			log.Fatalf("can't parse %s as a valid date/time", *startTime)
		}
		var et time.Time
		if *endTime != "" {
			endT, errr := timestampToTime(endTime)
			if errr != nil {
				log.Fatalf("can't parse %s as a valid date/time", *endTime)
			} else {
				et = endT
			}
		}
		out := make(chan *logEvent)

		var wg sync.WaitGroup

		triggerChannels := make([]chan<- time.Time, len(*logGroupStreamName))

		coordinator := &tailCoordinator{}
		for idx, gs := range *logGroupStreamName {
			trigger := make(chan time.Time, 1)
			go func(groupStream string) {
				tokens := strings.Split(groupStream, ":")
				var prefix string
				group := tokens[0]
				if len(tokens) > 1 && tokens[1] != "*" {
					prefix = tokens[1]
				}
				for c := range c.Tail(&group, &prefix, follow, &st, &et, grep, grepv, trigger) {
					out <- &logEvent{logEvent: *c, logGroup: group}
				}
				coordinator.remove(trigger)
				wg.Done()
			}(gs)
			triggerChannels[idx] = trigger
			wg.Add(1)
		}

		coordinator.start(triggerChannels)

		go func() {
			wg.Wait()
			if *debug {
				fmt.Println("closing main channel...")
			}
			close(out)
		}()

		for logEv := range out {
			fmt.Println(formatLogMsg(*logEv, printTimestamp, printStreamName, printGroupName))
		}
	}
}

type tailCoordinator struct {
	targets *ring.Ring
	sync.RWMutex
}

func (f *tailCoordinator) start(targets []chan<- time.Time) {
	f.targets = ring.New(len(targets))
	for i := 0; i < f.targets.Len(); i++ {
		f.targets.Value = targets[i]
		f.targets = f.targets.Next()
	}

	//AWS API accepts 5 reqs/sec for account
	ticker := time.NewTicker(205 * time.Millisecond)
	go func() {
		for range ticker.C {
			if f.targets == nil {
				if *debug {
					fmt.Println("coordinator: ring buffer is empty, exiting scheduler.")
				}
				return
			}
			f.Lock()
			x := f.targets.Value.(chan<- time.Time)
			x <- time.Now()
			f.targets = f.targets.Next()
			f.Unlock()
		}
	}()
}

func (f *tailCoordinator) remove(c chan<- time.Time) {
	f.RLock()
	initialLen := f.targets.Len()
	f.RUnlock()

	var visited int
	f.Lock()
	defer f.Unlock()

	if f.targets.Len() == 1 {
		f.targets = ring.New(0)
		if *debug {
			fmt.Println("coordinator: single node buffer: reset", f.targets.Len())
		}
		return
	}

	for visited = 0; visited < f.targets.Len(); visited++ {
		if f.targets.Value == c {
			targetChan := f.targets.Value.(chan<- time.Time)

			f.targets = f.targets.Prev()
			f.targets.Unlink(1)
			close(targetChan)
			if *debug {
				fmt.Printf("coordinator: channel found and removed at index: %d\n", visited)
			}
			break
		}
		f.targets = f.targets.Next()
	}

	if f.targets.Len() < initialLen {
		for i := 0; i < visited; i++ {
			f.targets = f.targets.Prev()
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os/exec"
)

func main() {
	cmd := exec.Cmd("mjpg_streamer", "-o 'output_http.so -w /home/johan/src/mjpg-streamer/mjpg-streamer-experimental/www'", "-i 'input_raspicam.so  -ex -hf -vf night'")
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	proxy := httputil.NewSingleHostReverseProxy(&url.URL{
		Scheme:   "http",
		Host:     "127.0.0.1:8080",
		Path:     "/",
		RawQuery: "action=stream",
	})
	http.ListenAndServe(":80", proxy)
}
<commit_msg>Added cmd to start webcam<commit_after>package main

import (
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os/exec"
)

func main() {
	cmd := exec.Command("mjpg_streamer", "-o 'output_http.so -w /home/johan/src/mjpg-streamer/mjpg-streamer-experimental/www'", "-i 'input_raspicam.so  -ex -hf -vf night'")
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	proxy := httputil.NewSingleHostReverseProxy(&url.URL{
		Scheme:   "http",
		Host:     "127.0.0.1:8080",
		Path:     "/",
		RawQuery: "action=stream",
	})
	http.ListenAndServe(":80", proxy)
}
<|endoftext|>


<commit_before>// Package main provides ...
package main

import (
	"fmt"
	"os/exec"
)

func fetchPages() {
	cmd := exec.Command("git", "clone", "https://github.com/tldr-pages/tldr", "/tmp/tldr-pages")
	err := cmd.Run()
	if err != nil {
		fmt.Println("ERROR: Can't fetch tldr repository")
	}
}

func main() {
	fmt.Println("Hello world!")
}
<commit_msg>feat(cache): create cache dir<commit_after>// Package main provides ...
package main

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
)

func fetchPages() {
	cmd := exec.Command("git", "clone", "https://github.com/tldr-pages/tldr", "/tmp/tldr-pages")
	err := cmd.Run()
	if err != nil {
		fmt.Println("ERROR: Can't fetch tldr repository")
	}
}

func getHomeDirectory() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}
	if usr.HomeDir == "" {
		return "", errors.New("Can't load user's home folder path")
	}

	return usr.HomeDir, nil
}

func createCacheDir() error {
	homeDir, err := getHomeDirectory()
	if err != nil {
		os.Stderr.WriteString("ERROR: " + err.Error() + "\n")
		return err
	}

	os.MkdirAll(homeDir+"/.tldr-go", 755)
	return nil
}

func main() {
	createCacheDir()
	fmt.Println("Hello world!")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"go.zanini.me/planet-updater/firmwares"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	typeFlag       = kingpin.Flag("type", "Lamp type").Required().PlaceHolder("[pro|compact]").Enum("pro", "compact")
	destinationArg = kingpin.Arg("destination", "The WiFish address").Required().String()
	portArg        = kingpin.Arg("port", "The WiFish IP port number").Required().Uint16()
)

const (
	assetPROFirmwareName     = "PRO-V14.bin"
	assetCompactFirmwareName = "Compact-V15.bin"
)

func main() {
	kingpin.Version("1.0.0")
	kingpin.Parse()

	fmt.Println(updatePlanet(*destinationArg, int(*portArg), firmwares.MustAsset("PRO-V14.bin")))
}
<commit_msg>Add type selection<commit_after>package main

import (
	"fmt"

	"go.zanini.me/planet-updater/firmwares"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	typeFlag       = kingpin.Flag("type", "Lamp type").Required().PlaceHolder("[pro|compact]").Enum("pro", "compact")
	destinationArg = kingpin.Arg("destination", "The WiFish address").Required().String()
	portArg        = kingpin.Arg("port", "The WiFish IP port number").Required().Uint16()
)

const (
	assetPROFirmwareName     = "PRO-V14.bin"
	assetCompactFirmwareName = "Compact-V15.bin"
)

func main() {
	kingpin.Version("1.0.0")
	kingpin.Parse()

	var firmwareAsset string

	if *typeFlag == "pro" {
		firmwareAsset = assetPROFirmwareName
	} else {
		firmwareAsset = assetCompactFirmwareName
	}

	fmt.Println(updatePlanet(*destinationArg, int(*portArg), firmwares.MustAsset(firmwareAsset)))
}
<|endoftext|>