<commit_before>package env

import (
	"encoding"
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/caarlos0/env/parsers"
)

// nolint: gochecknoglobals
var (
	// ErrNotAStructPtr is returned if you pass something that is not a pointer to a
	// Struct to Parse
	ErrNotAStructPtr = errors.New("env: expected a pointer to a Struct")
	// OnEnvVarSet is an optional convenience callback, such as for logging purposes.
	// If not nil, it's called after successfully setting the given field from the given value.
	OnEnvVarSet func(reflect.StructField, string)

	defaultBuiltInParsers = map[reflect.Kind]ParserFunc{
		reflect.Bool: func(v string) (interface{}, error) {
			return strconv.ParseBool(v)
		},
		reflect.String: func(v string) (interface{}, error) {
			return v, nil
		},
		reflect.Int: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int(i), err
		},
		reflect.Int16: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 16)
			return int16(i), err
		},
		reflect.Int32: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int32(i), err
		},
		reflect.Int64: func(v string) (interface{}, error) {
			return strconv.ParseInt(v, 10, 64)
		},
		reflect.Int8: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 8)
			return int8(i), err
		},
		reflect.Uint: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint(i), err
		},
		reflect.Uint16: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 16)
			return uint16(i), err
		},
		reflect.Uint32: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint32(i), err
		},
		reflect.Uint64: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 64)
			return i, err
		},
		reflect.Uint8: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 8)
			return uint8(i), err
		},
		reflect.Float64: func(v string) (interface{}, error) {
			return strconv.ParseFloat(v, 64)
		},
		reflect.Float32: func(v string) (interface{}, error) {
			f, err := strconv.ParseFloat(v, 32)
			return float32(f), err
		},
	}
)

func defaultCustomParsers() CustomParsers {
	return CustomParsers{
		parsers.URLType:      parsers.URLFunc,
		parsers.DurationType: parsers.DurationFunc,
	}
}

// CustomParsers is a friendly name for the type that `ParseWithFuncs()` accepts
type CustomParsers map[reflect.Type]ParserFunc

// ParserFunc defines the signature of a function that can be used within `CustomParsers`
type ParserFunc func(v string) (interface{}, error)

// Parse parses a struct containing `env` tags and loads its values from
// environment variables.
func Parse(v interface{}) error {
	return ParseWithFuncs(v, CustomParsers{})
}

// ParseWithFuncs is the same as `Parse` except it also allows the user to pass
// in custom parsers.
func ParseWithFuncs(v interface{}, funcMap CustomParsers) error {
	ptrRef := reflect.ValueOf(v)
	if ptrRef.Kind() != reflect.Ptr {
		return ErrNotAStructPtr
	}
	ref := ptrRef.Elem()
	if ref.Kind() != reflect.Struct {
		return ErrNotAStructPtr
	}
	var parsers = defaultCustomParsers()
	for k, v := range funcMap {
		parsers[k] = v
	}
	return doParse(ref, parsers)
}

func doParse(ref reflect.Value, funcMap CustomParsers) error {
	refType := ref.Type()

	for i := 0; i < refType.NumField(); i++ {
		refField := ref.Field(i)
		if reflect.Ptr == refField.Kind() && !refField.IsNil() && refField.CanSet() {
			err := Parse(refField.Interface())
			if nil != err {
				return err
			}
			continue
		}
		refTypeField := refType.Field(i)
		value, err := get(refTypeField)
		if err != nil {
			return err
		}
		if value == "" {
			if reflect.Struct == refField.Kind() {
				if err := doParse(refField, funcMap); err != nil {
					return err
				}
			}
			continue
		}
		if err := set(refField, refTypeField, value, funcMap); err != nil {
			return err
		}
		// TODO: change this to a param instead of global
		if OnEnvVarSet != nil {
			OnEnvVarSet(refTypeField, value)
		}
	}
	return nil
}

func get(field reflect.StructField) (string, error) {
	var (
		val string
		err error
	)

	key, opts := parseKeyForOption(field.Tag.Get("env"))

	defaultValue := field.Tag.Get("envDefault")
	val = getOr(key, defaultValue)

	expandVar := field.Tag.Get("envExpand")
	if strings.ToLower(expandVar) == "true" {
		val = os.ExpandEnv(val)
	}

	if len(opts) > 0 {
		for _, opt := range opts {
			// The only option supported is "required".
			switch opt {
			case "":
				break
			case "required":
				val, err = getRequired(key)
			default:
				err = fmt.Errorf("env: tag option %q not supported", opt)
			}
		}
	}

	return val, err
}

// split the env tag's key into the expected key and desired option, if any.
func parseKeyForOption(key string) (string, []string) {
	opts := strings.Split(key, ",")
	return opts[0], opts[1:]
}

func getRequired(key string) (string, error) {
	if value, ok := os.LookupEnv(key); ok {
		return value, nil
	}
	return "", fmt.Errorf(`env: required environment variable %q is not set`, key)
}

func getOr(key, defaultValue string) string {
	value, ok := os.LookupEnv(key)
	if ok {
		return value
	}
	return defaultValue
}

func set(field reflect.Value, sf reflect.StructField, value string, funcMap CustomParsers) error {
	if field.Kind() == reflect.Slice {
		return handleSlice(field, value, sf, funcMap)
	}

	parserFunc, ok := funcMap[sf.Type]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return newParseError(sf, err)
		}
		field.Set(reflect.ValueOf(val))
		return nil
	}

	parserFunc, ok = defaultBuiltInParsers[sf.Type.Kind()]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return newParseError(sf, err)
		}
		field.Set(reflect.ValueOf(val).Convert(sf.Type))
		return nil
	}

	return handleTextUnmarshaler(field, value, sf)
}

func handleSlice(field reflect.Value, value string, sf reflect.StructField, funcMap CustomParsers) error {
	var separator = sf.Tag.Get("envSeparator")
	if separator == "" {
		separator = ","
	}
	var parts = strings.Split(value, separator)

	var elemType = sf.Type.Elem()
	if elemType.Kind() == reflect.Ptr {
		elemType = elemType.Elem()
	}

	if _, ok := reflect.New(elemType).Interface().(encoding.TextUnmarshaler); ok {
		return parseTextUnmarshalers(field, parts, sf)
	}

	parserFunc, ok := funcMap[elemType]
	if !ok {
		parserFunc, ok = defaultBuiltInParsers[elemType.Kind()]
		if !ok {
			return newNoParserError(sf)
		}
	}

	var result = reflect.MakeSlice(sf.Type, 0, len(parts))
	for _, part := range parts {
		r, err := parserFunc(part)
		if err != nil {
			return newParseError(sf, err)
		}
		var v = reflect.ValueOf(r).Convert(elemType)
		if sf.Type.Elem().Kind() == reflect.Ptr {
			// TODO: add this!
			return fmt.Errorf("env: pointer slices of built-in and aliased types are not supported: %s %s", sf.Name, sf.Type)
		}
		result = reflect.Append(result, v)
	}

	field.Set(result)
	return nil
}

func handleTextUnmarshaler(field reflect.Value, value string, sf reflect.StructField) error {
	if reflect.Ptr == field.Kind() {
		if field.IsNil() {
			field.Set(reflect.New(field.Type().Elem()))
		}
	} else if field.CanAddr() {
		field = field.Addr()
	}

	tm, ok := field.Interface().(encoding.TextUnmarshaler)
	if !ok {
		return newNoParserError(sf)
	}
	var err = tm.UnmarshalText([]byte(value))
	return newParseError(sf, err)
}

func parseTextUnmarshalers(field reflect.Value, data []string, sf reflect.StructField) error {
	s := len(data)
	elemType := field.Type().Elem()
	slice := reflect.MakeSlice(reflect.SliceOf(elemType), s, s)
	for i, v := range data {
		sv := slice.Index(i)
		kind := sv.Kind()
		if kind == reflect.Ptr {
			sv = reflect.New(elemType.Elem())
		} else {
			sv = sv.Addr()
		}
		tm := sv.Interface().(encoding.TextUnmarshaler)
		if err := tm.UnmarshalText([]byte(v)); err != nil {
			return newParseError(sf, err)
		}
		if kind == reflect.Ptr {
			slice.Index(i).Set(sv)
		}
	}

	field.Set(slice)

	return nil
}

func newParseError(sf reflect.StructField, err error) error {
	if err == nil {
		return nil
	}
	return parseError{
		sf:  sf,
		err: err,
	}
}

type parseError struct {
	sf  reflect.StructField
	err error
}

func (e parseError) Error() string {
	return fmt.Sprintf(`env: parse error on field "%s" of type "%s": %v`, e.sf.Name, e.sf.Type, e.err)
}

func newNoParserError(sf reflect.StructField) error {
	return fmt.Errorf(`env: no parser found for field "%s" of type "%s"`, sf.Name, sf.Type)
}
<commit_msg>fix: remove OnEnvVarSet<commit_after>package env

import (
	"encoding"
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"

	"github.com/caarlos0/env/parsers"
)

// nolint: gochecknoglobals
var (
	// ErrNotAStructPtr is returned if you pass something that is not a pointer to a
	// Struct to Parse
	ErrNotAStructPtr = errors.New("env: expected a pointer to a Struct")

	defaultBuiltInParsers = map[reflect.Kind]ParserFunc{
		reflect.Bool: func(v string) (interface{}, error) {
			return strconv.ParseBool(v)
		},
		reflect.String: func(v string) (interface{}, error) {
			return v, nil
		},
		reflect.Int: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int(i), err
		},
		reflect.Int16: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 16)
			return int16(i), err
		},
		reflect.Int32: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 32)
			return int32(i), err
		},
		reflect.Int64: func(v string) (interface{}, error) {
			return strconv.ParseInt(v, 10, 64)
		},
		reflect.Int8: func(v string) (interface{}, error) {
			i, err := strconv.ParseInt(v, 10, 8)
			return int8(i), err
		},
		reflect.Uint: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint(i), err
		},
		reflect.Uint16: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 16)
			return uint16(i), err
		},
		reflect.Uint32: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 32)
			return uint32(i), err
		},
		reflect.Uint64: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 64)
			return i, err
		},
		reflect.Uint8: func(v string) (interface{}, error) {
			i, err := strconv.ParseUint(v, 10, 8)
			return uint8(i), err
		},
		reflect.Float64: func(v string) (interface{}, error) {
			return strconv.ParseFloat(v, 64)
		},
		reflect.Float32: func(v string) (interface{}, error) {
			f, err := strconv.ParseFloat(v, 32)
			return float32(f), err
		},
	}
)

func defaultCustomParsers() CustomParsers {
	return CustomParsers{
		parsers.URLType:      parsers.URLFunc,
		parsers.DurationType: parsers.DurationFunc,
	}
}

// CustomParsers is a friendly name for the type that `ParseWithFuncs()` accepts
type CustomParsers map[reflect.Type]ParserFunc

// ParserFunc defines the signature of a function that can be used within `CustomParsers`
type ParserFunc func(v string) (interface{}, error)

// Parse parses a struct containing `env` tags and loads its values from
// environment variables.
func Parse(v interface{}) error {
	return ParseWithFuncs(v, CustomParsers{})
}

// ParseWithFuncs is the same as `Parse` except it also allows the user to pass
// in custom parsers.
func ParseWithFuncs(v interface{}, funcMap CustomParsers) error {
	ptrRef := reflect.ValueOf(v)
	if ptrRef.Kind() != reflect.Ptr {
		return ErrNotAStructPtr
	}
	ref := ptrRef.Elem()
	if ref.Kind() != reflect.Struct {
		return ErrNotAStructPtr
	}
	var parsers = defaultCustomParsers()
	for k, v := range funcMap {
		parsers[k] = v
	}
	return doParse(ref, parsers)
}

func doParse(ref reflect.Value, funcMap CustomParsers) error {
	refType := ref.Type()

	for i := 0; i < refType.NumField(); i++ {
		refField := ref.Field(i)
		if reflect.Ptr == refField.Kind() && !refField.IsNil() && refField.CanSet() {
			err := Parse(refField.Interface())
			if nil != err {
				return err
			}
			continue
		}
		refTypeField := refType.Field(i)
		value, err := get(refTypeField)
		if err != nil {
			return err
		}
		if value == "" {
			if reflect.Struct == refField.Kind() {
				if err := doParse(refField, funcMap); err != nil {
					return err
				}
			}
			continue
		}
		if err := set(refField, refTypeField, value, funcMap); err != nil {
			return err
		}
	}
	return nil
}

func get(field reflect.StructField) (string, error) {
	var (
		val string
		err error
	)

	key, opts := parseKeyForOption(field.Tag.Get("env"))

	defaultValue := field.Tag.Get("envDefault")
	val = getOr(key, defaultValue)

	expandVar := field.Tag.Get("envExpand")
	if strings.ToLower(expandVar) == "true" {
		val = os.ExpandEnv(val)
	}

	if len(opts) > 0 {
		for _, opt := range opts {
			// The only option supported is "required".
			switch opt {
			case "":
				break
			case "required":
				val, err = getRequired(key)
			default:
				err = fmt.Errorf("env: tag option %q not supported", opt)
			}
		}
	}

	return val, err
}

// split the env tag's key into the expected key and desired option, if any.
func parseKeyForOption(key string) (string, []string) {
	opts := strings.Split(key, ",")
	return opts[0], opts[1:]
}

func getRequired(key string) (string, error) {
	if value, ok := os.LookupEnv(key); ok {
		return value, nil
	}
	return "", fmt.Errorf(`env: required environment variable %q is not set`, key)
}

func getOr(key, defaultValue string) string {
	value, ok := os.LookupEnv(key)
	if ok {
		return value
	}
	return defaultValue
}

func set(field reflect.Value, sf reflect.StructField, value string, funcMap CustomParsers) error {
	if field.Kind() == reflect.Slice {
		return handleSlice(field, value, sf, funcMap)
	}

	parserFunc, ok := funcMap[sf.Type]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return newParseError(sf, err)
		}
		field.Set(reflect.ValueOf(val))
		return nil
	}

	parserFunc, ok = defaultBuiltInParsers[sf.Type.Kind()]
	if ok {
		val, err := parserFunc(value)
		if err != nil {
			return newParseError(sf, err)
		}
		field.Set(reflect.ValueOf(val).Convert(sf.Type))
		return nil
	}

	return handleTextUnmarshaler(field, value, sf)
}

func handleSlice(field reflect.Value, value string, sf reflect.StructField, funcMap CustomParsers) error {
	var separator = sf.Tag.Get("envSeparator")
	if separator == "" {
		separator = ","
	}
	var parts = strings.Split(value, separator)

	var elemType = sf.Type.Elem()
	if elemType.Kind() == reflect.Ptr {
		elemType = elemType.Elem()
	}

	if _, ok := reflect.New(elemType).Interface().(encoding.TextUnmarshaler); ok {
		return parseTextUnmarshalers(field, parts, sf)
	}

	parserFunc, ok := funcMap[elemType]
	if !ok {
		parserFunc, ok = defaultBuiltInParsers[elemType.Kind()]
		if !ok {
			return newNoParserError(sf)
		}
	}

	var result = reflect.MakeSlice(sf.Type, 0, len(parts))
	for _, part := range parts {
		r, err := parserFunc(part)
		if err != nil {
			return newParseError(sf, err)
		}
		var v = reflect.ValueOf(r).Convert(elemType)
		if sf.Type.Elem().Kind() == reflect.Ptr {
			// TODO: add this!
			return fmt.Errorf("env: pointer slices of built-in and aliased types are not supported: %s %s", sf.Name, sf.Type)
		}
		result = reflect.Append(result, v)
	}

	field.Set(result)
	return nil
}

func handleTextUnmarshaler(field reflect.Value, value string, sf reflect.StructField) error {
	if reflect.Ptr == field.Kind() {
		if field.IsNil() {
			field.Set(reflect.New(field.Type().Elem()))
		}
	} else if field.CanAddr() {
		field = field.Addr()
	}

	tm, ok := field.Interface().(encoding.TextUnmarshaler)
	if !ok {
		return newNoParserError(sf)
	}
	var err = tm.UnmarshalText([]byte(value))
	return newParseError(sf, err)
}

func parseTextUnmarshalers(field reflect.Value, data []string, sf reflect.StructField) error {
	s := len(data)
	elemType := field.Type().Elem()
	slice := reflect.MakeSlice(reflect.SliceOf(elemType), s, s)
	for i, v := range data {
		sv := slice.Index(i)
		kind := sv.Kind()
		if kind == reflect.Ptr {
			sv = reflect.New(elemType.Elem())
		} else {
			sv = sv.Addr()
		}
		tm := sv.Interface().(encoding.TextUnmarshaler)
		if err := tm.UnmarshalText([]byte(v)); err != nil {
			return newParseError(sf, err)
		}
		if kind == reflect.Ptr {
			slice.Index(i).Set(sv)
		}
	}

	field.Set(slice)

	return nil
}

func newParseError(sf reflect.StructField, err error) error {
	if err == nil {
		return nil
	}
	return parseError{
		sf:  sf,
		err: err,
	}
}

type parseError struct {
	sf  reflect.StructField
	err error
}

func (e parseError) Error() string {
	return fmt.Sprintf(`env: parse error on field "%s" of type "%s": %v`, e.sf.Name, e.sf.Type, e.err)
}

func newNoParserError(sf reflect.StructField) error {
	return fmt.Errorf(`env: no parser found for field "%s" of type "%s"`, sf.Name, sf.Type)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var cmdFix = &Command{
	UsageLine: "fix",
	Short:     "fix the beego application to compatibel with beego 1.6",
	Long: `
As from beego1.6, there's some incompatible code with the old version.

bee fix help to upgrade the application to beego 1.6
`,
}

func init() {
	cmdFix.Run = runFix
}

func runFix(cmd *Command, args []string) int {
	dir, err := os.Getwd()
	if err != nil {
		ColorLog("GetCurrent Path:%s\n", err)
	}
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}
			return nil
		}
		if err != nil {
			return err
		}
		ColorLog("%s\n", path)
		err = fixFile(path)
		if err != nil {
			ColorLog("fixFile:%s\n", err)
		}
		return err
	})
	return 0
}

var rules = []string{
	"beego.AppName", "beego.BConfig.AppName",
	"beego.RunMode", "beego.BConfig.RunMode",
	"beego.RecoverPanic", "beego.BConfig.RecoverPanic",
	"beego.RouterCaseSensitive", "beego.BConfig.RouterCaseSensitive",
	"beego.BeegoServerName", "beego.BConfig.ServerName",
	"beego.EnableGzip", "beego.BConfig.EnableGzip",
	"beego.ErrorsShow", "beego.BConfig.EnableErrorsShow",
	"beego.CopyRequestBody", "beego.BConfig.CopyRequestBody",
	"beego.MaxMemory", "beego.BConfig.MaxMemory",
	"beego.Graceful", "beego.BConfig.Listen.Graceful",
	"beego.HttpAddr", "beego.BConfig.Listen.HTTPAddr",
	"beego.HttpPort", "beego.BConfig.Listen.HTTPPort",
	"beego.ListenTCP4", "beego.BConfig.Listen.ListenTCP4",
	"beego.EnableHttpListen", "beego.BConfig.Listen.HTTPEnable",
	"beego.EnableHttpTLS", "beego.BConfig.Listen.HTTPSEnable",
	"beego.HttpsAddr", "beego.BConfig.Listen.HTTPSAddr",
	"beego.HttpsPort", "beego.BConfig.Listen.HTTPSPort",
	"beego.HttpCertFile", "beego.BConfig.Listen.HTTPSCertFile",
	"beego.HttpKeyFile", "beego.BConfig.Listen.HTTPSKeyFile",
	"beego.EnableAdmin", "beego.BConfig.Listen.AdminEnable",
	"beego.AdminHttpAddr", "beego.BConfig.Listen.AdminAddr",
	"beego.AdminHttpPort", "beego.BConfig.Listen.AdminPort",
	"beego.UseFcgi", "beego.BConfig.Listen.EnableFcgi",
	"beego.HttpServerTimeOut", "beego.BConfig.Listen.ServerTimeOut",
	"beego.AutoRender", "beego.BConfig.WebConfig.AutoRender",
	"beego.ViewsPath", "beego.BConfig.WebConfig.ViewsPath",
	"beego.StaticDir", "beego.BConfig.WebConfig.StaticDir",
	"beego.StaticExtensionsToGzip", "beego.BConfig.WebConfig.StaticExtensionsToGzip",
	"beego.DirectoryIndex", "beego.BConfig.WebConfig.DirectoryIndex",
	"beego.FlashName", "beego.BConfig.WebConfig.FlashName",
	"beego.FlashSeperator", "beego.BConfig.WebConfig.FlashSeperator",
	"beego.EnableDocs", "beego.BConfig.WebConfig.EnableDocs",
	"beego.XSRFKEY", "beego.BConfig.WebConfig.XSRFKEY",
	"beego.EnableXSRF", "beego.BConfig.WebConfig.EnableXSRF",
	"beego.XSRFExpire", "beego.BConfig.WebConfig.XSRFExpire",
	"beego.TemplateLeft", "beego.BConfig.WebConfig.TemplateLeft",
	"beego.TemplateRight", "beego.BConfig.WebConfig.TemplateRight",
	"beego.SessionOn", "beego.BConfig.WebConfig.Session.SessionOn",
	"beego.SessionProvider", "beego.BConfig.WebConfig.Session.SessionProvider",
	"beego.SessionName", "beego.BConfig.WebConfig.Session.SessionName",
	"beego.SessionGCMaxLifetime", "beego.BConfig.WebConfig.Session.SessionGCMaxLifetime",
	"beego.SessionSavePath", "beego.BConfig.WebConfig.Session.SessionProviderConfig",
	"beego.SessionCookieLifeTime", "beego.BConfig.WebConfig.Session.SessionCookieLifeTime",
	"beego.SessionAutoSetCookie", "beego.BConfig.WebConfig.Session.SessionAutoSetCookie",
	"beego.SessionDomain", "beego.BConfig.WebConfig.Session.SessionDomain",
	"Ctx.Input.CopyBody(", "Ctx.Input.CopyBody(beego.BConfig.MaxMemory",
	".UrlFor(", ".URLFor(",
	".ServeJson(", ".ServeJSON(",
	".ServeXml(", ".ServeXML(",
	".XsrfToken(", ".XSRFToken(",
	".CheckXsrfCookie(", ".CheckXSRFCookie(",
	".XsrfFormHtml(", ".XSRFFormHTML(",
	"beego.UrlFor(", "beego.URLFor(",
	"beego.GlobalDocApi", "beego.GlobalDocAPI",
	"beego.Errorhandler", "beego.ErrorHandler",
	"Output.Jsonp(", "Output.JSONP(",
	"Output.Json(", "Output.JSON(",
	"Output.Xml(", "Output.XML(",
	"Input.Uri()", "Input.URI()",
	"Input.Url()", "Input.URL()",
	"Input.AcceptsHtml()", "Input.AcceptsHTML()",
	"Input.AcceptsXml()", "Input.AcceptsXML()",
	"Input.AcceptsJson()", "Input.AcceptsJSON()",
	"Ctx.XsrfToken()", "Ctx.XSRFToken()",
	"Ctx.CheckXsrfCookie()", "Ctx.CheckXSRFCookie()",
	"session.SessionStore", "session.Store",
	".TplNames", ".TplName",
	"swagger.ApiRef", "swagger.APIRef",
	"swagger.ApiDeclaration", "swagger.APIDeclaration",
	"swagger.Api", "swagger.API",
	"swagger.ApiRef", "swagger.APIRef",
	"toolbox.UrlMap", "toolbox.URLMap",
	"logs.LoggerInterface", "logs.Logger",
	"Input.Request", "Input.Context.Request",
	"Input.Params)", "Input.Params())",
	"httplib.BeegoHttpSettings", "httplib.BeegoHTTPSettings",
	"httplib.BeegoHttpRequest", "httplib.BeegoHTTPRequest",
	".TlsClientConfig", ".TLSClientConfig",
	".JsonBody", ".JSONBody",
	".ToJson", ".ToJSON",
	".ToXml", ".ToXML",
	"beego.Html2str", "beego.HTML2str",
	"beego.AssetsCss", "beego.AssetsCSS",
	"orm.DR_Sqlite", "orm.DRSqlite",
	"orm.DR_Postgres", "orm.DRPostgres",
	"orm.DR_MySQL", "orm.DRMySQL",
	"orm.DR_Oracle", "orm.DROracle",
	"orm.Col_Add", "orm.ColAdd",
	"orm.Col_Minus", "orm.ColMinus",
	"orm.Col_Multiply", "orm.ColMultiply",
	"orm.Col_Except", "orm.ColExcept",
	"GenerateOperatorSql", "GenerateOperatorSQL",
	"OperatorSql", "OperatorSQL",
	"orm.Debug_Queries", "orm.DebugQueries",
	"orm.COMMA_SPACE", "orm.CommaSpace",
}

func fixFile(file string) error {
	rp := strings.NewReplacer(rules...)
	content, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	fixed := rp.Replace(string(content))

	// forword the RequestBody from the replace
	// "Input.Request", "Input.Context.Request",
	fixed = strings.Replace(fixed, "Input.Context.RequestBody", "Input.RequestBody", -1)

	// regexp replace
	pareg := regexp.MustCompile(`(Input.Params\[")(.*)("])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Param(\"$2\")")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])(\s)(=)(\s)(.*)`)
	fixed = pareg.ReplaceAllString(fixed, "Input.SetData(\"$2\", $7)")
	// replace the v.Apis in docs.go
	if strings.Contains(file, "docs.go") {
		fixed = strings.Replace(fixed, "v.Apis", "v.APIs", -1)
	}

	err = os.Truncate(file, 0)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(file, []byte(fixed), 0666)
}
<commit_msg>fix data<commit_after>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var cmdFix = &Command{
	UsageLine: "fix",
	Short:     "fix the beego application to compatibel with beego 1.6",
	Long: `
As from beego1.6, there's some incompatible code with the old version.

bee fix help to upgrade the application to beego 1.6
`,
}

func init() {
	cmdFix.Run = runFix
}

func runFix(cmd *Command, args []string) int {
	dir, err := os.Getwd()
	if err != nil {
		ColorLog("GetCurrent Path:%s\n", err)
	}
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}
			return nil
		}
		if err != nil {
			return err
		}
		ColorLog("%s\n", path)
		err = fixFile(path)
		if err != nil {
			ColorLog("fixFile:%s\n", err)
		}
		return err
	})
	return 0
}

var rules = []string{
	"beego.AppName", "beego.BConfig.AppName",
	"beego.RunMode", "beego.BConfig.RunMode",
	"beego.RecoverPanic", "beego.BConfig.RecoverPanic",
	"beego.RouterCaseSensitive", "beego.BConfig.RouterCaseSensitive",
	"beego.BeegoServerName", "beego.BConfig.ServerName",
	"beego.EnableGzip", "beego.BConfig.EnableGzip",
	"beego.ErrorsShow", "beego.BConfig.EnableErrorsShow",
	"beego.CopyRequestBody", "beego.BConfig.CopyRequestBody",
	"beego.MaxMemory", "beego.BConfig.MaxMemory",
	"beego.Graceful", "beego.BConfig.Listen.Graceful",
	"beego.HttpAddr", "beego.BConfig.Listen.HTTPAddr",
	"beego.HttpPort", "beego.BConfig.Listen.HTTPPort",
	"beego.ListenTCP4", "beego.BConfig.Listen.ListenTCP4",
	"beego.EnableHttpListen", "beego.BConfig.Listen.HTTPEnable",
	"beego.EnableHttpTLS", "beego.BConfig.Listen.HTTPSEnable",
	"beego.HttpsAddr", "beego.BConfig.Listen.HTTPSAddr",
	"beego.HttpsPort", "beego.BConfig.Listen.HTTPSPort",
	"beego.HttpCertFile", "beego.BConfig.Listen.HTTPSCertFile",
	"beego.HttpKeyFile", "beego.BConfig.Listen.HTTPSKeyFile",
	"beego.EnableAdmin", "beego.BConfig.Listen.AdminEnable",
	"beego.AdminHttpAddr", "beego.BConfig.Listen.AdminAddr",
	"beego.AdminHttpPort", "beego.BConfig.Listen.AdminPort",
	"beego.UseFcgi", "beego.BConfig.Listen.EnableFcgi",
	"beego.HttpServerTimeOut", "beego.BConfig.Listen.ServerTimeOut",
	"beego.AutoRender", "beego.BConfig.WebConfig.AutoRender",
	"beego.ViewsPath", "beego.BConfig.WebConfig.ViewsPath",
	"beego.StaticDir", "beego.BConfig.WebConfig.StaticDir",
	"beego.StaticExtensionsToGzip", "beego.BConfig.WebConfig.StaticExtensionsToGzip",
	"beego.DirectoryIndex", "beego.BConfig.WebConfig.DirectoryIndex",
	"beego.FlashName", "beego.BConfig.WebConfig.FlashName",
	"beego.FlashSeperator", "beego.BConfig.WebConfig.FlashSeperator",
	"beego.EnableDocs", "beego.BConfig.WebConfig.EnableDocs",
	"beego.XSRFKEY", "beego.BConfig.WebConfig.XSRFKEY",
	"beego.EnableXSRF", "beego.BConfig.WebConfig.EnableXSRF",
	"beego.XSRFExpire", "beego.BConfig.WebConfig.XSRFExpire",
	"beego.TemplateLeft", "beego.BConfig.WebConfig.TemplateLeft",
	"beego.TemplateRight", "beego.BConfig.WebConfig.TemplateRight",
	"beego.SessionOn", "beego.BConfig.WebConfig.Session.SessionOn",
	"beego.SessionProvider", "beego.BConfig.WebConfig.Session.SessionProvider",
	"beego.SessionName", "beego.BConfig.WebConfig.Session.SessionName",
	"beego.SessionGCMaxLifetime", "beego.BConfig.WebConfig.Session.SessionGCMaxLifetime",
	"beego.SessionSavePath", "beego.BConfig.WebConfig.Session.SessionProviderConfig",
	"beego.SessionCookieLifeTime", "beego.BConfig.WebConfig.Session.SessionCookieLifeTime",
	"beego.SessionAutoSetCookie", "beego.BConfig.WebConfig.Session.SessionAutoSetCookie",
	"beego.SessionDomain", "beego.BConfig.WebConfig.Session.SessionDomain",
	"Ctx.Input.CopyBody(", "Ctx.Input.CopyBody(beego.BConfig.MaxMemory",
	".UrlFor(", ".URLFor(",
	".ServeJson(", ".ServeJSON(",
	".ServeXml(", ".ServeXML(",
	".XsrfToken(", ".XSRFToken(",
	".CheckXsrfCookie(", ".CheckXSRFCookie(",
	".XsrfFormHtml(", ".XSRFFormHTML(",
	"beego.UrlFor(", "beego.URLFor(",
	"beego.GlobalDocApi", "beego.GlobalDocAPI",
	"beego.Errorhandler", "beego.ErrorHandler",
	"Output.Jsonp(", "Output.JSONP(",
	"Output.Json(", "Output.JSON(",
	"Output.Xml(", "Output.XML(",
	"Input.Uri()", "Input.URI()",
	"Input.Url()", "Input.URL()",
	"Input.AcceptsHtml()", "Input.AcceptsHTML()",
	"Input.AcceptsXml()", "Input.AcceptsXML()",
	"Input.AcceptsJson()", "Input.AcceptsJSON()",
	"Ctx.XsrfToken()", "Ctx.XSRFToken()",
	"Ctx.CheckXsrfCookie()", "Ctx.CheckXSRFCookie()",
	"session.SessionStore", "session.Store",
	".TplNames", ".TplName",
	"swagger.ApiRef", "swagger.APIRef",
	"swagger.ApiDeclaration", "swagger.APIDeclaration",
	"swagger.Api", "swagger.API",
	"swagger.ApiRef", "swagger.APIRef",
	"toolbox.UrlMap", "toolbox.URLMap",
	"logs.LoggerInterface", "logs.Logger",
	"Input.Request", "Input.Context.Request",
	"Input.Params)", "Input.Params())",
	"httplib.BeegoHttpSettings", "httplib.BeegoHTTPSettings",
	"httplib.BeegoHttpRequest", "httplib.BeegoHTTPRequest",
	".TlsClientConfig", ".TLSClientConfig",
	".JsonBody", ".JSONBody",
	".ToJson", ".ToJSON",
	".ToXml", ".ToXML",
	"beego.Html2str", "beego.HTML2str",
	"beego.AssetsCss", "beego.AssetsCSS",
	"orm.DR_Sqlite", "orm.DRSqlite",
	"orm.DR_Postgres", "orm.DRPostgres",
	"orm.DR_MySQL", "orm.DRMySQL",
	"orm.DR_Oracle", "orm.DROracle",
	"orm.Col_Add", "orm.ColAdd",
	"orm.Col_Minus", "orm.ColMinus",
	"orm.Col_Multiply", "orm.ColMultiply",
	"orm.Col_Except", "orm.ColExcept",
	"GenerateOperatorSql", "GenerateOperatorSQL",
	"OperatorSql", "OperatorSQL",
	"orm.Debug_Queries", "orm.DebugQueries",
	"orm.COMMA_SPACE", "orm.CommaSpace",
}

func fixFile(file string) error {
	rp := strings.NewReplacer(rules...)
	content, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	fixed := rp.Replace(string(content))

	// forword the RequestBody from the replace
	// "Input.Request", "Input.Context.Request",
	fixed = strings.Replace(fixed, "Input.Context.RequestBody", "Input.RequestBody", -1)

	// regexp replace
	pareg := regexp.MustCompile(`(Input.Params\[")(.*)("])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Param(\"$2\")")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])(\s)(=)(\s)(.*)`)
	fixed = pareg.ReplaceAllString(fixed, "Input.SetData(\"$2\", $7)")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Data(\"$2\")")
	// replace the v.Apis in docs.go
	if strings.Contains(file, "docs.go") {
		fixed = strings.Replace(fixed, "v.Apis", "v.APIs", -1)
	}

	err = os.Truncate(file, 0)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(file, []byte(fixed), 0666)
}
<|endoftext|>


<commit_before>package blockdevice

import (
	"fmt"
	"github.com/ceph/go-ceph/rados"
	"github.com/ceph/go-ceph/rbd"
	"os/exec"
	"strings"
)

const (
	DefaultPoolName       = "rbd"
	DefaultFileSystemType = "xfs"
)

type Connection struct {
	*rados.Conn
	context  *rados.IOContext
	pool     string
	username string
	cluster  string
}

type Image struct {
	*rbd.Image
	*rbd.ImageInfo
	*Connection
	name string
}

type Device struct {
	path           string
	isMounted      bool
	fileSystemType string
	mountPoint     string
}

func (d *Device) GetPath() string {
	return d.path
}

func (d *Device) GetMountPoint() string {
	return d.mountPoint
}

func (d *Device) Mount(mountPoint string) (string, error) {
	if d.isMounted && d.mountPoint == mountPoint {
		return "", fmt.Errorf("Device: %s is already mounted on path: %s", d.path, d.mountPoint)
	}

	if !d.IsAlreadyFormatted() {
		if err := d.Format(); err != nil {
			return "", err
		}
	}

	if _, err := RunCommand("mount", "-t", d.fileSystemType, d.path, mountPoint); err != nil {
		return "", err
	}

	d.isMounted = true
	return mountPoint, nil
}

func (d *Device) Format() error {
	mkfs, err := exec.LookPath("mkfs." + d.fileSystemType)
	if err != nil {
		return fmt.Errorf("Cannot format device:%s, Error: %s", d.path, err)
	}

	if _, err = RunCommand(mkfs, d.path); err != nil {
		return fmt.Errorf("Cannot format device:%s, Error: %s", d.path, err)
	}
	return nil
}

func (d *Device) GetFileSystemType() (string, error) {
	format, err := RunCommand("blkid", "-o", "value", "-s", "TYPE", d.path)
	if err != nil {
		return "", err
	}
	return format, nil
}

func (d *Device) IsAlreadyFormatted() bool {
	if current, _ := d.GetFileSystemType(); current == d.fileSystemType {
		return true
	}
	return false
}

func (d *Device) UnMap() error {
	if d.isMounted {
		if err := d.UnMount(); err != nil {
			return err
		}
	}

	if _, err := RunCommand("rbd", "unmap", d.path); err != nil {
		return err
	}
	return nil
}

func (d *Device) UnMount() error {
	if _, err := RunCommand("unmount", d.path); err != nil {
		return err
	}
	return nil
}

func NewDevice(image *Image, fsType string, mountPoint string) (*Device, error) {
	device, err := RunCommand("rbd", "map", "--id", image.username, "--pool", image.pool, image.name)
	if err != nil {
		return nil, err
	}

	if fsType == "" {
		fsType = DefaultFileSystemType
	}

	new_device := &Device{device, false, fsType, mountPoint}

	if err = new_device.Format(); err != nil {
		return nil, err
	}

	if mountPoint != "" {
		if _, err = new_device.Mount(mountPoint); err != nil {
			return nil, err
		}
	}

	return new_device, nil
}

func toMegs(size uint64) uint64 {
	return size * 1024 * 1024
}

func RunCommand(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	out, err := cmd.Output()
	return strings.Trim(string(out), " \n"), err
}

func (i *Image) MapToDevice(fsType string, mountPoint string) (*Device, error) {
	device, err := NewDevice(i, fsType, mountPoint)
	if err != nil {
		return nil, err
	}
	return device, err
}

func NewImage(image *rbd.Image, connection *Connection, name string) (*Image, error) {
	stat, err := image.Stat()
	if err != nil {
		return nil, err
	}

	err = image.Open(true)
	if err != nil {
		return nil, err
	}

	return &Image{
		image,
		stat,
		connection,
		name,
	}, nil
}

func (c *Connection) GetImageByName(name string) (*Image, error) {
	image := rbd.GetImage(c.context, name)
	if image == nil {
		return nil, fmt.Errorf("Image:%s not found on pool:%s", name, c.pool)
	}

	return NewImage(image, c, name)
}

func (c *Connection) GetOrCreateImage(name string, size uint64) (*Image, error) {
	image, err := c.GetImageByName(name)
	if image != nil {
		return image, nil
	}

	new_image, err := rbd.Create(c.context, name, toMegs(size))
	if err != nil {
		return nil, err
	}

	return NewImage(new_image, c, name)
}

func NewConnection(username string, pool string, cluster string, configFile string) (*Connection, error) {
	var conn *rados.Conn
	var err error

	if cluster != "" && username != "" {
		conn, err = rados.NewConnWithClusterAndUser(cluster, username)
	} else if username != "" {
		conn, err = rados.NewConnWithUser(username)
	} else {
		conn, err = rados.NewConn()
	}

	if err != nil {
		return nil, err
	}

	if configFile != "" {
		err = conn.ReadConfigFile(configFile)
	} else {
		err = conn.ReadDefaultConfigFile()
	}

	if err != nil {
		return nil, err
	}

	context, err := conn.OpenIOContext(pool)
	if err != nil {
		return nil, err
	}

	return &Connection{
		conn,
		context,
		pool,
		username,
		cluster,
	}, nil
}

func (c *Connection) Shutdown() {
	if c.context != nil {
		c.context.Destroy()
	}

	c.Shutdown()
}
<commit_msg>- NewDevice Sets isMounted equals true in case the device is mounted.<commit_after>package blockdevice

import (
	"fmt"
	"github.com/ceph/go-ceph/rados"
	"github.com/ceph/go-ceph/rbd"
	"os/exec"
	"strings"
)

const (
	DefaultPoolName       = "rbd"
	DefaultFileSystemType = "xfs"
)

type Connection struct {
	*rados.Conn
	context  *rados.IOContext
	pool     string
	username string
	cluster  string
}

type Image struct {
	*rbd.Image
	*rbd.ImageInfo
	*Connection
	name string
}

type Device struct {
	path           string
	isMounted      bool
	fileSystemType string
	mountPoint     string
}

func (d *Device) GetPath() string {
	return d.path
}

func (d *Device) GetMountPoint() string {
	return d.mountPoint
}

func (d *Device) Mount(mountPoint string) (string, error) {
	if d.isMounted && d.mountPoint == mountPoint {
		return "", fmt.Errorf("Device: %s is already mounted on path: %s", d.path, d.mountPoint)
	}

	if !d.IsAlreadyFormatted() {
		if err := d.Format(); err != nil {
			return "", err
		}
	}

	if _, err := RunCommand("mount", "-t", d.fileSystemType, d.path, mountPoint); err != nil {
		return "", err
	}

	d.isMounted = true
	return mountPoint, nil
}

func (d *Device) Format() error {
	mkfs, err := exec.LookPath("mkfs." + d.fileSystemType)
	if err != nil {
		return fmt.Errorf("Cannot format device:%s, Error: %s", d.path, err)
	}

	if _, err = RunCommand(mkfs, d.path); err != nil {
		return fmt.Errorf("Cannot format device:%s, Error: %s", d.path, err)
	}
	return nil
}

func (d *Device) GetFileSystemType() (string, error) {
	format, err := RunCommand("blkid", "-o", "value", "-s", "TYPE", d.path)
	if err != nil {
		return "", err
	}
	return format, nil
}

func (d *Device) IsAlreadyFormatted() bool {
	if current, _ := d.GetFileSystemType(); current == d.fileSystemType {
		return true
	}
	return false
}

func (d *Device) UnMap() error {
	if d.isMounted {
		if err := d.UnMount(); err != nil {
			return err
		}
	}

	if _, err := RunCommand("rbd", "unmap", d.path); err != nil {
		return err
	}
	return nil
}

func (d *Device) UnMount() error {
	if _, err := RunCommand("unmount", d.path); err != nil {
		return err
	}
	return nil
}

func NewDevice(image *Image, fsType string, mountPoint string) (*Device, error) {
	device, err := RunCommand("rbd", "map", "--id", image.username, "--pool", image.pool, image.name)
	if err != nil {
		return nil, err
	}

	if fsType == "" {
		fsType = DefaultFileSystemType
	}

	new_device := &Device{device, false, fsType, mountPoint}

	if err = new_device.Format(); err != nil {
		return nil, err
	}

	if mountPoint != "" {
		if _, err = new_device.Mount(mountPoint); err != nil {
			return nil, err
		}

		new_device.isMounted = true
	}

	return new_device, nil
}

func toMegs(size uint64) uint64 {
	return size * 1024 * 1024
}

func RunCommand(name string, args ...string) (string, error) {
	cmd := exec.Command(name, args...)
	out, err := cmd.Output()
	return strings.Trim(string(out), " \n"), err
}

func (i *Image) MapToDevice(fsType string, mountPoint string) (*Device, error) {
	device, err := NewDevice(i, fsType, mountPoint)
	if err != nil {
		return nil, err
	}
	return device, err
}

func NewImage(image *rbd.Image, connection *Connection, name string) (*Image, error) {
	stat, err := image.Stat()
	if err != nil {
		return nil, err
	}

	err = image.Open(true)
	if err != nil {
		return nil, err
	}

	return &Image{
		image,
		stat,
		connection,
		name,
	}, nil
}

func (c *Connection) GetImageByName(name string) (*Image, error) {
	image := rbd.GetImage(c.context, name)
	if image == nil {
		return nil, fmt.Errorf("Image:%s not found on pool:%s", name, c.pool)
	}

	return NewImage(image, c, name)
}

func (c *Connection) GetOrCreateImage(name string, size uint64) (*Image, error) {
	image, err := c.GetImageByName(name)
	if image != nil {
		return image, nil
	}

	new_image, err := rbd.Create(c.context, name, toMegs(size))
	if err != nil {
		return nil, err
	}

	return NewImage(new_image, c, name)
}

func NewConnection(username string, pool string, cluster string, configFile string) (*Connection, error) {
	var conn *rados.Conn
	var err error

	if cluster != "" && username != "" {
		conn, err = rados.NewConnWithClusterAndUser(cluster, username)
	} else if username != "" {
		conn, err = rados.NewConnWithUser(username)
	} else {
		conn, err = rados.NewConn()
	}

	if err != nil {
		return nil, err
	}

	if configFile != "" {
		err = conn.ReadConfigFile(configFile)
	} else {
		err = conn.ReadDefaultConfigFile()
	}

	if err != nil {
		return nil, err
	}

	context, err := conn.OpenIOContext(pool)
	if err != nil {
		return nil, err
	}

	return &Connection{
		conn,
		context,
		pool,
		username,
		cluster,
	}, nil
}

func (c *Connection) Shutdown() {
	if c.context != nil {
		c.context.Destroy()
	}

	c.Shutdown()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() { }
<commit_msg>Add map example<commit_after>package main

import (
	"fmt"
)

func main() {
	m := make(map[string]int)

	m["k1"] = 7
	m["k2"] = 13

	fmt.Println("map:", m)

    fmt.Println("len", len(m))

    delete(m, "k2")

	fmt.Println("map:", m)

    _, prs := m["k2"]
    fmt.Println("prs:", prs)
    n := map[string]int{"foo": 1, "bar": 2}
    fmt.Println("map:", n)

}
<|endoftext|>


<commit_before>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xsd

import (
	"fmt"
	"io"
	"sort"
	"strings"

	"bitbucket.org/pkg/inflect"
)

func (s *Schema) Gen(w io.Writer) {
	c := newCollector()
	s.collect(c)
	sort.Sort(c.types)
	for _, typ := range c.types {
		typ.Gen(w)
	}
}

func (t *enumType) Gen(w io.Writer) {
	fmt.Fprintf(w, "type %s %s\n", t.Name, t.Type)
	fmt.Fprintln(w, "const (")
	for _, kv := range t.KV {
		fmt.Fprintf(w, "%s %s = %s\n", kv.Key, t.Name, kv.Value)
	}
	fmt.Fprintln(w, ")")
}

func (t pluralType) Gen(w io.Writer) {
	p(w, "type ", t.Name, " []", t.Type)
	p(w)
}

func cleanDoc(s string) string {
	ss := strings.Split(s, "\n")
	for i := range ss {
		ss[i] = strings.TrimSpace(ss[i])
	}
	return strings.Join(ss, " ")
}

func (t ComplexType) Gen(w io.Writer) {
	if doc := t.Annotation.Documentation; doc != "" {
		doc = cleanDoc(doc)
		if !strings.HasPrefix(doc, t.GoName()) {
			doc = t.GoName() + " is " + doc
		}
		p(w, "// "+doc)
	}
	p(w, "type ", t.GoName(), " struct {")
	if t.SimpleContent != nil {
		t.SimpleContent.Gen(w, t.GoName())
	}
	for _, attr := range t.Attributes {
		attr.Gen(w, t.GoName())
	}
	for _, seq := range t.Sequences {
		seq.Gen(w, false)
	}
	for _, choice := range t.Choices {
		choice.Gen(w, false)
	}
	p(w, "}")
	p(w, "")
}

func (s *SimpleContent) Gen(w io.Writer, namespace string) {
	typ := goType(s.Extension.Base)
	if typ == "AnyURI" {
		p(w, "Value []byte `xml:\",innerxml\"`")
	} else {
		p(w, "Value ", typ, " `xml:\",chardata\"`")
	}
	for _, attr := range s.Extension.Attributes {
		attr.Gen(w, namespace)
	}
}

func (t ComplexType) GoName() string {
	return goType(t.Name)
}

func (a Attribute) Gen(w io.Writer, namespace string) {
	omitempty := ""
	typ := a.GoType(namespace)
	if a.Use == "optional" {
		omitempty = ",omitempty"
		typ = omitType(typ)
	}

	if a.Annotation.Documentation != "" {
		p(w, "")
		doc := cleanDoc(a.Annotation.Documentation)
		p(w, "// "+doc)
	}
	p(w, a.GoName(), " ", typ, " `xml:\"", a.Name, ",attr"+omitempty+"\"`")
}

func (a Attribute) GoName() string {
	return snakeToCamel(a.Name)
}

func (a Attribute) GoType(namespace string) string {
	if a.Type != "" {
		return goType(a.Type)
	}
	if goType(a.SimpleType.Restriction.Base) == "NMTOKEN" {
		return namespace + a.GoName()
	}
	return goType(a.SimpleType.Restriction.Base)
}

func (s Sequence) Gen(w io.Writer, plural bool) {
	if s.MaxOccurs == "unbounded" {
		plural = true
	}
	for _, seq := range s.Sequences {
		seq.Gen(w, plural)
	}
	for _, choice := range s.Choices {
		choice.Gen(w, plural)
	}
	for _, elem := range s.Elements {
		elem.Gen(w, plural)
	}
}

func (c Choice) Gen(w io.Writer, plural bool) {
	for _, e := range c.Elements {
		e.Gen(w, plural)
	}
}

func (e Element) Gen(w io.Writer, plural bool) {
	omitempty := ""
	if e.MinOccurs == "0" {
		omitempty = ",omitempty"
	}
	if e.MaxOccurs == "unbounded" {
		plural = true
	}
	if e.GoType() == "" {
		e.Type = e.Name
		defer func() { e.Type = "" }()
	}
	if e.Annotation.Documentation != "" {
		p(w, "")
		p(w, "// "+e.Annotation.Documentation)
	}
	if plural {
		pluralName := inflect.Pluralize(e.GoName())
		pluralType := "[]" + e.GoType()
		p(w, pluralName, " ", pluralType, " `xml:\"", e.Name, omitempty+"\"`")
	} else {
		typ := e.GoType()
		if e.MinOccurs == "0" {
			typ = omitType(typ)
		}
		p(w, e.GoName(), " ", typ, " `xml:\"", e.Name, omitempty+"\"`")
	}
}

func (e Element) GoName() string {
	return snakeToCamel(e.Name)
}

func (e Element) GoType() string {
	return goType(e.Type)
}

func trimNamespace(s string) string {
	m := strings.Split(s, ":")
	if len(m) == 2 {
		return m[1]
	}
	return s
}

func snakeToCamel(s string) string {
	ss := strings.Split(s, "_")
	for i := range ss {
		ss[i] = strings.Title(ss[i])
	}
	return strings.Join(ss, "")
}

func omitType(s string) string {
	switch s {
	case "int", "string", "bool":
		return s
	}
	return "*" + s
}

func goType(s string) string {
	s = trimNamespace(s)
	switch s {
	case "integer":
		return "int"
	case "boolean":
		return "bool"
	case "string":
		return "string"
	case "decimal":
		return "float32"
	}
	s = strings.TrimSuffix(s, "Type")
	s = strings.TrimSuffix(s, "type")
	return snakeToCamel(s)
}

func p(w io.Writer, v ...interface{}) {
	fmt.Fprint(w, v...)
	fmt.Fprintln(w)
}
<commit_msg>CDATA should not be encouraged.<commit_after>// Copyright 2014, Hǎiliàng Wáng. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xsd

import (
	"fmt"
	"io"
	"sort"
	"strings"

	"bitbucket.org/pkg/inflect"
)

func (s *Schema) Gen(w io.Writer) {
	c := newCollector()
	s.collect(c)
	sort.Sort(c.types)
	for _, typ := range c.types {
		typ.Gen(w)
	}
}

func (t *enumType) Gen(w io.Writer) {
	fmt.Fprintf(w, "type %s %s\n", t.Name, t.Type)
	fmt.Fprintln(w, "const (")
	for _, kv := range t.KV {
		fmt.Fprintf(w, "%s %s = %s\n", kv.Key, t.Name, kv.Value)
	}
	fmt.Fprintln(w, ")")
}

func (t pluralType) Gen(w io.Writer) {
	p(w, "type ", t.Name, " []", t.Type)
	p(w)
}

func cleanDoc(s string) string {
	ss := strings.Split(s, "\n")
	for i := range ss {
		ss[i] = strings.TrimSpace(ss[i])
	}
	return strings.Join(ss, " ")
}

func (t ComplexType) Gen(w io.Writer) {
	if doc := t.Annotation.Documentation; doc != "" {
		doc = cleanDoc(doc)
		if !strings.HasPrefix(doc, t.GoName()) {
			doc = t.GoName() + " is " + doc
		}
		p(w, "// "+doc)
	}
	p(w, "type ", t.GoName(), " struct {")
	if t.SimpleContent != nil {
		t.SimpleContent.Gen(w, t.GoName())
	}
	for _, attr := range t.Attributes {
		attr.Gen(w, t.GoName())
	}
	for _, seq := range t.Sequences {
		seq.Gen(w, false)
	}
	for _, choice := range t.Choices {
		choice.Gen(w, false)
	}
	p(w, "}")
	p(w, "")
}

func (s *SimpleContent) Gen(w io.Writer, namespace string) {
	typ := goType(s.Extension.Base)
	p(w, "Value ", typ, " `xml:\",chardata\"`")
	for _, attr := range s.Extension.Attributes {
		attr.Gen(w, namespace)
	}
}

func (t ComplexType) GoName() string {
	return goType(t.Name)
}

func (a Attribute) Gen(w io.Writer, namespace string) {
	omitempty := ""
	typ := a.GoType(namespace)
	if a.Use == "optional" {
		omitempty = ",omitempty"
		typ = omitType(typ)
	}

	if a.Annotation.Documentation != "" {
		p(w, "")
		doc := cleanDoc(a.Annotation.Documentation)
		p(w, "// "+doc)
	}
	p(w, a.GoName(), " ", typ, " `xml:\"", a.Name, ",attr"+omitempty+"\"`")
}

func (a Attribute) GoName() string {
	return snakeToCamel(a.Name)
}

func (a Attribute) GoType(namespace string) string {
	if a.Type != "" {
		return goType(a.Type)
	}
	if goType(a.SimpleType.Restriction.Base) == "NMTOKEN" {
		return namespace + a.GoName()
	}
	return goType(a.SimpleType.Restriction.Base)
}

func (s Sequence) Gen(w io.Writer, plural bool) {
	if s.MaxOccurs == "unbounded" {
		plural = true
	}
	for _, seq := range s.Sequences {
		seq.Gen(w, plural)
	}
	for _, choice := range s.Choices {
		choice.Gen(w, plural)
	}
	for _, elem := range s.Elements {
		elem.Gen(w, plural)
	}
}

func (c Choice) Gen(w io.Writer, plural bool) {
	for _, e := range c.Elements {
		e.Gen(w, plural)
	}
}

func (e Element) Gen(w io.Writer, plural bool) {
	omitempty := ""
	if e.MinOccurs == "0" {
		omitempty = ",omitempty"
	}
	if e.MaxOccurs == "unbounded" {
		plural = true
	}
	if e.GoType() == "" {
		e.Type = e.Name
		defer func() { e.Type = "" }()
	}
	if e.Annotation.Documentation != "" {
		p(w, "")
		p(w, "// "+e.Annotation.Documentation)
	}
	if plural {
		pluralName := inflect.Pluralize(e.GoName())
		pluralType := "[]" + e.GoType()
		p(w, pluralName, " ", pluralType, " `xml:\"", e.Name, omitempty+"\"`")
	} else {
		typ := e.GoType()
		if e.MinOccurs == "0" {
			typ = omitType(typ)
		}
		p(w, e.GoName(), " ", typ, " `xml:\"", e.Name, omitempty+"\"`")
	}
}

func (e Element) GoName() string {
	return snakeToCamel(e.Name)
}

func (e Element) GoType() string {
	return goType(e.Type)
}

func trimNamespace(s string) string {
	m := strings.Split(s, ":")
	if len(m) == 2 {
		return m[1]
	}
	return s
}

func snakeToCamel(s string) string {
	ss := strings.Split(s, "_")
	for i := range ss {
		ss[i] = strings.Title(ss[i])
	}
	return strings.Join(ss, "")
}

func omitType(s string) string {
	switch s {
	case "int", "string", "bool":
		return s
	}
	return "*" + s
}

func goType(s string) string {
	s = trimNamespace(s)
	switch s {
	case "integer":
		return "int"
	case "boolean":
		return "bool"
	case "string":
		return "string"
	case "decimal":
		return "float32"
	}
	s = strings.TrimSuffix(s, "Type")
	s = strings.TrimSuffix(s, "type")
	return snakeToCamel(s)
}

func p(w io.Writer, v ...interface{}) {
	fmt.Fprint(w, v...)
	fmt.Fprintln(w)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/evoL/gif/config"
	"github.com/evoL/gif/store"
	"github.com/evoL/gif/version"
	"net/url"
	"os"
	"regexp"
	"strings"
)

func main() {
	typeFlags := []cli.Flag{
		cli.BoolFlag{
			Name:  "tag, t",
			Usage: "Enforces searching by tag.",
		},
		cli.BoolFlag{
			Name:  "untagged",
			Usage: "Lists only images that have no tag.",
		},
		cli.BoolFlag{
			Name:  "local",
			Usage: "Lists only images that are local, that is not avaliable remotely.",
		},
	}
	getFlags := append(
		typeFlags,
		cli.BoolFlag{
			Name:  "all, a",
			Usage: "Gets all matching images.",
		},
		cli.StringFlag{
			Name:  "order, sort, s",
			Usage: "Specifies the order of images. Must be one of: random, newest, oldest.",
			Value: "random",
		},
	)
	removeFlags := append(
		typeFlags,
		cli.BoolFlag{
			Name:  "all, a",
			Usage: "Removes all matching images.",
		},
		cli.BoolFlag{
			Name:  "really",
			Usage: "Doesn't ask for confirmation.",
		},
	)
	exportFlags := []cli.Flag{
		cli.StringFlag{
			Name:  "output, o",
			Usage: "Target output file. Set to '-' for stdout.",
			Value: "-",
		},
		cli.BoolFlag{
			Name:  "bundle",
			Usage: "Export a bundle containing all images and metadata.",
		},
	}
	importFlags := []cli.Flag{
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "When importing directories, do it recursively.",
		},
	}
	uploadFlags := append(
		typeFlags,
		cli.BoolFlag{
			Name:  "really",
			Usage: "Doesn't ask for confirmation.",
		},
	)

	app := cli.NewApp()
	app.Name = "gif"
	app.Usage = "a stupid gif manager"
	app.Author = "Rafał Hirsz"
	app.Email = "rafal@hirsz.co"
	app.Version = version.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "default",
			Usage: "Path to the configuration file",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "add",
			Usage:  "Adds an image",
			Action: AddCommand,
		},
		{
			Name:   "config",
			Usage:  "Prints the current configuration",
			Action: ConfigCommand,
		},
		{
			Name:   "export",
			Usage:  "Exports the database",
			Action: ExportCommand,
			Flags:  exportFlags,
		},
		{
			Name:   "import",
			Usage:  "Imports multiple images into the database",
			Action: ImportCommand,
			Flags:  importFlags,
		},
		{
			Name:   "list",
			Usage:  "Lists stored images",
			Action: ListCommand,
			Flags:  typeFlags,
		},
		{
			Name:   "path",
			Usage:  "Lists paths to images",
			Action: PathCommand,
			Flags:  getFlags,
		},
		{
			Name:    "remove",
			Aliases: []string{"rm"},
			Usage:   "Removes images",
			Action:  RemoveCommand,
			Flags:   removeFlags,
		},
		{
			Name:   "tag",
			Usage:  "Enables to change tags for images",
			Action: TagCommand,
			Flags:  typeFlags,
		},
		{
			Name:   "tags",
			Usage:  "Lists tags available in the database along with their image count",
			Action: TagsCommand,
		},
		{
			Name:   "url",
			Usage:  "Lists URLs of images",
			Action: UrlCommand,
			Flags:  getFlags,
		},
		{
			Name:   "upload",
			Usage:  "Uploads images to a server and saves the URLs for later use",
			Action: UploadCommand,
			Flags:  uploadFlags,
		},
		{
			Name:   "path",
			Usage:  "Lists paths to images",
			Action: PathCommand,
			Flags:  getFlags,
		},
	}
	app.Before = func(c *cli.Context) (err error) {
		err = loadConfig(c.String("config"))
		return
	}

	app.Run(os.Args)
}

func ConfigCommand(c *cli.Context) {
	config.Global.Print()
}

///////////////////////////////////////////////////////////////////////////////

type locationType int

const (
	invalidLocation locationType = iota
	fileLocation
	directoryLocation
	urlLocation
)

func loadConfig(arg string) (err error) {
	if arg == "default" {
		err = config.Default()
	} else {
		err = config.Load(arg)
	}

	if err != nil {
		fmt.Println("Error while loading the configuration file: " + err.Error())
	}
	return
}

func getStore() *store.Store {
	s, err := store.Default()
	if err != nil {
		fmt.Println("Cannot create store: " + err.Error())
		os.Exit(1)
	}
	return s
}

func typeFilter(c *cli.Context) (filter store.Filter) {
	if c.Args().Present() {
		arg := strings.Join(c.Args(), " ")

		if !c.Bool("tag") && regexp.MustCompile("^[0-9a-f]+$").MatchString(arg) {
			filter = store.IdOrTagFilter{Id: arg}
		} else {
			filter = store.TagFilter{Tag: arg}
		}
	} else if c.Bool("untagged") {
		filter = store.UntaggedFilter{}
	} else {
		filter = store.NullFilter{}
	}

	if c.Bool("local") {
		filter = store.LocalFilter{Filter: filter}
	}

	return
}

func listFilter(c *cli.Context) store.Filter {
	return store.DateOrderer{
		Filter:    typeFilter(c),
		Direction: store.Descending,
	}
}

func orderAndLimit(input store.Filter, c *cli.Context) (filter store.Filter) {
	switch c.String("order") {
	case "random":
		filter = store.RandomOrderer{Filter: input}
	case "newest":
		filter = store.DateOrderer{Filter: input, Direction: store.Descending}
	case "oldest":
		filter = store.DateOrderer{Filter: input, Direction: store.Ascending}
	default:
		fmt.Println("Invalid order.")
		os.Exit(1)
	}

	if !c.Bool("all") {
		filter = store.Limiter{Filter: filter, Limit: 1}
	}

	return
}

func parseLocation(location string) (locationType, error) {
	if location == "" {
		return invalidLocation, errors.New("No location specified")
	}

	// Check for URL
	u, err := url.Parse(location)
	if err == nil {
		if u.Scheme == "http" || u.Scheme == "https" {
			return urlLocation, nil
		} else if u.Scheme != "" {
			return urlLocation, errors.New("Only HTTP and HTTPS URLs are supported")
		}
	}

	// Check for path
	fileInfo, err := os.Stat(location)
	if err == nil {
		if fileInfo.IsDir() {
			return directoryLocation, nil
		}
		return fileLocation, nil
	}

	return invalidLocation, errors.New("Invalid location")
}
<commit_msg>Fixed a typo.<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/evoL/gif/config"
	"github.com/evoL/gif/store"
	"github.com/evoL/gif/version"
	"net/url"
	"os"
	"regexp"
	"strings"
)

func main() {
	typeFlags := []cli.Flag{
		cli.BoolFlag{
			Name:  "tag, t",
			Usage: "Enforces searching by tag.",
		},
		cli.BoolFlag{
			Name:  "untagged",
			Usage: "Lists only images that have no tag.",
		},
		cli.BoolFlag{
			Name:  "local",
			Usage: "Lists only images that are local, that is not available remotely.",
		},
	}
	getFlags := append(
		typeFlags,
		cli.BoolFlag{
			Name:  "all, a",
			Usage: "Gets all matching images.",
		},
		cli.StringFlag{
			Name:  "order, sort, s",
			Usage: "Specifies the order of images. Must be one of: random, newest, oldest.",
			Value: "random",
		},
	)
	removeFlags := append(
		typeFlags,
		cli.BoolFlag{
			Name:  "all, a",
			Usage: "Removes all matching images.",
		},
		cli.BoolFlag{
			Name:  "really",
			Usage: "Doesn't ask for confirmation.",
		},
	)
	exportFlags := []cli.Flag{
		cli.StringFlag{
			Name:  "output, o",
			Usage: "Target output file. Set to '-' for stdout.",
			Value: "-",
		},
		cli.BoolFlag{
			Name:  "bundle",
			Usage: "Export a bundle containing all images and metadata.",
		},
	}
	importFlags := []cli.Flag{
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "When importing directories, do it recursively.",
		},
	}
	uploadFlags := append(
		typeFlags,
		cli.BoolFlag{
			Name:  "really",
			Usage: "Doesn't ask for confirmation.",
		},
	)

	app := cli.NewApp()
	app.Name = "gif"
	app.Usage = "a stupid gif manager"
	app.Author = "Rafał Hirsz"
	app.Email = "rafal@hirsz.co"
	app.Version = version.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "default",
			Usage: "Path to the configuration file",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "add",
			Usage:  "Adds an image",
			Action: AddCommand,
		},
		{
			Name:   "config",
			Usage:  "Prints the current configuration",
			Action: ConfigCommand,
		},
		{
			Name:   "export",
			Usage:  "Exports the database",
			Action: ExportCommand,
			Flags:  exportFlags,
		},
		{
			Name:   "import",
			Usage:  "Imports multiple images into the database",
			Action: ImportCommand,
			Flags:  importFlags,
		},
		{
			Name:   "list",
			Usage:  "Lists stored images",
			Action: ListCommand,
			Flags:  typeFlags,
		},
		{
			Name:   "path",
			Usage:  "Lists paths to images",
			Action: PathCommand,
			Flags:  getFlags,
		},
		{
			Name:    "remove",
			Aliases: []string{"rm"},
			Usage:   "Removes images",
			Action:  RemoveCommand,
			Flags:   removeFlags,
		},
		{
			Name:   "tag",
			Usage:  "Enables to change tags for images",
			Action: TagCommand,
			Flags:  typeFlags,
		},
		{
			Name:   "tags",
			Usage:  "Lists tags available in the database along with their image count",
			Action: TagsCommand,
		},
		{
			Name:   "url",
			Usage:  "Lists URLs of images",
			Action: UrlCommand,
			Flags:  getFlags,
		},
		{
			Name:   "upload",
			Usage:  "Uploads images to a server and saves the URLs for later use",
			Action: UploadCommand,
			Flags:  uploadFlags,
		},
		{
			Name:   "path",
			Usage:  "Lists paths to images",
			Action: PathCommand,
			Flags:  getFlags,
		},
	}
	app.Before = func(c *cli.Context) (err error) {
		err = loadConfig(c.String("config"))
		return
	}

	app.Run(os.Args)
}

func ConfigCommand(c *cli.Context) {
	config.Global.Print()
}

///////////////////////////////////////////////////////////////////////////////

type locationType int

const (
	invalidLocation locationType = iota
	fileLocation
	directoryLocation
	urlLocation
)

func loadConfig(arg string) (err error) {
	if arg == "default" {
		err = config.Default()
	} else {
		err = config.Load(arg)
	}

	if err != nil {
		fmt.Println("Error while loading the configuration file: " + err.Error())
	}
	return
}

func getStore() *store.Store {
	s, err := store.Default()
	if err != nil {
		fmt.Println("Cannot create store: " + err.Error())
		os.Exit(1)
	}
	return s
}

func typeFilter(c *cli.Context) (filter store.Filter) {
	if c.Args().Present() {
		arg := strings.Join(c.Args(), " ")

		if !c.Bool("tag") && regexp.MustCompile("^[0-9a-f]+$").MatchString(arg) {
			filter = store.IdOrTagFilter{Id: arg}
		} else {
			filter = store.TagFilter{Tag: arg}
		}
	} else if c.Bool("untagged") {
		filter = store.UntaggedFilter{}
	} else {
		filter = store.NullFilter{}
	}

	if c.Bool("local") {
		filter = store.LocalFilter{Filter: filter}
	}

	return
}

func listFilter(c *cli.Context) store.Filter {
	return store.DateOrderer{
		Filter:    typeFilter(c),
		Direction: store.Descending,
	}
}

func orderAndLimit(input store.Filter, c *cli.Context) (filter store.Filter) {
	switch c.String("order") {
	case "random":
		filter = store.RandomOrderer{Filter: input}
	case "newest":
		filter = store.DateOrderer{Filter: input, Direction: store.Descending}
	case "oldest":
		filter = store.DateOrderer{Filter: input, Direction: store.Ascending}
	default:
		fmt.Println("Invalid order.")
		os.Exit(1)
	}

	if !c.Bool("all") {
		filter = store.Limiter{Filter: filter, Limit: 1}
	}

	return
}

func parseLocation(location string) (locationType, error) {
	if location == "" {
		return invalidLocation, errors.New("No location specified")
	}

	// Check for URL
	u, err := url.Parse(location)
	if err == nil {
		if u.Scheme == "http" || u.Scheme == "https" {
			return urlLocation, nil
		} else if u.Scheme != "" {
			return urlLocation, errors.New("Only HTTP and HTTPS URLs are supported")
		}
	}

	// Check for path
	fileInfo, err := os.Stat(location)
	if err == nil {
		if fileInfo.IsDir() {
			return directoryLocation, nil
		}
		return fileLocation, nil
	}

	return invalidLocation, errors.New("Invalid location")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/xorpaul/uiprogress"
)

func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
	defer timeTrack(time.Now(), funcName())
	if len(uniqueGitModules) <= 0 {
		Debugf("uniqueGitModules[] is empty, skipping...")
		return
	}
	bar := uiprogress.AddBar(len(uniqueGitModules)).AppendCompleted().PrependElapsed()
	bar.PrependFunc(func(b *uiprogress.Bar) string {
		return fmt.Sprintf("Resolving Git modules (%d/%d)", b.Current(), len(uniqueGitModules))
	})
	// Dummy channel to coordinate the number of concurrent goroutines.
	// This channel should be buffered otherwise we will be immediately blocked
	// when trying to fill it.

	Debugf("Resolving " + strconv.Itoa(len(uniqueGitModules)) + " Git modules with " + strconv.Itoa(config.Maxworker) + " workers")
	concurrentGoroutines := make(chan struct{}, config.Maxworker)
	// Fill the dummy channel with config.Maxworker empty struct.
	for i := 0; i < config.Maxworker; i++ {
		concurrentGoroutines <- struct{}{}
	}

	// The done channel indicates when a single goroutine has finished its job.
	done := make(chan bool)
	// The waitForAllJobs channel allows the main program
	// to wait until we have indeed done all the jobs.
	waitForAllJobs := make(chan bool)
	// Collect all the jobs, and since the job is finished, we can
	// release another spot for a goroutine.
	go func() {
		for _, gm := range uniqueGitModules {
			go func(gm GitModule) {
				<-done
				// Say that another goroutine can now start.
				concurrentGoroutines <- struct{}{}
			}(gm)
		}
		// We have collected all the jobs, the program can now terminate
		waitForAllJobs <- true
	}()
	wg := sync.WaitGroup{}
	wg.Add(len(uniqueGitModules))

	for url, gm := range uniqueGitModules {
		Debugf("git repo url " + url)
		privateKey := gm.privateKey
		go func(url string, gm GitModule, bar *uiprogress.Bar) {
			// Try to receive from the concurrentGoroutines channel. When we have something,
			// it means we can start a new goroutine because another one finished.
			// Otherwise, it will block the execution until an execution
			// spot is available.
			<-concurrentGoroutines
			defer bar.Incr()
			defer wg.Done()

			if len(gm.privateKey) > 0 {
				Debugf("git repo url " + url + " with ssh key " + privateKey)
			} else {
				Debugf("git repo url " + url + " without ssh key")
			}

			//log.Println(config)
			// create save directory name from Git repo name
			repoDir := strings.Replace(strings.Replace(url, "/", "_", -1), ":", "-", -1)
			workDir := filepath.Join(config.ModulesCacheDir, repoDir)

			success := doMirrorOrUpdate(gm, workDir, 0)
			if !success && config.UseCacheFallback == false {
				Fatalf("Fatal: Could not reach git repository " + url)
			}
			//	doCloneOrPull(source, workDir, targetDir, sa.Remote, branch, sa.PrivateKey)
			done <- true
		}(url, gm, bar)
	}

	// Wait for all jobs to finish
	<-waitForAllJobs
	wg.Wait()
}

func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool {
	needSSHKey := true
	if strings.Contains(gitModule.git, "github.com") || len(gitModule.privateKey) == 0 {
		needSSHKey = false
	}
	isControlRepo := strings.HasPrefix(workDir, config.EnvCacheDir)
	isInModulesCacheDir := strings.HasPrefix(workDir, config.ModulesCacheDir)

	er := ExecResult{}
	gitCmd := "git clone --mirror " + gitModule.git + " " + workDir
	if config.CloneGitModules && !isControlRepo && !isInModulesCacheDir {
		//fmt.Printf("%+v\n", gitModule)
		gitCmd = "git clone --single-branch --branch " + gitModule.tree + " " + gitModule.git + " " + workDir
	}
	if isDir(workDir) {
		gitCmd = "git --git-dir " + workDir + " remote update --prune"
	}

	if needSSHKey {
		er = executeCommand("ssh-agent bash -c 'ssh-add "+gitModule.privateKey+"; "+gitCmd+"'", config.Timeout, gitModule.ignoreUnreachable)
	} else {
		er = executeCommand(gitCmd, config.Timeout, gitModule.ignoreUnreachable)
	}

	if er.returnCode != 0 {
		if config.UseCacheFallback {
			Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment!")
			Warnf("WARN: Trying to use cache for " + gitModule.git + " git repository")
			return false
		} else if config.RetryGitCommands && retryCount > -1 {
			Warnf("WARN: git command failed: " + gitCmd + " deleting local cached repository and retrying...")
			purgeDir(workDir, "doMirrorOrUpdate, because git command failed, retrying")
			return doMirrorOrUpdate(gitModule, workDir, retryCount-1)
		}
		Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment!")
		return false
	}
	return true
}

func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, correspondingPuppetEnvironment string) bool {
	startedAt := time.Now()
	mutex.Lock()
	syncGitCount++
	mutex.Unlock()
	if !isDir(srcDir) {
		if config.UseCacheFallback {
			Fatalf("Could not find cached git module " + srcDir)
		}
	}
	logCmd := "git --git-dir " + srcDir + " rev-parse --verify '" + gitModule.tree
	if config.GitObjectSyntaxNotSupported != true {
		logCmd = logCmd + "^{object}'"
	} else {
		logCmd = logCmd + "'"
	}

	isControlRepo := strings.HasPrefix(srcDir, config.EnvCacheDir)

	er := executeCommand(logCmd, config.Timeout, gitModule.ignoreUnreachable)
	hashFile := filepath.Join(targetDir, ".latest_commit")
	deployFile := filepath.Join(targetDir, ".g10k-deploy.json")
	needToSync := true
	if er.returnCode != 0 {
		if gitModule.ignoreUnreachable {
			Debugf("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
			purgeDir(targetDir, "syncToModuleDir, because ignore-unreachable is set for this module")
		}
		return false
	}

	if len(er.output) > 0 {
		if strings.HasPrefix(srcDir, config.EnvCacheDir) {
			mutex.Lock()
			desiredContent = append(desiredContent, deployFile)
			mutex.Unlock()
			if fileExists(deployFile) {
				dr := readDeployResultFile(deployFile)
				if dr.Signature == strings.TrimSuffix(er.output, "\n") {
					needToSync = false
					// need to get the content of the git repository to detect and purge unmanaged files
					addDesiredContent(srcDir, gitModule.tree, targetDir)
				}
			}
		} else {
			Debugf("adding path to managed content: " + targetDir)
			mutex.Lock()
			desiredContent = append(desiredContent, hashFile)
			desiredContent = append(desiredContent, targetDir)
			mutex.Unlock()
			targetHashByte, _ := ioutil.ReadFile(hashFile)
			targetHash := string(targetHashByte)
			if targetHash == strings.TrimSuffix(er.output, "\n") {
				needToSync = false
				mutex.Lock()
				unchangedModuleDirs = append(unchangedModuleDirs, targetDir)
				mutex.Unlock()
				//Debugf("Skipping, because no diff found between " + srcDir + "(" + er.output + ") and " + targetDir + "(" + string(targetHash) + ")")
			}
		}

	}
	if needToSync && er.returnCode == 0 {
		Infof("Need to sync " + targetDir)
		mutex.Lock()
		needSyncDirs = append(needSyncDirs, targetDir)
		if _, ok := needSyncEnvs[correspondingPuppetEnvironment]; !ok {
			needSyncEnvs[correspondingPuppetEnvironment] = empty
		}
		needSyncGitCount++
		mutex.Unlock()

		if !dryRun && !config.CloneGitModules || isControlRepo {
			if pfMode {
				purgeDir(targetDir, "git dir with changes in -puppetfile mode")
			}
			checkDirAndCreate(targetDir, "git dir")
			gitArchiveArgs := []string{"--git-dir", srcDir, "archive", gitModule.tree}
			cmd := exec.Command("git", gitArchiveArgs...)
			Debugf("Executing git --git-dir " + srcDir + " archive " + gitModule.tree)
			cmdOut, err := cmd.StdoutPipe()
			if err != nil {
				if !gitModule.ignoreUnreachable {
					Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
				} else {
					return false
				}
				Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + gitModule.tree + " Error: " + err.Error())
			}
			cmd.Start()

			before := time.Now()
			unTar(cmdOut, targetDir)
			duration := time.Since(before).Seconds()
			mutex.Lock()
			ioGitTime += duration
			mutex.Unlock()

			err = cmd.Wait()
			if err != nil {
				Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + gitModule.tree + " Error: " + err.Error())
				//"\nIf you are using GitLab please ensure that you've added your deploy key to your repository." +
				//"\nThe Puppet environment which is using this unresolveable repository is " + correspondingPuppetEnvironment)
			}

			Verbosef("syncToModuleDir(): Executing git --git-dir " + srcDir + " archive " + gitModule.tree + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")

			er = executeCommand(logCmd, config.Timeout, false)
			if er.returnCode != 0 {
				Fatalf("executeCommand(): git command failed: " + logCmd + " " + err.Error() + "\nOutput: " + er.output)
			}
			if len(er.output) > 0 {
				commitHash := strings.TrimSuffix(er.output, "\n")
				if isControlRepo {
					Debugf("Writing to deploy file " + deployFile)
					dr := DeployResult{
						Name:      gitModule.tree,
						Signature: commitHash,
						StartedAt: startedAt,
					}
					writeStructJSONFile(deployFile, dr)
				} else {
					Debugf("Writing hash " + commitHash + " from command " + logCmd + " to " + hashFile)
					f, _ := os.Create(hashFile)
					defer f.Close()
					f.WriteString(commitHash)
					f.Sync()
				}

			}
		} else if config.CloneGitModules {
			return doMirrorOrUpdate(gitModule, targetDir, 0)
		}
	}
	return true
}

// addDesiredContent takes the given git repository directory and the
// relevant reference (branch, commit hash, tag) and adds its content to
// the global desiredContent slice so that it doesn't get purged by g10k
func addDesiredContent(gitDir string, tree string, targetDir string) {
	treeCmd := "git --git-dir " + gitDir + " ls-tree --full-tree -r -t --name-only " + tree
	er := executeCommand(treeCmd, config.Timeout, false)
	foundGitFiles := strings.Split(er.output, "\n")
	mutex.Lock()
	for _, desiredFile := range foundGitFiles[:len(foundGitFiles)-1] {
		desiredContent = append(desiredContent, filepath.Join(targetDir, desiredFile))

		// because we're using -r which prints git managed files in subfolders like this: foo/test3
		// we have to split up the given string and add the possible parent directories (foo in this case)
		parentDirs := strings.Split(desiredFile, "/")
		if len(parentDirs) > 1 {
			for _, dir := range parentDirs[:len(parentDirs)-1] {
				desiredContent = append(desiredContent, filepath.Join(targetDir, dir))
			}
		}
	}
	mutex.Unlock()

}
<commit_msg>allow ssh key even for github.com repositories if it is the control repo, fixes #165<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/xorpaul/uiprogress"
)

func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
	defer timeTrack(time.Now(), funcName())
	if len(uniqueGitModules) <= 0 {
		Debugf("uniqueGitModules[] is empty, skipping...")
		return
	}
	bar := uiprogress.AddBar(len(uniqueGitModules)).AppendCompleted().PrependElapsed()
	bar.PrependFunc(func(b *uiprogress.Bar) string {
		return fmt.Sprintf("Resolving Git modules (%d/%d)", b.Current(), len(uniqueGitModules))
	})
	// Dummy channel to coordinate the number of concurrent goroutines.
	// This channel should be buffered otherwise we will be immediately blocked
	// when trying to fill it.

	Debugf("Resolving " + strconv.Itoa(len(uniqueGitModules)) + " Git modules with " + strconv.Itoa(config.Maxworker) + " workers")
	concurrentGoroutines := make(chan struct{}, config.Maxworker)
	// Fill the dummy channel with config.Maxworker empty struct.
	for i := 0; i < config.Maxworker; i++ {
		concurrentGoroutines <- struct{}{}
	}

	// The done channel indicates when a single goroutine has finished its job.
	done := make(chan bool)
	// The waitForAllJobs channel allows the main program
	// to wait until we have indeed done all the jobs.
	waitForAllJobs := make(chan bool)
	// Collect all the jobs, and since the job is finished, we can
	// release another spot for a goroutine.
	go func() {
		for _, gm := range uniqueGitModules {
			go func(gm GitModule) {
				<-done
				// Say that another goroutine can now start.
				concurrentGoroutines <- struct{}{}
			}(gm)
		}
		// We have collected all the jobs, the program can now terminate
		waitForAllJobs <- true
	}()
	wg := sync.WaitGroup{}
	wg.Add(len(uniqueGitModules))

	for url, gm := range uniqueGitModules {
		Debugf("git repo url " + url)
		privateKey := gm.privateKey
		go func(url string, gm GitModule, bar *uiprogress.Bar) {
			// Try to receive from the concurrentGoroutines channel. When we have something,
			// it means we can start a new goroutine because another one finished.
			// Otherwise, it will block the execution until an execution
			// spot is available.
			<-concurrentGoroutines
			defer bar.Incr()
			defer wg.Done()

			if len(gm.privateKey) > 0 {
				Debugf("git repo url " + url + " with ssh key " + privateKey)
			} else {
				Debugf("git repo url " + url + " without ssh key")
			}

			//log.Println(config)
			// create save directory name from Git repo name
			repoDir := strings.Replace(strings.Replace(url, "/", "_", -1), ":", "-", -1)
			workDir := filepath.Join(config.ModulesCacheDir, repoDir)

			success := doMirrorOrUpdate(gm, workDir, 0)
			if !success && config.UseCacheFallback == false {
				Fatalf("Fatal: Could not reach git repository " + url)
			}
			//	doCloneOrPull(source, workDir, targetDir, sa.Remote, branch, sa.PrivateKey)
			done <- true
		}(url, gm, bar)
	}

	// Wait for all jobs to finish
	<-waitForAllJobs
	wg.Wait()
}

func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool {
	isControlRepo := strings.HasPrefix(workDir, config.EnvCacheDir)
	isInModulesCacheDir := strings.HasPrefix(workDir, config.ModulesCacheDir)

	needSSHKey := true
	if len(gitModule.privateKey) == 0 || strings.Contains(gitModule.git, "github.com") {
		if isControlRepo {
			needSSHKey = true
		} else {
			needSSHKey = false
		}
	}
	fmt.Println("needSSHKey is set to:", needSSHKey)
	er := ExecResult{}
	gitCmd := "git clone --mirror " + gitModule.git + " " + workDir
	if config.CloneGitModules && !isControlRepo && !isInModulesCacheDir {
		//fmt.Printf("%+v\n", gitModule)
		gitCmd = "git clone --single-branch --branch " + gitModule.tree + " " + gitModule.git + " " + workDir
	}
	if isDir(workDir) {
		gitCmd = "git --git-dir " + workDir + " remote update --prune"
	}

	if needSSHKey {
		er = executeCommand("ssh-agent bash -c 'ssh-add "+gitModule.privateKey+"; "+gitCmd+"'", config.Timeout, gitModule.ignoreUnreachable)
	} else {
		er = executeCommand(gitCmd, config.Timeout, gitModule.ignoreUnreachable)
	}

	if er.returnCode != 0 {
		if config.UseCacheFallback {
			Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment!")
			Warnf("WARN: Trying to use cache for " + gitModule.git + " git repository")
			return false
		} else if config.RetryGitCommands && retryCount > -1 {
			Warnf("WARN: git command failed: " + gitCmd + " deleting local cached repository and retrying...")
			purgeDir(workDir, "doMirrorOrUpdate, because git command failed, retrying")
			return doMirrorOrUpdate(gitModule, workDir, retryCount-1)
		}
		Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment!")
		return false
	}
	return true
}

func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, correspondingPuppetEnvironment string) bool {
	startedAt := time.Now()
	mutex.Lock()
	syncGitCount++
	mutex.Unlock()
	if !isDir(srcDir) {
		if config.UseCacheFallback {
			Fatalf("Could not find cached git module " + srcDir)
		}
	}
	logCmd := "git --git-dir " + srcDir + " rev-parse --verify '" + gitModule.tree
	if config.GitObjectSyntaxNotSupported != true {
		logCmd = logCmd + "^{object}'"
	} else {
		logCmd = logCmd + "'"
	}

	isControlRepo := strings.HasPrefix(srcDir, config.EnvCacheDir)

	er := executeCommand(logCmd, config.Timeout, gitModule.ignoreUnreachable)
	hashFile := filepath.Join(targetDir, ".latest_commit")
	deployFile := filepath.Join(targetDir, ".g10k-deploy.json")
	needToSync := true
	if er.returnCode != 0 {
		if gitModule.ignoreUnreachable {
			Debugf("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
			purgeDir(targetDir, "syncToModuleDir, because ignore-unreachable is set for this module")
		}
		return false
	}

	if len(er.output) > 0 {
		if strings.HasPrefix(srcDir, config.EnvCacheDir) {
			mutex.Lock()
			desiredContent = append(desiredContent, deployFile)
			mutex.Unlock()
			if fileExists(deployFile) {
				dr := readDeployResultFile(deployFile)
				if dr.Signature == strings.TrimSuffix(er.output, "\n") {
					needToSync = false
					// need to get the content of the git repository to detect and purge unmanaged files
					addDesiredContent(srcDir, gitModule.tree, targetDir)
				}
			}
		} else {
			Debugf("adding path to managed content: " + targetDir)
			mutex.Lock()
			desiredContent = append(desiredContent, hashFile)
			desiredContent = append(desiredContent, targetDir)
			mutex.Unlock()
			targetHashByte, _ := ioutil.ReadFile(hashFile)
			targetHash := string(targetHashByte)
			if targetHash == strings.TrimSuffix(er.output, "\n") {
				needToSync = false
				mutex.Lock()
				unchangedModuleDirs = append(unchangedModuleDirs, targetDir)
				mutex.Unlock()
				//Debugf("Skipping, because no diff found between " + srcDir + "(" + er.output + ") and " + targetDir + "(" + string(targetHash) + ")")
			}
		}

	}
	if needToSync && er.returnCode == 0 {
		Infof("Need to sync " + targetDir)
		mutex.Lock()
		needSyncDirs = append(needSyncDirs, targetDir)
		if _, ok := needSyncEnvs[correspondingPuppetEnvironment]; !ok {
			needSyncEnvs[correspondingPuppetEnvironment] = empty
		}
		needSyncGitCount++
		mutex.Unlock()

		if !dryRun && !config.CloneGitModules || isControlRepo {
			if pfMode {
				purgeDir(targetDir, "git dir with changes in -puppetfile mode")
			}
			checkDirAndCreate(targetDir, "git dir")
			gitArchiveArgs := []string{"--git-dir", srcDir, "archive", gitModule.tree}
			cmd := exec.Command("git", gitArchiveArgs...)
			Debugf("Executing git --git-dir " + srcDir + " archive " + gitModule.tree)
			cmdOut, err := cmd.StdoutPipe()
			if err != nil {
				if !gitModule.ignoreUnreachable {
					Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
				} else {
					return false
				}
				Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + gitModule.tree + " Error: " + err.Error())
			}
			cmd.Start()

			before := time.Now()
			unTar(cmdOut, targetDir)
			duration := time.Since(before).Seconds()
			mutex.Lock()
			ioGitTime += duration
			mutex.Unlock()

			err = cmd.Wait()
			if err != nil {
				Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + gitModule.tree + " Error: " + err.Error())
				//"\nIf you are using GitLab please ensure that you've added your deploy key to your repository." +
				//"\nThe Puppet environment which is using this unresolveable repository is " + correspondingPuppetEnvironment)
			}

			Verbosef("syncToModuleDir(): Executing git --git-dir " + srcDir + " archive " + gitModule.tree + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")

			er = executeCommand(logCmd, config.Timeout, false)
			if er.returnCode != 0 {
				Fatalf("executeCommand(): git command failed: " + logCmd + " " + err.Error() + "\nOutput: " + er.output)
			}
			if len(er.output) > 0 {
				commitHash := strings.TrimSuffix(er.output, "\n")
				if isControlRepo {
					Debugf("Writing to deploy file " + deployFile)
					dr := DeployResult{
						Name:      gitModule.tree,
						Signature: commitHash,
						StartedAt: startedAt,
					}
					writeStructJSONFile(deployFile, dr)
				} else {
					Debugf("Writing hash " + commitHash + " from command " + logCmd + " to " + hashFile)
					f, _ := os.Create(hashFile)
					defer f.Close()
					f.WriteString(commitHash)
					f.Sync()
				}

			}
		} else if config.CloneGitModules {
			return doMirrorOrUpdate(gitModule, targetDir, 0)
		}
	}
	return true
}

// addDesiredContent takes the given git repository directory and the
// relevant reference (branch, commit hash, tag) and adds its content to
// the global desiredContent slice so that it doesn't get purged by g10k
func addDesiredContent(gitDir string, tree string, targetDir string) {
	treeCmd := "git --git-dir " + gitDir + " ls-tree --full-tree -r -t --name-only " + tree
	er := executeCommand(treeCmd, config.Timeout, false)
	foundGitFiles := strings.Split(er.output, "\n")
	mutex.Lock()
	for _, desiredFile := range foundGitFiles[:len(foundGitFiles)-1] {
		desiredContent = append(desiredContent, filepath.Join(targetDir, desiredFile))

		// because we're using -r which prints git managed files in subfolders like this: foo/test3
		// we have to split up the given string and add the possible parent directories (foo in this case)
		parentDirs := strings.Split(desiredFile, "/")
		if len(parentDirs) > 1 {
			for _, dir := range parentDirs[:len(parentDirs)-1] {
				desiredContent = append(desiredContent, filepath.Join(targetDir, dir))
			}
		}
	}
	mutex.Unlock()

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"

	. "github.com/reujab/bronze/types"
	"gopkg.in/libgit2/git2go.v26"
)

// reformat scp-like url (e. g. ssh)
// https://github.com/motemen/ghq/blob/master/url.go
func fixUrl(url string) string {
	hasSchemePattern := regexp.MustCompile("^[^:]+://")
	scpLikeUrlPattern := regexp.MustCompile("^([^@]+@)?([^:]+):/?(.+)$")

	if !hasSchemePattern.MatchString(url) && scpLikeUrlPattern.MatchString(url) {
		matched := scpLikeUrlPattern.FindStringSubmatch(url)
		user := matched[1]
		host := matched[2]
		path := matched[3]
		return fmt.Sprintf("ssh://%s%s/%s", user, host, path)
	}

	return url
}

// the git segment provides useful information about a git repository such as the domain of the "origin" remote (with an icon), the current branch, and whether the HEAD is dirty
func gitSegment(segment *Segment) {
	dir, err := os.Getwd()
	die(err)
	repo, err := git.OpenRepositoryExtended(dir, 0, "/")
	if err != nil {
		return
	}
	defer repo.Free()

	var domainName string
	remote, err := repo.Remotes.Lookup("origin")
	if err == nil {
		remoteUrl := fixUrl(remote.Url())
		uri, err := url.Parse(remoteUrl)
		if err == nil && len(uri.Hostname()) > 4 {
			// strip the tld off the hostname
			domainName = uri.Hostname()[:len(uri.Hostname())-4]
		}
		remote.Free()
	}

	var stashes int
	repo.Stashes.Foreach(func(int, string, *git.Oid) error {
		stashes++
		return nil
	})

	var ahead, behind int
	var branch string
	head, err := repo.Head()
	if err == nil {
		upstream, err := head.Branch().Upstream()
		if err == nil {
			ahead, behind, err = repo.AheadBehind(head.Branch().Target(), upstream.Target())
			die(err)
		}

		branch, err = head.Branch().Name()
		if err != nil {
			// head is detached
			branch = head.Branch().Target().String()[:7]
		}
		head.Free()
	}

	var dirty, modified, staged bool
	status, err := repo.StatusList(&git.StatusOptions{
		Flags: git.StatusOptIncludeUntracked,
	})
	if err != nil {
		// bare repository
		return
	}
	count, err := status.EntryCount()
	die(err)
	if count != 0 {
		dirty = true
	}
	for i := 0; i < count; i++ {
		entry, err := status.ByIndex(i)
		die(err)
		if entry.Status&git.StatusWtNew != 0 || entry.Status&git.StatusWtModified != 0 || entry.Status&git.StatusWtDeleted != 0 || entry.Status&git.StatusWtTypeChange != 0 || entry.Status&git.StatusWtRenamed != 0 {
			modified = true
		}
		if entry.Status&git.StatusIndexNew != 0 || entry.Status&git.StatusIndexModified != 0 || entry.Status&git.StatusIndexDeleted != 0 || entry.Status&git.StatusIndexRenamed != 0 || entry.Status&git.StatusIndexTypeChange != 0 {
			staged = true
		}
	}
	status.Free()

	var segments []string
	domainIcon := icons[domainName]
	if domainIcon == "" {
		domainIcon = icons["git"]
	}
	if domainIcon != "" {
		segments = append(segments, domainIcon)
	}
	if stashes != 0 || ahead != 0 || behind != 0 {
		section := strings.Repeat(icons["stash"], stashes) + strings.Repeat(icons["ahead"], ahead) + strings.Repeat(icons["behind"], behind)
		if section != "" {
			segments = append(segments, section)
		}
	}
	if branch != "" {
		segments = append(segments, branch)
	}
	if dirty {
		segment.Background = "yellow"

		var section string
		if modified {
			section += icons["modified"]
		}
		if staged {
			section += icons["staged"]
		}
		if section != "" {
			segments = append(segments, section)
		}
	}
	segment.Value = strings.Join(segments, " ")
}
<commit_msg>upgraded to git2go.v27<commit_after>package main

import (
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"

	. "github.com/reujab/bronze/types"
	"gopkg.in/libgit2/git2go.v27"
)

// reformat scp-like url (e. g. ssh)
// https://github.com/motemen/ghq/blob/master/url.go
func fixUrl(url string) string {
	hasSchemePattern := regexp.MustCompile("^[^:]+://")
	scpLikeUrlPattern := regexp.MustCompile("^([^@]+@)?([^:]+):/?(.+)$")

	if !hasSchemePattern.MatchString(url) && scpLikeUrlPattern.MatchString(url) {
		matched := scpLikeUrlPattern.FindStringSubmatch(url)
		user := matched[1]
		host := matched[2]
		path := matched[3]
		return fmt.Sprintf("ssh://%s%s/%s", user, host, path)
	}

	return url
}

// the git segment provides useful information about a git repository such as the domain of the "origin" remote (with an icon), the current branch, and whether the HEAD is dirty
func gitSegment(segment *Segment) {
	dir, err := os.Getwd()
	die(err)
	repo, err := git.OpenRepositoryExtended(dir, 0, "/")
	if err != nil {
		return
	}
	defer repo.Free()

	var domainName string
	remote, err := repo.Remotes.Lookup("origin")
	if err == nil {
		remoteUrl := fixUrl(remote.Url())
		uri, err := url.Parse(remoteUrl)
		if err == nil && len(uri.Hostname()) > 4 {
			// strip the tld off the hostname
			domainName = uri.Hostname()[:len(uri.Hostname())-4]
		}
		remote.Free()
	}

	var stashes int
	repo.Stashes.Foreach(func(int, string, *git.Oid) error {
		stashes++
		return nil
	})

	var ahead, behind int
	var branch string
	head, err := repo.Head()
	if err == nil {
		upstream, err := head.Branch().Upstream()
		if err == nil {
			ahead, behind, err = repo.AheadBehind(head.Branch().Target(), upstream.Target())
			die(err)
		}

		branch, err = head.Branch().Name()
		if err != nil {
			// head is detached
			branch = head.Branch().Target().String()[:7]
		}
		head.Free()
	}

	var dirty, modified, staged bool
	status, err := repo.StatusList(&git.StatusOptions{
		Flags: git.StatusOptIncludeUntracked,
	})
	if err != nil {
		// bare repository
		return
	}
	count, err := status.EntryCount()
	die(err)
	if count != 0 {
		dirty = true
	}
	for i := 0; i < count; i++ {
		entry, err := status.ByIndex(i)
		die(err)
		if entry.Status&git.StatusWtNew != 0 || entry.Status&git.StatusWtModified != 0 || entry.Status&git.StatusWtDeleted != 0 || entry.Status&git.StatusWtTypeChange != 0 || entry.Status&git.StatusWtRenamed != 0 {
			modified = true
		}
		if entry.Status&git.StatusIndexNew != 0 || entry.Status&git.StatusIndexModified != 0 || entry.Status&git.StatusIndexDeleted != 0 || entry.Status&git.StatusIndexRenamed != 0 || entry.Status&git.StatusIndexTypeChange != 0 {
			staged = true
		}
	}
	status.Free()

	var segments []string
	domainIcon := icons[domainName]
	if domainIcon == "" {
		domainIcon = icons["git"]
	}
	if domainIcon != "" {
		segments = append(segments, domainIcon)
	}
	if stashes != 0 || ahead != 0 || behind != 0 {
		section := strings.Repeat(icons["stash"], stashes) + strings.Repeat(icons["ahead"], ahead) + strings.Repeat(icons["behind"], behind)
		if section != "" {
			segments = append(segments, section)
		}
	}
	if branch != "" {
		segments = append(segments, branch)
	}
	if dirty {
		segment.Background = "yellow"

		var section string
		if modified {
			section += icons["modified"]
		}
		if staged {
			section += icons["staged"]
		}
		if section != "" {
			segments = append(segments, section)
		}
	}
	segment.Value = strings.Join(segments, " ")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/FreekKalter/text/columnswriter"
	"github.com/str1ngs/ansi/color"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
)

var wg sync.WaitGroup

type colorFunc func(interface{}) *color.Escape

var colorMap map[string]colorFunc = map[string]colorFunc{
	"no_version_control": func(i interface{}) *color.Escape { return color.Bold(color.Blue(i)) },                  // Blue
	"dirty":              func(i interface{}) *color.Escape { return color.Bold(color.Red(i)) },                   // Red
	"no_remote":          func(i interface{}) *color.Escape { return color.BgBlue(color.Bold(color.Red(i))) },     // Red on Blue
	"fetch_failed":       func(i interface{}) *color.Escape { return color.BgRed(color.Bold(color.Blue(i))) },     // Red on Blue
	"branch_ahead":       func(i interface{}) *color.Escape { return color.BgYellow(color.Bold(color.Green(i))) }, // Green on Yellow
	"branch_behind":      func(i interface{}) *color.Escape { return color.BgYellow(color.Bold(color.Red(i))) },   // Red on Yellow
}

// Struct returned by gls go-routines
type Project struct {
	Name, State string
}

type Projects []*Project

func (projects Projects) Len() int      { return len(projects) }
func (projects Projects) Swap(i, j int) { projects[i], projects[j] = projects[j], projects[i] }

type ByName struct{ Projects }

func (s ByName) Less(i, j int) bool {
	return strings.ToLower(s.Projects[i].Name) < strings.ToLower(s.Projects[j].Name)
}

var (
	cleanGitRegex = regexp.MustCompile("nothing to commit")
	fetchErrors   = regexp.MustCompile("^fatal")
	branchAhead   = regexp.MustCompile("branch is ahead of")
	branchBehind  = regexp.MustCompile("branch is behind")
)

func main() {
	var help bool
	flag.BoolVar(&help, "help", false, "print help message")
	flag.Parse()
	if help {
		for k, v := range colorMap {
			fmt.Println(v(k))
		}
		return
	}
	files, err := filepath.Glob("*")
	if err != nil {
		panic(err)
	}
	glsResults := make(chan Project, 1000)

	var projects Projects
	for _, file := range files {
		file_info, _ := os.Stat(file)
		if file_info.IsDir() {
			wg.Add(1)
			go gls(file, glsResults)
		} else {
			projects = append(projects, &Project{Name: file, State: "ok"})
		}
	}
	wg.Wait()
	close(glsResults)

	for res := range glsResults {
		// make a copy to add to []projects, because res always points to the same address space
		toAppend := res
		toAppend.Name = filepath.Base(res.Name)
		projects = append(projects, &toAppend)
	}
	sort.Sort(ByName{projects})

	var projectsString string
	for _, p := range projects {
		if p.State == "ok" {
			projectsString = fmt.Sprintf("%s\t%s", projectsString, p.Name)
		} else {
			projectsString = fmt.Sprintf("%s\t%s", projectsString, colorMap[p.State](p.Name))
		}
	}

	w := columnswriter.New(os.Stdout, '\t', 0, 2)
	fmt.Fprint(w, projectsString)
	w.Flush()
}

func gls(dirName string, result chan Project) {
	defer wg.Done()
	var ret Project = Project{Name: dirName}

	// First chek, is the directory under (git) version control
	if ok, _ := exists(filepath.Join(dirName, ".git")); !ok {
		ret.State = "no_version_control"
		result <- ret
		return
	}

	gitDir := fmt.Sprintf("--git-dir=%s", filepath.Join(dirName, ".git"))
	gitTree := fmt.Sprintf("--work-tree=%s", dirName)
	output, err := exec.Command("git", gitDir, gitTree, "status").Output() //, gitDir, gitTree, "status")
	if err != nil {
		panic(err)
	}
	// Are there uncommitted changes is the directory (dirty)
	if !cleanGitRegex.MatchString(strings.TrimSpace(string(output))) {
		ret.State = "dirty"
		result <- ret
		return
	}

	// Check if the repo has a remote
	output, err = exec.Command("git", gitDir, gitTree, "remote", "-v").Output()
	if err != nil {
		panic(err)
	}
	if len(output) == 0 {
		ret.State = "no_remote"
		result <- ret
		return
	}

	// Fetch latest changes from remote
	output, err = exec.Command("git", gitDir, gitTree, "fetch").Output()
	if err != nil {
		ret.State = "fetch_failed"
		result <- ret
		return
	}
	outputStr := strings.TrimSpace(string(output))
	if fetchErrors.MatchString(outputStr) {
		ret.State = "fetch_failed"
		result <- ret
		return
	}

	output, err = exec.Command("git", gitDir, gitTree, "status").Output()
	if err != nil {
		panic(err)
	}
	outputStr = strings.TrimSpace(string(output))

	// Is branch ahead of behind of remote
	if branchAhead.MatchString(outputStr) {
		ret.State = "branch_ahead"
		result <- ret
		return
	} else if branchBehind.MatchString(outputStr) {
		ret.State = "branch_behind"
		result <- ret
		return
	}

	ret.State = "ok"
	result <- ret
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
<commit_msg>added list option<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/FreekKalter/text/columnswriter"
	"github.com/str1ngs/ansi/color"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
)

var wg sync.WaitGroup

type colorFunc func(interface{}) *color.Escape

var colorMap map[string]colorFunc = map[string]colorFunc{
	"no_version_control": func(i interface{}) *color.Escape { return color.Bold(color.Blue(i)) },                  // Blue
	"dirty":              func(i interface{}) *color.Escape { return color.Bold(color.Red(i)) },                   // Red
	"no_remote":          func(i interface{}) *color.Escape { return color.BgBlue(color.Bold(color.Red(i))) },     // Red on Blue
	"fetch_failed":       func(i interface{}) *color.Escape { return color.BgRed(color.Bold(color.Blue(i))) },     // Blue on Red
	"branch_ahead":       func(i interface{}) *color.Escape { return color.BgYellow(color.Bold(color.Green(i))) }, // Green on Yellow
	"branch_behind":      func(i interface{}) *color.Escape { return color.BgYellow(color.Bold(color.Red(i))) },   // Red on Yellow
}

// Struct returned by gls go-routines
type Project struct {
	Name, State string
}

type Projects []*Project

func (projects Projects) Len() int      { return len(projects) }
func (projects Projects) Swap(i, j int) { projects[i], projects[j] = projects[j], projects[i] }

type ByName struct{ Projects }

func (s ByName) Less(i, j int) bool {
	return strings.ToLower(s.Projects[i].Name) < strings.ToLower(s.Projects[j].Name)
}

var (
	cleanGitRegex = regexp.MustCompile("nothing to commit")
	fetchErrors   = regexp.MustCompile("^fatal")
	branchAhead   = regexp.MustCompile("branch is ahead of")
	branchBehind  = regexp.MustCompile("branch is behind")
)

func main() {
	var help, list bool
	//TODO: Add sort by name or state
	flag.BoolVar(&help, "help", false, "print help message")
	flag.BoolVar(&list, "l", false, "display results in 1 long list")
	flag.Parse()
	if help {
		for k, v := range colorMap {
			fmt.Println(v(k))
		}
		return
	}
	files, err := filepath.Glob("*")
	if err != nil {
		panic(err)
	}
	glsResults := make(chan Project, 1000)

	var projects Projects
	for _, file := range files {
		file_info, _ := os.Stat(file)
		if file_info.IsDir() {
			wg.Add(1)
			go gls(file, glsResults)
		} else {
			projects = append(projects, &Project{Name: file, State: "ok"})
		}
	}
	wg.Wait()
	close(glsResults)

	for res := range glsResults {
		// make a copy to add to []projects, because res always points to the same address space
		toAppend := res
		toAppend.Name = filepath.Base(res.Name)
		projects = append(projects, &toAppend)
	}
	sort.Sort(ByName{projects})

	if list {
		for _, p := range projects {
			if p.State == "ok" {
				fmt.Printf("%s\n", p.Name)
			} else {
				fmt.Printf("%s\n", colorMap[p.State](p.Name))
			}
		}
	} else {

		var projectsString string
		for _, p := range projects {
			if p.State == "ok" {
				projectsString = fmt.Sprintf("%s\t%s", projectsString, p.Name)
			} else {
				projectsString = fmt.Sprintf("%s\t%s", projectsString, colorMap[p.State](p.Name))
			}
		}

		w := columnswriter.New(os.Stdout, '\t', 0, 2)
		fmt.Fprint(w, projectsString)
		w.Flush()
	}
}

func gls(dirName string, result chan Project) {
	defer wg.Done()
	var ret Project = Project{Name: dirName}

	// First chek, is the directory under (git) version control
	if ok, _ := exists(filepath.Join(dirName, ".git")); !ok {
		ret.State = "no_version_control"
		result <- ret
		return
	}

	gitDir := fmt.Sprintf("--git-dir=%s", filepath.Join(dirName, ".git"))
	gitTree := fmt.Sprintf("--work-tree=%s", dirName)
	output, err := exec.Command("git", gitDir, gitTree, "status").Output() //, gitDir, gitTree, "status")
	if err != nil {
		panic(err)
	}
	// Are there uncommitted changes is the directory (dirty)
	if !cleanGitRegex.MatchString(strings.TrimSpace(string(output))) {
		ret.State = "dirty"
		result <- ret
		return
	}

	// Check if the repo has a remote
	output, err = exec.Command("git", gitDir, gitTree, "remote", "-v").Output()
	if err != nil {
		panic(err)
	}
	if len(output) == 0 {
		ret.State = "no_remote"
		result <- ret
		return
	}

	// Fetch latest changes from remote
	output, err = exec.Command("git", gitDir, gitTree, "fetch").Output()
	if err != nil {
		ret.State = "fetch_failed"
		result <- ret
		return
	}
	outputStr := strings.TrimSpace(string(output))
	if fetchErrors.MatchString(outputStr) {
		ret.State = "fetch_failed"
		result <- ret
		return
	}

	output, err = exec.Command("git", gitDir, gitTree, "status").Output()
	if err != nil {
		panic(err)
	}
	outputStr = strings.TrimSpace(string(output))

	// Is branch ahead of behind of remote
	if branchAhead.MatchString(outputStr) {
		ret.State = "branch_ahead"
		result <- ret
		return
	} else if branchBehind.MatchString(outputStr) {
		ret.State = "branch_behind"
		result <- ret
		return
	}

	ret.State = "ok"
	result <- ret
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
<|endoftext|>