<commit_before>package robo

import (
	"errors"
)

var (
	errEmptyPattern   = errors.New("robo: empty pattern")
	errEmptyParameter = errors.New("robo: empty parameter name")
	errEmptyCharset   = errors.New("robo: empty charset")

	errCharsetHasSlash = errors.New("robo: parameter charset includes '/'")
	errIllegalWildcard = errors.New("robo: illegal '*' position")
	errImpossibleRange = errors.New("robo: impossible charset range")

	errUnexpectedHyphen   = errors.New("robo: unexpected '-'")
	errUnexpectedLBracket = errors.New("robo: unexpected '['")
	errUnexpectedRBracket = errors.New("robo: unexpected ']'")
	errMissingRBrace      = errors.New("robo: missing closing '}'")
	errMissingRBracket    = errors.New("robo: missing closing ']'")
)

// The pathMatcher interface is used to match the paths of incoming requests.
// Any captured parameters will be appended to buf and returned as the second
// return value.
type pathMatcher interface {
	match(path string, buf []string) (bool, []string)
}

// compileMatcher compiles a pathMatcher from a pattern string.
func compileMatcher(pattern string) (pathMatcher, error) {
	var fs []fragment

	if pattern == "" {
		return nil, errEmptyPattern
	}

	for pattern != "" {
		f, n, err := compileFragment(pattern)
		if err != nil {
			return nil, err
		}

		fs = append(fs, f)
		pattern = pattern[n:]
	}

	return &fragmentMatcher{fs}, nil
}

// compileFragment compiles a fragment matcher from a prefix of a pattern
// string. It returns the compiled fragment, and how many bytes of the input
// string were consumed.
func compileFragment(pattern string) (fragment, int, error) {
	switch pattern[0] {
	default:
		return compileLiteralFragment(pattern)
	case '*':
		return compileWildcardFragment(pattern)
	case '{':
		return compileParameterFragment(pattern)
	}
}

func compileLiteralFragment(pattern string) (fragment, int, error) {
	var i int
	var e bool

	for i = 0; i < len(pattern); i++ {
		switch c := pattern[i]; {
		case e:
			e = false
		case c == '\\':
			e = true
		case c == '*', c == '{':
			return fragment{t: literalFragment, s: pattern[:i], n: i}, i, nil
		}
	}

	return fragment{t: literalFragment, s: pattern[:i], n: i}, i, nil
}

func compileWildcardFragment(pattern string) (fragment, int, error) {
	if pattern != "*" {
		return fragment{}, 0, errIllegalWildcard
	}
	return fragment{t: wildcardFragment}, 1, nil
}

func compileParameterFragment(pattern string) (fragment, int, error) {
	var i int
	var e bool
	var f fragment

	for i = 1; i < len(pattern); i++ {
		switch c := pattern[i]; {
		case e:
			e = false

		case c == '\\':
			e = true

		case c == '[':
			chars, n, err := compileCharsetFragment(pattern[i:])
			if err != nil {
				return f, 0, err
			}

			// before returning, make sure the charset block is
			// followed by a '}'
			if i := i + n; i == len(pattern) || pattern[i] != '}' {
				return fragment{}, 0, errMissingRBrace
			}

			f = fragment{t: inclusiveFragment, s: pattern[1:i], r: chars}
			return f, i + n + 1, nil

		case c == '}':
			if i == 1 {
				return fragment{}, 0, errEmptyParameter
			}

			f = fragment{t: exclusiveFragment, s: pattern[1:i], r: []rune{'/'}}

			// if available, add the next rune to exclusive parameters (for
			// example: '-' for {foo} in "/{foo}-bar")
			for _, r := range pattern[i+1:] {
				if r != '/' {
					f.r = append(f.r, r)
				}
				break
			}

			return f, i + 1, nil
		}
	}

	return f, 0, errMissingRBrace
}

func compileCharsetFragment(pattern string) ([]rune, int, error) {
	var o []rune
	var e bool

	// begin with some rudimentary error checking
	if len(pattern) < 2 {
		return nil, 0, errMissingRBracket
	} else if pattern[1] == ']' {
		return nil, 0, errEmptyCharset
	}

loop:
	for i, r := range pattern[1:] {
		if !e {
			switch r {
			case '\\':
				e = true
				continue loop

			case '[':
				return nil, 0, errUnexpectedLBracket

			case ']':
				if len(o)&1 != 0 {
					return nil, 0, errUnexpectedRBracket
				}

				// make sure the charset doesn't include forward slash
				for i := 0; i < len(o); i += 2 {
					if o[i] <= '/' && '/' <= o[i+1] {
						return nil, 0, errCharsetHasSlash
					}
				}

				return simplifyCharset(o), i + 2, nil

			case '-':
				// catch ambiguous range statements like "a-c-e"
				if len(o) == 0 || o[len(o)-1] != o[len(o)-2] {
					return nil, 0, errUnexpectedHyphen
				}

				o = o[:len(o)-1]
				continue loop
			}
		}

		// determine whether or not this character is the upper bound
		// of a charset range
		if len(o)&1 != 0 {
			if r < o[len(o)-1] {
				return nil, 0, errImpossibleRange
			}
			o = append(o, r)
		} else {
			o = append(o, r, r)
		}

		e = false
	}

	return nil, 0, errMissingRBracket
}

// simplifyCharset merges overlapping rune ranges in the input charset.
func simplifyCharset(a []rune) []rune {
	if len(a) == 0 {
		return a
	}

	// first, sort the pairs in the charset
	for i := 2; i < len(a); i += 2 {
		for j := i; j > 0; j -= 2 {
			if a[j] > a[j-2] || (a[j] == a[j-2] && a[j] > a[j-2]) {
				break
			}
			a[j-2], a[j+0] = a[j+0], a[j-2]
			a[j-1], a[j+1] = a[j+1], a[j-1]
		}
	}

	// then merge overlapping pairs
	var r, w = 2, 2

	for ; r < len(a); r += 2 {
		if a[r] <= a[w-2] {
			if a[w-2] <= a[r+1]+1 {
				goto merge
			}
		} else if a[r]-1 <= a[w-1] {
			goto merge
		}

		a[w] = a[r]
		a[w+1] = a[r+1]
		w += 2
		continue

	merge:
		if a[r] < a[w-2] {
			a[w-2] = a[r]
		}
		if a[r+1] > a[w-1] {
			a[w-1] = a[r+1]
		}
	}

	return a[:w]
}

// fragmentMatcher is an implementation of the pathMatcher interface,
// which matches input strings using precompiled fragments.
type fragmentMatcher struct {
	fs []fragment
}

func (f *fragmentMatcher) match(path string, buf []string) (bool, []string) {
	var n int

	for _, f := range f.fs {
		n, buf = f.matchPrefix(path, buf)
		if n < 0 {
			return false, nil
		}
		path = path[n:]
	}

	// make sure the whole string has been consumed
	if path != "" {
		return false, nil
	}

	return true, buf
}

type fragment struct {
	t int
	s string
	n int
	r []rune
}

const (
	literalFragment = iota
	exclusiveFragment
	inclusiveFragment
	wildcardFragment
)

func (f *fragment) matchPrefix(pattern string, buf []string) (int, []string) {
	switch f.t {
	case literalFragment:
		if len(pattern) < f.n || pattern[:f.n] != f.s {
			return -1, nil
		}
		return f.n, buf

	case exclusiveFragment:
		for i, r := range pattern {
			for _, excl := range f.r {
				if r == excl {
					return nonZero(i), append(buf, f.s, pattern[:i])
				}
			}
		}
		return nonZero(len(pattern)), append(buf, f.s, pattern)

	case inclusiveFragment:
		for i, r := range pattern {
			for j := 0; j < len(f.r); j += 2 {
				if f.r[j] <= r && r <= f.r[j+1] {
					goto ok
				}
			}
			return nonZero(i), append(buf, f.s, pattern[:i])
		ok:
		}
		return nonZero(len(pattern)), append(buf, f.s, pattern)

	case wildcardFragment:
		return len(pattern), append(buf, "*", pattern)
	}

	panic("unreachable")
}

// nonZero return -1 instead of n if n == 0.
func nonZero(n int) int {
	if n == 0 {
		return -1
	}
	return n
}
<commit_msg>Implement two specialized path matchers<commit_after>package robo

import (
	"errors"
)

var (
	errEmptyPattern   = errors.New("robo: empty pattern")
	errEmptyParameter = errors.New("robo: empty parameter name")
	errEmptyCharset   = errors.New("robo: empty charset")

	errCharsetHasSlash = errors.New("robo: parameter charset includes '/'")
	errIllegalWildcard = errors.New("robo: illegal '*' position")
	errImpossibleRange = errors.New("robo: impossible charset range")

	errUnexpectedHyphen   = errors.New("robo: unexpected '-'")
	errUnexpectedLBracket = errors.New("robo: unexpected '['")
	errUnexpectedRBracket = errors.New("robo: unexpected ']'")
	errMissingRBrace      = errors.New("robo: missing closing '}'")
	errMissingRBracket    = errors.New("robo: missing closing ']'")
)

// The pathMatcher interface is used to match the paths of incoming requests.
// Any captured parameters will be appended to buf and returned as the second
// return value.
type pathMatcher interface {
	match(path string, buf []string) (bool, []string)
}

// compileMatcher compiles a pathMatcher from a pattern string.
func compileMatcher(pattern string) (pathMatcher, error) {
	var fs []fragment

	if pattern == "" {
		return nil, errEmptyPattern
	}

	for pattern != "" {
		f, n, err := compileFragment(pattern)
		if err != nil {
			return nil, err
		}

		fs = append(fs, f)
		pattern = pattern[n:]
	}

	// replace the standard fragmentMatcher with faster equivalents
	// when possible
	switch {
	case len(fs) == 1 && fs[0].t == literalFragment:
		return &literalMatcher{fs[0].s}, nil
	case len(fs) == 2 && fs[0].t == literalFragment &&
		fs[1].t == wildcardFragment:
		return &prefixMatcher{fs[0].s, fs[0].n}, nil
	}

	return &fragmentMatcher{fs}, nil
}

// compileFragment compiles a fragment matcher from a prefix of a pattern
// string. It returns the compiled fragment, and how many bytes of the input
// string were consumed.
func compileFragment(pattern string) (fragment, int, error) {
	switch pattern[0] {
	default:
		return compileLiteralFragment(pattern)
	case '*':
		return compileWildcardFragment(pattern)
	case '{':
		return compileParameterFragment(pattern)
	}
}

func compileLiteralFragment(pattern string) (fragment, int, error) {
	var i int
	var e bool

	for i = 0; i < len(pattern); i++ {
		switch c := pattern[i]; {
		case e:
			e = false
		case c == '\\':
			e = true
		case c == '*', c == '{':
			return fragment{t: literalFragment, s: pattern[:i], n: i}, i, nil
		}
	}

	return fragment{t: literalFragment, s: pattern[:i], n: i}, i, nil
}

func compileWildcardFragment(pattern string) (fragment, int, error) {
	if pattern != "*" {
		return fragment{}, 0, errIllegalWildcard
	}
	return fragment{t: wildcardFragment}, 1, nil
}

func compileParameterFragment(pattern string) (fragment, int, error) {
	var i int
	var e bool
	var f fragment

	for i = 1; i < len(pattern); i++ {
		switch c := pattern[i]; {
		case e:
			e = false

		case c == '\\':
			e = true

		case c == '[':
			chars, n, err := compileCharsetFragment(pattern[i:])
			if err != nil {
				return f, 0, err
			}

			// before returning, make sure the charset block is
			// followed by a '}'
			if i := i + n; i == len(pattern) || pattern[i] != '}' {
				return fragment{}, 0, errMissingRBrace
			}

			f = fragment{t: inclusiveFragment, s: pattern[1:i], r: chars}
			return f, i + n + 1, nil

		case c == '}':
			if i == 1 {
				return fragment{}, 0, errEmptyParameter
			}

			f = fragment{t: exclusiveFragment, s: pattern[1:i], r: []rune{'/'}}

			// if available, add the next rune to exclusive parameters (for
			// example: '-' for {foo} in "/{foo}-bar")
			for _, r := range pattern[i+1:] {
				if r != '/' {
					f.r = append(f.r, r)
				}
				break
			}

			return f, i + 1, nil
		}
	}

	return f, 0, errMissingRBrace
}

func compileCharsetFragment(pattern string) ([]rune, int, error) {
	var o []rune
	var e bool

	// begin with some rudimentary error checking
	if len(pattern) < 2 {
		return nil, 0, errMissingRBracket
	} else if pattern[1] == ']' {
		return nil, 0, errEmptyCharset
	}

loop:
	for i, r := range pattern[1:] {
		if !e {
			switch r {
			case '\\':
				e = true
				continue loop

			case '[':
				return nil, 0, errUnexpectedLBracket

			case ']':
				if len(o)&1 != 0 {
					return nil, 0, errUnexpectedRBracket
				}

				// make sure the charset doesn't include forward slash
				for i := 0; i < len(o); i += 2 {
					if o[i] <= '/' && '/' <= o[i+1] {
						return nil, 0, errCharsetHasSlash
					}
				}

				return simplifyCharset(o), i + 2, nil

			case '-':
				// catch ambiguous range statements like "a-c-e"
				if len(o) == 0 || o[len(o)-1] != o[len(o)-2] {
					return nil, 0, errUnexpectedHyphen
				}

				o = o[:len(o)-1]
				continue loop
			}
		}

		// determine whether or not this character is the upper bound
		// of a charset range
		if len(o)&1 != 0 {
			if r < o[len(o)-1] {
				return nil, 0, errImpossibleRange
			}
			o = append(o, r)
		} else {
			o = append(o, r, r)
		}

		e = false
	}

	return nil, 0, errMissingRBracket
}

// simplifyCharset merges overlapping rune ranges in the input charset.
func simplifyCharset(a []rune) []rune {
	if len(a) == 0 {
		return a
	}

	// first, sort the pairs in the charset
	for i := 2; i < len(a); i += 2 {
		for j := i; j > 0; j -= 2 {
			if a[j] > a[j-2] || (a[j] == a[j-2] && a[j] > a[j-2]) {
				break
			}
			a[j-2], a[j+0] = a[j+0], a[j-2]
			a[j-1], a[j+1] = a[j+1], a[j-1]
		}
	}

	// then merge overlapping pairs
	var r, w = 2, 2

	for ; r < len(a); r += 2 {
		if a[r] <= a[w-2] {
			if a[w-2] <= a[r+1]+1 {
				goto merge
			}
		} else if a[r]-1 <= a[w-1] {
			goto merge
		}

		a[w] = a[r]
		a[w+1] = a[r+1]
		w += 2
		continue

	merge:
		if a[r] < a[w-2] {
			a[w-2] = a[r]
		}
		if a[r+1] > a[w-1] {
			a[w-1] = a[r+1]
		}
	}

	return a[:w]
}

// literalMatcher matches an exact path.
type literalMatcher struct {
	s string
}

func (lm *literalMatcher) match(in string, buf []string) (bool, []string) {
	return in == lm.s, buf
}

// prefixMatcher matches a path prefix.
type prefixMatcher struct {
	s string
	n int
}

func (pm *prefixMatcher) match(in string, buf []string) (bool, []string) {
	if len(in) >= pm.n && in[:pm.n] == pm.s {
		return true, append(buf, "*", in[pm.n:])
	}
	return false, nil
}

// fragmentMatcher is an implementation of the pathMatcher interface,
// which matches input strings using precompiled fragments.
type fragmentMatcher struct {
	fs []fragment
}

func (fm *fragmentMatcher) match(path string, buf []string) (bool, []string) {
	var n int

	for _, f := range fm.fs {
		n, buf = f.matchPrefix(path, buf)
		if n < 0 {
			return false, nil
		}
		path = path[n:]
	}

	// make sure the whole string has been consumed
	if path != "" {
		return false, nil
	}

	return true, buf
}

type fragment struct {
	t int
	s string
	n int
	r []rune
}

const (
	literalFragment = iota
	exclusiveFragment
	inclusiveFragment
	wildcardFragment
)

func (f *fragment) matchPrefix(pattern string, buf []string) (int, []string) {
	switch f.t {
	case literalFragment:
		if len(pattern) < f.n || pattern[:f.n] != f.s {
			return -1, nil
		}
		return f.n, buf

	case exclusiveFragment:
		for i, r := range pattern {
			for _, excl := range f.r {
				if r == excl {
					return nonZero(i), append(buf, f.s, pattern[:i])
				}
			}
		}
		return nonZero(len(pattern)), append(buf, f.s, pattern)

	case inclusiveFragment:
		for i, r := range pattern {
			for j := 0; j < len(f.r); j += 2 {
				if f.r[j] <= r && r <= f.r[j+1] {
					goto ok
				}
			}
			return nonZero(i), append(buf, f.s, pattern[:i])
		ok:
		}
		return nonZero(len(pattern)), append(buf, f.s, pattern)

	case wildcardFragment:
		return len(pattern), append(buf, "*", pattern)
	}

	panic("unreachable")
}

// nonZero return -1 instead of n if n == 0.
func nonZero(n int) int {
	if n == 0 {
		return -1
	}
	return n
}
<|endoftext|>


<commit_before>package mongodb

import (
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"

	"github.com/microservices-demo/user/users"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	name                string
	password            string
	host                string
	db                  = "users"
	ErrInvalidHexID     = errors.New("Invalid Id Hex")
	ErrorSavingCardData = errors.New("There was a problem saving some card data")
	ErrorSavingAddrData = errors.New("There was a problem saving some address data")
)

func init() {
	flag.StringVar(&name, "mongo-user", os.Getenv("MONGO_USER"), "Mongo user")
	flag.StringVar(&password, "mongo-password", os.Getenv("MONGO_PASS"), "Mongo password")
	flag.StringVar(&host, "mongo-host", os.Getenv("MONGO_HOST"), "Mongo host")
}

type Mongo struct {
	Session *mgo.Session
}

func (m *Mongo) Init() error {
	u := getURL()
	var err error
	m.Session, err = mgo.Dial(u.String())
	if err != nil {
		return err
	}
	return m.EnsureIndexes()
}

type MongoUser struct {
	users.User `bson:",inline"`
	ID         bson.ObjectId   `bson:"_id"`
	AddressIDs []bson.ObjectId `bson:"addresses"`
	CardIDs    []bson.ObjectId `bson:"cards"`
}

func New() MongoUser {
	u := users.New()
	return MongoUser{
		User:       u,
		AddressIDs: make([]bson.ObjectId, 0),
		CardIDs:    make([]bson.ObjectId, 0),
	}
}

func (mu *MongoUser) AddUserIDs() {
	if mu.User.Addresses == nil {
		mu.User.Addresses = make([]users.Address, 0)
	}
	for _, id := range mu.AddressIDs {
		mu.User.Addresses = append(mu.User.Addresses, users.Address{
			ID: id.Hex(),
		})
	}
	if mu.User.Cards == nil {
		mu.User.Cards = make([]users.Card, 0)
	}
	for _, id := range mu.CardIDs {
		mu.User.Cards = append(mu.User.Cards, users.Card{ID: id.Hex()})
	}
	mu.User.UserID = mu.ID.Hex()
}

type MongoAddress struct {
	users.Address `bson:",inline"`
	ID            bson.ObjectId `bson:"_id"`
}

func (m *MongoAddress) AddID() {
	m.Address.ID = m.ID.Hex()
}

type MongoCard struct {
	users.Card `bson:",inline"`
	ID         bson.ObjectId `bson:"_id"`
}

func (m *MongoCard) AddID() {
	m.Card.ID = m.ID.Hex()
}

func (m *Mongo) CreateUser(u *users.User) error {
	s := m.Session.Copy()
	defer s.Close()
	id := bson.NewObjectId()
	mu := New()
	mu.User = *u
	mu.ID = id
	var carderr error
	var addrerr error
	mu.CardIDs, carderr = m.createCards(u.Cards)
	mu.AddressIDs, addrerr = m.createAddresses(u.Addresses)
	c := s.DB("").C("customers")
	_, err := c.UpsertId(mu.ID, mu)
	if err != nil {
		// Gonna clean up if we can, ignore error
		// because the user save error takes precedence.
		m.cleanAttributes(mu)
		return err
	}
	mu.User.UserID = mu.ID.Hex()
	// Cheap err for attributes
	if carderr != nil || addrerr != nil {
		return fmt.Errorf("%v %v", carderr, addrerr)
	}
	u = &mu.User
	return nil
}

func (m *Mongo) createCards(cs []users.Card) ([]bson.ObjectId, error) {
	s := m.Session.Copy()
	defer s.Close()
	ids := make([]bson.ObjectId, 0)
	defer s.Close()
	for k, ca := range cs {
		id := bson.NewObjectId()
		mc := MongoCard{Card: ca, ID: id}
		c := s.DB("").C("cards")
		_, err := c.UpsertId(mc.ID, mc)
		if err != nil {
			return ids, err
		}
		ids = append(ids, id)
		cs[k].ID = id.Hex()
	}
	return ids, nil
}

func (m *Mongo) createAddresses(as []users.Address) ([]bson.ObjectId, error) {
	ids := make([]bson.ObjectId, 0)
	s := m.Session.Copy()
	defer s.Close()
	for k, a := range as {
		id := bson.NewObjectId()
		ma := MongoAddress{Address: a, ID: id}
		c := s.DB("").C("addresses")
		_, err := c.UpsertId(ma.ID, ma)
		if err != nil {
			return ids, err
		}
		ids = append(ids, id)
		as[k].ID = id.Hex()
	}
	return ids, nil
}

func (m *Mongo) cleanAttributes(mu MongoUser) error {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("addresses")
	_, err := c.RemoveAll(bson.M{"_id": bson.M{"$in": mu.AddressIDs}})
	c = s.DB("").C("cards")
	_, err = c.RemoveAll(bson.M{"_id": bson.M{"$in": mu.CardIDs}})
	return err
}

func (m *Mongo) appendAttributeId(attr string, id bson.ObjectId, userid string) error {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C(attr)
	return c.Update(bson.M{"_id": bson.ObjectIdHex(userid)},
		bson.M{"$addToSet": bson.M{"addresses": id}})
}

func (m *Mongo) GetUserByName(name string) (users.User, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("customers")
	mu := New()
	err := c.Find(bson.M{"username": name}).One(&mu)
	mu.AddUserIDs()
	return mu.User, err
}

func (m *Mongo) GetUser(id string) (users.User, error) {
	s := m.Session.Copy()
	defer s.Close()
	if !bson.IsObjectIdHex(id) {
		return users.New(), errors.New("Invalid Id Hex")
	}
	c := s.DB("").C("customers")
	mu := New()
	err := c.FindId(bson.ObjectIdHex(id)).One(&mu)
	mu.AddUserIDs()
	return mu.User, err
}

func (m *Mongo) GetUsers() ([]users.User, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("customers")
	var mus []MongoUser
	err := c.Find(nil).All(&mus)
	us := make([]users.User, 0)
	for _, mu := range mus {
		mu.AddUserIDs()
		us = append(us, mu.User)
	}
	return us, err
}

func (m *Mongo) GetUserAttributes(u *users.User) error {
	s := m.Session.Copy()
	defer s.Close()
	ids := make([]bson.ObjectId, 0)
	for _, a := range u.Addresses {
		if !bson.IsObjectIdHex(a.ID) {
			return ErrInvalidHexID
		}
		ids = append(ids, bson.ObjectIdHex(a.ID))
	}
	var ma []MongoAddress
	c := s.DB("").C("addresses")
	err := c.Find(bson.M{"_id": bson.M{"$in": ids}}).All(&ma)
	if err != nil {
		return err
	}
	na := make([]users.Address, 0)
	for _, a := range ma {
		a.Address.ID = a.ID.Hex()
		na = append(na, a.Address)
	}
	u.Addresses = na

	ids = make([]bson.ObjectId, 0)
	for _, c := range u.Cards {
		if !bson.IsObjectIdHex(c.ID) {
			return ErrInvalidHexID
		}
		ids = append(ids, bson.ObjectIdHex(c.ID))
	}
	var mc []MongoCard
	c = s.DB("").C("cards")
	err = c.Find(bson.M{"_id": bson.M{"$in": ids}}).All(&mc)
	if err != nil {
		return err
	}

	nc := make([]users.Card, 0)
	for _, ca := range mc {
		ca.Card.ID = ca.ID.Hex()
		nc = append(nc, ca.Card)
	}
	u.Cards = nc
	return nil
}

func (m *Mongo) GetCard(id string) (users.Card, error) {
	s := m.Session.Copy()
	defer s.Close()
	if !bson.IsObjectIdHex(id) {
		return users.Card{}, errors.New("Invalid Id Hex")
	}
	c := s.DB("").C("cards")
	mc := MongoCard{}
	err := c.FindId(bson.ObjectIdHex(id)).One(&mc)
	mc.AddID()
	return mc.Card, err
}
func (m *Mongo) GetCards() ([]users.Card, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("cards")
	var mcs []MongoCard
	err := c.Find(nil).All(&mcs)
	cs := make([]users.Card, 0)
	for _, mc := range mcs {
		mc.AddID()
		cs = append(cs, mc.Card)
	}
	return cs, err
}
func (m *Mongo) CreateCard(ca *users.Card, userid string) error {
	if !bson.IsObjectIdHex(userid) {
		return errors.New("Invalid Id Hex")
	}
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("cards")
	id := bson.NewObjectId()
	mc := MongoCard{Card: *ca, ID: id}
	_, err := c.UpsertId(mc.ID, mc)
	if err != nil {
		return err
	}
	err = m.appendAttributeId("cards", mc.ID, userid)
	if err != nil {
		return err
	}
	mc.AddID()
	ca = &mc.Card
	return err
}

func (m *Mongo) GetAddress(id string) (users.Address, error) {
	s := m.Session.Copy()
	defer s.Close()
	if !bson.IsObjectIdHex(id) {
		return users.Address{}, errors.New("Invalid Id Hex")
	}
	c := s.DB("").C("addresses")
	ma := MongoAddress{}
	err := c.FindId(bson.ObjectIdHex(id)).One(&ma)
	ma.AddID()
	return ma.Address, err
}

func (m *Mongo) GetAddresses() ([]users.Address, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("addresses")
	var mas []MongoAddress
	err := c.Find(nil).All(&mas)
	as := make([]users.Address, 0)
	for _, ma := range mas {
		ma.AddID()
		as = append(as, ma.Address)
	}
	return as, err
}

func (m *Mongo) CreateAddress(a *users.Address, userid string) error {
	if !bson.IsObjectIdHex(userid) {
		return errors.New("Invalid Id Hex")
	}
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("addresses")
	id := bson.NewObjectId()
	ma := MongoAddress{Address: *a, ID: id}
	_, err := c.UpsertId(ma.ID, ma)
	if err != nil {
		return err
	}
	err = m.appendAttributeId("addresses", ma.ID, userid)
	if err != nil {
		return err
	}

	ma.AddID()
	a = &ma.Address
	return err
}
func getURL() url.URL {
	u := url.UserPassword(name, password)
	return url.URL{
		Scheme: "mongodb",
		User:   u,
		Host:   host,
		Path:   db,
	}
}

func (m *Mongo) EnsureIndexes() error {
	s := m.Session.Copy()
	defer s.Close()
	i := mgo.Index{
		Key:        []string{"username"},
		Unique:     true,
		DropDups:   true,
		Background: true,
		Sparse:     false,
	}
	c := s.DB("").C("users")
	return c.EnsureIndex(i)
}
<commit_msg>added the ability to add anonymous addresses<commit_after>package mongodb

import (
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"

	"github.com/microservices-demo/user/users"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	name                string
	password            string
	host                string
	db                  = "users"
	ErrInvalidHexID     = errors.New("Invalid Id Hex")
	ErrorSavingCardData = errors.New("There was a problem saving some card data")
	ErrorSavingAddrData = errors.New("There was a problem saving some address data")
)

func init() {
	flag.StringVar(&name, "mongo-user", os.Getenv("MONGO_USER"), "Mongo user")
	flag.StringVar(&password, "mongo-password", os.Getenv("MONGO_PASS"), "Mongo password")
	flag.StringVar(&host, "mongo-host", os.Getenv("MONGO_HOST"), "Mongo host")
}

type Mongo struct {
	Session *mgo.Session
}

func (m *Mongo) Init() error {
	u := getURL()
	var err error
	m.Session, err = mgo.Dial(u.String())
	if err != nil {
		return err
	}
	return m.EnsureIndexes()
}

type MongoUser struct {
	users.User `bson:",inline"`
	ID         bson.ObjectId   `bson:"_id"`
	AddressIDs []bson.ObjectId `bson:"addresses"`
	CardIDs    []bson.ObjectId `bson:"cards"`
}

func New() MongoUser {
	u := users.New()
	return MongoUser{
		User:       u,
		AddressIDs: make([]bson.ObjectId, 0),
		CardIDs:    make([]bson.ObjectId, 0),
	}
}

func (mu *MongoUser) AddUserIDs() {
	if mu.User.Addresses == nil {
		mu.User.Addresses = make([]users.Address, 0)
	}
	for _, id := range mu.AddressIDs {
		mu.User.Addresses = append(mu.User.Addresses, users.Address{
			ID: id.Hex(),
		})
	}
	if mu.User.Cards == nil {
		mu.User.Cards = make([]users.Card, 0)
	}
	for _, id := range mu.CardIDs {
		mu.User.Cards = append(mu.User.Cards, users.Card{ID: id.Hex()})
	}
	mu.User.UserID = mu.ID.Hex()
}

type MongoAddress struct {
	users.Address `bson:",inline"`
	ID            bson.ObjectId `bson:"_id"`
}

func (m *MongoAddress) AddID() {
	m.Address.ID = m.ID.Hex()
}

type MongoCard struct {
	users.Card `bson:",inline"`
	ID         bson.ObjectId `bson:"_id"`
}

func (m *MongoCard) AddID() {
	m.Card.ID = m.ID.Hex()
}

func (m *Mongo) CreateUser(u *users.User) error {
	s := m.Session.Copy()
	defer s.Close()
	id := bson.NewObjectId()
	mu := New()
	mu.User = *u
	mu.ID = id
	var carderr error
	var addrerr error
	mu.CardIDs, carderr = m.createCards(u.Cards)
	mu.AddressIDs, addrerr = m.createAddresses(u.Addresses)
	c := s.DB("").C("customers")
	_, err := c.UpsertId(mu.ID, mu)
	if err != nil {
		// Gonna clean up if we can, ignore error
		// because the user save error takes precedence.
		m.cleanAttributes(mu)
		return err
	}
	mu.User.UserID = mu.ID.Hex()
	// Cheap err for attributes
	if carderr != nil || addrerr != nil {
		return fmt.Errorf("%v %v", carderr, addrerr)
	}
	u = &mu.User
	return nil
}

func (m *Mongo) createCards(cs []users.Card) ([]bson.ObjectId, error) {
	s := m.Session.Copy()
	defer s.Close()
	ids := make([]bson.ObjectId, 0)
	defer s.Close()
	for k, ca := range cs {
		id := bson.NewObjectId()
		mc := MongoCard{Card: ca, ID: id}
		c := s.DB("").C("cards")
		_, err := c.UpsertId(mc.ID, mc)
		if err != nil {
			return ids, err
		}
		ids = append(ids, id)
		cs[k].ID = id.Hex()
	}
	return ids, nil
}

func (m *Mongo) createAddresses(as []users.Address) ([]bson.ObjectId, error) {
	ids := make([]bson.ObjectId, 0)
	s := m.Session.Copy()
	defer s.Close()
	for k, a := range as {
		id := bson.NewObjectId()
		ma := MongoAddress{Address: a, ID: id}
		c := s.DB("").C("addresses")
		_, err := c.UpsertId(ma.ID, ma)
		if err != nil {
			return ids, err
		}
		ids = append(ids, id)
		as[k].ID = id.Hex()
	}
	return ids, nil
}

func (m *Mongo) cleanAttributes(mu MongoUser) error {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("addresses")
	_, err := c.RemoveAll(bson.M{"_id": bson.M{"$in": mu.AddressIDs}})
	c = s.DB("").C("cards")
	_, err = c.RemoveAll(bson.M{"_id": bson.M{"$in": mu.CardIDs}})
	return err
}

func (m *Mongo) appendAttributeId(attr string, id bson.ObjectId, userid string) error {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C(attr)
	return c.Update(bson.M{"_id": bson.ObjectIdHex(userid)},
		bson.M{"$addToSet": bson.M{"addresses": id}})
}

func (m *Mongo) GetUserByName(name string) (users.User, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("customers")
	mu := New()
	err := c.Find(bson.M{"username": name}).One(&mu)
	mu.AddUserIDs()
	return mu.User, err
}

func (m *Mongo) GetUser(id string) (users.User, error) {
	s := m.Session.Copy()
	defer s.Close()
	if !bson.IsObjectIdHex(id) {
		return users.New(), errors.New("Invalid Id Hex")
	}
	c := s.DB("").C("customers")
	mu := New()
	err := c.FindId(bson.ObjectIdHex(id)).One(&mu)
	mu.AddUserIDs()
	return mu.User, err
}

func (m *Mongo) GetUsers() ([]users.User, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("customers")
	var mus []MongoUser
	err := c.Find(nil).All(&mus)
	us := make([]users.User, 0)
	for _, mu := range mus {
		mu.AddUserIDs()
		us = append(us, mu.User)
	}
	return us, err
}

func (m *Mongo) GetUserAttributes(u *users.User) error {
	s := m.Session.Copy()
	defer s.Close()
	ids := make([]bson.ObjectId, 0)
	for _, a := range u.Addresses {
		if !bson.IsObjectIdHex(a.ID) {
			return ErrInvalidHexID
		}
		ids = append(ids, bson.ObjectIdHex(a.ID))
	}
	var ma []MongoAddress
	c := s.DB("").C("addresses")
	err := c.Find(bson.M{"_id": bson.M{"$in": ids}}).All(&ma)
	if err != nil {
		return err
	}
	na := make([]users.Address, 0)
	for _, a := range ma {
		a.Address.ID = a.ID.Hex()
		na = append(na, a.Address)
	}
	u.Addresses = na

	ids = make([]bson.ObjectId, 0)
	for _, c := range u.Cards {
		if !bson.IsObjectIdHex(c.ID) {
			return ErrInvalidHexID
		}
		ids = append(ids, bson.ObjectIdHex(c.ID))
	}
	var mc []MongoCard
	c = s.DB("").C("cards")
	err = c.Find(bson.M{"_id": bson.M{"$in": ids}}).All(&mc)
	if err != nil {
		return err
	}

	nc := make([]users.Card, 0)
	for _, ca := range mc {
		ca.Card.ID = ca.ID.Hex()
		nc = append(nc, ca.Card)
	}
	u.Cards = nc
	return nil
}

func (m *Mongo) GetCard(id string) (users.Card, error) {
	s := m.Session.Copy()
	defer s.Close()
	if !bson.IsObjectIdHex(id) {
		return users.Card{}, errors.New("Invalid Id Hex")
	}
	c := s.DB("").C("cards")
	mc := MongoCard{}
	err := c.FindId(bson.ObjectIdHex(id)).One(&mc)
	mc.AddID()
	return mc.Card, err
}
func (m *Mongo) GetCards() ([]users.Card, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("cards")
	var mcs []MongoCard
	err := c.Find(nil).All(&mcs)
	cs := make([]users.Card, 0)
	for _, mc := range mcs {
		mc.AddID()
		cs = append(cs, mc.Card)
	}
	return cs, err
}
func (m *Mongo) CreateCard(ca *users.Card, userid string) error {
	if !bson.IsObjectIdHex(userid) {
		return errors.New("Invalid Id Hex")
	}
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("cards")
	id := bson.NewObjectId()
	mc := MongoCard{Card: *ca, ID: id}
	_, err := c.UpsertId(mc.ID, mc)
	if err != nil {
		return err
	}
	// Address for anonymous user
	if id != "" {
		err = m.appendAttributeId("cards", mc.ID, userid)
		if err != nil {
			return err
		}
	}
	mc.AddID()
	ca = &mc.Card
	return err
}

func (m *Mongo) GetAddress(id string) (users.Address, error) {
	s := m.Session.Copy()
	defer s.Close()
	if !bson.IsObjectIdHex(id) {
		return users.Address{}, errors.New("Invalid Id Hex")
	}
	c := s.DB("").C("addresses")
	ma := MongoAddress{}
	err := c.FindId(bson.ObjectIdHex(id)).One(&ma)
	ma.AddID()
	return ma.Address, err
}

func (m *Mongo) GetAddresses() ([]users.Address, error) {
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("addresses")
	var mas []MongoAddress
	err := c.Find(nil).All(&mas)
	as := make([]users.Address, 0)
	for _, ma := range mas {
		ma.AddID()
		as = append(as, ma.Address)
	}
	return as, err
}

func (m *Mongo) CreateAddress(a *users.Address, userid string) error {
	if !bson.IsObjectIdHex(userid) {
		return errors.New("Invalid Id Hex")
	}
	s := m.Session.Copy()
	defer s.Close()
	c := s.DB("").C("addresses")
	id := bson.NewObjectId()
	ma := MongoAddress{Address: *a, ID: id}
	_, err := c.UpsertId(ma.ID, ma)
	if err != nil {
		return err
	}
	// Address for anonymous user
	if id != "" {
		err = m.appendAttributeId("addresses", ma.ID, userid)
		if err != nil {
			return err
		}
	}
	ma.AddID()
	a = &ma.Address
	return err
}
func getURL() url.URL {
	u := url.UserPassword(name, password)
	return url.URL{
		Scheme: "mongodb",
		User:   u,
		Host:   host,
		Path:   db,
	}
}

func (m *Mongo) EnsureIndexes() error {
	s := m.Session.Copy()
	defer s.Close()
	i := mgo.Index{
		Key:        []string{"username"},
		Unique:     true,
		DropDups:   true,
		Background: true,
		Sparse:     false,
	}
	c := s.DB("").C("users")
	return c.EnsureIndex(i)
}
<|endoftext|>


<commit_before>// The command line tool for running Imsto bootstrap.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"text/template"
	"wpst.me/calf/config"
)

// Cribbed from the genius organization of the "go" command.
type Command struct {
	Run                    func(args []string) bool
	UsageLine, Short, Long string
	// Flag is a set of flags specific to this command.
	Flag    flag.FlagSet
	IsDebug *bool
}

func (cmd *Command) Name() string {
	name := cmd.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (cmd *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Usage: imsto %s\n", cmd.UsageLine)
	fmt.Fprintf(os.Stderr, "Default Usage:\n")
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "Description:\n")
	fmt.Fprintf(os.Stderr, "  %s\n", strings.TrimSpace(cmd.Long))
	os.Exit(2)
}

const (
	VERSION = "0.0.4"
)

// main
var (
	IsDebug    *bool
	exitStatus = 0
	exitMu     sync.Mutex
	cfgDir     string
	logDir     string
)

var commands = []*Command{
	cmdImport,
	cmdExport,
	cmdOptimize,
	cmdTiring,
	cmdStage,
	cmdView,
	cmdTest,
	cmdAuth,
}

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

func init() {
	flag.StringVar(&cfgDir, "conf", "/etc/imsto", "app config dir")
	flag.StringVar(&logDir, "logs", "/var/log/imsto", "app logs dir")
	flag.Parse()
	if cfgDir != "" {
		config.SetRoot(cfgDir)
	}
	err := config.Load()
	if err != nil {
		log.Print("config load error: ", err)
	}
	if logDir == "" {
		dir := config.Root()
		log.Printf("conf %s", dir)
		if err != nil {
			log.Print("getwd error: ", err)
		}
		if dir != "" {
			dir = path.Join(dir, "logs")
			// log.Printf("log dir: %s", dir)
			if fi, err := os.Stat(dir); err == nil {
				if fi.IsDir() {
					logDir = dir
				}
			}
		}
	}
}

func main() {
	// fmt.Fprintf(os.Stdout, header)
	flag.Usage = func() { usage(1) }
	args := flag.Args()

	if len(args) < 1 || args[0] == "help" {
		if len(args) == 1 {
			usage(0)
		}
		if len(args) > 1 {
			for _, cmd := range commands {
				if cmd.Name() == args[1] {
					tmpl(os.Stdout, helpTemplate, cmd)
					return
				}
			}
		}
		usage(2)
	}

	if logDir != "" {
		_, err := os.Stat(logDir)
		if os.IsNotExist(err) {
			if err := os.Mkdir(logDir, os.FileMode(0755)); err != nil {
				log.Printf("mkdir '%s' error or access denied", logDir)
				return
			}
		} else if os.IsPermission(err) {
			log.Printf("dir '%s' access denied", logDir)
			return
		}

	}

	for _, cmd := range commands {
		name := cmd.Name()
		if name == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			IsDebug = cmd.IsDebug
			// if IsDebug != nil && *IsDebug {
			log.SetFlags(log.LstdFlags | log.Lshortfile)
			// log.Printf("log dir: %s", logDir)
			if logDir != "" {
				logfile := path.Join(logDir, name+"_log")
				// log.Printf("logfile: %s", logfile)
				fd, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err != nil {
					log.Printf("logfile %s create failed", logfile)
					return
				}
				log.SetOutput(fd)
			} else {
				log.Print("log dir is empty")
			}

			// }
			if !cmd.Run(args) {
				fmt.Fprintf(os.Stderr, "\n")
				cmd.Flag.Usage()
				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
				cmd.Flag.PrintDefaults()
			}
			// exit()
			return
		}
	}

	errorf("unknown command %q\nRun 'imsto help' for usage.\n", args[0])
}

func errorf(format string, args ...interface{}) {
	// Ensure the user's command prompt starts on the next line.
	if !strings.HasSuffix(format, "\n") {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, args...)
}

// const header = `
// Welcome

// `

const usageTemplate = `usage: imsto command [arguments]

The commands are:
{{range .}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}

Use "imsto help [command]" for more information.
`

var helpTemplate = `usage: imsto {{.UsageLine}}
{{.Long}}
`

func usage(exitCode int) {
	tmpl(os.Stderr, usageTemplate, commands)
	os.Exit(exitCode)
}

func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}

var atExitFuncs []func()

func atExit(f func()) {
	atExitFuncs = append(atExitFuncs, f)
}

func exit() {
	for _, f := range atExitFuncs {
		f()
	}
	os.Exit(exitStatus)
}

type apiRes map[string]interface{}
type apiMeta map[string]interface{}

func newApiRes(meta apiMeta, data interface{}) apiRes {
	res := make(apiRes)
	res["meta"] = meta
	res["data"] = data
	return res
}

func newApiMeta(ok bool) apiMeta {
	meta := make(apiMeta)
	meta["ok"] = ok
	return meta
}

func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err error) {
	w.Header().Set("Content-Type", "application/json")
	var bytes []byte
	if r.FormValue("pretty") != "" {
		bytes, err = json.MarshalIndent(obj, "", "  ")
	} else {
		bytes, err = json.Marshal(obj)
	}
	if err != nil {
		return
	}
	callback := r.FormValue("callback")
	if callback == "" {
		_, err = w.Write(bytes)
	} else {
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}
	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, obj interface{}) {
	if err := writeJson(w, r, obj); err != nil {
		log.Printf("error writing JSON %s: %s", obj, err.Error())
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, err error) {
	m := newApiMeta(false)
	m["error"] = err.Error()
	writeJsonQuiet(w, r, newApiRes(m, nil))
}

func debug(params ...interface{}) {
	if *IsDebug {
		log.Print(params)
	}
}
func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if len(whiteList) == 0 {
			f(w, r)
			return
		}
		host, _, err := net.SplitHostPort(r.RemoteAddr)
		if err == nil {
			for _, ip := range whiteList {
				if ip == host {
					f(w, r)
					return
				}
			}
		}
		w.WriteHeader(http.StatusForbidden)
		writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
	}
}
<commit_msg>fixes api result errors<commit_after>// The command line tool for running Imsto bootstrap.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"text/template"
	"wpst.me/calf/config"
)

// Cribbed from the genius organization of the "go" command.
type Command struct {
	Run                    func(args []string) bool
	UsageLine, Short, Long string
	// Flag is a set of flags specific to this command.
	Flag    flag.FlagSet
	IsDebug *bool
}

func (cmd *Command) Name() string {
	name := cmd.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (cmd *Command) Usage() {
	fmt.Fprintf(os.Stderr, "Usage: imsto %s\n", cmd.UsageLine)
	fmt.Fprintf(os.Stderr, "Default Usage:\n")
	cmd.Flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "Description:\n")
	fmt.Fprintf(os.Stderr, "  %s\n", strings.TrimSpace(cmd.Long))
	os.Exit(2)
}

const (
	VERSION = "0.0.4"
)

// main
var (
	IsDebug    *bool
	exitStatus = 0
	exitMu     sync.Mutex
	cfgDir     string
	logDir     string
)

var commands = []*Command{
	cmdImport,
	cmdExport,
	cmdOptimize,
	cmdTiring,
	cmdStage,
	cmdView,
	cmdTest,
	cmdAuth,
}

func setExitStatus(n int) {
	exitMu.Lock()
	if exitStatus < n {
		exitStatus = n
	}
	exitMu.Unlock()
}

func init() {
	flag.StringVar(&cfgDir, "conf", "/etc/imsto", "app config dir")
	flag.StringVar(&logDir, "logs", "/var/log/imsto", "app logs dir")
	flag.Parse()
	if cfgDir != "" {
		config.SetRoot(cfgDir)
	}
	err := config.Load()
	if err != nil {
		log.Print("config load error: ", err)
	}
	if logDir == "" {
		dir := config.Root()
		log.Printf("conf %s", dir)
		if err != nil {
			log.Print("getwd error: ", err)
		}
		if dir != "" {
			dir = path.Join(dir, "logs")
			// log.Printf("log dir: %s", dir)
			if fi, err := os.Stat(dir); err == nil {
				if fi.IsDir() {
					logDir = dir
				}
			}
		}
	}
}

func main() {
	// fmt.Fprintf(os.Stdout, header)
	flag.Usage = func() { usage(1) }
	args := flag.Args()

	if len(args) < 1 || args[0] == "help" {
		if len(args) == 1 {
			usage(0)
		}
		if len(args) > 1 {
			for _, cmd := range commands {
				if cmd.Name() == args[1] {
					tmpl(os.Stdout, helpTemplate, cmd)
					return
				}
			}
		}
		usage(2)
	}

	if logDir != "" {
		_, err := os.Stat(logDir)
		if os.IsNotExist(err) {
			if err := os.Mkdir(logDir, os.FileMode(0755)); err != nil {
				log.Printf("mkdir '%s' error or access denied", logDir)
				return
			}
		} else if os.IsPermission(err) {
			log.Printf("dir '%s' access denied", logDir)
			return
		}

	}

	for _, cmd := range commands {
		name := cmd.Name()
		if name == args[0] && cmd.Run != nil {
			cmd.Flag.Usage = func() { cmd.Usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			IsDebug = cmd.IsDebug
			// if IsDebug != nil && *IsDebug {
			log.SetFlags(log.LstdFlags | log.Lshortfile)
			// log.Printf("log dir: %s", logDir)
			if logDir != "" {
				logfile := path.Join(logDir, name+"_log")
				// log.Printf("logfile: %s", logfile)
				fd, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err != nil {
					log.Printf("logfile %s create failed", logfile)
					return
				}
				log.SetOutput(fd)
			} else {
				log.Print("log dir is empty")
			}

			// }
			if !cmd.Run(args) {
				fmt.Fprintf(os.Stderr, "\n")
				cmd.Flag.Usage()
				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
				cmd.Flag.PrintDefaults()
			}
			// exit()
			return
		}
	}

	errorf("unknown command %q\nRun 'imsto help' for usage.\n", args[0])
}

func errorf(format string, args ...interface{}) {
	// Ensure the user's command prompt starts on the next line.
	if !strings.HasSuffix(format, "\n") {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, args...)
}

// const header = `
// Welcome

// `

const usageTemplate = `usage: imsto command [arguments]

The commands are:
{{range .}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}

Use "imsto help [command]" for more information.
`

var helpTemplate = `usage: imsto {{.UsageLine}}
{{.Long}}
`

func usage(exitCode int) {
	tmpl(os.Stderr, usageTemplate, commands)
	os.Exit(exitCode)
}

func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}

var atExitFuncs []func()

func atExit(f func()) {
	atExitFuncs = append(atExitFuncs, f)
}

func exit() {
	for _, f := range atExitFuncs {
		f()
	}
	os.Exit(exitStatus)
}

type apiRes map[string]interface{}
type apiMeta map[string]interface{}

func newApiRes(meta apiMeta, data interface{}) apiRes {
	res := make(apiRes)
	res["meta"] = meta
	res["data"] = data
	return res
}

func newApiMeta(ok bool) apiMeta {
	meta := make(apiMeta)
	meta["ok"] = ok
	return meta
}

func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err error) {
	w.Header().Set("Content-Type", "application/json")
	var bytes []byte
	if r.FormValue("pretty") != "" {
		bytes, err = json.MarshalIndent(obj, "", "  ")
	} else {
		bytes, err = json.Marshal(obj)
	}
	if err != nil {
		return
	}
	callback := r.FormValue("callback")
	if callback == "" {
		_, err = w.Write(bytes)
	} else {
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}
	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, obj interface{}) {
	if err := writeJson(w, r, obj); err != nil {
		log.Printf("error writing JSON %s: %s", obj, err.Error())
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, err error) {
	res := newApiRes(newApiMeta(false), nil)
	res["error"] = err.Error()
	writeJsonQuiet(w, r, res)
}

func debug(params ...interface{}) {
	if *IsDebug {
		log.Print(params)
	}
}
func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if len(whiteList) == 0 {
			f(w, r)
			return
		}
		host, _, err := net.SplitHostPort(r.RemoteAddr)
		if err == nil {
			for _, ip := range whiteList {
				if ip == host {
					f(w, r)
					return
				}
			}
		}
		w.WriteHeader(http.StatusForbidden)
		writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
	}
}
<|endoftext|>


<commit_before>package maps

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

type BBOX struct {
	MinX, MinY, MaxX, MaxY float64
}

type Request struct {
	HTTP        *http.Request
	Query       url.Values
	Width       int
	Height      int
	BBOX        BBOX
	EPSGCode    int
	Format      string
	ScaleFactor float64
}

type MissingParamError struct {
	Param string
}

func (e *MissingParamError) Error() string {
	return fmt.Sprintf("Missing parameter '%s'", e.Param)
}

type InvalidParamError struct {
	Param string
	Value string
	Msg   string
}

func (e *InvalidParamError) Error() string {
	return fmt.Sprintf("Invalid parameter '%s=%s' %v", e.Param, e.Value, e.Msg)
}

func parseQueryUpper(q string) (url.Values, error) {
	origQuery, err := url.ParseQuery(q)
	if err != nil {
		return nil, err
	}

	query := make(url.Values)
	for k, vs := range origQuery {
		for _, v := range vs {
			query.Add(strings.ToUpper(k), v)
		}
	}
	return query, nil
}

func ParseMapRequest(r *http.Request) (*Request, error) {
	query, err := parseQueryUpper(r.URL.RawQuery)
	if err != nil {
		return nil, err
	}

	req := &Request{
		HTTP:     r,
		Query:    query,
		EPSGCode: 3857,
	}

	req.Width, req.Height, err = parseSize(req.Query)
	if err != nil {
		return nil, err
	}
	req.BBOX, err = parseBBOX(req.Query)
	if err != nil {
		return nil, err
	}

	req.Format, err = parseFormat(req.Query)
	if err != nil {
		return nil, err
	}

	req.ScaleFactor, err = parseScaleFactor(req.Query)
	if err != nil {
		return nil, err
	}
	return req, nil
}

func parseSize(q url.Values) (width, height int, err error) {

	w := q.Get("WIDTH")
	if w == "" {
		return width, height, &MissingParamError{"WIDTH"}
	}

	h := q.Get("HEIGHT")
	if h == "" {
		return width, height, &MissingParamError{"HEIGHT"}
	}
	var tmp uint64
	tmp, err = strconv.ParseUint(w, 10, 32)
	if err != nil || tmp == 0 {
		return width, height, &InvalidParamError{Param: "WIDTH", Value: w}
	}
	width = int(tmp)
	tmp, err = strconv.ParseUint(h, 10, 32)
	if err != nil || tmp == 0 {
		return width, height, &InvalidParamError{Param: "HEIGHT", Value: h}
	}
	height = int(tmp)
	return width, height, nil
}

func parseBBOX(q url.Values) (BBOX, error) {
	var err error
	var bbox BBOX

	bboxStr := q.Get("BBOX")
	if bboxStr == "" {
		return bbox, &MissingParamError{"BBOX"}
	}

	bboxParts := strings.Split(bboxStr, ",")
	if len(bboxParts) != 4 {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}

	bbox.MinX, err = strconv.ParseFloat(bboxParts[0], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}
	bbox.MinY, err = strconv.ParseFloat(bboxParts[1], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}
	bbox.MaxX, err = strconv.ParseFloat(bboxParts[2], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}
	bbox.MaxY, err = strconv.ParseFloat(bboxParts[3], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}

	return bbox, nil
}

func parseFormat(q url.Values) (string, error) {
	format := q.Get("FORMAT")
	if format == "" {
		return "png256", nil
	}

	if strings.HasPrefix(format, "image/png") {
		if format == "image/png; mode=24bit" {
			return "png32", nil
		}
		return "png256", nil
	} else if strings.HasPrefix(format, "image/jpeg") {
		return "jpeg", nil
	} else {
		return "png256", &InvalidParamError{Param: "FORMAT", Value: format}
	}
}

func parseScaleFactor(q url.Values) (float64, error) {
	v := q.Get("SCALE_FACTOR")
	if v == "" {
		return 1.0, nil
	}
	scalef, err := strconv.ParseFloat(v, 64)
	if err != nil || scalef <= 0.01 || scalef >= 100.0 {
		return 1.0, &InvalidParamError{Param: "SCALE_FACTOR", Value: v}
	}

	return scalef, nil
}
<commit_msg>app: parse SRS from request<commit_after>package maps

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

type BBOX struct {
	MinX, MinY, MaxX, MaxY float64
}

type Request struct {
	HTTP        *http.Request
	Query       url.Values
	Width       int
	Height      int
	BBOX        BBOX
	EPSGCode    int
	Format      string
	ScaleFactor float64
}

type MissingParamError struct {
	Param string
}

func (e *MissingParamError) Error() string {
	return fmt.Sprintf("Missing parameter '%s'", e.Param)
}

type InvalidParamError struct {
	Param string
	Value string
	Msg   string
}

func (e *InvalidParamError) Error() string {
	return fmt.Sprintf("Invalid parameter '%s=%s' %v", e.Param, e.Value, e.Msg)
}

func parseQueryUpper(q string) (url.Values, error) {
	origQuery, err := url.ParseQuery(q)
	if err != nil {
		return nil, err
	}

	query := make(url.Values)
	for k, vs := range origQuery {
		for _, v := range vs {
			query.Add(strings.ToUpper(k), v)
		}
	}
	return query, nil
}

func ParseMapRequest(r *http.Request) (*Request, error) {
	query, err := parseQueryUpper(r.URL.RawQuery)
	if err != nil {
		return nil, err
	}

	req := &Request{
		HTTP:  r,
		Query: query,
	}

	req.Width, req.Height, err = parseSize(req.Query)
	if err != nil {
		return nil, err
	}

	req.BBOX, err = parseBBOX(req.Query)
	if err != nil {
		return nil, err
	}

	req.EPSGCode, err = parseEPSGCode(req.Query)
	if err != nil {
		return nil, err
	}

	req.Format, err = parseFormat(req.Query)
	if err != nil {
		return nil, err
	}

	req.ScaleFactor, err = parseScaleFactor(req.Query)
	if err != nil {
		return nil, err
	}
	return req, nil
}

func parseSize(q url.Values) (width, height int, err error) {

	w := q.Get("WIDTH")
	if w == "" {
		return width, height, &MissingParamError{"WIDTH"}
	}

	h := q.Get("HEIGHT")
	if h == "" {
		return width, height, &MissingParamError{"HEIGHT"}
	}
	var tmp uint64
	tmp, err = strconv.ParseUint(w, 10, 32)
	if err != nil || tmp == 0 {
		return width, height, &InvalidParamError{Param: "WIDTH", Value: w}
	}
	width = int(tmp)
	tmp, err = strconv.ParseUint(h, 10, 32)
	if err != nil || tmp == 0 {
		return width, height, &InvalidParamError{Param: "HEIGHT", Value: h}
	}
	height = int(tmp)
	return width, height, nil
}

func parseBBOX(q url.Values) (BBOX, error) {
	var err error
	var bbox BBOX

	bboxStr := q.Get("BBOX")
	if bboxStr == "" {
		return bbox, &MissingParamError{"BBOX"}
	}

	bboxParts := strings.Split(bboxStr, ",")
	if len(bboxParts) != 4 {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}

	bbox.MinX, err = strconv.ParseFloat(bboxParts[0], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}
	bbox.MinY, err = strconv.ParseFloat(bboxParts[1], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}
	bbox.MaxX, err = strconv.ParseFloat(bboxParts[2], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}
	bbox.MaxY, err = strconv.ParseFloat(bboxParts[3], 64)
	if err != nil {
		return bbox, &InvalidParamError{Param: "BBOX", Value: bboxStr}
	}

	return bbox, nil
}

func parseEPSGCode(q url.Values) (int, error) {
	srsStr := q.Get("SRS")
	if srsStr == "" {
		return 0, &MissingParamError{"SRS"}
	}

	if srsStr == "CRS:84" {
		return 4326, nil
	}

	srsParts := strings.Split(srsStr, ":")
	if len(srsParts) != 2 {
		return 0, &InvalidParamError{Param: "SRS", Value: srsStr}
	}

	if !strings.HasPrefix(strings.ToUpper(srsStr), "EPSG") {
		return 0, &InvalidParamError{Param: "SRS", Value: srsStr}
	}

	epsgCode, err := strconv.ParseUint(srsParts[1], 10, 32)
	if err != nil {
		return 0, &InvalidParamError{Param: "SRS", Value: srsStr}
	}

	if epsgCode == 900913 {
		epsgCode = 3857
	}

	return int(epsgCode), nil
}

func parseFormat(q url.Values) (string, error) {
	format := q.Get("FORMAT")
	if format == "" {
		return "png256", nil
	}

	if strings.HasPrefix(format, "image/png") {
		if format == "image/png; mode=24bit" {
			return "png32", nil
		}
		return "png256", nil
	} else if strings.HasPrefix(format, "image/jpeg") {
		return "jpeg", nil
	} else {
		return "png256", &InvalidParamError{Param: "FORMAT", Value: format}
	}
}

func parseScaleFactor(q url.Values) (float64, error) {
	v := q.Get("SCALE_FACTOR")
	if v == "" {
		return 1.0, nil
	}
	scalef, err := strconv.ParseFloat(v, 64)
	if err != nil || scalef <= 0.01 || scalef >= 100.0 {
		return 1.0, &InvalidParamError{Param: "SCALE_FACTOR", Value: v}
	}

	return scalef, nil
}
<|endoftext|>


<commit_before>package marshal

import (
	"encoding/json"
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/types"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

//ss is []string
func MarshalJSON(ss []interface{}, bgn int, end int, schemaHandler *schema.SchemaHandler) (tb *map[string]*layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unknown error")
			}
		}
	}()

	res := make(map[string]*layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = layout.NewEmptyTable()
			res[pathStr].Path = common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Type = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]].GetType()
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := bgn; i < end; i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		var ui interface{}

		// `useNumber`causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
		d := json.NewDecoder(strings.NewReader(ss[i].(string)))
		d.UseNumber()
		d.Decode(&ui)

		node.Val = reflect.ValueOf(ui)
		node.PathMap = pathMap

		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node = stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()

			pathStr := node.PathMap.Path

			schemaIndex, ok := schemaHandler.MapIndex[pathStr]
			//no schema item will be ignored
			if !ok {
				continue
			}

			info := schemaHandler.Infos[schemaIndex]

			if tk == reflect.Map {
				keys := node.Val.MapKeys()

				if info.Type == "MAP" { //real map
					pathStr = pathStr + ".Key_value"
					if len(keys) <= 0 {
						for key, table := range res {
							if len(key) >= len(node.PathMap.Path) &&
								key[:len(node.PathMap.Path)] == node.PathMap.Path {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}

					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))
					for j := len(keys) - 1; j >= 0; j-- {
						key := keys[j]
						value := node.Val.MapIndex(key).Elem()

						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Key"]
						newNode.Val = key
						newNode.DL = node.DL + 1
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)

						newNode = nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Value"]
						newNode.Val = value
						newNode.DL = node.DL + 1
						newPathStr := newNode.PathMap.Path // check again
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newInfo := schemaHandler.Infos[newSchemaIndex]
						if newInfo.RepetitionType == parquet.FieldRepetitionType_OPTIONAL { //map value only be :optional or required
							newNode.DL++
						}

						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)
					}

				} else { //struct
					keysMap := make(map[string]int)
					for j := 0; j < len(keys); j++ {
						//ExName to InName
						keysMap[common.HeadToUpper(keys[j].String())] = j
					}
					for key, _ := range node.PathMap.Children {
						ki, ok := keysMap[key]

						if ok && node.Val.MapIndex(keys[ki]).Elem().IsValid() {
							newNode := nodeBuf.GetNode()
							newNode.PathMap = node.PathMap.Children[key]
							newNode.Val = node.Val.MapIndex(keys[ki]).Elem()
							newNode.RL = node.RL
							newNode.DL = node.DL
							newPathStr := newNode.PathMap.Path
							newSchemaIndex := schemaHandler.MapIndex[newPathStr]
							newInfo := schemaHandler.Infos[newSchemaIndex]
							if newInfo.RepetitionType == parquet.FieldRepetitionType_OPTIONAL {
								newNode.DL++
							}
							stack = append(stack, newNode)

						} else {
							newPathStr := node.PathMap.Children[key].Path
							for path, table := range res {
								if strings.HasPrefix(path, newPathStr) &&
									(len(path) == len(newPathStr) || path[len(newPathStr)] == '.') {

									table.Values = append(table.Values, nil)
									table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
									table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
								}
							}
						}
					}
				}

			} else if tk == reflect.Slice {
				ln := node.Val.Len()

				if info.Type == "LIST" { //real LIST
					pathStr = pathStr + ".List" + ".Element"
					if ln <= 0 {
						for key, table := range res {
							if len(key) >= len(node.PathMap.Path) &&
								key[:len(node.PathMap.Path)] == node.PathMap.Path {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["List"].Children["Element"]
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1

						newPathStr := newNode.PathMap.Path
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newInfo := schemaHandler.Infos[newSchemaIndex]
						if newInfo.RepetitionType == parquet.FieldRepetitionType_OPTIONAL { //element of LIST can only be optional or required
							newNode.DL++
						}

						stack = append(stack, newNode)
					}

				} else { //Repeated
					if ln <= 0 {
						for key, table := range res {
							if len(key) >= len(node.PathMap.Path) &&
								key[:len(node.PathMap.Path)] == node.PathMap.Path {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1
						stack = append(stack, newNode)
					}
				}

			} else {
				table := res[node.PathMap.Path]
				pT, cT := types.TypeNameToParquetType(info.Type, info.BaseType)
				val := types.JSONTypeToParquetType(node.Val, pT, cT, int(info.Length), int(info.Scale))

				table.Values = append(table.Values, val)
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
			}
		}
	}

	return &res, nil

}
<commit_msg>update marshal json using schema instead of info<commit_after>package marshal

import (
	"encoding/json"
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/types"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

//ss is []string
func MarshalJSON(ss []interface{}, bgn int, end int, schemaHandler *schema.SchemaHandler) (tb *map[string]*layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unknown error")
			}
		}
	}()

	res := make(map[string]*layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = layout.NewEmptyTable()
			res[pathStr].Path = common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Type = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]].GetType()
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := bgn; i < end; i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		var ui interface{}

		// `useNumber`causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
		d := json.NewDecoder(strings.NewReader(ss[i].(string)))
		d.UseNumber()
		d.Decode(&ui)

		node.Val = reflect.ValueOf(ui)
		node.PathMap = pathMap

		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node = stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()

			pathStr := node.PathMap.Path

			schemaIndex, ok := schemaHandler.MapIndex[pathStr]
			//no schema item will be ignored
			if !ok {
				continue
			}

			schema := schemaHandler.SchemaElements[schemaIndex]

			if tk == reflect.Map {
				keys := node.Val.MapKeys()

				if schema.GetConvertedType() == parquet.ConvertedType_MAP { //real map
					pathStr = pathStr + ".Key_value"
					if len(keys) <= 0 {
						for key, table := range res {
							if len(key) >= len(node.PathMap.Path) &&
								key[:len(node.PathMap.Path)] == node.PathMap.Path {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}

					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))
					for j := len(keys) - 1; j >= 0; j-- {
						key := keys[j]
						value := node.Val.MapIndex(key).Elem()

						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Key"]
						newNode.Val = key
						newNode.DL = node.DL + 1
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)

						newNode = nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Value"]
						newNode.Val = value
						newNode.DL = node.DL + 1
						newPathStr := newNode.PathMap.Path // check again
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newSchema := schemaHandler.SchemaElements[newSchemaIndex]
						if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL { //map value only be :optional or required
							newNode.DL++
						}

						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)
					}

				} else { //struct
					keysMap := make(map[string]int)
					for j := 0; j < len(keys); j++ {
						//ExName to InName
						keysMap[common.HeadToUpper(keys[j].String())] = j
					}
					for key, _ := range node.PathMap.Children {
						ki, ok := keysMap[key]

						if ok && node.Val.MapIndex(keys[ki]).Elem().IsValid() {
							newNode := nodeBuf.GetNode()
							newNode.PathMap = node.PathMap.Children[key]
							newNode.Val = node.Val.MapIndex(keys[ki]).Elem()
							newNode.RL = node.RL
							newNode.DL = node.DL
							newPathStr := newNode.PathMap.Path
							newSchemaIndex := schemaHandler.MapIndex[newPathStr]
							newSchema := schemaHandler.SchemaElements[newSchemaIndex]
							if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
								newNode.DL++
							}
							stack = append(stack, newNode)

						} else {
							newPathStr := node.PathMap.Children[key].Path
							for path, table := range res {
								if strings.HasPrefix(path, newPathStr) &&
									(len(path) == len(newPathStr) || path[len(newPathStr)] == '.') {

									table.Values = append(table.Values, nil)
									table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
									table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
								}
							}
						}
					}
				}

			} else if tk == reflect.Slice {
				ln := node.Val.Len()

				if schema.GetConvertedType() == parquet.ConvertedType_LIST { // real LIST
					pathStr = pathStr + ".List" + ".Element"
					if ln <= 0 {
						for key, table := range res {
							if len(key) >= len(node.PathMap.Path) &&
								key[:len(node.PathMap.Path)] == node.PathMap.Path {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["List"].Children["Element"]
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1

						newPathStr := newNode.PathMap.Path
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newSchema := schemaHandler.SchemaElements[newSchemaIndex]
						if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL { //element of LIST can only be optional or required
							newNode.DL++
						}

						stack = append(stack, newNode)
					}

				} else { //Repeated
					if ln <= 0 {
						for key, table := range res {
							if len(key) >= len(node.PathMap.Path) &&
								key[:len(node.PathMap.Path)] == node.PathMap.Path {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1
						stack = append(stack, newNode)
					}
				}

			} else {
				table := res[node.PathMap.Path]
				pT, cT := schema.Type, schema.ConvertedType
				val := types.JSONTypeToParquetType(node.Val, pT, cT, int(schema.GetTypeLength()), int(schema.GetScale()))

				table.Values = append(table.Values, val)
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
			}
		}
	}

	return &res, nil

}
<|endoftext|>


<commit_before>package rev

import (
	"log"
	"reflect"
)

// An "interceptor" is a function that is invoked by the framework at a
// designated time (BEFORE or AFTER) an action invcation.
//
// Since an interceptor may be used across many user Controllers, it is a
// function that takes the base Controller, rather than a method on a user
// controller.
//
// An interceptor may optionally return a Result (instead of nil).  Depending on
// when the interceptor was invoked, the response is different:
// 1. BEFORE:  No further interceptors are invoked, and neither is the action.
// 2. AFTER: Further interceptors are still run.
// In all cases, any returned Result will take the place of any existing Result.
// But in the BEFORE case, that returned Result is guaranteed to be final, while
// in the AFTER case it is possible that a further interceptor could emit its
// own Result.
//
// Interceptors are called in the order that they are added.
//
// ***
//
// Two types of interceptors are provided: Funcs and Methods
//
// Func Interceptors may apply to any / all Controllers.
//
//   func example(*rev.Controller) rev.Result
//
// Method Interceptors are provided so that properties can be set on application
// controllers.
//
//   func (c AppController) example() rev.Result
//   func (c *AppController) example() rev.Result
//
type InterceptorFunc func(*Controller) Result
type InterceptorMethod interface{}
type InterceptTime int

const (
	BEFORE InterceptTime = iota
	AFTER
)

type InterceptTarget int

const (
	ALL_CONTROLLERS InterceptTarget = iota
)

type Interception struct {
	When InterceptTime

	function InterceptorFunc
	method   InterceptorMethod

	callable     reflect.Value
	target       reflect.Type
	interceptAll bool
}

// Perform the given interception.
// val is a pointer to the App Controller.
func (i Interception) Invoke(val reflect.Value) reflect.Value {
	var arg reflect.Value
	if i.function == nil {
		// If it's an InterceptorMethod, then we have to pass in the target type.
		arg = findTarget(val, i.target)
	} else {
		// If it's an InterceptorFunc, then the type must be *Controller.
		// We can find that by following the embedded types up the chain.
		for val.Type() != controllerPtrType {
			if val.Kind() == reflect.Ptr {
				val = val.Elem()
			}
			val = val.Field(0)
		}
		arg = val
	}

	vals := i.callable.Call([]reflect.Value{arg})
	return vals[0]
}

var interceptors []*Interception

// Install a general interceptor.
// This can be applied to any Controller.
// It must have the signature of:
//   func example(c *rev.Controller) rev.Result
func InterceptFunc(intc InterceptorFunc, when InterceptTime, target interface{}) {
	interceptors = append(interceptors, &Interception{
		When:         when,
		function:     intc,
		callable:     reflect.ValueOf(intc),
		target:       reflect.TypeOf(target),
		interceptAll: target == ALL_CONTROLLERS,
	})
}

// Install an interceptor method that applies to its own Controller.
//   func (c AppController) example() rev.Result
//   func (c *AppController) example() rev.Result
func InterceptMethod(intc InterceptorMethod, when InterceptTime) {
	methodType := reflect.TypeOf(intc)
	if methodType.Kind() != reflect.Func || methodType.NumOut() != 1 || methodType.NumIn() != 1 {
		log.Fatalln("Interceptor method should have signature like",
			"'func (c *AppController) example() rev.Result' but was", methodType)
	}
	interceptors = append(interceptors, &Interception{
		When:     when,
		method:   intc,
		callable: reflect.ValueOf(intc),
		target:   methodType.In(0),
	})
}

func getInterceptors(when InterceptTime, val reflect.Value) []*Interception {
	result := []*Interception{}
	for _, intc := range interceptors {
		if intc.When != when {
			continue
		}

		if intc.interceptAll || findTarget(val, intc.target).IsValid() {
			result = append(result, intc)
		}
	}
	return result
}

// Find the value of the target, starting from val and including embedded types.
// Also, convert between any difference in indirection.
// If the target couldn't be found, the returned Value will have IsValid() == false
func findTarget(val reflect.Value, target reflect.Type) reflect.Value {
	// Look through the embedded types (until we reach the *rev.Controller at the top).
	for {
		// Check if val is of a similar type to the target type.
		if val.Type() == target {
			return val
		}
		if val.Kind() == reflect.Ptr && val.Elem().Type() == target {
			return val.Elem()
		}
		if target.Kind() == reflect.Ptr && target.Elem() == val.Type() {
			return val.Addr()
		}

		// If we reached the *rev.Controller and still didn't find what we were
		// looking for, give up.
		if val.Type() == controllerPtrType {
			break
		}

		// Else, drill into the first field (which had better be an embedded type).
		if val.Kind() == reflect.Ptr {
			val = val.Elem()
		}
		val = val.Field(0)
	}

	return reflect.Value{}
}
<commit_msg>Small update to interceptor docs.<commit_after>package rev

import (
	"log"
	"reflect"
)

// An "interceptor" is functionality invoked by the framework BEFORE or AFTER
// an action.
//
// An interceptor may optionally return a Result (instead of nil).  Depending on
// when the interceptor was invoked, the response is different:
// 1. BEFORE:  No further interceptors are invoked, and neither is the action.
// 2. AFTER: Further interceptors are still run.
// In all cases, any returned Result will take the place of any existing Result.
//
// In the BEFORE case, that returned Result is guaranteed to be final, while
// in the AFTER case it is possible that a further interceptor could emit its
// own Result.
//
// Interceptors are called in the order that they are added.
//
// ***
//
// Two types of interceptors are provided: Funcs and Methods
//
// Func Interceptors may apply to any / all Controllers.
//
//   func example(*rev.Controller) rev.Result
//
// Method Interceptors are provided so that properties can be set on application
// controllers.
//
//   func (c AppController) example() rev.Result
//   func (c *AppController) example() rev.Result
//
type InterceptorFunc func(*Controller) Result
type InterceptorMethod interface{}
type InterceptTime int

const (
	BEFORE InterceptTime = iota
	AFTER
)

type InterceptTarget int

const (
	ALL_CONTROLLERS InterceptTarget = iota
)

type Interception struct {
	When InterceptTime

	function InterceptorFunc
	method   InterceptorMethod

	callable     reflect.Value
	target       reflect.Type
	interceptAll bool
}

// Perform the given interception.
// val is a pointer to the App Controller.
func (i Interception) Invoke(val reflect.Value) reflect.Value {
	var arg reflect.Value
	if i.function == nil {
		// If it's an InterceptorMethod, then we have to pass in the target type.
		arg = findTarget(val, i.target)
	} else {
		// If it's an InterceptorFunc, then the type must be *Controller.
		// We can find that by following the embedded types up the chain.
		for val.Type() != controllerPtrType {
			if val.Kind() == reflect.Ptr {
				val = val.Elem()
			}
			val = val.Field(0)
		}
		arg = val
	}

	vals := i.callable.Call([]reflect.Value{arg})
	return vals[0]
}

var interceptors []*Interception

// Install a general interceptor.
// This can be applied to any Controller.
// It must have the signature of:
//   func example(c *rev.Controller) rev.Result
func InterceptFunc(intc InterceptorFunc, when InterceptTime, target interface{}) {
	interceptors = append(interceptors, &Interception{
		When:         when,
		function:     intc,
		callable:     reflect.ValueOf(intc),
		target:       reflect.TypeOf(target),
		interceptAll: target == ALL_CONTROLLERS,
	})
}

// Install an interceptor method that applies to its own Controller.
//   func (c AppController) example() rev.Result
//   func (c *AppController) example() rev.Result
func InterceptMethod(intc InterceptorMethod, when InterceptTime) {
	methodType := reflect.TypeOf(intc)
	if methodType.Kind() != reflect.Func || methodType.NumOut() != 1 || methodType.NumIn() != 1 {
		log.Fatalln("Interceptor method should have signature like",
			"'func (c *AppController) example() rev.Result' but was", methodType)
	}
	interceptors = append(interceptors, &Interception{
		When:     when,
		method:   intc,
		callable: reflect.ValueOf(intc),
		target:   methodType.In(0),
	})
}

func getInterceptors(when InterceptTime, val reflect.Value) []*Interception {
	result := []*Interception{}
	for _, intc := range interceptors {
		if intc.When != when {
			continue
		}

		if intc.interceptAll || findTarget(val, intc.target).IsValid() {
			result = append(result, intc)
		}
	}
	return result
}

// Find the value of the target, starting from val and including embedded types.
// Also, convert between any difference in indirection.
// If the target couldn't be found, the returned Value will have IsValid() == false
func findTarget(val reflect.Value, target reflect.Type) reflect.Value {
	// Look through the embedded types (until we reach the *rev.Controller at the top).
	for {
		// Check if val is of a similar type to the target type.
		if val.Type() == target {
			return val
		}
		if val.Kind() == reflect.Ptr && val.Elem().Type() == target {
			return val.Elem()
		}
		if target.Kind() == reflect.Ptr && target.Elem() == val.Type() {
			return val.Addr()
		}

		// If we reached the *rev.Controller and still didn't find what we were
		// looking for, give up.
		if val.Type() == controllerPtrType {
			break
		}

		// Else, drill into the first field (which had better be an embedded type).
		if val.Kind() == reflect.Ptr {
			val = val.Elem()
		}
		val = val.Field(0)
	}

	return reflect.Value{}
}
<|endoftext|>


<commit_before>package hcsshim

import (
	"encoding/json"
	"io"
	"time"
)

// ProcessConfig is used as both the input of Container.CreateProcess
// and to convert the parameters to JSON for passing onto the HCS
type ProcessConfig struct {
	ApplicationName   string            `json:",omitempty"`
	CommandLine       string            `json:",omitempty"`
	CommandArgs       []string          `json:",omitempty"` // Used by Linux Containers on Windows
	User              string            `json:",omitempty"`
	WorkingDirectory  string            `json:",omitempty"`
	Environment       map[string]string `json:",omitempty"`
	EmulateConsole    bool              `json:",omitempty"`
	CreateStdInPipe   bool              `json:",omitempty"`
	CreateStdOutPipe  bool              `json:",omitempty"`
	CreateStdErrPipe  bool              `json:",omitempty"`
	ConsoleSize       [2]uint           `json:",omitempty"`
	CreateInUtilityVm bool              `json:",omitempty"` // Used by Linux Containers on Windows
	OCISpecification  *json.RawMessage  `json:",omitempty"` // Used by Linux Containers on Windows
}

type Layer struct {
	ID   string
	Path string
}

type MappedDir struct {
	HostPath         string
	ContainerPath    string
	ReadOnly         bool
	BandwidthMaximum uint64
	IOPSMaximum      uint64
}

type MappedPipe struct {
	HostPath          string
	ContainerPipeName string
}

type HvRuntime struct {
	ImagePath           string `json:",omitempty"`
	SkipTemplate        bool   `json:",omitempty"`
	LinuxInitrdFile     string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM
	LinuxKernelFile     string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM
	LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode
	BootSource          string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD
	WritableBootSource  bool   `json:",omitempty"` // Linux Utility VM booting from VHD
}

type MappedVirtualDisk struct {
	HostPath          string `json:",omitempty"` // Path to VHD on the host
	ContainerPath     string // Platform-specific mount point path in the container
	CreateInUtilityVM bool   `json:",omitempty"`
	ReadOnly          bool   `json:",omitempty"`
	Cache             string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing"
	AttachOnly        bool   `json:",omitempty:`
}

// ContainerConfig is used as both the input of CreateContainer
// and to convert the parameters to JSON for passing onto the HCS
type ContainerConfig struct {
	SystemType                  string              // HCS requires this to be hard-coded to "Container"
	Name                        string              // Name of the container. We use the docker ID.
	Owner                       string              `json:",omitempty"` // The management platform that created this container
	VolumePath                  string              `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID}
	IgnoreFlushesDuringBoot     bool                `json:",omitempty"` // Optimization hint for container startup in Windows
	LayerFolderPath             string              `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format  %root%\windowsfilter\containerID
	Layers                      []Layer             // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID
	Credentials                 string              `json:",omitempty"` // Credentials information
	ProcessorCount              uint32              `json:",omitempty"` // Number of processors to assign to the container.
	ProcessorWeight             uint64              `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares.
	ProcessorMaximum            int64               `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit.
	StorageIOPSMaximum          uint64              `json:",omitempty"` // Maximum Storage IOPS
	StorageBandwidthMaximum     uint64              `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
	StorageSandboxSize          uint64              `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
	MemoryMaximumInMB           int64               `json:",omitempty"` // Maximum memory available to the container in Megabytes
	HostName                    string              `json:",omitempty"` // Hostname
	MappedDirectories           []MappedDir         `json:",omitempty"` // List of mapped directories (volumes/mounts)
	MappedPipes                 []MappedPipe        `json:",omitempty"` // List of mapped Windows named pipes
	HvPartition                 bool                // True if it a Hyper-V Container
	NetworkSharedContainerName  string              `json:",omitempty"` // Name (ID) of the container that we will share the network stack with.
	EndpointList                []string            `json:",omitempty"` // List of networking endpoints to be attached to container
	HvRuntime                   *HvRuntime          `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
	Servicing                   bool                `json:",omitempty"` // True if this container is for servicing
	AllowUnqualifiedDNSQuery    bool                `json:",omitempty"` // True to allow unqualified DNS name resolution
	DNSSearchList               string              `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
	ContainerType               string              `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise.
	TerminateOnLastHandleClosed bool                `json:",omitempty"` // Should HCS terminate the container once all handles have been closed
	MappedVirtualDisks          []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start
}

type ComputeSystemQuery struct {
	IDs    []string `json:"Ids,omitempty"`
	Types  []string `json:",omitempty"`
	Names  []string `json:",omitempty"`
	Owners []string `json:",omitempty"`
}

// Container represents a created (but not necessarily running) container.
type Container interface {
	// Start synchronously starts the container.
	Start() error

	// Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds.
	Shutdown() error

	// Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds.
	Terminate() error

	// Waits synchronously waits for the container to shutdown or terminate.
	Wait() error

	// WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It
	// returns false if timeout occurs.
	WaitTimeout(time.Duration) error

	// Pause pauses the execution of a container.
	Pause() error

	// Resume resumes the execution of a container.
	Resume() error

	// HasPendingUpdates returns true if the container has updates pending to install.
	HasPendingUpdates() (bool, error)

	// Statistics returns statistics for a container.
	Statistics() (Statistics, error)

	// ProcessList returns details for the processes in a container.
	ProcessList() ([]ProcessListItem, error)

	// MappedVirtualDisks returns virtual disks mapped to a utility VM, indexed by controller
	MappedVirtualDisks() (map[int]MappedVirtualDiskController, error)

	// CreateProcess launches a new process within the container.
	CreateProcess(c *ProcessConfig) (Process, error)

	// OpenProcess gets an interface to an existing process within the container.
	OpenProcess(pid int) (Process, error)

	// Close cleans up any state associated with the container but does not terminate or wait for it.
	Close() error

	// Modify the System
	Modify(config *ResourceModificationRequestResponse) error
}

// Process represents a running or exited process.
type Process interface {
	// Pid returns the process ID of the process within the container.
	Pid() int

	// Kill signals the process to terminate but does not wait for it to finish terminating.
	Kill() error

	// Wait waits for the process to exit.
	Wait() error

	// WaitTimeout waits for the process to exit or the duration to elapse. It returns
	// false if timeout occurs.
	WaitTimeout(time.Duration) error

	// ExitCode returns the exit code of the process. The process must have
	// already terminated.
	ExitCode() (int, error)

	// ResizeConsole resizes the console of the process.
	ResizeConsole(width, height uint16) error

	// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing
	// these pipes does not close the underlying pipes; it should be possible to
	// call this multiple times to get multiple interfaces.
	Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error)

	// CloseStdin closes the write side of the stdin pipe so that the process is
	// notified on the read side that there is no more data in stdin.
	CloseStdin() error

	// Close cleans up any state associated with the process but does not kill
	// or wait on it.
	Close() error
}
<commit_msg>Add CreateInUtilityVM to MappedDir<commit_after>package hcsshim

import (
	"encoding/json"
	"io"
	"time"
)

// ProcessConfig is used as both the input of Container.CreateProcess
// and to convert the parameters to JSON for passing onto the HCS
type ProcessConfig struct {
	ApplicationName   string            `json:",omitempty"`
	CommandLine       string            `json:",omitempty"`
	CommandArgs       []string          `json:",omitempty"` // Used by Linux Containers on Windows
	User              string            `json:",omitempty"`
	WorkingDirectory  string            `json:",omitempty"`
	Environment       map[string]string `json:",omitempty"`
	EmulateConsole    bool              `json:",omitempty"`
	CreateStdInPipe   bool              `json:",omitempty"`
	CreateStdOutPipe  bool              `json:",omitempty"`
	CreateStdErrPipe  bool              `json:",omitempty"`
	ConsoleSize       [2]uint           `json:",omitempty"`
	CreateInUtilityVm bool              `json:",omitempty"` // Used by Linux Containers on Windows
	OCISpecification  *json.RawMessage  `json:",omitempty"` // Used by Linux Containers on Windows
}

type Layer struct {
	ID   string
	Path string
}

type MappedDir struct {
	HostPath          string
	ContainerPath     string
	ReadOnly          bool
	BandwidthMaximum  uint64
	IOPSMaximum       uint64
	CreateInUtilityVM bool
}

type MappedPipe struct {
	HostPath          string
	ContainerPipeName string
}

type HvRuntime struct {
	ImagePath           string `json:",omitempty"`
	SkipTemplate        bool   `json:",omitempty"`
	LinuxInitrdFile     string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM
	LinuxKernelFile     string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM
	LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode
	BootSource          string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD
	WritableBootSource  bool   `json:",omitempty"` // Linux Utility VM booting from VHD
}

type MappedVirtualDisk struct {
	HostPath          string `json:",omitempty"` // Path to VHD on the host
	ContainerPath     string // Platform-specific mount point path in the container
	CreateInUtilityVM bool   `json:",omitempty"`
	ReadOnly          bool   `json:",omitempty"`
	Cache             string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing"
	AttachOnly        bool   `json:",omitempty:`
}

// ContainerConfig is used as both the input of CreateContainer
// and to convert the parameters to JSON for passing onto the HCS
type ContainerConfig struct {
	SystemType                  string              // HCS requires this to be hard-coded to "Container"
	Name                        string              // Name of the container. We use the docker ID.
	Owner                       string              `json:",omitempty"` // The management platform that created this container
	VolumePath                  string              `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID}
	IgnoreFlushesDuringBoot     bool                `json:",omitempty"` // Optimization hint for container startup in Windows
	LayerFolderPath             string              `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format  %root%\windowsfilter\containerID
	Layers                      []Layer             // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID
	Credentials                 string              `json:",omitempty"` // Credentials information
	ProcessorCount              uint32              `json:",omitempty"` // Number of processors to assign to the container.
	ProcessorWeight             uint64              `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares.
	ProcessorMaximum            int64               `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit.
	StorageIOPSMaximum          uint64              `json:",omitempty"` // Maximum Storage IOPS
	StorageBandwidthMaximum     uint64              `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
	StorageSandboxSize          uint64              `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
	MemoryMaximumInMB           int64               `json:",omitempty"` // Maximum memory available to the container in Megabytes
	HostName                    string              `json:",omitempty"` // Hostname
	MappedDirectories           []MappedDir         `json:",omitempty"` // List of mapped directories (volumes/mounts)
	MappedPipes                 []MappedPipe        `json:",omitempty"` // List of mapped Windows named pipes
	HvPartition                 bool                // True if it a Hyper-V Container
	NetworkSharedContainerName  string              `json:",omitempty"` // Name (ID) of the container that we will share the network stack with.
	EndpointList                []string            `json:",omitempty"` // List of networking endpoints to be attached to container
	HvRuntime                   *HvRuntime          `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
	Servicing                   bool                `json:",omitempty"` // True if this container is for servicing
	AllowUnqualifiedDNSQuery    bool                `json:",omitempty"` // True to allow unqualified DNS name resolution
	DNSSearchList               string              `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
	ContainerType               string              `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise.
	TerminateOnLastHandleClosed bool                `json:",omitempty"` // Should HCS terminate the container once all handles have been closed
	MappedVirtualDisks          []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start
}

type ComputeSystemQuery struct {
	IDs    []string `json:"Ids,omitempty"`
	Types  []string `json:",omitempty"`
	Names  []string `json:",omitempty"`
	Owners []string `json:",omitempty"`
}

// Container represents a created (but not necessarily running) container.
type Container interface {
	// Start synchronously starts the container.
	Start() error

	// Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds.
	Shutdown() error

	// Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds.
	Terminate() error

	// Waits synchronously waits for the container to shutdown or terminate.
	Wait() error

	// WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It
	// returns false if timeout occurs.
	WaitTimeout(time.Duration) error

	// Pause pauses the execution of a container.
	Pause() error

	// Resume resumes the execution of a container.
	Resume() error

	// HasPendingUpdates returns true if the container has updates pending to install.
	HasPendingUpdates() (bool, error)

	// Statistics returns statistics for a container.
	Statistics() (Statistics, error)

	// ProcessList returns details for the processes in a container.
	ProcessList() ([]ProcessListItem, error)

	// MappedVirtualDisks returns virtual disks mapped to a utility VM, indexed by controller
	MappedVirtualDisks() (map[int]MappedVirtualDiskController, error)

	// CreateProcess launches a new process within the container.
	CreateProcess(c *ProcessConfig) (Process, error)

	// OpenProcess gets an interface to an existing process within the container.
	OpenProcess(pid int) (Process, error)

	// Close cleans up any state associated with the container but does not terminate or wait for it.
	Close() error

	// Modify the System
	Modify(config *ResourceModificationRequestResponse) error
}

// Process represents a running or exited process.
type Process interface {
	// Pid returns the process ID of the process within the container.
	Pid() int

	// Kill signals the process to terminate but does not wait for it to finish terminating.
	Kill() error

	// Wait waits for the process to exit.
	Wait() error

	// WaitTimeout waits for the process to exit or the duration to elapse. It returns
	// false if timeout occurs.
	WaitTimeout(time.Duration) error

	// ExitCode returns the exit code of the process. The process must have
	// already terminated.
	ExitCode() (int, error)

	// ResizeConsole resizes the console of the process.
	ResizeConsole(width, height uint16) error

	// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing
	// these pipes does not close the underlying pipes; it should be possible to
	// call this multiple times to get multiple interfaces.
	Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error)

	// CloseStdin closes the write side of the stdin pipe so that the process is
	// notified on the read side that there is no more data in stdin.
	CloseStdin() error

	// Close cleans up any state associated with the process but does not kill
	// or wait on it.
	Close() error
}
<|endoftext|>