<commit_before>// +build appengine

package datastore

import (
	"fmt"
	"reflect"
	"strings"

	"gnd.la/config"
	"gnd.la/log"
	"gnd.la/orm/driver"
	"gnd.la/orm/operation"
	"gnd.la/orm/query"
	"gnd.la/util/types"

	"appengine"
	"appengine/datastore"
)

type Driver struct {
	c             appengine.Context
	logger        *log.Logger
	inTransaction bool
}

func (d *Driver) Check() error {
	return nil
}

func (d *Driver) Initialize(ms []driver.Model) error {
	// No need to create tables in the datastore. Instead,
	// check that the models can be stored.
	return nil
}

func (d *Driver) Query(m driver.Model, q query.Q, opts driver.QueryOptions) driver.Iter {
	dq, err := d.makeQuery(m, q, &opts)
	if err != nil {
		return &Iter{err: err}
	}
	return &Iter{iter: dq.Run(d.c)}
}

func (d *Driver) Count(m driver.Model, q query.Q, opts driver.QueryOptions) (uint64, error) {
	dq, err := d.makeQuery(m, q, &opts)
	if err != nil {
		return 0, err
	}
	c, err := dq.Count(d.c)
	return uint64(c), err
}

func (d *Driver) Exists(m driver.Model, q query.Q) (bool, error) {
	dq, err := d.makeQuery(m, q, nil)
	if err != nil {
		return false, err
	}
	c, err := dq.Count(d.c)
	return c != 0, err
}

func (d *Driver) Insert(m driver.Model, data interface{}) (driver.Result, error) {
	var id int64
	fields := m.Fields()
	var pkVal *reflect.Value
	// TODO: If the PK is supplied by the user rather than auto-assigned, it
	// might conflict with PKs generated by datastore.AllocateIDs().
	if fields.PrimaryKey >= 0 {
		p := d.primaryKey(fields, data)
		if p.IsValid() && types.Kind(p.Kind()) == types.Int {
			id = p.Int()
			if id == 0 {
				// Must assign PK field value after calling AllocateIDs
				pkVal = &p
			}
		}
	}
	name := m.Table()
	// Make all objects of a given kind ancestors of the same key. While
	// this hurts scalability, it makes all reads strongly consistent.
	parent := d.parentKey(m)
	var err error
	if id == 0 {
		id, _, err = datastore.AllocateIDs(d.c, name, parent, 1)
		if err != nil {
			return nil, err
		}
	}
	if fields.AutoincrementPk && pkVal != nil {
		pkVal.SetInt(int64(id))
	}
	key := datastore.NewKey(d.c, name, "", id, parent)
	log.Debugf("DATASTORE: put %s %v", key, data)
	_, err = datastore.Put(d.c, key, data)
	if err != nil {
		return nil, err
	}
	return &result{key: key, count: 1}, nil
}

func (d *Driver) Operate(m driver.Model, q query.Q, ops []*operation.Operation) (driver.Result, error) {
	return nil, fmt.Errorf("datastore driver does not support Operate")
}

func (d *Driver) Update(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	src := make([]interface{}, len(keys))
	for ii := range src {
		src[ii] = data
	}
	// Multi variants need to be run in transactions, otherwise some
	// might fail while others succeed
	err = d.runInTransaction(func(c appengine.Context) error {
		_, e := datastore.PutMulti(c, keys, src)
		return e
	})
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) Upsert(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	return nil, nil
}

func (d *Driver) Delete(m driver.Model, q query.Q) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	// See comment around PutMulti
	err = d.runInTransaction(func(c appengine.Context) error {
		return datastore.DeleteMulti(c, keys)
	})
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) getKeys(m driver.Model, q query.Q) ([]*datastore.Key, error) {
	dq, err := d.makeQuery(m, q, nil)
	if err != nil {
		return nil, err
	}
	iter := dq.KeysOnly().Run(d.c)
	var keys []*datastore.Key
	for {
		key, err := iter.Next(nil)
		if err != nil {
			if err == datastore.Done {
				break
			}
			return nil, err
		}
		keys = append(keys, key)
	}
	return keys, nil
}

func (d *Driver) makeQuery(m driver.Model, q query.Q, opts *driver.QueryOptions) (*datastore.Query, error) {
	if m.Join() != nil {
		return nil, errJoinNotSupported
	}
	dq := datastore.NewQuery(m.Table()).Ancestor(d.parentKey(m))
	var err error
	if dq, err = d.applyQuery(m, dq, q); err != nil {
		return nil, err
	}
	if opts != nil {
		if opts.Distinct {
			dq = dq.Distinct()
		}
		for _, v := range opt.Sort {
			field := v.Field()
			if v.Direction() == driver.DESC {
				field = "-" + field
			}
			dq = dq.Order(field)
		}
		if opts.Limit >= 0 {
			dq = dq.Limit(limit)
		}
		if opts.Offset > 0 {
			dq = dq.Offset(limit)
		}
	}
	return dq, nil
}

func (d *Driver) parentKey(m driver.Model) *datastore.Key {
	return datastore.NewKey(d.c, m.Table(), "", -1, nil)
}

func (d *Driver) applyQuery(m driver.Model, dq *datastore.Query, q query.Q) (*datastore.Query, error) {
	var field *query.Field
	var op string
	switch x := q.(type) {
	case *query.Eq:
		field = &x.Field
		op = " ="
	case *query.Lt:
		field = &x.Field
		op = " <"
	case *query.Lte:
		field = &x.Field
		op = " <="
	case *query.Gt:
		field = &x.Field
		op = " >"
	case *query.Gte:
		field = &x.Field
		op = " >="
	case *query.And:
		var err error
		for _, v := range x.Conditions {
			dq, err = d.applyQuery(m, dq, v)
			if err != nil {
				return nil, err
			}
		}
	case nil:
	default:
		return nil, fmt.Errorf("datastore does not support %T queries", q)
	}
	if field != nil {
		if _, ok := field.Value.(query.F); ok {
			return nil, fmt.Errorf("datastore queries can't reference other properties (%v)", field.Value)
		}
		name := field.Field
		fields := m.Fields()
		idx, ok := fields.QNameMap[name]
		if !ok {
			return nil, fmt.Errorf("can't map field %q to a datastore name", name)
		}
		if strings.IndexByte(name, '.') >= 0 {
			// GAE flattens embedded fields, so we must remove
			// the parts of the field which refer to a flattened
			// field.
			indexes := fields.Indexes[idx]
			parts := strings.Split(name, ".")
			if len(indexes) == len(parts) {
				var final []string
				typ := fields.Type
				for ii, v := range indexes {
					f := typ.Field(v)
					if !f.Anonymous {
						final = append(final, parts[ii])
					}
					typ = f.Type
				}
				name = strings.Join(final, ".")
			}
		}
		log.Debugf("DATASTORE: filter %s %s %v", m, name+op, field.Value)
		dq = dq.Filter(name+op, field.Value)
	}
	return dq, nil
}

func (d *Driver) Close() error {
	return nil
}

func (d *Driver) Upserts() bool {
	return false
}

func (d *Driver) Tags() []string {
	return []string{"datastore"}
}

func (d *Driver) SetLogger(logger *log.Logger) {
	d.logger = logger
}

func (d *Driver) SetContext(ctx appengine.Context) {
	d.c = ctx
}

func (d *Driver) Begin() (driver.Tx, error) {
	return nil, errTransactionNotSupported
}

func (d *Driver) Commit() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) Rollback() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) runInTransaction(f func(c appengine.Context) error) error {
	if d.inTransaction {
		return f(d.c)
	}
	return d.Transaction(func(drv driver.Driver) error {
		return f(drv.(*Driver).c)
	})
}

func (d *Driver) Transaction(f func(driver.Driver) error) error {
	// While not optimal, it's valid to request a cross group transaction
	// when the transaction affects a single entity group. Since the ORM
	// framework does not have an option for specifying if the transaction
	// will affect multiple entities (and it's not clear if we should have
	// it, since the datastore driver would be the only one using it), for
	// now we have to make all transactions XG.
	return datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		drv := *d
		drv.c = c
		drv.inTransaction = true
		return f(&drv)
	}, &datastore.TransactionOptions{XG: true})
}

func (d *Driver) Capabilities() driver.Capability {
	return driver.CAP_TRANSACTION | driver.CAP_AUTO_ID | driver.CAP_EVENTUAL | driver.CAP_PK
}

func (d *Driver) primaryKey(f *driver.Fields, data interface{}) reflect.Value {
	return fieldByIndex(reflect.ValueOf(data), f.Indexes[f.PrimaryKey])
}

func (d *Driver) HasFunc(fname string, retType reflect.Type) bool {
	return false
}

func (d *Driver) Connection() interface{} {
	return d.c
}

func fieldByIndex(val reflect.Value, indexes []int) reflect.Value {
	for _, v := range indexes {
		if val.Type().Kind() == reflect.Ptr {
			if val.IsNil() {
				return reflect.Value{}
			}
			val = val.Elem()
		}
		val = val.Field(v)
	}
	return val
}

func datastoreOpener(url *config.URL) (driver.Driver, error) {
	return &Driver{}, nil
}

func init() {
	driver.Register("datastore", datastoreOpener)
}
<commit_msg>Fix compilation of datastore ORM driver<commit_after>// +build appengine

package datastore

import (
	"fmt"
	"reflect"
	"strings"

	"gnd.la/config"
	"gnd.la/log"
	"gnd.la/orm/driver"
	"gnd.la/orm/operation"
	"gnd.la/orm/query"
	"gnd.la/util/types"

	"appengine"
	"appengine/datastore"
)

type Driver struct {
	c             appengine.Context
	logger        *log.Logger
	inTransaction bool
}

func (d *Driver) Check() error {
	return nil
}

func (d *Driver) Initialize(ms []driver.Model) error {
	// No need to create tables in the datastore. Instead,
	// check that the models can be stored.
	return nil
}

func (d *Driver) Query(m driver.Model, q query.Q, opts driver.QueryOptions) driver.Iter {
	dq, err := d.makeQuery(m, q, &opts)
	if err != nil {
		return &Iter{err: err}
	}
	return &Iter{iter: dq.Run(d.c)}
}

func (d *Driver) Count(m driver.Model, q query.Q, opts driver.QueryOptions) (uint64, error) {
	dq, err := d.makeQuery(m, q, &opts)
	if err != nil {
		return 0, err
	}
	c, err := dq.Count(d.c)
	return uint64(c), err
}

func (d *Driver) Exists(m driver.Model, q query.Q) (bool, error) {
	dq, err := d.makeQuery(m, q, nil)
	if err != nil {
		return false, err
	}
	c, err := dq.Count(d.c)
	return c != 0, err
}

func (d *Driver) Insert(m driver.Model, data interface{}) (driver.Result, error) {
	var id int64
	fields := m.Fields()
	var pkVal *reflect.Value
	// TODO: If the PK is supplied by the user rather than auto-assigned, it
	// might conflict with PKs generated by datastore.AllocateIDs().
	if fields.PrimaryKey >= 0 {
		p := d.primaryKey(fields, data)
		if p.IsValid() && types.Kind(p.Kind()) == types.Int {
			id = p.Int()
			if id == 0 {
				// Must assign PK field value after calling AllocateIDs
				pkVal = &p
			}
		}
	}
	name := m.Table()
	// Make all objects of a given kind ancestors of the same key. While
	// this hurts scalability, it makes all reads strongly consistent.
	parent := d.parentKey(m)
	var err error
	if id == 0 {
		id, _, err = datastore.AllocateIDs(d.c, name, parent, 1)
		if err != nil {
			return nil, err
		}
	}
	if fields.AutoincrementPk && pkVal != nil {
		pkVal.SetInt(int64(id))
	}
	key := datastore.NewKey(d.c, name, "", id, parent)
	log.Debugf("DATASTORE: put %s %v", key, data)
	_, err = datastore.Put(d.c, key, data)
	if err != nil {
		return nil, err
	}
	return &result{key: key, count: 1}, nil
}

func (d *Driver) Operate(m driver.Model, q query.Q, ops []*operation.Operation) (driver.Result, error) {
	return nil, fmt.Errorf("datastore driver does not support Operate")
}

func (d *Driver) Update(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	src := make([]interface{}, len(keys))
	for ii := range src {
		src[ii] = data
	}
	// Multi variants need to be run in transactions, otherwise some
	// might fail while others succeed
	err = d.runInTransaction(func(c appengine.Context) error {
		_, e := datastore.PutMulti(c, keys, src)
		return e
	})
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) Upsert(m driver.Model, q query.Q, data interface{}) (driver.Result, error) {
	return nil, nil
}

func (d *Driver) Delete(m driver.Model, q query.Q) (driver.Result, error) {
	keys, err := d.getKeys(m, q)
	if err != nil {
		return nil, err
	}
	// See comment around PutMulti
	err = d.runInTransaction(func(c appengine.Context) error {
		return datastore.DeleteMulti(c, keys)
	})
	if err != nil {
		return nil, err
	}
	return &result{count: len(keys)}, nil
}

func (d *Driver) getKeys(m driver.Model, q query.Q) ([]*datastore.Key, error) {
	dq, err := d.makeQuery(m, q, nil)
	if err != nil {
		return nil, err
	}
	iter := dq.KeysOnly().Run(d.c)
	var keys []*datastore.Key
	for {
		key, err := iter.Next(nil)
		if err != nil {
			if err == datastore.Done {
				break
			}
			return nil, err
		}
		keys = append(keys, key)
	}
	return keys, nil
}

func (d *Driver) makeQuery(m driver.Model, q query.Q, opts *driver.QueryOptions) (*datastore.Query, error) {
	if m.Join() != nil {
		return nil, errJoinNotSupported
	}
	dq := datastore.NewQuery(m.Table()).Ancestor(d.parentKey(m))
	var err error
	if dq, err = d.applyQuery(m, dq, q); err != nil {
		return nil, err
	}
	if opts != nil {
		if opts.Distinct {
			dq = dq.Distinct()
		}
		for _, v := range opts.Sort {
			field := v.Field()
			if v.Direction() == driver.DESC {
				field = "-" + field
			}
			dq = dq.Order(field)
		}
		if opts.Limit >= 0 {
			dq = dq.Limit(opts.Limit)
		}
		if opts.Offset > 0 {
			dq = dq.Offset(opts.Offset)
		}
	}
	return dq, nil
}

func (d *Driver) parentKey(m driver.Model) *datastore.Key {
	return datastore.NewKey(d.c, m.Table(), "", -1, nil)
}

func (d *Driver) applyQuery(m driver.Model, dq *datastore.Query, q query.Q) (*datastore.Query, error) {
	var field *query.Field
	var op string
	switch x := q.(type) {
	case *query.Eq:
		field = &x.Field
		op = " ="
	case *query.Lt:
		field = &x.Field
		op = " <"
	case *query.Lte:
		field = &x.Field
		op = " <="
	case *query.Gt:
		field = &x.Field
		op = " >"
	case *query.Gte:
		field = &x.Field
		op = " >="
	case *query.And:
		var err error
		for _, v := range x.Conditions {
			dq, err = d.applyQuery(m, dq, v)
			if err != nil {
				return nil, err
			}
		}
	case nil:
	default:
		return nil, fmt.Errorf("datastore does not support %T queries", q)
	}
	if field != nil {
		if _, ok := field.Value.(query.F); ok {
			return nil, fmt.Errorf("datastore queries can't reference other properties (%v)", field.Value)
		}
		name := field.Field
		fields := m.Fields()
		idx, ok := fields.QNameMap[name]
		if !ok {
			return nil, fmt.Errorf("can't map field %q to a datastore name", name)
		}
		if strings.IndexByte(name, '.') >= 0 {
			// GAE flattens embedded fields, so we must remove
			// the parts of the field which refer to a flattened
			// field.
			indexes := fields.Indexes[idx]
			parts := strings.Split(name, ".")
			if len(indexes) == len(parts) {
				var final []string
				typ := fields.Type
				for ii, v := range indexes {
					f := typ.Field(v)
					if !f.Anonymous {
						final = append(final, parts[ii])
					}
					typ = f.Type
				}
				name = strings.Join(final, ".")
			}
		}
		log.Debugf("DATASTORE: filter %s %s %v", m, name+op, field.Value)
		dq = dq.Filter(name+op, field.Value)
	}
	return dq, nil
}

func (d *Driver) Close() error {
	return nil
}

func (d *Driver) Upserts() bool {
	return false
}

func (d *Driver) Tags() []string {
	return []string{"datastore"}
}

func (d *Driver) SetLogger(logger *log.Logger) {
	d.logger = logger
}

func (d *Driver) SetContext(ctx appengine.Context) {
	d.c = ctx
}

func (d *Driver) Begin() (driver.Tx, error) {
	return nil, errTransactionNotSupported
}

func (d *Driver) Commit() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) Rollback() error {
	return driver.ErrNotInTransaction
}

func (d *Driver) runInTransaction(f func(c appengine.Context) error) error {
	if d.inTransaction {
		return f(d.c)
	}
	return d.Transaction(func(drv driver.Driver) error {
		return f(drv.(*Driver).c)
	})
}

func (d *Driver) Transaction(f func(driver.Driver) error) error {
	// While not optimal, it's valid to request a cross group transaction
	// when the transaction affects a single entity group. Since the ORM
	// framework does not have an option for specifying if the transaction
	// will affect multiple entities (and it's not clear if we should have
	// it, since the datastore driver would be the only one using it), for
	// now we have to make all transactions XG.
	return datastore.RunInTransaction(d.c, func(c appengine.Context) error {
		drv := *d
		drv.c = c
		drv.inTransaction = true
		return f(&drv)
	}, &datastore.TransactionOptions{XG: true})
}

func (d *Driver) Capabilities() driver.Capability {
	return driver.CAP_TRANSACTION | driver.CAP_AUTO_ID | driver.CAP_EVENTUAL | driver.CAP_PK
}

func (d *Driver) primaryKey(f *driver.Fields, data interface{}) reflect.Value {
	return fieldByIndex(reflect.ValueOf(data), f.Indexes[f.PrimaryKey])
}

func (d *Driver) HasFunc(fname string, retType reflect.Type) bool {
	return false
}

func (d *Driver) Connection() interface{} {
	return d.c
}

func fieldByIndex(val reflect.Value, indexes []int) reflect.Value {
	for _, v := range indexes {
		if val.Type().Kind() == reflect.Ptr {
			if val.IsNil() {
				return reflect.Value{}
			}
			val = val.Elem()
		}
		val = val.Field(v)
	}
	return val
}

func datastoreOpener(url *config.URL) (driver.Driver, error) {
	return &Driver{}, nil
}

func init() {
	driver.Register("datastore", datastoreOpener)
}
<|endoftext|>


<commit_before>// Copyright 2016 Qiang Xue. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package file provides handlers that serve static files for the ozzo routing package.
package file

import (
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/go-ozzo/ozzo-routing"
)

// ServerOptions defines the possible options for the Server handler.
type ServerOptions struct {
	// The path that all files to be served should be located within. The path map passed to the Server method
	// are all relative to this path. This property can be specified as an absolute file path or a path relative
	// to the current working path. If not set, this property defaults to the current working path.
	RootPath string
	// The file (e.g. index.html) to be served when the current request corresponds to a directory.
	// If not set, the handler will return a 404 HTTP error when the request corresponds to a directory.
	IndexFile string
	// A function that checks if the requested file path is allowed. If allowed, the function
	// may do additional work such as setting Expires HTTP header.
	// The function should return a boolean indicating whether the file should be served or not.
	// If false, a 404 HTTP error will be returned by the handler.
	Allow func(*routing.Context, string) bool
}

// PathMap specifies the mapping between URL paths (keys) and file paths (keys).
// The file paths are relative to Options.RootPath
type PathMap map[string]string

// RootPath stores the current working path
var RootPath string

func init() {
	RootPath, _ = os.Getwd()
}

// Server returns a handler that serves the files as the response content.
// The files being served are determined using the current URL path and the specified path map.
// For example, if the path map is {"/css": "/www/css", "/js": "/www/js"} and the current URL path
// "/css/main.css", the file "<working dir>/www/css/main.css" will be served.
// If a URL path matches multiple prefixes in the path map, the most specific prefix will take precedence.
// For example, if the path map contains both "/css" and "/css/img", and the URL path is "/css/img/logo.gif",
// then the path mapped by "/css/img" will be used.
//
//     import (
//         "log"
//         "github.com/go-ozzo/ozzo-routing"
//         "github.com/go-ozzo/ozzo-routing/file"
//     )
//
//     r := routing.New()
//     r.Get("/*", file.Server(file.PathMap{
//          "/css": "/ui/dist/css",
//          "/js": "/ui/dist/js",
//     })
func Server(pathMap PathMap, opts ...ServerOptions) routing.Handler {
	var options ServerOptions
	if len(opts) > 0 {
		options = opts[0]
	}
	if !filepath.IsAbs(options.RootPath) {
		options.RootPath = filepath.Join(RootPath, options.RootPath)
	}
	from, to := parsePathMap(pathMap)

	// security measure: limit the files within options.RootPath
	dir := http.Dir(options.RootPath)

	return func(c *routing.Context) error {
		if c.Request.Method != "GET" && c.Request.Method != "HEAD" {
			return routing.NewHTTPError(http.StatusMethodNotAllowed)
		}
		path, found := matchPath(c.Request.URL.Path, from, to)
		if !found || options.Allow != nil && !options.Allow(c, path) {
			return routing.NewHTTPError(http.StatusNotFound)
		}

		var (
			file  http.File
			fstat os.FileInfo
			err   error
		)

		if file, err = dir.Open(path); err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		}
		defer file.Close()

		if fstat, err = file.Stat(); err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		}

		if fstat.IsDir() {
			if options.IndexFile == "" {
				return routing.NewHTTPError(http.StatusNotFound)
			}
			path = filepath.Join(path, options.IndexFile)
			if file, err = dir.Open(path); err != nil {
				return routing.NewHTTPError(http.StatusNotFound, err.Error())
			}
			defer file.Close()

			fstat, err = file.Stat()

			if err != nil {
				return routing.NewHTTPError(http.StatusNotFound, err.Error())
			} else if fstat.IsDir() {
				return routing.NewHTTPError(http.StatusNotFound)
			}
		}

		http.ServeContent(c.Response, c.Request, path, fstat.ModTime(), file)
		return nil
	}
}

// Content returns a handler that serves the content of the specified file as the response.
// The file to be served can be specified as an absolute file path or a path relative to RootPath (which
// defaults to the current working path).
// If the specified file does not exist, the handler will pass the control to the next available handler.
func Content(path string) routing.Handler {
	if !filepath.IsAbs(path) {
		path = filepath.Join(RootPath, path)
	}
	return func(c *routing.Context) error {
		if c.Request.Method != "GET" && c.Request.Method != "HEAD" {
			return routing.NewHTTPError(http.StatusMethodNotAllowed)
		}
		file, err := os.Open(path)
		if err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		}
		defer file.Close()
		fstat, err := file.Stat()
		if err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		} else if fstat.IsDir() {
			return routing.NewHTTPError(http.StatusNotFound)
		}
		http.ServeContent(c.Response, c.Request, path, fstat.ModTime(), file)
		return nil
	}
}

func parsePathMap(pathMap PathMap) (from, to []string) {
	from = make([]string, len(pathMap))
	to = make([]string, len(pathMap))
	n := 0
	for i := range pathMap {
		from[n] = i
		n++
	}
	sort.Strings(from)
	for i, s := range from {
		to[i] = pathMap[s]
	}
	return
}

func matchPath(path string, from, to []string) (string, bool) {
	for i := len(from) - 1; i >= 0; i-- {
		prefix := from[i]
		if strings.HasPrefix(path, prefix) {
			return to[i] + path[len(prefix):], true
		}
	}
	return "", false
}
<commit_msg>fixed typo in doc<commit_after>// Copyright 2016 Qiang Xue. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package file provides handlers that serve static files for the ozzo routing package.
package file

import (
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/go-ozzo/ozzo-routing"
)

// ServerOptions defines the possible options for the Server handler.
type ServerOptions struct {
	// The path that all files to be served should be located within. The path map passed to the Server method
	// are all relative to this path. This property can be specified as an absolute file path or a path relative
	// to the current working path. If not set, this property defaults to the current working path.
	RootPath string
	// The file (e.g. index.html) to be served when the current request corresponds to a directory.
	// If not set, the handler will return a 404 HTTP error when the request corresponds to a directory.
	IndexFile string
	// A function that checks if the requested file path is allowed. If allowed, the function
	// may do additional work such as setting Expires HTTP header.
	// The function should return a boolean indicating whether the file should be served or not.
	// If false, a 404 HTTP error will be returned by the handler.
	Allow func(*routing.Context, string) bool
}

// PathMap specifies the mapping between URL paths (keys) and file paths (keys).
// The file paths are relative to Options.RootPath
type PathMap map[string]string

// RootPath stores the current working path
var RootPath string

func init() {
	RootPath, _ = os.Getwd()
}

// Server returns a handler that serves the files as the response content.
// The files being served are determined using the current URL path and the specified path map.
// For example, if the path map is {"/css": "/www/css", "/js": "/www/js"} and the current URL path
// "/css/main.css", the file "<working dir>/www/css/main.css" will be served.
// If a URL path matches multiple prefixes in the path map, the most specific prefix will take precedence.
// For example, if the path map contains both "/css" and "/css/img", and the URL path is "/css/img/logo.gif",
// then the path mapped by "/css/img" will be used.
//
//     import (
//         "log"
//         "github.com/go-ozzo/ozzo-routing"
//         "github.com/go-ozzo/ozzo-routing/file"
//     )
//
//     r := routing.New()
//     r.Get("/*", file.Server(file.PathMap{
//          "/css": "/ui/dist/css",
//          "/js": "/ui/dist/js",
//     }))
func Server(pathMap PathMap, opts ...ServerOptions) routing.Handler {
	var options ServerOptions
	if len(opts) > 0 {
		options = opts[0]
	}
	if !filepath.IsAbs(options.RootPath) {
		options.RootPath = filepath.Join(RootPath, options.RootPath)
	}
	from, to := parsePathMap(pathMap)

	// security measure: limit the files within options.RootPath
	dir := http.Dir(options.RootPath)

	return func(c *routing.Context) error {
		if c.Request.Method != "GET" && c.Request.Method != "HEAD" {
			return routing.NewHTTPError(http.StatusMethodNotAllowed)
		}
		path, found := matchPath(c.Request.URL.Path, from, to)
		if !found || options.Allow != nil && !options.Allow(c, path) {
			return routing.NewHTTPError(http.StatusNotFound)
		}

		var (
			file  http.File
			fstat os.FileInfo
			err   error
		)

		if file, err = dir.Open(path); err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		}
		defer file.Close()

		if fstat, err = file.Stat(); err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		}

		if fstat.IsDir() {
			if options.IndexFile == "" {
				return routing.NewHTTPError(http.StatusNotFound)
			}
			path = filepath.Join(path, options.IndexFile)
			if file, err = dir.Open(path); err != nil {
				return routing.NewHTTPError(http.StatusNotFound, err.Error())
			}
			defer file.Close()

			fstat, err = file.Stat()

			if err != nil {
				return routing.NewHTTPError(http.StatusNotFound, err.Error())
			} else if fstat.IsDir() {
				return routing.NewHTTPError(http.StatusNotFound)
			}
		}

		http.ServeContent(c.Response, c.Request, path, fstat.ModTime(), file)
		return nil
	}
}

// Content returns a handler that serves the content of the specified file as the response.
// The file to be served can be specified as an absolute file path or a path relative to RootPath (which
// defaults to the current working path).
// If the specified file does not exist, the handler will pass the control to the next available handler.
func Content(path string) routing.Handler {
	if !filepath.IsAbs(path) {
		path = filepath.Join(RootPath, path)
	}
	return func(c *routing.Context) error {
		if c.Request.Method != "GET" && c.Request.Method != "HEAD" {
			return routing.NewHTTPError(http.StatusMethodNotAllowed)
		}
		file, err := os.Open(path)
		if err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		}
		defer file.Close()
		fstat, err := file.Stat()
		if err != nil {
			return routing.NewHTTPError(http.StatusNotFound, err.Error())
		} else if fstat.IsDir() {
			return routing.NewHTTPError(http.StatusNotFound)
		}
		http.ServeContent(c.Response, c.Request, path, fstat.ModTime(), file)
		return nil
	}
}

func parsePathMap(pathMap PathMap) (from, to []string) {
	from = make([]string, len(pathMap))
	to = make([]string, len(pathMap))
	n := 0
	for i := range pathMap {
		from[n] = i
		n++
	}
	sort.Strings(from)
	for i, s := range from {
		to[i] = pathMap[s]
	}
	return
}

func matchPath(path string, from, to []string) (string, bool) {
	for i := len(from) - 1; i >= 0; i-- {
		prefix := from[i]
		if strings.HasPrefix(path, prefix) {
			return to[i] + path[len(prefix):], true
		}
	}
	return "", false
}
<|endoftext|>


<commit_before>package zipfs

import (
	"archive/zip"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"
)

var (
	errNotImplemented   = errors.New("not implemented yet")
	errFileClosed       = errors.New("file closed")
	errFileSystemClosed = errors.New("filesystem closed")
	errNotDirectory     = errors.New("not a directory")
	errDirectory        = errors.New("is a directory")
)

// FileSystem is a file system based on a ZIP file.
// It implements the http.FileSystem interface.
type FileSystem struct {
	readerAt  io.ReaderAt
	reader    *zip.Reader
	closer    io.Closer
	fileInfos fileInfoMap
}

// New will open the Zip file specified by name and
// return a new FileSystem based on that Zip file.
func New(name string) (*FileSystem, error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}
	zipReader, err := zip.NewReader(file, fi.Size())
	if err != nil {
		return nil, err
	}
	fs := &FileSystem{
		closer:    file,
		readerAt:  file,
		reader:    zipReader,
		fileInfos: fileInfoMap{},
	}

	// Build a map of file paths to speed lookup.
	// Note that this assumes that there are not a very
	// large number of files in the ZIP file.
	//
	// Because we iterate through the map it seems reasonable
	// to attach each fileInfo to it's parent directory. Once again,
	// reasonable if the ZIP file does not contain a very large number
	// of entries.
	for _, zf := range fs.reader.File {
		fi := fs.fileInfos.FindOrCreate(zf.Name)
		fi.zipFile = zf
		fiParent := fs.fileInfos.FindOrCreateParent(zf.Name)
		fiParent.fileInfos = append(fiParent.fileInfos, fi)
	}

	for _, fi := range fs.fileInfos {
		if len(fi.fileInfos) > 1 {
			sort.Sort(fi.fileInfos)
		}
	}

	return fs, nil
}

// Open implements the http.FileSystem interface.
// A http.File is returned, which can be served by
// the http.FileServer implementation.
func (fs *FileSystem) Open(name string) (http.File, error) {
	fi, err := fs.openFileInfo(name)
	if err != nil {
		return nil, err
	}

	return fi.openReader(name), nil
}

// Close closes the file system's underlying ZIP file and
// releases all memory allocated to internal data structures.
func (fs *FileSystem) Close() error {
	fs.reader = nil
	fs.readerAt = nil
	var err error
	if fs.closer != nil {
		err = fs.closer.Close()
		fs.closer = nil
	}
	fs.fileInfos = nil
	return err
}

type fileInfoList []*fileInfo

func (fl fileInfoList) Len() int {
	return len(fl)
}

func (fl fileInfoList) Less(i, j int) bool {
	name1 := fl[i].Name()
	name2 := fl[j].Name()
	return name1 < name2
}

func (fl fileInfoList) Swap(i, j int) {
	fi := fl[i]
	fl[i] = fl[j]
	fl[j] = fi
}

func (fs *FileSystem) openFileInfo(name string) (*fileInfo, error) {
	if fs.readerAt == nil {
		return nil, errFileSystemClosed
	}
	name = path.Clean(name)
	trimmedName := strings.TrimLeft(name, "/")
	fi := fs.fileInfos[trimmedName]
	if fi == nil {
		return nil, &os.PathError{Op: "Open", Path: name, Err: os.ErrNotExist}
	}

	return fi, nil
}

// fileMap keeps track of fileInfos
type fileInfoMap map[string]*fileInfo

func (fm fileInfoMap) FindOrCreate(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	fi := fm[name]
	if fi == nil {
		fi = &fileInfo{
			name: name,
		}
		fm[name] = fi
		if strippedName != name {
			// directories get two entries: with and without trailing slash
			fm[strippedName] = fi
		}
	}
	return fi
}

func (fm fileInfoMap) FindOrCreateParent(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	dirName := path.Dir(strippedName)
	if dirName == "." {
		dirName = "/"
	} else if !strings.HasSuffix(dirName, "/") {
		dirName = dirName + "/"
	}
	return fm.FindOrCreate(dirName)
}

// fileInfo implements the os.FileInfo interface.
type fileInfo struct {
	name      string
	fs        *FileSystem
	zipFile   *zip.File
	fileInfos fileInfoList
	tempPath  string
	mutex     sync.Mutex
}

func (fi *fileInfo) Name() string {
	return path.Base(fi.name)
}

func (fi *fileInfo) Size() int64 {
	if fi.zipFile == nil {
		return 0
	}
	if fi.zipFile.UncompressedSize64 == 0 {
		return int64(fi.zipFile.UncompressedSize)
	}
	return int64(fi.zipFile.UncompressedSize64)
}

func (fi *fileInfo) Mode() os.FileMode {
	if fi.zipFile == nil || fi.IsDir() {
		return 0555 | os.ModeDir
	}
	return 0444
}

var dirTime = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC)

func (fi *fileInfo) ModTime() time.Time {
	if fi.zipFile == nil {
		return dirTime
	}
	return fi.zipFile.ModTime()
}

func (fi *fileInfo) IsDir() bool {
	if fi.zipFile == nil {
		return true
	}
	return fi.zipFile.Mode().IsDir()
}

func (fi *fileInfo) Sys() interface{} {
	return fi.zipFile
}

func (fi *fileInfo) openReader(name string) *fileReader {
	return &fileReader{
		fileInfo: fi,
		name:     name,
	}
}

func (fi *fileInfo) readdir() ([]os.FileInfo, error) {
	if !fi.Mode().IsDir() {
		return nil, errNotDirectory
	}

	v := make([]os.FileInfo, len(fi.fileInfos))
	for i, fi := range fi.fileInfos {
		v[i] = fi
	}
	return v, nil
}

type fileReader struct {
	name     string // the name used to open
	fileInfo *fileInfo
	reader   io.ReadCloser
	file     *os.File
	closed   bool
	readdir  []os.FileInfo
}

func (f *fileReader) Close() error {
	var errs []error
	if f.reader != nil {
		err := f.reader.Close()
		errs = append(errs, err)
	}
	var tempFile string
	if f.file != nil {
		tempFile = f.file.Name()
		err := f.file.Close()
		errs = append(errs, err)
	}
	if tempFile != "" {
		err := os.Remove(tempFile)
		errs = append(errs, err)
	}

	f.closed = true

	for _, err := range errs {
		if err != nil {
			return f.pathError("Close", err)
		}
	}
	return nil
}

func (f *fileReader) Read(p []byte) (n int, err error) {
	if f.closed {
		return 0, f.pathError("Read", errFileClosed)
	}
	if f.file != nil {
		return f.file.Read(p)
	}
	if f.reader == nil {
		f.reader, err = f.fileInfo.zipFile.Open()
		if err != nil {
			return 0, err
		}
	}
	return f.reader.Read(p)
}

func (f *fileReader) Seek(offset int64, whence int) (int64, error) {
	if f.closed {
		return 0, f.pathError("Seek", errFileClosed)
	}

	// The reader cannot seek, so close it.
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return 0, err
		}
	}

	// A special case for when there is no file created and the seek is
	// to the beginning of the file. Just open (or re-open) the reader
	// at the beginning of the file.
	if f.file == nil && offset == 0 && whence == 0 {
		var err error
		f.reader, err = f.fileInfo.zipFile.Open()
		return 0, err
	}

	if err := f.createTempFile(); err != nil {
		return 0, err
	}

	return f.file.Seek(offset, whence)
}

func (f *fileReader) Readdir(count int) ([]os.FileInfo, error) {
	var err error
	var osFileInfos []os.FileInfo

	if count > 0 {
		if f.readdir == nil {
			f.readdir, err = f.fileInfo.readdir()
			if err != nil {
				return nil, f.pathError("Readdir", err)
			}
		}
		if len(f.readdir) >= count {
			osFileInfos = f.readdir[0:count]
			f.readdir = f.readdir[count:]
		} else {
			osFileInfos = f.readdir
			f.readdir = nil
			err = io.EOF
		}
	} else {
		osFileInfos, err = f.fileInfo.readdir()
		if err != nil {
			return nil, f.pathError("Readdir", err)
		}
	}

	return osFileInfos, err
}

func (f *fileReader) Stat() (os.FileInfo, error) {
	return f.fileInfo, nil
}

func (f *fileReader) createTempFile() error {
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return err
		}
		f.reader = nil
	}
	if f.file == nil {
		// Open a file that contains the contents of the zip file.
		osFile, err := createTempFile(f.fileInfo.zipFile)
		if err != nil {
			return err
		}

		f.file = osFile
	}
	return nil
}

func (f *fileReader) pathError(op string, err error) error {
	return &os.PathError{
		Op:   op,
		Path: f.name,
		Err:  err,
	}
}

// createTempFile creates a temporary file with the contents of the
// zip file. Used to implement io.Seeker interface.
func createTempFile(f *zip.File) (*os.File, error) {
	reader, err := f.Open()
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	tempFile, err := ioutil.TempFile("", "zipfs")
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(tempFile, reader)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}
	_, err = tempFile.Seek(0, os.SEEK_SET)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}

	return tempFile, nil
}
<commit_msg>Review and comments.<commit_after>package zipfs

import (
	"archive/zip"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"time"
)

var (
	errFileClosed       = errors.New("file closed")
	errFileSystemClosed = errors.New("filesystem closed")
	errNotDirectory     = errors.New("not a directory")
	errDirectory        = errors.New("is a directory")
)

// FileSystem is a file system based on a ZIP file.
// It implements the http.FileSystem interface.
type FileSystem struct {
	readerAt  io.ReaderAt
	closer    io.Closer
	reader    *zip.Reader
	fileInfos fileInfoMap
}

// New will open the Zip file specified by name and
// return a new FileSystem based on that Zip file.
func New(name string) (*FileSystem, error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	fi, err := file.Stat()
	if err != nil {
		return nil, err
	}
	zipReader, err := zip.NewReader(file, fi.Size())
	if err != nil {
		return nil, err
	}

	// Separate the file into an io.ReaderAt and an io.Closer.
	// Earlier versions of the code allowed for opening a filesystem
	// just with an io.ReaderAt. Not also that thw zip.Reader is
	// not actually used outside of this function so it probably
	// does not need to be in the FileSystem structure. Keeping it
	// there for now but may remove it in future.
	fs := &FileSystem{
		closer:    file,
		readerAt:  file,
		reader:    zipReader,
		fileInfos: fileInfoMap{},
	}

	// Build a map of file paths to speed lookup.
	// Note that this assumes that there are not a very
	// large number of files in the ZIP file.
	//
	// Because we iterate through the map it seems reasonable
	// to attach each fileInfo to it's parent directory. Once again,
	// reasonable if the ZIP file does not contain a very large number
	// of entries.
	for _, zf := range fs.reader.File {
		fi := fs.fileInfos.FindOrCreate(zf.Name)
		fi.zipFile = zf
		fiParent := fs.fileInfos.FindOrCreateParent(zf.Name)
		fiParent.fileInfos = append(fiParent.fileInfos, fi)
	}

	// Sort all of the list of fileInfos in each directory.
	for _, fi := range fs.fileInfos {
		if len(fi.fileInfos) > 1 {
			sort.Sort(fi.fileInfos)
		}
	}

	return fs, nil
}

// Open implements the http.FileSystem interface.
// A http.File is returned, which can be served by
// the http.FileServer implementation.
func (fs *FileSystem) Open(name string) (http.File, error) {
	fi, err := fs.openFileInfo(name)
	if err != nil {
		return nil, err
	}

	return fi.openReader(name), nil
}

// Close closes the file system's underlying ZIP file and
// releases all memory allocated to internal data structures.
func (fs *FileSystem) Close() error {
	fs.reader = nil
	fs.readerAt = nil
	var err error
	if fs.closer != nil {
		err = fs.closer.Close()
		fs.closer = nil
	}
	fs.fileInfos = nil
	return err
}

type fileInfoList []*fileInfo

func (fl fileInfoList) Len() int {
	return len(fl)
}

func (fl fileInfoList) Less(i, j int) bool {
	name1 := fl[i].Name()
	name2 := fl[j].Name()
	return name1 < name2
}

func (fl fileInfoList) Swap(i, j int) {
	fi := fl[i]
	fl[i] = fl[j]
	fl[j] = fi
}

func (fs *FileSystem) openFileInfo(name string) (*fileInfo, error) {
	if fs.readerAt == nil {
		return nil, errFileSystemClosed
	}
	name = path.Clean(name)
	trimmedName := strings.TrimLeft(name, "/")
	fi := fs.fileInfos[trimmedName]
	if fi == nil {
		return nil, &os.PathError{Op: "Open", Path: name, Err: os.ErrNotExist}
	}

	return fi, nil
}

// fileMap keeps track of fileInfos
type fileInfoMap map[string]*fileInfo

func (fm fileInfoMap) FindOrCreate(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	fi := fm[name]
	if fi == nil {
		fi = &fileInfo{
			name: name,
		}
		fm[name] = fi
		if strippedName != name {
			// directories get two entries: with and without trailing slash
			fm[strippedName] = fi
		}
	}
	return fi
}

func (fm fileInfoMap) FindOrCreateParent(name string) *fileInfo {
	strippedName := strings.TrimRight(name, "/")
	dirName := path.Dir(strippedName)
	if dirName == "." {
		dirName = "/"
	} else if !strings.HasSuffix(dirName, "/") {
		dirName = dirName + "/"
	}
	return fm.FindOrCreate(dirName)
}

// fileInfo implements the os.FileInfo interface.
type fileInfo struct {
	name      string
	fs        *FileSystem
	zipFile   *zip.File
	fileInfos fileInfoList
	tempPath  string
	mutex     sync.Mutex
}

func (fi *fileInfo) Name() string {
	return path.Base(fi.name)
}

func (fi *fileInfo) Size() int64 {
	if fi.zipFile == nil {
		return 0
	}
	if fi.zipFile.UncompressedSize64 == 0 {
		return int64(fi.zipFile.UncompressedSize)
	}
	return int64(fi.zipFile.UncompressedSize64)
}

func (fi *fileInfo) Mode() os.FileMode {
	if fi.zipFile == nil || fi.IsDir() {
		return 0555 | os.ModeDir
	}
	return 0444
}

var dirTime = time.Date(2001, 1, 1, 0, 0, 0, 0, time.UTC)

func (fi *fileInfo) ModTime() time.Time {
	if fi.zipFile == nil {
		return dirTime
	}
	return fi.zipFile.ModTime()
}

func (fi *fileInfo) IsDir() bool {
	if fi.zipFile == nil {
		return true
	}
	return fi.zipFile.Mode().IsDir()
}

func (fi *fileInfo) Sys() interface{} {
	return fi.zipFile
}

func (fi *fileInfo) openReader(name string) *fileReader {
	return &fileReader{
		fileInfo: fi,
		name:     name,
	}
}

func (fi *fileInfo) readdir() ([]os.FileInfo, error) {
	if !fi.Mode().IsDir() {
		return nil, errNotDirectory
	}

	v := make([]os.FileInfo, len(fi.fileInfos))
	for i, fi := range fi.fileInfos {
		v[i] = fi
	}
	return v, nil
}

type fileReader struct {
	name     string // the name used to open
	fileInfo *fileInfo
	reader   io.ReadCloser
	file     *os.File
	closed   bool
	readdir  []os.FileInfo
}

func (f *fileReader) Close() error {
	var errs []error
	if f.reader != nil {
		err := f.reader.Close()
		errs = append(errs, err)
	}
	var tempFile string
	if f.file != nil {
		tempFile = f.file.Name()
		err := f.file.Close()
		errs = append(errs, err)
	}
	if tempFile != "" {
		err := os.Remove(tempFile)
		errs = append(errs, err)
	}

	f.closed = true

	for _, err := range errs {
		if err != nil {
			return f.pathError("Close", err)
		}
	}
	return nil
}

func (f *fileReader) Read(p []byte) (n int, err error) {
	if f.closed {
		return 0, f.pathError("Read", errFileClosed)
	}
	if f.file != nil {
		return f.file.Read(p)
	}
	if f.reader == nil {
		f.reader, err = f.fileInfo.zipFile.Open()
		if err != nil {
			return 0, err
		}
	}
	return f.reader.Read(p)
}

func (f *fileReader) Seek(offset int64, whence int) (int64, error) {
	if f.closed {
		return 0, f.pathError("Seek", errFileClosed)
	}

	// The reader cannot seek, so close it.
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return 0, err
		}
	}

	// A special case for when there is no file created and the seek is
	// to the beginning of the file. Just open (or re-open) the reader
	// at the beginning of the file.
	if f.file == nil && offset == 0 && whence == 0 {
		var err error
		f.reader, err = f.fileInfo.zipFile.Open()
		return 0, err
	}

	if err := f.createTempFile(); err != nil {
		return 0, err
	}

	return f.file.Seek(offset, whence)
}

func (f *fileReader) Readdir(count int) ([]os.FileInfo, error) {
	var err error
	var osFileInfos []os.FileInfo

	if count > 0 {
		if f.readdir == nil {
			f.readdir, err = f.fileInfo.readdir()
			if err != nil {
				return nil, f.pathError("Readdir", err)
			}
		}
		if len(f.readdir) >= count {
			osFileInfos = f.readdir[0:count]
			f.readdir = f.readdir[count:]
		} else {
			osFileInfos = f.readdir
			f.readdir = nil
			err = io.EOF
		}
	} else {
		osFileInfos, err = f.fileInfo.readdir()
		if err != nil {
			return nil, f.pathError("Readdir", err)
		}
	}

	return osFileInfos, err
}

func (f *fileReader) Stat() (os.FileInfo, error) {
	return f.fileInfo, nil
}

func (f *fileReader) createTempFile() error {
	if f.reader != nil {
		if err := f.reader.Close(); err != nil {
			return err
		}
		f.reader = nil
	}
	if f.file == nil {
		// Open a file that contains the contents of the zip file.
		osFile, err := createTempFile(f.fileInfo.zipFile)
		if err != nil {
			return err
		}

		f.file = osFile
	}
	return nil
}

func (f *fileReader) pathError(op string, err error) error {
	return &os.PathError{
		Op:   op,
		Path: f.name,
		Err:  err,
	}
}

// createTempFile creates a temporary file with the contents of the
// zip file. Used to implement io.Seeker interface.
func createTempFile(f *zip.File) (*os.File, error) {
	reader, err := f.Open()
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	tempFile, err := ioutil.TempFile("", "zipfs")
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(tempFile, reader)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}
	_, err = tempFile.Seek(0, os.SEEK_SET)
	if err != nil {
		tempFile.Close()
		os.Remove(tempFile.Name())
		return nil, err
	}

	return tempFile, nil
}
<|endoftext|>


<commit_before>package isolated

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("table alignment", func() {
	BeforeEach(func() {
		helpers.LoginCF()
	})

	When("output is in English", func() {
		BeforeEach(func() {
			helpers.LoginCF()
			helpers.TargetOrgAndSpace(ReadOnlyOrg, ReadOnlySpace)
		})

		// Developer note: The spacing in this test is significant and explicit. Do
		// not replace with a regex.
		It("aligns the table correctly", func() {
			username, _ := helpers.GetCredentials()
			session := helpers.CF("target")
			Eventually(session).Should(Say("API endpoint:   %s", apiURL))
			Eventually(session).Should(Say(`API version:    [\d.]+`))
			Eventually(session).Should(Say("user:           %s", username))
			Eventually(session).Should(Say("org:            %s", ReadOnlyOrg))
			Eventually(session).Should(Say("space:          %s", ReadOnlySpace))
			Eventually(session).Should(Exit(0))
		})
	})

	When("output is in language with multibyte characters", func() {
		BeforeEach(func() {
			helpers.LoginCF()
			helpers.TargetOrgAndSpace(ReadOnlyOrg, ReadOnlySpace)
		})

		// Developer note: The spacing in this test is significant and explicit. Do
		// not replace with a regex.
		It("aligns the table correctly", func() {
			username, _ := helpers.GetCredentials()
			session := helpers.CFWithEnv(map[string]string{"LANG": "ja-JP.utf8"}, "target")
			Eventually(session).Should(Say("API エンドポイント:   %s", apiURL))
			// TODO: "version" here should be translated for all languages. We have translation resources for "api version"
			// (lowercase), which is what this said in V6, but we don't yet have them for "API version" (uppercase).
			Eventually(session).Should(Say(`API version:      [\d.]+`))
			Eventually(session).Should(Say("ユーザー:            %s", username))
			Eventually(session).Should(Say("組織:                %s", ReadOnlyOrg))
			Eventually(session).Should(Say("スペース:            %s", ReadOnlySpace))
			Eventually(session).Should(Exit(0))
		})
	})
})
<commit_msg>🐞: Table alignment test has correct alignment<commit_after>package isolated

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("table alignment", func() {
	BeforeEach(func() {
		helpers.LoginCF()
	})

	When("output is in English", func() {
		BeforeEach(func() {
			helpers.LoginCF()
			helpers.TargetOrgAndSpace(ReadOnlyOrg, ReadOnlySpace)
		})

		// Developer note: The spacing in this test is significant and explicit. Do
		// not replace with a regex.
		It("aligns the table correctly", func() {
			username, _ := helpers.GetCredentials()
			session := helpers.CF("target")
			Eventually(session).Should(Say("API endpoint:   %s", apiURL))
			Eventually(session).Should(Say(`API version:    [\d.]+`))
			Eventually(session).Should(Say("user:           %s", username))
			Eventually(session).Should(Say("org:            %s", ReadOnlyOrg))
			Eventually(session).Should(Say("space:          %s", ReadOnlySpace))
			Eventually(session).Should(Exit(0))
		})
	})

	When("output is in language with multibyte characters", func() {
		BeforeEach(func() {
			helpers.LoginCF()
			helpers.TargetOrgAndSpace(ReadOnlyOrg, ReadOnlySpace)
		})

		// Developer note: The spacing in this test is significant and explicit. Do
		// not replace with a regex.
		It("aligns the table correctly", func() {
			username, _ := helpers.GetCredentials()
			session := helpers.CFWithEnv(map[string]string{"LANG": "ja-JP.utf8"}, "target")
			Eventually(session).Should(Say("API エンドポイント:   %s", apiURL))
			// TODO: "version" here should be translated for all languages. We have translation resources for "api version"
			// (lowercase), which is what this said in V6, but we don't yet have them for "API version" (uppercase).
			Eventually(session).Should(Say(`API version:          [\d.]+`))
			Eventually(session).Should(Say("ユーザー:            %s", username))
			Eventually(session).Should(Say("組織:                %s", ReadOnlyOrg))
			Eventually(session).Should(Say("スペース:            %s", ReadOnlySpace))
			Eventually(session).Should(Exit(0))
		})
	})
})
<|endoftext|>


<commit_before>package gopcap

import (
	"bytes"
	"io"
	"time"
)

// checkMagicNum checks the first four bytes of a pcap file, searching for the magic number
// and checking the byte order. Returns three values: whether the file is a pcap file, whether
// the byte order needs flipping, and any error that was encountered. If error is returned,
// the other values are invalid.
func checkMagicNum(src io.Reader) (bool, bool, error) {
	// These magic numbers form the header of a pcap file.
	magic := []byte{0xa1, 0xb2, 0xc3, 0xd4}
	magic_reverse := []byte{0xd4, 0xc3, 0xb2, 0xa1}

	buffer := make([]byte, 4)
	read_count, err := src.Read(buffer)

	if read_count != 4 {
		return false, false, InsufficientLength
	}
	if (err != nil) && (err != io.EOF) {
		return false, false, err
	}

	if bytes.Compare(buffer, magic) == 0 {
		return true, false, nil
	} else if bytes.Compare(buffer, magic_reverse) == 0 {
		return true, true, nil
	}

	return false, false, NotAPcapFile
}

// parsePacket parses a full packet out of the pcap file. It returns an error if any problems were
// encountered.
func parsePacket(pkt *Packet, src io.Reader, flipped bool) error {
	err := populatePacketHeader(pkt, src, flipped)

	if err != nil {
		return err
	}

	data := make([]byte, pkt.IncludedLen)
	readlen, err := src.Read(data)
	pkt.Data = data

	if uint32(readlen) != pkt.IncludedLen {
		err = UnexpectedEOF
	}

	return err
}

// populateFileHeader reads the next 20 bytes out of the .pcap file and uses it to populate the
// PcapFile structure.
func populateFileHeader(file *PcapFile, src io.Reader, flipped bool) error {
	buffer := make([]byte, 20)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 20 {
		return InsufficientLength
	}

	// First two bytes are the major version number.
	file.MajorVersion = getUint16(buffer[0:2], flipped)

	// Next two are the minor version number.
	file.MinorVersion = getUint16(buffer[2:4], flipped)

	// GMT to local correction, in seconds east of UTC.
	file.TZCorrection = getInt32(buffer[4:8], flipped)

	// Next is the number of significant figures in the timestamps. Almost always zero.
	file.SigFigs = getUint32(buffer[8:12], flipped)

	// Now the maximum length of the captured packet data.
	file.MaxLen = getUint32(buffer[12:16], flipped)

	// And the link type.
	file.LinkType = Link(getUint32(buffer[16:20], flipped))

	return nil
}

// populatePacketHeader reads the next 16 bytes out of the file and builds it into a
// packet header.
func populatePacketHeader(packet *Packet, src io.Reader, flipped bool) error {
	buffer := make([]byte, 16)
	read_count, err := src.Read(buffer)

	if read_count != 16 {
		return InsufficientLength
	}

	// First is a pair of fields that build up the timestamp.
	ts_seconds := getUint32(buffer[0:4], flipped)
	ts_micros := getUint32(buffer[4:8], flipped)
	packet.Timestamp = (time.Duration(ts_seconds) * time.Second) + (time.Duration(ts_micros) * time.Microsecond)

	// Next is the length of the data segment.
	packet.IncludedLen = getUint32(buffer[8:12], flipped)

	// Then the original length of the packet.
	packet.ActualLen = getUint32(buffer[12:16], flipped)

	return err
}
<commit_msg>Catch IO errors too.<commit_after>package gopcap

import (
	"bytes"
	"io"
	"time"
)

// checkMagicNum checks the first four bytes of a pcap file, searching for the magic number
// and checking the byte order. Returns three values: whether the file is a pcap file, whether
// the byte order needs flipping, and any error that was encountered. If error is returned,
// the other values are invalid.
func checkMagicNum(src io.Reader) (bool, bool, error) {
	// These magic numbers form the header of a pcap file.
	magic := []byte{0xa1, 0xb2, 0xc3, 0xd4}
	magic_reverse := []byte{0xd4, 0xc3, 0xb2, 0xa1}

	buffer := make([]byte, 4)
	read_count, err := src.Read(buffer)

	if read_count != 4 {
		return false, false, InsufficientLength
	}
	if (err != nil) && (err != io.EOF) {
		return false, false, err
	}

	if bytes.Compare(buffer, magic) == 0 {
		return true, false, nil
	} else if bytes.Compare(buffer, magic_reverse) == 0 {
		return true, true, nil
	}

	return false, false, NotAPcapFile
}

// parsePacket parses a full packet out of the pcap file. It returns an error if any problems were
// encountered.
func parsePacket(pkt *Packet, src io.Reader, flipped bool) error {
	err := populatePacketHeader(pkt, src, flipped)

	if err != nil {
		return err
	}

	data := make([]byte, pkt.IncludedLen)
	readlen, err := src.Read(data)
	pkt.Data = data

	if uint32(readlen) != pkt.IncludedLen {
		err = UnexpectedEOF
	}

	return err
}

// populateFileHeader reads the next 20 bytes out of the .pcap file and uses it to populate the
// PcapFile structure.
func populateFileHeader(file *PcapFile, src io.Reader, flipped bool) error {
	buffer := make([]byte, 20)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 20 {
		return InsufficientLength
	}

	// First two bytes are the major version number.
	file.MajorVersion = getUint16(buffer[0:2], flipped)

	// Next two are the minor version number.
	file.MinorVersion = getUint16(buffer[2:4], flipped)

	// GMT to local correction, in seconds east of UTC.
	file.TZCorrection = getInt32(buffer[4:8], flipped)

	// Next is the number of significant figures in the timestamps. Almost always zero.
	file.SigFigs = getUint32(buffer[8:12], flipped)

	// Now the maximum length of the captured packet data.
	file.MaxLen = getUint32(buffer[12:16], flipped)

	// And the link type.
	file.LinkType = Link(getUint32(buffer[16:20], flipped))

	return nil
}

// populatePacketHeader reads the next 16 bytes out of the file and builds it into a
// packet header.
func populatePacketHeader(packet *Packet, src io.Reader, flipped bool) error {
	buffer := make([]byte, 16)
	read_count, err := src.Read(buffer)

	if err != nil {
		return err
	} else if read_count != 16 {
		return InsufficientLength
	}

	// First is a pair of fields that build up the timestamp.
	ts_seconds := getUint32(buffer[0:4], flipped)
	ts_micros := getUint32(buffer[4:8], flipped)
	packet.Timestamp = (time.Duration(ts_seconds) * time.Second) + (time.Duration(ts_micros) * time.Microsecond)

	// Next is the length of the data segment.
	packet.IncludedLen = getUint32(buffer[8:12], flipped)

	// Then the original length of the packet.
	packet.ActualLen = getUint32(buffer[12:16], flipped)

	return err
}
<|endoftext|>


<commit_before>package shells

import (
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
)

type AbstractShell struct {
}

func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
	features.Artifacts = true
	features.Cache = true
}

func (b *AbstractShell) GetSupportedOptions() []string {
	return []string{"artifacts", "cache", "dependencies"}
}

func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
	w.Cd(info.Build.FullProjectDir())
}

func (b *AbstractShell) writeExports(w ShellWriter, info common.ShellScriptInfo) {
	for _, variable := range info.Build.GetAllVariables() {
		w.Variable(variable)
	}
}

func (b *AbstractShell) writeTLSCAInfo(w ShellWriter, build *common.Build, key string) {
	if build.TLSCAChain != "" {
		w.Variable(common.BuildVariable{
			Key:      key,
			Value:    build.TLSCAChain,
			Public:   true,
			Internal: true,
			File:     true,
		})
	}
}

func (b *AbstractShell) writeCloneCmd(w ShellWriter, build *common.Build, projectDir string) {
	w.Notice("Cloning repository...")
	w.RmDir(projectDir)
	w.Command("git", "clone", build.RepoURL, projectDir)
	w.Cd(projectDir)
}

func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projectDir string, gitDir string) {
	w.IfDirectory(gitDir)
	w.Notice("Fetching changes...")
	w.Cd(projectDir)
	w.Command("git", "clean", "-ffdx")
	w.Command("git", "reset", "--hard")
	w.Command("git", "remote", "set-url", "origin", build.RepoURL)
	w.Command("git", "fetch", "origin", "--prune", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*")
	w.Else()
	b.writeCloneCmd(w, build, projectDir)
	w.EndIf()
}

func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
	w.Notice("Checking out %s as %s...", build.Sha[0:8], build.RefName)
	// We remove a git index file, this is required if `git checkout` is terminated
	w.RmFile(".git/index.lock")
	w.Command("git", "checkout", build.Sha)
}

func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, file string) {
	// For tags we don't create cache
	if build.Tag || build.CacheDir == "" {
		return
	}

	// Deduce cache key
	key = path.Join(build.Name, build.RefName)
	if userKey != "" {
		key = build.GetAllVariables().ExpandValue(userKey)
	}

	// Ignore cache without the key
	if key == "" {
		return
	}

	file = path.Join(build.CacheDir, key, "cache.zip")
	file, err := filepath.Rel(build.BuildDir, file)
	if err != nil {
		return "", ""
	}
	return
}

func (o *archivingOptions) CommandArguments() (args []string) {
	for _, path := range o.Paths {
		args = append(args, "--path", path)
	}

	if o.Untracked {
		args = append(args, "--untracked")
	}
	return
}

func (b *AbstractShell) cacheExtractor(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.RunnerCommand == "" {
		w.Warning("The cache is not supported in this executor.")
		return
	}

	// Skip restoring cache if no cache is defined
	if archiverArgs := options.CommandArguments(); len(archiverArgs) == 0 {
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-extractor",
		"--file", cacheFile,
	}

	// Generate cache download address
	if url := getCacheDownloadURL(info.Build, cacheKey); url != "" {
		args = append(args, "--url", url)
	}

	// Execute archive command
	w.Notice("Checking cache for %s...", cacheKey)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) downloadArtifacts(w ShellWriter, build *common.BuildInfo, info common.ShellScriptInfo) {
	if info.RunnerCommand == "" {
		w.Warning("The artifacts downloading is not supported in this executor.")
		return
	}

	args := []string{
		"artifacts-downloader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		build.Token,
		"--id",
		strconv.Itoa(build.ID),
	}

	w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) downloadAllArtifacts(w ShellWriter, dependencies *dependencies, info common.ShellScriptInfo) {
	for _, otherBuild := range info.Build.DependsOnBuilds {
		if otherBuild.Artifacts == nil || otherBuild.Artifacts.Filename == "" {
			continue
		}
		if !dependencies.IsDependent(otherBuild.Name) {
			continue
		}
		b.downloadArtifacts(w, &otherBuild, info)
	}
}

func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptInfo) {
	b.writeExports(w, info)

	build := info.Build
	projectDir := build.FullProjectDir()
	gitDir := path.Join(build.FullProjectDir(), ".git")

	b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	if build.AllowGitFetch {
		b.writeFetchCmd(w, build, projectDir, gitDir)
	} else {
		b.writeCloneCmd(w, build, projectDir)
	}

	b.writeCheckoutCmd(w, build)

	// Parse options
	var options shellOptions
	info.Build.Options.Decode(&options)

	// Try to restore from main cache, if not found cache for master
	b.cacheExtractor(w, options.Cache, info)

	// Process all artifacts
	b.downloadAllArtifacts(w, options.Dependencies, info)
}

func (b *AbstractShell) GenerateCommands(w ShellWriter, info common.ShellScriptInfo) {
	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)

	commands := info.Build.Commands
	commands = strings.TrimSpace(commands)
	for _, command := range strings.Split(commands, "\n") {
		command = strings.TrimSpace(command)
		if command != "" {
			w.Notice("$ %s", command)
		} else {
			w.EmptyLine()
		}
		w.Line(command)
	}
}

func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.RunnerCommand == "" {
		w.Warning("The cache is not supported in this executor.")
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-archiver",
		"--file", cacheFile,
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Generate cache upload address
	if url := getCacheUploadURL(info.Build, cacheKey); url != "" {
		args = append(args, "--url", url)
	}

	// Execute archive command
	w.Notice("Creating cache %s...", cacheKey)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.Build.Runner.URL == "" {
		return
	}
	if info.RunnerCommand == "" {
		w.Warning("The artifacts uploading is not supported in this executor.")
		return
	}

	args := []string{
		"artifacts-uploader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		info.Build.Token,
		"--id",
		strconv.Itoa(info.Build.ID),
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Get artifacts:name
	if name, ok := helpers.GetMapKey(info.Build.Options["artifacts"], "name"); ok {
		if nameValue, ok := name.(string); ok && nameValue != "" {
			args = append(args, "--name", nameValue)
		}
	}

	w.Notice("Uploading artifacts...")
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) GeneratePostBuild(w ShellWriter, info common.ShellScriptInfo) {
	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	// Parse options
	var options shellOptions
	info.Build.Options.Decode(&options)

	// Find cached files and archive them
	b.cacheArchiver(w, options.Cache, info)

	// Upload artifacts
	b.uploadArtifacts(w, options.Artifacts, info)
}
<commit_msg>Fix compilation error<commit_after>package shells

import (
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
)

type AbstractShell struct {
}

func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
	features.Artifacts = true
	features.Cache = true
}

func (b *AbstractShell) GetSupportedOptions() []string {
	return []string{"artifacts", "cache", "dependencies"}
}

func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
	w.Cd(info.Build.FullProjectDir())
}

func (b *AbstractShell) writeExports(w ShellWriter, info common.ShellScriptInfo) {
	for _, variable := range info.Build.GetAllVariables() {
		w.Variable(variable)
	}
}

func (b *AbstractShell) writeTLSCAInfo(w ShellWriter, build *common.Build, key string) {
	if build.TLSCAChain != "" {
		w.Variable(common.BuildVariable{
			Key:      key,
			Value:    build.TLSCAChain,
			Public:   true,
			Internal: true,
			File:     true,
		})
	}
}

func (b *AbstractShell) writeCloneCmd(w ShellWriter, build *common.Build, projectDir string) {
	w.Notice("Cloning repository...")
	w.RmDir(projectDir)
	w.Command("git", "clone", build.RepoURL, projectDir)
	w.Cd(projectDir)
}

func (b *AbstractShell) writeFetchCmd(w ShellWriter, build *common.Build, projectDir string, gitDir string) {
	w.IfDirectory(gitDir)
	w.Notice("Fetching changes...")
	w.Cd(projectDir)
	w.Command("git", "clean", "-ffdx")
	w.Command("git", "reset", "--hard")
	w.Command("git", "remote", "set-url", "origin", build.RepoURL)
	w.Command("git", "fetch", "origin", "--prune", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*")
	w.Else()
	b.writeCloneCmd(w, build, projectDir)
	w.EndIf()
}

func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
	w.Notice("Checking out %s as %s...", build.Sha[0:8], build.RefName)
	// We remove a git index file, this is required if `git checkout` is terminated
	w.RmFile(".git/index.lock")
	w.Command("git", "checkout", build.Sha)
}

func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, file string) {
	// For tags we don't create cache
	if build.Tag || build.CacheDir == "" {
		return
	}

	// Deduce cache key
	key = path.Join(build.Name, build.RefName)
	if userKey != "" {
		key = build.GetAllVariables().ExpandValue(userKey)
	}

	// Ignore cache without the key
	if key == "" {
		return
	}

	file = path.Join(build.CacheDir, key, "cache.zip")
	file, err := filepath.Rel(build.BuildDir, file)
	if err != nil {
		return "", ""
	}
	return
}

func (o *archivingOptions) CommandArguments() (args []string) {
	for _, path := range o.Paths {
		args = append(args, "--path", path)
	}

	if o.Untracked {
		args = append(args, "--untracked")
	}
	return
}

func (b *AbstractShell) cacheExtractor(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.RunnerCommand == "" {
		w.Warning("The cache is not supported in this executor.")
		return
	}

	// Skip restoring cache if no cache is defined
	if archiverArgs := options.CommandArguments(); len(archiverArgs) == 0 {
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-extractor",
		"--file", cacheFile,
	}

	// Generate cache download address
	if url := getCacheDownloadURL(info.Build, cacheKey); url != "" {
		args = append(args, "--url", url)
	}

	// Execute archive command
	w.Notice("Checking cache for %s...", cacheKey)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) downloadArtifacts(w ShellWriter, build *common.BuildInfo, info common.ShellScriptInfo) {
	if info.RunnerCommand == "" {
		w.Warning("The artifacts downloading is not supported in this executor.")
		return
	}

	args := []string{
		"artifacts-downloader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		build.Token,
		"--id",
		strconv.Itoa(build.ID),
	}

	w.Notice("Downloading artifacts for %s (%d)...", build.Name, build.ID)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) downloadAllArtifacts(w ShellWriter, dependencies *dependencies, info common.ShellScriptInfo) {
	for _, otherBuild := range info.Build.DependsOnBuilds {
		if otherBuild.Artifacts == nil || otherBuild.Artifacts.Filename == "" {
			continue
		}
		if !dependencies.IsDependent(otherBuild.Name) {
			continue
		}
		b.downloadArtifacts(w, &otherBuild, info)
	}
}

func (b *AbstractShell) GeneratePreBuild(w ShellWriter, info common.ShellScriptInfo) {
	b.writeExports(w, info)

	build := info.Build
	projectDir := build.FullProjectDir()
	gitDir := path.Join(build.FullProjectDir(), ".git")

	b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	if build.AllowGitFetch {
		b.writeFetchCmd(w, build, projectDir, gitDir)
	} else {
		b.writeCloneCmd(w, build, projectDir)
	}

	b.writeCheckoutCmd(w, build)

	// Parse options
	var options shellOptions
	info.Build.Options.Decode(&options)

	// Try to restore from main cache, if not found cache for master
	b.cacheExtractor(w, options.Cache, info)

	// Process all artifacts
	b.downloadAllArtifacts(w, options.Dependencies, info)
}

func (b *AbstractShell) GenerateCommands(w ShellWriter, info common.ShellScriptInfo) {
	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)

	commands := info.Build.Commands
	commands = strings.TrimSpace(commands)
	for _, command := range strings.Split(commands, "\n") {
		command = strings.TrimSpace(command)
		if command != "" {
			w.Notice("$ %s", command)
		} else {
			w.EmptyLine()
		}
		w.Line(command)
	}
}

func (b *AbstractShell) cacheArchiver(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.RunnerCommand == "" {
		w.Warning("The cache is not supported in this executor.")
		return
	}

	// Skip archiving if no cache is defined
	cacheKey, cacheFile := b.cacheFile(info.Build, options.Key)
	if cacheKey == "" {
		return
	}

	args := []string{
		"cache-archiver",
		"--file", cacheFile,
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Generate cache upload address
	if url := getCacheUploadURL(info.Build, cacheKey); url != "" {
		args = append(args, "--url", url)
	}

	// Execute archive command
	w.Notice("Creating cache %s...", cacheKey)
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) uploadArtifacts(w ShellWriter, options *archivingOptions, info common.ShellScriptInfo) {
	if options == nil {
		return
	}
	if info.Build.Runner.URL == "" {
		return
	}
	if info.RunnerCommand == "" {
		w.Warning("The artifacts uploading is not supported in this executor.")
		return
	}

	args := []string{
		"artifacts-uploader",
		"--url",
		info.Build.Runner.URL,
		"--token",
		info.Build.Token,
		"--id",
		strconv.Itoa(info.Build.ID),
	}

	// Create list of files to archive
	archiverArgs := options.CommandArguments()
	if len(archiverArgs) == 0 {
		// Skip creating archive
		return
	}
	args = append(args, archiverArgs...)

	// Get artifacts:name
	if name, ok := info.Build.Options.GetString("artifacts", "name"); ok && name != "" {
		args = append(args, "--name", name)
	}

	w.Notice("Uploading artifacts...")
	w.Command(info.RunnerCommand, args...)
}

func (b *AbstractShell) GeneratePostBuild(w ShellWriter, info common.ShellScriptInfo) {
	b.writeExports(w, info)
	b.writeCdBuildDir(w, info)
	b.writeTLSCAInfo(w, info.Build, "CI_SERVER_TLS_CA_FILE")

	// Parse options
	var options shellOptions
	info.Build.Options.Decode(&options)

	// Find cached files and archive them
	b.cacheArchiver(w, options.Cache, info)

	// Upload artifacts
	b.uploadArtifacts(w, options.Artifacts, info)
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"errors"
	"fmt"

	"github.com/GoogleCloudPlatform/healthcare/deploy/config/tfconfig"
)

func (p *Project) initTerraform(auditProject *Project) error {
	if err := p.initTerraformAuditResources(auditProject); err != nil {
		return fmt.Errorf("failed to init audit resources: %v", err)
	}
	if err := p.initServices(); err != nil {
		return fmt.Errorf("failed to init services: %v", err)
	}

	p.addDefaultIAM()
	p.addDefaultMonitoring()

	// At least have one owner access set to override default accesses
	// (https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets).
	for _, d := range p.BigqueryDatasets {
		d.Accesses = append(d.Accesses, &tfconfig.Access{Role: "OWNER", GroupByEmail: p.OwnersGroup})
	}

	for _, r := range p.TerraformResources() {
		if err := r.Init(p.ID); err != nil {
			return fmt.Errorf("failed to init %q (%v): %v", r.ResourceType(), r, err)
		}
	}
	return nil
}

func (p *Project) initTerraformAuditResources(auditProject *Project) error {
	d := p.Audit.LogsBigqueryDataset
	if d == nil {
		return errors.New("audit.logs_bigquery_dataset must be set")
	}

	if err := d.Init(auditProject.ID); err != nil {
		return fmt.Errorf("failed to init logs bq dataset: %v", err)
	}

	d.Accesses = append(d.Accesses,
		&tfconfig.Access{Role: "OWNER", GroupByEmail: auditProject.OwnersGroup},
		&tfconfig.Access{Role: "READER", GroupByEmail: p.AuditorsGroup},
	)

	p.BQLogSinkTF = &tfconfig.LoggingSink{
		Name:                 "audit-logs-to-bigquery",
		Destination:          fmt.Sprintf("bigquery.googleapis.com/projects/%s/datasets/%s", auditProject.ID, d.DatasetID),
		Filter:               `logName:"logs/cloudaudit.googleapis.com"`,
		UniqueWriterIdentity: true,
	}
	if err := p.BQLogSinkTF.Init(p.ID); err != nil {
		return fmt.Errorf("failed to init bigquery log sink: %v", err)
	}

	b := p.Audit.LogsStorageBucket
	if b == nil {
		return nil
	}

	if err := b.Init(auditProject.ID); err != nil {
		return fmt.Errorf("failed to init logs gcs bucket: %v", err)
	}

	b.IAMMembers = append(b.IAMMembers,
		&tfconfig.StorageIAMMember{Role: "roles/storage.admin", Member: "group:" + auditProject.OwnersGroup},
		&tfconfig.StorageIAMMember{Role: "roles/storage.objectCreator", Member: accessLogsWriter},
		&tfconfig.StorageIAMMember{Role: "roles/storage.objectViewer", Member: "group:" + p.AuditorsGroup})
	return nil
}

func (p *Project) initServices() error {
	if p.Services == nil {
		p.Services = new(tfconfig.ProjectServices)
	}

	svcs := []string{
		"bigquery-json.googleapis.com", // For bigquery audit logs and datasets.
		"bigquerystorage.googleapis.com",
		"cloudresourcemanager.googleapis.com", // For project level iam policy updates.
		"logging.googleapis.com",              // For default logging metrics.
	}
	if len(p.ComputeInstances) > 0 || len(p.ComputeImages) > 0 {
		svcs = append(svcs, "compute.googleapis.com")
	}
	if len(p.HealthcareDatasets) > 0 {
		svcs = append(svcs, "healthcare.googleapis.com")
	}
	if len(p.NotificationChannels) > 0 {
		svcs = append(svcs, "monitoring.googleapis.com")
	}

	for _, svc := range svcs {
		p.Services.Services = append(p.Services.Services, &tfconfig.ProjectService{Service: svc})
	}
	// Note: services will be de-duplicated when being marshalled.
	if err := p.Services.Init(p.ID); err != nil {
		return fmt.Errorf("failed to init services: %v", err)
	}
	return nil
}

func (p *Project) addDefaultIAM() {
	// Enable all possible audit log collection.
	p.IAMAuditConfig = &tfconfig.ProjectIAMAuditConfig{
		Service: "allServices",
		AuditLogConfigs: []*tfconfig.AuditLogConfig{
			{LogType: "DATA_READ"},
			{LogType: "DATA_WRITE"},
			{LogType: "ADMIN_READ"},
		},
	}
	if p.IAMMembers == nil {
		p.IAMMembers = new(tfconfig.ProjectIAMMembers)
	}

	p.IAMMembers.Members = append(p.IAMMembers.Members,
		&tfconfig.ProjectIAMMember{Role: "roles/owner", Member: "group:" + p.OwnersGroup},
		&tfconfig.ProjectIAMMember{Role: "roles/iam.securityReviewer", Member: "group:" + p.AuditorsGroup},
	)
	if p.Audit.LogsStorageBucket != nil || len(p.StorageBuckets) > 0 {
		// roles/owner does not grant storage.buckets.setIamPolicy, so we need to add storage admin role on the owners group.
		p.IAMMembers.Members = append(p.IAMMembers.Members, &tfconfig.ProjectIAMMember{
			Role: "roles/storage.admin", Member: "group:" + p.OwnersGroup,
		})
	}
}

func (p *Project) addDefaultMonitoring() {
	type metricAndAlert struct {
		metric *tfconfig.LoggingMetric
		alert  *tfconfig.MonitoringAlertPolicy
	}

	metricAndAlerts := []metricAndAlert{
		{
			metric: &tfconfig.LoggingMetric{
				Name:        "bigquery-settings-change-count",
				Description: "Count of bigquery permission changes.",
				Filter:      `resource.type="bigquery_resource" AND protoPayload.methodName="datasetservice.update"`,
			},
			alert: &tfconfig.MonitoringAlertPolicy{
				DisplayName: "Bigquery Update Alert",
				Documentation: &tfconfig.Documentation{
					Content: "This policy ensures the designated user/group is notified when Bigquery dataset settings are altered.",
				},
				Conditions: []*tfconfig.Condition{{
					DisplayName: "No tolerance on bigquery-settings-change-count!",
					ConditionThreshold: &tfconfig.ConditionThreshold{
						Filter: `resource.type="global" AND metric.type="logging.googleapis.com/user/${google_logging_metric.bigquery-settings-change-count.name}"`,
					},
				}},
			},
		},
		{
			metric: &tfconfig.LoggingMetric{
				Name:        "bucket-permission-change-count",
				Description: "Count of GCS permissions changes.",
				Filter: `resource.type=gcs_bucket AND protoPayload.serviceName=storage.googleapis.com AND
(protoPayload.methodName=storage.setIamPermissions OR protoPayload.methodName=storage.objects.update)`,
			},
			alert: &tfconfig.MonitoringAlertPolicy{
				DisplayName: "Bucket Permission Change Alert",
				Documentation: &tfconfig.Documentation{
					Content: "This policy ensures the designated user/group is notified when bucket/object permissions are altered.",
				},
				Conditions: []*tfconfig.Condition{{
					DisplayName: "No tolerance on bucket-permission-change-count!",
					ConditionThreshold: &tfconfig.ConditionThreshold{
						Filter: `resource.type="gcs_bucket" AND metric.type="logging.googleapis.com/user/${google_logging_metric.bucket-permission-change-count.name}"`,
					},
				}},
			},
		},
		{
			metric: &tfconfig.LoggingMetric{
				Name:        "iam-policy-change-count",
				Description: "Count of IAM policy changes.",
				Filter:      `protoPayload.methodName="SetIamPolicy" OR protoPayload.methodName:".setIamPolicy"`,
			},
			alert: &tfconfig.MonitoringAlertPolicy{
				DisplayName: "IAM Policy Change Alert",
				Documentation: &tfconfig.Documentation{
					Content: "This policy ensures the designated user/group is notified when IAM policies are altered.",
				},
				Conditions: []*tfconfig.Condition{{
					DisplayName: "No tolerance on iam-policy-change-count!",
					ConditionThreshold: &tfconfig.ConditionThreshold{
						Filter: `resource.type=one_of("global","pubsub_topic","pubsub_subscription","gce_instance") AND metric.type="logging.googleapis.com/user/${google_logging_metric.iam-policy-change-count.name}"`,
					},
				}},
			},
		},
	}

	for _, ma := range metricAndAlerts {
		ma.metric.MetricDescriptor = &tfconfig.MetricDescriptor{
			MetricKind: "DELTA",
			ValueType:  "INT64",
			Labels: []*tfconfig.Label{{
				Key:         "user",
				ValueType:   "STRING",
				Description: "Unexpected user",
			}},
		}
		ma.metric.LabelExtractors = map[string]string{
			"user": "EXTRACT(protoPayload.authenticationInfo.principalEmail)",
		}

		ma.alert.Documentation.MimeType = "text/markdown"
		ma.alert.Combiner = "AND"
		for _, c := range ma.alert.Conditions {
			c.ConditionThreshold.Comparison = "COMPARISON_GT"
			c.ConditionThreshold.Duration = "0s"
		}

		p.DefaultLoggingMetrics = append(p.DefaultLoggingMetrics, ma.metric)

		if len(p.NotificationChannels) > 0 {
			for _, c := range p.NotificationChannels {
				ref := fmt.Sprintf("${%s.%s.name}", c.ResourceType(), c.ID())
				ma.alert.NotificationChannels = append(ma.alert.NotificationChannels, ref)
			}
			p.DefaultAlertPolicies = append(p.DefaultAlertPolicies, ma.alert)
		}
	}
}

// TerraformResources gets all terraform resources in this project.
func (p *Project) TerraformResources() []tfconfig.Resource {
	var rs []tfconfig.Resource
	// Put default resources first to make it easier to write tests.
	if p.IAMAuditConfig != nil {
		rs = append(rs, p.IAMAuditConfig)
	}
	if p.IAMMembers != nil {
		rs = append(rs, p.IAMMembers)
	}
	for _, r := range p.DefaultLoggingMetrics {
		rs = append(rs, r)
	}
	for _, r := range p.DefaultAlertPolicies {
		rs = append(rs, r)
	}

	for _, r := range p.BigqueryDatasets {
		rs = append(rs, r)
	}
	for _, r := range p.ComputeFirewalls {
		rs = append(rs, r)
	}
	for _, r := range p.ComputeImages {
		rs = append(rs, r)
	}
	for _, r := range p.ComputeInstances {
		rs = append(rs, r)
	}
	for _, r := range p.HealthcareDatasets {
		rs = append(rs, r)
	}
	for _, r := range p.IAMCustomRoles {
		rs = append(rs, r)
	}
	for _, r := range p.NotificationChannels {
		rs = append(rs, r)
	}
	for _, r := range p.PubsubTopics {
		rs = append(rs, r)
	}
	for _, r := range p.ResourceManagerLiens {
		rs = append(rs, r)
	}
	for _, r := range p.ServiceAccounts {
		rs = append(rs, r)
	}
	for _, r := range p.StorageBuckets {
		rs = append(rs, r)
	}
	return rs
}
<commit_msg>enable pubsub api if pubsub topics are set<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"errors"
	"fmt"

	"github.com/GoogleCloudPlatform/healthcare/deploy/config/tfconfig"
)

func (p *Project) initTerraform(auditProject *Project) error {
	if err := p.initTerraformAuditResources(auditProject); err != nil {
		return fmt.Errorf("failed to init audit resources: %v", err)
	}
	if err := p.initServices(); err != nil {
		return fmt.Errorf("failed to init services: %v", err)
	}

	p.addDefaultIAM()
	p.addDefaultMonitoring()

	// At least have one owner access set to override default accesses
	// (https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets).
	for _, d := range p.BigqueryDatasets {
		d.Accesses = append(d.Accesses, &tfconfig.Access{Role: "OWNER", GroupByEmail: p.OwnersGroup})
	}

	for _, r := range p.TerraformResources() {
		if err := r.Init(p.ID); err != nil {
			return fmt.Errorf("failed to init %q (%v): %v", r.ResourceType(), r, err)
		}
	}
	return nil
}

func (p *Project) initTerraformAuditResources(auditProject *Project) error {
	d := p.Audit.LogsBigqueryDataset
	if d == nil {
		return errors.New("audit.logs_bigquery_dataset must be set")
	}

	if err := d.Init(auditProject.ID); err != nil {
		return fmt.Errorf("failed to init logs bq dataset: %v", err)
	}

	d.Accesses = append(d.Accesses,
		&tfconfig.Access{Role: "OWNER", GroupByEmail: auditProject.OwnersGroup},
		&tfconfig.Access{Role: "READER", GroupByEmail: p.AuditorsGroup},
	)

	p.BQLogSinkTF = &tfconfig.LoggingSink{
		Name:                 "audit-logs-to-bigquery",
		Destination:          fmt.Sprintf("bigquery.googleapis.com/projects/%s/datasets/%s", auditProject.ID, d.DatasetID),
		Filter:               `logName:"logs/cloudaudit.googleapis.com"`,
		UniqueWriterIdentity: true,
	}
	if err := p.BQLogSinkTF.Init(p.ID); err != nil {
		return fmt.Errorf("failed to init bigquery log sink: %v", err)
	}

	b := p.Audit.LogsStorageBucket
	if b == nil {
		return nil
	}

	if err := b.Init(auditProject.ID); err != nil {
		return fmt.Errorf("failed to init logs gcs bucket: %v", err)
	}

	b.IAMMembers = append(b.IAMMembers,
		&tfconfig.StorageIAMMember{Role: "roles/storage.admin", Member: "group:" + auditProject.OwnersGroup},
		&tfconfig.StorageIAMMember{Role: "roles/storage.objectCreator", Member: accessLogsWriter},
		&tfconfig.StorageIAMMember{Role: "roles/storage.objectViewer", Member: "group:" + p.AuditorsGroup})
	return nil
}

func (p *Project) initServices() error {
	if p.Services == nil {
		p.Services = new(tfconfig.ProjectServices)
	}

	svcs := []string{
		"bigquery-json.googleapis.com", // For bigquery audit logs and datasets.
		"bigquerystorage.googleapis.com",
		"cloudresourcemanager.googleapis.com", // For project level iam policy updates.
		"logging.googleapis.com",              // For default logging metrics.
	}
	if len(p.ComputeInstances) > 0 || len(p.ComputeImages) > 0 {
		svcs = append(svcs, "compute.googleapis.com")
	}
	if len(p.HealthcareDatasets) > 0 {
		svcs = append(svcs, "healthcare.googleapis.com")
	}
	if len(p.NotificationChannels) > 0 {
		svcs = append(svcs, "monitoring.googleapis.com")
	}
	if len(p.PubsubTopics) > 0 {
		svcs = append(svcs, "pubsub.googleapis.com")
	}

	for _, svc := range svcs {
		p.Services.Services = append(p.Services.Services, &tfconfig.ProjectService{Service: svc})
	}
	// Note: services will be de-duplicated when being marshalled.
	if err := p.Services.Init(p.ID); err != nil {
		return fmt.Errorf("failed to init services: %v", err)
	}
	return nil
}

func (p *Project) addDefaultIAM() {
	// Enable all possible audit log collection.
	p.IAMAuditConfig = &tfconfig.ProjectIAMAuditConfig{
		Service: "allServices",
		AuditLogConfigs: []*tfconfig.AuditLogConfig{
			{LogType: "DATA_READ"},
			{LogType: "DATA_WRITE"},
			{LogType: "ADMIN_READ"},
		},
	}
	if p.IAMMembers == nil {
		p.IAMMembers = new(tfconfig.ProjectIAMMembers)
	}

	p.IAMMembers.Members = append(p.IAMMembers.Members,
		&tfconfig.ProjectIAMMember{Role: "roles/owner", Member: "group:" + p.OwnersGroup},
		&tfconfig.ProjectIAMMember{Role: "roles/iam.securityReviewer", Member: "group:" + p.AuditorsGroup},
	)
	if p.Audit.LogsStorageBucket != nil || len(p.StorageBuckets) > 0 {
		// roles/owner does not grant storage.buckets.setIamPolicy, so we need to add storage admin role on the owners group.
		p.IAMMembers.Members = append(p.IAMMembers.Members, &tfconfig.ProjectIAMMember{
			Role: "roles/storage.admin", Member: "group:" + p.OwnersGroup,
		})
	}
}

func (p *Project) addDefaultMonitoring() {
	type metricAndAlert struct {
		metric *tfconfig.LoggingMetric
		alert  *tfconfig.MonitoringAlertPolicy
	}

	metricAndAlerts := []metricAndAlert{
		{
			metric: &tfconfig.LoggingMetric{
				Name:        "bigquery-settings-change-count",
				Description: "Count of bigquery permission changes.",
				Filter:      `resource.type="bigquery_resource" AND protoPayload.methodName="datasetservice.update"`,
			},
			alert: &tfconfig.MonitoringAlertPolicy{
				DisplayName: "Bigquery Update Alert",
				Documentation: &tfconfig.Documentation{
					Content: "This policy ensures the designated user/group is notified when Bigquery dataset settings are altered.",
				},
				Conditions: []*tfconfig.Condition{{
					DisplayName: "No tolerance on bigquery-settings-change-count!",
					ConditionThreshold: &tfconfig.ConditionThreshold{
						Filter: `resource.type="global" AND metric.type="logging.googleapis.com/user/${google_logging_metric.bigquery-settings-change-count.name}"`,
					},
				}},
			},
		},
		{
			metric: &tfconfig.LoggingMetric{
				Name:        "bucket-permission-change-count",
				Description: "Count of GCS permissions changes.",
				Filter: `resource.type=gcs_bucket AND protoPayload.serviceName=storage.googleapis.com AND
(protoPayload.methodName=storage.setIamPermissions OR protoPayload.methodName=storage.objects.update)`,
			},
			alert: &tfconfig.MonitoringAlertPolicy{
				DisplayName: "Bucket Permission Change Alert",
				Documentation: &tfconfig.Documentation{
					Content: "This policy ensures the designated user/group is notified when bucket/object permissions are altered.",
				},
				Conditions: []*tfconfig.Condition{{
					DisplayName: "No tolerance on bucket-permission-change-count!",
					ConditionThreshold: &tfconfig.ConditionThreshold{
						Filter: `resource.type="gcs_bucket" AND metric.type="logging.googleapis.com/user/${google_logging_metric.bucket-permission-change-count.name}"`,
					},
				}},
			},
		},
		{
			metric: &tfconfig.LoggingMetric{
				Name:        "iam-policy-change-count",
				Description: "Count of IAM policy changes.",
				Filter:      `protoPayload.methodName="SetIamPolicy" OR protoPayload.methodName:".setIamPolicy"`,
			},
			alert: &tfconfig.MonitoringAlertPolicy{
				DisplayName: "IAM Policy Change Alert",
				Documentation: &tfconfig.Documentation{
					Content: "This policy ensures the designated user/group is notified when IAM policies are altered.",
				},
				Conditions: []*tfconfig.Condition{{
					DisplayName: "No tolerance on iam-policy-change-count!",
					ConditionThreshold: &tfconfig.ConditionThreshold{
						Filter: `resource.type=one_of("global","pubsub_topic","pubsub_subscription","gce_instance") AND metric.type="logging.googleapis.com/user/${google_logging_metric.iam-policy-change-count.name}"`,
					},
				}},
			},
		},
	}

	for _, ma := range metricAndAlerts {
		ma.metric.MetricDescriptor = &tfconfig.MetricDescriptor{
			MetricKind: "DELTA",
			ValueType:  "INT64",
			Labels: []*tfconfig.Label{{
				Key:         "user",
				ValueType:   "STRING",
				Description: "Unexpected user",
			}},
		}
		ma.metric.LabelExtractors = map[string]string{
			"user": "EXTRACT(protoPayload.authenticationInfo.principalEmail)",
		}

		ma.alert.Documentation.MimeType = "text/markdown"
		ma.alert.Combiner = "AND"
		for _, c := range ma.alert.Conditions {
			c.ConditionThreshold.Comparison = "COMPARISON_GT"
			c.ConditionThreshold.Duration = "0s"
		}

		p.DefaultLoggingMetrics = append(p.DefaultLoggingMetrics, ma.metric)

		if len(p.NotificationChannels) > 0 {
			for _, c := range p.NotificationChannels {
				ref := fmt.Sprintf("${%s.%s.name}", c.ResourceType(), c.ID())
				ma.alert.NotificationChannels = append(ma.alert.NotificationChannels, ref)
			}
			p.DefaultAlertPolicies = append(p.DefaultAlertPolicies, ma.alert)
		}
	}
}

// TerraformResources gets all terraform resources in this project.
func (p *Project) TerraformResources() []tfconfig.Resource {
	var rs []tfconfig.Resource
	// Put default resources first to make it easier to write tests.
	if p.IAMAuditConfig != nil {
		rs = append(rs, p.IAMAuditConfig)
	}
	if p.IAMMembers != nil {
		rs = append(rs, p.IAMMembers)
	}
	for _, r := range p.DefaultLoggingMetrics {
		rs = append(rs, r)
	}
	for _, r := range p.DefaultAlertPolicies {
		rs = append(rs, r)
	}

	for _, r := range p.BigqueryDatasets {
		rs = append(rs, r)
	}
	for _, r := range p.ComputeFirewalls {
		rs = append(rs, r)
	}
	for _, r := range p.ComputeImages {
		rs = append(rs, r)
	}
	for _, r := range p.ComputeInstances {
		rs = append(rs, r)
	}
	for _, r := range p.HealthcareDatasets {
		rs = append(rs, r)
	}
	for _, r := range p.IAMCustomRoles {
		rs = append(rs, r)
	}
	for _, r := range p.NotificationChannels {
		rs = append(rs, r)
	}
	for _, r := range p.PubsubTopics {
		rs = append(rs, r)
	}
	for _, r := range p.ResourceManagerLiens {
		rs = append(rs, r)
	}
	for _, r := range p.ServiceAccounts {
		rs = append(rs, r)
	}
	for _, r := range p.StorageBuckets {
		rs = append(rs, r)
	}
	return rs
}
<|endoftext|>