<commit_before>package service

import (
	"context"
	"errors"

	"github.com/go-kit/kit/endpoint"
	"github.com/kolide/fleet/server/contexts/viewer"
	"github.com/kolide/fleet/server/kolide"
)

type appConfigRequest struct {
	Payload kolide.AppConfigPayload
}

type appConfigResponse struct {
	OrgInfo            *kolide.OrgInfo             `json:"org_info,omitemtpy"`
	ServerSettings     *kolide.ServerSettings      `json:"server_settings,omitempty"`
	SMTPSettings       *kolide.SMTPSettingsPayload `json:"smtp_settings,omitempty"`
	SSOSettings        *kolide.SSOSettingsPayload  `json:"sso_settings,omitempty"`
	HostExpirySettings *kolide.HostExpirySettings  `json:"host_expiry_settings,omitempty"`
	Err                error                       `json:"error,omitempty"`
}

func (r appConfigResponse) error() error { return r.Err }

func makeGetAppConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		vc, ok := viewer.FromContext(ctx)
		if !ok {
			return nil, errors.New("could not fetch user")
		}
		config, err := svc.AppConfig(ctx)
		if err != nil {
			return nil, err
		}
		var smtpSettings *kolide.SMTPSettingsPayload
		var ssoSettings *kolide.SSOSettingsPayload
		var hostExpirySettings *kolide.HostExpirySettings
		// only admin can see smtp, sso, and host expiry settings
		if vc.CanPerformAdminActions() {
			smtpSettings = smtpSettingsFromAppConfig(config)
			if smtpSettings.SMTPPassword != nil {
				*smtpSettings.SMTPPassword = "********"
			}
			ssoSettings = &kolide.SSOSettingsPayload{
				EntityID:    &config.EntityID,
				IssuerURI:   &config.IssuerURI,
				IDPImageURL: &config.IDPImageURL,
				Metadata:    &config.Metadata,
				MetadataURL: &config.MetadataURL,
				IDPName:     &config.IDPName,
				EnableSSO:   &config.EnableSSO,
			}
			hostExpirySettings = &kolide.HostExpirySettings{
				HostExpiryEnabled: &config.HostExpiryEnabled,
				HostExpiryWindow:  &config.HostExpiryWindow,
			}
		}
		response := appConfigResponse{
			OrgInfo: &kolide.OrgInfo{
				OrgName:    &config.OrgName,
				OrgLogoURL: &config.OrgLogoURL,
			},
			ServerSettings: &kolide.ServerSettings{
				KolideServerURL:    &config.KolideServerURL,
				EnrollSecret:       &config.EnrollSecret,
				LiveQueryDisabled: &config.LiveQueryDisabled,
			},
			SMTPSettings:       smtpSettings,
			SSOSettings:        ssoSettings,
			HostExpirySettings: hostExpirySettings,
		}
		return response, nil
	}
}

func makeModifyAppConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		req := request.(appConfigRequest)
		config, err := svc.ModifyAppConfig(ctx, req.Payload)
		if err != nil {
			return appConfigResponse{Err: err}, nil
		}
		response := appConfigResponse{
			OrgInfo: &kolide.OrgInfo{
				OrgName:    &config.OrgName,
				OrgLogoURL: &config.OrgLogoURL,
			},
			ServerSettings: &kolide.ServerSettings{
				KolideServerURL:    &config.KolideServerURL,
				EnrollSecret:       &config.EnrollSecret,
				LiveQueryDisabled: &config.LiveQueryDisabled,
			},
			SMTPSettings: smtpSettingsFromAppConfig(config),
			SSOSettings: &kolide.SSOSettingsPayload{
				EntityID:    &config.EntityID,
				IssuerURI:   &config.IssuerURI,
				IDPImageURL: &config.IDPImageURL,
				Metadata:    &config.Metadata,
				MetadataURL: &config.MetadataURL,
				IDPName:     &config.IDPName,
				EnableSSO:   &config.EnableSSO,
			},
			HostExpirySettings: &kolide.HostExpirySettings{
				HostExpiryEnabled: &config.HostExpiryEnabled,
				HostExpiryWindow:  &config.HostExpiryWindow,
			},
		}
		if response.SMTPSettings.SMTPPassword != nil {
			*response.SMTPSettings.SMTPPassword = "********"
		}
		return response, nil
	}
}

func smtpSettingsFromAppConfig(config *kolide.AppConfig) *kolide.SMTPSettingsPayload {
	authType := config.SMTPAuthenticationType.String()
	authMethod := config.SMTPAuthenticationMethod.String()
	return &kolide.SMTPSettingsPayload{
		SMTPEnabled:              &config.SMTPConfigured,
		SMTPConfigured:           &config.SMTPConfigured,
		SMTPSenderAddress:        &config.SMTPSenderAddress,
		SMTPServer:               &config.SMTPServer,
		SMTPPort:                 &config.SMTPPort,
		SMTPAuthenticationType:   &authType,
		SMTPUserName:             &config.SMTPUserName,
		SMTPPassword:             &config.SMTPPassword,
		SMTPEnableTLS:            &config.SMTPEnableTLS,
		SMTPAuthenticationMethod: &authMethod,
		SMTPDomain:               &config.SMTPDomain,
		SMTPVerifySSLCerts:       &config.SMTPVerifySSLCerts,
		SMTPEnableStartTLS:       &config.SMTPEnableStartTLS,
	}
}
<commit_msg>server/service: Fix appConfigResponse.OrgInfo json tag (#2212)<commit_after>package service

import (
	"context"
	"errors"

	"github.com/go-kit/kit/endpoint"
	"github.com/kolide/fleet/server/contexts/viewer"
	"github.com/kolide/fleet/server/kolide"
)

type appConfigRequest struct {
	Payload kolide.AppConfigPayload
}

type appConfigResponse struct {
	OrgInfo            *kolide.OrgInfo             `json:"org_info,omitempty"`
	ServerSettings     *kolide.ServerSettings      `json:"server_settings,omitempty"`
	SMTPSettings       *kolide.SMTPSettingsPayload `json:"smtp_settings,omitempty"`
	SSOSettings        *kolide.SSOSettingsPayload  `json:"sso_settings,omitempty"`
	HostExpirySettings *kolide.HostExpirySettings  `json:"host_expiry_settings,omitempty"`
	Err                error                       `json:"error,omitempty"`
}

func (r appConfigResponse) error() error { return r.Err }

func makeGetAppConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		vc, ok := viewer.FromContext(ctx)
		if !ok {
			return nil, errors.New("could not fetch user")
		}
		config, err := svc.AppConfig(ctx)
		if err != nil {
			return nil, err
		}
		var smtpSettings *kolide.SMTPSettingsPayload
		var ssoSettings *kolide.SSOSettingsPayload
		var hostExpirySettings *kolide.HostExpirySettings
		// only admin can see smtp, sso, and host expiry settings
		if vc.CanPerformAdminActions() {
			smtpSettings = smtpSettingsFromAppConfig(config)
			if smtpSettings.SMTPPassword != nil {
				*smtpSettings.SMTPPassword = "********"
			}
			ssoSettings = &kolide.SSOSettingsPayload{
				EntityID:    &config.EntityID,
				IssuerURI:   &config.IssuerURI,
				IDPImageURL: &config.IDPImageURL,
				Metadata:    &config.Metadata,
				MetadataURL: &config.MetadataURL,
				IDPName:     &config.IDPName,
				EnableSSO:   &config.EnableSSO,
			}
			hostExpirySettings = &kolide.HostExpirySettings{
				HostExpiryEnabled: &config.HostExpiryEnabled,
				HostExpiryWindow:  &config.HostExpiryWindow,
			}
		}
		response := appConfigResponse{
			OrgInfo: &kolide.OrgInfo{
				OrgName:    &config.OrgName,
				OrgLogoURL: &config.OrgLogoURL,
			},
			ServerSettings: &kolide.ServerSettings{
				KolideServerURL:   &config.KolideServerURL,
				EnrollSecret:      &config.EnrollSecret,
				LiveQueryDisabled: &config.LiveQueryDisabled,
			},
			SMTPSettings:       smtpSettings,
			SSOSettings:        ssoSettings,
			HostExpirySettings: hostExpirySettings,
		}
		return response, nil
	}
}

func makeModifyAppConfigEndpoint(svc kolide.Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		req := request.(appConfigRequest)
		config, err := svc.ModifyAppConfig(ctx, req.Payload)
		if err != nil {
			return appConfigResponse{Err: err}, nil
		}
		response := appConfigResponse{
			OrgInfo: &kolide.OrgInfo{
				OrgName:    &config.OrgName,
				OrgLogoURL: &config.OrgLogoURL,
			},
			ServerSettings: &kolide.ServerSettings{
				KolideServerURL:   &config.KolideServerURL,
				EnrollSecret:      &config.EnrollSecret,
				LiveQueryDisabled: &config.LiveQueryDisabled,
			},
			SMTPSettings: smtpSettingsFromAppConfig(config),
			SSOSettings: &kolide.SSOSettingsPayload{
				EntityID:    &config.EntityID,
				IssuerURI:   &config.IssuerURI,
				IDPImageURL: &config.IDPImageURL,
				Metadata:    &config.Metadata,
				MetadataURL: &config.MetadataURL,
				IDPName:     &config.IDPName,
				EnableSSO:   &config.EnableSSO,
			},
			HostExpirySettings: &kolide.HostExpirySettings{
				HostExpiryEnabled: &config.HostExpiryEnabled,
				HostExpiryWindow:  &config.HostExpiryWindow,
			},
		}
		if response.SMTPSettings.SMTPPassword != nil {
			*response.SMTPSettings.SMTPPassword = "********"
		}
		return response, nil
	}
}

func smtpSettingsFromAppConfig(config *kolide.AppConfig) *kolide.SMTPSettingsPayload {
	authType := config.SMTPAuthenticationType.String()
	authMethod := config.SMTPAuthenticationMethod.String()
	return &kolide.SMTPSettingsPayload{
		SMTPEnabled:              &config.SMTPConfigured,
		SMTPConfigured:           &config.SMTPConfigured,
		SMTPSenderAddress:        &config.SMTPSenderAddress,
		SMTPServer:               &config.SMTPServer,
		SMTPPort:                 &config.SMTPPort,
		SMTPAuthenticationType:   &authType,
		SMTPUserName:             &config.SMTPUserName,
		SMTPPassword:             &config.SMTPPassword,
		SMTPEnableTLS:            &config.SMTPEnableTLS,
		SMTPAuthenticationMethod: &authMethod,
		SMTPDomain:               &config.SMTPDomain,
		SMTPVerifySSLCerts:       &config.SMTPVerifySSLCerts,
		SMTPEnableStartTLS:       &config.SMTPEnableStartTLS,
	}
}
<|endoftext|>


<commit_before>package tape

import (
	"fmt"
	"net/http"
	"reflect"
	"strings"

	"github.com/labstack/echo"
)

// OK throw exception if !ok
func (e *exception) OK(ok bool, message interface{}) {
	if !ok {
		panic(message)
	}
}

// Err throw exception if err!=nil
func (e *exception) Err(err error, message interface{}) {
	if err != nil {
		if message == "" {
			switch {
			case strings.HasPrefix(err.Error(), "sql:"):
				message = "数据查询失败"
			case strings.HasPrefix(err.Error(), "Error 1"):
				message = "数据操作异常"
			default:
				message = err.Error()
			}
		}
		fmt.Println(reflect.TypeOf(err), err.Error(), message)
		panic(message)
	}
}

// Catch throw exception if recover err
func (e *exception) Catch(ctx echo.Context) func() {
	return func() {
		if r := recover(); r != nil {
			ctx.(echo.Context).JSON(http.StatusOK, Hash{
				STATUS:  FAIL,
				MESSAGE: r,
			})
		}
	}
}
<commit_msg>remove type<commit_after>package tape

import (
	"fmt"
	"net/http"
	"reflect"
	"strings"

	"github.com/labstack/echo"
)

// OK throw exception if !ok
func (e *exception) OK(ok bool, message interface{}) {
	if !ok {
		panic(message)
	}
}

// Err throw exception if err!=nil
func (e *exception) Err(err error, message interface{}) {
	if err != nil {
		if message == "" {
			switch {
			case strings.HasPrefix(err.Error(), "sql:"):
				message = "数据查询失败"
			case strings.HasPrefix(err.Error(), "Error 1"):
				message = "数据操作异常"
			default:
				message = err.Error()
			}
		}
		fmt.Println(reflect.TypeOf(err), err.Error(), message)
		panic(message)
	}
}

// Catch throw exception if recover err
func (e *exception) Catch(ctx echo.Context) func() {
	return func() {
		if r := recover(); r != nil {
			ctx.JSON(http.StatusOK, Hash{
				STATUS:  FAIL,
				MESSAGE: r,
			})
		}
	}
}
<|endoftext|>


<commit_before>package regressiontests

import "testing"

func TestStructcheck(t *testing.T) {
	t.Parallel()
	source := `package test

type test struct {
	unused int
}
`
	expected := Issues{
		{Linter: "structcheck", Severity: "warning", Path: "test.go", Line: 4, Col: 2, Message: "unused struct field github.com/alecthomas/gometalinter/regressiontests/.test.unused"},
	}
	ExpectIssues(t, "structcheck", source, expected)
}
<commit_msg>make tests compatible with forks<commit_after>package regressiontests

import (
	"reflect"
	"testing"
)

type Empty struct{}

func TestStructcheck(t *testing.T) {
	t.Parallel()
	source := `package test

type test struct {
	unused int
}
`
	pkgName := reflect.TypeOf(Empty{}).PkgPath()
	expected := Issues{
		{Linter: "structcheck", Severity: "warning", Path: "test.go", Line: 4, Col: 2, Message: "unused struct field " + pkgName + "/.test.unused"},
	}
	ExpectIssues(t, "structcheck", source, expected)
}
<|endoftext|>


<commit_before>package filesystem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/naoty/todo/todo"
)

// FileSystem represents a repository backed by file system.
type FileSystem struct {
	root string
}

type index struct {
	Todos map[string][]int `json:"todos"`
}

// New returns a new FileSystem.
func New() (*FileSystem, error) {
	home := os.Getenv("TODOS_PATH")
	if home == "" {
		var err error
		home, err = os.UserHomeDir()
		if err != nil {
			return nil, err
		}
	}

	root := filepath.Join(home, ".todos")
	if _, err := os.Stat(root); os.IsNotExist(err) {
		err := os.Mkdir(root, 0755)
		if err != nil {
			return nil, err
		}
	}

	return &FileSystem{root: root}, nil
}

// Get implements Repository interface.
func (repo *FileSystem) Get(id int) (*todo.Todo, error) {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("TODO not found: %d", id)
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td, err := Parse(string(content))
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td.ID = id
	return td, nil
}

// List implements Repository interface.
func (repo *FileSystem) List() ([]*todo.Todo, error) {
	todos := map[int]*todo.Todo{}

	err := filepath.Walk(repo.root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !strings.HasSuffix(path, ".md") {
			return nil
		}

		id, err := parseID(path)
		if err != nil {
			return err
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		content, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		td, err := Parse(string(content))
		if err != nil {
			return err
		}

		td.ID = id
		todos[id] = td

		return nil
	})

	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	i, err := repo.readIndex()
	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	ordered := orderTodos(todos, i, "")

	return ordered, nil
}

// Add implements Repository interface.
func (repo *FileSystem) Add(title string, parent *int) error {
	todos, err := repo.List()
	if err != nil {
		return fmt.Errorf("failed to get next id: %w", err)
	}

	nextID := maxID(todos) + 1
	filename := fmt.Sprintf("%d.md", nextID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); err == nil {
		return fmt.Errorf("already exist: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}
	defer file.Close()

	content := newContent(title)
	_, err = file.WriteString(content)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	parentID := ""
	if *parent > 0 {
		parentID = fmt.Sprintf("%d", *parent)
	}

	st.Todos[parentID] = append(st.Todos[parentID], nextID)
	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	return nil
}

// Update implements Repository interface.
func (repo *FileSystem) Update(td *todo.Todo) error {
	filename := fmt.Sprintf("%d.md", td.ID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0644)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}
	defer file.Close()

	data := Marshal(td)
	_, err = file.Write(data)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}

	return nil
}

// Delete implements Repository interface.
func (repo *FileSystem) Delete(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	err := os.Remove(path)
	if err != nil {
		return fmt.Errorf("failed to delete TODO: %w", err)
	}

	index, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	key := fmt.Sprintf("%d", id)
	subIDs, ok := index.Todos[key]

	if ok {
		for _, id := range subIDs {
			filename := fmt.Sprintf("%d.md", id)
			path := filepath.Join(repo.root, filename)

			if _, err := os.Stat(path); os.IsNotExist(err) {
				return fmt.Errorf("file not found: %s", path)
			}

			err := os.Remove(path)
			if err != nil {
				return fmt.Errorf("failed to delete TODO: %w", err)
			}
		}
	}

	delete(index.Todos, key)

	for k, todos := range index.Todos {
		var ids []int
		for _, _id := range todos {
			if _id == id {
				continue
			}
			ids = append(ids, _id)
		}

		if len(ids) == 0 {
			delete(index.Todos, k)
			continue
		}

		index.Todos[k] = ids
	}

	err = repo.writeIndex(index)
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	return nil
}

// Open implements Repository interface.
func (repo *FileSystem) Open(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	cmd := exec.Command("open", path)
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("failed to open %s: %w", path, err)
	}

	return nil
}

// Move implements Repository interface.
func (repo *FileSystem) Move(id, position int) error {
	if position < 1 {
		return fmt.Errorf("position number must be larger than 0: %d", position)
	}

	to := position - 1

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	ids, _ := st.Todos[""]
	if to >= len(ids) {
		return fmt.Errorf("position number is too large: %d", position)
	}

	var from int
	for j, _id := range ids {
		if _id == id {
			from = j
		}
	}

	st.Todos[""] = swapped(ids, from, to)

	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	return nil
}

func parseID(path string) (int, error) {
	text := strings.TrimRight(filepath.Base(path), filepath.Ext(path))
	return strconv.Atoi(text)
}

func newContent(title string) string {
	return strings.Trim(fmt.Sprintf(`
---
title: %s
state: undone
---
`, title), "\n")
}

func (repo *FileSystem) readIndex() (*index, error) {
	path := filepath.Join(repo.root, "index.json")
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return &index{
			Todos: map[string][]int{
				"": {},
			},
		}, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	var i index
	err = json.Unmarshal(content, &i)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	return &i, nil
}

func (repo *FileSystem) writeIndex(i *index) error {
	path := filepath.Join(repo.root, "index.json")

	data, err := json.MarshalIndent(*i, "", "  ")
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	err = ioutil.WriteFile(path, data, 0644)
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	return nil
}

func swapped(s []int, from, to int) []int {
	if from < 0 || from >= len(s) {
		return s
	}

	if to < 0 || to >= len(s) {
		return s
	}

	copied := make([]int, len(s), cap(s))
	copy(copied, s)

	e := copied[from]
	copied = append(copied[:from], copied[(from+1):]...)
	copied = append(copied[:to], append([]int{e}, copied[to:]...)...)
	return copied
}

func maxID(todos []*todo.Todo) int {
	result := 0

	for _, td := range todos {
		if td.ID > result {
			result = td.ID
		}

		maxSubID := maxID(td.Todos)
		if maxSubID > result {
			result = maxSubID
		}
	}

	return result
}

func orderTodos(todos map[int]*todo.Todo, i *index, key string) []*todo.Todo {
	var result []*todo.Todo

	for _, id := range i.Todos[key] {
		td, ok := todos[id]
		if !ok {
			continue
		}

		newKey := fmt.Sprintf("%d", td.ID)
		td.Todos = orderTodos(todos, i, newKey)
		result = append(result, td)
	}

	return result
}
<commit_msg>Return error when parent is not found<commit_after>package filesystem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/naoty/todo/todo"
)

// FileSystem represents a repository backed by file system.
type FileSystem struct {
	root string
}

type index struct {
	Todos map[string][]int `json:"todos"`
}

// New returns a new FileSystem.
func New() (*FileSystem, error) {
	home := os.Getenv("TODOS_PATH")
	if home == "" {
		var err error
		home, err = os.UserHomeDir()
		if err != nil {
			return nil, err
		}
	}

	root := filepath.Join(home, ".todos")
	if _, err := os.Stat(root); os.IsNotExist(err) {
		err := os.Mkdir(root, 0755)
		if err != nil {
			return nil, err
		}
	}

	return &FileSystem{root: root}, nil
}

// Get implements Repository interface.
func (repo *FileSystem) Get(id int) (*todo.Todo, error) {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("TODO not found: %d", id)
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td, err := Parse(string(content))
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td.ID = id
	return td, nil
}

// List implements Repository interface.
func (repo *FileSystem) List() ([]*todo.Todo, error) {
	todos := map[int]*todo.Todo{}

	err := filepath.Walk(repo.root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !strings.HasSuffix(path, ".md") {
			return nil
		}

		id, err := parseID(path)
		if err != nil {
			return err
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		content, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		td, err := Parse(string(content))
		if err != nil {
			return err
		}

		td.ID = id
		todos[id] = td

		return nil
	})

	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	i, err := repo.readIndex()
	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	ordered := orderTodos(todos, i, "")

	return ordered, nil
}

// Add implements Repository interface.
func (repo *FileSystem) Add(title string, parent *int) error {
	todos, err := repo.List()
	if err != nil {
		return fmt.Errorf("failed to get next id: %w", err)
	}

	if *parent > 0 && !isExists(todos, *parent) {
		return fmt.Errorf("parent not found: %d", *parent)
	}

	nextID := maxID(todos) + 1
	filename := fmt.Sprintf("%d.md", nextID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); err == nil {
		return fmt.Errorf("already exist: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}
	defer file.Close()

	content := newContent(title)
	_, err = file.WriteString(content)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	parentID := ""
	if *parent > 0 {
		parentID = fmt.Sprintf("%d", *parent)
	}

	st.Todos[parentID] = append(st.Todos[parentID], nextID)
	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	return nil
}

// Update implements Repository interface.
func (repo *FileSystem) Update(td *todo.Todo) error {
	filename := fmt.Sprintf("%d.md", td.ID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0644)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}
	defer file.Close()

	data := Marshal(td)
	_, err = file.Write(data)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}

	return nil
}

// Delete implements Repository interface.
func (repo *FileSystem) Delete(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	err := os.Remove(path)
	if err != nil {
		return fmt.Errorf("failed to delete TODO: %w", err)
	}

	index, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	key := fmt.Sprintf("%d", id)
	subIDs, ok := index.Todos[key]

	if ok {
		for _, id := range subIDs {
			filename := fmt.Sprintf("%d.md", id)
			path := filepath.Join(repo.root, filename)

			if _, err := os.Stat(path); os.IsNotExist(err) {
				return fmt.Errorf("file not found: %s", path)
			}

			err := os.Remove(path)
			if err != nil {
				return fmt.Errorf("failed to delete TODO: %w", err)
			}
		}
	}

	delete(index.Todos, key)

	for k, todos := range index.Todos {
		var ids []int
		for _, _id := range todos {
			if _id == id {
				continue
			}
			ids = append(ids, _id)
		}

		if len(ids) == 0 {
			delete(index.Todos, k)
			continue
		}

		index.Todos[k] = ids
	}

	err = repo.writeIndex(index)
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	return nil
}

// Open implements Repository interface.
func (repo *FileSystem) Open(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	cmd := exec.Command("open", path)
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("failed to open %s: %w", path, err)
	}

	return nil
}

// Move implements Repository interface.
func (repo *FileSystem) Move(id, position int) error {
	if position < 1 {
		return fmt.Errorf("position number must be larger than 0: %d", position)
	}

	to := position - 1

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	ids, _ := st.Todos[""]
	if to >= len(ids) {
		return fmt.Errorf("position number is too large: %d", position)
	}

	var from int
	for j, _id := range ids {
		if _id == id {
			from = j
		}
	}

	st.Todos[""] = swapped(ids, from, to)

	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	return nil
}

func parseID(path string) (int, error) {
	text := strings.TrimRight(filepath.Base(path), filepath.Ext(path))
	return strconv.Atoi(text)
}

func newContent(title string) string {
	return strings.Trim(fmt.Sprintf(`
---
title: %s
state: undone
---
`, title), "\n")
}

func (repo *FileSystem) readIndex() (*index, error) {
	path := filepath.Join(repo.root, "index.json")
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return &index{
			Todos: map[string][]int{
				"": {},
			},
		}, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	var i index
	err = json.Unmarshal(content, &i)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	return &i, nil
}

func (repo *FileSystem) writeIndex(i *index) error {
	path := filepath.Join(repo.root, "index.json")

	data, err := json.MarshalIndent(*i, "", "  ")
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	err = ioutil.WriteFile(path, data, 0644)
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	return nil
}

func swapped(s []int, from, to int) []int {
	if from < 0 || from >= len(s) {
		return s
	}

	if to < 0 || to >= len(s) {
		return s
	}

	copied := make([]int, len(s), cap(s))
	copy(copied, s)

	e := copied[from]
	copied = append(copied[:from], copied[(from+1):]...)
	copied = append(copied[:to], append([]int{e}, copied[to:]...)...)
	return copied
}

func maxID(todos []*todo.Todo) int {
	result := 0

	for _, td := range todos {
		if td.ID > result {
			result = td.ID
		}

		maxSubID := maxID(td.Todos)
		if maxSubID > result {
			result = maxSubID
		}
	}

	return result
}

func orderTodos(todos map[int]*todo.Todo, i *index, key string) []*todo.Todo {
	var result []*todo.Todo

	for _, id := range i.Todos[key] {
		td, ok := todos[id]
		if !ok {
			continue
		}

		newKey := fmt.Sprintf("%d", td.ID)
		td.Todos = orderTodos(todos, i, newKey)
		result = append(result, td)
	}

	return result
}

func isExists(todos []*todo.Todo, id int) bool {
	for _, td := range todos {
		if td.ID == id {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package sarama

// OffsetMethod is passed in ConsumerConfig to tell the consumer how to determine the starting offset.
type OffsetMethod int

const (
	// OffsetMethodManual causes the consumer to interpret the OffsetValue in the ConsumerConfig as the
	// offset at which to start, allowing the user to manually specify their desired starting offset.
	OffsetMethodManual OffsetMethod = iota
	// OffsetMethodNewest causes the consumer to start at the most recent available offset, as
	// determined by querying the broker.
	OffsetMethodNewest
	// OffsetMethodOldest causes the consumer to start at the oldest available offset, as
	// determined by querying the broker.
	OffsetMethodOldest
)

// ConsumerConfig is used to pass multiple configuration options to NewConsumer.
type ConsumerConfig struct {
	// The default (maximum) amount of data to fetch from the broker in each request. The default of 0 is treated as 1024 bytes.
	DefaultFetchSize int32
	// The minimum amount of data to fetch in a request - the broker will wait until at least this many bytes are available.
	// The default of 0 is treated as 'at least one' to prevent the consumer from spinning when no messages are available.
	MinFetchSize int32
	// The maximum permittable message size - messages larger than this will return MessageTooLarge. The default of 0 is
	// treated as no limit.
	MaxMessageSize int32
	// The maximum amount of time (in ms) the broker will wait for MinFetchSize bytes to become available before it
	// returns fewer than that anyways. The default of 0 causes Kafka to return immediately, which is rarely desirable
	// as it causes the Consumer to spin when no events are available. 100-500ms is a reasonable range for most cases.
	MaxWaitTime int32

	// The method used to determine at which offset to begin consuming messages.
	OffsetMethod OffsetMethod
	// Interpreted differently according to the value of OffsetMethod.
	OffsetValue int64

	// The number of events to buffer in the Events channel. Setting this can let the
	// consumer continue fetching messages in the background while local code consumes events,
	// greatly improving throughput.
	EventBufferSize int
}

// ConsumerEvent is what is provided to the user when an event occurs. It is either an error (in which case Err is non-nil) or
// a message (in which case Err is nil and the other fields are all set).
type ConsumerEvent struct {
	Key, Value []byte
	Topic      string
	Partition  int32
	Offset     int64
	Err        error
}

// Consumer processes Kafka messages from a given topic and partition.
// You MUST call Close() on a consumer to avoid leaks, it will not be garbage-collected automatically when
// it passes out of scope (this is in addition to calling Close on the underlying client, which is still necessary).
type Consumer struct {
	client *Client

	topic     string
	partition int32
	group     string
	config    ConsumerConfig

	offset        int64
	broker        *Broker
	stopper, done chan bool
	events        chan *ConsumerEvent
}

// NewConsumer creates a new consumer attached to the given client. It will read messages from the given topic and partition, as
// part of the named consumer group.
func NewConsumer(client *Client, topic string, partition int32, group string, config *ConsumerConfig) (*Consumer, error) {
	if config == nil {
		config = new(ConsumerConfig)
	}

	if config.DefaultFetchSize < 0 {
		return nil, ConfigurationError("Invalid DefaultFetchSize")
	} else if config.DefaultFetchSize == 0 {
		config.DefaultFetchSize = 1024
	}

	if config.MinFetchSize < 0 {
		return nil, ConfigurationError("Invalid MinFetchSize")
	} else if config.MinFetchSize == 0 {
		config.MinFetchSize = 1
	}

	if config.MaxMessageSize < 0 {
		return nil, ConfigurationError("Invalid MaxMessageSize")
	}

	if config.MaxWaitTime <= 0 {
		return nil, ConfigurationError("Invalid MaxWaitTime")
	} else if config.MaxWaitTime < 100 {
		Logger.Println("ConsumerConfig.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
	}

	if config.EventBufferSize < 0 {
		return nil, ConfigurationError("Invalid EventBufferSize")
	}

	if topic == "" {
		return nil, ConfigurationError("Empty topic")
	}

	broker, err := client.Leader(topic, partition)
	if err != nil {
		return nil, err
	}

	c := &Consumer{
		client:    client,
		topic:     topic,
		partition: partition,
		group:     group,
		config:    *config,
		broker:    broker,
		stopper:   make(chan bool),
		done:      make(chan bool),
		events:    make(chan *ConsumerEvent, config.EventBufferSize),
	}

	switch config.OffsetMethod {
	case OffsetMethodManual:
		if config.OffsetValue < 0 {
			return nil, ConfigurationError("OffsetValue cannot be < 0 when OffsetMethod is MANUAL")
		}
		c.offset = config.OffsetValue
	case OffsetMethodNewest:
		c.offset, err = c.getOffset(LatestOffsets, true)
		if err != nil {
			return nil, err
		}
	case OffsetMethodOldest:
		c.offset, err = c.getOffset(EarliestOffset, true)
		if err != nil {
			return nil, err
		}
	default:
		return nil, ConfigurationError("Invalid OffsetMethod")
	}

	go withRecover(c.fetchMessages)

	return c, nil
}

// Events returns the read channel for any events (messages or errors) that might be returned by the broker.
func (c *Consumer) Events() <-chan *ConsumerEvent {
	return c.events
}

// Close stops the consumer from fetching messages. It is required to call this function before
// a consumer object passes out of scope, as it will otherwise leak memory. You must call this before
// calling Close on the underlying client.
func (c *Consumer) Close() error {
	close(c.stopper)
	<-c.done
	return nil
}

// helper function for safely sending an error on the errors channel
// if it returns true, the error was sent (or was nil)
// if it returns false, the stopper channel signaled that your goroutine should return!
func (c *Consumer) sendError(err error) bool {
	if err == nil {
		return true
	}

	select {
	case <-c.stopper:
		close(c.events)
		close(c.done)
		return false
	case c.events <- &ConsumerEvent{Err: err}:
		return true
	}
}

func (c *Consumer) fetchMessages() {

	fetchSize := c.config.DefaultFetchSize

	for {
		request := new(FetchRequest)
		request.MinBytes = c.config.MinFetchSize
		request.MaxWaitTime = c.config.MaxWaitTime
		request.AddBlock(c.topic, c.partition, c.offset, fetchSize)

		response, err := c.broker.Fetch(c.client.id, request)
		switch {
		case err == nil:
			break
		case err == EncodingError:
			if c.sendError(err) {
				continue
			} else {
				return
			}
		default:
			Logger.Printf("Unexpected error processing FetchRequest; disconnecting broker %s: %s\n", c.broker.addr, err)
			c.client.disconnectBroker(c.broker)
			for c.broker, err = c.client.Leader(c.topic, c.partition); err != nil; c.broker, err = c.client.Leader(c.topic, c.partition) {
				if !c.sendError(err) {
					return
				}
			}
			continue
		}

		block := response.GetBlock(c.topic, c.partition)
		if block == nil {
			if c.sendError(IncompleteResponse) {
				continue
			} else {
				return
			}
		}

		switch block.Err {
		case NoError:
			break
		case UnknownTopicOrPartition, NotLeaderForPartition, LeaderNotAvailable:
			err = c.client.RefreshTopicMetadata(c.topic)
			if c.sendError(err) {
				for c.broker, err = c.client.Leader(c.topic, c.partition); err != nil; c.broker, err = c.client.Leader(c.topic, c.partition) {
					if !c.sendError(err) {
						return
					}
				}
				continue
			} else {
				return
			}
		default:
			if c.sendError(block.Err) {
				continue
			} else {
				return
			}
		}

		if len(block.MsgSet.Messages) == 0 {
			// We got no messages. If we got a trailing one then we need to ask for more data.
			// Otherwise we just poll again and wait for one to be produced...
			if block.MsgSet.PartialTrailingMessage {
				if c.config.MaxMessageSize == 0 {
					fetchSize *= 2
				} else {
					if fetchSize == c.config.MaxMessageSize {
						if c.sendError(MessageTooLarge) {
							continue
						} else {
							return
						}
					} else {
						fetchSize *= 2
						if fetchSize > c.config.MaxMessageSize {
							fetchSize = c.config.MaxMessageSize
						}
					}
				}
			}
			select {
			case <-c.stopper:
				close(c.events)
				close(c.done)
				return
			default:
				continue
			}
		} else {
			fetchSize = c.config.DefaultFetchSize
		}

		for _, msgBlock := range block.MsgSet.Messages {
			for _, msg := range msgBlock.Messages() {
				select {
				case <-c.stopper:
					close(c.events)
					close(c.done)
					return
				case c.events <- &ConsumerEvent{Key: msg.Msg.Key, Value: msg.Msg.Value, Offset: msg.Offset, Topic: c.topic, Partition: c.partition}:
					c.offset++
				}
			}
		}
	}
}

func (c *Consumer) getOffset(where OffsetTime, retry bool) (int64, error) {
	request := &OffsetRequest{}
	request.AddBlock(c.topic, c.partition, where, 1)

	response, err := c.broker.GetAvailableOffsets(c.client.id, request)
	switch err {
	case nil:
		break
	case EncodingError:
		return -1, err
	default:
		if !retry {
			return -1, err
		}
		Logger.Printf("Unexpected error processing OffsetRequest; disconnecting broker %s: %s\n", c.broker.addr, err)
		c.client.disconnectBroker(c.broker)
		c.broker, err = c.client.Leader(c.topic, c.partition)
		if err != nil {
			return -1, err
		}
		return c.getOffset(where, false)
	}

	block := response.GetBlock(c.topic, c.partition)
	if block == nil {
		return -1, IncompleteResponse
	}

	switch block.Err {
	case NoError:
		if len(block.Offsets) < 1 {
			return -1, IncompleteResponse
		}
		return block.Offsets[0], nil
	case UnknownTopicOrPartition, NotLeaderForPartition, LeaderNotAvailable:
		if !retry {
			return -1, block.Err
		}
		err = c.client.RefreshTopicMetadata(c.topic)
		if err != nil {
			return -1, err
		}
		c.broker, err = c.client.Leader(c.topic, c.partition)
		if err != nil {
			return -1, err
		}
		return c.getOffset(where, false)
	}

	return -1, block.Err
}
<commit_msg>Update docs and sendError<commit_after>package sarama

// OffsetMethod is passed in ConsumerConfig to tell the consumer how to determine the starting offset.
type OffsetMethod int

const (
	// OffsetMethodManual causes the consumer to interpret the OffsetValue in the ConsumerConfig as the
	// offset at which to start, allowing the user to manually specify their desired starting offset.
	OffsetMethodManual OffsetMethod = iota
	// OffsetMethodNewest causes the consumer to start at the most recent available offset, as
	// determined by querying the broker.
	OffsetMethodNewest
	// OffsetMethodOldest causes the consumer to start at the oldest available offset, as
	// determined by querying the broker.
	OffsetMethodOldest
)

// ConsumerConfig is used to pass multiple configuration options to NewConsumer.
type ConsumerConfig struct {
	// The default (maximum) amount of data to fetch from the broker in each request. The default of 0 is treated as 1024 bytes.
	DefaultFetchSize int32
	// The minimum amount of data to fetch in a request - the broker will wait until at least this many bytes are available.
	// The default of 0 is treated as 'at least one' to prevent the consumer from spinning when no messages are available.
	MinFetchSize int32
	// The maximum permittable message size - messages larger than this will return MessageTooLarge. The default of 0 is
	// treated as no limit.
	MaxMessageSize int32
	// The maximum amount of time (in ms) the broker will wait for MinFetchSize bytes to become available before it
	// returns fewer than that anyways. The default of 0 causes Kafka to return immediately, which is rarely desirable
	// as it causes the Consumer to spin when no events are available. 100-500ms is a reasonable range for most cases.
	MaxWaitTime int32

	// The method used to determine at which offset to begin consuming messages.
	OffsetMethod OffsetMethod
	// Interpreted differently according to the value of OffsetMethod.
	OffsetValue int64

	// The number of events to buffer in the Events channel. Setting this can let the
	// consumer continue fetching messages in the background while local code consumes events,
	// greatly improving throughput.
	EventBufferSize int
}

// ConsumerEvent is what is provided to the user when an event occurs. It is either an error (in which case Err is non-nil) or
// a message (in which case Err is nil and Offset, Key, and Value are set). Topic and Partition are always set.
type ConsumerEvent struct {
	Key, Value []byte
	Topic      string
	Partition  int32
	Offset     int64
	Err        error
}

// Consumer processes Kafka messages from a given topic and partition.
// You MUST call Close() on a consumer to avoid leaks, it will not be garbage-collected automatically when
// it passes out of scope (this is in addition to calling Close on the underlying client, which is still necessary).
type Consumer struct {
	client *Client

	topic     string
	partition int32
	group     string
	config    ConsumerConfig

	offset        int64
	broker        *Broker
	stopper, done chan bool
	events        chan *ConsumerEvent
}

// NewConsumer creates a new consumer attached to the given client. It will read messages from the given topic and partition, as
// part of the named consumer group.
func NewConsumer(client *Client, topic string, partition int32, group string, config *ConsumerConfig) (*Consumer, error) {
	if config == nil {
		config = new(ConsumerConfig)
	}

	if config.DefaultFetchSize < 0 {
		return nil, ConfigurationError("Invalid DefaultFetchSize")
	} else if config.DefaultFetchSize == 0 {
		config.DefaultFetchSize = 1024
	}

	if config.MinFetchSize < 0 {
		return nil, ConfigurationError("Invalid MinFetchSize")
	} else if config.MinFetchSize == 0 {
		config.MinFetchSize = 1
	}

	if config.MaxMessageSize < 0 {
		return nil, ConfigurationError("Invalid MaxMessageSize")
	}

	if config.MaxWaitTime <= 0 {
		return nil, ConfigurationError("Invalid MaxWaitTime")
	} else if config.MaxWaitTime < 100 {
		Logger.Println("ConsumerConfig.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.")
	}

	if config.EventBufferSize < 0 {
		return nil, ConfigurationError("Invalid EventBufferSize")
	}

	if topic == "" {
		return nil, ConfigurationError("Empty topic")
	}

	broker, err := client.Leader(topic, partition)
	if err != nil {
		return nil, err
	}

	c := &Consumer{
		client:    client,
		topic:     topic,
		partition: partition,
		group:     group,
		config:    *config,
		broker:    broker,
		stopper:   make(chan bool),
		done:      make(chan bool),
		events:    make(chan *ConsumerEvent, config.EventBufferSize),
	}

	switch config.OffsetMethod {
	case OffsetMethodManual:
		if config.OffsetValue < 0 {
			return nil, ConfigurationError("OffsetValue cannot be < 0 when OffsetMethod is MANUAL")
		}
		c.offset = config.OffsetValue
	case OffsetMethodNewest:
		c.offset, err = c.getOffset(LatestOffsets, true)
		if err != nil {
			return nil, err
		}
	case OffsetMethodOldest:
		c.offset, err = c.getOffset(EarliestOffset, true)
		if err != nil {
			return nil, err
		}
	default:
		return nil, ConfigurationError("Invalid OffsetMethod")
	}

	go withRecover(c.fetchMessages)

	return c, nil
}

// Events returns the read channel for any events (messages or errors) that might be returned by the broker.
func (c *Consumer) Events() <-chan *ConsumerEvent {
	return c.events
}

// Close stops the consumer from fetching messages. It is required to call this function before
// a consumer object passes out of scope, as it will otherwise leak memory. You must call this before
// calling Close on the underlying client.
func (c *Consumer) Close() error {
	close(c.stopper)
	<-c.done
	return nil
}

// helper function for safely sending an error on the errors channel
// if it returns true, the error was sent (or was nil)
// if it returns false, the stopper channel signaled that your goroutine should return!
func (c *Consumer) sendError(err error) bool {
	if err == nil {
		return true
	}

	select {
	case <-c.stopper:
		close(c.events)
		close(c.done)
		return false
	case c.events <- &ConsumerEvent{Err: err, Topic: c.topic, Partition: c.partition}:
		return true
	}
}

func (c *Consumer) fetchMessages() {

	fetchSize := c.config.DefaultFetchSize

	for {
		request := new(FetchRequest)
		request.MinBytes = c.config.MinFetchSize
		request.MaxWaitTime = c.config.MaxWaitTime
		request.AddBlock(c.topic, c.partition, c.offset, fetchSize)

		response, err := c.broker.Fetch(c.client.id, request)
		switch {
		case err == nil:
			break
		case err == EncodingError:
			if c.sendError(err) {
				continue
			} else {
				return
			}
		default:
			Logger.Printf("Unexpected error processing FetchRequest; disconnecting broker %s: %s\n", c.broker.addr, err)
			c.client.disconnectBroker(c.broker)
			for c.broker, err = c.client.Leader(c.topic, c.partition); err != nil; c.broker, err = c.client.Leader(c.topic, c.partition) {
				if !c.sendError(err) {
					return
				}
			}
			continue
		}

		block := response.GetBlock(c.topic, c.partition)
		if block == nil {
			if c.sendError(IncompleteResponse) {
				continue
			} else {
				return
			}
		}

		switch block.Err {
		case NoError:
			break
		case UnknownTopicOrPartition, NotLeaderForPartition, LeaderNotAvailable:
			err = c.client.RefreshTopicMetadata(c.topic)
			if c.sendError(err) {
				for c.broker, err = c.client.Leader(c.topic, c.partition); err != nil; c.broker, err = c.client.Leader(c.topic, c.partition) {
					if !c.sendError(err) {
						return
					}
				}
				continue
			} else {
				return
			}
		default:
			if c.sendError(block.Err) {
				continue
			} else {
				return
			}
		}

		if len(block.MsgSet.Messages) == 0 {
			// We got no messages. If we got a trailing one then we need to ask for more data.
			// Otherwise we just poll again and wait for one to be produced...
			if block.MsgSet.PartialTrailingMessage {
				if c.config.MaxMessageSize == 0 {
					fetchSize *= 2
				} else {
					if fetchSize == c.config.MaxMessageSize {
						if c.sendError(MessageTooLarge) {
							continue
						} else {
							return
						}
					} else {
						fetchSize *= 2
						if fetchSize > c.config.MaxMessageSize {
							fetchSize = c.config.MaxMessageSize
						}
					}
				}
			}
			select {
			case <-c.stopper:
				close(c.events)
				close(c.done)
				return
			default:
				continue
			}
		} else {
			fetchSize = c.config.DefaultFetchSize
		}

		for _, msgBlock := range block.MsgSet.Messages {
			for _, msg := range msgBlock.Messages() {
				select {
				case <-c.stopper:
					close(c.events)
					close(c.done)
					return
				case c.events <- &ConsumerEvent{Key: msg.Msg.Key, Value: msg.Msg.Value, Offset: msg.Offset, Topic: c.topic, Partition: c.partition}:
					c.offset++
				}
			}
		}
	}
}

func (c *Consumer) getOffset(where OffsetTime, retry bool) (int64, error) {
	request := &OffsetRequest{}
	request.AddBlock(c.topic, c.partition, where, 1)

	response, err := c.broker.GetAvailableOffsets(c.client.id, request)
	switch err {
	case nil:
		break
	case EncodingError:
		return -1, err
	default:
		if !retry {
			return -1, err
		}
		Logger.Printf("Unexpected error processing OffsetRequest; disconnecting broker %s: %s\n", c.broker.addr, err)
		c.client.disconnectBroker(c.broker)
		c.broker, err = c.client.Leader(c.topic, c.partition)
		if err != nil {
			return -1, err
		}
		return c.getOffset(where, false)
	}

	block := response.GetBlock(c.topic, c.partition)
	if block == nil {
		return -1, IncompleteResponse
	}

	switch block.Err {
	case NoError:
		if len(block.Offsets) < 1 {
			return -1, IncompleteResponse
		}
		return block.Offsets[0], nil
	case UnknownTopicOrPartition, NotLeaderForPartition, LeaderNotAvailable:
		if !retry {
			return -1, block.Err
		}
		err = c.client.RefreshTopicMetadata(c.topic)
		if err != nil {
			return -1, err
		}
		c.broker, err = c.client.Leader(c.topic, c.partition)
		if err != nil {
			return -1, err
		}
		return c.getOffset(where, false)
	}

	return -1, block.Err
}
<|endoftext|>


<commit_before>package google

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
	"google.golang.org/api/googleapi"
)

// resourceGoogleProject returns a *schema.Resource that allows a customer
// to declare a Google Cloud Project resource.
//
// This example shows a project with a policy declared in config:
//
// resource "google_project" "my-project" {
//    project = "a-project-id"
//    policy = "${data.google_iam_policy.admin.policy}"
// }
func resourceGoogleProject() *schema.Resource {
	return &schema.Resource{
		SchemaVersion: 1,

		Create: resourceGoogleProjectCreate,
		Read:   resourceGoogleProjectRead,
		Update: resourceGoogleProjectUpdate,
		Delete: resourceGoogleProjectDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		MigrateState: resourceGoogleProjectMigrateState,

		Schema: map[string]*schema.Schema{
			"id": &schema.Schema{
				Type:       schema.TypeString,
				Optional:   true,
				Computed:   true,
				Deprecated: "The id field has unexpected behaviour and probably doesn't do what you expect. See https://www.terraform.io/docs/providers/google/r/google_project.html#id-field for more information. Please use project_id instead; future versions of Terraform will remove the id field.",
			},
			"project_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
					// This suppresses the diff if project_id is not set
					if new == "" {
						return true
					}
					return false
				},
			},
			"skip_delete": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"org_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"policy_data": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				Computed:         true,
				Deprecated:       "Use the 'google_project_iam_policy' resource to define policies for a Google Project",
				DiffSuppressFunc: jsonPolicyDiffSuppress,
			},
			"policy_etag": &schema.Schema{
				Type:       schema.TypeString,
				Computed:   true,
				Deprecated: "Use the the 'google_project_iam_policy' resource to define policies for a Google Project",
			},
			"number": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceGoogleProjectCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	var pid string
	var err error
	pid = d.Get("project_id").(string)
	if pid == "" {
		pid, err = getProject(d, config)
		if err != nil {
			return fmt.Errorf("Error getting project ID: %v", err)
		}
		if pid == "" {
			return fmt.Errorf("'project_id' must be set in the config")
		}
	}

	// we need to check if name and org_id are set, and throw an error if they aren't
	// we can't just set these as required on the object, however, as that would break
	// all configs that used previous iterations of the resource.
	// TODO(paddy): remove this for 0.9 and set these attributes as required.
	name, org_id := d.Get("name").(string), d.Get("org_id").(string)
	if name == "" {
		return fmt.Errorf("`name` must be set in the config if you're creating a project.")
	}
	if org_id == "" {
		return fmt.Errorf("`org_id` must be set in the config if you're creating a project.")
	}

	log.Printf("[DEBUG]: Creating new project %q", pid)
	project := &cloudresourcemanager.Project{
		ProjectId: pid,
		Name:      d.Get("name").(string),
		Parent: &cloudresourcemanager.ResourceId{
			Id:   d.Get("org_id").(string),
			Type: "organization",
		},
	}

	op, err := config.clientResourceManager.Projects.Create(project).Do()
	if err != nil {
		return fmt.Errorf("Error creating project %s (%s): %s.", project.ProjectId, project.Name, err)
	}

	d.SetId(pid)

	// Wait for the operation to complete
	waitErr := resourceManagerOperationWait(config, op, "project to create")
	if waitErr != nil {
		return waitErr
	}

	// Apply the IAM policy if it is set
	if pString, ok := d.GetOk("policy_data"); ok {
		// The policy string is just a marshaled cloudresourcemanager.Policy.
		// Unmarshal it to a struct.
		var policy cloudresourcemanager.Policy
		if err := json.Unmarshal([]byte(pString.(string)), &policy); err != nil {
			return err
		}
		log.Printf("[DEBUG] Got policy from config: %#v", policy.Bindings)

		// Retrieve existing IAM policy from project. This will be merged
		// with the policy defined here.
		p, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return err
		}
		log.Printf("[DEBUG] Got existing bindings from project: %#v", p.Bindings)

		// Merge the existing policy bindings with those defined in this manifest.
		p.Bindings = mergeBindings(append(p.Bindings, policy.Bindings...))

		// Apply the merged policy
		log.Printf("[DEBUG] Setting new policy for project: %#v", p)
		_, err = config.clientResourceManager.Projects.SetIamPolicy(pid,
			&cloudresourcemanager.SetIamPolicyRequest{Policy: p}).Do()

		if err != nil {
			return fmt.Errorf("Error applying IAM policy for project %q: %s", pid, err)
		}
	}

	return resourceGoogleProjectRead(d, meta)
}

func resourceGoogleProjectRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()

	// Read the project
	p, err := config.clientResourceManager.Projects.Get(pid).Do()
	if err != nil {
		if v, ok := err.(*googleapi.Error); ok && v.Code == http.StatusNotFound {
			return fmt.Errorf("Project %q does not exist.", pid)
		}
		return fmt.Errorf("Error checking project %q: %s", pid, err)
	}

	d.Set("project_id", pid)
	d.Set("number", strconv.FormatInt(int64(p.ProjectNumber), 10))
	d.Set("name", p.Name)

	if p.Parent != nil {
		d.Set("org_id", p.Parent.Id)
	}

	// Read the IAM policy
	pol, err := getProjectIamPolicy(pid, config)
	if err != nil {
		return err
	}

	polBytes, err := json.Marshal(pol)
	if err != nil {
		return err
	}

	d.Set("policy_etag", pol.Etag)
	d.Set("policy_data", string(polBytes))

	return nil
}

func resourceGoogleProjectUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()

	// Read the project
	// we need the project even though refresh has already been called
	// because the API doesn't support patch, so we need the actual object
	p, err := config.clientResourceManager.Projects.Get(pid).Do()
	if err != nil {
		if v, ok := err.(*googleapi.Error); ok && v.Code == http.StatusNotFound {
			return fmt.Errorf("Project %q does not exist.", pid)
		}
		return fmt.Errorf("Error checking project %q: %s", pid, err)
	}

	// Project name has changed
	if ok := d.HasChange("name"); ok {
		p.Name = d.Get("name").(string)
		// Do update on project
		p, err = config.clientResourceManager.Projects.Update(p.ProjectId, p).Do()
		if err != nil {
			return fmt.Errorf("Error updating project %q: %s", p.Name, err)
		}
	}

	return updateProjectIamPolicy(d, config, pid)
}

func resourceGoogleProjectDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	// Only delete projects if skip_delete isn't set
	if !d.Get("skip_delete").(bool) {
		pid := d.Id()
		_, err := config.clientResourceManager.Projects.Delete(pid).Do()
		if err != nil {
			return fmt.Errorf("Error deleting project %q: %s", pid, err)
		}
	}
	d.SetId("")
	return nil
}

func updateProjectIamPolicy(d *schema.ResourceData, config *Config, pid string) error {
	// Policy has changed
	if ok := d.HasChange("policy_data"); ok {
		// The policy string is just a marshaled cloudresourcemanager.Policy.
		// Unmarshal it to a struct that contains the old and new policies
		oldP, newP := d.GetChange("policy_data")
		oldPString := oldP.(string)
		newPString := newP.(string)

		// JSON Unmarshaling would fail
		if oldPString == "" {
			oldPString = "{}"
		}
		if newPString == "" {
			newPString = "{}"
		}

		log.Printf("[DEBUG]: Old policy: %q\nNew policy: %q", oldPString, newPString)

		var oldPolicy, newPolicy cloudresourcemanager.Policy
		if err := json.Unmarshal([]byte(newPString), &newPolicy); err != nil {
			return err
		}
		if err := json.Unmarshal([]byte(oldPString), &oldPolicy); err != nil {
			return err
		}

		// Find any Roles and Members that were removed (i.e., those that are present
		// in the old but absent in the new
		oldMap := rolesToMembersMap(oldPolicy.Bindings)
		newMap := rolesToMembersMap(newPolicy.Bindings)
		deleted := make(map[string]map[string]bool)

		// Get each role and its associated members in the old state
		for role, members := range oldMap {
			// Initialize map for role
			if _, ok := deleted[role]; !ok {
				deleted[role] = make(map[string]bool)
			}
			// The role exists in the new state
			if _, ok := newMap[role]; ok {
				// Check each memeber
				for member, _ := range members {
					// Member does not exist in new state, so it was deleted
					if _, ok = newMap[role][member]; !ok {
						deleted[role][member] = true
					}
				}
			} else {
				// This indicates an entire role was deleted. Mark all members
				// for delete.
				for member, _ := range members {
					deleted[role][member] = true
				}
			}
		}
		log.Printf("[DEBUG] Roles and Members to be deleted: %#v", deleted)

		// Retrieve existing IAM policy from project. This will be merged
		// with the policy in the current state
		// TODO(evanbrown): Add an 'authoritative' flag that allows policy
		// in manifest to overwrite existing policy.
		p, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return err
		}
		log.Printf("[DEBUG] Got existing bindings from project: %#v", p.Bindings)

		// Merge existing policy with policy in the current state
		log.Printf("[DEBUG] Merging new bindings from project: %#v", newPolicy.Bindings)
		mergedBindings := mergeBindings(append(p.Bindings, newPolicy.Bindings...))

		// Remove any roles and members that were explicitly deleted
		mergedBindingsMap := rolesToMembersMap(mergedBindings)
		for role, members := range deleted {
			for member, _ := range members {
				delete(mergedBindingsMap[role], member)
			}
		}

		p.Bindings = rolesToMembersBinding(mergedBindingsMap)
		dump, _ := json.MarshalIndent(p.Bindings, " ", "  ")
		log.Printf("[DEBUG] Setting new policy for project: %#v:\n%s", p, string(dump))

		_, err = config.clientResourceManager.Projects.SetIamPolicy(pid,
			&cloudresourcemanager.SetIamPolicyRequest{Policy: p}).Do()

		if err != nil {
			return fmt.Errorf("Error applying IAM policy for project %q: %s", pid, err)
		}
	}
	return nil
}
<commit_msg>providers/google: Fix google_project IAM bug<commit_after>package google

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
	"google.golang.org/api/googleapi"
)

// resourceGoogleProject returns a *schema.Resource that allows a customer
// to declare a Google Cloud Project resource.
//
// This example shows a project with a policy declared in config:
//
// resource "google_project" "my-project" {
//    project = "a-project-id"
//    policy = "${data.google_iam_policy.admin.policy}"
// }
func resourceGoogleProject() *schema.Resource {
	return &schema.Resource{
		SchemaVersion: 1,

		Create: resourceGoogleProjectCreate,
		Read:   resourceGoogleProjectRead,
		Update: resourceGoogleProjectUpdate,
		Delete: resourceGoogleProjectDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		MigrateState: resourceGoogleProjectMigrateState,

		Schema: map[string]*schema.Schema{
			"id": &schema.Schema{
				Type:       schema.TypeString,
				Optional:   true,
				Computed:   true,
				Deprecated: "The id field has unexpected behaviour and probably doesn't do what you expect. See https://www.terraform.io/docs/providers/google/r/google_project.html#id-field for more information. Please use project_id instead; future versions of Terraform will remove the id field.",
			},
			"project_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
					// This suppresses the diff if project_id is not set
					if new == "" {
						return true
					}
					return false
				},
			},
			"skip_delete": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"org_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"policy_data": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				Computed:         true,
				Deprecated:       "Use the 'google_project_iam_policy' resource to define policies for a Google Project",
				DiffSuppressFunc: jsonPolicyDiffSuppress,
			},
			"policy_etag": &schema.Schema{
				Type:       schema.TypeString,
				Computed:   true,
				Deprecated: "Use the the 'google_project_iam_policy' resource to define policies for a Google Project",
			},
			"number": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceGoogleProjectCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	var pid string
	var err error
	pid = d.Get("project_id").(string)
	if pid == "" {
		pid, err = getProject(d, config)
		if err != nil {
			return fmt.Errorf("Error getting project ID: %v", err)
		}
		if pid == "" {
			return fmt.Errorf("'project_id' must be set in the config")
		}
	}

	// we need to check if name and org_id are set, and throw an error if they aren't
	// we can't just set these as required on the object, however, as that would break
	// all configs that used previous iterations of the resource.
	// TODO(paddy): remove this for 0.9 and set these attributes as required.
	name, org_id := d.Get("name").(string), d.Get("org_id").(string)
	if name == "" {
		return fmt.Errorf("`name` must be set in the config if you're creating a project.")
	}
	if org_id == "" {
		return fmt.Errorf("`org_id` must be set in the config if you're creating a project.")
	}

	log.Printf("[DEBUG]: Creating new project %q", pid)
	project := &cloudresourcemanager.Project{
		ProjectId: pid,
		Name:      d.Get("name").(string),
		Parent: &cloudresourcemanager.ResourceId{
			Id:   d.Get("org_id").(string),
			Type: "organization",
		},
	}

	op, err := config.clientResourceManager.Projects.Create(project).Do()
	if err != nil {
		return fmt.Errorf("Error creating project %s (%s): %s.", project.ProjectId, project.Name, err)
	}

	d.SetId(pid)

	// Wait for the operation to complete
	waitErr := resourceManagerOperationWait(config, op, "project to create")
	if waitErr != nil {
		return waitErr
	}

	// Apply the IAM policy if it is set
	if pString, ok := d.GetOk("policy_data"); ok {
		// The policy string is just a marshaled cloudresourcemanager.Policy.
		// Unmarshal it to a struct.
		var policy cloudresourcemanager.Policy
		if err := json.Unmarshal([]byte(pString.(string)), &policy); err != nil {
			return err
		}
		log.Printf("[DEBUG] Got policy from config: %#v", policy.Bindings)

		// Retrieve existing IAM policy from project. This will be merged
		// with the policy defined here.
		p, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return err
		}
		log.Printf("[DEBUG] Got existing bindings from project: %#v", p.Bindings)

		// Merge the existing policy bindings with those defined in this manifest.
		p.Bindings = mergeBindings(append(p.Bindings, policy.Bindings...))

		// Apply the merged policy
		log.Printf("[DEBUG] Setting new policy for project: %#v", p)
		_, err = config.clientResourceManager.Projects.SetIamPolicy(pid,
			&cloudresourcemanager.SetIamPolicyRequest{Policy: p}).Do()

		if err != nil {
			return fmt.Errorf("Error applying IAM policy for project %q: %s", pid, err)
		}
	}

	return resourceGoogleProjectRead(d, meta)
}

func resourceGoogleProjectRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()

	// Read the project
	p, err := config.clientResourceManager.Projects.Get(pid).Do()
	if err != nil {
		if v, ok := err.(*googleapi.Error); ok && v.Code == http.StatusNotFound {
			return fmt.Errorf("Project %q does not exist.", pid)
		}
		return fmt.Errorf("Error checking project %q: %s", pid, err)
	}

	d.Set("project_id", pid)
	d.Set("number", strconv.FormatInt(int64(p.ProjectNumber), 10))
	d.Set("name", p.Name)

	if p.Parent != nil {
		d.Set("org_id", p.Parent.Id)
	}

	return nil
}

func resourceGoogleProjectUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()

	// Read the project
	// we need the project even though refresh has already been called
	// because the API doesn't support patch, so we need the actual object
	p, err := config.clientResourceManager.Projects.Get(pid).Do()
	if err != nil {
		if v, ok := err.(*googleapi.Error); ok && v.Code == http.StatusNotFound {
			return fmt.Errorf("Project %q does not exist.", pid)
		}
		return fmt.Errorf("Error checking project %q: %s", pid, err)
	}

	// Project name has changed
	if ok := d.HasChange("name"); ok {
		p.Name = d.Get("name").(string)
		// Do update on project
		p, err = config.clientResourceManager.Projects.Update(p.ProjectId, p).Do()
		if err != nil {
			return fmt.Errorf("Error updating project %q: %s", p.Name, err)
		}
	}

	return updateProjectIamPolicy(d, config, pid)
}

func resourceGoogleProjectDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	// Only delete projects if skip_delete isn't set
	if !d.Get("skip_delete").(bool) {
		pid := d.Id()
		_, err := config.clientResourceManager.Projects.Delete(pid).Do()
		if err != nil {
			return fmt.Errorf("Error deleting project %q: %s", pid, err)
		}
	}
	d.SetId("")
	return nil
}

func updateProjectIamPolicy(d *schema.ResourceData, config *Config, pid string) error {
	// Policy has changed
	if ok := d.HasChange("policy_data"); ok {
		// The policy string is just a marshaled cloudresourcemanager.Policy.
		// Unmarshal it to a struct that contains the old and new policies
		oldP, newP := d.GetChange("policy_data")
		oldPString := oldP.(string)
		newPString := newP.(string)

		// JSON Unmarshaling would fail
		if oldPString == "" {
			oldPString = "{}"
		}
		if newPString == "" {
			newPString = "{}"
		}

		log.Printf("[DEBUG]: Old policy: %q\nNew policy: %q", oldPString, newPString)

		var oldPolicy, newPolicy cloudresourcemanager.Policy
		if err := json.Unmarshal([]byte(newPString), &newPolicy); err != nil {
			return err
		}
		if err := json.Unmarshal([]byte(oldPString), &oldPolicy); err != nil {
			return err
		}

		// Find any Roles and Members that were removed (i.e., those that are present
		// in the old but absent in the new
		oldMap := rolesToMembersMap(oldPolicy.Bindings)
		newMap := rolesToMembersMap(newPolicy.Bindings)
		deleted := make(map[string]map[string]bool)

		// Get each role and its associated members in the old state
		for role, members := range oldMap {
			// Initialize map for role
			if _, ok := deleted[role]; !ok {
				deleted[role] = make(map[string]bool)
			}
			// The role exists in the new state
			if _, ok := newMap[role]; ok {
				// Check each memeber
				for member, _ := range members {
					// Member does not exist in new state, so it was deleted
					if _, ok = newMap[role][member]; !ok {
						deleted[role][member] = true
					}
				}
			} else {
				// This indicates an entire role was deleted. Mark all members
				// for delete.
				for member, _ := range members {
					deleted[role][member] = true
				}
			}
		}
		log.Printf("[DEBUG] Roles and Members to be deleted: %#v", deleted)

		// Retrieve existing IAM policy from project. This will be merged
		// with the policy in the current state
		// TODO(evanbrown): Add an 'authoritative' flag that allows policy
		// in manifest to overwrite existing policy.
		p, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return err
		}
		log.Printf("[DEBUG] Got existing bindings from project: %#v", p.Bindings)

		// Merge existing policy with policy in the current state
		log.Printf("[DEBUG] Merging new bindings from project: %#v", newPolicy.Bindings)
		mergedBindings := mergeBindings(append(p.Bindings, newPolicy.Bindings...))

		// Remove any roles and members that were explicitly deleted
		mergedBindingsMap := rolesToMembersMap(mergedBindings)
		for role, members := range deleted {
			for member, _ := range members {
				delete(mergedBindingsMap[role], member)
			}
		}

		p.Bindings = rolesToMembersBinding(mergedBindingsMap)
		dump, _ := json.MarshalIndent(p.Bindings, " ", "  ")
		log.Printf("[DEBUG] Setting new policy for project: %#v:\n%s", p, string(dump))

		_, err = config.clientResourceManager.Projects.SetIamPolicy(pid,
			&cloudresourcemanager.SetIamPolicyRequest{Policy: p}).Do()

		if err != nil {
			return fmt.Errorf("Error applying IAM policy for project %q: %s", pid, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Package codename generates a random code name meant for naming software releases
// if you run short of inspiration.
//
// Currently based on the pattern "[Superb] [Superhero]".
//
// The words were taken from:
// - https://github.com/sindresorhus/superb
// - https://github.com/sindresorhus/superheroes
//
// The words are included in the binary with go-bindata, to regenerate
// the dictionary, run in the codename folder:
//     # go-bindata -o words.go -pkg codename data/
package codename

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"regexp"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"
)

const (
	// SpacedString returns "Awesome Hero".
	SpacedString FormatType = 1
	// Sanitized returns a safe string, "awesome-hero".
	Sanitized FormatType = 2

	SuperbFilePath      = "data/superb.json"
	SuperheroesFilePath = "data/superheroes.json"
)

type FormatType uint
type JSONData []interface{}

// GetCodename generates a code name with the given format.
func Get(format FormatType) (string, error) {
	// Load Superb words
	sp, err := Asset(SuperbFilePath)
	if err != nil {
		return "", err
	}

	var spa JSONData
	json.Unmarshal(sp, &spa)

	// Load Superheroes words
	sh, err := Asset(SuperheroesFilePath)
	if err != nil {
		return "", err
	}
	var sha JSONData
	json.Unmarshal(sh, &sha)

	// Seed the randomizer
	rand.Seed(time.Now().UTC().UnixNano())

	// Pick random items from each list
	fw := spa[rand.Intn(len(spa))]
	sw := sha[rand.Intn(len(sha))]

	codename := fmt.Sprintf("%s %s", fw, sw)

	switch format {
	case SpacedString:
		upperCaseFirst := func(s string) string {
			if len(s) == 0 {
				return ""
			}
			r, n := utf8.DecodeRuneInString(s)
			return string(unicode.ToUpper(r)) + s[n:]
		}

		cs, csn := strings.Split(codename, " "), []string{}
		for _, csi := range cs {
			csn = append(csn, upperCaseFirst(csi))
		}
		codename = strings.Join(csn, " ")
		break
	case Sanitized:
		codename = strings.ToLower(strings.Trim(codename, "-"))
		reg, err := regexp.Compile("[^a-z0-9]+")
		if err != nil {
			return "", err
		}
		codename = reg.ReplaceAllString(codename, "-")
		break
	}

	return codename, nil
}
<commit_msg>style(codename): add a line break in package doc<commit_after>// Package codename generates a random code name meant for naming software releases
// if you run short of inspiration.
//
// Currently based on the pattern "[Superb] [Superhero]".
//
// The dictionaries are from:
//
// - https://github.com/sindresorhus/superb
// - https://github.com/sindresorhus/superheroes
//
// The words are included in the binary with go-bindata, to regenerate
// the dictionary, run in the codename folder:
//     # go-bindata -o words.go -pkg codename data/
package codename

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"regexp"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"
)

const (
	// SpacedString returns "Awesome Hero".
	SpacedString FormatType = 1
	// Sanitized returns a safe string, "awesome-hero".
	Sanitized FormatType = 2

	SuperbFilePath      = "data/superb.json"
	SuperheroesFilePath = "data/superheroes.json"
)

type FormatType uint
type JSONData []interface{}

// GetCodename generates a code name with the given format.
func Get(format FormatType) (string, error) {
	// Load Superb words
	sp, err := Asset(SuperbFilePath)
	if err != nil {
		return "", err
	}

	var spa JSONData
	json.Unmarshal(sp, &spa)

	// Load Superheroes words
	sh, err := Asset(SuperheroesFilePath)
	if err != nil {
		return "", err
	}
	var sha JSONData
	json.Unmarshal(sh, &sha)

	// Seed the randomizer
	rand.Seed(time.Now().UTC().UnixNano())

	// Pick random items from each list
	fw := spa[rand.Intn(len(spa))]
	sw := sha[rand.Intn(len(sha))]

	codename := fmt.Sprintf("%s %s", fw, sw)

	switch format {
	case SpacedString:
		upperCaseFirst := func(s string) string {
			if len(s) == 0 {
				return ""
			}
			r, n := utf8.DecodeRuneInString(s)
			return string(unicode.ToUpper(r)) + s[n:]
		}

		cs, csn := strings.Split(codename, " "), []string{}
		for _, csi := range cs {
			csn = append(csn, upperCaseFirst(csi))
		}
		codename = strings.Join(csn, " ")
		break
	case Sanitized:
		codename = strings.ToLower(strings.Trim(codename, "-"))
		reg, err := regexp.Compile("[^a-z0-9]+")
		if err != nil {
			return "", err
		}
		codename = reg.ReplaceAllString(codename, "-")
		break
	}

	return codename, nil
}
<|endoftext|>


<commit_before>// Package core implements Chain Core and its API.
package core

import (
	"context"
	"expvar"
	"fmt"
	"net/http"
	"net/http/pprof"
	"sync"
	"time"

	"chain/core/accesstoken"
	"chain/core/account"
	"chain/core/asset"
	"chain/core/config"
	"chain/core/leader"
	"chain/core/pin"
	"chain/core/query"
	"chain/core/rpc"
	"chain/core/txbuilder"
	"chain/core/txdb"
	"chain/core/txfeed"
	"chain/database/pg"
	"chain/encoding/json"
	"chain/errors"
	"chain/generated/dashboard"
	"chain/generated/docs"
	"chain/net/http/gzip"
	"chain/net/http/httpjson"
	"chain/net/http/limit"
	"chain/net/http/reqid"
	"chain/net/http/static"
	"chain/protocol"
	"chain/protocol/bc"
)

const (
	defGenericPageSize = 100
)

// TODO(kr): change this to "network" or something.
const networkRPCPrefix = "/rpc/"

var (
	errNotFound       = errors.New("not found")
	errRateLimited    = errors.New("request limit exceeded")
	errLeaderElection = errors.New("no leader; pending election")
)

// Handler serves the Chain HTTP API
type API struct {
	Chain         *protocol.Chain
	Store         *txdb.Store
	PinStore      *pin.Store
	Assets        *asset.Registry
	Accounts      *account.Manager
	Indexer       *query.Indexer
	TxFeeds       *txfeed.Tracker
	AccessTokens  *accesstoken.CredentialStore
	Config        *config.Config
	Submitter     txbuilder.Submitter
	DB            pg.DB
	Addr          string
	AltAuth       func(*http.Request) bool
	Signer        func(context.Context, *bc.Block) ([]byte, error)
	RequestLimits []RequestLimit

	healthMu     sync.Mutex
	healthErrors map[string]interface{}
}

type RequestLimit struct {
	Key       func(*http.Request) string
	Burst     int
	PerSecond int
}

func maxBytes(h http.Handler) http.Handler {
	const maxReqSize = 1e7 // 10MB
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		// A block can easily be bigger than maxReqSize, but everything
		// else should be pretty small.
		if req.URL.Path != networkRPCPrefix+"signer/sign-block" {
			req.Body = http.MaxBytesReader(w, req.Body, maxReqSize)
		}
		h.ServeHTTP(w, req)
	})
}

func (a *API) needConfig() func(f interface{}) http.Handler {
	if a.Config == nil {
		return func(f interface{}) http.Handler {
			return alwaysError(errUnconfigured)
		}
	}
	return jsonHandler
}

func Handler(a *API, register func(*http.ServeMux, *API)) http.Handler {
	// Setup the muxer.
	needConfig := a.needConfig()

	devOnly := func(h http.Handler) http.Handler { return h }
	if config.Production {
		devOnly = func(h http.Handler) http.Handler { return alwaysError(errProduction) }
	}

	m := http.NewServeMux()
	m.Handle("/", alwaysError(errNotFound))

	m.Handle("/create-account", needConfig(a.createAccount))
	m.Handle("/create-asset", needConfig(a.createAsset))
	m.Handle("/build-transaction", needConfig(a.build))
	m.Handle("/submit-transaction", needConfig(a.submit))
	m.Handle("/create-control-program", needConfig(a.createControlProgram)) // DEPRECATED
	m.Handle("/create-account-receiver", needConfig(a.createAccountReceiver))
	m.Handle("/create-transaction-feed", needConfig(a.createTxFeed))
	m.Handle("/get-transaction-feed", needConfig(a.getTxFeed))
	m.Handle("/update-transaction-feed", needConfig(a.updateTxFeed))
	m.Handle("/delete-transaction-feed", needConfig(a.deleteTxFeed))
	m.Handle("/mockhsm", alwaysError(errProduction))
	m.Handle("/list-accounts", needConfig(a.listAccounts))
	m.Handle("/list-assets", needConfig(a.listAssets))
	m.Handle("/list-transaction-feeds", needConfig(a.listTxFeeds))
	m.Handle("/list-transactions", needConfig(a.listTransactions))
	m.Handle("/list-balances", needConfig(a.listBalances))
	m.Handle("/list-unspent-outputs", needConfig(a.listUnspentOutputs))
	m.Handle("/reset", devOnly(needConfig(a.reset)))

	m.Handle(networkRPCPrefix+"submit", needConfig(func(ctx context.Context, tx *bc.Tx) error {
		return a.Submitter.Submit(ctx, tx)
	}))
	m.Handle(networkRPCPrefix+"get-blocks", needConfig(a.getBlocksRPC)) // DEPRECATED: use get-block instead
	m.Handle(networkRPCPrefix+"get-block", needConfig(a.getBlockRPC))
	m.Handle(networkRPCPrefix+"get-snapshot-info", needConfig(a.getSnapshotInfoRPC))
	m.Handle(networkRPCPrefix+"get-snapshot", http.HandlerFunc(a.getSnapshotRPC))
	m.Handle(networkRPCPrefix+"signer/sign-block", needConfig(a.leaderSignHandler(a.Signer)))
	m.Handle(networkRPCPrefix+"block-height", needConfig(func(ctx context.Context) map[string]uint64 {
		h := a.Chain.Height()
		return map[string]uint64{
			"block_height": h,
		}
	}))

	m.Handle("/create-access-token", jsonHandler(a.createAccessToken))
	m.Handle("/list-access-tokens", jsonHandler(a.listAccessTokens))
	m.Handle("/delete-access-token", jsonHandler(a.deleteAccessToken))
	m.Handle("/configure", jsonHandler(a.configure))
	m.Handle("/info", jsonHandler(a.info))

	m.Handle("/debug/vars", http.HandlerFunc(expvarHandler))
	m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	if register != nil {
		register(m, a)
	}

	latencyHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if l := latency(m, req); l != nil {
			defer l.RecordSince(time.Now())
		}
		m.ServeHTTP(w, req)
	})

	var handler = (&apiAuthn{
		tokens:   a.AccessTokens,
		tokenMap: make(map[string]tokenResult),
		alt:      a.AltAuth,
	}).handler(latencyHandler)
	handler = maxBytes(handler)
	handler = webAssetsHandler(handler)
	handler = healthHandler(handler)
	for _, l := range a.RequestLimits {
		handler = limit.Handler(handler, alwaysError(errRateLimited), l.PerSecond, l.Burst, l.Key)
	}
	handler = gzip.Handler{Handler: handler}
	handler = coreCounter(handler)
	handler = reqid.Handler(handler)
	handler = timeoutContextHandler(handler)

	return handler
}

// Used as a request object for api queries
type requestQuery struct {
	Filter       string        `json:"filter,omitempty"`
	FilterParams []interface{} `json:"filter_params,omitempty"`
	SumBy        []string      `json:"sum_by,omitempty"`
	PageSize     int           `json:"page_size"`

	// AscLongPoll and Timeout are used by /list-transactions
	// to facilitate notifications.
	AscLongPoll bool          `json:"ascending_with_long_poll,omitempty"`
	Timeout     json.Duration `json:"timeout"`

	// After is a completely opaque cursor, indicating that only
	// items in the result set after the one identified by `After`
	// should be included. It has no relationship to time.
	After string `json:"after"`

	// These two are used for time-range queries like /list-transactions
	StartTimeMS uint64 `json:"start_time,omitempty"`
	EndTimeMS   uint64 `json:"end_time,omitempty"`

	// This is used for point-in-time queries like /list-balances
	// TODO(bobg): Different request structs for endpoints with different needs
	TimestampMS uint64 `json:"timestamp,omitempty"`

	// This is used for filtering results from /list-access-tokens
	// Value must be "client" or "network"
	Type string `json:"type"`

	// Aliases is used to filter results from /mockshm/list-keys
	Aliases []string `json:"aliases,omitempty"`
}

// Used as a response object for api queries
type page struct {
	Items    interface{}  `json:"items"`
	Next     requestQuery `json:"next"`
	LastPage bool         `json:"last_page"`
}

// timeoutContextHandler propagates the timeout, if any, provided as a header
// in the http request.
func timeoutContextHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		timeout, err := time.ParseDuration(req.Header.Get(rpc.HeaderTimeout))
		if err != nil {
			handler.ServeHTTP(w, req) // unmodified
			return
		}

		ctx := req.Context()
		ctx, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()
		handler.ServeHTTP(w, req.WithContext(ctx))
	})
}

func webAssetsHandler(next http.Handler) http.Handler {
	mux := http.NewServeMux()
	mux.Handle("/dashboard/", http.StripPrefix("/dashboard/", static.Handler{
		Assets:  dashboard.Files,
		Default: "index.html",
	}))
	mux.Handle("/docs/", http.StripPrefix("/docs/", static.Handler{
		Assets: docs.Files,
		Index:  "index.html",
	}))
	mux.Handle("/", next)

	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/" {
			http.Redirect(w, req, "/dashboard/", http.StatusFound)
			return
		}

		mux.ServeHTTP(w, req)
	})
}

func (a *API) leaderSignHandler(f func(context.Context, *bc.Block) ([]byte, error)) func(context.Context, *bc.Block) ([]byte, error) {
	return func(ctx context.Context, b *bc.Block) ([]byte, error) {
		if f == nil {
			return nil, errNotFound // TODO(kr): is this really the right error here?
		}
		if leader.IsLeading() {
			return f(ctx, b)
		}
		var resp []byte
		err := a.forwardToLeader(ctx, "/rpc/signer/sign-block", b, &resp)
		return resp, err
	}
}

// forwardToLeader forwards the current request to the core's leader
// process. It propagates the same credentials used in the current
// request. For that reason, it cannot be used outside of a request-
// handling context.
func (a *API) forwardToLeader(ctx context.Context, path string, body interface{}, resp interface{}) error {
	addr, err := leader.Address(ctx, a.DB)
	if err != nil {
		return errors.Wrap(err)
	}

	// Don't infinite loop if the leader's address is our own address.
	// This is possible if we just became the leader. The client should
	// just retry.
	if addr == a.Addr {
		return errLeaderElection
	}

	// TODO(jackson): If using TLS, use https:// here.
	l := &rpc.Client{
		BaseURL: "http://" + addr,
	}

	// Forward the request credentials if we have them.
	// TODO(jackson): Don't use the incoming request's credentials and
	// have an alternative authentication scheme between processes of the
	// same Core. For now, we only call the leader for the purpose of
	// forwarding a request, so this is OK.
	req := httpjson.Request(ctx)
	user, pass, ok := req.BasicAuth()
	if ok {
		l.AccessToken = fmt.Sprintf("%s:%s", user, pass)
	}

	return l.Call(ctx, path, body, resp)
}

// expvarHandler is copied from the expvar package.
// TODO(jackson): In Go 1.8, use expvar.Handler.
// https://go-review.googlesource.com/#/c/24722/
func expvarHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprintf(w, "{\n")
	first := true
	expvar.Do(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprintf(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "\n}\n")
}

func healthHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/health" {
			return
		}
		handler.ServeHTTP(w, req)
	})
}
<commit_msg>core: use expvar.Handler<commit_after>// Package core implements Chain Core and its API.
package core

import (
	"context"
	"expvar"
	"fmt"
	"net/http"
	"net/http/pprof"
	"sync"
	"time"

	"chain/core/accesstoken"
	"chain/core/account"
	"chain/core/asset"
	"chain/core/config"
	"chain/core/leader"
	"chain/core/pin"
	"chain/core/query"
	"chain/core/rpc"
	"chain/core/txbuilder"
	"chain/core/txdb"
	"chain/core/txfeed"
	"chain/database/pg"
	"chain/encoding/json"
	"chain/errors"
	"chain/generated/dashboard"
	"chain/generated/docs"
	"chain/net/http/gzip"
	"chain/net/http/httpjson"
	"chain/net/http/limit"
	"chain/net/http/reqid"
	"chain/net/http/static"
	"chain/protocol"
	"chain/protocol/bc"
)

const (
	defGenericPageSize = 100
)

// TODO(kr): change this to "network" or something.
const networkRPCPrefix = "/rpc/"

var (
	errNotFound       = errors.New("not found")
	errRateLimited    = errors.New("request limit exceeded")
	errLeaderElection = errors.New("no leader; pending election")
)

// Handler serves the Chain HTTP API
type API struct {
	Chain         *protocol.Chain
	Store         *txdb.Store
	PinStore      *pin.Store
	Assets        *asset.Registry
	Accounts      *account.Manager
	Indexer       *query.Indexer
	TxFeeds       *txfeed.Tracker
	AccessTokens  *accesstoken.CredentialStore
	Config        *config.Config
	Submitter     txbuilder.Submitter
	DB            pg.DB
	Addr          string
	AltAuth       func(*http.Request) bool
	Signer        func(context.Context, *bc.Block) ([]byte, error)
	RequestLimits []RequestLimit

	healthMu     sync.Mutex
	healthErrors map[string]interface{}
}

type RequestLimit struct {
	Key       func(*http.Request) string
	Burst     int
	PerSecond int
}

func maxBytes(h http.Handler) http.Handler {
	const maxReqSize = 1e7 // 10MB
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		// A block can easily be bigger than maxReqSize, but everything
		// else should be pretty small.
		if req.URL.Path != networkRPCPrefix+"signer/sign-block" {
			req.Body = http.MaxBytesReader(w, req.Body, maxReqSize)
		}
		h.ServeHTTP(w, req)
	})
}

func (a *API) needConfig() func(f interface{}) http.Handler {
	if a.Config == nil {
		return func(f interface{}) http.Handler {
			return alwaysError(errUnconfigured)
		}
	}
	return jsonHandler
}

func Handler(a *API, register func(*http.ServeMux, *API)) http.Handler {
	// Setup the muxer.
	needConfig := a.needConfig()

	devOnly := func(h http.Handler) http.Handler { return h }
	if config.Production {
		devOnly = func(h http.Handler) http.Handler { return alwaysError(errProduction) }
	}

	m := http.NewServeMux()
	m.Handle("/", alwaysError(errNotFound))

	m.Handle("/create-account", needConfig(a.createAccount))
	m.Handle("/create-asset", needConfig(a.createAsset))
	m.Handle("/build-transaction", needConfig(a.build))
	m.Handle("/submit-transaction", needConfig(a.submit))
	m.Handle("/create-control-program", needConfig(a.createControlProgram)) // DEPRECATED
	m.Handle("/create-account-receiver", needConfig(a.createAccountReceiver))
	m.Handle("/create-transaction-feed", needConfig(a.createTxFeed))
	m.Handle("/get-transaction-feed", needConfig(a.getTxFeed))
	m.Handle("/update-transaction-feed", needConfig(a.updateTxFeed))
	m.Handle("/delete-transaction-feed", needConfig(a.deleteTxFeed))
	m.Handle("/mockhsm", alwaysError(errProduction))
	m.Handle("/list-accounts", needConfig(a.listAccounts))
	m.Handle("/list-assets", needConfig(a.listAssets))
	m.Handle("/list-transaction-feeds", needConfig(a.listTxFeeds))
	m.Handle("/list-transactions", needConfig(a.listTransactions))
	m.Handle("/list-balances", needConfig(a.listBalances))
	m.Handle("/list-unspent-outputs", needConfig(a.listUnspentOutputs))
	m.Handle("/reset", devOnly(needConfig(a.reset)))

	m.Handle(networkRPCPrefix+"submit", needConfig(func(ctx context.Context, tx *bc.Tx) error {
		return a.Submitter.Submit(ctx, tx)
	}))
	m.Handle(networkRPCPrefix+"get-blocks", needConfig(a.getBlocksRPC)) // DEPRECATED: use get-block instead
	m.Handle(networkRPCPrefix+"get-block", needConfig(a.getBlockRPC))
	m.Handle(networkRPCPrefix+"get-snapshot-info", needConfig(a.getSnapshotInfoRPC))
	m.Handle(networkRPCPrefix+"get-snapshot", http.HandlerFunc(a.getSnapshotRPC))
	m.Handle(networkRPCPrefix+"signer/sign-block", needConfig(a.leaderSignHandler(a.Signer)))
	m.Handle(networkRPCPrefix+"block-height", needConfig(func(ctx context.Context) map[string]uint64 {
		h := a.Chain.Height()
		return map[string]uint64{
			"block_height": h,
		}
	}))

	m.Handle("/create-access-token", jsonHandler(a.createAccessToken))
	m.Handle("/list-access-tokens", jsonHandler(a.listAccessTokens))
	m.Handle("/delete-access-token", jsonHandler(a.deleteAccessToken))
	m.Handle("/configure", jsonHandler(a.configure))
	m.Handle("/info", jsonHandler(a.info))

	m.Handle("/debug/vars", expvar.Handler())
	m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	if register != nil {
		register(m, a)
	}

	latencyHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if l := latency(m, req); l != nil {
			defer l.RecordSince(time.Now())
		}
		m.ServeHTTP(w, req)
	})

	var handler = (&apiAuthn{
		tokens:   a.AccessTokens,
		tokenMap: make(map[string]tokenResult),
		alt:      a.AltAuth,
	}).handler(latencyHandler)
	handler = maxBytes(handler)
	handler = webAssetsHandler(handler)
	handler = healthHandler(handler)
	for _, l := range a.RequestLimits {
		handler = limit.Handler(handler, alwaysError(errRateLimited), l.PerSecond, l.Burst, l.Key)
	}
	handler = gzip.Handler{Handler: handler}
	handler = coreCounter(handler)
	handler = reqid.Handler(handler)
	handler = timeoutContextHandler(handler)

	return handler
}

// Used as a request object for api queries
type requestQuery struct {
	Filter       string        `json:"filter,omitempty"`
	FilterParams []interface{} `json:"filter_params,omitempty"`
	SumBy        []string      `json:"sum_by,omitempty"`
	PageSize     int           `json:"page_size"`

	// AscLongPoll and Timeout are used by /list-transactions
	// to facilitate notifications.
	AscLongPoll bool          `json:"ascending_with_long_poll,omitempty"`
	Timeout     json.Duration `json:"timeout"`

	// After is a completely opaque cursor, indicating that only
	// items in the result set after the one identified by `After`
	// should be included. It has no relationship to time.
	After string `json:"after"`

	// These two are used for time-range queries like /list-transactions
	StartTimeMS uint64 `json:"start_time,omitempty"`
	EndTimeMS   uint64 `json:"end_time,omitempty"`

	// This is used for point-in-time queries like /list-balances
	// TODO(bobg): Different request structs for endpoints with different needs
	TimestampMS uint64 `json:"timestamp,omitempty"`

	// This is used for filtering results from /list-access-tokens
	// Value must be "client" or "network"
	Type string `json:"type"`

	// Aliases is used to filter results from /mockshm/list-keys
	Aliases []string `json:"aliases,omitempty"`
}

// Used as a response object for api queries
type page struct {
	Items    interface{}  `json:"items"`
	Next     requestQuery `json:"next"`
	LastPage bool         `json:"last_page"`
}

// timeoutContextHandler propagates the timeout, if any, provided as a header
// in the http request.
func timeoutContextHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		timeout, err := time.ParseDuration(req.Header.Get(rpc.HeaderTimeout))
		if err != nil {
			handler.ServeHTTP(w, req) // unmodified
			return
		}

		ctx := req.Context()
		ctx, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()
		handler.ServeHTTP(w, req.WithContext(ctx))
	})
}

func webAssetsHandler(next http.Handler) http.Handler {
	mux := http.NewServeMux()
	mux.Handle("/dashboard/", http.StripPrefix("/dashboard/", static.Handler{
		Assets:  dashboard.Files,
		Default: "index.html",
	}))
	mux.Handle("/docs/", http.StripPrefix("/docs/", static.Handler{
		Assets: docs.Files,
		Index:  "index.html",
	}))
	mux.Handle("/", next)

	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/" {
			http.Redirect(w, req, "/dashboard/", http.StatusFound)
			return
		}

		mux.ServeHTTP(w, req)
	})
}

func (a *API) leaderSignHandler(f func(context.Context, *bc.Block) ([]byte, error)) func(context.Context, *bc.Block) ([]byte, error) {
	return func(ctx context.Context, b *bc.Block) ([]byte, error) {
		if f == nil {
			return nil, errNotFound // TODO(kr): is this really the right error here?
		}
		if leader.IsLeading() {
			return f(ctx, b)
		}
		var resp []byte
		err := a.forwardToLeader(ctx, "/rpc/signer/sign-block", b, &resp)
		return resp, err
	}
}

// forwardToLeader forwards the current request to the core's leader
// process. It propagates the same credentials used in the current
// request. For that reason, it cannot be used outside of a request-
// handling context.
func (a *API) forwardToLeader(ctx context.Context, path string, body interface{}, resp interface{}) error {
	addr, err := leader.Address(ctx, a.DB)
	if err != nil {
		return errors.Wrap(err)
	}

	// Don't infinite loop if the leader's address is our own address.
	// This is possible if we just became the leader. The client should
	// just retry.
	if addr == a.Addr {
		return errLeaderElection
	}

	// TODO(jackson): If using TLS, use https:// here.
	l := &rpc.Client{
		BaseURL: "http://" + addr,
	}

	// Forward the request credentials if we have them.
	// TODO(jackson): Don't use the incoming request's credentials and
	// have an alternative authentication scheme between processes of the
	// same Core. For now, we only call the leader for the purpose of
	// forwarding a request, so this is OK.
	req := httpjson.Request(ctx)
	user, pass, ok := req.BasicAuth()
	if ok {
		l.AccessToken = fmt.Sprintf("%s:%s", user, pass)
	}

	return l.Call(ctx, path, body, resp)
}

func healthHandler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/health" {
			return
		}
		handler.ServeHTTP(w, req)
	})
}
<|endoftext|>