<commit_before>package agent

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"code.google.com/p/go.net/context"
	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/homedir"
	"github.com/facebookgo/stackerr"
	dockerclient "github.com/fsouza/go-dockerclient"

	"github.com/bearded-web/bearded/models/agent"
	"github.com/bearded-web/bearded/models/plugin"
	"github.com/bearded-web/bearded/models/report"
	"github.com/bearded-web/bearded/models/scan"
	"github.com/bearded-web/bearded/pkg/client"
	"github.com/bearded-web/bearded/pkg/docker"
	"github.com/bearded-web/bearded/pkg/transport/mango"
	"github.com/bearded-web/bearded/pkg/utils"
)

type Agent struct {
	// client helps to communicate with bearded api
	api     *client.Client
	name    string
	dclient *docker.Docker
}

func New(api *client.Client, dclient *docker.Docker, name string) (*Agent, error) {
	a := &Agent{
		api:     api,
		name:    name,
		dclient: dclient,
	}
	return a, nil
}

func (a *Agent) Serve(ctx context.Context) error {
	var resultErr error

	agnt := &agent.Agent{
		Name:   a.name,
		Type:   agent.System,
		Status: agent.StatusUndefined,
	}
	prevStatus := agnt.Status
loop:
	for {
		timeout := 0
		if prevStatus != agnt.Status {
			logrus.Debugf("Agent status: %s -> %s", prevStatus, agnt.Status)
			prevStatus = agnt.Status
		}
		switch agnt.Status {
		case agent.StatusUndefined:
			err := a.Register(ctx, agnt)
			if err != nil {
				logrus.Errorf("Registration error: %v", err)
				timeout = 5
			}
			logrus.Infof("Agent Id: %s", client.FromId(agnt.Id))
		case agent.StatusRegistered:
			err := a.Retrieve(ctx, agnt)
			if err != nil {
				logrus.Errorf("Retrieve error: %v", err)
				timeout = 5
			}
			if agnt.Status == agent.StatusRegistered {
				timeout = 5
			}
		case agent.StatusApproved:
			err := a.GetJobs(ctx, agnt)
			if err != nil {
				logrus.Errorf("GetJobs error: %v", err)
				timeout = 5
			}
		case agent.StatusBlocked:
			resultErr = fmt.Errorf("Agent is blocked")
			break loop
		default:
			resultErr = fmt.Errorf("Unknown agent status: %s", agnt.Status)
			break loop
		}
		if timeout != 0 {
			logrus.Debugf("Timeout: %d", timeout)
		}
		select {
		case <-ctx.Done():
			if ctx.Err() != context.Canceled {
				resultErr = ctx.Err()
			}
			break loop
		case <-time.After(time.Duration(timeout) * time.Second):

		}
	}
	return resultErr
}

func (a *Agent) Register(ctx context.Context, agnt *agent.Agent) error {
	logrus.Info("Register")
	if created, err := a.api.Agents.Create(ctx, agnt); err != nil {
		if !client.IsConflicted(err) {
			logrus.Error(err)
			return err
		}
		logrus.Info("Already existed")
		// agent is already existed
		// retrieve existed
		logrus.Info("Get by name and type")
		agentList, err := a.api.Agents.List(ctx, &client.AgentsListOpts{Name: agnt.Name, Type: agnt.Type})
		if err != nil {
			return err
		}
		if agentList.Count != 1 {
			err := fmt.Errorf("expected 1 agent, but actual is %d", agentList.Count)
			return err
		}
		*agnt = *agentList.Results[0]
	} else {
		*agnt = *created
	}
	return nil
}

func (a *Agent) Retrieve(ctx context.Context, agnt *agent.Agent) error {
	logrus.Info("Retrieve")
	if agnt.Id == "" {
		agnt.Status = agent.StatusUndefined
		return fmt.Errorf("ageng.Id shouldn't be empty")
	}
	got, err := a.api.Agents.Get(ctx, client.FromId(agnt.Id))
	if err != nil {
		return err
	}
	*agnt = *got
	return nil
}

func (a *Agent) GetJobs(ctx context.Context, agnt *agent.Agent) error {
	//	logrus.Debug("Request jobs")
	jobs, err := a.api.Agents.GetJobs(ctx, agnt)
	if err != nil {
		return err
	}
	//	logrus.Debugf("Got %d jobs", len(jobs))
	for _, job := range jobs {
		if err := a.HandleJob(ctx, job); err != nil {
			// TODO (m0sth8): return scan failed status
			// what should I do if backend server is unavailable?
			logrus.Error(err)
		}
	}
	return nil
}

func (a *Agent) HandleJob(ctx context.Context, job *agent.Job) error {
	logrus.Debugf("Job: %s", job)
	if job.Cmd == agent.CmdScan {
		go func() {
			scanCtx, cancel := context.WithCancel(ctx)
			defer cancel()
			if err := a.HandleScan(scanCtx, job.Scan); err != nil {
				logrus.Error(stackerr.Wrap(err))
			}
		}()
	}
	return nil
}

func (a *Agent) HandleScan(ctx context.Context, sess *scan.Session) error {
	// take a plugin
	pl, err := a.api.Plugins.Get(ctx, client.FromId(sess.Plugin))
	if err != nil {
		return stackerr.Wrap(err)
	}
	logrus.Infof("plugin: %s", pl)
	logrus.Info("set session to working state")
	sess.Status = scan.StatusWorking
	if sess, err = a.api.Scans.SessionUpdate(ctx, sess); err != nil {
		return err
	}

	setFailed := func(err error) error {
		logrus.Info("set session to failed state, due to %s", err)
		sess.Status = scan.StatusFailed
		if sess, err = a.api.Scans.SessionUpdate(ctx, sess); err != nil {
			return err
		}
		return err
	}
	// we have a couple of hack for boot2docker network
	isBoot2Docker := utils.IsBoot2Docker()

	hostCfg := &dockerclient.HostConfig{}
	args := sess.Step.Conf.CommandArgs
	cfg := &dockerclient.Config{
		Image: pl.Container.Image,
		Tty:   true,
		Cmd:   strings.Split(args, " "),
	}

	switch pl.Type {
	case plugin.Util:
	case plugin.Script:
		if hostCfg.PortBindings == nil {
			hostCfg.PortBindings = map[dockerclient.Port][]dockerclient.PortBinding{}
		}
		hostIp := "127.0.0.1"
		if isBoot2Docker {
			// we should listen on external boot2docker virtual machine interface
			hostIp = ""
		}
		hostCfg.PortBindings["9238/tcp"] = []dockerclient.PortBinding{dockerclient.PortBinding{HostIP: hostIp}}
	default:
		return setFailed(fmt.Errorf("Unexpected plugin type %v", pl.Type))
	}

	if sharedFiles := sess.Step.Conf.SharedFiles; sharedFiles != nil && len(sharedFiles) > 0 {
		tmpRoot := os.TempDir()
		if isBoot2Docker {
			// in mac os boot2docker, our binded directories must be inside the /Users home directory
			// TODO (m0sth8): exclude tmp root to config files
			tmpRoot = filepath.Join(homedir.Get(), "Library/Caches/bearded-web")
			err := os.MkdirAll(tmpRoot, 0755)
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't create a tmp directory %s", tmpRoot))
			}
		}
		tmpDir, err := ioutil.TempDir(tmpRoot, "bearded-volume-")
		if err != nil {
			logrus.Error(stackerr.Wrap(err))
			return setFailed(fmt.Errorf("Can't create a temp directory"))
		}
		defer func() {
			os.RemoveAll(tmpDir)
		}()
		shareDir := filepath.Join(tmpDir, "share")
		err = os.MkdirAll(shareDir, 0755)
		if err != nil {
			logrus.Error(stackerr.Wrap(err))
			return setFailed(fmt.Errorf("Can't create a share directory"))
		}
		for _, sharedFile := range sharedFiles {
			base := filepath.Base(sharedFile.Path)
			dir := filepath.Dir(sharedFile.Path)
			dir = filepath.Join(shareDir, filepath.Join("/", dir))
			err := os.MkdirAll(dir, 0755)
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't create a directory"))
			}
			err = ioutil.WriteFile(filepath.Join(dir, base), []byte(sharedFile.Text), 0644)
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't create a temporary file"))
			}
			println("put file", filepath.Join(dir, base))
		}
		hostCfg.Binds = append(hostCfg.Binds, fmt.Sprintf("%s:/share:r", shareDir))
		println("bind", fmt.Sprintf("%s:/share:ro", shareDir))
	}

	takeFiles := []string{}
	if sess.Step.Conf.TakeFiles != nil {
		for _, f := range sess.Step.Conf.TakeFiles {
			takeFiles = append(takeFiles, f.Path)
		}
	}
	ch := a.dclient.RunImage(ctx, cfg, hostCfg, takeFiles)
	// creating container
	var container *dockerclient.Container
	select {
	case <-ctx.Done():
		return setFailed(ctx.Err())
	case res := <-ch:
		// container info
		if res.Err != nil {
			return setFailed(res.Err)
		}
		container = res.Container
	}
	// get ports
	var serv *RemoteServer
	if pl.Type == plugin.Script {
		// setup transport between agent and script
		// script should expose 9238
		port := container.NetworkSettings.Ports["9238/tcp"][0].HostPort
		if port == "" {
			return setFailed(stackerr.New("Unexpected empty port"))
		}
		host := "127.0.0.1"
		// TODO (m0sth8): extract this logic
		if isBoot2Docker {
			bootIp, err := utils.Boot2DocketIp()
			if err != nil {
				return setFailed(stackerr.Wrap(err))
			}
			host = string(bootIp)
		}
		logrus.Infof("script addr is %s:%s", host, port)
		//		transp := websocket.NewClient(fmt.Sprintf("ws://%s:%s", host, port))
		transp, err := mango.NewClient(fmt.Sprintf("tcp://%s:%s", host, port))
		if err != nil {
			return setFailed(stackerr.Wrap(err))
		}
		// setup remote server
		serv, _ = NewRemoteServer(transp, a.api, sess)
		go transp.Serve(ctx, serv)
		err = serv.Connect(ctx)
		if err != nil {
			return setFailed(stackerr.Wrap(err))
		}
	}

	var (
		res docker.ContainerResponse
		//		closed bool
	)
	// running
	select {
	case <-ctx.Done():
		return setFailed(ctx.Err())
	case res = <-ch:
		//		if closed {
		//			// TODO (m0sth8): handle closed channel from docker container
		//			return setFailed(stackerr.Newf("Docker channel is closed, %v", res))
		//		}
	}
	if res.Err != nil {
		logrus.Error(res.Err)
		return setFailed(stackerr.Wrap(res.Err))
	}
	var rep *report.Report
	raw := &report.Report{
		Type: report.TypeRaw,
		Raw:  report.Raw{Raw: string(res.Log)},
	}
	// handle files from container
	if sess.Step.Conf.TakeFiles != nil && res.Files != nil {
		for _, f := range sess.Step.Conf.TakeFiles {
			if data, found := res.Files[f.Path]; found {
				name := f.Path
				if f.Name != "" {
					name = f.Name
				}
				meta, err := a.api.Files.Create(ctx, name, data)
				if err != nil {
					logrus.Error(stackerr.Wrap(err))
					continue
				}
				raw.Files = append(raw.Files, meta)
			}
		}
	}

	rep = raw
	switch pl.Type {
	case plugin.Script:
		if serv != nil && serv.Rep != nil {
			if serv.Rep.Type != report.TypeMulti {
				rep = &report.Report{
					Type: report.TypeMulti,
				}
				rep.Multi = append(rep.Multi, serv.Rep)
			} else {
				rep = serv.Rep
			}

			rep.Multi = append(rep.Multi, raw)
		}
	}

	_, err = a.api.Scans.SessionReportCreate(ctx, sess, rep)
	if err != nil {
		return setFailed(stackerr.Wrap(err))
	}

	logrus.Info("finished")
	sess.Status = scan.StatusFinished
	if sess, err = a.api.Scans.SessionUpdate(ctx, sess); err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix homedir bug<commit_after>package agent

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"code.google.com/p/go.net/context"
	"github.com/Sirupsen/logrus"
	"github.com/facebookgo/stackerr"
	dockerclient "github.com/fsouza/go-dockerclient"

	"github.com/bearded-web/bearded/models/agent"
	"github.com/bearded-web/bearded/models/plugin"
	"github.com/bearded-web/bearded/models/report"
	"github.com/bearded-web/bearded/models/scan"
	"github.com/bearded-web/bearded/pkg/client"
	"github.com/bearded-web/bearded/pkg/docker"
	"github.com/bearded-web/bearded/pkg/transport/mango"
	"github.com/bearded-web/bearded/pkg/utils"
	"github.com/bearded-web/bearded/vendor/homedir"
)

type Agent struct {
	// client helps to communicate with bearded api
	api     *client.Client
	name    string
	dclient *docker.Docker
}

func New(api *client.Client, dclient *docker.Docker, name string) (*Agent, error) {
	a := &Agent{
		api:     api,
		name:    name,
		dclient: dclient,
	}
	return a, nil
}

func (a *Agent) Serve(ctx context.Context) error {
	var resultErr error

	agnt := &agent.Agent{
		Name:   a.name,
		Type:   agent.System,
		Status: agent.StatusUndefined,
	}
	prevStatus := agnt.Status
loop:
	for {
		timeout := 0
		if prevStatus != agnt.Status {
			logrus.Debugf("Agent status: %s -> %s", prevStatus, agnt.Status)
			prevStatus = agnt.Status
		}
		switch agnt.Status {
		case agent.StatusUndefined:
			err := a.Register(ctx, agnt)
			if err != nil {
				logrus.Errorf("Registration error: %v", err)
				timeout = 5
			}
			logrus.Infof("Agent Id: %s", client.FromId(agnt.Id))
		case agent.StatusRegistered:
			err := a.Retrieve(ctx, agnt)
			if err != nil {
				logrus.Errorf("Retrieve error: %v", err)
				timeout = 5
			}
			if agnt.Status == agent.StatusRegistered {
				timeout = 5
			}
		case agent.StatusApproved:
			err := a.GetJobs(ctx, agnt)
			if err != nil {
				logrus.Errorf("GetJobs error: %v", err)
				timeout = 5
			}
		case agent.StatusBlocked:
			resultErr = fmt.Errorf("Agent is blocked")
			break loop
		default:
			resultErr = fmt.Errorf("Unknown agent status: %s", agnt.Status)
			break loop
		}
		if timeout != 0 {
			logrus.Debugf("Timeout: %d", timeout)
		}
		select {
		case <-ctx.Done():
			if ctx.Err() != context.Canceled {
				resultErr = ctx.Err()
			}
			break loop
		case <-time.After(time.Duration(timeout) * time.Second):

		}
	}
	return resultErr
}

func (a *Agent) Register(ctx context.Context, agnt *agent.Agent) error {
	logrus.Info("Register")
	if created, err := a.api.Agents.Create(ctx, agnt); err != nil {
		if !client.IsConflicted(err) {
			logrus.Error(err)
			return err
		}
		logrus.Info("Already existed")
		// agent is already existed
		// retrieve existed
		logrus.Info("Get by name and type")
		agentList, err := a.api.Agents.List(ctx, &client.AgentsListOpts{Name: agnt.Name, Type: agnt.Type})
		if err != nil {
			return err
		}
		if agentList.Count != 1 {
			err := fmt.Errorf("expected 1 agent, but actual is %d", agentList.Count)
			return err
		}
		*agnt = *agentList.Results[0]
	} else {
		*agnt = *created
	}
	return nil
}

func (a *Agent) Retrieve(ctx context.Context, agnt *agent.Agent) error {
	logrus.Info("Retrieve")
	if agnt.Id == "" {
		agnt.Status = agent.StatusUndefined
		return fmt.Errorf("ageng.Id shouldn't be empty")
	}
	got, err := a.api.Agents.Get(ctx, client.FromId(agnt.Id))
	if err != nil {
		return err
	}
	*agnt = *got
	return nil
}

func (a *Agent) GetJobs(ctx context.Context, agnt *agent.Agent) error {
	//	logrus.Debug("Request jobs")
	jobs, err := a.api.Agents.GetJobs(ctx, agnt)
	if err != nil {
		return err
	}
	//	logrus.Debugf("Got %d jobs", len(jobs))
	for _, job := range jobs {
		if err := a.HandleJob(ctx, job); err != nil {
			// TODO (m0sth8): return scan failed status
			// what should I do if backend server is unavailable?
			logrus.Error(err)
		}
	}
	return nil
}

func (a *Agent) HandleJob(ctx context.Context, job *agent.Job) error {
	logrus.Debugf("Job: %s", job)
	if job.Cmd == agent.CmdScan {
		go func() {
			scanCtx, cancel := context.WithCancel(ctx)
			defer cancel()
			if err := a.HandleScan(scanCtx, job.Scan); err != nil {
				logrus.Error(stackerr.Wrap(err))
			}
		}()
	}
	return nil
}

func (a *Agent) HandleScan(ctx context.Context, sess *scan.Session) error {
	// take a plugin
	pl, err := a.api.Plugins.Get(ctx, client.FromId(sess.Plugin))
	if err != nil {
		return stackerr.Wrap(err)
	}
	logrus.Infof("plugin: %s", pl)
	logrus.Info("set session to working state")
	sess.Status = scan.StatusWorking
	if sess, err = a.api.Scans.SessionUpdate(ctx, sess); err != nil {
		return err
	}

	setFailed := func(err error) error {
		logrus.Info("set session to failed state, due to %s", err)
		sess.Status = scan.StatusFailed
		if sess, err = a.api.Scans.SessionUpdate(ctx, sess); err != nil {
			return err
		}
		return err
	}
	// we have a couple of hack for boot2docker network
	isBoot2Docker := utils.IsBoot2Docker()

	hostCfg := &dockerclient.HostConfig{}
	args := sess.Step.Conf.CommandArgs
	cfg := &dockerclient.Config{
		Image: pl.Container.Image,
		Tty:   true,
		Cmd:   strings.Split(args, " "),
	}

	switch pl.Type {
	case plugin.Util:
	case plugin.Script:
		if hostCfg.PortBindings == nil {
			hostCfg.PortBindings = map[dockerclient.Port][]dockerclient.PortBinding{}
		}
		hostIp := "127.0.0.1"
		if isBoot2Docker {
			// we should listen on external boot2docker virtual machine interface
			hostIp = ""
		}
		hostCfg.PortBindings["9238/tcp"] = []dockerclient.PortBinding{dockerclient.PortBinding{HostIP: hostIp}}
	default:
		return setFailed(fmt.Errorf("Unexpected plugin type %v", pl.Type))
	}

	if sharedFiles := sess.Step.Conf.SharedFiles; sharedFiles != nil && len(sharedFiles) > 0 {
		tmpRoot := os.TempDir()
		if isBoot2Docker {
			// in mac os boot2docker, our binded directories must be inside the /Users home directory
			// TODO (m0sth8): exclude tmp root to config files
			home, err := homedir.Dir()
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't get a home directory"))
			}
			tmpRoot = filepath.Join(home, "Library/Caches/bearded-web")
			err = os.MkdirAll(tmpRoot, 0755)
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't create a tmp directory %s", tmpRoot))
			}
		}
		tmpDir, err := ioutil.TempDir(tmpRoot, "bearded-volume-")
		if err != nil {
			logrus.Error(stackerr.Wrap(err))
			return setFailed(fmt.Errorf("Can't create a temp directory"))
		}
		defer func() {
			os.RemoveAll(tmpDir)
		}()
		shareDir := filepath.Join(tmpDir, "share")
		err = os.MkdirAll(shareDir, 0755)
		if err != nil {
			logrus.Error(stackerr.Wrap(err))
			return setFailed(fmt.Errorf("Can't create a share directory"))
		}
		for _, sharedFile := range sharedFiles {
			base := filepath.Base(sharedFile.Path)
			dir := filepath.Dir(sharedFile.Path)
			dir = filepath.Join(shareDir, filepath.Join("/", dir))
			err := os.MkdirAll(dir, 0755)
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't create a directory"))
			}
			err = ioutil.WriteFile(filepath.Join(dir, base), []byte(sharedFile.Text), 0644)
			if err != nil {
				logrus.Error(stackerr.Wrap(err))
				return setFailed(fmt.Errorf("Can't create a temporary file"))
			}
			println("put file", filepath.Join(dir, base))
		}
		hostCfg.Binds = append(hostCfg.Binds, fmt.Sprintf("%s:/share:r", shareDir))
		println("bind", fmt.Sprintf("%s:/share:ro", shareDir))
	}

	takeFiles := []string{}
	if sess.Step.Conf.TakeFiles != nil {
		for _, f := range sess.Step.Conf.TakeFiles {
			takeFiles = append(takeFiles, f.Path)
		}
	}
	ch := a.dclient.RunImage(ctx, cfg, hostCfg, takeFiles)
	// creating container
	var container *dockerclient.Container
	select {
	case <-ctx.Done():
		return setFailed(ctx.Err())
	case res := <-ch:
		// container info
		if res.Err != nil {
			return setFailed(res.Err)
		}
		container = res.Container
	}
	// get ports
	var serv *RemoteServer
	if pl.Type == plugin.Script {
		// setup transport between agent and script
		// script should expose 9238
		port := container.NetworkSettings.Ports["9238/tcp"][0].HostPort
		if port == "" {
			return setFailed(stackerr.New("Unexpected empty port"))
		}
		host := "127.0.0.1"
		// TODO (m0sth8): extract this logic
		if isBoot2Docker {
			bootIp, err := utils.Boot2DocketIp()
			if err != nil {
				return setFailed(stackerr.Wrap(err))
			}
			host = string(bootIp)
		}
		logrus.Infof("script addr is %s:%s", host, port)
		//		transp := websocket.NewClient(fmt.Sprintf("ws://%s:%s", host, port))
		transp, err := mango.NewClient(fmt.Sprintf("tcp://%s:%s", host, port))
		if err != nil {
			return setFailed(stackerr.Wrap(err))
		}
		// setup remote server
		serv, _ = NewRemoteServer(transp, a.api, sess)
		go transp.Serve(ctx, serv)
		err = serv.Connect(ctx)
		if err != nil {
			return setFailed(stackerr.Wrap(err))
		}
	}

	var (
		res docker.ContainerResponse
		//		closed bool
	)
	// running
	select {
	case <-ctx.Done():
		return setFailed(ctx.Err())
	case res = <-ch:
		//		if closed {
		//			// TODO (m0sth8): handle closed channel from docker container
		//			return setFailed(stackerr.Newf("Docker channel is closed, %v", res))
		//		}
	}
	if res.Err != nil {
		logrus.Error(res.Err)
		return setFailed(stackerr.Wrap(res.Err))
	}
	var rep *report.Report
	raw := &report.Report{
		Type: report.TypeRaw,
		Raw:  report.Raw{Raw: string(res.Log)},
	}
	// handle files from container
	if sess.Step.Conf.TakeFiles != nil && res.Files != nil {
		for _, f := range sess.Step.Conf.TakeFiles {
			if data, found := res.Files[f.Path]; found {
				name := f.Path
				if f.Name != "" {
					name = f.Name
				}
				meta, err := a.api.Files.Create(ctx, name, data)
				if err != nil {
					logrus.Error(stackerr.Wrap(err))
					continue
				}
				raw.Files = append(raw.Files, meta)
			}
		}
	}

	rep = raw
	switch pl.Type {
	case plugin.Script:
		if serv != nil && serv.Rep != nil {
			if serv.Rep.Type != report.TypeMulti {
				rep = &report.Report{
					Type: report.TypeMulti,
				}
				rep.Multi = append(rep.Multi, serv.Rep)
			} else {
				rep = serv.Rep
			}

			rep.Multi = append(rep.Multi, raw)
		}
	}

	_, err = a.api.Scans.SessionReportCreate(ctx, sess, rep)
	if err != nil {
		return setFailed(stackerr.Wrap(err))
	}

	logrus.Info("finished")
	sess.Status = scan.StatusFinished
	if sess, err = a.api.Scans.SessionUpdate(ctx, sess); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/middleware"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
)

func GetPluginList(c *middleware.Context) Response {
	typeFilter := c.Query("type")
	enabledFilter := c.Query("enabled")
	embeddedFilter := c.Query("embedded")

	pluginSettingsMap, err := plugins.GetPluginSettings(c.OrgId)

	if err != nil {
		return ApiError(500, "Failed to get list of plugins", err)
	}

	result := make([]*dtos.PluginListItem, 0)
	for _, pluginDef := range plugins.Plugins {
		// filter out app sub plugins
		if embeddedFilter == "0" && pluginDef.IncludedInAppId != "" {
			continue
		}

		// filter on type
		if typeFilter != "" && typeFilter != pluginDef.Type {
			continue
		}

		listItem := &dtos.PluginListItem{
			Id:   pluginDef.Id,
			Name: pluginDef.Name,
			Type: pluginDef.Type,
			Info: &pluginDef.Info,
		}

		if pluginSetting, exists := pluginSettingsMap[pluginDef.Id]; exists {
			listItem.Enabled = pluginSetting.Enabled
			listItem.Pinned = pluginSetting.Pinned
		}

		// filter out disabled
		if enabledFilter == "1" && !listItem.Enabled {
			continue
		}

		result = append(result, listItem)
	}

	return Json(200, result)
}

func GetPluginSettingById(c *middleware.Context) Response {
	pluginId := c.Params(":pluginId")

	if def, exists := plugins.Plugins[pluginId]; !exists {
		return ApiError(404, "Plugin not found, no installed plugin with that id", nil)
	} else {

		dto := &dtos.PluginSetting{
			Type:         def.Type,
			Id:           def.Id,
			Name:         def.Name,
			Info:         &def.Info,
			Dependencies: &def.Dependencies,
			Includes:     def.Includes,
			BaseUrl:      def.BaseUrl,
			Module:       def.Module,
		}

		if app, exists := plugins.Apps[pluginId]; exists {
			dto.Pages = app.Pages
		}

		query := m.GetPluginSettingByIdQuery{PluginId: pluginId, OrgId: c.OrgId}
		if err := bus.Dispatch(&query); err != nil {
			if err != m.ErrPluginSettingNotFound {
				return ApiError(500, "Failed to get login settings", nil)
			}
		} else {
			dto.Enabled = query.Result.Enabled
			dto.Pinned = query.Result.Pinned
			dto.JsonData = query.Result.JsonData
		}

		return Json(200, dto)
	}
}

func UpdatePluginSetting(c *middleware.Context, cmd m.UpdatePluginSettingCmd) Response {
	pluginId := c.Params(":pluginId")

	cmd.OrgId = c.OrgId
	cmd.PluginId = pluginId

	if _, ok := plugins.Apps[cmd.PluginId]; !ok {
		return ApiError(404, "Plugin not installed.", nil)
	}

	if err := bus.Dispatch(&cmd); err != nil {
		return ApiError(500, "Failed to update plugin setting", err)
	}

	return ApiSuccess("Plugin settings updated")
}

func GetPluginDashboards(c *middleware.Context) Response {
	pluginId := c.Params(":pluginId")

	if list, err := plugins.GetPluginDashboards(c.OrgId, pluginId); err != nil {
		if notfound, ok := err.(plugins.PluginNotFoundError); ok {
			return ApiError(404, notfound.Error(), nil)
		}

		return ApiError(500, "Failed to get plugin dashboards", err)
	} else {
		return Json(200, list)
	}
}

func GetPluginReadme(c *middleware.Context) Response {
	pluginId := c.Params(":pluginId")

	if content, err := plugins.GetPluginReadme(pluginId); err != nil {
		if notfound, ok := err.(plugins.PluginNotFoundError); ok {
			return ApiError(404, notfound.Error(), nil)
		}

		return ApiError(500, "Could not get readme", err)
	} else {
		return Respond(200, content)
	}
}

func ImportDashboard(c *middleware.Context, apiCmd dtos.ImportDashboardCommand) Response {

	cmd := plugins.ImportDashboardCommand{
		OrgId:    c.OrgId,
		UserId:   c.UserId,
		PluginId: apiCmd.PluginId,
		Path:     apiCmd.Path,
		Inputs:   apiCmd.Inputs,
	}

	if err := bus.Dispatch(&cmd); err != nil {
		return ApiError(500, "Failed to install dashboard", err)
	}

	return Json(200, cmd.Result)
}
<commit_msg>fix(datasource): filter out built in data sources from data source type dropdown<commit_after>package api

import (
	"github.com/grafana/grafana/pkg/api/dtos"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/middleware"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
)

func GetPluginList(c *middleware.Context) Response {
	typeFilter := c.Query("type")
	enabledFilter := c.Query("enabled")
	embeddedFilter := c.Query("embedded")

	pluginSettingsMap, err := plugins.GetPluginSettings(c.OrgId)

	if err != nil {
		return ApiError(500, "Failed to get list of plugins", err)
	}

	result := make([]*dtos.PluginListItem, 0)
	for _, pluginDef := range plugins.Plugins {
		// filter out app sub plugins
		if embeddedFilter == "0" && pluginDef.IncludedInAppId != "" {
			continue
		}

		// filter on type
		if typeFilter != "" && typeFilter != pluginDef.Type {
			continue
		}

		listItem := &dtos.PluginListItem{
			Id:   pluginDef.Id,
			Name: pluginDef.Name,
			Type: pluginDef.Type,
			Info: &pluginDef.Info,
		}

		if pluginSetting, exists := pluginSettingsMap[pluginDef.Id]; exists {
			listItem.Enabled = pluginSetting.Enabled
			listItem.Pinned = pluginSetting.Pinned
		}

		// filter out disabled
		if enabledFilter == "1" && !listItem.Enabled {
			continue
		}

		// filter out built in data sources
		if ds, exists := plugins.DataSources[pluginDef.Id]; exists {
			if ds.BuiltIn {
				continue
			}
		}

		result = append(result, listItem)
	}

	return Json(200, result)
}

func GetPluginSettingById(c *middleware.Context) Response {
	pluginId := c.Params(":pluginId")

	if def, exists := plugins.Plugins[pluginId]; !exists {
		return ApiError(404, "Plugin not found, no installed plugin with that id", nil)
	} else {

		dto := &dtos.PluginSetting{
			Type:         def.Type,
			Id:           def.Id,
			Name:         def.Name,
			Info:         &def.Info,
			Dependencies: &def.Dependencies,
			Includes:     def.Includes,
			BaseUrl:      def.BaseUrl,
			Module:       def.Module,
		}

		if app, exists := plugins.Apps[pluginId]; exists {
			dto.Pages = app.Pages
		}

		query := m.GetPluginSettingByIdQuery{PluginId: pluginId, OrgId: c.OrgId}
		if err := bus.Dispatch(&query); err != nil {
			if err != m.ErrPluginSettingNotFound {
				return ApiError(500, "Failed to get login settings", nil)
			}
		} else {
			dto.Enabled = query.Result.Enabled
			dto.Pinned = query.Result.Pinned
			dto.JsonData = query.Result.JsonData
		}

		return Json(200, dto)
	}
}

func UpdatePluginSetting(c *middleware.Context, cmd m.UpdatePluginSettingCmd) Response {
	pluginId := c.Params(":pluginId")

	cmd.OrgId = c.OrgId
	cmd.PluginId = pluginId

	if _, ok := plugins.Apps[cmd.PluginId]; !ok {
		return ApiError(404, "Plugin not installed.", nil)
	}

	if err := bus.Dispatch(&cmd); err != nil {
		return ApiError(500, "Failed to update plugin setting", err)
	}

	return ApiSuccess("Plugin settings updated")
}

func GetPluginDashboards(c *middleware.Context) Response {
	pluginId := c.Params(":pluginId")

	if list, err := plugins.GetPluginDashboards(c.OrgId, pluginId); err != nil {
		if notfound, ok := err.(plugins.PluginNotFoundError); ok {
			return ApiError(404, notfound.Error(), nil)
		}

		return ApiError(500, "Failed to get plugin dashboards", err)
	} else {
		return Json(200, list)
	}
}

func GetPluginReadme(c *middleware.Context) Response {
	pluginId := c.Params(":pluginId")

	if content, err := plugins.GetPluginReadme(pluginId); err != nil {
		if notfound, ok := err.(plugins.PluginNotFoundError); ok {
			return ApiError(404, notfound.Error(), nil)
		}

		return ApiError(500, "Could not get readme", err)
	} else {
		return Respond(200, content)
	}
}

func ImportDashboard(c *middleware.Context, apiCmd dtos.ImportDashboardCommand) Response {

	cmd := plugins.ImportDashboardCommand{
		OrgId:    c.OrgId,
		UserId:   c.UserId,
		PluginId: apiCmd.PluginId,
		Path:     apiCmd.Path,
		Inputs:   apiCmd.Inputs,
	}

	if err := bus.Dispatch(&cmd); err != nil {
		return ApiError(500, "Failed to install dashboard", err)
	}

	return Json(200, cmd.Result)
}
<|endoftext|>


<commit_before>package model

import (
	"bytes"
	"context"
	"database/sql"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/ViBiOh/httputils/pkg/db"
	"github.com/ViBiOh/httputils/pkg/errors"
	"github.com/ViBiOh/httputils/pkg/httperror"
	"github.com/ViBiOh/httputils/pkg/httpjson"
	"github.com/ViBiOh/httputils/pkg/logger"
	"github.com/ViBiOh/httputils/pkg/tools"
	opentracing "github.com/opentracing/opentracing-go"
)

const (
	maxConcurrentFetcher = 20
	listPrefix           = "/list"
)

// Config of package
type Config struct {
	infos *string
}

// App of package
type App interface {
	Health() bool
	Handler() http.Handler
	ListFunds() []Fund
	GetFundsAbove(float64, map[string]*Alert) ([]*Fund, error)
	GetFundsBelow(map[string]*Alert) ([]*Fund, error)
	GetCurrentAlerts() (map[string]*Alert, error)
	SaveAlert(*Alert, *sql.Tx) error
	Do(context.Context, time.Time) error
}

type app struct {
	dbConnexion *sql.DB
	fundsURL    string
	fundsMap    sync.Map
}

// Flags adds flags for configuring package
func Flags(fs *flag.FlagSet, prefix string) Config {
	return Config{
		infos: fs.String(tools.ToCamel(fmt.Sprintf("%sInfos", prefix)), "", "[funds] Informations URL"),
	}
}

// New creates new App from Config
func New(config Config, dbConfig db.Config) (App, error) {
	app := &app{
		fundsURL: strings.TrimSpace(*config.infos),
		fundsMap: sync.Map{},
	}

	fundsDB, err := db.New(dbConfig)
	if err != nil {
		logger.Error("%#v", errors.WithStack(err))
	} else {
		app.dbConnexion = fundsDB
	}

	return app, nil
}

// Do do scheduler task of refreshing data
func (a *app) Do(ctx context.Context, _ time.Time) error {
	if a.fundsURL == "" {
		return nil
	}

	logger.Info("Refresh started")
	defer logger.Info("Refresh ended")

	if err := a.refreshData(ctx); err != nil {
		logger.Error("%#v", err)
	}

	if a.dbConnexion != nil {
		if err := a.saveData(); err != nil {
			return err
		}
	}

	return nil
}

func (a *app) refreshData(ctx context.Context) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "Fetch Funds")
	defer span.Finish()

	inputs, results := tools.ConcurrentAction(0, func(ID interface{}) (interface{}, error) {
		return fetchFund(ctx, a.fundsURL, ID.([]byte))
	})

	go func() {
		defer close(inputs)

		for _, fundID := range fundsIds {
			inputs <- fundID
		}
	}()

	errorIds := make([][]byte, 0)

	for {
		result, ok := <-results

		if !ok {
			break
		}

		if result.Err != nil {
			errorIds = append(errorIds, result.Input.([]byte))
		} else {
			content := result.Output.(Fund)
			a.fundsMap.Store(content.ID, content)
		}
	}

	if len(errorIds) > 0 {
		return errors.New("errors with ids %s", bytes.Join(errorIds, []byte(",")))
	}

	return nil
}

func (a *app) saveData() (err error) {
	var tx *sql.Tx
	if tx, err = db.GetTx(a.dbConnexion, nil); err != nil {
		return
	}

	defer func() {
		err = db.EndTx(tx, err)
	}()

	a.fundsMap.Range(func(_ interface{}, value interface{}) bool {
		fund := value.(Fund)
		err = a.saveFund(&fund, tx)

		return err == nil
	})

	return
}

// Health check health
func (a *app) Health() bool {
	return db.Ping(a.dbConnexion)
}

// ListFunds return content of funds' map
func (a *app) ListFunds() []Fund {
	funds := make([]Fund, 0, len(fundsIds))

	a.fundsMap.Range(func(_ interface{}, value interface{}) bool {
		funds = append(funds, value.(Fund))
		return true
	})

	return funds
}

func (a *app) listHandler(w http.ResponseWriter, r *http.Request) {
	if err := httpjson.ResponseArrayJSON(w, http.StatusOK, a.ListFunds(), httpjson.IsPretty(r)); err != nil {
		httperror.InternalServerError(w, err)
	}
}

// Handler for model request. Should be use with net/http
func (a *app) Handler() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodOptions {
			if _, err := w.Write(nil); err != nil {
				httperror.InternalServerError(w, errors.WithStack(err))
			}
			return
		}

		if strings.HasPrefix(r.URL.Path, listPrefix) {
			if r.Method == http.MethodGet {
				a.listHandler(w, r)
			} else {
				w.WriteHeader(http.StatusMethodNotAllowed)
			}
		} else {
			w.WriteHeader(http.StatusNotFound)
		}
	})
}
<commit_msg>Updating concurrent fetcher value<commit_after>package model

import (
	"bytes"
	"context"
	"database/sql"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/ViBiOh/httputils/pkg/db"
	"github.com/ViBiOh/httputils/pkg/errors"
	"github.com/ViBiOh/httputils/pkg/httperror"
	"github.com/ViBiOh/httputils/pkg/httpjson"
	"github.com/ViBiOh/httputils/pkg/logger"
	"github.com/ViBiOh/httputils/pkg/tools"
	opentracing "github.com/opentracing/opentracing-go"
)

const (
	maxConcurrentFetcher = 8
	listPrefix           = "/list"
)

// Config of package
type Config struct {
	infos *string
}

// App of package
type App interface {
	Health() bool
	Handler() http.Handler
	ListFunds() []Fund
	GetFundsAbove(float64, map[string]*Alert) ([]*Fund, error)
	GetFundsBelow(map[string]*Alert) ([]*Fund, error)
	GetCurrentAlerts() (map[string]*Alert, error)
	SaveAlert(*Alert, *sql.Tx) error
	Do(context.Context, time.Time) error
}

type app struct {
	dbConnexion *sql.DB
	fundsURL    string
	fundsMap    sync.Map
}

// Flags adds flags for configuring package
func Flags(fs *flag.FlagSet, prefix string) Config {
	return Config{
		infos: fs.String(tools.ToCamel(fmt.Sprintf("%sInfos", prefix)), "", "[funds] Informations URL"),
	}
}

// New creates new App from Config
func New(config Config, dbConfig db.Config) (App, error) {
	app := &app{
		fundsURL: strings.TrimSpace(*config.infos),
		fundsMap: sync.Map{},
	}

	fundsDB, err := db.New(dbConfig)
	if err != nil {
		logger.Error("%#v", errors.WithStack(err))
	} else {
		app.dbConnexion = fundsDB
	}

	return app, nil
}

// Do do scheduler task of refreshing data
func (a *app) Do(ctx context.Context, _ time.Time) error {
	if a.fundsURL == "" {
		return nil
	}

	logger.Info("Refresh started")
	defer logger.Info("Refresh ended")

	if err := a.refreshData(ctx); err != nil {
		logger.Error("%#v", err)
	}

	if a.dbConnexion != nil {
		if err := a.saveData(); err != nil {
			return err
		}
	}

	return nil
}

func (a *app) refreshData(ctx context.Context) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "Fetch Funds")
	defer span.Finish()

	inputs, results := tools.ConcurrentAction(maxConcurrentFetcher, func(ID interface{}) (interface{}, error) {
		return fetchFund(ctx, a.fundsURL, ID.([]byte))
	})

	go func() {
		defer close(inputs)

		for _, fundID := range fundsIds {
			inputs <- fundID
		}
	}()

	errorIds := make([][]byte, 0)

	for {
		result, ok := <-results

		if !ok {
			break
		}

		if result.Err != nil {
			errorIds = append(errorIds, result.Input.([]byte))
		} else {
			content := result.Output.(Fund)
			a.fundsMap.Store(content.ID, content)
		}
	}

	if len(errorIds) > 0 {
		return errors.New("errors with ids %s", bytes.Join(errorIds, []byte(",")))
	}

	return nil
}

func (a *app) saveData() (err error) {
	var tx *sql.Tx
	if tx, err = db.GetTx(a.dbConnexion, nil); err != nil {
		return
	}

	defer func() {
		err = db.EndTx(tx, err)
	}()

	a.fundsMap.Range(func(_ interface{}, value interface{}) bool {
		fund := value.(Fund)
		err = a.saveFund(&fund, tx)

		return err == nil
	})

	return
}

// Health check health
func (a *app) Health() bool {
	return db.Ping(a.dbConnexion)
}

// ListFunds return content of funds' map
func (a *app) ListFunds() []Fund {
	funds := make([]Fund, 0, len(fundsIds))

	a.fundsMap.Range(func(_ interface{}, value interface{}) bool {
		funds = append(funds, value.(Fund))
		return true
	})

	return funds
}

func (a *app) listHandler(w http.ResponseWriter, r *http.Request) {
	if err := httpjson.ResponseArrayJSON(w, http.StatusOK, a.ListFunds(), httpjson.IsPretty(r)); err != nil {
		httperror.InternalServerError(w, err)
	}
}

// Handler for model request. Should be use with net/http
func (a *app) Handler() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodOptions {
			if _, err := w.Write(nil); err != nil {
				httperror.InternalServerError(w, errors.WithStack(err))
			}
			return
		}

		if strings.HasPrefix(r.URL.Path, listPrefix) {
			if r.Method == http.MethodGet {
				a.listHandler(w, r)
			} else {
				w.WriteHeader(http.StatusMethodNotAllowed)
			}
		} else {
			w.WriteHeader(http.StatusNotFound)
		}
	})
}
<|endoftext|>


<commit_before>package move

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/vfs"
	vcardParser "github.com/emersion/go-vcard"
)

// ContactName is a struct describing a name of a contact
type ContactName struct {
	FamilyName     string `json:"familyName,omitempty"`
	GivenName      string `json:"givenName,omitempty"`
	AdditionalName string `json:"additionalName,omitempty"`
	NamePrefix     string `json:"namePrefix,omitempty"`
	NameSuffix     string `json:"nameSuffix,omitempty"`
}

// ContactEmail is a struct describing an email of a contact
type ContactEmail struct {
	Address string `json:"address"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactAddress is a struct describing an address of a contact
type ContactAddress struct {
	Street           string `json:"street,omitempty"`
	Pobox            string `json:"pobox,omitempty"`
	City             string `json:"city,omitempty"`
	Region           string `json:"region,omitempty"`
	Postcode         string `json:"postcode,omitempty"`
	Country          string `json:"country,omitempty"`
	Type             string `json:"type,omitempty"`
	Primary          bool   `json:"primary,omitempty"`
	Label            string `json:"label,omitempty"`
	FormattedAddress string `json:"formattedAddress,omitempty"`
}

// ContactPhone is a struct describing a phone of a contact
type ContactPhone struct {
	Number  string `json:"number"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactCozy is a struct describing a cozy instance of a contact
type ContactCozy struct {
	URL     string `json:"url"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// Contact is a struct containing all the informations about a contact
type Contact struct {
	DocID  string `json:"_id,omitempty"`
	DocRev string `json:"_rev,omitempty"`

	FullName string           `json:"fullname,omitempty"`
	Name     ContactName      `json:"name,omitempty"`
	Birthday string           `json:"birthday,omitempty"`
	Note     string           `json:"note,omitempty"`
	Email    []ContactEmail   `json:"email,omitempty"`
	Address  []ContactAddress `json:"address,omitempty"`
	Phone    []ContactPhone   `json:"phone,omitempty"`
	Cozy     []ContactCozy    `json:"cozy,omitempty"`
}

// ID returns the contact qualified identifier
func (c *Contact) ID() string { return c.DocID }

// Rev returns the contact revision
func (c *Contact) Rev() string { return c.DocRev }

// DocType returns the contact document type
func (c *Contact) DocType() string { return consts.Contacts }

// Clone implements couchdb.Doc
func (c *Contact) Clone() couchdb.Doc {
	cloned := *c
	cloned.FullName = c.FullName
	cloned.Name = c.Name

	cloned.Email = make([]ContactEmail, len(c.Email))
	copy(cloned.Email, c.Email)

	cloned.Address = make([]ContactAddress, len(c.Address))
	copy(cloned.Address, c.Address)

	cloned.Phone = make([]ContactPhone, len(c.Phone))
	copy(cloned.Phone, c.Phone)

	cloned.Cozy = make([]ContactCozy, len(c.Cozy))
	copy(cloned.Cozy, c.Cozy)

	return &cloned
}

// SetID changes the contact qualified identifier
func (c *Contact) SetID(id string) { c.DocID = id }

// SetRev changes the contact revision
func (c *Contact) SetRev(rev string) { c.DocRev = rev }

func createContact(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, db couchdb.Database) error {
	decoder := vcardParser.NewDecoder(tr)
	vcard, err := decoder.Decode()
	if err != nil {
		return err
	}

	fullname := "John Doe"
	contactname := ContactName{
		GivenName:  "John",
		FamilyName: "Doe",
	}

	name := vcard.Name()
	if name != nil {
		contactname = ContactName{
			FamilyName:     name.FamilyName,
			GivenName:      name.GivenName,
			AdditionalName: name.AdditionalName,
			NamePrefix:     name.HonorificPrefix,
			NameSuffix:     name.HonorificSuffix,
		}
		fullname = name.Value
	}
	if names := vcard.FormattedNames(); len(names) > 0 {
		fullname = names[0].Value
	}

	var bday string
	if field := vcard.Get("BDAY"); field != nil {
		bday = field.Value
	}

	var note string
	if field := vcard.Get("NOTE"); field != nil {
		note = field.Value
	}

	var contactemail []ContactEmail
	for _, mail := range vcard.Values("EMAIL") {
		ce := ContactEmail{
			Address: mail,
		}
		contactemail = append(contactemail, ce)
	}

	var contactphone []ContactPhone
	for _, phone := range vcard.Values("TEL") {
		cp := ContactPhone{
			Number: phone,
		}
		contactphone = append(contactphone, cp)
	}

	var contactaddress []ContactAddress
	for _, address := range vcard.Addresses() {
		ca := ContactAddress{
			Street:           address.StreetAddress,
			Pobox:            address.PostOfficeBox,
			City:             address.Locality,
			Region:           address.Region,
			Postcode:         address.PostalCode,
			Country:          address.Country,
			FormattedAddress: address.Value,
		}
		contactaddress = append(contactaddress, ca)
	}

	contact := &Contact{
		FullName: fullname,
		Name:     contactname,
		Birthday: bday,
		Note:     note,
		Address:  contactaddress,
		Email:    contactemail,
		Phone:    contactphone,
	}

	return couchdb.CreateDoc(db, contact)
}

func createAlbum(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc, db couchdb.Database) error {
	m := make(map[string]*couchdb.DocReference)

	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		jsondoc := &couchdb.JSONDoc{}
		err := jsondoc.UnmarshalJSON(bs.Bytes())
		if err != nil {
			return err
		}
		doctype, ok := jsondoc.M["type"].(string)
		if ok {
			jsondoc.Type = doctype
		}
		delete(jsondoc.M, "type")

		id := jsondoc.ID()
		jsondoc.SetID("")
		jsondoc.SetRev("")

		err = couchdb.CreateDoc(db, jsondoc)
		if err != nil {
			return err
		}

		m[id] = &couchdb.DocReference{
			ID:   jsondoc.ID(),
			Type: jsondoc.DocType(),
		}

	}

	_, err := tr.Next()
	if err != nil {
		return err
	}

	bs = bufio.NewScanner(tr)
	for bs.Scan() {
		ref := &Reference{}
		err := json.Unmarshal(bs.Bytes(), &ref)
		if err != nil {
			return err
		}

		file, err := fs.FileByPath(dstDoc.Fullpath + ref.Filepath)
		if err != nil {
			return err
		}

		if m[ref.Albumid] != nil {
			file.AddReferencedBy(*m[ref.Albumid])
			if err = couchdb.UpdateDoc(db, file); err != nil {
				return err
			}
		}

	}

	return nil

}

func createFile(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc) error {
	name := strings.TrimPrefix(hdr.Name, "files/")
	filename := path.Base(name)
	mime, class := vfs.ExtractMimeAndClassFromFilename(filename)
	now := time.Now()
	executable := hdr.FileInfo().Mode()&0100 != 0

	dirname := path.Join(dstDoc.Fullpath, path.Dir(name))
	dirDoc, err := fs.DirByPath(dirname)
	if err != nil {
		// XXX Tarball from cozy v2 exports can have files in a non-existant directory
		if dirDoc, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
			return err
		}
	}
	fileDoc, err := vfs.NewFileDoc(filename, dirDoc.ID(), hdr.Size, nil, mime, class, now, executable, false, nil)
	if err != nil {
		return err
	}

	file, err := fs.CreateFile(fileDoc, nil)
	if err != nil {
		ext := path.Ext(fileDoc.DocName)
		fileName := fileDoc.DocName[0 : len(fileDoc.DocName)-len(ext)]
		fileDoc.DocName = fmt.Sprintf("%s-conflict-%d%s", fileName, rand.Int(), ext)
		file, err = fs.CreateFile(fileDoc, nil)
		if err != nil {
			return err
		}
	}

	_, err = io.Copy(file, tr)
	cerr := file.Close()
	if err != nil {
		return err
	}
	return cerr
}

// untar untar doc directory
func untar(r io.Reader, dst *vfs.DirDoc, instance *instance.Instance) error {
	fs := instance.VFS()

	// tar+gzip reader
	gr, err := gzip.NewReader(r)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Can't open gzip reader for import: %s", err)
		return err
	}
	defer gr.Close()
	tgz := tar.NewReader(gr)

	for {
		hdr, err := tgz.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Error on import: %s", err)
			return err
		}

		parts := strings.SplitN(path.Clean(hdr.Name), "/", 2)
		var name, doctype string
		if len(parts) > 1 {
			doctype = parts[0]
			name = parts[1]
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			if doctype == "files" {
				dirname := path.Join(dst.Fullpath, name)
				if _, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import directory %s: %s", hdr.Name, err)
					return err
				}
			}

		case tar.TypeReg:
			if doctype == "albums" && name == albumsFile {
				err = createAlbum(fs, hdr, tgz, dst, instance)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
					return err
				}
			} else if doctype == "contacts" {
				if err := createContact(fs, hdr, tgz, instance); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import contact %s: %s", hdr.Name, err)
					return err
				}
			} else if doctype == "files" {
				if err := createFile(fs, hdr, tgz, dst); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import file %s: %s", hdr.Name, err)
					return err
				}
			}

		default:
			logger.WithDomain(instance.Domain).Errorf("Unknown typeflag for import: %v", hdr.Typeflag)
			return errors.New("Unknown typeflag")
		}
	}

	return nil
}

// Import is used to import a tarball with files, photos, contacts to an instance
func Import(instance *instance.Instance, filename, destination string) error {
	r, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer r.Close()

	fs := instance.VFS()
	exist, err := vfs.DirExists(fs, destination)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Error for destination %s: %s", destination, err)
		return err
	}
	var dst *vfs.DirDoc
	if !exist {
		dst, err = vfs.Mkdir(fs, destination, nil)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't create destination directory %s: %s", destination, err)
			return err
		}
	} else {
		dst, err = fs.DirByPath(destination)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't find destination directory %s: %s", destination, err)
			return err
		}
	}

	return untar(r, dst, instance)
}
<commit_msg>Fix import of albums with missing photos<commit_after>package move

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/vfs"
	vcardParser "github.com/emersion/go-vcard"
)

// ContactName is a struct describing a name of a contact
type ContactName struct {
	FamilyName     string `json:"familyName,omitempty"`
	GivenName      string `json:"givenName,omitempty"`
	AdditionalName string `json:"additionalName,omitempty"`
	NamePrefix     string `json:"namePrefix,omitempty"`
	NameSuffix     string `json:"nameSuffix,omitempty"`
}

// ContactEmail is a struct describing an email of a contact
type ContactEmail struct {
	Address string `json:"address"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactAddress is a struct describing an address of a contact
type ContactAddress struct {
	Street           string `json:"street,omitempty"`
	Pobox            string `json:"pobox,omitempty"`
	City             string `json:"city,omitempty"`
	Region           string `json:"region,omitempty"`
	Postcode         string `json:"postcode,omitempty"`
	Country          string `json:"country,omitempty"`
	Type             string `json:"type,omitempty"`
	Primary          bool   `json:"primary,omitempty"`
	Label            string `json:"label,omitempty"`
	FormattedAddress string `json:"formattedAddress,omitempty"`
}

// ContactPhone is a struct describing a phone of a contact
type ContactPhone struct {
	Number  string `json:"number"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactCozy is a struct describing a cozy instance of a contact
type ContactCozy struct {
	URL     string `json:"url"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// Contact is a struct containing all the informations about a contact
type Contact struct {
	DocID  string `json:"_id,omitempty"`
	DocRev string `json:"_rev,omitempty"`

	FullName string           `json:"fullname,omitempty"`
	Name     ContactName      `json:"name,omitempty"`
	Birthday string           `json:"birthday,omitempty"`
	Note     string           `json:"note,omitempty"`
	Email    []ContactEmail   `json:"email,omitempty"`
	Address  []ContactAddress `json:"address,omitempty"`
	Phone    []ContactPhone   `json:"phone,omitempty"`
	Cozy     []ContactCozy    `json:"cozy,omitempty"`
}

// ID returns the contact qualified identifier
func (c *Contact) ID() string { return c.DocID }

// Rev returns the contact revision
func (c *Contact) Rev() string { return c.DocRev }

// DocType returns the contact document type
func (c *Contact) DocType() string { return consts.Contacts }

// Clone implements couchdb.Doc
func (c *Contact) Clone() couchdb.Doc {
	cloned := *c
	cloned.FullName = c.FullName
	cloned.Name = c.Name

	cloned.Email = make([]ContactEmail, len(c.Email))
	copy(cloned.Email, c.Email)

	cloned.Address = make([]ContactAddress, len(c.Address))
	copy(cloned.Address, c.Address)

	cloned.Phone = make([]ContactPhone, len(c.Phone))
	copy(cloned.Phone, c.Phone)

	cloned.Cozy = make([]ContactCozy, len(c.Cozy))
	copy(cloned.Cozy, c.Cozy)

	return &cloned
}

// SetID changes the contact qualified identifier
func (c *Contact) SetID(id string) { c.DocID = id }

// SetRev changes the contact revision
func (c *Contact) SetRev(rev string) { c.DocRev = rev }

func createContact(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, db couchdb.Database) error {
	decoder := vcardParser.NewDecoder(tr)
	vcard, err := decoder.Decode()
	if err != nil {
		return err
	}

	fullname := "John Doe"
	contactname := ContactName{
		GivenName:  "John",
		FamilyName: "Doe",
	}

	name := vcard.Name()
	if name != nil {
		contactname = ContactName{
			FamilyName:     name.FamilyName,
			GivenName:      name.GivenName,
			AdditionalName: name.AdditionalName,
			NamePrefix:     name.HonorificPrefix,
			NameSuffix:     name.HonorificSuffix,
		}
		fullname = name.Value
	}
	if names := vcard.FormattedNames(); len(names) > 0 {
		fullname = names[0].Value
	}

	var bday string
	if field := vcard.Get("BDAY"); field != nil {
		bday = field.Value
	}

	var note string
	if field := vcard.Get("NOTE"); field != nil {
		note = field.Value
	}

	var contactemail []ContactEmail
	for _, mail := range vcard.Values("EMAIL") {
		ce := ContactEmail{
			Address: mail,
		}
		contactemail = append(contactemail, ce)
	}

	var contactphone []ContactPhone
	for _, phone := range vcard.Values("TEL") {
		cp := ContactPhone{
			Number: phone,
		}
		contactphone = append(contactphone, cp)
	}

	var contactaddress []ContactAddress
	for _, address := range vcard.Addresses() {
		ca := ContactAddress{
			Street:           address.StreetAddress,
			Pobox:            address.PostOfficeBox,
			City:             address.Locality,
			Region:           address.Region,
			Postcode:         address.PostalCode,
			Country:          address.Country,
			FormattedAddress: address.Value,
		}
		contactaddress = append(contactaddress, ca)
	}

	contact := &Contact{
		FullName: fullname,
		Name:     contactname,
		Birthday: bday,
		Note:     note,
		Address:  contactaddress,
		Email:    contactemail,
		Phone:    contactphone,
	}

	return couchdb.CreateDoc(db, contact)
}

func createAlbums(i *instance.Instance, tr *tar.Reader, albums *AlbumReferences) error {
	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		jsondoc := &couchdb.JSONDoc{}
		if err := jsondoc.UnmarshalJSON(bs.Bytes()); err != nil {
			return err
		}
		delete(jsondoc.M, "type")
		id := jsondoc.ID()
		jsondoc.SetID("")
		jsondoc.SetRev("")
		jsondoc.Type = consts.PhotosAlbums

		if err := couchdb.CreateDoc(i, jsondoc); err != nil {
			return err
		}
		(*albums)[id] = couchdb.DocReference{
			ID:   jsondoc.ID(),
			Type: consts.PhotosAlbums,
		}
	}

	return nil
}

// AlbumReferences is used to associate photos to their albums, though we don't
// force the ID of the albums to the values in the tarball.
type AlbumReferences map[string]couchdb.DocReference

func fillAlbums(i *instance.Instance, tr *tar.Reader, dstDoc *vfs.DirDoc, albums *AlbumReferences) error {
	fs := i.VFS()
	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		ref := Reference{}
		if err := json.Unmarshal(bs.Bytes(), &ref); err != nil {
			return err
		}

		file, err := fs.FileByPath(dstDoc.Fullpath + ref.Filepath)
		if err != nil {
			// XXX Ignore missing photos (we have this for migrating some cozy v2)
			continue
		}

		if docRef, ok := (*albums)[ref.Albumid]; ok {
			file.AddReferencedBy(docRef)
			if err = couchdb.UpdateDoc(i, file); err != nil {
				return err
			}
		}
	}

	return nil
}

func createFile(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc) error {
	name := strings.TrimPrefix(hdr.Name, "files/")
	filename := path.Base(name)
	mime, class := vfs.ExtractMimeAndClassFromFilename(filename)
	now := time.Now()
	executable := hdr.FileInfo().Mode()&0100 != 0

	dirname := path.Join(dstDoc.Fullpath, path.Dir(name))
	dirDoc, err := fs.DirByPath(dirname)
	if err != nil {
		// XXX Tarball from cozy v2 exports can have files in a non-existant directory
		if dirDoc, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
			return err
		}
	}
	fileDoc, err := vfs.NewFileDoc(filename, dirDoc.ID(), hdr.Size, nil, mime, class, now, executable, false, nil)
	if err != nil {
		return err
	}

	file, err := fs.CreateFile(fileDoc, nil)
	if err != nil {
		ext := path.Ext(fileDoc.DocName)
		fileName := fileDoc.DocName[0 : len(fileDoc.DocName)-len(ext)]
		fileDoc.DocName = fmt.Sprintf("%s-conflict-%d%s", fileName, rand.Int(), ext)
		file, err = fs.CreateFile(fileDoc, nil)
		if err != nil {
			return err
		}
	}

	_, err = io.Copy(file, tr)
	cerr := file.Close()
	if err != nil {
		return err
	}
	return cerr
}

// untar untar doc directory
func untar(r io.Reader, dst *vfs.DirDoc, instance *instance.Instance) error {
	fs := instance.VFS()

	// tar+gzip reader
	gr, err := gzip.NewReader(r)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Can't open gzip reader for import: %s", err)
		return err
	}
	defer gr.Close()
	tgz := tar.NewReader(gr)

	albumsRef := make(AlbumReferences)

	for {
		hdr, err := tgz.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Error on import: %s", err)
			return err
		}

		parts := strings.SplitN(path.Clean(hdr.Name), "/", 2)
		var name, doctype string
		if len(parts) > 1 {
			doctype = parts[0]
			name = parts[1]
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			if doctype == "files" {
				dirname := path.Join(dst.Fullpath, name)
				if _, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import directory %s: %s", hdr.Name, err)
					return err
				}
			}

		case tar.TypeReg:
			if doctype == "albums" && name == albumsFile {
				err = createAlbums(instance, tgz, &albumsRef)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
					return err
				}
			} else if doctype == "albums" && name == referencesFile {
				err = fillAlbums(instance, tgz, dst, &albumsRef)
				if err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import album %s: %s", hdr.Name, err)
					return err
				}
			} else if doctype == "contacts" {
				if err := createContact(fs, hdr, tgz, instance); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import contact %s: %s", hdr.Name, err)
					return err
				}
			} else if doctype == "files" {
				if err := createFile(fs, hdr, tgz, dst); err != nil {
					logger.WithDomain(instance.Domain).Errorf("Can't import file %s: %s", hdr.Name, err)
					return err
				}
			}

		default:
			logger.WithDomain(instance.Domain).Errorf("Unknown typeflag for import: %v", hdr.Typeflag)
			return errors.New("Unknown typeflag")
		}
	}

	return nil
}

// Import is used to import a tarball with files, photos, contacts to an instance
func Import(instance *instance.Instance, filename, destination string) error {
	r, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer r.Close()

	fs := instance.VFS()
	exist, err := vfs.DirExists(fs, destination)
	if err != nil {
		logger.WithDomain(instance.Domain).Errorf("Error for destination %s: %s", destination, err)
		return err
	}
	var dst *vfs.DirDoc
	if !exist {
		dst, err = vfs.Mkdir(fs, destination, nil)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't create destination directory %s: %s", destination, err)
			return err
		}
	} else {
		dst, err = fs.DirByPath(destination)
		if err != nil {
			logger.WithDomain(instance.Domain).Errorf("Can't find destination directory %s: %s", destination, err)
			return err
		}
	}

	return untar(r, dst, instance)
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"html"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/misc"
)

var queueSyncInterval = 5 * time.Second

const maxErrors = 20

var _ = log.Printf

// TODO: rate control + tunable
// TODO: expose copierPoolSize as tunable
type SyncHandler struct {
	fromName, fromqName, toName string
	from, fromq, to             blobserver.Storage

	copierPoolSize int

	lk             sync.Mutex // protects following
	status         string
	blobStatus     map[string]fmt.Stringer // stringer called with lk held
	recentErrors   []timestampedError
	recentCopyTime time.Time
	totalCopies    int64
	totalCopyBytes int64
	totalErrors    int64
}

func init() {
	blobserver.RegisterHandlerConstructor("sync", newSyncFromConfig)
}

func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	from := conf.RequiredString("from")
	to := conf.RequiredString("to")
	fullSync := conf.OptionalBool("fullSyncOnStart", false)
	blockFullSync := conf.OptionalBool("blockingFullSyncOnStart", false)
	if err = conf.Validate(); err != nil {
		return
	}
	fromBs, err := ld.GetStorage(from)
	if err != nil {
		return
	}
	toBs, err := ld.GetStorage(to)
	if err != nil {
		return
	}
	fromQsc, ok := fromBs.(blobserver.StorageQueueCreator)
	if !ok {
		return nil, fmt.Errorf("Prefix %s (type %T) does not support being efficient replication source (queueing)", from, fromBs)
	}
	synch, err := createSyncHandler(from, to, fromQsc, toBs)
	if err != nil {
		return
	}

	if fullSync || blockFullSync {
		didFullSync := make(chan bool, 1)
		go func() {
			n := synch.runSync("queue", fromQsc, 0)
			log.Printf("Queue sync copied %d blobs", n)
			n = synch.runSync("full", fromBs, 0)
			log.Printf("Full sync copied %d blobs", n)
			didFullSync <- true
			synch.syncQueueLoop()
		}()
		if blockFullSync {
			log.Printf("Blocking startup, waiting for full sync from %q to %q", from, to)
			<-didFullSync
			log.Printf("Full sync complete.")
		}
	} else {
		go synch.syncQueueLoop()
	}
	return synch, nil
}

type timestampedError struct {
	t   time.Time
	err error
}

func createSyncHandler(fromName, toName string, from blobserver.StorageQueueCreator, to blobserver.Storage) (*SyncHandler, error) {
	h := &SyncHandler{
		copierPoolSize: 3,
		from:           from,
		to:             to,
		fromName:       fromName,
		toName:         toName,
		status:         "not started",
		blobStatus:     make(map[string]fmt.Stringer),
	}
	h.fromqName = strings.Replace(strings.Trim(toName, "/"), "/", "-", -1)
	var err error
	h.fromq, err = from.CreateQueue(h.fromqName)
	if err != nil {
		return nil, fmt.Errorf("Prefix %s (type %T) failed to create queue %q: %v",
			fromName, from, h.fromqName, err)
	}
	return h, nil
}

func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	sh.lk.Lock()
	defer sh.lk.Unlock()

	fmt.Fprintf(rw, "
%s to %s Sync Status

Current status: %s
",
		sh.fromName, sh.toName, html.EscapeString(sh.status))

	fmt.Fprintf(rw, "
Stats:

")
	fmt.Fprintf(rw, "
Blobs copied: %d
", sh.totalCopies)
	fmt.Fprintf(rw, "
Bytes copied: %d
", sh.totalCopyBytes)
	if !sh.recentCopyTime.IsZero() {
		fmt.Fprintf(rw, "
Most recent copy: %s
", sh.recentCopyTime.Format(time.RFC3339))
	}
	fmt.Fprintf(rw, "
Copy errors: %d
", sh.totalErrors)
	fmt.Fprintf(rw, "
")

	if len(sh.blobStatus) > 0 {
		fmt.Fprintf(rw, "
Current Copies:

")
		for blobstr, sfn := range sh.blobStatus {
			fmt.Fprintf(rw, "
%s: %s
\n",
				blobstr, html.EscapeString(sfn.String()))
		}
		fmt.Fprintf(rw, "
")
	}

	if len(sh.recentErrors) > 0 {
		fmt.Fprintf(rw, "
Recent Errors:

")
		for _, te := range sh.recentErrors {
			fmt.Fprintf(rw, "
%s: %s
\n",
				te.t.Format(time.RFC3339),
				html.EscapeString(te.err.Error()))
		}
		fmt.Fprintf(rw, "
")
	}
}

func (sh *SyncHandler) setStatus(s string, args ...interface{}) {
	s = time.Now().UTC().Format(time.RFC3339) + ": " + fmt.Sprintf(s, args...)
	sh.lk.Lock()
	defer sh.lk.Unlock()
	sh.status = s
}

func (sh *SyncHandler) setBlobStatus(blobref string, s fmt.Stringer) {
	sh.lk.Lock()
	defer sh.lk.Unlock()
	if s != nil {
		sh.blobStatus[blobref] = s
	} else {
		delete(sh.blobStatus, blobref)
	}
}

func (sh *SyncHandler) addErrorToLog(err error) {
	log.Printf(err.Error())
	sh.lk.Lock()
	defer sh.lk.Unlock()
	sh.recentErrors = append(sh.recentErrors, timestampedError{time.Now().UTC(), err})
	if len(sh.recentErrors) > maxErrors {
		// Kinda lame, but whatever. Only for errors, rare.
		copy(sh.recentErrors[:maxErrors], sh.recentErrors[1:maxErrors+1])
		sh.recentErrors = sh.recentErrors[:maxErrors]
	}
}

type copyResult struct {
	sb  blobref.SizedBlobRef
	err error
}

func (sh *SyncHandler) runSync(srcName string, enumSrc blobserver.Storage, longPollWait time.Duration) int {
	if longPollWait != 0 {
		sh.setStatus("Idle; waiting for new blobs")
	}
	enumch := make(chan blobref.SizedBlobRef)
	errch := make(chan error, 1)
	go func() {
		errch <- enumSrc.EnumerateBlobs(enumch, "", 1000, longPollWait)
	}()

	nCopied := 0
	toCopy := 0

	workch := make(chan blobref.SizedBlobRef, 1000)
	resch := make(chan copyResult, 8)
	for sb := range enumch {
		toCopy++
		workch <- sb
		if toCopy <= sh.copierPoolSize {
			go sh.copyWorker(resch, workch)
		}
		sh.setStatus("Enumerating queued blobs: %d", toCopy)
	}
	close(workch)
	for i := 0; i < toCopy; i++ {
		sh.setStatus("Copied %d/%d of batch of queued blobs", nCopied, toCopy)
		res := <-resch
		nCopied++
		sh.lk.Lock()
		if res.err == nil {
			sh.totalCopies++
			sh.totalCopyBytes += res.sb.Size
			sh.recentCopyTime = time.Now().UTC()
		} else {
			sh.totalErrors++
		}
		sh.lk.Unlock()
	}

	if err := <-errch; err != nil {
		sh.addErrorToLog(fmt.Errorf("replication error for source %q, enumerate from source: %v", srcName, err))
		return nCopied
	}
	return nCopied
}

func (sh *SyncHandler) syncQueueLoop() {
	every(queueSyncInterval, func() {
		for sh.runSync(sh.fromqName, sh.fromq, queueSyncInterval) > 0 {
			// Loop, before sleeping.
		}
		sh.setStatus("Sleeping briefly before next long poll.")
	})
}

func (sh *SyncHandler) copyWorker(res chan<- copyResult, work <-chan blobref.SizedBlobRef) {
	for sb := range work {
		res <- copyResult{sb, sh.copyBlob(sb)}
	}
}

type statusFunc func() string

func (sf statusFunc) String() string {
	return sf()
}

type status string

func (s status) String() string {
	return string(s)
}

func (sh *SyncHandler) copyBlob(sb blobref.SizedBlobRef) error {
	key := sb.BlobRef.String()
	set := func(s fmt.Stringer) {
		sh.setBlobStatus(key, s)
	}
	defer set(nil)

	errorf := func(s string, args ...interface{}) error {
		// TODO: increment error stats
		pargs := []interface{}{sh.fromqName, sb.BlobRef}
		pargs = append(pargs, args...)
		err := fmt.Errorf("replication error for queue %q, blob %s: "+s, pargs...)
		sh.addErrorToLog(err)
		return err
	}

	set(status("sending GET to source"))
	blobReader, fromSize, err := sh.from.FetchStreaming(sb.BlobRef)
	if err != nil {
		return errorf("source fetch: %v", err)
	}
	if fromSize != sb.Size {
		return errorf("source fetch size mismatch: get=%d, enumerate=%d", fromSize, sb.Size)
	}

	bytesCopied := int64(0) // accessed without locking; minor, just for status display
	set(statusFunc(func() string {
		return fmt.Sprintf("copying: %d/%d bytes", bytesCopied, sb.Size)
	}))
	newsb, err := sh.to.ReceiveBlob(sb.BlobRef, misc.CountingReader{blobReader, &bytesCopied})
	if err != nil {
		return errorf("dest write: %v", err)
	}
	if newsb.Size != sb.Size {
		return errorf("write size mismatch: source_read=%d but dest_write=%d", sb.Size, newsb.Size)
	}
	set(status("copied; removing from queue"))
	err = sh.fromq.RemoveBlobs([]*blobref.BlobRef{sb.BlobRef})
	if err != nil {
		return errorf("source queue delete: %v", err)
	}
	return nil
}

func every(interval time.Duration, f func()) {
	for {
		t1 := time.Now()
		f()
		sleepUntil := t1.Add(interval)
		if sleep := sleepUntil.Sub(time.Now()); sleep > 0 {
			time.Sleep(sleep)
		}
	}
}
<commit_msg>sync: fix fd leak, update a TODO<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"fmt"
	"html"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/blobserver"
	"camlistore.org/pkg/jsonconfig"
	"camlistore.org/pkg/misc"
)

var queueSyncInterval = 5 * time.Second

const maxErrors = 20

var _ = log.Printf

// TODO: rate control + tunable
// TODO: expose copierPoolSize as tunable
type SyncHandler struct {
	fromName, fromqName, toName string
	from, fromq, to             blobserver.Storage

	copierPoolSize int

	lk             sync.Mutex // protects following
	status         string
	blobStatus     map[string]fmt.Stringer // stringer called with lk held
	recentErrors   []timestampedError
	recentCopyTime time.Time
	totalCopies    int64
	totalCopyBytes int64
	totalErrors    int64
}

func init() {
	blobserver.RegisterHandlerConstructor("sync", newSyncFromConfig)
}

func newSyncFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err error) {
	from := conf.RequiredString("from")
	to := conf.RequiredString("to")
	fullSync := conf.OptionalBool("fullSyncOnStart", false)
	blockFullSync := conf.OptionalBool("blockingFullSyncOnStart", false)
	if err = conf.Validate(); err != nil {
		return
	}
	fromBs, err := ld.GetStorage(from)
	if err != nil {
		return
	}
	toBs, err := ld.GetStorage(to)
	if err != nil {
		return
	}
	fromQsc, ok := fromBs.(blobserver.StorageQueueCreator)
	if !ok {
		return nil, fmt.Errorf("Prefix %s (type %T) does not support being efficient replication source (queueing)", from, fromBs)
	}
	synch, err := createSyncHandler(from, to, fromQsc, toBs)
	if err != nil {
		return
	}

	if fullSync || blockFullSync {
		didFullSync := make(chan bool, 1)
		go func() {
			n := synch.runSync("queue", fromQsc, 0)
			log.Printf("Queue sync copied %d blobs", n)
			n = synch.runSync("full", fromBs, 0)
			log.Printf("Full sync copied %d blobs", n)
			didFullSync <- true
			synch.syncQueueLoop()
		}()
		if blockFullSync {
			log.Printf("Blocking startup, waiting for full sync from %q to %q", from, to)
			<-didFullSync
			log.Printf("Full sync complete.")
		}
	} else {
		go synch.syncQueueLoop()
	}
	return synch, nil
}

type timestampedError struct {
	t   time.Time
	err error
}

func createSyncHandler(fromName, toName string, from blobserver.StorageQueueCreator, to blobserver.Storage) (*SyncHandler, error) {
	h := &SyncHandler{
		copierPoolSize: 3,
		from:           from,
		to:             to,
		fromName:       fromName,
		toName:         toName,
		status:         "not started",
		blobStatus:     make(map[string]fmt.Stringer),
	}
	h.fromqName = strings.Replace(strings.Trim(toName, "/"), "/", "-", -1)
	var err error
	h.fromq, err = from.CreateQueue(h.fromqName)
	if err != nil {
		return nil, fmt.Errorf("Prefix %s (type %T) failed to create queue %q: %v",
			fromName, from, h.fromqName, err)
	}
	return h, nil
}

func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	sh.lk.Lock()
	defer sh.lk.Unlock()

	fmt.Fprintf(rw, "
%s to %s Sync Status

Current status: %s
",
		sh.fromName, sh.toName, html.EscapeString(sh.status))

	fmt.Fprintf(rw, "
Stats:

")
	fmt.Fprintf(rw, "
Blobs copied: %d
", sh.totalCopies)
	fmt.Fprintf(rw, "
Bytes copied: %d
", sh.totalCopyBytes)
	if !sh.recentCopyTime.IsZero() {
		fmt.Fprintf(rw, "
Most recent copy: %s
", sh.recentCopyTime.Format(time.RFC3339))
	}
	fmt.Fprintf(rw, "
Copy errors: %d
", sh.totalErrors)
	fmt.Fprintf(rw, "
")

	if len(sh.blobStatus) > 0 {
		fmt.Fprintf(rw, "
Current Copies:

")
		for blobstr, sfn := range sh.blobStatus {
			fmt.Fprintf(rw, "
%s: %s
\n",
				blobstr, html.EscapeString(sfn.String()))
		}
		fmt.Fprintf(rw, "
")
	}

	if len(sh.recentErrors) > 0 {
		fmt.Fprintf(rw, "
Recent Errors:

")
		for _, te := range sh.recentErrors {
			fmt.Fprintf(rw, "
%s: %s
\n",
				te.t.Format(time.RFC3339),
				html.EscapeString(te.err.Error()))
		}
		fmt.Fprintf(rw, "
")
	}
}

func (sh *SyncHandler) setStatus(s string, args ...interface{}) {
	s = time.Now().UTC().Format(time.RFC3339) + ": " + fmt.Sprintf(s, args...)
	sh.lk.Lock()
	defer sh.lk.Unlock()
	sh.status = s
}

func (sh *SyncHandler) setBlobStatus(blobref string, s fmt.Stringer) {
	sh.lk.Lock()
	defer sh.lk.Unlock()
	if s != nil {
		sh.blobStatus[blobref] = s
	} else {
		delete(sh.blobStatus, blobref)
	}
}

func (sh *SyncHandler) addErrorToLog(err error) {
	log.Printf(err.Error())
	sh.lk.Lock()
	defer sh.lk.Unlock()
	sh.recentErrors = append(sh.recentErrors, timestampedError{time.Now().UTC(), err})
	if len(sh.recentErrors) > maxErrors {
		// Kinda lame, but whatever. Only for errors, rare.
		copy(sh.recentErrors[:maxErrors], sh.recentErrors[1:maxErrors+1])
		sh.recentErrors = sh.recentErrors[:maxErrors]
	}
}

type copyResult struct {
	sb  blobref.SizedBlobRef
	err error
}

func (sh *SyncHandler) runSync(srcName string, enumSrc blobserver.Storage, longPollWait time.Duration) int {
	if longPollWait != 0 {
		sh.setStatus("Idle; waiting for new blobs")
	}
	enumch := make(chan blobref.SizedBlobRef)
	errch := make(chan error, 1)
	go func() {
		errch <- enumSrc.EnumerateBlobs(enumch, "", 1000, longPollWait)
	}()

	nCopied := 0
	toCopy := 0

	workch := make(chan blobref.SizedBlobRef, 1000)
	resch := make(chan copyResult, 8)
	for sb := range enumch {
		toCopy++
		workch <- sb
		if toCopy <= sh.copierPoolSize {
			go sh.copyWorker(resch, workch)
		}
		sh.setStatus("Enumerating queued blobs: %d", toCopy)
	}
	close(workch)
	for i := 0; i < toCopy; i++ {
		sh.setStatus("Copied %d/%d of batch of queued blobs", nCopied, toCopy)
		res := <-resch
		nCopied++
		sh.lk.Lock()
		if res.err == nil {
			sh.totalCopies++
			sh.totalCopyBytes += res.sb.Size
			sh.recentCopyTime = time.Now().UTC()
		} else {
			sh.totalErrors++
		}
		sh.lk.Unlock()
	}

	if err := <-errch; err != nil {
		sh.addErrorToLog(fmt.Errorf("replication error for source %q, enumerate from source: %v", srcName, err))
		return nCopied
	}
	return nCopied
}

func (sh *SyncHandler) syncQueueLoop() {
	every(queueSyncInterval, func() {
		for sh.runSync(sh.fromqName, sh.fromq, queueSyncInterval) > 0 {
			// Loop, before sleeping.
		}
		sh.setStatus("Sleeping briefly before next long poll.")
	})
}

func (sh *SyncHandler) copyWorker(res chan<- copyResult, work <-chan blobref.SizedBlobRef) {
	for sb := range work {
		res <- copyResult{sb, sh.copyBlob(sb)}
	}
}

type statusFunc func() string

func (sf statusFunc) String() string { return sf() }

type status string

func (s status) String() string { return string(s) }

func (sh *SyncHandler) copyBlob(sb blobref.SizedBlobRef) error {
	key := sb.BlobRef.String()
	set := func(s fmt.Stringer) {
		sh.setBlobStatus(key, s)
	}
	defer set(nil)

	errorf := func(s string, args ...interface{}) error {
		// TODO: increment error stats
		pargs := []interface{}{sh.fromqName, sb.BlobRef}
		pargs = append(pargs, args...)
		err := fmt.Errorf("replication error for queue %q, blob %s: "+s, pargs...)
		sh.addErrorToLog(err)
		return err
	}

	set(status("sending GET to source"))
	rc, fromSize, err := sh.from.FetchStreaming(sb.BlobRef)
	if err != nil {
		return errorf("source fetch: %v", err)
	}
	defer rc.Close()
	if fromSize != sb.Size {
		return errorf("source fetch size mismatch: get=%d, enumerate=%d", fromSize, sb.Size)
	}

	bytesCopied := int64(0) // TODO: data race, accessed without locking in statusFunc below.
	set(statusFunc(func() string {
		return fmt.Sprintf("copying: %d/%d bytes", bytesCopied, sb.Size)
	}))
	newsb, err := sh.to.ReceiveBlob(sb.BlobRef, misc.CountingReader{rc, &bytesCopied})
	if err != nil {
		return errorf("dest write: %v", err)
	}
	if newsb.Size != sb.Size {
		return errorf("write size mismatch: source_read=%d but dest_write=%d", sb.Size, newsb.Size)
	}
	set(status("copied; removing from queue"))
	err = sh.fromq.RemoveBlobs([]*blobref.BlobRef{sb.BlobRef})
	if err != nil {
		return errorf("source queue delete: %v", err)
	}
	return nil
}

func every(interval time.Duration, f func()) {
	for {
		t1 := time.Now()
		f()
		sleepUntil := t1.Add(interval)
		if sleep := sleepUntil.Sub(time.Now()); sleep > 0 {
			time.Sleep(sleep)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/chat1"
	"github.com/keybase/client/go/protocol/keybase1"
	"golang.org/x/net/context"
)

type CmdChatAPI struct {
	libkb.Contextified
	indent     bool
	inputFile  string
	outputFile string
	message    string
}

func newCmdChatAPI(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "api",
		Usage: "JSON api",
		Action: func(c *cli.Context) {
			cmd := &CmdChatAPI{Contextified: libkb.NewContextified(g)}
			cl.ChooseCommand(cmd, "api", c)
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "p, pretty",
				Usage: "Output pretty (indented) JSON.",
			},
			cli.StringFlag{
				Name:  "m",
				Usage: "Specify JSON as string instead of stdin",
			},
			cli.StringFlag{
				Name:  "i, infile",
				Usage: "Specify JSON input file (stdin default)",
			},
			cli.StringFlag{
				Name:  "o, outfile",
				Usage: "Specify output file (stdout default)",
			},
		},
	}
}

func (c *CmdChatAPI) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) != 0 {
		return errors.New("api takes no arguments")
	}
	c.indent = ctx.Bool("pretty")
	c.inputFile = ctx.String("infile")
	c.outputFile = ctx.String("outfile")
	c.message = ctx.String("message")

	if len(c.message) > 0 && len(c.inputFile) > 0 {
		return errors.New("specify -m or -i, but not both")
	}

	return nil
}

func (c *CmdChatAPI) Run() error {
	d := NewChatAPIDecoder(&ChatAPI{svcHandler: c, indent: c.indent})

	var r io.Reader
	r = os.Stdin
	if len(c.message) > 0 {
		r = strings.NewReader(c.message)
	} else if len(c.inputFile) > 0 {
		f, err := os.Open(c.inputFile)
		if err != nil {
			return err
		}
		defer f.Close()
		r = f
	}

	var w io.Writer
	w = os.Stdout
	if len(c.outputFile) > 0 {
		f, err := os.Create(c.outputFile)
		if err != nil {
			return err
		}
		defer f.Close()
		w = f
	}

	if err := d.Decode(context.Background(), r, w); err != nil {
		return err
	}

	return nil
}

func (c *CmdChatAPI) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:       true,
		KbKeyring: true,
		Config:    true,
	}
}

type ConvSummary struct {
	ID      chat1.ConversationID `json:"id"`
	Channel ChatChannel          `json:"channel"`
}

type ChatList struct {
	Conversations []ConvSummary `json:"conversations"`
}

// ListV1 implements ChatServiceHandler.ListV1.
func (c *CmdChatAPI) ListV1(ctx context.Context) Reply {
	client, err := GetChatLocalClient(c.G())
	if err != nil {
		return c.errReply(err)
	}

	inbox, err := client.GetInboxLocal(ctx, keybase1.GetInboxLocalArg{})
	if err != nil {
		return c.errReply(err)
	}

	var cl ChatList
	cl.Conversations = make([]ConvSummary, len(inbox.Conversations))
	for i, conv := range inbox.Conversations {
		if len(conv.MaxMsgs) == 0 {
			return c.errReply(fmt.Errorf("conversation %d had no max msgs", conv.Metadata.ConversationID))
		}
		tlf := conv.MaxMsgs[0].ClientHeader.TlfName
		cl.Conversations[i] = ConvSummary{
			ID: conv.Metadata.ConversationID,
			Channel: ChatChannel{
				Name:      tlf,
				TopicType: strings.ToLower(conv.Metadata.IdTriple.TopicType.String()),
			},
		}
	}

	return Reply{Result: cl}
}

type MsgSender struct {
	UID        string `json:"uid"`
	Username   string `json:"username,omitempty"`
	DeviceID   string `json:"device_id"`
	DeviceName string `json:"device_name,omitempty"`
}

// unfortunately need this...
type MsgContent struct {
	TypeName   string                                `json:"type"`
	Text       *keybase1.MessageText                 `json:"text,omitempty"`
	Attachment *keybase1.MessageAttachment           `json:"attachment,omitempty"`
	Edit       *keybase1.MessageEdit                 `json:"edit,omitempty"`
	Delete     *keybase1.MessageDelete               `json:"delete,omitempty"`
	Metadata   *keybase1.MessageConversationMetadata `json:"metadata,omitempty"`
}

type MsgSummary struct {
	ID       chat1.MessageID `json:"id"`
	Channel  ChatChannel     `json:"channel"`
	Sender   MsgSender       `json:"sender"`
	SentAt   int64           `json:"sent_at"`
	SentAtMs int64           `json:"sent_at_ms"`
	Content  MsgContent      `json:"content"`
}

type Thread struct {
	Messages []MsgSummary `json:"messages"`
}

// ReadV1 implements ChatServiceHandler.ReadV1.
func (c *CmdChatAPI) ReadV1(ctx context.Context, opts readOptionsV1) Reply {
	client, err := GetChatLocalClient(c.G())
	if err != nil {
		return c.errReply(err)
	}

	if opts.ConversationID == 0 {
		// resolve conversation id
		cinfo := keybase1.ConversationInfoLocal{
			TlfName:   opts.Channel.Name,
			TopicType: opts.Channel.TopicTypeEnum(),
			TopicName: opts.Channel.TopicName,
		}
		existing, err := client.ResolveConversationLocal(ctx, cinfo)
		if err != nil {
			return c.errReply(err)
		}
		if len(existing) > 1 {
			return c.errReply(fmt.Errorf("multiple conversations matched %q", opts.Channel.Name))
		}
		if len(existing) == 0 {
			return c.errReply(fmt.Errorf("no conversations matched %q", opts.Channel.Name))
		}
		opts.ConversationID = existing[0].Id
	}

	arg := keybase1.GetThreadLocalArg{
		ConversationID: opts.ConversationID,
		Query: &chat1.GetThreadQuery{
			MarkAsRead: true,
		},
	}
	threadView, err := client.GetThreadLocal(ctx, arg)
	if err != nil {
		return c.errReply(err)
	}

	var thread Thread
	thread.Messages = make([]MsgSummary, len(threadView.Messages))
	for i, m := range threadView.Messages {
		thread.Messages[i] = MsgSummary{
			ID: m.ServerHeader.MessageID,
			Channel: ChatChannel{
				Name:      m.MessagePlaintext.ClientHeader.TlfName,
				TopicType: strings.ToLower(m.MessagePlaintext.ClientHeader.Conv.TopicType.String()),
			},
			Sender: MsgSender{
				UID:      m.MessagePlaintext.ClientHeader.Sender.String(),
				DeviceID: m.MessagePlaintext.ClientHeader.SenderDevice.String(),
			},
			SentAt:   int64(m.ServerHeader.Ctime / 1000),
			SentAtMs: int64(m.ServerHeader.Ctime),
		}
		if len(m.MessagePlaintext.MessageBodies) > 0 {
			thread.Messages[i].Content = c.convertMsgBody(m.MessagePlaintext.MessageBodies[0])
		}
		if len(m.MessagePlaintext.MessageBodies) > 1 {
			c.G().Log.Warning("message %v had multiple bodies", m.ServerHeader.MessageID)
		}
	}

	return Reply{Result: thread}
}

// SendV1 implements ChatServiceHandler.SendV1.
func (c *CmdChatAPI) SendV1(ctx context.Context, opts sendOptionsV1) Reply {
	client, err := GetChatLocalClient(c.G())
	if err != nil {
		return c.errReply(err)
	}

	var cinfo keybase1.ConversationInfoLocal
	if opts.ConversationID > 0 {
		cinfo = keybase1.ConversationInfoLocal{
			Id: opts.ConversationID,
		}
	} else {
		cinfo = keybase1.ConversationInfoLocal{
			TlfName:   opts.Channel.Name,
			TopicType: opts.Channel.TopicTypeEnum(),
			TopicName: opts.Channel.TopicName,
		}
	}

	// find the conversation
	existing, err := client.ResolveConversationLocal(ctx, cinfo)
	if err != nil {
		return c.errReply(err)
	}
	var conversation keybase1.ConversationInfoLocal
	switch len(existing) {
	case 0:
		conversation, err = client.NewConversationLocal(ctx, cinfo)
		if err != nil {
			return c.errReply(err)
		}
	case 1:
		conversation = existing[0]
	default:
		return c.errReply(fmt.Errorf("multiple conversations matched"))
	}

	// XXX ResolveConversationLocal, NewConversationLocal need to return
	// topic id and tlf id.  In order to fill in conversation id triple
	// below.
	// ticket CORE-3746

	postArg := keybase1.PostLocalArg{
		ConversationID: conversation.Id,
		MessagePlaintext: keybase1.MessagePlaintext{
			ClientHeader: chat1.MessageClientHeader{
				Conv: chat1.ConversationIDTriple{
					TopicType: conversation.TopicType,
				},
				TlfName:     conversation.TlfName,
				MessageType: chat1.MessageType_TEXT,
			},
			MessageBodies: []keybase1.MessageBody{
				keybase1.NewMessageBodyWithText(keybase1.MessageText{Body: opts.Message.Body}),
			},
		},
	}
	if err := client.PostLocal(ctx, postArg); err != nil {
		return c.errReply(err)
	}

	return Reply{Result: "message sent"}
}

func (c *CmdChatAPI) errReply(err error) Reply {
	return Reply{Error: &CallError{Message: err.Error()}}
}

// need this to get message type name
func (c *CmdChatAPI) convertMsgBody(mb keybase1.MessageBody) MsgContent {
	return MsgContent{
		TypeName:   strings.ToLower(chat1.MessageTypeRevMap[mb.MessageType__]),
		Text:       mb.Text__,
		Attachment: mb.Attachment__,
		Edit:       mb.Edit__,
		Delete:     mb.Delete__,
		Metadata:   mb.Metadata__,
	}
}
<commit_msg>Fix -m flag parse<commit_after>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/chat1"
	"github.com/keybase/client/go/protocol/keybase1"
	"golang.org/x/net/context"
)

type CmdChatAPI struct {
	libkb.Contextified
	indent     bool
	inputFile  string
	outputFile string
	message    string
}

func newCmdChatAPI(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "api",
		Usage: "JSON api",
		Action: func(c *cli.Context) {
			cmd := &CmdChatAPI{Contextified: libkb.NewContextified(g)}
			cl.ChooseCommand(cmd, "api", c)
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "p, pretty",
				Usage: "Output pretty (indented) JSON.",
			},
			cli.StringFlag{
				Name:  "m",
				Usage: "Specify JSON as string instead of stdin",
			},
			cli.StringFlag{
				Name:  "i, infile",
				Usage: "Specify JSON input file (stdin default)",
			},
			cli.StringFlag{
				Name:  "o, outfile",
				Usage: "Specify output file (stdout default)",
			},
		},
	}
}

func (c *CmdChatAPI) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) != 0 {
		return errors.New("api takes no arguments")
	}
	c.indent = ctx.Bool("pretty")
	c.inputFile = ctx.String("infile")
	c.outputFile = ctx.String("outfile")
	c.message = ctx.String("m")

	if len(c.message) > 0 && len(c.inputFile) > 0 {
		return errors.New("specify -m or -i, but not both")
	}

	return nil
}

func (c *CmdChatAPI) Run() error {
	d := NewChatAPIDecoder(&ChatAPI{svcHandler: c, indent: c.indent})

	var r io.Reader
	r = os.Stdin
	if len(c.message) > 0 {
		r = strings.NewReader(c.message + "\n")
	} else if len(c.inputFile) > 0 {
		f, err := os.Open(c.inputFile)
		if err != nil {
			return err
		}
		defer f.Close()
		r = f
	}

	var w io.Writer
	w = os.Stdout
	if len(c.outputFile) > 0 {
		f, err := os.Create(c.outputFile)
		if err != nil {
			return err
		}
		defer f.Close()
		w = f
	}

	if err := d.Decode(context.Background(), r, w); err != nil {
		return err
	}

	return nil
}

func (c *CmdChatAPI) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:       true,
		KbKeyring: true,
		Config:    true,
	}
}

type ConvSummary struct {
	ID      chat1.ConversationID `json:"id"`
	Channel ChatChannel          `json:"channel"`
}

type ChatList struct {
	Conversations []ConvSummary `json:"conversations"`
}

// ListV1 implements ChatServiceHandler.ListV1.
func (c *CmdChatAPI) ListV1(ctx context.Context) Reply {
	client, err := GetChatLocalClient(c.G())
	if err != nil {
		return c.errReply(err)
	}

	inbox, err := client.GetInboxLocal(ctx, keybase1.GetInboxLocalArg{})
	if err != nil {
		return c.errReply(err)
	}

	var cl ChatList
	cl.Conversations = make([]ConvSummary, len(inbox.Conversations))
	for i, conv := range inbox.Conversations {
		if len(conv.MaxMsgs) == 0 {
			return c.errReply(fmt.Errorf("conversation %d had no max msgs", conv.Metadata.ConversationID))
		}
		tlf := conv.MaxMsgs[0].ClientHeader.TlfName
		cl.Conversations[i] = ConvSummary{
			ID: conv.Metadata.ConversationID,
			Channel: ChatChannel{
				Name:      tlf,
				TopicType: strings.ToLower(conv.Metadata.IdTriple.TopicType.String()),
			},
		}
	}

	return Reply{Result: cl}
}

type MsgSender struct {
	UID        string `json:"uid"`
	Username   string `json:"username,omitempty"`
	DeviceID   string `json:"device_id"`
	DeviceName string `json:"device_name,omitempty"`
}

// unfortunately need this...
type MsgContent struct {
	TypeName   string                                `json:"type"`
	Text       *keybase1.MessageText                 `json:"text,omitempty"`
	Attachment *keybase1.MessageAttachment           `json:"attachment,omitempty"`
	Edit       *keybase1.MessageEdit                 `json:"edit,omitempty"`
	Delete     *keybase1.MessageDelete               `json:"delete,omitempty"`
	Metadata   *keybase1.MessageConversationMetadata `json:"metadata,omitempty"`
}

type MsgSummary struct {
	ID       chat1.MessageID `json:"id"`
	Channel  ChatChannel     `json:"channel"`
	Sender   MsgSender       `json:"sender"`
	SentAt   int64           `json:"sent_at"`
	SentAtMs int64           `json:"sent_at_ms"`
	Content  MsgContent      `json:"content"`
}

type Thread struct {
	Messages []MsgSummary `json:"messages"`
}

// ReadV1 implements ChatServiceHandler.ReadV1.
func (c *CmdChatAPI) ReadV1(ctx context.Context, opts readOptionsV1) Reply {
	client, err := GetChatLocalClient(c.G())
	if err != nil {
		return c.errReply(err)
	}

	if opts.ConversationID == 0 {
		// resolve conversation id
		cinfo := keybase1.ConversationInfoLocal{
			TlfName:   opts.Channel.Name,
			TopicType: opts.Channel.TopicTypeEnum(),
			TopicName: opts.Channel.TopicName,
		}
		existing, err := client.ResolveConversationLocal(ctx, cinfo)
		if err != nil {
			return c.errReply(err)
		}
		if len(existing) > 1 {
			return c.errReply(fmt.Errorf("multiple conversations matched %q", opts.Channel.Name))
		}
		if len(existing) == 0 {
			return c.errReply(fmt.Errorf("no conversations matched %q", opts.Channel.Name))
		}
		opts.ConversationID = existing[0].Id
	}

	arg := keybase1.GetThreadLocalArg{
		ConversationID: opts.ConversationID,
		Query: &chat1.GetThreadQuery{
			MarkAsRead: true,
		},
	}
	threadView, err := client.GetThreadLocal(ctx, arg)
	if err != nil {
		return c.errReply(err)
	}

	var thread Thread
	thread.Messages = make([]MsgSummary, len(threadView.Messages))
	for i, m := range threadView.Messages {
		thread.Messages[i] = MsgSummary{
			ID: m.ServerHeader.MessageID,
			Channel: ChatChannel{
				Name:      m.MessagePlaintext.ClientHeader.TlfName,
				TopicType: strings.ToLower(m.MessagePlaintext.ClientHeader.Conv.TopicType.String()),
			},
			Sender: MsgSender{
				UID:      m.MessagePlaintext.ClientHeader.Sender.String(),
				DeviceID: m.MessagePlaintext.ClientHeader.SenderDevice.String(),
			},
			SentAt:   int64(m.ServerHeader.Ctime / 1000),
			SentAtMs: int64(m.ServerHeader.Ctime),
		}
		if len(m.MessagePlaintext.MessageBodies) > 0 {
			thread.Messages[i].Content = c.convertMsgBody(m.MessagePlaintext.MessageBodies[0])
		}
		if len(m.MessagePlaintext.MessageBodies) > 1 {
			c.G().Log.Warning("message %v had multiple bodies", m.ServerHeader.MessageID)
		}
	}

	return Reply{Result: thread}
}

// SendV1 implements ChatServiceHandler.SendV1.
func (c *CmdChatAPI) SendV1(ctx context.Context, opts sendOptionsV1) Reply {
	client, err := GetChatLocalClient(c.G())
	if err != nil {
		return c.errReply(err)
	}

	var cinfo keybase1.ConversationInfoLocal
	if opts.ConversationID > 0 {
		cinfo = keybase1.ConversationInfoLocal{
			Id: opts.ConversationID,
		}
	} else {
		cinfo = keybase1.ConversationInfoLocal{
			TlfName:   opts.Channel.Name,
			TopicType: opts.Channel.TopicTypeEnum(),
			TopicName: opts.Channel.TopicName,
		}
	}

	// find the conversation
	existing, err := client.ResolveConversationLocal(ctx, cinfo)
	if err != nil {
		return c.errReply(err)
	}
	var conversation keybase1.ConversationInfoLocal
	switch len(existing) {
	case 0:
		conversation, err = client.NewConversationLocal(ctx, cinfo)
		if err != nil {
			return c.errReply(err)
		}
	case 1:
		conversation = existing[0]
	default:
		return c.errReply(fmt.Errorf("multiple conversations matched"))
	}

	// XXX ResolveConversationLocal, NewConversationLocal need to return
	// topic id and tlf id.  In order to fill in conversation id triple
	// below.
	// ticket CORE-3746

	postArg := keybase1.PostLocalArg{
		ConversationID: conversation.Id,
		MessagePlaintext: keybase1.MessagePlaintext{
			ClientHeader: chat1.MessageClientHeader{
				Conv: chat1.ConversationIDTriple{
					TopicType: conversation.TopicType,
				},
				TlfName:     conversation.TlfName,
				MessageType: chat1.MessageType_TEXT,
			},
			MessageBodies: []keybase1.MessageBody{
				keybase1.NewMessageBodyWithText(keybase1.MessageText{Body: opts.Message.Body}),
			},
		},
	}
	if err := client.PostLocal(ctx, postArg); err != nil {
		return c.errReply(err)
	}

	return Reply{Result: "message sent"}
}

func (c *CmdChatAPI) errReply(err error) Reply {
	return Reply{Error: &CallError{Message: err.Error()}}
}

// need this to get message type name
func (c *CmdChatAPI) convertMsgBody(mb keybase1.MessageBody) MsgContent {
	return MsgContent{
		TypeName:   strings.ToLower(chat1.MessageTypeRevMap[mb.MessageType__]),
		Text:       mb.Text__,
		Attachment: mb.Attachment__,
		Edit:       mb.Edit__,
		Delete:     mb.Delete__,
		Metadata:   mb.Metadata__,
	}
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestConfig(t *testing.T) {
	configFile := "./doesnotexist"
	_, err := NewConfig(configFile)
	assert.NotNil(t, err, "config must return an error, when file does not exist")

	configFile = "../../example.goophry.config.json"
	conf, err := NewConfig(configFile)
	assert.Nil(t, err, "config should not return an error, when file exists and has valid json")

	assert.NotEqual(t, "", conf.RedisNetwork, "RedisNetwork should have a default value")
}
<commit_msg>small improvement to testing for config value RedisNetwork<commit_after>package config

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestConfig(t *testing.T) {
	configFile := "./doesnotexist"
	_, err := NewConfig(configFile)
	assert.NotNil(t, err, "config must return an error, when file does not exist")

	configFile = "../../example.goophry.config.json"
	conf, err := NewConfig(configFile)
	assert.Nil(t, err, "config should not return an error, when file exists and has valid json")

	assert.Contains(t, []string{"tcp", "udp"}, conf.RedisNetwork, "RedisNetwork should always have a (valid) value")
}
<|endoftext|>