<commit_before>package server

import (
	"bytes"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/golang/snappy"
	"golang.org/x/net/context"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/admin"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/pbutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
)

type apiServer struct {
	log.Logger
	address        string
	pachClient     *client.APIClient
	pachClientOnce sync.Once
	clusterInfo    *admin.ClusterInfo
}

func (a *apiServer) InspectCluster(ctx context.Context, request *types.Empty) (*admin.ClusterInfo, error) {
	return a.clusterInfo, nil
}

func (a *apiServer) Extract(request *admin.ExtractRequest, extractServer admin.API_ExtractServer) (retErr error) {
	func() { a.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { a.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	pachClient := a.getPachClient()
	pachClient = pachClient.WithCtx(extractServer.Context())
	handleOp := extractServer.Send
	if request.URL != "" {
		url, err := obj.ParseURL(request.URL)
		if err != nil {
			return fmt.Errorf("error parsing url %v: %v", request.URL, err)
		}
		objClient, err := obj.NewClientFromURLAndSecret(extractServer.Context(), url)
		if err != nil {
			return err
		}
		objW, err := objClient.Writer(url.Object)
		if err != nil {
			return err
		}
		snappyW := snappy.NewBufferedWriter(objW)
		defer func() {
			if err := snappyW.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		w := pbutil.NewWriter(snappyW)
		handleOp = func(op *admin.Op) error { return w.Write(op) }
	}
	if !request.NoObjects {
		w := extractObjectWriter(handleOp)
		if err := pachClient.ListObject(func(object *pfs.Object) error {
			fmt.Printf("Extract object: %s\n", object.Hash)
			if err := pachClient.GetObject(object.Hash, w); err != nil {
				return err
			}
			// empty PutObjectRequest to indicate EOF
			return handleOp(&admin.Op{Op1_7: &admin.Op1_7{Object: &pfs.PutObjectRequest{}}})
		}); err != nil {
			return err
		}
		if err := pachClient.ListTag(func(resp *pfs.ListTagsResponse) error {
			return handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Tag: &pfs.TagObjectRequest{
					Object: resp.Object,
					Tags:   []*pfs.Tag{resp.Tag},
				},
			}})
		}); err != nil {
			return err
		}
	}
	var repos []*pfs.Repo
	if !request.NoRepos {
		ris, err := pachClient.ListRepo(nil)
		if err != nil {
			return err
		}
	repos:
		for _, ri := range ris {
			bis, err := pachClient.ListBranch(ri.Repo.Name)
			if err != nil {
				return err
			}
			for _, bi := range bis {
				if len(bi.Provenance) > 0 {
					continue repos
				}
			}
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Repo: &pfs.CreateRepoRequest{
					Repo:        ri.Repo,
					Provenance:  ri.Provenance,
					Description: ri.Description,
				}},
			}); err != nil {
				return err
			}
			repos = append(repos, ri.Repo)
		}
	}
	if !request.NoPipelines {
		pis, err := pachClient.ListPipeline()
		if err != nil {
			return err
		}
		for _, pi := range pis {
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Pipeline: &pps.CreatePipelineRequest{
					Pipeline:           pi.Pipeline,
					Transform:          pi.Transform,
					ParallelismSpec:    pi.ParallelismSpec,
					Egress:             pi.Egress,
					OutputBranch:       pi.OutputBranch,
					ScaleDownThreshold: pi.ScaleDownThreshold,
					ResourceRequests:   pi.ResourceRequests,
					ResourceLimits:     pi.ResourceLimits,
					Input:              pi.Input,
					Description:        pi.Description,
					Incremental:        pi.Incremental,
					CacheSize:          pi.CacheSize,
					EnableStats:        pi.EnableStats,
					Batch:              pi.Batch,
					MaxQueueSize:       pi.MaxQueueSize,
					Service:            pi.Service,
					ChunkSpec:          pi.ChunkSpec,
					DatumTimeout:       pi.DatumTimeout,
					JobTimeout:         pi.JobTimeout,
					Salt:               pi.Salt,
				}}}); err != nil {
				return err
			}
		}
	}
	// We send the actual commits last, that way pipelines will have already
	// been created and will recreate output commits for historical outputs.
	for _, repo := range repos {
		cis, err := pachClient.ListCommit(repo.Name, "", "", 0)
		if err != nil {
			return err
		}
		bis, err := pachClient.ListBranch(repo.Name)
		if err != nil {
			return err
		}
		for _, bcr := range buildCommitRequests(cis, bis) {
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{Commit: bcr}}); err != nil {
				return err
			}
		}
		for _, bi := range bis {
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Branch: &pfs.CreateBranchRequest{
					Head:   bi.Head,
					Branch: client.NewBranch(bi.Head.Repo.Name, bi.Name),
				},
			}}); err != nil {
				return err
			}
		}
	}
	return nil
}

func buildCommitRequests(cis []*pfs.CommitInfo, bis []*pfs.BranchInfo) []*pfs.BuildCommitRequest {
	cis = sortCommitInfos(cis)
	result := make([]*pfs.BuildCommitRequest, len(cis))
	commitToBranch := make(map[string]string)
	for _, bi := range bis {
		if _, ok := commitToBranch[bi.Head.ID]; !ok || bi.Name == "master" {
			commitToBranch[bi.Head.ID] = bi.Name
		}
	}
	for i := range cis {
		ci := cis[len(cis)-i-1]
		branch := commitToBranch[ci.Commit.ID]
		// Even without a parent, ParentCommit is used to indicate which
		// repo to make the commit in.
		if ci.ParentCommit == nil {
			ci.ParentCommit = client.NewCommit(ci.Commit.Repo.Name, "")
		}
		result[len(cis)-i-1] = &pfs.BuildCommitRequest{
			Parent: ci.ParentCommit,
			Tree:   ci.Tree,
			ID:     ci.Commit.ID,
			Branch: branch,
		}
		if _, ok := commitToBranch[ci.ParentCommit.ID]; !ok || branch == "master" {
			commitToBranch[ci.ParentCommit.ID] = branch
		}
	}
	return result
}

func sortCommitInfos(cis []*pfs.CommitInfo) []*pfs.CommitInfo {
	commitMap := make(map[string]*pfs.CommitInfo)
	for _, ci := range cis {
		commitMap[ci.Commit.ID] = ci
	}
	var result []*pfs.CommitInfo
	for _, ci := range cis {
		if commitMap[ci.Commit.ID] == nil {
			continue
		}
		var localResult []*pfs.CommitInfo
		for ci != nil {
			localResult = append(localResult, ci)
			delete(commitMap, ci.Commit.ID)
			if ci.ParentCommit != nil {
				ci = commitMap[ci.ParentCommit.ID]
			} else {
				ci = nil
			}
		}
		for i := range localResult {
			result = append(result, localResult[len(localResult)-i-1])
		}
	}
	return result
}

func (a *apiServer) Restore(restoreServer admin.API_RestoreServer) (retErr error) {
	func() { a.Log(nil, nil, nil, 0) }()
	defer func(start time.Time) { a.Log(nil, nil, retErr, time.Since(start)) }(time.Now())
	ctx := restoreServer.Context()
	pachClient := a.getPachClient()
	defer func() {
		for {
			_, err := restoreServer.Recv()
			if err != nil {
				break
			}
		}
		if err := restoreServer.SendAndClose(&types.Empty{}); err != nil && retErr == nil {
			retErr = err
		}
	}()
	var r pbutil.Reader
	for {
		var op *admin.Op
		if r == nil {
			req, err := restoreServer.Recv()
			if err != nil {
				if err == io.EOF {
					return nil
				}
				return err
			}
			if req.URL != "" {
				url, err := obj.ParseURL(req.URL)
				if err != nil {
					return fmt.Errorf("error parsing url %v: %v", req.URL, err)
				}
				objClient, err := obj.NewClientFromURLAndSecret(restoreServer.Context(), url)
				if err != nil {
					return err
				}
				objR, err := objClient.Reader(url.Object, 0, 0)
				if err != nil {
					return err
				}
				snappyR := snappy.NewReader(objR)
				r = pbutil.NewReader(snappyR)
				continue
			} else {
				op = req.Op
			}
		} else {
			op = &admin.Op{}
			if err := r.Read(op); err != nil {
				if err == io.EOF {
					return nil
				}
				return err
			}
		}
		switch {
		case op.Op1_7 != nil && op.Op1_7.Object != nil:
			r := &extractObjectReader{adminAPIRestoreServer: restoreServer}
			r.buf.Write(op.Op1_7.Object.Value)
			if _, _, err := pachClient.PutObject(r); err != nil {
				return fmt.Errorf("error putting object: %v", err)
			}
		case op.Op1_7 != nil && op.Op1_7.Tag != nil:
			if _, err := pachClient.ObjectAPIClient.TagObject(ctx, op.Op1_7.Tag); err != nil {
				return fmt.Errorf("error tagging object: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Repo != nil:
			if _, err := pachClient.PfsAPIClient.CreateRepo(ctx, op.Op1_7.Repo); err != nil {
				return fmt.Errorf("error creating repo: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Commit != nil:
			if _, err := pachClient.PfsAPIClient.BuildCommit(ctx, op.Op1_7.Commit); err != nil {
				return fmt.Errorf("error creating commit: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Branch != nil:
			if op.Op1_7.Branch.Branch == nil {
				op.Op1_7.Branch.Branch = client.NewBranch(op.Op1_7.Branch.Head.Repo.Name, op.Op1_7.Branch.SBranch)
			}
			if _, err := pachClient.PfsAPIClient.CreateBranch(ctx, op.Op1_7.Branch); err != nil {
				return fmt.Errorf("error creating branch: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Pipeline != nil:
			if _, err := pachClient.PpsAPIClient.CreatePipeline(ctx, op.Op1_7.Pipeline); err != nil {
				return fmt.Errorf("error creating pipeline: %v", grpcutil.ScrubGRPC(err))
			}
		}
	}
}

func (a *apiServer) getPachClient() *client.APIClient {
	a.pachClientOnce.Do(func() {
		var err error
		a.pachClient, err = client.NewFromAddress(a.address)
		if err != nil {
			panic(fmt.Sprintf("pps failed to initialize pach client: %v", err))
		}
	})
	return a.pachClient
}

type extractObjectWriter func(*admin.Op) error

func (w extractObjectWriter) Write(p []byte) (int, error) {
	chunkSize := grpcutil.MaxMsgSize / 2
	var n int
	for i := 0; i*(chunkSize) < len(p); i++ {
		value := p[i*chunkSize:]
		if len(value) > chunkSize {
			value = value[:chunkSize]
		}
		if err := w(&admin.Op{Op1_7: &admin.Op1_7{Object: &pfs.PutObjectRequest{Value: value}}}); err != nil {
			return n, err
		}
		n += len(value)
	}
	return n, nil
}

type adminAPIRestoreServer admin.API_RestoreServer

type extractObjectReader struct {
	adminAPIRestoreServer
	buf bytes.Buffer
	eof bool
}

func (r *extractObjectReader) Read(p []byte) (int, error) {
	for len(p) > r.buf.Len() && !r.eof {
		request, err := r.Recv()
		if err != nil {
			return 0, grpcutil.ScrubGRPC(err)
		}
		op := request.Op
		if op.Op1_7.Object == nil {
			return 0, fmt.Errorf("expected an object, but got: %v", op)
		}
		r.buf.Write(op.Op1_7.Object.Value)
		if len(op.Op1_7.Object.Value) == 0 {
			r.eof = true
		}
	}
	return r.buf.Read(p)
}
<commit_msg>Make restore ignore already exists errors.<commit_after>package server

import (
	"bytes"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/golang/snappy"
	"golang.org/x/net/context"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/admin"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/pbutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/errutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
)

type apiServer struct {
	log.Logger
	address        string
	pachClient     *client.APIClient
	pachClientOnce sync.Once
	clusterInfo    *admin.ClusterInfo
}

func (a *apiServer) InspectCluster(ctx context.Context, request *types.Empty) (*admin.ClusterInfo, error) {
	return a.clusterInfo, nil
}

func (a *apiServer) Extract(request *admin.ExtractRequest, extractServer admin.API_ExtractServer) (retErr error) {
	func() { a.Log(request, nil, nil, 0) }()
	defer func(start time.Time) { a.Log(request, nil, retErr, time.Since(start)) }(time.Now())
	pachClient := a.getPachClient()
	pachClient = pachClient.WithCtx(extractServer.Context())
	handleOp := extractServer.Send
	if request.URL != "" {
		url, err := obj.ParseURL(request.URL)
		if err != nil {
			return fmt.Errorf("error parsing url %v: %v", request.URL, err)
		}
		objClient, err := obj.NewClientFromURLAndSecret(extractServer.Context(), url)
		if err != nil {
			return err
		}
		objW, err := objClient.Writer(url.Object)
		if err != nil {
			return err
		}
		snappyW := snappy.NewBufferedWriter(objW)
		defer func() {
			if err := snappyW.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		w := pbutil.NewWriter(snappyW)
		handleOp = func(op *admin.Op) error { return w.Write(op) }
	}
	if !request.NoObjects {
		w := extractObjectWriter(handleOp)
		if err := pachClient.ListObject(func(object *pfs.Object) error {
			if err := pachClient.GetObject(object.Hash, w); err != nil {
				return err
			}
			// empty PutObjectRequest to indicate EOF
			return handleOp(&admin.Op{Op1_7: &admin.Op1_7{Object: &pfs.PutObjectRequest{}}})
		}); err != nil {
			return err
		}
		if err := pachClient.ListTag(func(resp *pfs.ListTagsResponse) error {
			return handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Tag: &pfs.TagObjectRequest{
					Object: resp.Object,
					Tags:   []*pfs.Tag{resp.Tag},
				},
			}})
		}); err != nil {
			return err
		}
	}
	var repos []*pfs.Repo
	if !request.NoRepos {
		ris, err := pachClient.ListRepo(nil)
		if err != nil {
			return err
		}
	repos:
		for _, ri := range ris {
			bis, err := pachClient.ListBranch(ri.Repo.Name)
			if err != nil {
				return err
			}
			for _, bi := range bis {
				if len(bi.Provenance) > 0 {
					continue repos
				}
			}
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Repo: &pfs.CreateRepoRequest{
					Repo:        ri.Repo,
					Provenance:  ri.Provenance,
					Description: ri.Description,
				}},
			}); err != nil {
				return err
			}
			repos = append(repos, ri.Repo)
		}
	}
	if !request.NoPipelines {
		pis, err := pachClient.ListPipeline()
		if err != nil {
			return err
		}
		for _, pi := range pis {
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Pipeline: &pps.CreatePipelineRequest{
					Pipeline:           pi.Pipeline,
					Transform:          pi.Transform,
					ParallelismSpec:    pi.ParallelismSpec,
					Egress:             pi.Egress,
					OutputBranch:       pi.OutputBranch,
					ScaleDownThreshold: pi.ScaleDownThreshold,
					ResourceRequests:   pi.ResourceRequests,
					ResourceLimits:     pi.ResourceLimits,
					Input:              pi.Input,
					Description:        pi.Description,
					Incremental:        pi.Incremental,
					CacheSize:          pi.CacheSize,
					EnableStats:        pi.EnableStats,
					Batch:              pi.Batch,
					MaxQueueSize:       pi.MaxQueueSize,
					Service:            pi.Service,
					ChunkSpec:          pi.ChunkSpec,
					DatumTimeout:       pi.DatumTimeout,
					JobTimeout:         pi.JobTimeout,
					Salt:               pi.Salt,
				}}}); err != nil {
				return err
			}
		}
	}
	// We send the actual commits last, that way pipelines will have already
	// been created and will recreate output commits for historical outputs.
	for _, repo := range repos {
		cis, err := pachClient.ListCommit(repo.Name, "", "", 0)
		if err != nil {
			return err
		}
		bis, err := pachClient.ListBranch(repo.Name)
		if err != nil {
			return err
		}
		for _, bcr := range buildCommitRequests(cis, bis) {
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{Commit: bcr}}); err != nil {
				return err
			}
		}
		for _, bi := range bis {
			if err := handleOp(&admin.Op{Op1_7: &admin.Op1_7{
				Branch: &pfs.CreateBranchRequest{
					Head:   bi.Head,
					Branch: client.NewBranch(bi.Head.Repo.Name, bi.Name),
				},
			}}); err != nil {
				return err
			}
		}
	}
	return nil
}

func buildCommitRequests(cis []*pfs.CommitInfo, bis []*pfs.BranchInfo) []*pfs.BuildCommitRequest {
	cis = sortCommitInfos(cis)
	result := make([]*pfs.BuildCommitRequest, len(cis))
	commitToBranch := make(map[string]string)
	for _, bi := range bis {
		if _, ok := commitToBranch[bi.Head.ID]; !ok || bi.Name == "master" {
			commitToBranch[bi.Head.ID] = bi.Name
		}
	}
	for i := range cis {
		ci := cis[len(cis)-i-1]
		branch := commitToBranch[ci.Commit.ID]
		// Even without a parent, ParentCommit is used to indicate which
		// repo to make the commit in.
		if ci.ParentCommit == nil {
			ci.ParentCommit = client.NewCommit(ci.Commit.Repo.Name, "")
		}
		result[len(cis)-i-1] = &pfs.BuildCommitRequest{
			Parent: ci.ParentCommit,
			Tree:   ci.Tree,
			ID:     ci.Commit.ID,
			Branch: branch,
		}
		if _, ok := commitToBranch[ci.ParentCommit.ID]; !ok || branch == "master" {
			commitToBranch[ci.ParentCommit.ID] = branch
		}
	}
	return result
}

func sortCommitInfos(cis []*pfs.CommitInfo) []*pfs.CommitInfo {
	commitMap := make(map[string]*pfs.CommitInfo)
	for _, ci := range cis {
		commitMap[ci.Commit.ID] = ci
	}
	var result []*pfs.CommitInfo
	for _, ci := range cis {
		if commitMap[ci.Commit.ID] == nil {
			continue
		}
		var localResult []*pfs.CommitInfo
		for ci != nil {
			localResult = append(localResult, ci)
			delete(commitMap, ci.Commit.ID)
			if ci.ParentCommit != nil {
				ci = commitMap[ci.ParentCommit.ID]
			} else {
				ci = nil
			}
		}
		for i := range localResult {
			result = append(result, localResult[len(localResult)-i-1])
		}
	}
	return result
}

func (a *apiServer) Restore(restoreServer admin.API_RestoreServer) (retErr error) {
	func() { a.Log(nil, nil, nil, 0) }()
	defer func(start time.Time) { a.Log(nil, nil, retErr, time.Since(start)) }(time.Now())
	ctx := restoreServer.Context()
	pachClient := a.getPachClient()
	defer func() {
		for {
			_, err := restoreServer.Recv()
			if err != nil {
				break
			}
		}
		if err := restoreServer.SendAndClose(&types.Empty{}); err != nil && retErr == nil {
			retErr = err
		}
	}()
	var r pbutil.Reader
	for {
		var op *admin.Op
		if r == nil {
			req, err := restoreServer.Recv()
			if err != nil {
				if err == io.EOF {
					return nil
				}
				return err
			}
			if req.URL != "" {
				url, err := obj.ParseURL(req.URL)
				if err != nil {
					return fmt.Errorf("error parsing url %v: %v", req.URL, err)
				}
				objClient, err := obj.NewClientFromURLAndSecret(restoreServer.Context(), url)
				if err != nil {
					return err
				}
				objR, err := objClient.Reader(url.Object, 0, 0)
				if err != nil {
					return err
				}
				snappyR := snappy.NewReader(objR)
				r = pbutil.NewReader(snappyR)
				continue
			} else {
				op = req.Op
			}
		} else {
			op = &admin.Op{}
			if err := r.Read(op); err != nil {
				if err == io.EOF {
					return nil
				}
				return err
			}
		}
		switch {
		case op.Op1_7 != nil && op.Op1_7.Object != nil:
			r := &extractObjectReader{adminAPIRestoreServer: restoreServer}
			r.buf.Write(op.Op1_7.Object.Value)
			if _, _, err := pachClient.PutObject(r); err != nil {
				return fmt.Errorf("error putting object: %v", err)
			}
		case op.Op1_7 != nil && op.Op1_7.Tag != nil:
			if _, err := pachClient.ObjectAPIClient.TagObject(ctx, op.Op1_7.Tag); err != nil {
				return fmt.Errorf("error tagging object: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Repo != nil:
			if _, err := pachClient.PfsAPIClient.CreateRepo(ctx, op.Op1_7.Repo); err != nil && !errutil.IsAlreadyExistError(err) {
				return fmt.Errorf("error creating repo: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Commit != nil:
			if _, err := pachClient.PfsAPIClient.BuildCommit(ctx, op.Op1_7.Commit); err != nil && !errutil.IsAlreadyExistError(err) {
				return fmt.Errorf("error creating commit: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Branch != nil:
			if op.Op1_7.Branch.Branch == nil {
				op.Op1_7.Branch.Branch = client.NewBranch(op.Op1_7.Branch.Head.Repo.Name, op.Op1_7.Branch.SBranch)
			}
			if _, err := pachClient.PfsAPIClient.CreateBranch(ctx, op.Op1_7.Branch); err != nil && !errutil.IsAlreadyExistError(err) {
				return fmt.Errorf("error creating branch: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Op1_7 != nil && op.Op1_7.Pipeline != nil:
			if _, err := pachClient.PpsAPIClient.CreatePipeline(ctx, op.Op1_7.Pipeline); err != nil && !errutil.IsAlreadyExistError(err) {
				return fmt.Errorf("error creating pipeline: %v", grpcutil.ScrubGRPC(err))
			}
		}
	}
}

func (a *apiServer) getPachClient() *client.APIClient {
	a.pachClientOnce.Do(func() {
		var err error
		a.pachClient, err = client.NewFromAddress(a.address)
		if err != nil {
			panic(fmt.Sprintf("pps failed to initialize pach client: %v", err))
		}
	})
	return a.pachClient
}

type extractObjectWriter func(*admin.Op) error

func (w extractObjectWriter) Write(p []byte) (int, error) {
	chunkSize := grpcutil.MaxMsgSize / 2
	var n int
	for i := 0; i*(chunkSize) < len(p); i++ {
		value := p[i*chunkSize:]
		if len(value) > chunkSize {
			value = value[:chunkSize]
		}
		if err := w(&admin.Op{Op1_7: &admin.Op1_7{Object: &pfs.PutObjectRequest{Value: value}}}); err != nil {
			return n, err
		}
		n += len(value)
	}
	return n, nil
}

type adminAPIRestoreServer admin.API_RestoreServer

type extractObjectReader struct {
	adminAPIRestoreServer
	buf bytes.Buffer
	eof bool
}

func (r *extractObjectReader) Read(p []byte) (int, error) {
	for len(p) > r.buf.Len() && !r.eof {
		request, err := r.Recv()
		if err != nil {
			return 0, grpcutil.ScrubGRPC(err)
		}
		op := request.Op
		if op.Op1_7.Object == nil {
			return 0, fmt.Errorf("expected an object, but got: %v", op)
		}
		r.buf.Write(op.Op1_7.Object.Value)
		if len(op.Op1_7.Object.Value) == 0 {
			r.eof = true
		}
	}
	return r.buf.Read(p)
}
<|endoftext|>


<commit_before><commit_msg>update the response builder to handle pointers<commit_after><|endoftext|>


<commit_before>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"github.com/cilium/cilium/pkg/cidr"
	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/datapath"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mtu"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/node/addressing"
	nodemanager "github.com/cilium/cilium/pkg/node/manager"
	nodestore "github.com/cilium/cilium/pkg/node/store"
	"github.com/cilium/cilium/pkg/option"
)

type nodeDiscovery struct {
	manager     *nodemanager.Manager
	localConfig datapath.LocalNodeConfiguration
	registrar   nodestore.NodeRegistrar
	localNode   node.Node
	registered  chan struct{}
}

func enableLocalNodeRoute() bool {
	if option.Config.IsFlannelMasterDeviceSet() {
		return false
	}

	return true
}

func newNodeDiscovery(manager *nodemanager.Manager, mtuConfig mtu.Configuration) *nodeDiscovery {
	auxPrefixes := []*cidr.CIDR{}

	if option.Config.IPv4ServiceRange != AutoCIDR {
		serviceCIDR, err := cidr.ParseCIDR(option.Config.IPv4ServiceRange)
		if err != nil {
			log.WithError(err).WithField(logfields.V4Prefix, option.Config.IPv4ServiceRange).Fatal("Invalid IPv4 service prefix")
		}

		auxPrefixes = append(auxPrefixes, serviceCIDR)
	}

	if option.Config.IPv6ServiceRange != AutoCIDR {
		serviceCIDR, err := cidr.ParseCIDR(option.Config.IPv6ServiceRange)
		if err != nil {
			log.WithError(err).WithField(logfields.V6Prefix, option.Config.IPv6ServiceRange).Fatal("Invalid IPv6 service prefix")
		}

		auxPrefixes = append(auxPrefixes, serviceCIDR)
	}

	return &nodeDiscovery{
		manager: manager,
		localConfig: datapath.LocalNodeConfiguration{
			MtuConfig:               mtuConfig,
			UseSingleClusterRoute:   option.Config.UseSingleClusterRoute,
			EnableIPv4:              option.Config.EnableIPv4,
			EnableIPv6:              option.Config.EnableIPv6,
			EnableEncapsulation:     option.Config.Tunnel != option.TunnelDisabled,
			EnableAutoDirectRouting: option.Config.EnableAutoDirectRouting,
			EnableLocalNodeRoute:    enableLocalNodeRoute(),
			AuxiliaryPrefixes:       auxPrefixes,
			EnableIPSec:             option.Config.EnableIPSec,
		},
		localNode: node.Node{
			Source: node.FromLocalNode,
		},
		registered: make(chan struct{}),
	}
}

// start configures the local node and starts node discovery. This is called on
// agent startup to configure the local node based on the configuration options
// passed to the agent
func (n *nodeDiscovery) startDiscovery() {
	n.localNode.Name = node.GetName()
	n.localNode.Cluster = option.Config.ClusterName
	n.localNode.IPAddresses = []node.Address{
		{
			Type: addressing.NodeInternalIP,
			IP:   node.GetExternalIPv4(),
		},
	}
	n.localNode.IPv4AllocCIDR = node.GetIPv4AllocRange()
	n.localNode.IPv6AllocCIDR = node.GetIPv6AllocRange()
	n.localNode.ClusterID = option.Config.ClusterID
	n.localNode.IPv4GW = node.GetInternalIPv4()

	n.manager.NodeUpdated(n.localNode)

	go func() {
		log.Info("Adding local node to cluster")
		if err := n.registrar.RegisterNode(&n.localNode, n.manager); err != nil {
			log.WithError(err).Fatal("Unable to initialize local node")
		}
		close(n.registered)
	}()

	go func() {
		select {
		case <-n.registered:
		case <-time.NewTimer(defaults.NodeInitTimeout).C:
			log.Fatalf("Unable to initialize local node due to timeout")
		}
	}()

	go func() {
		<-n.registered
		controller.NewManager().UpdateController("propagating local node change to kv-store",
			controller.ControllerParams{
				DoFunc: func() error {
					err := n.registrar.UpdateLocalKeySync(&n.localNode)
					if err != nil {
						log.WithError(err).Error("Unable to propagate local node change to kvstore")
					}
					return err
				},
			})
	}()
}

// Close shuts down the node discovery engine
func (n *nodeDiscovery) Close() {
	n.manager.Close()
}
<commit_msg>agent: Announce both IPv4 and IPv6 node address via node discovery<commit_after>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"time"

	"github.com/cilium/cilium/pkg/cidr"
	"github.com/cilium/cilium/pkg/controller"
	"github.com/cilium/cilium/pkg/datapath"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mtu"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/node/addressing"
	nodemanager "github.com/cilium/cilium/pkg/node/manager"
	nodestore "github.com/cilium/cilium/pkg/node/store"
	"github.com/cilium/cilium/pkg/option"
)

type nodeDiscovery struct {
	manager     *nodemanager.Manager
	localConfig datapath.LocalNodeConfiguration
	registrar   nodestore.NodeRegistrar
	localNode   node.Node
	registered  chan struct{}
}

func enableLocalNodeRoute() bool {
	if option.Config.IsFlannelMasterDeviceSet() {
		return false
	}

	return true
}

func newNodeDiscovery(manager *nodemanager.Manager, mtuConfig mtu.Configuration) *nodeDiscovery {
	auxPrefixes := []*cidr.CIDR{}

	if option.Config.IPv4ServiceRange != AutoCIDR {
		serviceCIDR, err := cidr.ParseCIDR(option.Config.IPv4ServiceRange)
		if err != nil {
			log.WithError(err).WithField(logfields.V4Prefix, option.Config.IPv4ServiceRange).Fatal("Invalid IPv4 service prefix")
		}

		auxPrefixes = append(auxPrefixes, serviceCIDR)
	}

	if option.Config.IPv6ServiceRange != AutoCIDR {
		serviceCIDR, err := cidr.ParseCIDR(option.Config.IPv6ServiceRange)
		if err != nil {
			log.WithError(err).WithField(logfields.V6Prefix, option.Config.IPv6ServiceRange).Fatal("Invalid IPv6 service prefix")
		}

		auxPrefixes = append(auxPrefixes, serviceCIDR)
	}

	return &nodeDiscovery{
		manager: manager,
		localConfig: datapath.LocalNodeConfiguration{
			MtuConfig:               mtuConfig,
			UseSingleClusterRoute:   option.Config.UseSingleClusterRoute,
			EnableIPv4:              option.Config.EnableIPv4,
			EnableIPv6:              option.Config.EnableIPv6,
			EnableEncapsulation:     option.Config.Tunnel != option.TunnelDisabled,
			EnableAutoDirectRouting: option.Config.EnableAutoDirectRouting,
			EnableLocalNodeRoute:    enableLocalNodeRoute(),
			AuxiliaryPrefixes:       auxPrefixes,
			EnableIPSec:             option.Config.EnableIPSec,
		},
		localNode: node.Node{
			Source: node.FromLocalNode,
		},
		registered: make(chan struct{}),
	}
}

// start configures the local node and starts node discovery. This is called on
// agent startup to configure the local node based on the configuration options
// passed to the agent
func (n *nodeDiscovery) startDiscovery() {
	n.localNode.Name = node.GetName()
	n.localNode.Cluster = option.Config.ClusterName
	n.localNode.IPAddresses = []node.Address{}
	n.localNode.IPv4AllocCIDR = node.GetIPv4AllocRange()
	n.localNode.IPv6AllocCIDR = node.GetIPv6AllocRange()
	n.localNode.ClusterID = option.Config.ClusterID
	n.localNode.IPv4GW = node.GetInternalIPv4()

	if node.GetExternalIPv4() != nil {
		n.localNode.IPAddresses = append(n.localNode.IPAddresses, node.Address{
			Type: addressing.NodeInternalIP,
			IP:   node.GetExternalIPv4(),
		})
	}

	if node.GetIPv6() != nil {
		n.localNode.IPAddresses = append(n.localNode.IPAddresses, node.Address{
			Type: addressing.NodeInternalIP,
			IP:   node.GetIPv6(),
		})
	}

	n.manager.NodeUpdated(n.localNode)

	go func() {
		log.Info("Adding local node to cluster")
		if err := n.registrar.RegisterNode(&n.localNode, n.manager); err != nil {
			log.WithError(err).Fatal("Unable to initialize local node")
		}
		close(n.registered)
	}()

	go func() {
		select {
		case <-n.registered:
		case <-time.NewTimer(defaults.NodeInitTimeout).C:
			log.Fatalf("Unable to initialize local node due to timeout")
		}
	}()

	go func() {
		<-n.registered
		controller.NewManager().UpdateController("propagating local node change to kv-store",
			controller.ControllerParams{
				DoFunc: func() error {
					err := n.registrar.UpdateLocalKeySync(&n.localNode)
					if err != nil {
						log.WithError(err).Error("Unable to propagate local node change to kvstore")
					}
					return err
				},
			})
	}()
}

// Close shuts down the node discovery engine
func (n *nodeDiscovery) Close() {
	n.manager.Close()
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/otoolep/raft"
	"github.com/otoolep/rqlite/command"
	"github.com/otoolep/rqlite/db"
)

// isPretty returns whether the HTTP response body should be pretty-printed.
func isPretty(req *http.Request) (bool, error) {
	err := req.ParseForm()
	if err != nil {
		return false, err
	}
	if _, ok := req.Form["pretty"]; ok {
		return true, nil
	}
	return false, nil
}

// The raftd server is a combination of the Raft server and an HTTP
// server which acts as the transport.
type Server struct {
	name       string
	host       string
	port       int
	path       string
	router     *mux.Router
	raftServer raft.Server
	httpServer *http.Server
	db         *db.DB
	mutex      sync.RWMutex
}

// Creates a new server.
func New(dataDir string, dbfile string, host string, port int) *Server {
	s := &Server{
		host:   host,
		port:   port,
		path:   dataDir,
		db:     db.New(path.Join(dataDir, dbfile)),
		router: mux.NewRouter(),
	}

	// Read existing name or generate a new one.
	if b, err := ioutil.ReadFile(filepath.Join(dataDir, "name")); err == nil {
		s.name = string(b)
	} else {
		s.name = fmt.Sprintf("%07x", rand.Int())[0:7]
		if err = ioutil.WriteFile(filepath.Join(dataDir, "name"), []byte(s.name), 0644); err != nil {
			panic(err)
		}
	}

	return s
}

// Returns the connection string.
func (s *Server) connectionString() string {
	return fmt.Sprintf("http://%s:%d", s.host, s.port)
}

// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
	var err error

	log.Printf("Initializing Raft Server: %s", s.path)

	// Initialize and start Raft server.
	transporter := raft.NewHTTPTransporter("/raft", 200*time.Millisecond)
	s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.db, "")
	if err != nil {
		log.Fatal(err)
	}
	transporter.Install(s.raftServer, s)
	s.raftServer.Start()

	if leader != "" {
		// Join to leader if specified.

		log.Println("Attempting to join leader:", leader)

		if !s.raftServer.IsLogEmpty() {
			log.Fatal("Cannot join with an existing log")
		}
		if err := s.Join(leader); err != nil {
			log.Fatal(err)
		}

	} else if s.raftServer.IsLogEmpty() {
		// Initialize the server by joining itself.

		log.Println("Initializing new cluster")

		_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
			Name:             s.raftServer.Name(),
			ConnectionString: s.connectionString(),
		})
		if err != nil {
			log.Fatal(err)
		}

	} else {
		log.Println("Recovered from log")
	}

	log.Println("Initializing HTTP server")

	// Initialize and start HTTP server.
	s.httpServer = &http.Server{
		Addr:    fmt.Sprintf(":%d", s.port),
		Handler: s.router,
	}

	s.router.HandleFunc("/db", s.readHandler).Methods("GET")
	s.router.HandleFunc("/db", s.writeHandler).Methods("POST")
	s.router.HandleFunc("/join", s.joinHandler).Methods("POST")

	log.Println("Listening at:", s.connectionString())

	return s.httpServer.ListenAndServe()
}

// This is a hack around Gorilla mux not providing the correct net/http
// HandleFunc() interface.
func (s *Server) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) {
	s.router.HandleFunc(pattern, handler)
}

// Joins to the leader of an existing cluster.
func (s *Server) Join(leader string) error {
	command := &raft.DefaultJoinCommand{
		Name:             s.raftServer.Name(),
		ConnectionString: s.connectionString(),
	}

	var b bytes.Buffer
	json.NewEncoder(&b).Encode(command)
	resp, err := http.Post(fmt.Sprintf("http://%s/join", leader), "application/json", &b)
	if err != nil {
		return err
	}
	resp.Body.Close()

	return nil
}

func (s *Server) joinHandler(w http.ResponseWriter, req *http.Request) {
	command := &raft.DefaultJoinCommand{}

	if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if _, err := s.raftServer.Do(command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (s *Server) readHandler(w http.ResponseWriter, req *http.Request) {
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	r, err := s.db.Query(string(b))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(r, "", "    ")
	} else {
		b, err = json.Marshal(r)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write([]byte(b))
}

func (s *Server) writeHandler(w http.ResponseWriter, req *http.Request) {
	// Read the value from the POST body.
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	stmt := string(b)

	// Execute the command against the Raft server.
	_, err = s.raftServer.Do(command.NewWriteCommand(stmt))
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
}
<commit_msg>Factor out detection of URL query params<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/otoolep/raft"
	"github.com/otoolep/rqlite/command"
	"github.com/otoolep/rqlite/db"
)

// queryParam returns whether the given query param is set to true.
func queryParam(req *http.Request, param string) (bool, error) {
	err := req.ParseForm()
	if err != nil {
		return false, err
	}
	if _, ok := req.Form[param]; ok {
		return true, nil
	}
	return false, nil
}

// isPretty returns whether the HTTP response body should be pretty-printed.
func isPretty(req *http.Request) (bool, error) {
	return queryParam(req, "pretty")
}

// transactionRequested returns whether the client requested an explicit
// transaction for the request.
func transactionRequested(req *http.Request) (bool, error) {
	return queryParam(req, "transaction")
}

// The raftd server is a combination of the Raft server and an HTTP
// server which acts as the transport.
type Server struct {
	name       string
	host       string
	port       int
	path       string
	router     *mux.Router
	raftServer raft.Server
	httpServer *http.Server
	db         *db.DB
	mutex      sync.RWMutex
}

// Creates a new server.
func New(dataDir string, dbfile string, host string, port int) *Server {
	s := &Server{
		host:   host,
		port:   port,
		path:   dataDir,
		db:     db.New(path.Join(dataDir, dbfile)),
		router: mux.NewRouter(),
	}

	// Read existing name or generate a new one.
	if b, err := ioutil.ReadFile(filepath.Join(dataDir, "name")); err == nil {
		s.name = string(b)
	} else {
		s.name = fmt.Sprintf("%07x", rand.Int())[0:7]
		if err = ioutil.WriteFile(filepath.Join(dataDir, "name"), []byte(s.name), 0644); err != nil {
			panic(err)
		}
	}

	return s
}

// Returns the connection string.
func (s *Server) connectionString() string {
	return fmt.Sprintf("http://%s:%d", s.host, s.port)
}

// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
	var err error

	log.Printf("Initializing Raft Server: %s", s.path)

	// Initialize and start Raft server.
	transporter := raft.NewHTTPTransporter("/raft", 200*time.Millisecond)
	s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.db, "")
	if err != nil {
		log.Fatal(err)
	}
	transporter.Install(s.raftServer, s)
	s.raftServer.Start()

	if leader != "" {
		// Join to leader if specified.

		log.Println("Attempting to join leader:", leader)

		if !s.raftServer.IsLogEmpty() {
			log.Fatal("Cannot join with an existing log")
		}
		if err := s.Join(leader); err != nil {
			log.Fatal(err)
		}

	} else if s.raftServer.IsLogEmpty() {
		// Initialize the server by joining itself.

		log.Println("Initializing new cluster")

		_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
			Name:             s.raftServer.Name(),
			ConnectionString: s.connectionString(),
		})
		if err != nil {
			log.Fatal(err)
		}

	} else {
		log.Println("Recovered from log")
	}

	log.Println("Initializing HTTP server")

	// Initialize and start HTTP server.
	s.httpServer = &http.Server{
		Addr:    fmt.Sprintf(":%d", s.port),
		Handler: s.router,
	}

	s.router.HandleFunc("/db", s.readHandler).Methods("GET")
	s.router.HandleFunc("/db", s.writeHandler).Methods("POST")
	s.router.HandleFunc("/join", s.joinHandler).Methods("POST")

	log.Println("Listening at:", s.connectionString())

	return s.httpServer.ListenAndServe()
}

// This is a hack around Gorilla mux not providing the correct net/http
// HandleFunc() interface.
func (s *Server) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) {
	s.router.HandleFunc(pattern, handler)
}

// Joins to the leader of an existing cluster.
func (s *Server) Join(leader string) error {
	command := &raft.DefaultJoinCommand{
		Name:             s.raftServer.Name(),
		ConnectionString: s.connectionString(),
	}

	var b bytes.Buffer
	json.NewEncoder(&b).Encode(command)
	resp, err := http.Post(fmt.Sprintf("http://%s/join", leader), "application/json", &b)
	if err != nil {
		return err
	}
	resp.Body.Close()

	return nil
}

func (s *Server) joinHandler(w http.ResponseWriter, req *http.Request) {
	command := &raft.DefaultJoinCommand{}

	if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if _, err := s.raftServer.Do(command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (s *Server) readHandler(w http.ResponseWriter, req *http.Request) {
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	r, err := s.db.Query(string(b))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(r, "", "    ")
	} else {
		b, err = json.Marshal(r)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write([]byte(b))
}

func (s *Server) writeHandler(w http.ResponseWriter, req *http.Request) {
	// Read the value from the POST body.
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	stmt := string(b)

	// Execute the command against the Raft server.
	_, err = s.raftServer.Do(command.NewWriteCommand(stmt))
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package jujuclient

import (
	"io/ioutil"
	"os"

	"github.com/juju/errors"
	"gopkg.in/yaml.v2"

	"github.com/juju/juju/juju/osenv"
)

// JujuControllersPath is the location where controllers information is
// expected to be found.
// TODO (anastasiamac 2016-02-02) This needs to be updated to use new and shiny DATA instead of HOME...?
// Requires JUJU_DATA to be set.
func JujuControllersPath() string {
	return osenv.JujuXDGDataHomePath("controllers.yaml")
}

// ReadControllersFile loads all controllers defined in a given file.
// If the file is not found, it is not an error.
func ReadControllersFile(file string) (map[string]Controller, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	controllers, err := ParseControllers(data)
	if err != nil {
		return nil, err
	}
	return controllers.Controllers, nil
}

// WriteControllersFile marshals to YAML details of the given controllers
// and writes it to the controllers file.
func WriteControllersFile(controllers *Controllers) error {
	data, err := yaml.Marshal(controllers)
	if err != nil {
		return errors.Annotate(err, "cannot marshal yaml controllers")
	}
	return ioutil.WriteFile(JujuControllersPath(), data, os.FileMode(0600))
}

// ParseControllers parses the given YAML bytes into controllers metadata.
func ParseControllers(data []byte) (*Controllers, error) {
	var result Controllers
	err := yaml.Unmarshal(data, &result)
	if err != nil {
		return nil, errors.Annotate(err, "cannot unmarshal yaml controllers metadata")
	}
	return &result, nil
}
<commit_msg>Remove question.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package jujuclient

import (
	"io/ioutil"
	"os"

	"github.com/juju/errors"
	"gopkg.in/yaml.v2"

	"github.com/juju/juju/juju/osenv"
)

// JujuControllersPath is the location where controllers information is
// expected to be found.
// TODO (anastasiamac 2016-02-02) This needs to be updated to use new and shiny DATA instead of HOME.
// Requires JUJU_HOME to be set.
func JujuControllersPath() string {
	return osenv.JujuXDGDataHomePath("controllers.yaml")
}

// ReadControllersFile loads all controllers defined in a given file.
// If the file is not found, it is not an error.
func ReadControllersFile(file string) (map[string]Controller, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	controllers, err := ParseControllers(data)
	if err != nil {
		return nil, err
	}
	return controllers.Controllers, nil
}

// WriteControllersFile marshals to YAML details of the given controllers
// and writes it to the controllers file.
func WriteControllersFile(controllers *Controllers) error {
	data, err := yaml.Marshal(controllers)
	if err != nil {
		return errors.Annotate(err, "cannot marshal yaml controllers")
	}
	return ioutil.WriteFile(JujuControllersPath(), data, os.FileMode(0600))
}

// ParseControllers parses the given YAML bytes into controllers metadata.
func ParseControllers(data []byte) (*Controllers, error) {
	var result Controllers
	err := yaml.Unmarshal(data, &result)
	if err != nil {
		return nil, errors.Annotate(err, "cannot unmarshal yaml controllers metadata")
	}
	return &result, nil
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"strings"

	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/docker/distribution/registry/api/errcode"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type dockerFetcher struct {
	*dockerBase
}

func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(
		logrus.Fields{
			"base":   r.base.String(),
			"digest": desc.Digest,
		},
	))

	urls, err := r.getV2URLPaths(ctx, desc)
	if err != nil {
		return nil, err
	}

	ctx, err = contextWithRepositoryScope(ctx, r.refspec, false)
	if err != nil {
		return nil, err
	}

	return newHTTPReadSeeker(desc.Size, func(offset int64) (io.ReadCloser, error) {
		for _, u := range urls {
			rc, err := r.open(ctx, u, desc.MediaType, offset)
			if err != nil {
				if errdefs.IsNotFound(err) {
					continue // try one of the other urls.
				}

				return nil, err
			}

			return rc, nil
		}

		return nil, errors.Wrapf(errdefs.ErrNotFound,
			"could not fetch content descriptor %v (%v) from remote",
			desc.Digest, desc.MediaType)

	})
}

func (r dockerFetcher) open(ctx context.Context, u, mediatype string, offset int64) (io.ReadCloser, error) {
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Accept", strings.Join([]string{mediatype, `*`}, ", "))

	if offset > 0 {
		// Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints
		// will return the header without supporting the range. The content
		// range must always be checked.
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
	}

	resp, err := r.doRequestWithRetries(ctx, req, nil)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode > 299 {
		// TODO(stevvooe): When doing a offset specific request, we should
		// really distinguish between a 206 and a 200. In the case of 200, we
		// can discard the bytes, hiding the seek behavior from the
		// implementation.

		defer resp.Body.Close()
		if resp.StatusCode == http.StatusNotFound {
			return nil, errors.Wrapf(errdefs.ErrNotFound, "content at %v not found", u)
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err == nil {
			dockerErr := errcode.Errors{}
			err := json.Unmarshal(body, &dockerErr)
			if err == nil && dockerErr.Len() > 0 {
				return nil, errors.Errorf("unexpected status code %v: %s - Server message: %s", u, resp.Status, dockerErr.Error())
			}
		}
		return nil, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
	}
	if offset > 0 {
		cr := resp.Header.Get("content-range")
		if cr != "" {
			if !strings.HasPrefix(cr, fmt.Sprintf("bytes %d-", offset)) {
				return nil, errors.Errorf("unhandled content range in response: %v", cr)

			}
		} else {
			// TODO: Should any cases where use of content range
			// without the proper header be considered?
			// 206 responses?

			// Discard up to offset
			// Could use buffer pool here but this case should be rare
			n, err := io.Copy(ioutil.Discard, io.LimitReader(resp.Body, offset))
			if err != nil {
				return nil, errors.Wrap(err, "failed to discard to offset")
			}
			if n != offset {
				return nil, errors.Errorf("unable to discard to offset")
			}

		}
	}

	return resp.Body, nil
}

// getV2URLPaths generates the candidate urls paths for the object based on the
// set of hints and the provided object id. URLs are returned in the order of
// most to least likely succeed.
func (r *dockerFetcher) getV2URLPaths(ctx context.Context, desc ocispec.Descriptor) ([]string, error) {
	var urls []string

	if len(desc.URLs) > 0 {
		// handle fetch via external urls.
		for _, u := range desc.URLs {
			log.G(ctx).WithField("url", u).Debug("adding alternative url")
			urls = append(urls, u)
		}
	}

	switch desc.MediaType {
	case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
		images.MediaTypeDockerSchema1Manifest,
		ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
		urls = append(urls, r.url(path.Join("manifests", desc.Digest.String())))
	}

	// always fallback to attempting to get the object out of the blobs store.
	urls = append(urls, r.url(path.Join("blobs", desc.Digest.String())))

	return urls, nil
}
<commit_msg>Update handling of registry errors<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"strings"

	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/docker/distribution/registry/api/errcode"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type dockerFetcher struct {
	*dockerBase
}

func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(
		logrus.Fields{
			"base":   r.base.String(),
			"digest": desc.Digest,
		},
	))

	urls, err := r.getV2URLPaths(ctx, desc)
	if err != nil {
		return nil, err
	}

	ctx, err = contextWithRepositoryScope(ctx, r.refspec, false)
	if err != nil {
		return nil, err
	}

	return newHTTPReadSeeker(desc.Size, func(offset int64) (io.ReadCloser, error) {
		for _, u := range urls {
			rc, err := r.open(ctx, u, desc.MediaType, offset)
			if err != nil {
				if errdefs.IsNotFound(err) {
					continue // try one of the other urls.
				}

				return nil, err
			}

			return rc, nil
		}

		return nil, errors.Wrapf(errdefs.ErrNotFound,
			"could not fetch content descriptor %v (%v) from remote",
			desc.Digest, desc.MediaType)

	})
}

func (r dockerFetcher) open(ctx context.Context, u, mediatype string, offset int64) (io.ReadCloser, error) {
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Accept", strings.Join([]string{mediatype, `*`}, ", "))

	if offset > 0 {
		// Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints
		// will return the header without supporting the range. The content
		// range must always be checked.
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
	}

	resp, err := r.doRequestWithRetries(ctx, req, nil)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode > 299 {
		// TODO(stevvooe): When doing a offset specific request, we should
		// really distinguish between a 206 and a 200. In the case of 200, we
		// can discard the bytes, hiding the seek behavior from the
		// implementation.
		defer resp.Body.Close()

		if resp.StatusCode == http.StatusNotFound {
			return nil, errors.Wrapf(errdefs.ErrNotFound, "content at %v not found", u)
		}
		var registryErr errcode.Errors
		if err := json.NewDecoder(resp.Body).Decode(&registryErr); err != nil || registryErr.Len() < 1 {
			return nil, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
		}
		return nil, errors.Errorf("unexpected status code %v: %s - Server message: %s", u, resp.Status, registryErr.Error())
	}
	if offset > 0 {
		cr := resp.Header.Get("content-range")
		if cr != "" {
			if !strings.HasPrefix(cr, fmt.Sprintf("bytes %d-", offset)) {
				return nil, errors.Errorf("unhandled content range in response: %v", cr)

			}
		} else {
			// TODO: Should any cases where use of content range
			// without the proper header be considered?
			// 206 responses?

			// Discard up to offset
			// Could use buffer pool here but this case should be rare
			n, err := io.Copy(ioutil.Discard, io.LimitReader(resp.Body, offset))
			if err != nil {
				return nil, errors.Wrap(err, "failed to discard to offset")
			}
			if n != offset {
				return nil, errors.Errorf("unable to discard to offset")
			}

		}
	}

	return resp.Body, nil
}

// getV2URLPaths generates the candidate urls paths for the object based on the
// set of hints and the provided object id. URLs are returned in the order of
// most to least likely succeed.
func (r *dockerFetcher) getV2URLPaths(ctx context.Context, desc ocispec.Descriptor) ([]string, error) {
	var urls []string

	if len(desc.URLs) > 0 {
		// handle fetch via external urls.
		for _, u := range desc.URLs {
			log.G(ctx).WithField("url", u).Debug("adding alternative url")
			urls = append(urls, u)
		}
	}

	switch desc.MediaType {
	case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList,
		images.MediaTypeDockerSchema1Manifest,
		ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex:
		urls = append(urls, r.url(path.Join("manifests", desc.Digest.String())))
	}

	// always fallback to attempting to get the object out of the blobs store.
	urls = append(urls, r.url(path.Join("blobs", desc.Digest.String())))

	return urls, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/wsxiaoys/terminal/color"

	"github.com/xogeny/impact/config"
)

var version = "0.7.0"

type VersionCommand struct {
	Verbose bool `short:"v" long:"verbose" description:"Turn on verbose output"`
}

func (x VersionCommand) Execute(args []string) error {
	color.Printf("@{g}Impact version @{!g}%s\n", version)
	color.Printf("  Settings file: %s\n", config.SettingsFile())
	settings, err := config.ReadSettings()
	if err != nil {
		color.Printf("    @{!r}%s\n", err.Error())
	} else {
		color.Printf("@{g}%s\n", settings.List(""))
	}
	return nil
}
<commit_msg>Updated version<commit_after>package main

import (
	"github.com/wsxiaoys/terminal/color"

	"github.com/xogeny/impact/config"
)

var version = "0.8.0-dev"

type VersionCommand struct {
	Verbose bool `short:"v" long:"verbose" description:"Turn on verbose output"`
}

func (x VersionCommand) Execute(args []string) error {
	color.Printf("@{g}Impact version @{!g}%s\n", version)
	color.Printf("  Settings file: %s\n", config.SettingsFile())
	settings, err := config.ReadSettings()
	if err != nil {
		color.Printf("    @{!r}%s\n", err.Error())
	} else {
		color.Printf("@{g}%s\n", settings.List(""))
	}
	return nil
}
<|endoftext|>


<commit_before>package lists

import (
	"errors"
)

type StringSlice []string
type NestedSlice []interface{}

// P01 (*) Find the last element of a list.
// Example:
// ?- my_last(X,[a,b,c,d]).
// X = d

func (s StringSlice) LastElement() (string, error) {
	if len(s) < 1 {
		return "", errors.New("Nothing in the string slice!")
	}
	return s[len(s)-1], nil
}

// P02 (*) Find the last but one element of a list.
// (zweitletztes Element, l'avant-dernier élément)

func (s StringSlice) LastElementButOne() (string, error) {
	if len(s) < 2 {
		return "", errors.New("String slice isn't long enough!")
	}
	return s[len(s)-2], nil
}

// P03 (*) Find the K'th element of a list.
// The first element in the list is number 1.
// Example:
// ?- element_at(X,[a,b,c,d,e],3).
// X = c

func (s StringSlice) NthElement(index int) (string, error) {
	k := index - 1
	if len(s) < index {
		return "", errors.New("String slice isn't long enough!")
	}
	return s[k], nil
}

// P04 (*) Find the number of elements of a list.

func (s StringSlice) Length() int {
	return len(s)
}

// SIDEBAR: Rotate a list...
// Instead of full reversal, this will rotate the
// list by rotDist # of elements
// http://www.cs.bell-labs.com/cm/cs/pearls/rotate.c
// Programming Pearls 2.3

func (s StringSlice) ReversePortion(i, j int) StringSlice {
	result := make(StringSlice, len(s))
	copy(result, s)
	var t string
	for i < j {
		t = result[i]
		result[i] = result[j]
		result[j] = t
		i++
		j--
	}
	return result
}

func (s StringSlice) Rotate(rotDist, n int) StringSlice {
	result := make(StringSlice, len(s))
	copy(result, s)
	result = result.ReversePortion(0, rotDist-1)
	result = result.ReversePortion(rotDist, n-1)
	result = result.ReversePortion(0, n-1)
	return result
}

// P05 (*) Reverse a list.
// Uses ReversePortion() algorithm

func (s StringSlice) Reverse() StringSlice {
	result := make(StringSlice, len(s))
	copy(result, s)
	result = result.ReversePortion(0, s.Length()-1)
	return result
}

// P06 (*) Find out whether a list is a palindrome.
// A palindrome can be read forward or backward; e.g. [x,a,m,a,x].

func (s StringSlice) IsPalindrome() bool {
	var i, mid, last int
	i = 0
	mid = s.Length() / 2
	last = s.Length() - 1
	for i < mid {
		if s[i] != s[last-i] {
			return false
		}
		i++
	}
	return true
}

// P07 (**) Flatten a nested list structure.
// Transform a list, possibly holding lists as elements into a `flat' list by replacing each list with its elements (recursively).

// Example:
// ?- my_flatten([a, [b, [c, d], e]], X).
// X = [a, b, c, d, e]

// Hint: Use the predefined predicates is_list/1 and append/3

// http://rosettacode.org/wiki/Flatten_a_list#Go
// Uses type switching http://golang.org/doc/effective_go.html#type_switch

func (n NestedSlice) Flatten() StringSlice {
	var result []string
	for _, s := range n {
		switch i := s.(type) {
		case string:
			result = append(result, i)
		case NestedSlice:
			result = append(result, i.Flatten()...)
		}
	}
	return result
}

// P08 (**) Eliminate consecutive duplicates of list elements.
// If a list contains repeated elements they should be replaced with a single copy of the element. The order of the elements should not be changed.

// Example:
// ?- compress([a,a,a,a,b,c,c,a,a,d,e,e,e,e],X).
// X = [a,b,c,a,d,e]

func (s StringSlice) Compress() StringSlice {
	var result StringSlice
	for _, item := range s {
		if last, _ := result.LastElement(); item != last {
			result = append(result, item)
		}
	}
	return result
}

// P09 (**) Pack consecutive duplicates of list elements into sublists.
// If a list contains repeated elements they should be placed in separate sublists.

// Example:
// ?- pack([a,a,a,a,b,c,c,a,a,d,e,e,e,e],X).
// X = [[a,a,a,a],[b],[c,c],[a,a],[d],[e,e,e,e]]

func (s StringSlice) Pack() NestedSlice {
	var result NestedSlice
	var group = make(StringSlice, 0)
	for _, item := range s {
		if len(group) == 0 {
			group = append(group, item)
		} else if len(group) > 0 {
			if last := group[len(group)-1]; item != last {
				group = append(group, item)
			}
		}
		// } else {
		// 	result = append(result, group)
		// 	group = make(StringSlice, 0)
		// }
	}
	return result
}

// P10 (*) Run-length encoding of a list.
// Use the result of problem P09 to implement the so-called run-length encoding data compression method. Consecutive duplicates of elements are encoded as terms [N,E] where N is the number of duplicates of the element E.

// Example:
// ?- encode([a,a,a,a,b,c,c,a,a,d,e,e,e,e],X).
// X = [[4,a],[1,b],[2,c],[2,a],[1,d][4,e]]
<commit_msg>Pack method<commit_after>package lists

import (
	"errors"
	"fmt"
)

type StringSlice []string
type NestedSlice []interface{}

// P01 (*) Find the last element of a list.
// Example:
// ?- my_last(X,[a,b,c,d]).
// X = d

func (s StringSlice) LastElement() (string, error) {
	if len(s) < 1 {
		return "", errors.New("Nothing in the string slice!")
	}
	return s[len(s)-1], nil
}

// P02 (*) Find the last but one element of a list.
// (zweitletztes Element, l'avant-dernier élément)

func (s StringSlice) LastElementButOne() (string, error) {
	if len(s) < 2 {
		return "", errors.New("String slice isn't long enough!")
	}
	return s[len(s)-2], nil
}

// P03 (*) Find the K'th element of a list.
// The first element in the list is number 1.
// Example:
// ?- element_at(X,[a,b,c,d,e],3).
// X = c

func (s StringSlice) NthElement(index int) (string, error) {
	k := index - 1
	if len(s) < index {
		return "", errors.New("String slice isn't long enough!")
	}
	return s[k], nil
}

// P04 (*) Find the number of elements of a list.

func (s StringSlice) Length() int {
	return len(s)
}

// SIDEBAR: Rotate a list...
// Instead of full reversal, this will rotate the
// list by rotDist # of elements
// http://www.cs.bell-labs.com/cm/cs/pearls/rotate.c
// Programming Pearls 2.3

func (s StringSlice) ReversePortion(i, j int) StringSlice {
	result := make(StringSlice, len(s))
	copy(result, s)
	var t string
	for i < j {
		t = result[i]
		result[i] = result[j]
		result[j] = t
		i++
		j--
	}
	return result
}

func (s StringSlice) Rotate(rotDist, n int) StringSlice {
	result := make(StringSlice, len(s))
	copy(result, s)
	result = result.ReversePortion(0, rotDist-1)
	result = result.ReversePortion(rotDist, n-1)
	result = result.ReversePortion(0, n-1)
	return result
}

// P05 (*) Reverse a list.
// Uses ReversePortion() algorithm

func (s StringSlice) Reverse() StringSlice {
	result := make(StringSlice, len(s))
	copy(result, s)
	result = result.ReversePortion(0, s.Length()-1)
	return result
}

// P06 (*) Find out whether a list is a palindrome.
// A palindrome can be read forward or backward; e.g. [x,a,m,a,x].

func (s StringSlice) IsPalindrome() bool {
	var i, mid, last int
	i = 0
	mid = s.Length() / 2
	last = s.Length() - 1
	for i < mid {
		if s[i] != s[last-i] {
			return false
		}
		i++
	}
	return true
}

// P07 (**) Flatten a nested list structure.
// Transform a list, possibly holding lists as elements into a `flat' list by replacing each list with its elements (recursively).

// Example:
// ?- my_flatten([a, [b, [c, d], e]], X).
// X = [a, b, c, d, e]

// Hint: Use the predefined predicates is_list/1 and append/3

// http://rosettacode.org/wiki/Flatten_a_list#Go
// Uses type switching http://golang.org/doc/effective_go.html#type_switch

func (n NestedSlice) Flatten() StringSlice {
	var result []string
	for _, s := range n {
		switch i := s.(type) {
		case string:
			result = append(result, i)
		case NestedSlice:
			result = append(result, i.Flatten()...)
		}
	}
	return result
}

// P08 (**) Eliminate consecutive duplicates of list elements.
// If a list contains repeated elements they should be replaced with a single copy of the element. The order of the elements should not be changed.

// Example:
// ?- compress([a,a,a,a,b,c,c,a,a,d,e,e,e,e],X).
// X = [a,b,c,a,d,e]

func (s StringSlice) Compress() StringSlice {
	var result StringSlice
	for _, item := range s {
		if last, _ := result.LastElement(); item != last {
			result = append(result, item)
		}
	}
	return result
}

// P09 (**) Pack consecutive duplicates of list elements into sublists.
// If a list contains repeated elements they should be placed in separate sublists.

// Example:
// ?- pack([a,a,a,a,b,c,c,a,a,d,e,e,e,e],X).
// X = [[a,a,a,a],[b],[c,c],[a,a],[d],[e,e,e,e]]

func (s StringSlice) Pack() NestedSlice {
	var result NestedSlice
	var group StringSlice
	for _, item := range s {
		if len(group) == 0 {
			group = append(group, item)
		} else {
			if last := group[len(group)-1]; item != last {
				group = append(group, item)
			}
		}
		// } else {
		// 	result = append(result, group)
		// 	group = make(StringSlice, 0)
		// }
		fmt.Println(group)
	}
	return result
}

// P10 (*) Run-length encoding of a list.
// Use the result of problem P09 to implement the so-called run-length encoding data compression method. Consecutive duplicates of elements are encoded as terms [N,E] where N is the number of duplicates of the element E.

// Example:
// ?- encode([a,a,a,a,b,c,c,a,a,d,e,e,e,e],X).
// X = [[4,a],[1,b],[2,c],[2,a],[1,d][4,e]]
<|endoftext|>


<commit_before>package main

import (
	. "eaciit/wfdemo-git/library/helper"
	"flag"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	log      *tk.LogEngine
	mapField = map[string]MappingColumn{
		"windspeed":                   MappingColumn{"Wind Speed", "WindSpeed_ms", "m/s", 0.0, 50.0, "$avg"},
		"power":                       MappingColumn{"Power", "ActivePower_kW", "kW", -200, 2100.0 + (2100.0 * 0.10), "$sum"},
		"production":                  MappingColumn{"Production", "Production", "kWh", -200, 2100.0, "$sum"},
		"winddirection":               MappingColumn{"Wind Direction", "WindDirection", "Degree", 0.0, 360.0, "$avg"},
		"nacellepos":                  MappingColumn{"Nacelle Direction", "NacellePos", "Degree", 0.0, 360.0, "$avg"},
		"rotorrpm":                    MappingColumn{"Rotor RPM", "RotorSpeed_RPM", "RPM", 0.0, 30.0, "$avg"},
		"genrpm":                      MappingColumn{"Generator RPM", "GenSpeed_RPM", "RPM", 0.0, 30.0, "$avg"},
		"pitchangle":                  MappingColumn{"Pitch Angle", "PitchAngle", "Degree", -10.0, 120.0, "$avg"},
		"PitchCabinetTempBlade1":      MappingColumn{"Pitch Cabinet Temp Blade 1", "PitchCabinetTempBlade1", "Degree", -10.0, 120.0, "$avg"},
		"PitchCabinetTempBlade2":      MappingColumn{"Pitch Cabinet Temp Blade 2", "PitchCabinetTempBlade2", "Degree", -10.0, 120.0, "$avg"},
		"PitchCabinetTempBlade3":      MappingColumn{"Pitch Cabinet Temp Blade 3", "PitchCabinetTempBlade3", "Degree", -10.0, 120.0, "$avg"},
		"PitchConvInternalTempBlade1": MappingColumn{"Pitch Conv Internal Temp Blade 1", "PitchConvInternalTempBlade1", "Degree", -10.0, 120.0, "$avg"},
		"PitchConvInternalTempBlade2": MappingColumn{"Pitch Conv Internal Temp Blade 2", "PitchConvInternalTempBlade2", "Degree", -10.0, 120.0, "$avg"},
		"PitchConvInternalTempBlade3": MappingColumn{"Pitch Conv Internal Temp Blade 3", "PitchConvInternalTempBlade3", "Degree", -10.0, 120.0, "$avg"},
		"TempG1L1":                    MappingColumn{"TempG1 L1", "TempG1L1", "Degree", -10.0, 200.0, "$avg"},
		"TempG1L2":                    MappingColumn{"TempG1 L2", "TempG1L2", "Degree", -10.0, 200.0, "$avg"},
		"TempG1L3":                    MappingColumn{"TempG1 L3", "TempG1L3", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorBearingDE":      MappingColumn{"Temp Generator Bearing DE", "TempGeneratorBearingDE", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorBearingNDE":     MappingColumn{"Temp Generator Bearing NDE", "TempGeneratorBearingNDE", "Degree", -10.0, 200.0, "$avg"},
		"TempGearBoxOilSump":          MappingColumn{"Temp Gear Box Oil Sump", "TempGearBoxOilSump", "Degree", -10.0, 200.0, "$avg"},
		"TempHubBearing":              MappingColumn{"Temp Hub Bearing", "TempHubBearing", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorChoke":          MappingColumn{"Temp Generator Choke", "TempGeneratorChoke", "Degree", -10.0, 200.0, "$avg"},
		"TempGridChoke":               MappingColumn{"Temp Grid Choke", "TempGridChoke", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorCoolingUnit":    MappingColumn{"Temp Generator Cooling Unit", "TempGeneratorCoolingUnit", "Degree", -10.0, 200.0, "$avg"},
		"TempConvCabinet2":            MappingColumn{"Temp Conv Cabinet 2", "TempConvCabinet2", "Degree", -10.0, 200.0, "$avg"},
		"TempOutdoor":                 MappingColumn{"Temp Outdoor", "TempOutdoor", "Degree", -10.0, 200.0, "$avg"},
		"TempSlipRing":                MappingColumn{"Temp Slip Ring", "TempSlipRing", "Degree", -10.0, 200.0, "$avg"},
		"TransformerWindingTemp1":     MappingColumn{"Transformer Winding Temp 1", "TransformerWindingTemp1", "Degree", -10.0, 200.0, "$avg"},
		"TransformerWindingTemp2":     MappingColumn{"Transformer Winding Temp 2", "TransformerWindingTemp2", "Degree", -10.0, 200.0, "$avg"},
		"TransformerWindingTemp3":     MappingColumn{"Transformer Winding Temp 3", "TransformerWindingTemp3", "Degree", -10.0, 200.0, "$avg"},
		"TempShaftBearing1":           MappingColumn{"Temp Shaft Bearing 1", "TempShaftBearing1", "Degree", -10.0, 200.0, "$avg"},
		"TempShaftBearing2":           MappingColumn{"Temp Shaft Bearing 2", "TempShaftBearing2", "Degree", -10.0, 200.0, "$avg"},
		"TempShaftBearing3":           MappingColumn{"Temp Shaft Bearing 3", "TempShaftBearing3", "Degree", -10.0, 200.0, "$avg"},
		"TempGearBoxIMSDE":            MappingColumn{"Temp Gear Box IMS DE", "TempGearBoxIMSDE", "Degree", -10.0, 200.0, "$avg"},
		"TempBottomControlSection":    MappingColumn{"Temp Bottom Control Section", "TempBottomControlSection", "Degree", -10.0, 200.0, "$avg"},
		"TempBottomPowerSection":      MappingColumn{"Temp Bottom Power Section", "TempBottomPowerSection", "Degree", -10.0, 200.0, "$avg"},
		"TempCabinetTopBox":           MappingColumn{"Temp Cabinet Top Box", "TempCabinetTopBox", "Degree", -10.0, 200.0, "$avg"},
		"TempNacelle":                 MappingColumn{"Temp Nacelle", "TempNacelle", "Degree", -10.0, 200.0, "$avg"},
		"VoltageL1":                   MappingColumn{"Voltage L 1", "GridPPVPhaseAB", "Volt", -10.0, 1000.0, "$avg"},
		"VoltageL2":                   MappingColumn{"Voltage L 2", "GridPPVPhaseBC", "Volt", -10.0, 1000.0, "$avg"},
		"VoltageL3":                   MappingColumn{"Voltage L 3", "GridPPVPhaseCA", "Volt", -10.0, 1000.0, "$avg"},
		"PitchAccuV1":                 MappingColumn{"Blade Voltage V1", "PitchAccuV1", "Volt", -10.0, 1000.0, "$avg"},
		"PitchAccuV2":                 MappingColumn{"Blade Voltage V2", "PitchAccuV2", "Volt", -10.0, 1000.0, "$avg"},
		"PitchAccuV3":                 MappingColumn{"Blade Voltage V3", "PitchAccuV3", "Volt", -10.0, 1000.0, "$avg"},
	}
)

type MappingColumn struct {
	Name      string
	SecField  string
	Unit      string
	MinValue  float64
	MaxValue  float64
	Aggregate string
}

const (
	sError   = "ERROR"
	sInfo    = "INFO"
	sWarning = "WARNING"
)

func main() {
	logpath := ""
	flag.StringVar(&logpath, "log", "", "Log folder place")
	flag.Parse()
	config := ReadConfig()
	if logpath == "" {
		logpath, _ = config["logpathtimeseries"]
	}
	log, _ = tk.NewLog(false, true, logpath, "simpleHFDLog_%s", "20060102")
	ctx, e := PrepareConnection(config)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}

	tagList := []string{"_id", "timestamp", "dateinfo", "projectname", "turbine"}
	for _, val := range mapField {
		tagList = append(tagList, strings.ToLower(val.SecField))
	}

	csrLog, e := ctx.NewQuery().From("log_latestdaterun").
		Where(dbox.Eq("type", "databrowser")).Cursor(nil)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	defer csrLog.Close()
	lastData := []struct {
		ProjectName string
		LastDate    time.Time
	}{}
	e = csrLog.Fetch(&lastData, 0, false)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	lastDatePerProject := map[string]time.Time{}
	for _, val := range lastData {
		lastDatePerProject[val.ProjectName] = val.LastDate
	}

	csrProject, e := ctx.NewQuery().From("ref_project").
		Where(dbox.Eq("active", true)).Cursor(nil)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	defer csrProject.Close()
	projectList := []struct {
		ProjectID   string
		ProjectName string
	}{}
	e = csrProject.Fetch(&projectList, 0, false)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	var wgProject sync.WaitGroup
	wgProject.Add(len(projectList))

	for _, project := range projectList {
		go func(projectid string) {
			csrData, e := ctx.NewQuery().From("Scada10MinHFD").Select(tagList...).
				Where(dbox.And(
					dbox.Eq("projectname", projectid),
					dbox.Gte("timestamp", lastDatePerProject[projectid]),
					dbox.Eq("isnull", false))).
				Cursor(nil)
			if e != nil {
				log.AddLog(e.Error(), sError)
			}
			defer csrData.Close()

			maxTimeStamp := time.Time{}

			var wg sync.WaitGroup
			totalData := csrData.Count()
			totalWorker := runtime.NumCPU() * 2
			chanData := make(chan tk.M, totalData)
			step := getstep(totalData)
			tNow := time.Now()

			wg.Add(totalWorker)
			for i := 0; i < totalWorker; i++ {
				go func() {
					ctxWorker, e := PrepareConnection(config)
					if e != nil {
						log.AddLog(e.Error(), sError)
					}
					csrSave := ctxWorker.NewQuery().From("TimeSeriesHFD").SetConfig("multiexec", true).Save()
					defer csrSave.Close()
					for data := range chanData {
						if data.GetInt("count")%step == 0 {
							percent := tk.ToInt(tk.Div(float64(data.GetInt("count"))*100.0, float64(totalData)), tk.RoundingUp)
							log.AddLog(tk.Sprintf("[%s] Saving %d of %d (%d percent) in %s\n",
								strings.ToUpper(projectid), data.GetInt("count"), totalData, percent,
								time.Since(tNow).String()), sInfo)
						}
						data.Unset("count")
						csrSave.Exec(tk.M{"data": data})
					}
					wg.Done()
				}()
			}

			log.AddLog(tk.Sprintf("Processing %d data [%s] with %d step using %d CPU since %s",
				totalData, strings.ToUpper(projectid), step, totalWorker, lastDatePerProject[projectid].Format("20060102_150405")), sInfo)

			count := 0
			_data := tk.M{}
			currTimeStamp := time.Time{}
			for {
				count++
				_data = tk.M{}
				e = csrData.Fetch(&_data, 1, false)
				if e != nil {
					if !strings.Contains(e.Error(), "Not found") {
						log.AddLog(e.Error(), sError)
					}
					break
				}
				currTimeStamp = _data.Get("timestamp", time.Time{}).(time.Time).UTC()
				if currTimeStamp.After(maxTimeStamp) {
					maxTimeStamp = currTimeStamp
				}

				_data.Set("count", count)
				chanData <- _data

				// if count%step == 0 {
				// 	log.AddLog(tk.Sprintf("Processing %d of %d in %s\n",
				// 		count, totalData,
				// 		time.Since(tNow).String()), sInfo)
				// }
			}
			close(chanData)
			wg.Wait()

			if maxTimeStamp.Year() > 1 {
				e = ctx.NewQuery().From("log_latestdaterun").Save().
					Exec(tk.M{"data": tk.M{
						"_id":         "timeseries_hfd_" + projectid,
						"lastdate":    maxTimeStamp,
						"projectname": projectid,
						"type":        "timeseries",
					}})
				if e != nil {
					log.AddLog(e.Error(), sError)
				}
			}
		}(project.ProjectID)
	}
}

func getstep(count int) int {
	v := count / 5
	if v == 0 {
		return 1
	}
	return v
}

func PrepareConnection(config map[string]string) (dbox.IConnection, error) {
	ci := &dbox.ConnectionInfo{config["host"], config["database"], config["username"], config["password"], nil}
	c, e := dbox.NewConnection("mongo", ci)

	if e != nil {
		return nil, e
	}

	e = c.Connect()
	if e != nil {
		return nil, e
	}
	// log.AddLog(tk.Sprintf("DB Connect %s : %s", config["host"], config["database"]), sInfo)
	return c, nil
}
<commit_msg>bug fixing<commit_after>package main

import (
	. "eaciit/wfdemo-git/library/helper"
	"flag"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	log      *tk.LogEngine
	mapField = map[string]MappingColumn{
		"windspeed":                   MappingColumn{"Wind Speed", "WindSpeed_ms", "m/s", 0.0, 50.0, "$avg"},
		"power":                       MappingColumn{"Power", "ActivePower_kW", "kW", -200, 2100.0 + (2100.0 * 0.10), "$sum"},
		"production":                  MappingColumn{"Production", "Production", "kWh", -200, 2100.0, "$sum"},
		"winddirection":               MappingColumn{"Wind Direction", "WindDirection", "Degree", 0.0, 360.0, "$avg"},
		"nacellepos":                  MappingColumn{"Nacelle Direction", "NacellePos", "Degree", 0.0, 360.0, "$avg"},
		"rotorrpm":                    MappingColumn{"Rotor RPM", "RotorSpeed_RPM", "RPM", 0.0, 30.0, "$avg"},
		"genrpm":                      MappingColumn{"Generator RPM", "GenSpeed_RPM", "RPM", 0.0, 30.0, "$avg"},
		"pitchangle":                  MappingColumn{"Pitch Angle", "PitchAngle", "Degree", -10.0, 120.0, "$avg"},
		"PitchCabinetTempBlade1":      MappingColumn{"Pitch Cabinet Temp Blade 1", "PitchCabinetTempBlade1", "Degree", -10.0, 120.0, "$avg"},
		"PitchCabinetTempBlade2":      MappingColumn{"Pitch Cabinet Temp Blade 2", "PitchCabinetTempBlade2", "Degree", -10.0, 120.0, "$avg"},
		"PitchCabinetTempBlade3":      MappingColumn{"Pitch Cabinet Temp Blade 3", "PitchCabinetTempBlade3", "Degree", -10.0, 120.0, "$avg"},
		"PitchConvInternalTempBlade1": MappingColumn{"Pitch Conv Internal Temp Blade 1", "PitchConvInternalTempBlade1", "Degree", -10.0, 120.0, "$avg"},
		"PitchConvInternalTempBlade2": MappingColumn{"Pitch Conv Internal Temp Blade 2", "PitchConvInternalTempBlade2", "Degree", -10.0, 120.0, "$avg"},
		"PitchConvInternalTempBlade3": MappingColumn{"Pitch Conv Internal Temp Blade 3", "PitchConvInternalTempBlade3", "Degree", -10.0, 120.0, "$avg"},
		"TempG1L1":                    MappingColumn{"TempG1 L1", "TempG1L1", "Degree", -10.0, 200.0, "$avg"},
		"TempG1L2":                    MappingColumn{"TempG1 L2", "TempG1L2", "Degree", -10.0, 200.0, "$avg"},
		"TempG1L3":                    MappingColumn{"TempG1 L3", "TempG1L3", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorBearingDE":      MappingColumn{"Temp Generator Bearing DE", "TempGeneratorBearingDE", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorBearingNDE":     MappingColumn{"Temp Generator Bearing NDE", "TempGeneratorBearingNDE", "Degree", -10.0, 200.0, "$avg"},
		"TempGearBoxOilSump":          MappingColumn{"Temp Gear Box Oil Sump", "TempGearBoxOilSump", "Degree", -10.0, 200.0, "$avg"},
		"TempHubBearing":              MappingColumn{"Temp Hub Bearing", "TempHubBearing", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorChoke":          MappingColumn{"Temp Generator Choke", "TempGeneratorChoke", "Degree", -10.0, 200.0, "$avg"},
		"TempGridChoke":               MappingColumn{"Temp Grid Choke", "TempGridChoke", "Degree", -10.0, 200.0, "$avg"},
		"TempGeneratorCoolingUnit":    MappingColumn{"Temp Generator Cooling Unit", "TempGeneratorCoolingUnit", "Degree", -10.0, 200.0, "$avg"},
		"TempConvCabinet2":            MappingColumn{"Temp Conv Cabinet 2", "TempConvCabinet2", "Degree", -10.0, 200.0, "$avg"},
		"TempOutdoor":                 MappingColumn{"Temp Outdoor", "TempOutdoor", "Degree", -10.0, 200.0, "$avg"},
		"TempSlipRing":                MappingColumn{"Temp Slip Ring", "TempSlipRing", "Degree", -10.0, 200.0, "$avg"},
		"TransformerWindingTemp1":     MappingColumn{"Transformer Winding Temp 1", "TransformerWindingTemp1", "Degree", -10.0, 200.0, "$avg"},
		"TransformerWindingTemp2":     MappingColumn{"Transformer Winding Temp 2", "TransformerWindingTemp2", "Degree", -10.0, 200.0, "$avg"},
		"TransformerWindingTemp3":     MappingColumn{"Transformer Winding Temp 3", "TransformerWindingTemp3", "Degree", -10.0, 200.0, "$avg"},
		"TempShaftBearing1":           MappingColumn{"Temp Shaft Bearing 1", "TempShaftBearing1", "Degree", -10.0, 200.0, "$avg"},
		"TempShaftBearing2":           MappingColumn{"Temp Shaft Bearing 2", "TempShaftBearing2", "Degree", -10.0, 200.0, "$avg"},
		"TempShaftBearing3":           MappingColumn{"Temp Shaft Bearing 3", "TempShaftBearing3", "Degree", -10.0, 200.0, "$avg"},
		"TempGearBoxIMSDE":            MappingColumn{"Temp Gear Box IMS DE", "TempGearBoxIMSDE", "Degree", -10.0, 200.0, "$avg"},
		"TempBottomControlSection":    MappingColumn{"Temp Bottom Control Section", "TempBottomControlSection", "Degree", -10.0, 200.0, "$avg"},
		"TempBottomPowerSection":      MappingColumn{"Temp Bottom Power Section", "TempBottomPowerSection", "Degree", -10.0, 200.0, "$avg"},
		"TempCabinetTopBox":           MappingColumn{"Temp Cabinet Top Box", "TempCabinetTopBox", "Degree", -10.0, 200.0, "$avg"},
		"TempNacelle":                 MappingColumn{"Temp Nacelle", "TempNacelle", "Degree", -10.0, 200.0, "$avg"},
		"VoltageL1":                   MappingColumn{"Voltage L 1", "GridPPVPhaseAB", "Volt", -10.0, 1000.0, "$avg"},
		"VoltageL2":                   MappingColumn{"Voltage L 2", "GridPPVPhaseBC", "Volt", -10.0, 1000.0, "$avg"},
		"VoltageL3":                   MappingColumn{"Voltage L 3", "GridPPVPhaseCA", "Volt", -10.0, 1000.0, "$avg"},
		"PitchAccuV1":                 MappingColumn{"Blade Voltage V1", "PitchAccuV1", "Volt", -10.0, 1000.0, "$avg"},
		"PitchAccuV2":                 MappingColumn{"Blade Voltage V2", "PitchAccuV2", "Volt", -10.0, 1000.0, "$avg"},
		"PitchAccuV3":                 MappingColumn{"Blade Voltage V3", "PitchAccuV3", "Volt", -10.0, 1000.0, "$avg"},
	}
)

type MappingColumn struct {
	Name      string
	SecField  string
	Unit      string
	MinValue  float64
	MaxValue  float64
	Aggregate string
}

const (
	sError   = "ERROR"
	sInfo    = "INFO"
	sWarning = "WARNING"
)

func main() {
	logpath := ""
	flag.StringVar(&logpath, "log", "", "Log folder place")
	flag.Parse()
	config := ReadConfig()
	if logpath == "" {
		logpath, _ = config["logpathtimeseries"]
	}
	log, _ = tk.NewLog(false, true, logpath, "simpleHFDLog_%s", "20060102")
	ctx, e := PrepareConnection(config)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}

	tagList := []string{"_id", "timestamp", "dateinfo", "projectname", "turbine"}
	for _, val := range mapField {
		tagList = append(tagList, strings.ToLower(val.SecField))
	}

	csrLog, e := ctx.NewQuery().From("log_latestdaterun").
		Where(dbox.Eq("type", "timeseries")).Cursor(nil)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	defer csrLog.Close()
	lastData := []struct {
		ProjectName string
		LastDate    time.Time
	}{}
	e = csrLog.Fetch(&lastData, 0, false)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	lastDatePerProject := map[string]time.Time{}
	for _, val := range lastData {
		lastDatePerProject[val.ProjectName] = val.LastDate
	}

	csrProject, e := ctx.NewQuery().From("ref_project").
		Where(dbox.Eq("active", true)).Cursor(nil)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	defer csrProject.Close()
	projectList := []struct {
		ProjectID   string
		ProjectName string
	}{}
	e = csrProject.Fetch(&projectList, 0, false)
	if e != nil {
		log.AddLog(e.Error(), sError)
	}
	var wgProject sync.WaitGroup
	wgProject.Add(len(projectList))

	for _, project := range projectList {
		go func(projectid string) {
			csrData, e := ctx.NewQuery().From("Scada10MinHFD").Select(tagList...).
				Where(dbox.And(
					dbox.Eq("projectname", projectid),
					dbox.Gte("timestamp", lastDatePerProject[projectid]),
					dbox.Eq("isnull", false))).
				Cursor(nil)
			if e != nil {
				log.AddLog(e.Error(), sError)
			}
			defer csrData.Close()

			maxTimeStamp := time.Time{}

			var wg sync.WaitGroup
			totalData := csrData.Count()
			totalWorker := runtime.NumCPU() * 2
			chanData := make(chan tk.M, totalData)
			step := getstep(totalData)
			tNow := time.Now()

			wg.Add(totalWorker)
			for i := 0; i < totalWorker; i++ {
				go func() {
					ctxWorker, e := PrepareConnection(config)
					if e != nil {
						log.AddLog(e.Error(), sError)
					}
					csrSave := ctxWorker.NewQuery().From("TimeSeriesHFD").SetConfig("multiexec", true).Save()
					defer csrSave.Close()
					for data := range chanData {
						if data.GetInt("count")%step == 0 {
							percent := tk.ToInt(tk.Div(float64(data.GetInt("count"))*100.0, float64(totalData)), tk.RoundingUp)
							log.AddLog(tk.Sprintf("[%s] Saving %d of %d (%d percent) in %s\n",
								strings.ToUpper(projectid), data.GetInt("count"), totalData, percent,
								time.Since(tNow).String()), sInfo)
						}
						data.Unset("count")
						csrSave.Exec(tk.M{"data": data})
					}
					wg.Done()
				}()
			}

			log.AddLog(tk.Sprintf("Processing %d data [%s] with %d step using %d CPU since %s",
				totalData, strings.ToUpper(projectid), step, totalWorker, lastDatePerProject[projectid].Format("20060102_150405")), sInfo)

			count := 0
			_data := tk.M{}
			currTimeStamp := time.Time{}
			for {
				count++
				_data = tk.M{}
				e = csrData.Fetch(&_data, 1, false)
				if e != nil {
					if !strings.Contains(e.Error(), "Not found") {
						log.AddLog(e.Error(), sError)
					}
					break
				}
				currTimeStamp = _data.Get("timestamp", time.Time{}).(time.Time).UTC()
				if currTimeStamp.After(maxTimeStamp) {
					maxTimeStamp = currTimeStamp
				}

				_data.Set("count", count)
				chanData <- _data

				// if count%step == 0 {
				// 	log.AddLog(tk.Sprintf("Processing %d of %d in %s\n",
				// 		count, totalData,
				// 		time.Since(tNow).String()), sInfo)
				// }
			}
			close(chanData)
			wg.Wait()

			if maxTimeStamp.Year() > 1 {
				e = ctx.NewQuery().From("log_latestdaterun").Save().
					Exec(tk.M{"data": tk.M{
						"_id":         "timeseries_hfd_" + projectid,
						"lastdate":    maxTimeStamp,
						"projectname": projectid,
						"type":        "timeseries",
					}})
				if e != nil {
					log.AddLog(e.Error(), sError)
				}
			}
		}(project.ProjectID)
	}
}

func getstep(count int) int {
	v := count / 5
	if v == 0 {
		return 1
	}
	return v
}

func PrepareConnection(config map[string]string) (dbox.IConnection, error) {
	ci := &dbox.ConnectionInfo{config["host"], config["database"], config["username"], config["password"], nil}
	c, e := dbox.NewConnection("mongo", ci)

	if e != nil {
		return nil, e
	}

	e = c.Connect()
	if e != nil {
		return nil, e
	}
	// log.AddLog(tk.Sprintf("DB Connect %s : %s", config["host"], config["database"]), sInfo)
	return c, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
// static inline void incref(PyObject *obj) { Py_INCREF(obj); }
// static inline void decref(PyObject *obj) { Py_DECREF(obj); }
// static inline void xdecref(PyObject *obj) { Py_XDECREF(obj); }
import "C"

import (
	"fmt"
	"os"
	"unsafe"
)

type Error struct {
	kind *C.PyObject
	tb   *C.PyObject
	msg  string
}

func (e *Error) String() string {
	return fmt.Sprintf("<TYPE>: %s", e.msg)
}

func exception() os.Error {
	if C.PyErr_Occurred() == nil {
		return nil
	}

	var t, v, tb *C.PyObject

	defer C.xdecref(v)

	C.PyErr_Fetch(&t, &v, &tb)

	s := C.PyObject_Str(v)
	defer C.decref(s)

	return &Error{t, tb, C.GoString(C.PyString_AsString(s))}
}

func raise(err os.Error) {
	var msg string
	var exc = C.PyExc_Exception

	e, ok := err.(*Error)
	if ok {
		exc = e.kind
		msg = e.msg
	} else {
		msg = err.String()
	}

	s := C.CString(msg)
	defer C.free(unsafe.Pointer(s))

	C.PyErr_SetString(exc, s)
}

func Err_Format(f string, args ...interface{}) {
	s := C.CString(fmt.Sprintf(f, args))
	defer C.free(unsafe.Pointer(s))
	C.PyErr_SetString(C.PyExc_Exception, s)
}

func TypeError(format string, args ...interface{}) os.Error {
	msg := fmt.Sprintf(format, args...)
	C.incref(C.PyExc_TypeError)
	return &Error{C.PyExc_TypeError, nil, msg}
}

func KeyError(format string, args ...interface{}) os.Error {
	msg := fmt.Sprintf(format, args...)
	C.incref(C.PyExc_KeyError)
	return &Error{C.PyExc_KeyError, nil, msg}
}

func int2Err(i C.int) os.Error {
	if i < 0 {
		return exception()
	}
	return nil
}

func int2BoolErr(i C.int) (bool, os.Error) {
	if i < 0 {
		return false, exception()
	}
	return i > 0, nil
}

func ssize_t2Int64Err(s C.Py_ssize_t) (int64, os.Error) {
	if s < 0 {
		return 0, exception()
	}
	return int64(s), nil
}

func obj2ObjErr(obj *C.PyObject) (Object, os.Error) {
	if obj == nil {
		return nil, exception()
	}
	return newBaseObject(obj).actual(), nil
}
<commit_msg>Add AttributeError function<commit_after>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
// static inline void incref(PyObject *obj) { Py_INCREF(obj); }
// static inline void decref(PyObject *obj) { Py_DECREF(obj); }
// static inline void xdecref(PyObject *obj) { Py_XDECREF(obj); }
import "C"

import (
	"fmt"
	"os"
	"unsafe"
)

type Error struct {
	kind *C.PyObject
	tb   *C.PyObject
	msg  string
}

func (e *Error) String() string {
	return fmt.Sprintf("<TYPE>: %s", e.msg)
}

func exception() os.Error {
	if C.PyErr_Occurred() == nil {
		return nil
	}

	var t, v, tb *C.PyObject

	defer C.xdecref(v)

	C.PyErr_Fetch(&t, &v, &tb)

	s := C.PyObject_Str(v)
	defer C.decref(s)

	return &Error{t, tb, C.GoString(C.PyString_AsString(s))}
}

func raise(err os.Error) {
	var msg string
	var exc = C.PyExc_Exception

	e, ok := err.(*Error)
	if ok {
		exc = e.kind
		msg = e.msg
	} else {
		msg = err.String()
	}

	s := C.CString(msg)
	defer C.free(unsafe.Pointer(s))

	C.PyErr_SetString(exc, s)
}

func Err_Format(f string, args ...interface{}) {
	s := C.CString(fmt.Sprintf(f, args))
	defer C.free(unsafe.Pointer(s))
	C.PyErr_SetString(C.PyExc_Exception, s)
}

func TypeError(format string, args ...interface{}) os.Error {
	msg := fmt.Sprintf(format, args...)
	C.incref(C.PyExc_TypeError)
	return &Error{C.PyExc_TypeError, nil, msg}
}

func KeyError(format string, args ...interface{}) os.Error {
	msg := fmt.Sprintf(format, args...)
	C.incref(C.PyExc_KeyError)
	return &Error{C.PyExc_KeyError, nil, msg}
}

func AttributeError(format string, args ...interface{}) os.Error {
	msg := fmt.Sprintf(format, args...)
	C.incref(C.PyExc_AttributeError)
	return &Error{C.PyExc_AttributeError, nil, msg}
}

func int2Err(i C.int) os.Error {
	if i < 0 {
		return exception()
	}
	return nil
}

func int2BoolErr(i C.int) (bool, os.Error) {
	if i < 0 {
		return false, exception()
	}
	return i > 0, nil
}

func ssize_t2Int64Err(s C.Py_ssize_t) (int64, os.Error) {
	if s < 0 {
		return 0, exception()
	}
	return int64(s), nil
}

func obj2ObjErr(obj *C.PyObject) (Object, os.Error) {
	if obj == nil {
		return nil, exception()
	}
	return newBaseObject(obj).actual(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"math/rand"
	"time"
	"github.com/bertbaron/solve"
)

const (
	N = 8
)

type direction int

const (
	up direction = iota
	down direction = iota
	left direction = iota
	right direction = iota
)

func (d direction) String() string {
	switch d {
	case up: return "UP"
	case down: return "DOWN"
	case left: return "LEFT"
	case right: return "RIGHT"
	}
	panic(fmt.Sprintf("Invalid direction: %d", d))
}

type puzzleContext struct {
	width    int
	height   int
	solution [N][N]byte
}

type puzzleState struct {
	context *puzzleContext
	board   [N][N]byte
	x, y    int
	cost    float64
}

func initPuzzle(width, height int) puzzleState {
	var state puzzleState
	var value byte
	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			value++
			state.board[y][x] = value
		}
	}
	state.x = width - 1
	state.y = height - 1
	state.board[state.y][state.x] = 0
	state.context = &puzzleContext{width, height, state.board}
	return state
}

func byte2string(b byte) string {
	if b == 0 {
		return "  "
	}
	return fmt.Sprintf("%2d", b)
}

func (p puzzleState) draw() string {
	s := ""
	for y := 0; y < p.context.height; y++ {
		values := make([]string, p.context.width)
		for x := 0; x < p.context.height; x++ {
			values[x] = byte2string(p.board[y][x])
		}
		s += strings.Join(values, " ") + "\n"
	}
	return s
}

func move(p puzzleState, d direction) *puzzleState {
	x, y := p.x, p.y
	switch d {
	case up: y--
	case down: y++
	case left: x--
	case right: x++
	}
	if x < 0 || y < 0 || x >= p.context.width || y >= p.context.height {
		return nil
	}
	nw := p
	nw.board[p.y][p.x] = nw.board[y][x]
	nw.board[y][x] = 0
	nw.x = x
	nw.y = y
	return &nw
}

func shuffle(p puzzleState, shuffles int) puzzleState {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for i := 0; i < shuffles; i++ {
		dir := direction(r.Intn(4))
		if nw := move(p, dir); nw != nil {
			p = *nw
		}

	}
	return p
}

/*
 Implementation of solve.State
 */

func (p puzzleState) Cost() float64 {
	return p.cost
}

func (p puzzleState) Expand() []solve.State {
	children := make([]solve.State, 0)
	for d := 0; d < 4; d++ {
		if child := move(p, direction(d)); child != nil {
			child.cost = p.cost + 1
			children = append(children, *child)
		}
	}
	return children
}

func (p puzzleState) IsGoal() bool {
	return p.board == p.context.solution
}

func (p puzzleState) Heuristic() float64 {
	return 0
}

func (p puzzleState) Id() interface{} {
	return p.board
}

func main() {
	puzzle := shuffle(initPuzzle(3, 3), 1000)
	fmt.Println("Solving the puzzle:")
	fmt.Print(puzzle.draw())
	fmt.Println()
	result := solve.NewSolver(puzzle).
		Algorithm(solve.IDAstar).
		Constraint(solve.CHEAPEST_PATH).
		Solve()
	fmt.Printf("Result: %v", result)
}<commit_msg>A star puzzle complete, with simple manhattan distance heuristic. Performance can probably be improved.<commit_after>package main

import (
	"fmt"
	"strings"
	"math/rand"
	"github.com/bertbaron/solve"
	"math"
	"os"
	"log"
	"runtime/pprof"
)

const (
	N = 4
)

type direction byte

const (
	up direction = iota
	down direction = iota
	left direction = iota
	right direction = iota
)

func (d direction) String() string {
	switch d {
	case up: return "↑"
	case down: return "↓"
	case left: return "←"
	case right: return "→"
	}
	panic(fmt.Sprintf("Invalid direction: %d", d))
}

type puzzleContext struct {
	width    byte
	height   byte
	solution [N][N]byte
}

type puzzleState struct {
	context *puzzleContext
	board   [N][N]byte
	cost    int16
	x, y    byte
	dir     direction
}

func initPuzzle(width, height int) puzzleState {
	var state puzzleState
	var value byte
	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			value++
			state.board[y][x] = value
		}
	}
	state.x, state.y = byte(width - 1), byte(height - 1)
	state.board[state.y][state.x] = 0
	state.context = &puzzleContext{byte(width), byte(height), state.board}
	return state
}

func byte2string(b byte) string {
	if b == 0 {
		return "  "
	}
	return fmt.Sprintf("%2d", b)
}

func (p puzzleState) draw() string {
	s := ""
	for y := 0; y < int(p.context.height); y++ {
		values := make([]string, int(p.context.width))
		for x := 0; x < int(p.context.width); x++ {
			values[x] = byte2string(p.board[y][x])
		}
		s += strings.Join(values, " ") + "\n"
	}
	return s
}

func move(p puzzleState, d direction) *puzzleState {
	x, y := p.x, p.y
	switch d {
	case up: y--
	case down: y++
	case left: x--
	case right: x++
	}
	if x < 0 || y < 0 || x >= p.context.width || y >= p.context.height {
		return nil
	}
	nw := p
	nw.board[p.y][p.x], nw.board[y][x] = nw.board[y][x], 0
	nw.x, nw.y, nw.dir = x, y, d
	return &nw
}

func shuffle(seed int64, p puzzleState, shuffles int) puzzleState {
	r := rand.New(rand.NewSource(seed))
	for i := 0; i < shuffles; i++ {
		dir := direction(r.Intn(4))
		if nw := move(p, dir); nw != nil {
			p = *nw
		}

	}
	return p
}

/*
 Implementation of solve.State
 */

func (p puzzleState) Cost() float64 {
	return float64(p.cost)
}

func (p puzzleState) Expand() []solve.State {
	children := make([]solve.State, 0)
	for d := 0; d < 4; d++ {
		if child := move(p, direction(d)); child != nil {
			child.cost += 1
			children = append(children, *child)
		}
	}
	return children
}

func (p puzzleState) IsGoal() bool {
	return p.board == p.context.solution
}

func manhattanDistance(w, h, x, y, value int) float64 {
	if value == 0 {
		return 0
	}
	xx, yy := value % w, value / w
	return math.Abs(float64(xx - x)) + math.Abs(float64(yy - y))
}

func (p puzzleState) Heuristic() float64 {
	md := 0.0
	w, h := int(p.context.width), int(p.context.height)
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			md += (manhattanDistance(w, h, x, y, int(p.board[y][x])))
		}
	}
	return md
}

func (p puzzleState) Id() interface{} {
	return p.board
}

func main() {
	f, err := os.Create("cpu.prof")
	if err != nil {
		log.Fatal(err)
	}
	pprof.StartCPUProfile(f)
	defer pprof.StopCPUProfile()

	//seed := time.Now().UnixNano()
	var seed int64 = 3
	puzzle := shuffle(seed, initPuzzle(3, 3), 1000)
	fmt.Println("Solving the puzzle:")
	fmt.Print(puzzle.draw())
	fmt.Println()
	result := solve.NewSolver(puzzle).
		Algorithm(solve.IDAstar).
		Constraint(solve.NO_LOOP).
		Solve()
	n := len(result.Solution)
	if n == 0 {
		fmt.Println("No solution found")
	} else {
		moves := make([]string, n - 1)
		for i, state := range result.Solution[1:] {
			moves[i] = state.(puzzleState).dir.String()
		}
		fmt.Printf("Solution in %v steps: %s\n", result.Solution[n - 1].Cost(), strings.Join(moves, ", "))
		fmt.Printf("visited %d nodes\n", result.Visited)
	}
}<|endoftext|>