<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"test_service"

	vsecurity "v.io/x/ref/security"

	"v.io/v23"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/vdl"
	"v.io/v23/vom"
)

type invokeMethWCavIdImpl struct{}

var _ test_service.InvokeMethodWithCaveatedIdentityServerMethods = (*invokeMethWCavIdImpl)(nil)

func NewInvokeMethodWithCaveatedIdentityServer() test_service.InvokeMethodWithCaveatedIdentityServerMethods {
	return &invokeMethWCavIdImpl{}
}

// Invoke is a method on the InvokeMethodWithCaveatedIdentity service that
// invokes "AMethod" on the service with the provided name with an identity
// blessed with a caveat with the provided CaveatDescriptor.
func (i *invokeMethWCavIdImpl) Invoke(call rpc.ServerCall, name string, cavDesc security.CaveatDescriptor, cavParam *vdl.Value) error {
	ctx := call.Context()

	bytes, err := vom.Encode(cavParam)
	if err != nil {
		return err
	}

	cav := security.Caveat{
		Id:       cavDesc.Id,
		ParamVom: bytes,
	}
	p := v23.GetPrincipal(ctx)
	other, _ := security.RemoteBlessingNames(ctx)
	sharedWithOther := p.BlessingStore().ForPeer(other...)

	pWithCaveats, err := vsecurity.NewPrincipal()
	if err != nil {
		return err
	}
	// The "child" extension below is necessary for the blessing to be authorized
	// at the JavaScript server (which uses the default authorization policy).
	b, err := p.Bless(pWithCaveats.PublicKey(), sharedWithOther, "child", cav)
	if err != nil {
		return err
	}
	if err := vsecurity.SetDefaultBlessings(pWithCaveats, b); err != nil {
		return err
	}

	client := test_service.InvokableTestMethodClient(name)
	ctxWithCaveats, err := v23.SetPrincipal(ctx, pWithCaveats)
	if err != nil {
		return err
	}

	str, err := client.AMethod(ctxWithCaveats)
	if err != nil {
		return err
	}
	if str != "aResult" {
		return fmt.Errorf("Got wrong result %q", str)
	}
	return nil
}
<commit_msg>js.core: Restructure security packages<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"test_service"

	vsecurity "v.io/x/ref/lib/security"

	"v.io/v23"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/vdl"
	"v.io/v23/vom"
)

type invokeMethWCavIdImpl struct{}

var _ test_service.InvokeMethodWithCaveatedIdentityServerMethods = (*invokeMethWCavIdImpl)(nil)

func NewInvokeMethodWithCaveatedIdentityServer() test_service.InvokeMethodWithCaveatedIdentityServerMethods {
	return &invokeMethWCavIdImpl{}
}

// Invoke is a method on the InvokeMethodWithCaveatedIdentity service that
// invokes "AMethod" on the service with the provided name with an identity
// blessed with a caveat with the provided CaveatDescriptor.
func (i *invokeMethWCavIdImpl) Invoke(call rpc.ServerCall, name string, cavDesc security.CaveatDescriptor, cavParam *vdl.Value) error {
	ctx := call.Context()

	bytes, err := vom.Encode(cavParam)
	if err != nil {
		return err
	}

	cav := security.Caveat{
		Id:       cavDesc.Id,
		ParamVom: bytes,
	}
	p := v23.GetPrincipal(ctx)
	other, _ := security.RemoteBlessingNames(ctx)
	sharedWithOther := p.BlessingStore().ForPeer(other...)

	pWithCaveats, err := vsecurity.NewPrincipal()
	if err != nil {
		return err
	}
	// The "child" extension below is necessary for the blessing to be authorized
	// at the JavaScript server (which uses the default authorization policy).
	b, err := p.Bless(pWithCaveats.PublicKey(), sharedWithOther, "child", cav)
	if err != nil {
		return err
	}
	if err := vsecurity.SetDefaultBlessings(pWithCaveats, b); err != nil {
		return err
	}

	client := test_service.InvokableTestMethodClient(name)
	ctxWithCaveats, err := v23.SetPrincipal(ctx, pWithCaveats)
	if err != nil {
		return err
	}

	str, err := client.AMethod(ctxWithCaveats)
	if err != nil {
		return err
	}
	if str != "aResult" {
		return fmt.Errorf("Got wrong result %q", str)
	}
	return nil
}
<|endoftext|>


<commit_before>package data

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/flynn/flynn/controller/schema"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/pkg/postgres"
	"github.com/flynn/flynn/pkg/random"
	"github.com/flynn/que-go"
	"github.com/jackc/pgx"
)

type DeploymentRepo struct {
	db            *postgres.DB
	q             *que.Client
	appRepo       *AppRepo
	releaseRepo   *ReleaseRepo
	formationRepo *FormationRepo
}

func NewDeploymentRepo(db *postgres.DB, appRepo *AppRepo, releaseRepo *ReleaseRepo, formationRepo *FormationRepo) *DeploymentRepo {
	q := que.NewClient(db.ConnPool)
	return &DeploymentRepo{db: db, q: q, appRepo: appRepo, releaseRepo: releaseRepo, formationRepo: formationRepo}
}

func (r *DeploymentRepo) Add(appID, releaseID string) (*ct.Deployment, error) {
	ed, err := r.AddExpanded(appID, releaseID)
	if err != nil {
		return nil, err
	}
	var oldReleaseID string
	if ed.OldRelease != nil {
		oldReleaseID = ed.OldRelease.ID
	}
	return &ct.Deployment{
		ID:            ed.ID,
		AppID:         ed.AppID,
		OldReleaseID:  oldReleaseID,
		NewReleaseID:  ed.NewRelease.ID,
		Strategy:      ed.Strategy,
		Status:        ed.Status,
		Processes:     ed.Processes,
		Tags:          ed.Tags,
		DeployTimeout: ed.DeployTimeout,
		CreatedAt:     ed.CreatedAt,
		FinishedAt:    ed.FinishedAt,
	}, nil
}

func (r *DeploymentRepo) AddExpanded(appID, releaseID string) (*ct.ExpandedDeployment, error) {
	tx, err := r.db.Begin()
	if err != nil {
		return nil, err
	}

	app, err := r.appRepo.TxGet(tx, appID)
	if err != nil {
		if err == ErrNotFound {
			err = ct.ValidationError{
				Message: fmt.Sprintf("could not find app with ID %s", appID),
			}
		}
		tx.Rollback()
		return nil, err
	}

	release, err := r.releaseRepo.TxGet(tx, releaseID)
	if err != nil {
		if err == ErrNotFound {
			err = ct.ValidationError{
				Message: fmt.Sprintf("could not find release with ID %s", releaseID),
			}
		}
		tx.Rollback()
		return nil, err
	}

	oldRelease, err := r.appRepo.TxGetRelease(tx, app.ID)
	if err == ErrNotFound {
		oldRelease = nil
	} else if err != nil {
		tx.Rollback()
		return nil, err
	}

	oldFormation := &ct.Formation{}
	if oldRelease != nil {
		f, err := r.formationRepo.TxGet(tx, app.ID, oldRelease.ID)
		if err == nil {
			oldFormation = f
		} else if err != ErrNotFound {
			tx.Rollback()
			return nil, err
		}
	}
	procCount := 0
	for _, i := range oldFormation.Processes {
		procCount += i
	}

	releaseType := ct.ReleaseTypeCode
	if oldRelease != nil {
		if artifactsEqual(oldRelease.ArtifactIDs, release.ArtifactIDs) {
			releaseType = ct.ReleaseTypeConfig
		}
	} else if len(release.ArtifactIDs) == 0 {
		releaseType = ct.ReleaseTypeConfig
	}

	ed := &ct.ExpandedDeployment{
		AppID:         app.ID,
		NewRelease:    release,
		Type:          releaseType,
		Strategy:      app.Strategy,
		OldRelease:    oldRelease,
		Processes:     oldFormation.Processes,
		Tags:          oldFormation.Tags,
		DeployTimeout: app.DeployTimeout,
	}

	d := &ct.Deployment{
		AppID:           app.ID,
		NewReleaseID:    release.ID,
		Strategy:        app.Strategy,
		Processes:       oldFormation.Processes,
		Tags:            oldFormation.Tags,
		DeployTimeout:   app.DeployTimeout,
		DeployBatchSize: app.DeployBatchSize(),
	}
	if oldRelease != nil {
		d.OldReleaseID = oldRelease.ID
	}

	if err := schema.Validate(d); err != nil {
		tx.Rollback()
		return nil, err
	}
	if procCount == 0 {
		// immediately set app release
		if err := r.appRepo.TxSetRelease(tx, app, release.ID); err != nil {
			tx.Rollback()
			return nil, err
		}
		now := time.Now().Truncate(time.Microsecond) // postgres only has microsecond precision
		d.FinishedAt = &now
		ed.FinishedAt = &now
	}

	var oldReleaseID *string
	if oldRelease != nil {
		oldReleaseID = &oldRelease.ID
	}
	if d.ID == "" {
		d.ID = random.UUID()
	}
	ed.ID = d.ID
	if err := tx.QueryRow("deployment_insert", d.ID, d.AppID, oldReleaseID, d.NewReleaseID, string(releaseType), d.Strategy, d.Processes, d.Tags, d.DeployTimeout, d.DeployBatchSize).Scan(&d.CreatedAt); err != nil {
		tx.Rollback()
		if postgres.IsUniquenessError(err, "isolate_deploys") {
			return nil, ct.ValidationError{Message: "Cannot create deploy, there is already one in progress for this app."}
		}
		return nil, err
	}
	ed.CreatedAt = d.CreatedAt

	// fake initial deployment
	if d.FinishedAt != nil {
		if err := tx.Exec("deployment_update_finished_at", d.ID, d.FinishedAt); err != nil {
			tx.Rollback()
			return nil, err
		}
		if err = createDeploymentEvent(tx.Exec, d, "complete"); err != nil {
			tx.Rollback()
			return nil, err
		}
		ed.Status = "complete"
		return ed, tx.Commit()
	}

	args, err := json.Marshal(ct.DeployID{ID: d.ID})
	if err != nil {
		tx.Rollback()
		return nil, err
	}

	if err = createDeploymentEvent(tx.Exec, d, "pending"); err != nil {
		tx.Rollback()
		return nil, err
	}
	ed.Status = "pending"

	job := &que.Job{Type: "deployment", Args: args}
	if err := r.q.EnqueueInTx(job, tx.Tx); err != nil {
		tx.Rollback()
		return nil, err
	}
	return ed, tx.Commit()
}

func (r *DeploymentRepo) Get(id string) (*ct.Deployment, error) {
	row := r.db.QueryRow("deployment_select", id)
	return scanDeployment(row)
}

func (r *DeploymentRepo) GetExpanded(id string) (*ct.ExpandedDeployment, error) {
	row := r.db.QueryRow("deployment_select_expanded", id)
	return scanExpandedDeployment(row)
}

func (r *DeploymentRepo) List(appID string) ([]*ct.Deployment, error) {
	rows, err := r.db.Query("deployment_list", appID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var deployments []*ct.Deployment
	for rows.Next() {
		deployment, err := scanDeployment(rows)
		if err != nil {
			return nil, err
		}
		deployments = append(deployments, deployment)
	}
	return deployments, rows.Err()
}

type ListDeploymentOptions struct {
	PageToken     PageToken
	AppIDs        []string
	DeploymentIDs []string
	StatusFilters []string
	TypeFilters   []ct.ReleaseType
}

func (r *DeploymentRepo) ListPage(opts ListDeploymentOptions) ([]*ct.ExpandedDeployment, *PageToken, error) {
	pageSize := DEFAULT_PAGE_SIZE
	if opts.PageToken.Size > 0 {
		pageSize = opts.PageToken.Size
	}
	typeFilters := make([]string, 0, len(opts.TypeFilters))
	for _, t := range opts.TypeFilters {
		if t == ct.ReleaseTypeAny {
			typeFilters = []string{}
			break
		}
		typeFilters = append(typeFilters, string(t))
	}
	rows, err := r.db.Query("deployment_list_page", opts.AppIDs, opts.DeploymentIDs, opts.StatusFilters, typeFilters, opts.PageToken.BeforeID, pageSize+1)
	if err != nil {
		return nil, nil, err
	}
	defer rows.Close()
	var deployments []*ct.ExpandedDeployment
	for rows.Next() {
		deployment, err := scanExpandedDeployment(rows)
		if err != nil {
			return nil, nil, err
		}
		deployments = append(deployments, deployment)
	}
	if err := rows.Err(); err != nil {
		return nil, nil, err
	}
	var nextPageToken *PageToken
	if len(deployments) == pageSize+1 {
		nextPageToken = &PageToken{
			BeforeID: &deployments[pageSize].ID,
			Size:     pageSize,
		}
		deployments = deployments[0:pageSize]
	}
	return deployments, nextPageToken, nil
}

func scanExpandedDeployment(s postgres.Scanner) (*ct.ExpandedDeployment, error) {
	d := &ct.ExpandedDeployment{}
	oldRelease := &ct.Release{}
	newRelease := &ct.Release{}
	var oldArtifactIDs string
	var newArtifactIDs string
	var oldReleaseID *string
	var status *string
	err := s.Scan(
		&d.ID, &d.AppID, &oldReleaseID, &newRelease.ID, &d.Strategy, &status, &d.Processes, &d.Tags, &d.DeployTimeout, &d.CreatedAt, &d.FinishedAt,
		&oldArtifactIDs, &oldRelease.Env, &oldRelease.Processes, &oldRelease.Meta, &oldRelease.CreatedAt,
		&newArtifactIDs, &newRelease.Env, &newRelease.Processes, &newRelease.Meta, &newRelease.CreatedAt,
		&d.Type,
	)
	if err == pgx.ErrNoRows {
		err = ErrNotFound
	}
	if oldReleaseID != nil {
		oldRelease.ID = *oldReleaseID
		oldRelease.AppID = d.AppID
		if oldArtifactIDs != "" {
			oldRelease.ArtifactIDs = splitPGStringArray(oldArtifactIDs)
		}
		d.OldRelease = oldRelease
	}
	if newArtifactIDs != "" {
		newRelease.ArtifactIDs = splitPGStringArray(newArtifactIDs)
	}
	newRelease.AppID = d.AppID
	d.NewRelease = newRelease
	if status != nil {
		d.Status = *status
	}
	return d, err
}

func scanDeployment(s postgres.Scanner) (*ct.Deployment, error) {
	d := &ct.Deployment{}
	var oldReleaseID *string
	var status *string
	err := s.Scan(&d.ID, &d.AppID, &oldReleaseID, &d.NewReleaseID, &d.Strategy, &status, &d.Processes, &d.Tags, &d.DeployTimeout, &d.DeployBatchSize, &d.CreatedAt, &d.FinishedAt)
	if err == pgx.ErrNoRows {
		err = ErrNotFound
	}
	if oldReleaseID != nil {
		d.OldReleaseID = *oldReleaseID
	}
	if status != nil {
		d.Status = *status
	}
	return d, err
}

func createDeploymentEvent(dbExec func(string, ...interface{}) error, d *ct.Deployment, status string) error {
	e := ct.DeploymentEvent{
		AppID:        d.AppID,
		DeploymentID: d.ID,
		ReleaseID:    d.NewReleaseID,
		Status:       status,
	}
	return CreateEvent(dbExec, &ct.Event{
		AppID:      d.AppID,
		ObjectID:   d.ID,
		ObjectType: ct.EventTypeDeployment,
		Op:         ct.EventOpCreate,
	}, e)
}

func artifactsEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	if a == nil && b != nil {
		return false
	}
	if a != nil && b == nil {
		return false
	}
	for i := 0; i < len(a); i++ {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}
<commit_msg>controller: Fix error return in scanExpandedDeployment<commit_after>package data

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/flynn/flynn/controller/schema"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/pkg/postgres"
	"github.com/flynn/flynn/pkg/random"
	"github.com/flynn/que-go"
	"github.com/jackc/pgx"
)

type DeploymentRepo struct {
	db            *postgres.DB
	q             *que.Client
	appRepo       *AppRepo
	releaseRepo   *ReleaseRepo
	formationRepo *FormationRepo
}

func NewDeploymentRepo(db *postgres.DB, appRepo *AppRepo, releaseRepo *ReleaseRepo, formationRepo *FormationRepo) *DeploymentRepo {
	q := que.NewClient(db.ConnPool)
	return &DeploymentRepo{db: db, q: q, appRepo: appRepo, releaseRepo: releaseRepo, formationRepo: formationRepo}
}

func (r *DeploymentRepo) Add(appID, releaseID string) (*ct.Deployment, error) {
	ed, err := r.AddExpanded(appID, releaseID)
	if err != nil {
		return nil, err
	}
	var oldReleaseID string
	if ed.OldRelease != nil {
		oldReleaseID = ed.OldRelease.ID
	}
	return &ct.Deployment{
		ID:            ed.ID,
		AppID:         ed.AppID,
		OldReleaseID:  oldReleaseID,
		NewReleaseID:  ed.NewRelease.ID,
		Strategy:      ed.Strategy,
		Status:        ed.Status,
		Processes:     ed.Processes,
		Tags:          ed.Tags,
		DeployTimeout: ed.DeployTimeout,
		CreatedAt:     ed.CreatedAt,
		FinishedAt:    ed.FinishedAt,
	}, nil
}

func (r *DeploymentRepo) AddExpanded(appID, releaseID string) (*ct.ExpandedDeployment, error) {
	tx, err := r.db.Begin()
	if err != nil {
		return nil, err
	}

	app, err := r.appRepo.TxGet(tx, appID)
	if err != nil {
		if err == ErrNotFound {
			err = ct.ValidationError{
				Message: fmt.Sprintf("could not find app with ID %s", appID),
			}
		}
		tx.Rollback()
		return nil, err
	}

	release, err := r.releaseRepo.TxGet(tx, releaseID)
	if err != nil {
		if err == ErrNotFound {
			err = ct.ValidationError{
				Message: fmt.Sprintf("could not find release with ID %s", releaseID),
			}
		}
		tx.Rollback()
		return nil, err
	}

	oldRelease, err := r.appRepo.TxGetRelease(tx, app.ID)
	if err == ErrNotFound {
		oldRelease = nil
	} else if err != nil {
		tx.Rollback()
		return nil, err
	}

	oldFormation := &ct.Formation{}
	if oldRelease != nil {
		f, err := r.formationRepo.TxGet(tx, app.ID, oldRelease.ID)
		if err == nil {
			oldFormation = f
		} else if err != ErrNotFound {
			tx.Rollback()
			return nil, err
		}
	}
	procCount := 0
	for _, i := range oldFormation.Processes {
		procCount += i
	}

	releaseType := ct.ReleaseTypeCode
	if oldRelease != nil {
		if artifactsEqual(oldRelease.ArtifactIDs, release.ArtifactIDs) {
			releaseType = ct.ReleaseTypeConfig
		}
	} else if len(release.ArtifactIDs) == 0 {
		releaseType = ct.ReleaseTypeConfig
	}

	ed := &ct.ExpandedDeployment{
		AppID:         app.ID,
		NewRelease:    release,
		Type:          releaseType,
		Strategy:      app.Strategy,
		OldRelease:    oldRelease,
		Processes:     oldFormation.Processes,
		Tags:          oldFormation.Tags,
		DeployTimeout: app.DeployTimeout,
	}

	d := &ct.Deployment{
		AppID:           app.ID,
		NewReleaseID:    release.ID,
		Strategy:        app.Strategy,
		Processes:       oldFormation.Processes,
		Tags:            oldFormation.Tags,
		DeployTimeout:   app.DeployTimeout,
		DeployBatchSize: app.DeployBatchSize(),
	}
	if oldRelease != nil {
		d.OldReleaseID = oldRelease.ID
	}

	if err := schema.Validate(d); err != nil {
		tx.Rollback()
		return nil, err
	}
	if procCount == 0 {
		// immediately set app release
		if err := r.appRepo.TxSetRelease(tx, app, release.ID); err != nil {
			tx.Rollback()
			return nil, err
		}
		now := time.Now().Truncate(time.Microsecond) // postgres only has microsecond precision
		d.FinishedAt = &now
		ed.FinishedAt = &now
	}

	var oldReleaseID *string
	if oldRelease != nil {
		oldReleaseID = &oldRelease.ID
	}
	if d.ID == "" {
		d.ID = random.UUID()
	}
	ed.ID = d.ID
	if err := tx.QueryRow("deployment_insert", d.ID, d.AppID, oldReleaseID, d.NewReleaseID, string(releaseType), d.Strategy, d.Processes, d.Tags, d.DeployTimeout, d.DeployBatchSize).Scan(&d.CreatedAt); err != nil {
		tx.Rollback()
		if postgres.IsUniquenessError(err, "isolate_deploys") {
			return nil, ct.ValidationError{Message: "Cannot create deploy, there is already one in progress for this app."}
		}
		return nil, err
	}
	ed.CreatedAt = d.CreatedAt

	// fake initial deployment
	if d.FinishedAt != nil {
		if err := tx.Exec("deployment_update_finished_at", d.ID, d.FinishedAt); err != nil {
			tx.Rollback()
			return nil, err
		}
		if err = createDeploymentEvent(tx.Exec, d, "complete"); err != nil {
			tx.Rollback()
			return nil, err
		}
		ed.Status = "complete"
		return ed, tx.Commit()
	}

	args, err := json.Marshal(ct.DeployID{ID: d.ID})
	if err != nil {
		tx.Rollback()
		return nil, err
	}

	if err = createDeploymentEvent(tx.Exec, d, "pending"); err != nil {
		tx.Rollback()
		return nil, err
	}
	ed.Status = "pending"

	job := &que.Job{Type: "deployment", Args: args}
	if err := r.q.EnqueueInTx(job, tx.Tx); err != nil {
		tx.Rollback()
		return nil, err
	}
	return ed, tx.Commit()
}

func (r *DeploymentRepo) Get(id string) (*ct.Deployment, error) {
	row := r.db.QueryRow("deployment_select", id)
	return scanDeployment(row)
}

func (r *DeploymentRepo) GetExpanded(id string) (*ct.ExpandedDeployment, error) {
	row := r.db.QueryRow("deployment_select_expanded", id)
	return scanExpandedDeployment(row)
}

func (r *DeploymentRepo) List(appID string) ([]*ct.Deployment, error) {
	rows, err := r.db.Query("deployment_list", appID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var deployments []*ct.Deployment
	for rows.Next() {
		deployment, err := scanDeployment(rows)
		if err != nil {
			return nil, err
		}
		deployments = append(deployments, deployment)
	}
	return deployments, rows.Err()
}

type ListDeploymentOptions struct {
	PageToken     PageToken
	AppIDs        []string
	DeploymentIDs []string
	StatusFilters []string
	TypeFilters   []ct.ReleaseType
}

func (r *DeploymentRepo) ListPage(opts ListDeploymentOptions) ([]*ct.ExpandedDeployment, *PageToken, error) {
	pageSize := DEFAULT_PAGE_SIZE
	if opts.PageToken.Size > 0 {
		pageSize = opts.PageToken.Size
	}
	typeFilters := make([]string, 0, len(opts.TypeFilters))
	for _, t := range opts.TypeFilters {
		if t == ct.ReleaseTypeAny {
			typeFilters = []string{}
			break
		}
		typeFilters = append(typeFilters, string(t))
	}
	rows, err := r.db.Query("deployment_list_page", opts.AppIDs, opts.DeploymentIDs, opts.StatusFilters, typeFilters, opts.PageToken.BeforeID, pageSize+1)
	if err != nil {
		return nil, nil, err
	}
	defer rows.Close()
	var deployments []*ct.ExpandedDeployment
	for rows.Next() {
		deployment, err := scanExpandedDeployment(rows)
		if err != nil {
			return nil, nil, err
		}
		deployments = append(deployments, deployment)
	}
	if err := rows.Err(); err != nil {
		return nil, nil, err
	}
	var nextPageToken *PageToken
	if len(deployments) == pageSize+1 {
		nextPageToken = &PageToken{
			BeforeID: &deployments[pageSize].ID,
			Size:     pageSize,
		}
		deployments = deployments[0:pageSize]
	}
	return deployments, nextPageToken, nil
}

func scanExpandedDeployment(s postgres.Scanner) (*ct.ExpandedDeployment, error) {
	d := &ct.ExpandedDeployment{}
	oldRelease := &ct.Release{}
	newRelease := &ct.Release{}
	var oldArtifactIDs string
	var newArtifactIDs string
	var oldReleaseID *string
	var status *string
	err := s.Scan(
		&d.ID, &d.AppID, &oldReleaseID, &newRelease.ID, &d.Strategy, &status, &d.Processes, &d.Tags, &d.DeployTimeout, &d.CreatedAt, &d.FinishedAt,
		&oldArtifactIDs, &oldRelease.Env, &oldRelease.Processes, &oldRelease.Meta, &oldRelease.CreatedAt,
		&newArtifactIDs, &newRelease.Env, &newRelease.Processes, &newRelease.Meta, &newRelease.CreatedAt,
		&d.Type,
	)
	if err != nil {
		if err == pgx.ErrNoRows {
			err = ErrNotFound
		}
		return nil, err
	}
	if oldReleaseID != nil {
		oldRelease.ID = *oldReleaseID
		oldRelease.AppID = d.AppID
		if oldArtifactIDs != "" {
			oldRelease.ArtifactIDs = splitPGStringArray(oldArtifactIDs)
		}
		d.OldRelease = oldRelease
	}
	if newArtifactIDs != "" {
		newRelease.ArtifactIDs = splitPGStringArray(newArtifactIDs)
	}
	newRelease.AppID = d.AppID
	d.NewRelease = newRelease
	if status != nil {
		d.Status = *status
	}
	return d, err
}

func scanDeployment(s postgres.Scanner) (*ct.Deployment, error) {
	d := &ct.Deployment{}
	var oldReleaseID *string
	var status *string
	err := s.Scan(&d.ID, &d.AppID, &oldReleaseID, &d.NewReleaseID, &d.Strategy, &status, &d.Processes, &d.Tags, &d.DeployTimeout, &d.DeployBatchSize, &d.CreatedAt, &d.FinishedAt)
	if err == pgx.ErrNoRows {
		err = ErrNotFound
	}
	if oldReleaseID != nil {
		d.OldReleaseID = *oldReleaseID
	}
	if status != nil {
		d.Status = *status
	}
	return d, err
}

func createDeploymentEvent(dbExec func(string, ...interface{}) error, d *ct.Deployment, status string) error {
	e := ct.DeploymentEvent{
		AppID:        d.AppID,
		DeploymentID: d.ID,
		ReleaseID:    d.NewReleaseID,
		Status:       status,
	}
	return CreateEvent(dbExec, &ct.Event{
		AppID:      d.AppID,
		ObjectID:   d.ID,
		ObjectType: ct.EventTypeDeployment,
		Op:         ct.EventOpCreate,
	}, e)
}

func artifactsEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	if a == nil && b != nil {
		return false
	}
	if a != nil && b == nil {
		return false
	}
	for i := 0; i < len(a); i++ {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/astaxie/beego"

	"encoding/json"
	"github.com/scmo/apayment-backend/models"
	"github.com/scmo/apayment-backend/services"
)

// Operations about APaymentToken
type APaymentTokenController struct {
	beego.Controller
}

// @Title Creates a new APayment Token Transfer
// @Description Endpoint to transfer APayment Token from the System Account to the selected account
// @Param	body		body 	models.APaymentTokenTransfer	true		"body for request content"
// @Success 200 {Object} models.APaymentTokenTransfer
// @Failure 403 body is empty
// @router / [post]
func (this *APaymentTokenController) Transfer() {
	var aPaymentTokenTransfer models.APaymentTokenTransfer

	json.Unmarshal(this.Ctx.Input.RequestBody, &aPaymentTokenTransfer)

	claims, _ := services.ParseToken(this.Ctx.Request.Header.Get("Authorization"))
	user, err := services.GetUserByUsername(claims.Subject)
	if err != nil {
		this.CustomAbort(404, err.Error())
	}

	if !user.HasRole("Admin") {
		this.CustomAbort(404, "Unauthorization")
	}

	aPaymentTokenTransfer.From = beego.AppConfig.String("systemAccountAddress")

	err = services.Transfer(&aPaymentTokenTransfer, "")
	if err != nil {
		beego.Error("Error while tranfering tokens. ", err)
		this.CustomAbort(500, err.Error())
	}
	this.ServeJSON()
}

// @Title Get Transactions
// @Description get all transactions
// @Success 200 {Object} models.APaymentTokenTransaction
// @router /transactions [get]
func (this *APaymentTokenController) GetAllTransactions() {

	claims, _ := services.ParseToken(this.Ctx.Request.Header.Get("Authorization"))
	user, err := services.GetUserByUsername(claims.Subject)
	if err != nil {
		this.CustomAbort(404, err.Error())
	}

	if user.HasRole("Admin") || user.HasRole("Canton") == false {
		this.CustomAbort(401, "Unauthorized")
	}

	transactions, err := services.GetTransactions()
	if err != nil {
		beego.Error("Error while getting transactions. ", err)
		this.CustomAbort(500, err.Error())
	}
	this.Data["json"] = transactions
	this.ServeJSON()
}
<commit_msg>fix if formatting<commit_after>package controllers

import (
	"github.com/astaxie/beego"

	"encoding/json"
	"github.com/scmo/apayment-backend/models"
	"github.com/scmo/apayment-backend/services"
)

// Operations about APaymentToken
type APaymentTokenController struct {
	beego.Controller
}

// @Title Creates a new APayment Token Transfer
// @Description Endpoint to transfer APayment Token from the System Account to the selected account
// @Param	body		body 	models.APaymentTokenTransfer	true		"body for request content"
// @Success 200 {Object} models.APaymentTokenTransfer
// @Failure 403 body is empty
// @router / [post]
func (this *APaymentTokenController) Transfer() {
	var aPaymentTokenTransfer models.APaymentTokenTransfer

	json.Unmarshal(this.Ctx.Input.RequestBody, &aPaymentTokenTransfer)

	claims, _ := services.ParseToken(this.Ctx.Request.Header.Get("Authorization"))
	user, err := services.GetUserByUsername(claims.Subject)
	if err != nil {
		this.CustomAbort(404, err.Error())
	}

	if !user.HasRole("Admin") {
		this.CustomAbort(404, "Unauthorization")
	}

	aPaymentTokenTransfer.From = beego.AppConfig.String("systemAccountAddress")

	err = services.Transfer(&aPaymentTokenTransfer, "")
	if err != nil {
		beego.Error("Error while tranfering tokens. ", err)
		this.CustomAbort(500, err.Error())
	}
	this.ServeJSON()
}

// @Title Get Transactions
// @Description get all transactions
// @Success 200 {Object} models.APaymentTokenTransaction
// @router /transactions [get]
func (this *APaymentTokenController) GetAllTransactions() {

	claims, _ := services.ParseToken(this.Ctx.Request.Header.Get("Authorization"))
	user, err := services.GetUserByUsername(claims.Subject)
	if err != nil {
		this.CustomAbort(404, err.Error())
	}

	if (user.HasRole("Admin") || user.HasRole("Canton")) == false {
		this.CustomAbort(401, "Unauthorized")
	}

	transactions, err := services.GetTransactions()
	if err != nil {
		beego.Error("Error while getting transactions. ", err)
		this.CustomAbort(500, err.Error())
	}
	this.Data["json"] = transactions
	this.ServeJSON()
}
<|endoftext|>


<commit_before>package web

import (
	"fmt"
	logging "github.com/op/go-logging"
	"io"
	"mstree"
	"net"
	"net/http"
	"runtime"
	"sync/atomic"
	"time"
)

type Server struct {
	tree        *mstree.MSTree
	selfMonitor bool
}

type handlerCounters struct {
	add    uint64
	search uint64
	dump   uint64
}

type rpsCounters struct {
	add    float64
	search float64
	dump   float64
}

const (
	monitorHost = "127.0.0.1:42000"
)

var (
	log           *logging.Logger = logging.MustGetLogger("metricsearch")
	totalRequests handlerCounters
	lastRequests  handlerCounters
	rps           rpsCounters
)

func (s *Server) sendMetrics() {
	conn, err := net.Dial("tcp", monitorHost)
	if err != nil {
		return
	}
	defer conn.Close()
	ts := time.Now().Unix()
	sqs, _ := s.tree.SyncQueueSize()
	fmt.Fprintf(conn, "metricsearch.rps.add %.4f %d\n", rps.add, ts)
	fmt.Fprintf(conn, "metricsearch.rps.search %.4f %d\n", rps.search, ts)
	fmt.Fprintf(conn, "metricsearch.rps.dump %.4f %d\n", rps.dump, ts)
	fmt.Fprintf(conn, "metricsearch.reqs.add %.2f %d\n", float32(totalRequests.add), ts)
	fmt.Fprintf(conn, "metricsearch.reqs.search %.2f %d\n", float32(totalRequests.search), ts)
	fmt.Fprintf(conn, "metricsearch.reqs.dump %.2f %d\n", float32(totalRequests.dump), ts)
	fmt.Fprintf(conn, "metricsearch.metrics %.2f %d\n", float64(s.tree.TotalMetrics), ts)
	fmt.Fprintf(conn, "metricsearch.sync_queue %.2f %d\n", float64(sqs), ts)
}

func (s *Server) searchHandler(w http.ResponseWriter, r *http.Request) {
	atomic.AddUint64(&totalRequests.search, 1)
	w.Header().Set("Content-Type", "text/plain")
	r.ParseForm()
	query := r.Form.Get("query")
	tm := time.Now()
	data := s.tree.Search(query)
	dur := time.Now().Sub(tm)
	if dur > time.Millisecond {
		// slower than 1ms
		log.Debug("Searching %s took %s\n", query, dur.String())
	}
	for _, item := range data {
		io.WriteString(w, item+"\n")
	}
}

func (s *Server) addHandler(w http.ResponseWriter, r *http.Request) {
	atomic.AddUint64(&totalRequests.add, 1)
	w.Header().Set("Content-Type", "text/plain")
	r.ParseForm()
	name := r.Form.Get("name")
	if name == "" {
		w.WriteHeader(http.StatusBadRequest)
		io.WriteString(w, "Specify 'name' parameter")
		return
	}
	tm := time.Now()
	s.tree.Add(name)
	dur := time.Now().Sub(tm)
	if dur > time.Millisecond*100 {
		log.Debug("Indexing %s took %s\n", name, dur.String())
	}
	io.WriteString(w, "Ok")
}

func (s *Server) stackHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	buf := make([]byte, 65536)
	n := runtime.Stack(buf, true)
	w.Write(buf[:n])
}

func (s *Server) dumpHandler(w http.ResponseWriter, r *http.Request) {
	atomic.AddUint64(&totalRequests.dump, 1)
	w.Header().Set("Content-Type", "text/plain")
	s.tree.Root.TraverseDump("", w)
}

func (s *Server) statsHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	io.WriteString(w, "Total requests (online):\n=============================\n")
	io.WriteString(w, fmt.Sprintf("  add:    %d\n", totalRequests.add))
	io.WriteString(w, fmt.Sprintf("  search: %d\n", totalRequests.search))
	io.WriteString(w, fmt.Sprintf("  dump:   %d\n", totalRequests.dump))
	io.WriteString(w, "\n")
	io.WriteString(w, "RPS (refreshes every minute):\n=============================\n")
	io.WriteString(w, fmt.Sprintf("  add:    %.3f\n", rps.add))
	io.WriteString(w, fmt.Sprintf("  search: %.3f\n", rps.search))
	io.WriteString(w, fmt.Sprintf("  dump:   %.3f\n", rps.dump))
	io.WriteString(w, "\n")
	sqs, _ := s.tree.SyncQueueSize()
	io.WriteString(w, fmt.Sprintf("Total Metrics: %d\n", s.tree.TotalMetrics))
	io.WriteString(w, fmt.Sprintf("Sync Queue Size: %d\n", sqs))
}

func (s *Server) recalcRPS() {
	ticker := time.Tick(time.Minute)
	for _ = range ticker {
		rps.add = float64(totalRequests.add-lastRequests.add) / 60
		rps.dump = float64(totalRequests.dump-lastRequests.dump) / 60
		rps.search = float64(totalRequests.search-lastRequests.search) / 60
		lastRequests = totalRequests
		if s.selfMonitor {
			s.sendMetrics()
		}
	}
}

func NewServer(tree *mstree.MSTree, selfMonitor bool) *Server {
	server := &Server{tree, selfMonitor}
	http.HandleFunc("/search", server.searchHandler)
	http.HandleFunc("/add", server.addHandler)
	http.HandleFunc("/debug/stack", server.stackHandler)
	http.HandleFunc("/dump", server.dumpHandler)
	http.HandleFunc("/stats", server.statsHandler)
	return server
}

func (s *Server) Start(listenAddr string) {
	log.Notice("Starting background stats job")
	go s.recalcRPS()
	log.Notice("Starting HTTP")
	err := http.ListenAndServe(listenAddr, nil)
	if err != nil {
		log.Error(err.Error())
		panic(err)
	}
}
<commit_msg>hostname in self monitor<commit_after>package web

import (
	"fmt"
	logging "github.com/op/go-logging"
	"io"
	"mstree"
	"net"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync/atomic"
	"time"
)

type Server struct {
	tree        *mstree.MSTree
	selfMonitor bool
}

type handlerCounters struct {
	add    uint64
	search uint64
	dump   uint64
}

type rpsCounters struct {
	add    float64
	search float64
	dump   float64
}

const (
	monitorHost = "127.0.0.1:42000"
)

var (
	log           *logging.Logger = logging.MustGetLogger("metricsearch")
	totalRequests handlerCounters
	lastRequests  handlerCounters
	rps           rpsCounters
	selfHostname  string
)

func (s *Server) sendMetrics() {
	conn, err := net.Dial("tcp", monitorHost)
	if err != nil {
		return
	}
	defer conn.Close()
	ts := time.Now().Unix()
	sqs, _ := s.tree.SyncQueueSize()
	fmt.Fprintf(conn, "%s.metricsearch.rps.add %.4f %d\n", selfHostname, rps.add, ts)
	fmt.Fprintf(conn, "%s.metricsearch.rps.search %.4f %d\n", selfHostname, rps.search, ts)
	fmt.Fprintf(conn, "%s.metricsearch.rps.dump %.4f %d\n", selfHostname, rps.dump, ts)
	fmt.Fprintf(conn, "%s.metricsearch.reqs.add %.2f %d\n", selfHostname, float32(totalRequests.add), ts)
	fmt.Fprintf(conn, "%s.metricsearch.reqs.search %.2f %d\n", selfHostname, float32(totalRequests.search), ts)
	fmt.Fprintf(conn, "%s.metricsearch.reqs.dump %.2f %d\n", selfHostname, float32(totalRequests.dump), ts)
	fmt.Fprintf(conn, "%s.metricsearch.metrics %.2f %d\n", selfHostname, float64(s.tree.TotalMetrics), ts)
	fmt.Fprintf(conn, "%s.metricsearch.sync_queue %.2f %d\n", selfHostname, float64(sqs), ts)
}

func (s *Server) searchHandler(w http.ResponseWriter, r *http.Request) {
	atomic.AddUint64(&totalRequests.search, 1)
	w.Header().Set("Content-Type", "text/plain")
	r.ParseForm()
	query := r.Form.Get("query")
	tm := time.Now()
	data := s.tree.Search(query)
	dur := time.Now().Sub(tm)
	if dur > time.Millisecond {
		// slower than 1ms
		log.Debug("Searching %s took %s\n", query, dur.String())
	}
	for _, item := range data {
		io.WriteString(w, item+"\n")
	}
}

func (s *Server) addHandler(w http.ResponseWriter, r *http.Request) {
	atomic.AddUint64(&totalRequests.add, 1)
	w.Header().Set("Content-Type", "text/plain")
	r.ParseForm()
	name := r.Form.Get("name")
	if name == "" {
		w.WriteHeader(http.StatusBadRequest)
		io.WriteString(w, "Specify 'name' parameter")
		return
	}
	tm := time.Now()
	s.tree.Add(name)
	dur := time.Now().Sub(tm)
	if dur > time.Millisecond*100 {
		log.Debug("Indexing %s took %s\n", name, dur.String())
	}
	io.WriteString(w, "Ok")
}

func (s *Server) stackHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	buf := make([]byte, 65536)
	n := runtime.Stack(buf, true)
	w.Write(buf[:n])
}

func (s *Server) dumpHandler(w http.ResponseWriter, r *http.Request) {
	atomic.AddUint64(&totalRequests.dump, 1)
	w.Header().Set("Content-Type", "text/plain")
	s.tree.Root.TraverseDump("", w)
}

func (s *Server) statsHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	io.WriteString(w, "Total requests (online):\n=============================\n")
	io.WriteString(w, fmt.Sprintf("  add:    %d\n", totalRequests.add))
	io.WriteString(w, fmt.Sprintf("  search: %d\n", totalRequests.search))
	io.WriteString(w, fmt.Sprintf("  dump:   %d\n", totalRequests.dump))
	io.WriteString(w, "\n")
	io.WriteString(w, "RPS (refreshes every minute):\n=============================\n")
	io.WriteString(w, fmt.Sprintf("  add:    %.3f\n", rps.add))
	io.WriteString(w, fmt.Sprintf("  search: %.3f\n", rps.search))
	io.WriteString(w, fmt.Sprintf("  dump:   %.3f\n", rps.dump))
	io.WriteString(w, "\n")
	sqs, _ := s.tree.SyncQueueSize()
	io.WriteString(w, fmt.Sprintf("Total Metrics: %d\n", s.tree.TotalMetrics))
	io.WriteString(w, fmt.Sprintf("Sync Queue Size: %d\n", sqs))
}

func (s *Server) recalcRPS() {
	ticker := time.Tick(time.Minute)
	for _ = range ticker {
		rps.add = float64(totalRequests.add-lastRequests.add) / 60
		rps.dump = float64(totalRequests.dump-lastRequests.dump) / 60
		rps.search = float64(totalRequests.search-lastRequests.search) / 60
		lastRequests = totalRequests
		if s.selfMonitor {
			s.sendMetrics()
		}
	}
}

func NewServer(tree *mstree.MSTree, selfMonitor bool) *Server {
	host, err := os.Hostname()
	if err != nil {
		host = "localhost"
	}
	selfHostname = strings.Replace(host, ".", "_", -1)
	server := &Server{tree, selfMonitor}
	http.HandleFunc("/search", server.searchHandler)
	http.HandleFunc("/add", server.addHandler)
	http.HandleFunc("/debug/stack", server.stackHandler)
	http.HandleFunc("/dump", server.dumpHandler)
	http.HandleFunc("/stats", server.statsHandler)
	return server
}

func (s *Server) Start(listenAddr string) {
	log.Notice("Starting background stats job")
	go s.recalcRPS()
	log.Notice("Starting HTTP")
	err := http.ListenAndServe(listenAddr, nil)
	if err != nil {
		log.Error(err.Error())
		panic(err)
	}
}
<|endoftext|>


<commit_before>package relay

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/_third_party/github.com/mreiferson/go-httpclient"
	"github.com/StackExchange/bosun/search"
)

func RelayHTTP(addr, dest string) error {
	mux := http.NewServeMux()
	mux.HandleFunc("/", Handle(dest))
	log.Println("OpenTSDB relay listening on:", addr)
	log.Println("OpenTSDB destination:", dest)
	return http.ListenAndServe(addr, mux)
}

var client = &http.Client{
	Transport: &httpclient.Transport{
		RequestTimeout: time.Minute,
	},
}

func Handle(dest string) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		orig, _ := ioutil.ReadAll(r.Body)
		if r.URL.Path == "/api/put" {
			var reader io.Reader = bytes.NewReader(orig)
			if r, err := gzip.NewReader(reader); err == nil {
				reader = r
				defer r.Close()
			}
			body, _ := ioutil.ReadAll(reader)
			var dp opentsdb.DataPoint
			var mdp opentsdb.MultiDataPoint
			if err := json.Unmarshal(body, &mdp); err == nil {
			} else if err = json.Unmarshal(body, &dp); err == nil {
				mdp = opentsdb.MultiDataPoint{&dp}
			}
			if len(mdp) > 0 {
				search.HTTPExtract(mdp)
			}
		}
		durl := url.URL{
			Scheme: "http",
			Host:   dest,
		}
		durl.Path = r.URL.Path
		durl.RawQuery = r.URL.RawQuery
		durl.Fragment = r.URL.Fragment
		req, err := http.NewRequest(r.Method, durl.String(), bytes.NewReader(orig))
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}
		req.Header = r.Header
		req.TransferEncoding = r.TransferEncoding
		req.ContentLength = r.ContentLength
		resp, err := client.Do(req)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}
		b, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		w.WriteHeader(resp.StatusCode)
		w.Write(b)
	}
}
<commit_msg>Allow search of non-gzipped data<commit_after>package relay

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/_third_party/github.com/mreiferson/go-httpclient"
	"github.com/StackExchange/bosun/search"
)

func RelayHTTP(addr, dest string) error {
	mux := http.NewServeMux()
	mux.HandleFunc("/", Handle(dest))
	log.Println("OpenTSDB relay listening on:", addr)
	log.Println("OpenTSDB destination:", dest)
	return http.ListenAndServe(addr, mux)
}

var client = &http.Client{
	Transport: &httpclient.Transport{
		RequestTimeout: time.Minute,
	},
}

func Handle(dest string) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		orig, _ := ioutil.ReadAll(r.Body)
		if r.URL.Path == "/api/put" {
			var body []byte
			if r, err := gzip.NewReader(bytes.NewReader(orig)); err == nil {
				body, _ = ioutil.ReadAll(r)
				r.Close()
			} else {
				body = orig
			}
			var dp opentsdb.DataPoint
			var mdp opentsdb.MultiDataPoint
			if err := json.Unmarshal(body, &mdp); err == nil {
			} else if err = json.Unmarshal(body, &dp); err == nil {
				mdp = opentsdb.MultiDataPoint{&dp}
			}
			if len(mdp) > 0 {
				search.HTTPExtract(mdp)
			}
		}
		durl := url.URL{
			Scheme: "http",
			Host:   dest,
		}
		durl.Path = r.URL.Path
		durl.RawQuery = r.URL.RawQuery
		durl.Fragment = r.URL.Fragment
		req, err := http.NewRequest(r.Method, durl.String(), bytes.NewReader(orig))
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}
		req.Header = r.Header
		req.TransferEncoding = r.TransferEncoding
		req.ContentLength = r.ContentLength
		resp, err := client.Do(req)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}
		b, _ := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		w.WriteHeader(resp.StatusCode)
		w.Write(b)
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"errors"
	"sync"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/watch"
)

type consulWatcher struct {
	r        *consulRegistry
	wo       WatchOptions
	wp       *watch.Plan
	watchers map[string]*watch.Plan

	next chan *Result
	exit chan bool

	sync.RWMutex
	services map[string][]*Service
}

func newConsulWatcher(cr *consulRegistry, opts ...WatchOption) (Watcher, error) {
	var wo WatchOptions
	for _, o := range opts {
		o(&wo)
	}

	cw := &consulWatcher{
		r:        cr,
		wo:       wo,
		exit:     make(chan bool),
		next:     make(chan *Result, 10),
		watchers: make(map[string]*watch.Plan),
		services: make(map[string][]*Service),
	}

	wp, err := watch.Parse(map[string]interface{}{"type": "services"})
	if err != nil {
		return nil, err
	}

	wp.Handler = cw.handle
	go wp.Run(cr.Address)
	cw.wp = wp

	return cw, nil
}

func (cw *consulWatcher) serviceHandler(idx uint64, data interface{}) {
	entries, ok := data.([]*api.ServiceEntry)
	if !ok {
		return
	}

	serviceMap := map[string]*Service{}
	serviceName := ""

	for _, e := range entries {
		serviceName = e.Service.Service
		// version is now a tag
		version, _ := decodeVersion(e.Service.Tags)
		// service ID is now the node id
		id := e.Service.ID
		// key is always the version
		key := version
		// address is service address
		address := e.Service.Address

		// use node address
		if len(address) == 0 {
			address = e.Node.Address
		}

		svc, ok := serviceMap[key]
		if !ok {
			svc = &Service{
				Endpoints: decodeEndpoints(e.Service.Tags),
				Name:      e.Service.Service,
				Version:   version,
			}
			serviceMap[key] = svc
		}

		var del bool

		for _, check := range e.Checks {
			// delete the node if the status is critical
			if check.Status == "critical" {
				del = true
				break
			}
		}

		// if delete then skip the node
		if del {
			continue
		}

		svc.Nodes = append(svc.Nodes, &Node{
			Id:       id,
			Address:  address,
			Port:     e.Service.Port,
			Metadata: decodeMetadata(e.Service.Tags),
		})
	}

	cw.RLock()
	// make a copy
	rservices := make(map[string][]*Service)
	for k, v := range cw.services {
		rservices[k] = v
	}
	cw.RUnlock()

	var newServices []*Service

	// serviceMap is the new set of services keyed by name+version
	for _, newService := range serviceMap {
		// append to the new set of cached services
		newServices = append(newServices, newService)

		// check if the service exists in the existing cache
		oldServices, ok := rservices[serviceName]
		if !ok {
			// does not exist? then we're creating brand new entries
			cw.next <- &Result{Action: "create", Service: newService}
			continue
		}

		// service exists. ok let's figure out what to update and delete version wise
		action := "create"

		for _, oldService := range oldServices {
			// does this version exist?
			// no? then default to create
			if oldService.Version != newService.Version {
				continue
			}

			// yes? then it's an update
			action = "update"

			var nodes []*Node
			// check the old nodes to see if they've been deleted
			for _, oldNode := range oldService.Nodes {
				var seen bool
				for _, newNode := range newService.Nodes {
					if newNode.Id == oldNode.Id {
						seen = true
						break
					}
				}
				// does the old node exist in the new set of nodes
				// no? then delete that shit
				if !seen {
					nodes = append(nodes, oldNode)
				}
			}

			// it's an update rather than creation
			if len(nodes) > 0 {
				delService := oldService
				delService.Nodes = nodes
				cw.next <- &Result{Action: "delete", Service: delService}
			}
		}

		cw.next <- &Result{Action: action, Service: newService}
	}

	// Now check old versions that may not be in new services map
	for _, old := range rservices[serviceName] {
		// old version does not exist in new version map
		// kill it with fire!
		if _, ok := serviceMap[old.Version]; !ok {
			cw.next <- &Result{Action: "delete", Service: old}
		}
	}

	cw.Lock()
	cw.services[serviceName] = newServices
	cw.Unlock()
}

func (cw *consulWatcher) handle(idx uint64, data interface{}) {
	services, ok := data.(map[string][]string)
	if !ok {
		return
	}

	// add new watchers
	for service, _ := range services {
		// Filter on watch options
		// wo.Service: Only watch services we care about
		if len(cw.wo.Service) > 0 && service != cw.wo.Service {
			continue
		}

		if _, ok := cw.watchers[service]; ok {
			continue
		}
		wp, err := watch.Parse(map[string]interface{}{
			"type":    "service",
			"service": service,
		})
		if err == nil {
			wp.Handler = cw.serviceHandler
			go wp.Run(cw.r.Address)
			cw.watchers[service] = wp
			cw.next <- &Result{Action: "create", Service: &Service{Name: service}}
		}
	}

	cw.RLock()
	// make a copy
	rservices := make(map[string][]*Service)
	for k, v := range cw.services {
		rservices[k] = v
	}
	cw.RUnlock()

	// remove unknown services from registry
	for service, _ := range rservices {
		if _, ok := services[service]; !ok {
			cw.Lock()
			delete(cw.services, service)
			cw.Unlock()
		}
	}

	// remove unknown services from watchers
	for service, w := range cw.watchers {
		if _, ok := services[service]; !ok {
			w.Stop()
			delete(cw.watchers, service)
			cw.next <- &Result{Action: "delete", Service: &Service{Name: service}}
		}
	}
}

func (cw *consulWatcher) Next() (*Result, error) {
	select {
	case <-cw.exit:
		return nil, errors.New("result chan closed")
	case r, ok := <-cw.next:
		if !ok {
			return nil, errors.New("result chan closed")
		}
		return r, nil
	}
	return nil, errors.New("result chan closed")
}

func (cw *consulWatcher) Stop() {
	select {
	case <-cw.exit:
		return
	default:
		close(cw.exit)
		if cw.wp == nil {
			return
		}
		cw.wp.Stop()

		// drain results
		for {
			select {
			case <-cw.next:
			default:
				return
			}
		}
	}
}
<commit_msg>import  引入路径更新 解决install 报错<commit_after>package registry

import (
	"errors"
	"sync"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/api/watch"
)

type consulWatcher struct {
	r        *consulRegistry
	wo       WatchOptions
	wp       *watch.Plan
	watchers map[string]*watch.Plan

	next chan *Result
	exit chan bool

	sync.RWMutex
	services map[string][]*Service
}

func newConsulWatcher(cr *consulRegistry, opts ...WatchOption) (Watcher, error) {
	var wo WatchOptions
	for _, o := range opts {
		o(&wo)
	}

	cw := &consulWatcher{
		r:        cr,
		wo:       wo,
		exit:     make(chan bool),
		next:     make(chan *Result, 10),
		watchers: make(map[string]*watch.Plan),
		services: make(map[string][]*Service),
	}

	wp, err := watch.Parse(map[string]interface{}{"type": "services"})
	if err != nil {
		return nil, err
	}

	wp.Handler = cw.handle
	go wp.Run(cr.Address)
	cw.wp = wp

	return cw, nil
}

func (cw *consulWatcher) serviceHandler(idx uint64, data interface{}) {
	entries, ok := data.([]*api.ServiceEntry)
	if !ok {
		return
	}

	serviceMap := map[string]*Service{}
	serviceName := ""

	for _, e := range entries {
		serviceName = e.Service.Service
		// version is now a tag
		version, _ := decodeVersion(e.Service.Tags)
		// service ID is now the node id
		id := e.Service.ID
		// key is always the version
		key := version
		// address is service address
		address := e.Service.Address

		// use node address
		if len(address) == 0 {
			address = e.Node.Address
		}

		svc, ok := serviceMap[key]
		if !ok {
			svc = &Service{
				Endpoints: decodeEndpoints(e.Service.Tags),
				Name:      e.Service.Service,
				Version:   version,
			}
			serviceMap[key] = svc
		}

		var del bool

		for _, check := range e.Checks {
			// delete the node if the status is critical
			if check.Status == "critical" {
				del = true
				break
			}
		}

		// if delete then skip the node
		if del {
			continue
		}

		svc.Nodes = append(svc.Nodes, &Node{
			Id:       id,
			Address:  address,
			Port:     e.Service.Port,
			Metadata: decodeMetadata(e.Service.Tags),
		})
	}

	cw.RLock()
	// make a copy
	rservices := make(map[string][]*Service)
	for k, v := range cw.services {
		rservices[k] = v
	}
	cw.RUnlock()

	var newServices []*Service

	// serviceMap is the new set of services keyed by name+version
	for _, newService := range serviceMap {
		// append to the new set of cached services
		newServices = append(newServices, newService)

		// check if the service exists in the existing cache
		oldServices, ok := rservices[serviceName]
		if !ok {
			// does not exist? then we're creating brand new entries
			cw.next <- &Result{Action: "create", Service: newService}
			continue
		}

		// service exists. ok let's figure out what to update and delete version wise
		action := "create"

		for _, oldService := range oldServices {
			// does this version exist?
			// no? then default to create
			if oldService.Version != newService.Version {
				continue
			}

			// yes? then it's an update
			action = "update"

			var nodes []*Node
			// check the old nodes to see if they've been deleted
			for _, oldNode := range oldService.Nodes {
				var seen bool
				for _, newNode := range newService.Nodes {
					if newNode.Id == oldNode.Id {
						seen = true
						break
					}
				}
				// does the old node exist in the new set of nodes
				// no? then delete that shit
				if !seen {
					nodes = append(nodes, oldNode)
				}
			}

			// it's an update rather than creation
			if len(nodes) > 0 {
				delService := oldService
				delService.Nodes = nodes
				cw.next <- &Result{Action: "delete", Service: delService}
			}
		}

		cw.next <- &Result{Action: action, Service: newService}
	}

	// Now check old versions that may not be in new services map
	for _, old := range rservices[serviceName] {
		// old version does not exist in new version map
		// kill it with fire!
		if _, ok := serviceMap[old.Version]; !ok {
			cw.next <- &Result{Action: "delete", Service: old}
		}
	}

	cw.Lock()
	cw.services[serviceName] = newServices
	cw.Unlock()
}

func (cw *consulWatcher) handle(idx uint64, data interface{}) {
	services, ok := data.(map[string][]string)
	if !ok {
		return
	}

	// add new watchers
	for service, _ := range services {
		// Filter on watch options
		// wo.Service: Only watch services we care about
		if len(cw.wo.Service) > 0 && service != cw.wo.Service {
			continue
		}

		if _, ok := cw.watchers[service]; ok {
			continue
		}
		wp, err := watch.Parse(map[string]interface{}{
			"type":    "service",
			"service": service,
		})
		if err == nil {
			wp.Handler = cw.serviceHandler
			go wp.Run(cw.r.Address)
			cw.watchers[service] = wp
			cw.next <- &Result{Action: "create", Service: &Service{Name: service}}
		}
	}

	cw.RLock()
	// make a copy
	rservices := make(map[string][]*Service)
	for k, v := range cw.services {
		rservices[k] = v
	}
	cw.RUnlock()

	// remove unknown services from registry
	for service, _ := range rservices {
		if _, ok := services[service]; !ok {
			cw.Lock()
			delete(cw.services, service)
			cw.Unlock()
		}
	}

	// remove unknown services from watchers
	for service, w := range cw.watchers {
		if _, ok := services[service]; !ok {
			w.Stop()
			delete(cw.watchers, service)
			cw.next <- &Result{Action: "delete", Service: &Service{Name: service}}
		}
	}
}

func (cw *consulWatcher) Next() (*Result, error) {
	select {
	case <-cw.exit:
		return nil, errors.New("result chan closed")
	case r, ok := <-cw.next:
		if !ok {
			return nil, errors.New("result chan closed")
		}
		return r, nil
	}
	return nil, errors.New("result chan closed")
}

func (cw *consulWatcher) Stop() {
	select {
	case <-cw.exit:
		return
	default:
		close(cw.exit)
		if cw.wp == nil {
			return
		}
		cw.wp.Stop()

		// drain results
		for {
			select {
			case <-cw.next:
			default:
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"fmt"
	"os"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/factory"
	"github.com/docker/libtrust"
	"github.com/spf13/cobra"
)

func emit(format string, a ...interface{}) {
	if dryRun {
		fmt.Printf(format+"\n", a...)
	}
}

func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {

	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
	if !ok {
		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
	}

	// mark
	markSet := make(map[digest.Digest]struct{})
	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
		emit(repoName)

		var err error
		named, err := reference.ParseNamed(repoName)
		if err != nil {
			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
		}
		repository, err := registry.Repository(ctx, named)
		if err != nil {
			return fmt.Errorf("failed to construct repository: %v", err)
		}

		manifestService, err := repository.Manifests(ctx)
		if err != nil {
			return fmt.Errorf("failed to construct manifest service: %v", err)
		}

		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
		if !ok {
			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
		}

		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
			// Mark the manifest's blob
			emit("%s: marking manifest %s ", repoName, dgst)
			markSet[dgst] = struct{}{}

			manifest, err := manifestService.Get(ctx, dgst)
			if err != nil {
				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
			}

			descriptors := manifest.References()
			for _, descriptor := range descriptors {
				markSet[descriptor.Digest] = struct{}{}
				emit("%s: marking blob %s", repoName, descriptor.Digest)
			}

			switch manifest.(type) {
			case *schema1.SignedManifest:
				signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
				if !ok {
					return fmt.Errorf("unable to convert ManifestService into SignaturesGetter")
				}
				signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
				if err != nil {
					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
				}
				for _, signatureDigest := range signatures {
					emit("%s: marking signature %s", repoName, signatureDigest)
					markSet[signatureDigest] = struct{}{}
				}
				break
			case *schema2.DeserializedManifest:
				config := manifest.(*schema2.DeserializedManifest).Config
				emit("%s: marking configuration %s", repoName, config.Digest)
				markSet[config.Digest] = struct{}{}
				break
			}

			return nil
		})

		return err
	})

	if err != nil {
		return fmt.Errorf("failed to mark: %v\n", err)
	}

	// sweep
	blobService := registry.Blobs()
	deleteSet := make(map[digest.Digest]struct{})
	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
		// check if digest is in markSet. If not, delete it!
		if _, ok := markSet[dgst]; !ok {
			deleteSet[dgst] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error enumerating blobs: %v", err)
	}

	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
	// Construct vacuum
	vacuum := storage.NewVacuum(ctx, storageDriver)
	for dgst := range deleteSet {
		emit("blob eligible for deletion: %s", dgst)
		if dryRun {
			continue
		}
		err = vacuum.RemoveBlob(string(dgst))
		if err != nil {
			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
		}
	}

	return err
}

func init() {
	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything expect remove the blobs")
}

var dryRun bool

// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
	Use:   "garbage-collect <config>",
	Short: "`garbage-collect` deletes layers not referenced by any manifests",
	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
	Run: func(cmd *cobra.Command, args []string) {
		config, err := resolveConfiguration(args)
		if err != nil {
			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
			cmd.Usage()
			os.Exit(1)
		}

		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
			os.Exit(1)
		}

		ctx := context.Background()
		ctx, err = configureLogging(ctx, config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
			os.Exit(1)
		}

		k, err := libtrust.GenerateECP256PrivateKey()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}

		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
			os.Exit(1)
		}

		err = markAndSweep(ctx, driver, registry)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
			os.Exit(1)
		}
	},
}
<commit_msg>Fix wording for dry-run flag in useage message for garbage collector.<commit_after>package registry

import (
	"fmt"
	"os"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/factory"
	"github.com/docker/libtrust"
	"github.com/spf13/cobra"
)

func emit(format string, a ...interface{}) {
	if dryRun {
		fmt.Printf(format+"\n", a...)
	}
}

func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {

	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
	if !ok {
		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
	}

	// mark
	markSet := make(map[digest.Digest]struct{})
	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
		emit(repoName)

		var err error
		named, err := reference.ParseNamed(repoName)
		if err != nil {
			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
		}
		repository, err := registry.Repository(ctx, named)
		if err != nil {
			return fmt.Errorf("failed to construct repository: %v", err)
		}

		manifestService, err := repository.Manifests(ctx)
		if err != nil {
			return fmt.Errorf("failed to construct manifest service: %v", err)
		}

		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
		if !ok {
			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
		}

		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
			// Mark the manifest's blob
			emit("%s: marking manifest %s ", repoName, dgst)
			markSet[dgst] = struct{}{}

			manifest, err := manifestService.Get(ctx, dgst)
			if err != nil {
				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
			}

			descriptors := manifest.References()
			for _, descriptor := range descriptors {
				markSet[descriptor.Digest] = struct{}{}
				emit("%s: marking blob %s", repoName, descriptor.Digest)
			}

			switch manifest.(type) {
			case *schema1.SignedManifest:
				signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
				if !ok {
					return fmt.Errorf("unable to convert ManifestService into SignaturesGetter")
				}
				signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
				if err != nil {
					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
				}
				for _, signatureDigest := range signatures {
					emit("%s: marking signature %s", repoName, signatureDigest)
					markSet[signatureDigest] = struct{}{}
				}
				break
			case *schema2.DeserializedManifest:
				config := manifest.(*schema2.DeserializedManifest).Config
				emit("%s: marking configuration %s", repoName, config.Digest)
				markSet[config.Digest] = struct{}{}
				break
			}

			return nil
		})

		return err
	})

	if err != nil {
		return fmt.Errorf("failed to mark: %v\n", err)
	}

	// sweep
	blobService := registry.Blobs()
	deleteSet := make(map[digest.Digest]struct{})
	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
		// check if digest is in markSet. If not, delete it!
		if _, ok := markSet[dgst]; !ok {
			deleteSet[dgst] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error enumerating blobs: %v", err)
	}

	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
	// Construct vacuum
	vacuum := storage.NewVacuum(ctx, storageDriver)
	for dgst := range deleteSet {
		emit("blob eligible for deletion: %s", dgst)
		if dryRun {
			continue
		}
		err = vacuum.RemoveBlob(string(dgst))
		if err != nil {
			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
		}
	}

	return err
}

func init() {
	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything except remove the blobs")
}

var dryRun bool

// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
	Use:   "garbage-collect <config>",
	Short: "`garbage-collect` deletes layers not referenced by any manifests",
	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
	Run: func(cmd *cobra.Command, args []string) {
		config, err := resolveConfiguration(args)
		if err != nil {
			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
			cmd.Usage()
			os.Exit(1)
		}

		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
			os.Exit(1)
		}

		ctx := context.Background()
		ctx, err = configureLogging(ctx, config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
			os.Exit(1)
		}

		k, err := libtrust.GenerateECP256PrivateKey()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}

		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
			os.Exit(1)
		}

		err = markAndSweep(ctx, driver, registry)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
			os.Exit(1)
		}
	},
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/scaleway/scaleway-cli/pkg/api"
)

func resourceIp() *schema.Resource {
	return &schema.Resource{
		Create: resourceIpCreate,
		Read:   resourceIpRead,
		Update: resourceIpUpdate,
		Delete: resourceIpDelete,
		Schema: map[string]*schema.Schema{
			"server": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceIpCreate(d *schema.ResourceData, m interface{}) error {
	scaleway := m.(*api.ScalewayAPI)
	ip, err := scaleway.NewIP()
	if err != nil {
		return err
	}
	d.SetId(ip.IP.ID)
	return resourceIpUpdate(d, m)
}

func resourceIpRead(d *schema.ResourceData, m interface{}) error {
	scaleway := m.(*api.ScalewayAPI)
	ip, err := scaleway.GetIP(d.Id())
	if err != nil {
		return err
	}
	d.Set("ip", ip)
	return nil
}

func resourceIpUpdate(d *schema.ResourceData, m interface{}) error {
	server := m.(*api.ScalewayAPI)
	server.AttachIP(d.Id(), d.Get("server").(string))
	return resourceIpRead(d, m)
}

func resourceIpDelete(d *schema.ResourceData, m interface{}) error {
	server := m.(*api.ScalewayAPI)
	err := server.DeleteIP(d.Id())
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix scaleway_ip ip address attribute read<commit_after>package main

import (
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/scaleway/scaleway-cli/pkg/api"
)

func resourceIp() *schema.Resource {
	return &schema.Resource{
		Create: resourceIpCreate,
		Read:   resourceIpRead,
		Update: resourceIpUpdate,
		Delete: resourceIpDelete,
		Schema: map[string]*schema.Schema{
			"server": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceIpCreate(d *schema.ResourceData, m interface{}) error {
	scaleway := m.(*api.ScalewayAPI)
	ip, err := scaleway.NewIP()
	if err != nil {
		return err
	}
	d.SetId(ip.IP.ID)
	return resourceIpUpdate(d, m)
}

func resourceIpRead(d *schema.ResourceData, m interface{}) error {
	scaleway := m.(*api.ScalewayAPI)
	ip, err := scaleway.GetIP(d.Id())
	if err != nil {
		return err
	}
	d.Set("ip", ip.IP.Address)
	return nil
}

func resourceIpUpdate(d *schema.ResourceData, m interface{}) error {
	server := m.(*api.ScalewayAPI)
	server.AttachIP(d.Id(), d.Get("server").(string))
	return resourceIpRead(d, m)
}

func resourceIpDelete(d *schema.ResourceData, m interface{}) error {
	server := m.(*api.ScalewayAPI)
	err := server.DeleteIP(d.Id())
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// +build go1.12
// +build !386

/*
 *
 * Copyright 2020 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package xds_test contains e2e tests for xDS use.
package xds_test

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"testing"
	"time"

	"github.com/google/uuid"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/internal/grpctest"
	"google.golang.org/grpc/internal/leakcheck"
	"google.golang.org/grpc/internal/xds/env"
	"google.golang.org/grpc/testdata"
	"google.golang.org/grpc/xds/internal/testutils/e2e"

	xdsinternal "google.golang.org/grpc/internal/xds"
	testpb "google.golang.org/grpc/test/grpc_testing"
)

const (
	defaultTestTimeout      = 10 * time.Second
	defaultTestShortTimeout = 100 * time.Millisecond
)

type s struct {
	grpctest.Tester
}

func Test(t *testing.T) {
	grpctest.RunSubTests(t, s{})
}

type testService struct {
	testpb.TestServiceServer
}

func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
	return &testpb.Empty{}, nil
}

var (
	// Globals corresponding to the single instance of the xDS management server
	// which is spawned for all the tests in this package.
	managementServer *e2e.ManagementServer
	xdsClientNodeID  string
)

// TestMain sets up an xDS management server, runs all tests, and stops the
// management server.
func TestMain(m *testing.M) {
	// The management server is started and stopped from here, but the leakcheck
	// runs after every individual test. So, we need to skip the goroutine which
	// spawns the management server and is blocked on the call to `Serve()`.
	leakcheck.RegisterIgnoreGoroutine("e2e.StartManagementServer")

	cancel, err := setupManagementServer()
	if err != nil {
		log.Printf("setupManagementServer() failed: %v", err)
		os.Exit(1)
	}

	code := m.Run()
	cancel()
	os.Exit(code)
}

func createTmpFile(src, dst string) error {
	data, err := ioutil.ReadFile(src)
	if err != nil {
		return fmt.Errorf("ioutil.ReadFile(%q) failed: %v", src, err)
	}
	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
		return fmt.Errorf("ioutil.WriteFile(%q) failed: %v", dst, err)
	}
	return nil
}

// createTempDirWithFiles creates a temporary directory under the system default
// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
// rootSrc files are creates appropriate files under the newly create tempDir.
// Returns the name of the created tempDir.
func createTmpDirWithFiles(dirSuffix, certSrc, keySrc, rootSrc string) (string, error) {
	// Create a temp directory. Passing an empty string for the first argument
	// uses the system temp directory.
	dir, err := ioutil.TempDir("", dirSuffix)
	if err != nil {
		return "", fmt.Errorf("ioutil.TempDir() failed: %v", err)
	}

	if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil {
		return "", err
	}
	if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil {
		return "", err
	}
	if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil {
		return "", err
	}
	return dir, nil
}

// createClientTLSCredentials creates client-side TLS transport credentials.
func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
	t.Helper()

	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem"))
	if err != nil {
		t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err)
	}
	b, err := ioutil.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
	if err != nil {
		t.Fatalf("ioutil.ReadFile(x509/server_ca_cert.pem) failed: %v", err)
	}
	roots := x509.NewCertPool()
	if !roots.AppendCertsFromPEM(b) {
		t.Fatal("failed to append certificates")
	}
	return credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      roots,
		ServerName:   "x.test.example.com",
	})
}

// setupManagement server performs the following:
// - spin up an xDS management server on a local port
// - set up certificates for consumption by the file_watcher plugin
// - sets up the global variables which refer to this management server and the
//   nodeID to be used when talking to this management server.
//
// Returns a function to be invoked by the caller to stop the management server.
func setupManagementServer() (func(), error) {
	// Turn on the env var protection for client-side security.
	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
	env.ClientSideSecuritySupport = true

	// Spin up an xDS management server on a local port.
	var err error
	managementServer, err = e2e.StartManagementServer()
	if err != nil {
		return nil, err
	}

	// Create a directory to hold certs and key files used on the server side.
	serverDir, err := createTmpDirWithFiles("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
	if err != nil {
		managementServer.Stop()
		return nil, err
	}

	// Create a directory to hold certs and key files used on the client side.
	clientDir, err := createTmpDirWithFiles("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
	if err != nil {
		managementServer.Stop()
		return nil, err
	}

	// Create certificate providers section of the bootstrap config with entries
	// for both the client and server sides.
	cpc := map[string]json.RawMessage{
		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
		e2e.ClientSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)),
	}

	// Create a bootstrap file in a temporary directory.
	xdsClientNodeID = uuid.New().String()
	bootstrapCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
		Version:                            xdsinternal.TransportV3,
		NodeID:                             xdsClientNodeID,
		ServerURI:                          managementServer.Address,
		CertificateProviders:               cpc,
		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
	})
	if err != nil {
		managementServer.Stop()
		return nil, err
	}

	return func() {
		managementServer.Stop()
		bootstrapCleanup()
		env.ClientSideSecuritySupport = origClientSideSecurityEnvVar
	}, nil
}
<commit_msg>xds: workaround to deflake xds e2e tests (#4413)<commit_after>// +build go1.12
// +build !386

/*
 *
 * Copyright 2020 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package xds_test contains e2e tests for xDS use.
package xds_test

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"testing"
	"time"

	"github.com/google/uuid"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/internal/grpctest"
	"google.golang.org/grpc/internal/leakcheck"
	"google.golang.org/grpc/internal/xds/env"
	"google.golang.org/grpc/testdata"
	"google.golang.org/grpc/xds/internal/testutils/e2e"

	xdsinternal "google.golang.org/grpc/internal/xds"
	testpb "google.golang.org/grpc/test/grpc_testing"
)

const (
	defaultTestTimeout      = 10 * time.Second
	defaultTestShortTimeout = 100 * time.Millisecond
)

type s struct {
	grpctest.Tester
}

func Test(t *testing.T) {
	grpctest.RunSubTests(t, s{})
}

type testService struct {
	testpb.TestServiceServer
}

func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
	return &testpb.Empty{}, nil
}

var (
	// Globals corresponding to the single instance of the xDS management server
	// which is spawned for all the tests in this package.
	managementServer *e2e.ManagementServer
	xdsClientNodeID  string
)

// TestMain sets up an xDS management server, runs all tests, and stops the
// management server.
func TestMain(m *testing.M) {
	// The management server is started and stopped from here, but the leakcheck
	// runs after every individual test. So, we need to skip the goroutine which
	// spawns the management server and is blocked on the call to `Serve()`.
	leakcheck.RegisterIgnoreGoroutine("e2e.StartManagementServer")

	// Remove this once https://github.com/envoyproxy/go-control-plane/pull/430
	// is merged. For more information about this goroutine leak, see:
	// https://github.com/envoyproxy/go-control-plane/issues/429.
	leakcheck.RegisterIgnoreGoroutine("(*server).StreamHandler")

	cancel, err := setupManagementServer()
	if err != nil {
		log.Printf("setupManagementServer() failed: %v", err)
		os.Exit(1)
	}

	code := m.Run()
	cancel()
	os.Exit(code)
}

func createTmpFile(src, dst string) error {
	data, err := ioutil.ReadFile(src)
	if err != nil {
		return fmt.Errorf("ioutil.ReadFile(%q) failed: %v", src, err)
	}
	if err := ioutil.WriteFile(dst, data, os.ModePerm); err != nil {
		return fmt.Errorf("ioutil.WriteFile(%q) failed: %v", dst, err)
	}
	return nil
}

// createTempDirWithFiles creates a temporary directory under the system default
// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
// rootSrc files are creates appropriate files under the newly create tempDir.
// Returns the name of the created tempDir.
func createTmpDirWithFiles(dirSuffix, certSrc, keySrc, rootSrc string) (string, error) {
	// Create a temp directory. Passing an empty string for the first argument
	// uses the system temp directory.
	dir, err := ioutil.TempDir("", dirSuffix)
	if err != nil {
		return "", fmt.Errorf("ioutil.TempDir() failed: %v", err)
	}

	if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil {
		return "", err
	}
	if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil {
		return "", err
	}
	if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil {
		return "", err
	}
	return dir, nil
}

// createClientTLSCredentials creates client-side TLS transport credentials.
func createClientTLSCredentials(t *testing.T) credentials.TransportCredentials {
	t.Helper()

	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem"))
	if err != nil {
		t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err)
	}
	b, err := ioutil.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
	if err != nil {
		t.Fatalf("ioutil.ReadFile(x509/server_ca_cert.pem) failed: %v", err)
	}
	roots := x509.NewCertPool()
	if !roots.AppendCertsFromPEM(b) {
		t.Fatal("failed to append certificates")
	}
	return credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      roots,
		ServerName:   "x.test.example.com",
	})
}

// setupManagement server performs the following:
// - spin up an xDS management server on a local port
// - set up certificates for consumption by the file_watcher plugin
// - sets up the global variables which refer to this management server and the
//   nodeID to be used when talking to this management server.
//
// Returns a function to be invoked by the caller to stop the management server.
func setupManagementServer() (func(), error) {
	// Turn on the env var protection for client-side security.
	origClientSideSecurityEnvVar := env.ClientSideSecuritySupport
	env.ClientSideSecuritySupport = true

	// Spin up an xDS management server on a local port.
	var err error
	managementServer, err = e2e.StartManagementServer()
	if err != nil {
		return nil, err
	}

	// Create a directory to hold certs and key files used on the server side.
	serverDir, err := createTmpDirWithFiles("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
	if err != nil {
		managementServer.Stop()
		return nil, err
	}

	// Create a directory to hold certs and key files used on the client side.
	clientDir, err := createTmpDirWithFiles("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem")
	if err != nil {
		managementServer.Stop()
		return nil, err
	}

	// Create certificate providers section of the bootstrap config with entries
	// for both the client and server sides.
	cpc := map[string]json.RawMessage{
		e2e.ServerSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)),
		e2e.ClientSideCertProviderInstance: e2e.DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)),
	}

	// Create a bootstrap file in a temporary directory.
	xdsClientNodeID = uuid.New().String()
	bootstrapCleanup, err := xdsinternal.SetupBootstrapFile(xdsinternal.BootstrapOptions{
		Version:                            xdsinternal.TransportV3,
		NodeID:                             xdsClientNodeID,
		ServerURI:                          managementServer.Address,
		CertificateProviders:               cpc,
		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
	})
	if err != nil {
		managementServer.Stop()
		return nil, err
	}

	return func() {
		managementServer.Stop()
		bootstrapCleanup()
		env.ClientSideSecuritySupport = origClientSideSecurityEnvVar
	}, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"

	"github.com/ipfs/go-ipfs/core/coreunix"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"

	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName     = "quiet"
	silentOptionName    = "silent"
	progressOptionName  = "progress"
	trickleOptionName   = "trickle"
	wrapOptionName      = "wrap-with-directory"
	hiddenOptionName    = "hidden"
	onlyHashOptionName  = "only-hash"
	chunkerOptionName   = "chunker"
	pinOptionName       = "pin"
	rawLeavesOptionName = "raw-leaves"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file or directory to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
		cmds.BoolOption(silentOptionName, "Write no output."),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.  Default: true."),
		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
	},
	PreRun: func(req cmds.Request) error {
		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
			return nil
		}

		// ipfs cli progress bar defaults to true
		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, pin_found, _ := req.Option(pinOptionName).Bool()
		rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()

		if !pin_found { // default
			dopin = true
		}

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		dserv := n.DAG
		local, _, _ := req.Option("local").Bool()
		if local {
			offlineexch := offline.Exchange(n.Blockstore)
			bserv := blockservice.New(n.Blockstore, offlineexch)
			dserv = dag.NewDAGService(bserv)
		}

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent
		fileAdder.RawLeaves = rawblks

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, prgFound, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		silent, _, err := req.Option(silentOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		var showProgressBar bool
		if prgFound {
			showProgressBar = progress
		} else if !quiet && !silent {
			showProgressBar = true
		}

		var bar *pb.ProgressBar
		if showProgressBar {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if showProgressBar {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !showProgressBar {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if showProgressBar {
					bar.Update()
				}
			case size := <-sizeChan:
				if showProgressBar {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<commit_msg>add cmd: use .Default(true) for pin option.<commit_after>package commands

import (
	"fmt"
	"io"

	"github.com/ipfs/go-ipfs/core/coreunix"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"

	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName     = "quiet"
	silentOptionName    = "silent"
	progressOptionName  = "progress"
	trickleOptionName   = "trickle"
	wrapOptionName      = "wrap-with-directory"
	hiddenOptionName    = "hidden"
	onlyHashOptionName  = "only-hash"
	chunkerOptionName   = "chunker"
	pinOptionName       = "pin"
	rawLeavesOptionName = "raw-leaves"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file or directory to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
		cmds.BoolOption(silentOptionName, "Write no output."),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
	},
	PreRun: func(req cmds.Request) error {
		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
			return nil
		}

		// ipfs cli progress bar defaults to true
		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()
		rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		dserv := n.DAG
		local, _, _ := req.Option("local").Bool()
		if local {
			offlineexch := offline.Exchange(n.Blockstore)
			bserv := blockservice.New(n.Blockstore, offlineexch)
			dserv = dag.NewDAGService(bserv)
		}

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent
		fileAdder.RawLeaves = rawblks

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, prgFound, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		silent, _, err := req.Option(silentOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		var showProgressBar bool
		if prgFound {
			showProgressBar = progress
		} else if !quiet && !silent {
			showProgressBar = true
		}

		var bar *pb.ProgressBar
		if showProgressBar {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if showProgressBar {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !showProgressBar {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if showProgressBar {
					bar.Update()
				}
			case size := <-sizeChan:
				if showProgressBar {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<|endoftext|>