<commit_before>package datakit

import (
	"log"

	p9p "github.com/docker/go-p9p"
	"golang.org/x/net/context"
)

type transaction struct {
	client     *Client
	fromBranch string
	newBranch  string
}

// NewTransaction opens a new transaction originating from fromBranch, named
// newBranch.
func NewTransaction(ctx context.Context, client *Client, fromBranch string, newBranch string) (*transaction, error) {

	err := client.Mkdir(ctx, "branch", fromBranch)
	if err != nil {
		log.Println("Failed to Create branch/", fromBranch, err)
		return nil, err
	}
	err = client.Mkdir(ctx, "branch", fromBranch, "transactions", newBranch)
	if err != nil {
		log.Println("Failed to Create branch/", fromBranch, "/transactions/", newBranch, err)
		return nil, err
	}

	return &transaction{client: client, fromBranch: fromBranch, newBranch: newBranch}, nil
}

func (t *transaction) close(ctx context.Context) {
	// TODO: do we need to clear up unmerged branches?
}

// Abort ensures the update will not be committed.
func (t *transaction) Abort(ctx context.Context) {
	t.close(ctx)
}

// Commit merges the newBranch back into the fromBranch, or fails
func (t *transaction) Commit(ctx context.Context, msg string) error {
	// msg
	msgPath := []string{"branch", t.fromBranch, "transactions", t.newBranch, "msg"}
	msgFile, err := t.client.Open(ctx, p9p.ORDWR, msgPath...)
	if err != nil {
		log.Println("Failed to Open msg", err)
		return err
	}
	defer msgFile.Close(ctx)
	_, err = msgFile.Write(ctx, []byte(msg), 0)
	if err != nil {
		log.Println("Failed to Write msg", err)
	}

	// ctl
	ctlPath := []string{"branch", t.fromBranch, "transactions", t.newBranch, "ctl"}
	ctlFile, err := t.client.Open(ctx, p9p.ORDWR, ctlPath...)
	if err != nil {
		log.Println("Failed to Open ctl", err)
		return err
	}
	defer ctlFile.Close(ctx)
	_, err = ctlFile.Write(ctx, []byte("commit"), 0)
	if err != nil {
		log.Println("Failed to Write ctl", err)
		return err
	}
	return nil
}

// Write updates a key=value pair within the transaction.
func (t *transaction) Write(ctx context.Context, path []string, value string) error {
	p := []string{"branch", t.fromBranch, "transactions", t.newBranch, "rw"}
	for _, dir := range path[0 : len(path)-1] {
		p = append(p, dir)
	}
	err := t.client.Mkdir(ctx, p...)
	if err != nil {
		log.Println("Failed to Mkdir", p)
	}
	p = append(p, path[len(path)-1])
	file, err := t.client.Create(ctx, p...)
	if err != nil {
		log.Println("Failed to Create", p)
		return err
	}
	defer file.Close(ctx)
	writer := file.NewIOWriter(ctx, 0)
	_, err = writer.Write([]byte(value))
	if err != nil {
		log.Println("Failed to Write", path, "=", value, ":", err)
		return err
	}
	return nil
}

// Remove deletes a key within the transaction.
func (t *transaction) Remove(ctx context.Context, path []string) error {
	p := []string{"branch", t.fromBranch, "transactions", t.newBranch, "rw"}
	for _, dir := range path {
		p = append(p, dir)
	}
	err := t.client.Remove(ctx, p...)
	if err != nil {
		log.Println("Failed to Remove ", p)
	}
	return nil
}
<commit_msg>Add Read to transaction<commit_after>package datakit

import (
	"bytes"
	"io"
	"log"

	p9p "github.com/docker/go-p9p"
	"golang.org/x/net/context"
)

type transaction struct {
	client     *Client
	fromBranch string
	newBranch  string
}

// NewTransaction opens a new transaction originating from fromBranch, named
// newBranch.
func NewTransaction(ctx context.Context, client *Client, fromBranch string, newBranch string) (*transaction, error) {

	err := client.Mkdir(ctx, "branch", fromBranch)
	if err != nil {
		log.Println("Failed to Create branch/", fromBranch, err)
		return nil, err
	}
	err = client.Mkdir(ctx, "branch", fromBranch, "transactions", newBranch)
	if err != nil {
		log.Println("Failed to Create branch/", fromBranch, "/transactions/", newBranch, err)
		return nil, err
	}

	return &transaction{client: client, fromBranch: fromBranch, newBranch: newBranch}, nil
}

func (t *transaction) close(ctx context.Context) {
	// TODO: do we need to clear up unmerged branches?
}

// Abort ensures the update will not be committed.
func (t *transaction) Abort(ctx context.Context) {
	t.close(ctx)
}

// Commit merges the newBranch back into the fromBranch, or fails
func (t *transaction) Commit(ctx context.Context, msg string) error {
	// msg
	msgPath := []string{"branch", t.fromBranch, "transactions", t.newBranch, "msg"}
	msgFile, err := t.client.Open(ctx, p9p.ORDWR, msgPath...)
	if err != nil {
		log.Println("Failed to Open msg", err)
		return err
	}
	defer msgFile.Close(ctx)
	_, err = msgFile.Write(ctx, []byte(msg), 0)
	if err != nil {
		log.Println("Failed to Write msg", err)
	}

	// ctl
	ctlPath := []string{"branch", t.fromBranch, "transactions", t.newBranch, "ctl"}
	ctlFile, err := t.client.Open(ctx, p9p.ORDWR, ctlPath...)
	if err != nil {
		log.Println("Failed to Open ctl", err)
		return err
	}
	defer ctlFile.Close(ctx)
	_, err = ctlFile.Write(ctx, []byte("commit"), 0)
	if err != nil {
		log.Println("Failed to Write ctl", err)
		return err
	}
	return nil
}

// Write updates a key=value pair within the transaction.
func (t *transaction) Write(ctx context.Context, path []string, value string) error {
	p := []string{"branch", t.fromBranch, "transactions", t.newBranch, "rw"}
	for _, dir := range path[0 : len(path)-1] {
		p = append(p, dir)
	}
	err := t.client.Mkdir(ctx, p...)
	if err != nil {
		log.Println("Failed to Mkdir", p)
	}
	p = append(p, path[len(path)-1])
	file, err := t.client.Create(ctx, p...)
	if err != nil {
		log.Println("Failed to Create", p)
		return err
	}
	defer file.Close(ctx)
	writer := file.NewIOWriter(ctx, 0)
	_, err = writer.Write([]byte(value))
	if err != nil {
		log.Println("Failed to Write", path, "=", value, ":", err)
		return err
	}
	return nil
}

// Read reads a key within the transaction.
func (t *transaction) Read(ctx context.Context, path []string) (string, error) {
	p := []string{"branch", t.fromBranch, "transactions", t.newBranch, "rw"}
	for _, dir := range path[0 : len(path)-1] {
		p = append(p, dir)
	}
	file, err := t.client.Open(ctx, p9p.OREAD, p...)
	if err != nil {
		return "", err
	}
	defer file.Close(ctx)
	reader := file.NewIOReader(ctx, 0)
	buf := bytes.NewBuffer(nil)
	io.Copy(buf, reader)
	return string(buf.Bytes()), nil
}

// Remove deletes a key within the transaction.
func (t *transaction) Remove(ctx context.Context, path []string) error {
	p := []string{"branch", t.fromBranch, "transactions", t.newBranch, "rw"}
	for _, dir := range path {
		p = append(p, dir)
	}
	err := t.client.Remove(ctx, p...)
	if err != nil {
		log.Println("Failed to Remove ", p)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationmaster

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/juju/errors"
	"github.com/juju/httprequest"
	"github.com/juju/version"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
	"gopkg.in/juju/names.v2"
	"gopkg.in/macaroon.v1"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/api/common"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/core/migration"
	"github.com/juju/juju/resource"
	"github.com/juju/juju/watcher"
)

// NewWatcherFunc exists to let us unit test Facade without patching.
type NewWatcherFunc func(base.APICaller, params.NotifyWatchResult) watcher.NotifyWatcher

// NewClient returns a new Client based on an existing API connection.
func NewClient(caller base.APICaller, newWatcher NewWatcherFunc) *Client {
	return &Client{
		caller:            base.NewFacadeCaller(caller, "MigrationMaster"),
		newWatcher:        newWatcher,
		httpClientFactory: caller.HTTPClient,
	}
}

// Client describes the client side API for the MigrationMaster facade
// (used by the migrationmaster worker).
type Client struct {
	caller            base.FacadeCaller
	newWatcher        NewWatcherFunc
	httpClientFactory func() (*httprequest.Client, error)
}

// Watch returns a watcher which reports when a migration is active
// for the model associated with the API connection.
func (c *Client) Watch() (watcher.NotifyWatcher, error) {
	var result params.NotifyWatchResult
	err := c.caller.FacadeCall("Watch", nil, &result)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if result.Error != nil {
		return nil, result.Error
	}
	return c.newWatcher(c.caller.RawAPICaller(), result), nil
}

// MigrationStatus returns the details and progress of the latest
// model migration.
func (c *Client) MigrationStatus() (migration.MigrationStatus, error) {
	var empty migration.MigrationStatus
	var status params.MasterMigrationStatus
	err := c.caller.FacadeCall("MigrationStatus", nil, &status)
	if err != nil {
		return empty, errors.Trace(err)
	}

	modelTag, err := names.ParseModelTag(status.Spec.ModelTag)
	if err != nil {
		return empty, errors.Annotatef(err, "parsing model tag")
	}

	phase, ok := migration.ParsePhase(status.Phase)
	if !ok {
		return empty, errors.New("unable to parse phase")
	}

	target := status.Spec.TargetInfo
	controllerTag, err := names.ParseControllerTag(target.ControllerTag)
	if err != nil {
		return empty, errors.Annotatef(err, "parsing controller tag")
	}

	authTag, err := names.ParseUserTag(target.AuthTag)
	if err != nil {
		return empty, errors.Annotatef(err, "unable to parse auth tag")
	}

	var macs []macaroon.Slice
	if target.Macaroons != "" {
		if err := json.Unmarshal([]byte(target.Macaroons), &macs); err != nil {
			return empty, errors.Annotatef(err, "unmarshalling macaroon")
		}
	}

	return migration.MigrationStatus{
		MigrationId:      status.MigrationId,
		ModelUUID:        modelTag.Id(),
		ExternalControl:  status.Spec.ExternalControl,
		Phase:            phase,
		PhaseChangedTime: status.PhaseChangedTime,
		TargetInfo: migration.TargetInfo{
			ControllerTag: controllerTag,
			Addrs:         target.Addrs,
			CACert:        target.CACert,
			AuthTag:       authTag,
			Password:      target.Password,
			Macaroons:     macs,
		},
	}, nil
}

// SetPhase updates the phase of the currently active model migration.
func (c *Client) SetPhase(phase migration.Phase) error {
	args := params.SetMigrationPhaseArgs{
		Phase: phase.String(),
	}
	return c.caller.FacadeCall("SetPhase", args, nil)
}

// SetStatusMessage sets a human readable message regarding the
// progress of a migration.
func (c *Client) SetStatusMessage(message string) error {
	args := params.SetMigrationStatusMessageArgs{
		Message: message,
	}
	return c.caller.FacadeCall("SetStatusMessage", args, nil)
}

// ModelInfo return basic information about the model to migrated.
func (c *Client) ModelInfo() (migration.ModelInfo, error) {
	var info params.MigrationModelInfo
	err := c.caller.FacadeCall("ModelInfo", nil, &info)
	if err != nil {
		return migration.ModelInfo{}, errors.Trace(err)
	}
	owner, err := names.ParseUserTag(info.OwnerTag)
	if err != nil {
		return migration.ModelInfo{}, errors.Trace(err)
	}
	return migration.ModelInfo{
		UUID:         info.UUID,
		Name:         info.Name,
		Owner:        owner,
		AgentVersion: info.AgentVersion,
	}, nil
}

// Prechecks verifies that the source controller and model are healthy
// and able to participate in a migration.
func (c *Client) Prechecks() error {
	return c.caller.FacadeCall("Prechecks", nil, nil)
}

// Export returns a serialized representation of the model associated
// with the API connection. The charms used by the model are also
// returned.
func (c *Client) Export() (migration.SerializedModel, error) {
	var empty migration.SerializedModel
	var serialized params.SerializedModel
	err := c.caller.FacadeCall("Export", nil, &serialized)
	if err != nil {
		return empty, errors.Trace(err)
	}

	// Convert tools info to output map.
	tools := make(map[version.Binary]string)
	for _, toolsInfo := range serialized.Tools {
		v, err := version.ParseBinary(toolsInfo.Version)
		if err != nil {
			return migration.SerializedModel{}, errors.Annotate(err, "error parsing tools version")
		}
		tools[v] = toolsInfo.URI
	}

	resources, err := convertResources(serialized.Resources)
	if err != nil {
		return empty, errors.Trace(err)
	}

	return migration.SerializedModel{
		Bytes:     serialized.Bytes,
		Charms:    serialized.Charms,
		Tools:     tools,
		Resources: resources,
	}, nil
}

// OpenResource downloads the named resource for an application.
func (c *Client) OpenResource(application, name string) (io.ReadCloser, error) {
	httpClient, err := c.httpClientFactory()
	if err != nil {
		return nil, errors.Annotate(err, "unable to create HTTP client")
	}
	uri := fmt.Sprintf("/applications/%s/resources/%s", application, name)
	var resp *http.Response
	if err := httpClient.Get(uri, &resp); err != nil {
		return nil, errors.Annotate(err, "unable to retrieve resource")
	}
	return resp.Body, nil
}

// Reap removes the documents for the model associated with the API
// connection.
func (c *Client) Reap() error {
	return c.caller.FacadeCall("Reap", nil, nil)
}

// WatchMinionReports returns a watcher which reports when a migration
// minion has made a report for the current migration phase.
func (c *Client) WatchMinionReports() (watcher.NotifyWatcher, error) {
	var result params.NotifyWatchResult
	err := c.caller.FacadeCall("WatchMinionReports", nil, &result)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if result.Error != nil {
		return nil, result.Error
	}
	return c.newWatcher(c.caller.RawAPICaller(), result), nil
}

// MinionReports returns details of the reports made by migration
// minions to the controller for the current migration phase.
func (c *Client) MinionReports() (migration.MinionReports, error) {
	var in params.MinionReports
	var out migration.MinionReports

	err := c.caller.FacadeCall("MinionReports", nil, &in)
	if err != nil {
		return out, errors.Trace(err)
	}

	out.MigrationId = in.MigrationId

	phase, ok := migration.ParsePhase(in.Phase)
	if !ok {
		return out, errors.Errorf("invalid phase: %q", in.Phase)
	}
	out.Phase = phase

	out.SuccessCount = in.SuccessCount
	out.UnknownCount = in.UnknownCount

	out.SomeUnknownMachines, out.SomeUnknownUnits, err = groupTagIds(in.UnknownSample)
	if err != nil {
		return out, errors.Annotate(err, "processing unknown agents")
	}

	out.FailedMachines, out.FailedUnits, err = groupTagIds(in.Failed)
	if err != nil {
		return out, errors.Annotate(err, "processing failed agents")
	}

	return out, nil
}

func (c *Client) StreamModelLog(start time.Time) (<-chan common.LogMessage, error) {
	return common.StreamDebugLog(c.caller.RawAPICaller(), common.DebugLogParams{
		Replay:    true,
		NoTail:    true,
		StartTime: start,
	})
}

func groupTagIds(tagStrs []string) ([]string, []string, error) {
	var machines []string
	var units []string

	for i := 0; i < len(tagStrs); i++ {
		tag, err := names.ParseTag(tagStrs[i])
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
		switch t := tag.(type) {
		case names.MachineTag:
			machines = append(machines, t.Id())
		case names.UnitTag:
			units = append(units, t.Id())
		default:
			return nil, nil, errors.Errorf("unsupported tag: %q", tag)
		}
	}
	return machines, units, nil
}

func convertResources(in []params.SerializedModelResource) ([]migration.SerializedModelResource, error) {
	if len(in) == 0 {
		return nil, nil
	}
	out := make([]migration.SerializedModelResource, 0, len(in))
	for _, resource := range in {
		outResource, err := convertAppResource(resource)
		if err != nil {
			return nil, errors.Trace(err)
		}
		out = append(out, outResource)
	}
	return out, nil
}

func convertAppResource(in params.SerializedModelResource) (migration.SerializedModelResource, error) {
	var empty migration.SerializedModelResource
	appRev, err := convertResourceRevision(in.Application, in.Name, in.ApplicationRevision)
	if err != nil {
		return empty, errors.Annotate(err, "application revision")
	}
	csRev, err := convertResourceRevision(in.Application, in.Name, in.CharmStoreRevision)
	if err != nil {
		return empty, errors.Annotate(err, "charmstore revision")
	}
	return migration.SerializedModelResource{
		ApplicationRevision: appRev,
		CharmStoreRevision:  csRev,
	}, nil
}

func convertResourceRevision(app, name string, rev params.SerializedModelResourceRevision) (resource.Resource, error) {
	var empty resource.Resource
	type_, err := charmresource.ParseType(rev.Type)
	if err != nil {
		return empty, errors.Trace(err)
	}
	origin, err := charmresource.ParseOrigin(rev.Origin)
	if err != nil {
		return empty, errors.Trace(err)
	}
	fp, err := charmresource.ParseFingerprint(rev.FingerprintHex)
	if err != nil {
		return empty, errors.Annotate(err, "invalid fingerprint")
	}
	return resource.Resource{
		Resource: charmresource.Resource{
			Meta: charmresource.Meta{
				Name:        name,
				Type:        type_,
				Path:        rev.Path,
				Description: rev.Description,
			},
			Origin:      origin,
			Revision:    rev.Revision,
			Size:        rev.Size,
			Fingerprint: fp,
		},
		ApplicationID: app,
		Username:      rev.Username,
		Timestamp:     rev.Timestamp,
	}, nil
}
<commit_msg>api/migrationmaster: Add missing docstring<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationmaster

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/juju/errors"
	"github.com/juju/httprequest"
	"github.com/juju/version"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"
	"gopkg.in/juju/names.v2"
	"gopkg.in/macaroon.v1"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/api/common"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/core/migration"
	"github.com/juju/juju/resource"
	"github.com/juju/juju/watcher"
)

// NewWatcherFunc exists to let us unit test Facade without patching.
type NewWatcherFunc func(base.APICaller, params.NotifyWatchResult) watcher.NotifyWatcher

// NewClient returns a new Client based on an existing API connection.
func NewClient(caller base.APICaller, newWatcher NewWatcherFunc) *Client {
	return &Client{
		caller:            base.NewFacadeCaller(caller, "MigrationMaster"),
		newWatcher:        newWatcher,
		httpClientFactory: caller.HTTPClient,
	}
}

// Client describes the client side API for the MigrationMaster facade
// (used by the migrationmaster worker).
type Client struct {
	caller            base.FacadeCaller
	newWatcher        NewWatcherFunc
	httpClientFactory func() (*httprequest.Client, error)
}

// Watch returns a watcher which reports when a migration is active
// for the model associated with the API connection.
func (c *Client) Watch() (watcher.NotifyWatcher, error) {
	var result params.NotifyWatchResult
	err := c.caller.FacadeCall("Watch", nil, &result)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if result.Error != nil {
		return nil, result.Error
	}
	return c.newWatcher(c.caller.RawAPICaller(), result), nil
}

// MigrationStatus returns the details and progress of the latest
// model migration.
func (c *Client) MigrationStatus() (migration.MigrationStatus, error) {
	var empty migration.MigrationStatus
	var status params.MasterMigrationStatus
	err := c.caller.FacadeCall("MigrationStatus", nil, &status)
	if err != nil {
		return empty, errors.Trace(err)
	}

	modelTag, err := names.ParseModelTag(status.Spec.ModelTag)
	if err != nil {
		return empty, errors.Annotatef(err, "parsing model tag")
	}

	phase, ok := migration.ParsePhase(status.Phase)
	if !ok {
		return empty, errors.New("unable to parse phase")
	}

	target := status.Spec.TargetInfo
	controllerTag, err := names.ParseControllerTag(target.ControllerTag)
	if err != nil {
		return empty, errors.Annotatef(err, "parsing controller tag")
	}

	authTag, err := names.ParseUserTag(target.AuthTag)
	if err != nil {
		return empty, errors.Annotatef(err, "unable to parse auth tag")
	}

	var macs []macaroon.Slice
	if target.Macaroons != "" {
		if err := json.Unmarshal([]byte(target.Macaroons), &macs); err != nil {
			return empty, errors.Annotatef(err, "unmarshalling macaroon")
		}
	}

	return migration.MigrationStatus{
		MigrationId:      status.MigrationId,
		ModelUUID:        modelTag.Id(),
		ExternalControl:  status.Spec.ExternalControl,
		Phase:            phase,
		PhaseChangedTime: status.PhaseChangedTime,
		TargetInfo: migration.TargetInfo{
			ControllerTag: controllerTag,
			Addrs:         target.Addrs,
			CACert:        target.CACert,
			AuthTag:       authTag,
			Password:      target.Password,
			Macaroons:     macs,
		},
	}, nil
}

// SetPhase updates the phase of the currently active model migration.
func (c *Client) SetPhase(phase migration.Phase) error {
	args := params.SetMigrationPhaseArgs{
		Phase: phase.String(),
	}
	return c.caller.FacadeCall("SetPhase", args, nil)
}

// SetStatusMessage sets a human readable message regarding the
// progress of a migration.
func (c *Client) SetStatusMessage(message string) error {
	args := params.SetMigrationStatusMessageArgs{
		Message: message,
	}
	return c.caller.FacadeCall("SetStatusMessage", args, nil)
}

// ModelInfo return basic information about the model to migrated.
func (c *Client) ModelInfo() (migration.ModelInfo, error) {
	var info params.MigrationModelInfo
	err := c.caller.FacadeCall("ModelInfo", nil, &info)
	if err != nil {
		return migration.ModelInfo{}, errors.Trace(err)
	}
	owner, err := names.ParseUserTag(info.OwnerTag)
	if err != nil {
		return migration.ModelInfo{}, errors.Trace(err)
	}
	return migration.ModelInfo{
		UUID:         info.UUID,
		Name:         info.Name,
		Owner:        owner,
		AgentVersion: info.AgentVersion,
	}, nil
}

// Prechecks verifies that the source controller and model are healthy
// and able to participate in a migration.
func (c *Client) Prechecks() error {
	return c.caller.FacadeCall("Prechecks", nil, nil)
}

// Export returns a serialized representation of the model associated
// with the API connection. The charms used by the model are also
// returned.
func (c *Client) Export() (migration.SerializedModel, error) {
	var empty migration.SerializedModel
	var serialized params.SerializedModel
	err := c.caller.FacadeCall("Export", nil, &serialized)
	if err != nil {
		return empty, errors.Trace(err)
	}

	// Convert tools info to output map.
	tools := make(map[version.Binary]string)
	for _, toolsInfo := range serialized.Tools {
		v, err := version.ParseBinary(toolsInfo.Version)
		if err != nil {
			return migration.SerializedModel{}, errors.Annotate(err, "error parsing tools version")
		}
		tools[v] = toolsInfo.URI
	}

	resources, err := convertResources(serialized.Resources)
	if err != nil {
		return empty, errors.Trace(err)
	}

	return migration.SerializedModel{
		Bytes:     serialized.Bytes,
		Charms:    serialized.Charms,
		Tools:     tools,
		Resources: resources,
	}, nil
}

// OpenResource downloads the named resource for an application.
func (c *Client) OpenResource(application, name string) (io.ReadCloser, error) {
	httpClient, err := c.httpClientFactory()
	if err != nil {
		return nil, errors.Annotate(err, "unable to create HTTP client")
	}
	uri := fmt.Sprintf("/applications/%s/resources/%s", application, name)
	var resp *http.Response
	if err := httpClient.Get(uri, &resp); err != nil {
		return nil, errors.Annotate(err, "unable to retrieve resource")
	}
	return resp.Body, nil
}

// Reap removes the documents for the model associated with the API
// connection.
func (c *Client) Reap() error {
	return c.caller.FacadeCall("Reap", nil, nil)
}

// WatchMinionReports returns a watcher which reports when a migration
// minion has made a report for the current migration phase.
func (c *Client) WatchMinionReports() (watcher.NotifyWatcher, error) {
	var result params.NotifyWatchResult
	err := c.caller.FacadeCall("WatchMinionReports", nil, &result)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if result.Error != nil {
		return nil, result.Error
	}
	return c.newWatcher(c.caller.RawAPICaller(), result), nil
}

// MinionReports returns details of the reports made by migration
// minions to the controller for the current migration phase.
func (c *Client) MinionReports() (migration.MinionReports, error) {
	var in params.MinionReports
	var out migration.MinionReports

	err := c.caller.FacadeCall("MinionReports", nil, &in)
	if err != nil {
		return out, errors.Trace(err)
	}

	out.MigrationId = in.MigrationId

	phase, ok := migration.ParsePhase(in.Phase)
	if !ok {
		return out, errors.Errorf("invalid phase: %q", in.Phase)
	}
	out.Phase = phase

	out.SuccessCount = in.SuccessCount
	out.UnknownCount = in.UnknownCount

	out.SomeUnknownMachines, out.SomeUnknownUnits, err = groupTagIds(in.UnknownSample)
	if err != nil {
		return out, errors.Annotate(err, "processing unknown agents")
	}

	out.FailedMachines, out.FailedUnits, err = groupTagIds(in.Failed)
	if err != nil {
		return out, errors.Annotate(err, "processing failed agents")
	}

	return out, nil
}

// StreamModelLog takes a starting time and returns a channel that
// will yield the logs on or after that time - these are the logs that
// need to be transferred to the target after the migration is
// successful.
func (c *Client) StreamModelLog(start time.Time) (<-chan common.LogMessage, error) {
	return common.StreamDebugLog(c.caller.RawAPICaller(), common.DebugLogParams{
		Replay:    true,
		NoTail:    true,
		StartTime: start,
	})
}

func groupTagIds(tagStrs []string) ([]string, []string, error) {
	var machines []string
	var units []string

	for i := 0; i < len(tagStrs); i++ {
		tag, err := names.ParseTag(tagStrs[i])
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
		switch t := tag.(type) {
		case names.MachineTag:
			machines = append(machines, t.Id())
		case names.UnitTag:
			units = append(units, t.Id())
		default:
			return nil, nil, errors.Errorf("unsupported tag: %q", tag)
		}
	}
	return machines, units, nil
}

func convertResources(in []params.SerializedModelResource) ([]migration.SerializedModelResource, error) {
	if len(in) == 0 {
		return nil, nil
	}
	out := make([]migration.SerializedModelResource, 0, len(in))
	for _, resource := range in {
		outResource, err := convertAppResource(resource)
		if err != nil {
			return nil, errors.Trace(err)
		}
		out = append(out, outResource)
	}
	return out, nil
}

func convertAppResource(in params.SerializedModelResource) (migration.SerializedModelResource, error) {
	var empty migration.SerializedModelResource
	appRev, err := convertResourceRevision(in.Application, in.Name, in.ApplicationRevision)
	if err != nil {
		return empty, errors.Annotate(err, "application revision")
	}
	csRev, err := convertResourceRevision(in.Application, in.Name, in.CharmStoreRevision)
	if err != nil {
		return empty, errors.Annotate(err, "charmstore revision")
	}
	return migration.SerializedModelResource{
		ApplicationRevision: appRev,
		CharmStoreRevision:  csRev,
	}, nil
}

func convertResourceRevision(app, name string, rev params.SerializedModelResourceRevision) (resource.Resource, error) {
	var empty resource.Resource
	type_, err := charmresource.ParseType(rev.Type)
	if err != nil {
		return empty, errors.Trace(err)
	}
	origin, err := charmresource.ParseOrigin(rev.Origin)
	if err != nil {
		return empty, errors.Trace(err)
	}
	fp, err := charmresource.ParseFingerprint(rev.FingerprintHex)
	if err != nil {
		return empty, errors.Annotate(err, "invalid fingerprint")
	}
	return resource.Resource{
		Resource: charmresource.Resource{
			Meta: charmresource.Meta{
				Name:        name,
				Type:        type_,
				Path:        rev.Path,
				Description: rev.Description,
			},
			Origin:      origin,
			Revision:    rev.Revision,
			Size:        rev.Size,
			Fingerprint: fp,
		},
		ApplicationID: app,
		Username:      rev.Username,
		Timestamp:     rev.Timestamp,
	}, nil
}
<|endoftext|>


<commit_before>package oneandone_cloudserver_api

import (
	log "github.com/Sirupsen/logrus"
)

type ServerAppliance struct {
	withId
	withName
	OsImageType        string   `json:"os_image_type"`
	OsFamily           string   `json:"os_family"`
	Os                 string   `json:"os"`
	OsVersion          string   `json:"os_version"`
	MinHddSize         int      `json:"min_hdd_size"`
	Architecture       int      `json:"architecture"`
	Licenses           []string `json:"licenses"`
	IsAutomaticInstall bool     `json:"automatic_installation"`
	Type               string   `json:"type"`
	withApi
}

// GET /server_appliances
func (api *API) GetServerAppliances() []ServerAppliance {
	log.Debug("requesting information about server appliances")
	session := api.prepareSession()
	res := []ServerAppliance{}
	resp, _ := session.Get(createUrl(api, "server_appliances"), nil, &res, nil)
	logResult(resp, 200)
	for index, _ := range res {
		res[index].api = api
	}
	return res
}

// GET /server_appliances/{id}
func (api *API) GetServerAppliance(Id string) ServerAppliance {
	log.Debug("requesting information about server appliance", Id)
	session := api.prepareSession()
	res := ServerAppliance{}
	resp, _ := session.Get(createUrl(api, "server_appliances"), nil, &res, nil)
	logResult(resp, 200)
	res.api = api
	return res
}
<commit_msg>fixed issue when querying for a selected appliance<commit_after>package oneandone_cloudserver_api

import (
	log "github.com/Sirupsen/logrus"
)

type ServerAppliance struct {
	withId
	withName
	OsImageType        string   `json:"os_image_type"`
	OsFamily           string   `json:"os_family"`
	Os                 string   `json:"os"`
	OsVersion          string   `json:"os_version"`
	MinHddSize         int      `json:"min_hdd_size"`
	Architecture       int      `json:"architecture"`
	Licenses           []string `json:"licenses"`
	IsAutomaticInstall bool     `json:"automatic_installation"`
	Type               string   `json:"type"`
	withApi
}

// GET /server_appliances
func (api *API) GetServerAppliances() []ServerAppliance {
	log.Debug("requesting information about server appliances")
	session := api.prepareSession()
	res := []ServerAppliance{}
	resp, _ := session.Get(createUrl(api, "server_appliances"), nil, &res, nil)
	logResult(resp, 200)
	for index, _ := range res {
		res[index].api = api
	}
	return res
}

// GET /server_appliances/{id}
func (api *API) GetServerAppliance(Id string) ServerAppliance {
	log.Debug("requesting information about server appliance", Id)
	session := api.prepareSession()
	res := ServerAppliance{}
	resp, _ := session.Get(createUrl(api, "server_appliances", Id), nil, &res, nil)
	logResult(resp, 200)
	res.api = api
	return res
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/surgemq/message"
)

type netReader interface {
	io.Reader
	SetReadDeadline(t time.Time) error
}

type timeoutReader struct {
	d    time.Duration
	conn netReader
}

func (r timeoutReader) Read(b []byte) (int, error) {
	if err := r.conn.SetReadDeadline(time.Now().Add(r.d)); err != nil {
		return 0, err
	}
	return r.conn.Read(b)
}

// receiver() reads data from the network, and writes the data into the incoming buffer
func (this *service) receiver() {
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) receiver开始", this.cid())
	})
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic: %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping receiver", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting receiver", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		//Log.Debugc(func() string{ return fmt.Sprintf("server/handleConnection: Setting read deadline to %d", time.Second*time.Duration(this.keepAlive))})
		keepAlive := time.Second * time.Duration(this.keepAlive)
		r := timeoutReader{
			d:    keepAlive + (keepAlive / 2),
			conn: conn,
		}

		for {
			_, err := this.in.ReadFrom(r)
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sess is: %v", this.sess)})
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sessMgr is: %v", this.sessMgr)})

			/*if err != nil {
				Log.Infoc(func() string { return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err) })
				//         if err != io.EOF {
				//         }
				return
			}*/
			if err != nil {

				if err == io.EOF {
					Log.Errorc(func() string {
						return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err)
					})
					return
				} else {
					Log.Infoc(func() string {
						return fmt.Sprintf("(%s) info reading from connection: %v", this.cid(), err)
					})
				}

				continue
			} else {
				Log.Infoc(func() string {
					return fmt.Sprintf("(%s)向ringbuffer些数据成功！", this.cid())
				})
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket: %v", this.cid(), ErrInvalidConnectionType)})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) %v", this.cid(), ErrInvalidConnectionType)
		})
	}
}

// sender() writes data from the outgoing buffer to the network
func (this *service) sender() {
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) sender开始", this.cid())
	})
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic: %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping sender", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting sender", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		for {
			_, err := this.out.WriteTo(conn)

			if err != nil {
				Log.Errorc(func() string {
					return fmt.Sprintf("(%s)Error Sender  writing data: %v", this.cid(), err)
				})
				return
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket not supported", this.cid())})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Invalid connection type", this.cid())
		})
	}
}

// peekMessageSize() reads, but not commits, enough bytes to determine the size of
// the next message and returns the type and size.
func (this *service) peekMessageSize() (message.Message, int, error) {
	fmt.Println("this.in.buf=", this.in.buf)
	fmt.Println("this.out.buf=", this.out.buf)
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) peekMessageSize开始", this.cid())
	})
	var (
		b   []byte
		err error
	)

	if this.in == nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize this.in is nil", this.cid())
		})
		err = ErrBufferNotReady
		return nil, 0, err
	}

	// Peek cnt bytes from the input buffer.
	b, err = this.in.ReadWait()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize this.in.ReadWait falure:%v", this.cid(), err)
		})
		return nil, 0, err
	}
	//total := int(remlen) + 1 + m
	mtype := message.MessageType((b)[0] >> 4)
	//return mtype, total, err
	var msg message.Message
	msg, err = mtype.New()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize mtype.New() falure:%v", this.cid(), err)
		})
		return nil, 0, err
	}
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) 开始创建对象(%s)", this.cid(), msg.Name())
	})
	_, err = msg.Decode(b)
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize msg.Decode falure:%v", this.cid(), err)
		})
		return nil, 0, err
	}
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) peekMessageSize结束(%s)", this.cid(), msg.Name())
	})
	fmt.Println("this.in.readwait=", msg.Len())
	return msg, len(b), err
}

// peekMessage() reads a message from the buffer, but the bytes are NOT committed.
// This means the buffer still thinks the bytes are not read yet.
/*func (this *service) peekMessage(mtype message.MessageType, total int) (message.Message, int, error) {
	var (
		b    []byte
		err  error
		i, n int
		msg  message.Message
	)

	if this.in == nil {
		return nil, 0, ErrBufferNotReady
	}

	// Peek until we get total bytes
	for i = 0; ; i++ {
		// Peek remlen bytes from the input buffer.
		b, err = this.in.ReadWait(total)
		if err != nil && err != ErrBufferInsufficientData {
			return nil, 0, err
		}

		// If not enough bytes are returned, then continue until there's enough.
		if len(b) >= total {
			break
		}
	}

	msg, err = mtype.New()
	if err != nil {
		return nil, 0, err
	}

	n, err = msg.Decode(b)
	return msg, n, err
}*/

// readMessage() reads and copies a message from the buffer. The buffer bytes are
// committed as a result of the read.
/*
func (this *service) readMessage(mtype message.MessageType, total int) (message.Message, int, error) {
	var (
		b   []byte
		err error
		n   int
		msg message.Message
	)

	if this.in == nil {
		err = ErrBufferNotReady
		return nil, 0, err
	}

	if len(this.intmp) < total {
		this.intmp = make([]byte, total)
	}

	// Read until we get total bytes
	//l := 0
	//for l < total {
	n, err = this.in.Read(this.intmp[0:])
	//l += n
	Log.Debugc(func() string {
		return fmt.Sprintf("read %d bytes", n)
	})
	if err != nil {
		return nil, 0, err
	}
	//}

	b = this.intmp[:total]

	msg, err = mtype.New()
	if err != nil {
		return msg, 0, err
	}

	n, err = msg.Decode(b)
	return msg, n, err
}
*/

// writeMessage() writes a message to the outgoing buffer
func (this *service) writeMessage(msg message.Message) (int, error) {
	var (
		l     int = msg.Len()
		m, n  int
		err   error
		buf   []byte
		start int64
		//wrap bool
	)

	if this.out == nil {
		return 0, ErrBufferNotReady
	}

	// This is to serialize writes to the underlying buffer. Multiple goroutines could
	// potentially get here because of calling Publish() or Subscribe() or other
	// functions that will send messages. For example, if a message is received in
	// another connetion, and the message needs to be published to this client, then
	// the Publish() function is called, and at the same time, another client could
	// do exactly the same thing.
	//
	// Not an ideal fix though. If possible we should remove mutex and be lockfree.
	// Mainly because when there's a large number of goroutines that want to publish
	// to this client, then they will all block. However, this will do for now.
	//
	// FIXME: Try to find a better way than a mutex...if possible.
	this.wmu.Lock()
	defer this.wmu.Unlock()

	//buf, wrap, err = this.out.WriteWait(l)//zheliyouwenti
	start, _, err = this.out.waitForWriteSpace(l)
	if err != nil {
		return 0, err
	}
	buf = make([]byte, l)
	n, err = msg.Encode(buf[0:])
	if err != nil {
		return 0, err
	}
	this.out.buf[start] = ByteArray{bArray: buf}
	m, err = this.out.WriteCommit(n)
	if err != nil {
		return 0, err
	}

	this.outStat.increment(int64(m))

	return m, nil
}
<commit_msg>修改了buffer.go;sendrecv.go;process.go<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/surgemq/message"
)

type netReader interface {
	io.Reader
	SetReadDeadline(t time.Time) error
}

type timeoutReader struct {
	d    time.Duration
	conn netReader
}

func (r timeoutReader) Read(b []byte) (int, error) {
	if err := r.conn.SetReadDeadline(time.Now().Add(r.d)); err != nil {
		return 0, err
	}
	return r.conn.Read(b)
}

// receiver() reads data from the network, and writes the data into the incoming buffer
func (this *service) receiver() {
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) receiver开始", this.cid())
	})
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic: %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping receiver", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting receiver", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		//Log.Debugc(func() string{ return fmt.Sprintf("server/handleConnection: Setting read deadline to %d", time.Second*time.Duration(this.keepAlive))})
		keepAlive := time.Second * time.Duration(this.keepAlive)
		r := timeoutReader{
			d:    keepAlive + (keepAlive / 2),
			conn: conn,
		}

		for {
			_, err := this.in.ReadFrom(r)
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sess is: %v", this.sess)})
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sessMgr is: %v", this.sessMgr)})

			/*if err != nil {
				Log.Infoc(func() string { return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err) })
				//         if err != io.EOF {
				//         }
				return
			}*/
			if err != nil {

				if err == io.EOF {
					Log.Errorc(func() string {
						return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err)
					})
					return
				} else {
					Log.Infoc(func() string {
						return fmt.Sprintf("(%s) info reading from connection: %v", this.cid(), err)
					})
				}

				continue
			} else {
				Log.Infoc(func() string {
					return fmt.Sprintf("(%s)向ringbuffer些数据成功！", this.cid())
				})
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket: %v", this.cid(), ErrInvalidConnectionType)})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) %v", this.cid(), ErrInvalidConnectionType)
		})
	}
}

// sender() writes data from the outgoing buffer to the network
func (this *service) sender() {
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) sender开始", this.cid())
	})
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic: %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping sender", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting sender", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		for {
			_, err := this.out.WriteTo(conn)

			if err != nil {
				Log.Errorc(func() string {
					return fmt.Sprintf("(%s)Error Sender  writing data: %v", this.cid(), err)
				})
				return
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket not supported", this.cid())})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Invalid connection type", this.cid())
		})
	}
}

// peekMessageSize() reads, but not commits, enough bytes to determine the size of
// the next message and returns the type and size.
func (this *service) peekMessageSize() (message.Message, int, error) {
	fmt.Println("(", this.cid(), ")", "this.in.buf=", this.in.buf)
	fmt.Println("(", this.cid(), ")", "this.out.buf=", this.out.buf)
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) peekMessageSize开始", this.cid())
	})
	var (
		b   []byte
		err error
	)

	if this.in == nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize this.in is nil", this.cid())
		})
		err = ErrBufferNotReady
		return nil, 0, err
	}

	// Peek cnt bytes from the input buffer.
	b, err = this.in.ReadWait()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize this.in.ReadWait falure:%v", this.cid(), err)
		})
		return nil, 0, err
	}
	//total := int(remlen) + 1 + m
	mtype := message.MessageType((b)[0] >> 4)
	//return mtype, total, err
	var msg message.Message
	msg, err = mtype.New()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize mtype.New() falure:%v", this.cid(), err)
		})
		return nil, 0, err
	}
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) 开始创建对象(%s)", this.cid(), msg.Name())
	})
	_, err = msg.Decode(b)
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) peekMessageSize msg.Decode falure:%v", this.cid(), err)
		})
		return nil, 0, err
	}
	Log.Infoc(func() string {
		return fmt.Sprintf("(%s) peekMessageSize结束(%s)", this.cid(), msg.Name())
	})
	fmt.Println("this.in.readwait=", msg.Len())
	return msg, len(b), err
}

// peekMessage() reads a message from the buffer, but the bytes are NOT committed.
// This means the buffer still thinks the bytes are not read yet.
/*func (this *service) peekMessage(mtype message.MessageType, total int) (message.Message, int, error) {
	var (
		b    []byte
		err  error
		i, n int
		msg  message.Message
	)

	if this.in == nil {
		return nil, 0, ErrBufferNotReady
	}

	// Peek until we get total bytes
	for i = 0; ; i++ {
		// Peek remlen bytes from the input buffer.
		b, err = this.in.ReadWait(total)
		if err != nil && err != ErrBufferInsufficientData {
			return nil, 0, err
		}

		// If not enough bytes are returned, then continue until there's enough.
		if len(b) >= total {
			break
		}
	}

	msg, err = mtype.New()
	if err != nil {
		return nil, 0, err
	}

	n, err = msg.Decode(b)
	return msg, n, err
}*/

// readMessage() reads and copies a message from the buffer. The buffer bytes are
// committed as a result of the read.
/*
func (this *service) readMessage(mtype message.MessageType, total int) (message.Message, int, error) {
	var (
		b   []byte
		err error
		n   int
		msg message.Message
	)

	if this.in == nil {
		err = ErrBufferNotReady
		return nil, 0, err
	}

	if len(this.intmp) < total {
		this.intmp = make([]byte, total)
	}

	// Read until we get total bytes
	//l := 0
	//for l < total {
	n, err = this.in.Read(this.intmp[0:])
	//l += n
	Log.Debugc(func() string {
		return fmt.Sprintf("read %d bytes", n)
	})
	if err != nil {
		return nil, 0, err
	}
	//}

	b = this.intmp[:total]

	msg, err = mtype.New()
	if err != nil {
		return msg, 0, err
	}

	n, err = msg.Decode(b)
	return msg, n, err
}
*/

// writeMessage() writes a message to the outgoing buffer
func (this *service) writeMessage(msg message.Message) (int, error) {
	var (
		l     int = msg.Len()
		m, n  int
		err   error
		buf   []byte
		start int64
		//wrap bool
	)

	if this.out == nil {
		return 0, ErrBufferNotReady
	}

	// This is to serialize writes to the underlying buffer. Multiple goroutines could
	// potentially get here because of calling Publish() or Subscribe() or other
	// functions that will send messages. For example, if a message is received in
	// another connetion, and the message needs to be published to this client, then
	// the Publish() function is called, and at the same time, another client could
	// do exactly the same thing.
	//
	// Not an ideal fix though. If possible we should remove mutex and be lockfree.
	// Mainly because when there's a large number of goroutines that want to publish
	// to this client, then they will all block. However, this will do for now.
	//
	// FIXME: Try to find a better way than a mutex...if possible.
	this.wmu.Lock()
	defer this.wmu.Unlock()

	//buf, wrap, err = this.out.WriteWait(l)//zheliyouwenti
	start, _, err = this.out.waitForWriteSpace(l)
	if err != nil {
		return 0, err
	}
	buf = make([]byte, l)
	n, err = msg.Encode(buf[0:])
	if err != nil {
		return 0, err
	}
	this.out.buf[start] = ByteArray{bArray: buf}
	m, err = this.out.WriteCommit(n)
	if err != nil {
		return 0, err
	}

	this.outStat.increment(int64(m))

	return m, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate descriptor-adapter --descriptor-name Interface  --value-type *interfaces.Interface --meta-type *ifaceidx.IfaceMetadata --import "../model/interfaces" --import "ifaceidx" --output-dir "descriptor"
//go:generate descriptor-adapter --descriptor-name Unnumbered  --value-type *interfaces.Interface_Unnumbered --import "../model/interfaces" --output-dir "descriptor"

package ifplugin

import (
	"context"
	"os"
	"sync"

	"github.com/go-errors/errors"
	govppapi "git.fd.io/govpp.git/api"

	"github.com/ligato/cn-infra/infra"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/logging/measure"
	"github.com/ligato/cn-infra/idxmap"
	"github.com/ligato/cn-infra/health/statuscheck"
	"github.com/ligato/cn-infra/utils/safeclose"

	scheduler "github.com/ligato/vpp-agent/plugins/kvscheduler/api"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/descriptor"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/descriptor/adapter"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/ifaceidx"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/vppcalls"
	"github.com/ligato/vpp-agent/plugins/vppv2/model/interfaces"
	linux_ifcalls "github.com/ligato/vpp-agent/plugins/linuxv2/ifplugin/linuxcalls"
	linux_ifplugin "github.com/ligato/vpp-agent/plugins/linuxv2/ifplugin"
	"github.com/ligato/vpp-agent/plugins/vpp/binapi/dhcp"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

const (
	// vppStatusPublishersEnv is the name of the environment variable used to
	// override state publishers from the configuration file.
	vppStatusPublishersEnv = "VPP_STATUS_PUBLISHERS"
)

var (
	// noopWriter (no operation writer) helps avoiding NIL pointer based segmentation fault.
	// It is used as default if some dependency was not injected.
	noopWriter = datasync.KVProtoWriters{}

	// noopWatcher (no operation watcher) helps avoiding NIL pointer based segmentation fault.
	// It is used as default if some dependency was not injected.
	noopWatcher = datasync.KVProtoWatchers{}
)

// IfPlugin configures VPP interfaces using GoVPP.
type IfPlugin struct {
	Deps

	// GoVPP
	vppCh     govppapi.Channel

	// handlers
	ifHandler      vppcalls.IfVppAPI
	linuxIfHandler linux_ifcalls.NetlinkAPI

	// descriptors
	ifDescriptor   *descriptor.InterfaceDescriptor
	unIfDescriptor *descriptor.UnnumberedIfDescriptor
	dhcpDescriptor *descriptor.DHCPDescriptor

	// index maps
	intfIndex ifaceidx.IfaceMetadataIndex
	dhcpIndex idxmap.NamedMapping

	// From config file
	enableStopwatch bool
	stopwatch       *measure.Stopwatch // timer used to measure and store time
	defaultMtu      uint32

	// state data
	statusCheckReg   bool
	watchStatusReg   datasync.WatchRegistration
	resyncStatusChan chan datasync.ResyncEvent
	ifNotifChan      chan govppapi.Message
	ifStateChan      chan *interfaces.InterfaceNotification
	ifStateUpdater   *InterfaceStateUpdater

	// go routine management
	ctx    context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup
}

// Deps lists dependencies of the interface p.
type Deps struct {
	infra.PluginDeps
	Scheduler     scheduler.KVScheduler
	GoVppmux      govppmux.API

	/* optional, provide if AFPacket or TAP+AUTO_TAP interfaces are used */
	LinuxIfPlugin linux_ifplugin.API

	// state publishing
	StatusCheck       statuscheck.PluginStatusWriter
	PublishErrors     datasync.KeyProtoValWriter
	Watcher           datasync.KeyValProtoWatcher /* for resync of interface state data (PublishStatistics) */
	NotifyStatistics  datasync.KeyProtoValWriter  /* e.g. Kafka */
	PublishStatistics datasync.KeyProtoValWriter  /* e.g. ETCD (with resync) */
	DataSyncs         map[string]datasync.KeyProtoValWriter /* available DBs for PublishStatistics */
	// TODO: GRPCSvc           rpc.GRPCService
}

// Config holds the vpp-plugin configuration.
type Config struct {
	Mtu              uint32   `json:"mtu"`
	Stopwatch        bool     `json:"stopwatch"`
	StatusPublishers []string `json:"status-publishers"`
}

// Init loads configuration file and registers interface-related descriptors.
func (p *IfPlugin) Init() error {
	var err error
	// Read config file and set all related fields
	p.fromConfigFile()

	// Fills nil dependencies with default values
	p.fixNilPointers()

	// Plugin-wide stopwatch instance
	if p.enableStopwatch {
		p.stopwatch = measure.NewStopwatch(string(p.PluginName), p.Log)
	}

	// VPP channel
	if p.vppCh, err = p.GoVppmux.NewAPIChannel(); err != nil {
		return errors.Errorf("failed to create GoVPP API channel: %v", err)
	}

	// init handlers
	p.ifHandler = vppcalls.NewIfVppHandler(p.vppCh, p.Log, p.stopwatch)
	if p.LinuxIfPlugin != nil {
		p.linuxIfHandler = linux_ifcalls.NewNetLinkHandler(p.stopwatch)
	}

	// init descriptors
	p.ifDescriptor = descriptor.NewInterfaceDescriptor(p.ifHandler, p.defaultMtu,
		p.linuxIfHandler, p.LinuxIfPlugin, p.Log)
	ifDescriptor := adapter.NewInterfaceDescriptor(p.ifDescriptor.GetDescriptor())
	p.unIfDescriptor = descriptor.NewUnnumberedIfDescriptor(p.ifHandler, p.Log)
	unIfDescriptor := adapter.NewUnnumberedDescriptor(p.unIfDescriptor.GetDescriptor())
	p.dhcpDescriptor = descriptor.NewDHCPDescriptor(p.Scheduler, p.ifHandler, p.Log)
	dhcpDescriptor := p.dhcpDescriptor.GetDescriptor()

	// register descriptors
	p.Deps.Scheduler.RegisterKVDescriptor(ifDescriptor)
	p.Deps.Scheduler.RegisterKVDescriptor(unIfDescriptor)
	p.Deps.Scheduler.RegisterKVDescriptor(dhcpDescriptor)

	// obtain read-only reference to index map
	var withIndex bool
	metadataMap := p.Deps.Scheduler.GetMetadataMap(ifDescriptor.Name)
	p.intfIndex, withIndex = metadataMap.(ifaceidx.IfaceMetadataIndex)
	if !withIndex {
		return errors.New("missing index with interface metadata")
	}
	p.dhcpIndex = p.Deps.Scheduler.GetMetadataMap(dhcpDescriptor.Name)

	// pass read-only index map to descriptors
	p.ifDescriptor.SetInterfaceIndex(p.intfIndex)
	p.unIfDescriptor.SetInterfaceIndex(p.intfIndex)
	p.dhcpDescriptor.SetInterfaceIndex(p.intfIndex)

	// start watching for DHCP notifications
	dhcpChan := make(chan govppapi.Message, 1)
	if _, err := p.vppCh.SubscribeNotification(dhcpChan, &dhcp.DHCPComplEvent{}); err != nil {
		return err
	}
	p.dhcpDescriptor.StartWatchingDHCP(dhcpChan)

	// Create plugin context, save cancel function into the plugin handle.
	p.ctx, p.cancel = context.WithCancel(context.Background())

	// subscribe & watch for resync of interface state data
	p.resyncStatusChan = make(chan datasync.ResyncEvent)
	p.watchStatusReg, err = p.Watcher.
		Watch("VPP interface state data", nil, p.resyncStatusChan,
		interfaces.StatePrefix)
	if err != nil {
		return err
	}
	go p.watchStatusEvents()

	// start interface state publishing
	go p.publishIfStateEvents()

	// start interface state updater
	p.ifStateChan = make(chan *interfaces.InterfaceNotification, 100)
	// Interface state updater
	p.ifStateUpdater = &InterfaceStateUpdater{}
	if err := p.ifStateUpdater.Init(p.ctx, p.Log, p.GoVppmux, p.intfIndex, func(state *interfaces.InterfaceNotification) {
		select {
		case p.ifStateChan <- state:
			// OK
		default:
			p.Log.Debug("Unable to send to the ifStateNotifications channel - channel buffer full.")
		}
	}); err != nil {
		return p.ifStateUpdater.LogError(err)
	}

	p.Log.Debug("ifStateUpdater Initialized")

	return nil
}

// AfterInit delegates the call to ifStateUpdater.
func (p *IfPlugin) AfterInit() error {
	err := p.ifStateUpdater.AfterInit()
	if err != nil {
		return err
	}

	if p.StatusCheck != nil {
		// Register the plugin to status check. Periodical probe is not needed,
		// data change will be reported when changed
		p.StatusCheck.Register(p.PluginName, nil)
		// Notify that status check for the plugins was registered. It will
		// prevent status report errors in case resync is executed before AfterInit.
		p.statusCheckReg = true
	}

	return nil
}

// Close stops all go routines.
func (p *IfPlugin) Close() error {
	// stop publishing of state data
	p.cancel()
	p.wg.Wait()

	// stop watching of DHCP notifications
	p.dhcpDescriptor.StopWatchingDHCP()

	// close all channels
	safeclose.CloseAll(p.resyncStatusChan, p.ifStateChan, p.ifNotifChan)
	return nil
}

// GetInterfaceIndex gives read-only access to map with metadata of all configured
// VPP interfaces.
func (p *IfPlugin) GetInterfaceIndex() ifaceidx.IfaceMetadataIndex {
	return p.intfIndex
}

// GetDHCPIndex gives read-only access to (untyped) map with DHCP leases.
// Cast metadata to "github.com/ligato/vpp-agent/plugins/vppv2/model/interfaces".DHCPLease
func (p *IfPlugin) GetDHCPIndex() idxmap.NamedMapping {
	return p.dhcpIndex
}

// fromConfigFile loads plugin attributes from the configuration file.
func (p *IfPlugin) fromConfigFile() {
	config, err := p.loadConfig()
	if err != nil {
		p.Log.Errorf("Error reading %v config file: %v", p.PluginName, err)
		return
	}
	if config != nil {
		publishers := datasync.KVProtoWriters{}
		for _, pub := range config.StatusPublishers {
			db, found := p.Deps.DataSyncs[pub]
			if !found {
				p.Log.Warnf("Unknown status publisher %q from config", pub)
				continue
			}
			publishers = append(publishers, db)
			p.Log.Infof("Added status publisher %q from config", pub)
		}
		p.Deps.PublishStatistics = publishers
		if config.Mtu != 0 {
			p.defaultMtu = config.Mtu
			p.Log.Infof("Default MTU set to %v", p.defaultMtu)
		}

		if config.Stopwatch {
			p.enableStopwatch = true
			p.Log.Info("stopwatch enabled for %v", p.PluginName)
		} else {
			p.Log.Info("stopwatch disabled for %v", p.PluginName)
		}
	} else {
		p.Log.Infof("stopwatch disabled for %v", p.PluginName)
	}
}

// loadConfig loads configuration file.
func (p *IfPlugin) loadConfig() (*Config, error) {
	config := &Config{}

	found, err := p.Cfg.LoadValue(config)
	if err != nil {
		return nil, err
	} else if !found {
		p.Log.Debugf("%v config not found", p.PluginName)
		return nil, nil
	}
	p.Log.Debugf("%v config found: %+v", p.PluginName, config)

	if pubs := os.Getenv(vppStatusPublishersEnv); pubs != "" {
		p.Log.Debugf("status publishers from env: %v", pubs)
		config.StatusPublishers = append(config.StatusPublishers, pubs)
	}

	return config, err
}

// fixNilPointers sets noopWriter & nooWatcher for nil dependencies.
func (p *IfPlugin) fixNilPointers() {
	if p.Deps.PublishErrors == nil {
		p.Deps.PublishErrors = noopWriter
		p.Log.Debug("setting default noop writer for PublishErrors dependency")
	}
	if p.Deps.PublishStatistics == nil {
		p.Deps.PublishStatistics = noopWriter
		p.Log.Debug("setting default noop writer for PublishStatistics dependency")
	}
	if p.Deps.NotifyStatistics == nil {
		p.Deps.NotifyStatistics = noopWriter
		p.Log.Debug("setting default noop writer for NotifyStatistics dependency")
	}
	if p.Deps.Watcher == nil {
		p.Deps.Watcher = noopWatcher
		p.Log.Debug("setting default noop watcher for Watcher dependency")
	}
}<commit_msg>Remove unused channel.<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate descriptor-adapter --descriptor-name Interface  --value-type *interfaces.Interface --meta-type *ifaceidx.IfaceMetadata --import "../model/interfaces" --import "ifaceidx" --output-dir "descriptor"
//go:generate descriptor-adapter --descriptor-name Unnumbered  --value-type *interfaces.Interface_Unnumbered --import "../model/interfaces" --output-dir "descriptor"

package ifplugin

import (
	"context"
	"os"
	"sync"

	"github.com/go-errors/errors"
	govppapi "git.fd.io/govpp.git/api"

	"github.com/ligato/cn-infra/infra"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/logging/measure"
	"github.com/ligato/cn-infra/idxmap"
	"github.com/ligato/cn-infra/health/statuscheck"
	"github.com/ligato/cn-infra/utils/safeclose"

	scheduler "github.com/ligato/vpp-agent/plugins/kvscheduler/api"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/descriptor"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/descriptor/adapter"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/ifaceidx"
	"github.com/ligato/vpp-agent/plugins/vppv2/ifplugin/vppcalls"
	"github.com/ligato/vpp-agent/plugins/vppv2/model/interfaces"
	linux_ifcalls "github.com/ligato/vpp-agent/plugins/linuxv2/ifplugin/linuxcalls"
	linux_ifplugin "github.com/ligato/vpp-agent/plugins/linuxv2/ifplugin"
	"github.com/ligato/vpp-agent/plugins/vpp/binapi/dhcp"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

const (
	// vppStatusPublishersEnv is the name of the environment variable used to
	// override state publishers from the configuration file.
	vppStatusPublishersEnv = "VPP_STATUS_PUBLISHERS"
)

var (
	// noopWriter (no operation writer) helps avoiding NIL pointer based segmentation fault.
	// It is used as default if some dependency was not injected.
	noopWriter = datasync.KVProtoWriters{}

	// noopWatcher (no operation watcher) helps avoiding NIL pointer based segmentation fault.
	// It is used as default if some dependency was not injected.
	noopWatcher = datasync.KVProtoWatchers{}
)

// IfPlugin configures VPP interfaces using GoVPP.
type IfPlugin struct {
	Deps

	// GoVPP
	vppCh     govppapi.Channel

	// handlers
	ifHandler      vppcalls.IfVppAPI
	linuxIfHandler linux_ifcalls.NetlinkAPI

	// descriptors
	ifDescriptor   *descriptor.InterfaceDescriptor
	unIfDescriptor *descriptor.UnnumberedIfDescriptor
	dhcpDescriptor *descriptor.DHCPDescriptor

	// index maps
	intfIndex ifaceidx.IfaceMetadataIndex
	dhcpIndex idxmap.NamedMapping

	// From config file
	enableStopwatch bool
	stopwatch       *measure.Stopwatch // timer used to measure and store time
	defaultMtu      uint32

	// state data
	statusCheckReg   bool
	watchStatusReg   datasync.WatchRegistration
	resyncStatusChan chan datasync.ResyncEvent
	ifStateChan      chan *interfaces.InterfaceNotification
	ifStateUpdater   *InterfaceStateUpdater

	// go routine management
	ctx    context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup
}

// Deps lists dependencies of the interface p.
type Deps struct {
	infra.PluginDeps
	Scheduler     scheduler.KVScheduler
	GoVppmux      govppmux.API

	/* optional, provide if AFPacket or TAP+AUTO_TAP interfaces are used */
	LinuxIfPlugin linux_ifplugin.API

	// state publishing
	StatusCheck       statuscheck.PluginStatusWriter
	PublishErrors     datasync.KeyProtoValWriter
	Watcher           datasync.KeyValProtoWatcher /* for resync of interface state data (PublishStatistics) */
	NotifyStatistics  datasync.KeyProtoValWriter  /* e.g. Kafka */
	PublishStatistics datasync.KeyProtoValWriter  /* e.g. ETCD (with resync) */
	DataSyncs         map[string]datasync.KeyProtoValWriter /* available DBs for PublishStatistics */
	// TODO: GRPCSvc           rpc.GRPCService
}

// Config holds the vpp-plugin configuration.
type Config struct {
	Mtu              uint32   `json:"mtu"`
	Stopwatch        bool     `json:"stopwatch"`
	StatusPublishers []string `json:"status-publishers"`
}

// Init loads configuration file and registers interface-related descriptors.
func (p *IfPlugin) Init() error {
	var err error
	// Read config file and set all related fields
	p.fromConfigFile()

	// Fills nil dependencies with default values
	p.fixNilPointers()

	// Plugin-wide stopwatch instance
	if p.enableStopwatch {
		p.stopwatch = measure.NewStopwatch(string(p.PluginName), p.Log)
	}

	// VPP channel
	if p.vppCh, err = p.GoVppmux.NewAPIChannel(); err != nil {
		return errors.Errorf("failed to create GoVPP API channel: %v", err)
	}

	// init handlers
	p.ifHandler = vppcalls.NewIfVppHandler(p.vppCh, p.Log, p.stopwatch)
	if p.LinuxIfPlugin != nil {
		p.linuxIfHandler = linux_ifcalls.NewNetLinkHandler(p.stopwatch)
	}

	// init descriptors
	p.ifDescriptor = descriptor.NewInterfaceDescriptor(p.ifHandler, p.defaultMtu,
		p.linuxIfHandler, p.LinuxIfPlugin, p.Log)
	ifDescriptor := adapter.NewInterfaceDescriptor(p.ifDescriptor.GetDescriptor())
	p.unIfDescriptor = descriptor.NewUnnumberedIfDescriptor(p.ifHandler, p.Log)
	unIfDescriptor := adapter.NewUnnumberedDescriptor(p.unIfDescriptor.GetDescriptor())
	p.dhcpDescriptor = descriptor.NewDHCPDescriptor(p.Scheduler, p.ifHandler, p.Log)
	dhcpDescriptor := p.dhcpDescriptor.GetDescriptor()

	// register descriptors
	p.Deps.Scheduler.RegisterKVDescriptor(ifDescriptor)
	p.Deps.Scheduler.RegisterKVDescriptor(unIfDescriptor)
	p.Deps.Scheduler.RegisterKVDescriptor(dhcpDescriptor)

	// obtain read-only reference to index map
	var withIndex bool
	metadataMap := p.Deps.Scheduler.GetMetadataMap(ifDescriptor.Name)
	p.intfIndex, withIndex = metadataMap.(ifaceidx.IfaceMetadataIndex)
	if !withIndex {
		return errors.New("missing index with interface metadata")
	}
	p.dhcpIndex = p.Deps.Scheduler.GetMetadataMap(dhcpDescriptor.Name)

	// pass read-only index map to descriptors
	p.ifDescriptor.SetInterfaceIndex(p.intfIndex)
	p.unIfDescriptor.SetInterfaceIndex(p.intfIndex)
	p.dhcpDescriptor.SetInterfaceIndex(p.intfIndex)

	// start watching for DHCP notifications
	dhcpChan := make(chan govppapi.Message, 1)
	if _, err := p.vppCh.SubscribeNotification(dhcpChan, &dhcp.DHCPComplEvent{}); err != nil {
		return err
	}
	p.dhcpDescriptor.StartWatchingDHCP(dhcpChan)

	// Create plugin context, save cancel function into the plugin handle.
	p.ctx, p.cancel = context.WithCancel(context.Background())

	// subscribe & watch for resync of interface state data
	p.resyncStatusChan = make(chan datasync.ResyncEvent)
	p.watchStatusReg, err = p.Watcher.
		Watch("VPP interface state data", nil, p.resyncStatusChan,
		interfaces.StatePrefix)
	if err != nil {
		return err
	}
	go p.watchStatusEvents()

	// start interface state publishing
	go p.publishIfStateEvents()

	// start interface state updater
	p.ifStateChan = make(chan *interfaces.InterfaceNotification, 100)
	// Interface state updater
	p.ifStateUpdater = &InterfaceStateUpdater{}
	if err := p.ifStateUpdater.Init(p.ctx, p.Log, p.GoVppmux, p.intfIndex, func(state *interfaces.InterfaceNotification) {
		select {
		case p.ifStateChan <- state:
			// OK
		default:
			p.Log.Debug("Unable to send to the ifStateNotifications channel - channel buffer full.")
		}
	}); err != nil {
		return p.ifStateUpdater.LogError(err)
	}

	p.Log.Debug("ifStateUpdater Initialized")

	return nil
}

// AfterInit delegates the call to ifStateUpdater.
func (p *IfPlugin) AfterInit() error {
	err := p.ifStateUpdater.AfterInit()
	if err != nil {
		return err
	}

	if p.StatusCheck != nil {
		// Register the plugin to status check. Periodical probe is not needed,
		// data change will be reported when changed
		p.StatusCheck.Register(p.PluginName, nil)
		// Notify that status check for the plugins was registered. It will
		// prevent status report errors in case resync is executed before AfterInit.
		p.statusCheckReg = true
	}

	return nil
}

// Close stops all go routines.
func (p *IfPlugin) Close() error {
	// stop publishing of state data
	p.cancel()
	p.wg.Wait()

	// stop watching of DHCP notifications
	p.dhcpDescriptor.StopWatchingDHCP()

	// close all channels
	safeclose.CloseAll(p.resyncStatusChan, p.ifStateChan)
	return nil
}

// GetInterfaceIndex gives read-only access to map with metadata of all configured
// VPP interfaces.
func (p *IfPlugin) GetInterfaceIndex() ifaceidx.IfaceMetadataIndex {
	return p.intfIndex
}

// GetDHCPIndex gives read-only access to (untyped) map with DHCP leases.
// Cast metadata to "github.com/ligato/vpp-agent/plugins/vppv2/model/interfaces".DHCPLease
func (p *IfPlugin) GetDHCPIndex() idxmap.NamedMapping {
	return p.dhcpIndex
}

// fromConfigFile loads plugin attributes from the configuration file.
func (p *IfPlugin) fromConfigFile() {
	config, err := p.loadConfig()
	if err != nil {
		p.Log.Errorf("Error reading %v config file: %v", p.PluginName, err)
		return
	}
	if config != nil {
		publishers := datasync.KVProtoWriters{}
		for _, pub := range config.StatusPublishers {
			db, found := p.Deps.DataSyncs[pub]
			if !found {
				p.Log.Warnf("Unknown status publisher %q from config", pub)
				continue
			}
			publishers = append(publishers, db)
			p.Log.Infof("Added status publisher %q from config", pub)
		}
		p.Deps.PublishStatistics = publishers
		if config.Mtu != 0 {
			p.defaultMtu = config.Mtu
			p.Log.Infof("Default MTU set to %v", p.defaultMtu)
		}

		if config.Stopwatch {
			p.enableStopwatch = true
			p.Log.Info("stopwatch enabled for %v", p.PluginName)
		} else {
			p.Log.Info("stopwatch disabled for %v", p.PluginName)
		}
	} else {
		p.Log.Infof("stopwatch disabled for %v", p.PluginName)
	}
}

// loadConfig loads configuration file.
func (p *IfPlugin) loadConfig() (*Config, error) {
	config := &Config{}

	found, err := p.Cfg.LoadValue(config)
	if err != nil {
		return nil, err
	} else if !found {
		p.Log.Debugf("%v config not found", p.PluginName)
		return nil, nil
	}
	p.Log.Debugf("%v config found: %+v", p.PluginName, config)

	if pubs := os.Getenv(vppStatusPublishersEnv); pubs != "" {
		p.Log.Debugf("status publishers from env: %v", pubs)
		config.StatusPublishers = append(config.StatusPublishers, pubs)
	}

	return config, err
}

// fixNilPointers sets noopWriter & nooWatcher for nil dependencies.
func (p *IfPlugin) fixNilPointers() {
	if p.Deps.PublishErrors == nil {
		p.Deps.PublishErrors = noopWriter
		p.Log.Debug("setting default noop writer for PublishErrors dependency")
	}
	if p.Deps.PublishStatistics == nil {
		p.Deps.PublishStatistics = noopWriter
		p.Log.Debug("setting default noop writer for PublishStatistics dependency")
	}
	if p.Deps.NotifyStatistics == nil {
		p.Deps.NotifyStatistics = noopWriter
		p.Log.Debug("setting default noop writer for NotifyStatistics dependency")
	}
	if p.Deps.Watcher == nil {
		p.Deps.Watcher = noopWatcher
		p.Log.Debug("setting default noop watcher for Watcher dependency")
	}
}<|endoftext|>


<commit_before>package packfile

import (
	"bytes"
	"errors"
	"io"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/cache"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
)

var (
	// ErrReferenceDeltaNotFound is returned when the reference delta is not
	// found.
	ErrReferenceDeltaNotFound = errors.New("reference delta not found")

	// ErrNotSeekableSource is returned when the source for the parser is not
	// seekable and a storage was not provided, so it can't be parsed.
	ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided")

	// ErrDeltaNotCached is returned when the delta could not be found in cache.
	ErrDeltaNotCached = errors.New("delta could not be found in cache")
)

// Observer interface is implemented by index encoders.
type Observer interface {
	// OnHeader is called when a new packfile is opened.
	OnHeader(count uint32) error
	// OnInflatedObjectHeader is called for each object header read.
	OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error
	// OnInflatedObjectContent is called for each decoded object.
	OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, content []byte) error
	// OnFooter is called when decoding is done.
	OnFooter(h plumbing.Hash) error
}

// Parser decodes a packfile and calls any observer associated to it. Is used
// to generate indexes.
type Parser struct {
	storage          storer.EncodedObjectStorer
	scanner          *Scanner
	count            uint32
	oi               []*objectInfo
	oiByHash         map[plumbing.Hash]*objectInfo
	oiByOffset       map[int64]*objectInfo
	hashOffset       map[plumbing.Hash]int64
	pendingRefDeltas map[plumbing.Hash][]*objectInfo
	checksum         plumbing.Hash

	cache *cache.BufferLRU
	// delta content by offset, only used if source is not seekable
	deltas map[int64][]byte

	ob []Observer
}

// NewParser creates a new Parser. The Scanner source must be seekable.
// If it's not, NewParserWithStorage should be used instead.
func NewParser(scanner *Scanner, ob ...Observer) (*Parser, error) {
	return NewParserWithStorage(scanner, nil, ob...)
}

// NewParserWithStorage creates a new Parser. The scanner source must either
// be seekable or a storage must be provided.
func NewParserWithStorage(
	scanner *Scanner,
	storage storer.EncodedObjectStorer,
	ob ...Observer,
) (*Parser, error) {
	if !scanner.IsSeekable && storage == nil {
		return nil, ErrNotSeekableSource
	}

	var deltas map[int64][]byte
	if !scanner.IsSeekable {
		deltas = make(map[int64][]byte)
	}

	return &Parser{
		storage:          storage,
		scanner:          scanner,
		ob:               ob,
		count:            0,
		cache:            cache.NewBufferLRUDefault(),
		pendingRefDeltas: make(map[plumbing.Hash][]*objectInfo),
		deltas:           deltas,
	}, nil
}

func (p *Parser) forEachObserver(f func(o Observer) error) error {
	for _, o := range p.ob {
		if err := f(o); err != nil {
			return err
		}
	}
	return nil
}

func (p *Parser) onHeader(count uint32) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnHeader(count)
	})
}

func (p *Parser) onInflatedObjectHeader(
	t plumbing.ObjectType,
	objSize int64,
	pos int64,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectHeader(t, objSize, pos)
	})
}

func (p *Parser) onInflatedObjectContent(
	h plumbing.Hash,
	pos int64,
	crc uint32,
	content []byte,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectContent(h, pos, crc, content)
	})
}

func (p *Parser) onFooter(h plumbing.Hash) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnFooter(h)
	})
}

// Parse start decoding phase of the packfile.
func (p *Parser) Parse() (plumbing.Hash, error) {
	if err := p.init(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.indexObjects(); err != nil {
		return plumbing.ZeroHash, err
	}

	var err error
	p.checksum, err = p.scanner.Checksum()
	if err != nil && err != io.EOF {
		return plumbing.ZeroHash, err
	}

	if err := p.resolveDeltas(); err != nil {
		return plumbing.ZeroHash, err
	}

	if len(p.pendingRefDeltas) > 0 {
		return plumbing.ZeroHash, ErrReferenceDeltaNotFound
	}

	if err := p.onFooter(p.checksum); err != nil {
		return plumbing.ZeroHash, err
	}

	return p.checksum, nil
}

func (p *Parser) init() error {
	_, c, err := p.scanner.Header()
	if err != nil {
		return err
	}

	if err := p.onHeader(c); err != nil {
		return err
	}

	p.count = c
	p.oiByHash = make(map[plumbing.Hash]*objectInfo, p.count)
	p.oiByOffset = make(map[int64]*objectInfo, p.count)
	p.oi = make([]*objectInfo, p.count)

	return nil
}

func (p *Parser) indexObjects() error {
	buf := new(bytes.Buffer)

	for i := uint32(0); i < p.count; i++ {
		buf.Reset()

		oh, err := p.scanner.NextObjectHeader()
		if err != nil {
			return err
		}

		delta := false
		var ota *objectInfo
		switch t := oh.Type; t {
		case plumbing.OFSDeltaObject:
			delta = true

			parent, ok := p.oiByOffset[oh.OffsetReference]
			if !ok {
				return plumbing.ErrObjectNotFound
			}

			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)
		case plumbing.REFDeltaObject:
			delta = true

			parent, ok := p.oiByHash[oh.Reference]
			if ok {
				ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
				parent.Children = append(parent.Children, ota)
			} else {
				ota = newBaseObject(oh.Offset, oh.Length, t)
				p.pendingRefDeltas[oh.Reference] = append(
					p.pendingRefDeltas[oh.Reference],
					ota,
				)
			}
		default:
			ota = newBaseObject(oh.Offset, oh.Length, t)
		}

		_, crc, err := p.scanner.NextObject(buf)
		if err != nil {
			return err
		}

		ota.Crc32 = crc
		ota.Length = oh.Length

		data := buf.Bytes()
		if !delta {
			sha1, err := getSHA1(ota.Type, data)
			if err != nil {
				return err
			}

			ota.SHA1 = sha1
			p.oiByHash[ota.SHA1] = ota
		}

		if p.storage != nil && !delta {
			obj := new(plumbing.MemoryObject)
			obj.SetSize(oh.Length)
			obj.SetType(oh.Type)
			if _, err := obj.Write(data); err != nil {
				return err
			}

			if _, err := p.storage.SetEncodedObject(obj); err != nil {
				return err
			}
		}

		if delta && !p.scanner.IsSeekable {
			p.deltas[oh.Offset] = make([]byte, len(data))
			copy(p.deltas[oh.Offset], data)
		}

		p.oiByOffset[oh.Offset] = ota
		p.oi[i] = ota
	}

	return nil
}

func (p *Parser) resolveDeltas() error {
	for _, obj := range p.oi {
		content, err := p.get(obj)
		if err != nil {
			return err
		}

		if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil {
			return err
		}

		if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil {
			return err
		}

		if !obj.IsDelta() && len(obj.Children) > 0 {
			for _, child := range obj.Children {
				if _, err := p.resolveObject(child, content); err != nil {
					return err
				}
			}

			// Remove the delta from the cache.
			if obj.DiskType.IsDelta() && !p.scanner.IsSeekable {
				delete(p.deltas, obj.Offset)
			}
		}
	}

	return nil
}

func (p *Parser) get(o *objectInfo) ([]byte, error) {
	b, ok := p.cache.Get(o.Offset)
	// If it's not on the cache and is not a delta we can try to find it in the
	// storage, if there's one.
	if !ok && p.storage != nil && !o.Type.IsDelta() {
		var err error
		e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1)
		if err != nil {
			return nil, err
		}

		r, err := e.Reader()
		if err != nil {
			return nil, err
		}

		b = make([]byte, e.Size())
		if _, err = r.Read(b); err != nil {
			return nil, err
		}
	}

	if b != nil {
		return b, nil
	}

	var data []byte
	if o.DiskType.IsDelta() {
		base, err := p.get(o.Parent)
		if err != nil {
			return nil, err
		}

		data, err = p.resolveObject(o, base)
		if err != nil {
			return nil, err
		}
	} else {
		var err error
		data, err = p.readData(o)
		if err != nil {
			return nil, err
		}
	}

	if len(o.Children) > 0 {
		p.cache.Put(o.Offset, data)
	}

	return data, nil
}

func (p *Parser) resolveObject(
	o *objectInfo,
	base []byte,
) ([]byte, error) {
	if !o.DiskType.IsDelta() {
		return nil, nil
	}

	data, err := p.readData(o)
	if err != nil {
		return nil, err
	}

	data, err = applyPatchBase(o, data, base)
	if err != nil {
		return nil, err
	}

	if pending, ok := p.pendingRefDeltas[o.SHA1]; ok {
		for _, po := range pending {
			po.Parent = o
			o.Children = append(o.Children, po)
		}
		delete(p.pendingRefDeltas, o.SHA1)
	}

	if p.storage != nil {
		obj := new(plumbing.MemoryObject)
		obj.SetSize(o.Size())
		obj.SetType(o.Type)
		if _, err := obj.Write(data); err != nil {
			return nil, err
		}

		if _, err := p.storage.SetEncodedObject(obj); err != nil {
			return nil, err
		}
	}

	return data, nil
}

func (p *Parser) readData(o *objectInfo) ([]byte, error) {
	if !p.scanner.IsSeekable && o.DiskType.IsDelta() {
		data, ok := p.deltas[o.Offset]
		if !ok {
			return nil, ErrDeltaNotCached
		}

		return data, nil
	}

	if _, err := p.scanner.SeekFromStart(o.Offset); err != nil {
		return nil, err
	}

	if _, err := p.scanner.NextObjectHeader(); err != nil {
		return nil, err
	}

	buf := new(bytes.Buffer)
	if _, _, err := p.scanner.NextObject(buf); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) {
	patched, err := PatchDelta(base, data)
	if err != nil {
		return nil, err
	}

	if ota.SHA1 == plumbing.ZeroHash {
		ota.Type = ota.Parent.Type
		sha1, err := getSHA1(ota.Type, patched)
		if err != nil {
			return nil, err
		}

		ota.SHA1 = sha1
		ota.Length = int64(len(patched))
	}

	return patched, nil
}

func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) {
	hasher := plumbing.NewHasher(t, int64(len(data)))
	if _, err := hasher.Write(data); err != nil {
		return plumbing.ZeroHash, err
	}

	return hasher.Sum(), nil
}

type objectInfo struct {
	Offset   int64
	Length   int64
	Type     plumbing.ObjectType
	DiskType plumbing.ObjectType

	Crc32 uint32

	Parent   *objectInfo
	Children []*objectInfo
	SHA1     plumbing.Hash
}

func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo {
	return newDeltaObject(offset, length, t, nil)
}

func newDeltaObject(
	offset, length int64,
	t plumbing.ObjectType,
	parent *objectInfo,
) *objectInfo {
	obj := &objectInfo{
		Offset:   offset,
		Length:   length,
		Type:     t,
		DiskType: t,
		Crc32:    0,
		Parent:   parent,
	}

	return obj
}

func (o *objectInfo) IsDelta() bool {
	return o.Type.IsDelta()
}

func (o *objectInfo) Size() int64 {
	return o.Length
}
<commit_msg>plumbing/format/packfile: Fix broken "thin" packfile support. Fixes #991<commit_after>package packfile

import (
	"bytes"
	"errors"
	"io"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/cache"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
)

var (
	// ErrReferenceDeltaNotFound is returned when the reference delta is not
	// found.
	ErrReferenceDeltaNotFound = errors.New("reference delta not found")

	// ErrNotSeekableSource is returned when the source for the parser is not
	// seekable and a storage was not provided, so it can't be parsed.
	ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided")

	// ErrDeltaNotCached is returned when the delta could not be found in cache.
	ErrDeltaNotCached = errors.New("delta could not be found in cache")
)

// Observer interface is implemented by index encoders.
type Observer interface {
	// OnHeader is called when a new packfile is opened.
	OnHeader(count uint32) error
	// OnInflatedObjectHeader is called for each object header read.
	OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error
	// OnInflatedObjectContent is called for each decoded object.
	OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, content []byte) error
	// OnFooter is called when decoding is done.
	OnFooter(h plumbing.Hash) error
}

// Parser decodes a packfile and calls any observer associated to it. Is used
// to generate indexes.
type Parser struct {
	storage    storer.EncodedObjectStorer
	scanner    *Scanner
	count      uint32
	oi         []*objectInfo
	oiByHash   map[plumbing.Hash]*objectInfo
	oiByOffset map[int64]*objectInfo
	hashOffset map[plumbing.Hash]int64
	checksum   plumbing.Hash

	cache *cache.BufferLRU
	// delta content by offset, only used if source is not seekable
	deltas map[int64][]byte

	ob []Observer
}

// NewParser creates a new Parser. The Scanner source must be seekable.
// If it's not, NewParserWithStorage should be used instead.
func NewParser(scanner *Scanner, ob ...Observer) (*Parser, error) {
	return NewParserWithStorage(scanner, nil, ob...)
}

// NewParserWithStorage creates a new Parser. The scanner source must either
// be seekable or a storage must be provided.
func NewParserWithStorage(
	scanner *Scanner,
	storage storer.EncodedObjectStorer,
	ob ...Observer,
) (*Parser, error) {
	if !scanner.IsSeekable && storage == nil {
		return nil, ErrNotSeekableSource
	}

	var deltas map[int64][]byte
	if !scanner.IsSeekable {
		deltas = make(map[int64][]byte)
	}

	return &Parser{
		storage: storage,
		scanner: scanner,
		ob:      ob,
		count:   0,
		cache:   cache.NewBufferLRUDefault(),
		deltas:  deltas,
	}, nil
}

func (p *Parser) forEachObserver(f func(o Observer) error) error {
	for _, o := range p.ob {
		if err := f(o); err != nil {
			return err
		}
	}
	return nil
}

func (p *Parser) onHeader(count uint32) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnHeader(count)
	})
}

func (p *Parser) onInflatedObjectHeader(
	t plumbing.ObjectType,
	objSize int64,
	pos int64,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectHeader(t, objSize, pos)
	})
}

func (p *Parser) onInflatedObjectContent(
	h plumbing.Hash,
	pos int64,
	crc uint32,
	content []byte,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectContent(h, pos, crc, content)
	})
}

func (p *Parser) onFooter(h plumbing.Hash) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnFooter(h)
	})
}

// Parse start decoding phase of the packfile.
func (p *Parser) Parse() (plumbing.Hash, error) {
	if err := p.init(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.indexObjects(); err != nil {
		return plumbing.ZeroHash, err
	}

	var err error
	p.checksum, err = p.scanner.Checksum()
	if err != nil && err != io.EOF {
		return plumbing.ZeroHash, err
	}

	if err := p.resolveDeltas(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.onFooter(p.checksum); err != nil {
		return plumbing.ZeroHash, err
	}

	return p.checksum, nil
}

func (p *Parser) init() error {
	_, c, err := p.scanner.Header()
	if err != nil {
		return err
	}

	if err := p.onHeader(c); err != nil {
		return err
	}

	p.count = c
	p.oiByHash = make(map[plumbing.Hash]*objectInfo, p.count)
	p.oiByOffset = make(map[int64]*objectInfo, p.count)
	p.oi = make([]*objectInfo, p.count)

	return nil
}

func (p *Parser) indexObjects() error {
	buf := new(bytes.Buffer)

	for i := uint32(0); i < p.count; i++ {
		buf.Reset()

		oh, err := p.scanner.NextObjectHeader()
		if err != nil {
			return err
		}

		delta := false
		var ota *objectInfo
		switch t := oh.Type; t {
		case plumbing.OFSDeltaObject:
			delta = true

			parent, ok := p.oiByOffset[oh.OffsetReference]
			if !ok {
				return plumbing.ErrObjectNotFound
			}

			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)
		case plumbing.REFDeltaObject:
			delta = true
			parent, ok := p.oiByHash[oh.Reference]
			if !ok {
				// can't find referenced object in this pack file
				// this must be a "thin" pack.
				parent = &objectInfo{ //Placeholder parent
					SHA1:        oh.Reference,
					ExternalRef: true, // mark as an external reference that must be resolved
					Type:        plumbing.AnyObject,
					DiskType:    plumbing.AnyObject,
				}
				p.oiByHash[oh.Reference] = parent
			}
			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)

		default:
			ota = newBaseObject(oh.Offset, oh.Length, t)
		}

		_, crc, err := p.scanner.NextObject(buf)
		if err != nil {
			return err
		}

		ota.Crc32 = crc
		ota.Length = oh.Length

		data := buf.Bytes()
		if !delta {
			sha1, err := getSHA1(ota.Type, data)
			if err != nil {
				return err
			}

			ota.SHA1 = sha1
			p.oiByHash[ota.SHA1] = ota
		}

		if p.storage != nil && !delta {
			obj := new(plumbing.MemoryObject)
			obj.SetSize(oh.Length)
			obj.SetType(oh.Type)
			if _, err := obj.Write(data); err != nil {
				return err
			}

			if _, err := p.storage.SetEncodedObject(obj); err != nil {
				return err
			}
		}

		if delta && !p.scanner.IsSeekable {
			p.deltas[oh.Offset] = make([]byte, len(data))
			copy(p.deltas[oh.Offset], data)
		}

		p.oiByOffset[oh.Offset] = ota
		p.oi[i] = ota
	}

	return nil
}

func (p *Parser) resolveDeltas() error {
	for _, obj := range p.oi {
		content, err := p.get(obj)
		if err != nil {
			return err
		}

		if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil {
			return err
		}

		if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil {
			return err
		}

		if !obj.IsDelta() && len(obj.Children) > 0 {
			for _, child := range obj.Children {
				if _, err := p.resolveObject(child, content); err != nil {
					return err
				}
			}

			// Remove the delta from the cache.
			if obj.DiskType.IsDelta() && !p.scanner.IsSeekable {
				delete(p.deltas, obj.Offset)
			}
		}
	}

	return nil
}

func (p *Parser) get(o *objectInfo) (b []byte, err error) {
	var ok bool
	if !o.ExternalRef { // skip cache check for placeholder parents
		b, ok = p.cache.Get(o.Offset)
	}

	// If it's not on the cache and is not a delta we can try to find it in the
	// storage, if there's one. External refs must enter here.
	if !ok && p.storage != nil && !o.Type.IsDelta() {
		e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1)
		if err != nil {
			return nil, err
		}
		o.Type = e.Type()

		r, err := e.Reader()
		if err != nil {
			return nil, err
		}

		b = make([]byte, e.Size())
		if _, err = r.Read(b); err != nil {
			return nil, err
		}
	}

	if b != nil {
		return b, nil
	}

	if o.ExternalRef {
		// we were not able to resolve a ref in a thin pack
		return nil, ErrReferenceDeltaNotFound
	}

	var data []byte
	if o.DiskType.IsDelta() {
		base, err := p.get(o.Parent)
		if err != nil {
			return nil, err
		}

		data, err = p.resolveObject(o, base)
		if err != nil {
			return nil, err
		}
	} else {
		data, err = p.readData(o)
		if err != nil {
			return nil, err
		}
	}

	if len(o.Children) > 0 {
		p.cache.Put(o.Offset, data)
	}

	return data, nil
}

func (p *Parser) resolveObject(
	o *objectInfo,
	base []byte,
) ([]byte, error) {
	if !o.DiskType.IsDelta() {
		return nil, nil
	}

	data, err := p.readData(o)
	if err != nil {
		return nil, err
	}

	data, err = applyPatchBase(o, data, base)
	if err != nil {
		return nil, err
	}

	if p.storage != nil {
		obj := new(plumbing.MemoryObject)
		obj.SetSize(o.Size())
		obj.SetType(o.Type)
		if _, err := obj.Write(data); err != nil {
			return nil, err
		}

		if _, err := p.storage.SetEncodedObject(obj); err != nil {
			return nil, err
		}
	}

	return data, nil
}

func (p *Parser) readData(o *objectInfo) ([]byte, error) {
	if !p.scanner.IsSeekable && o.DiskType.IsDelta() {
		data, ok := p.deltas[o.Offset]
		if !ok {
			return nil, ErrDeltaNotCached
		}

		return data, nil
	}

	if _, err := p.scanner.SeekFromStart(o.Offset); err != nil {
		return nil, err
	}

	if _, err := p.scanner.NextObjectHeader(); err != nil {
		return nil, err
	}

	buf := new(bytes.Buffer)
	if _, _, err := p.scanner.NextObject(buf); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) {
	patched, err := PatchDelta(base, data)
	if err != nil {
		return nil, err
	}

	if ota.SHA1 == plumbing.ZeroHash {
		ota.Type = ota.Parent.Type
		sha1, err := getSHA1(ota.Type, patched)
		if err != nil {
			return nil, err
		}

		ota.SHA1 = sha1
		ota.Length = int64(len(patched))
	}

	return patched, nil
}

func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) {
	hasher := plumbing.NewHasher(t, int64(len(data)))
	if _, err := hasher.Write(data); err != nil {
		return plumbing.ZeroHash, err
	}

	return hasher.Sum(), nil
}

type objectInfo struct {
	Offset      int64
	Length      int64
	Type        plumbing.ObjectType
	DiskType    plumbing.ObjectType
	ExternalRef bool // indicates this is an external reference in a thin pack file

	Crc32 uint32

	Parent   *objectInfo
	Children []*objectInfo
	SHA1     plumbing.Hash
}

func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo {
	return newDeltaObject(offset, length, t, nil)
}

func newDeltaObject(
	offset, length int64,
	t plumbing.ObjectType,
	parent *objectInfo,
) *objectInfo {
	obj := &objectInfo{
		Offset:   offset,
		Length:   length,
		Type:     t,
		DiskType: t,
		Crc32:    0,
		Parent:   parent,
	}

	return obj
}

func (o *objectInfo) IsDelta() bool {
	return o.Type.IsDelta()
}

func (o *objectInfo) Size() int64 {
	return o.Length
}
<|endoftext|>