<commit_before>package transfer

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/github/git-lfs/tools"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

// Struct to capture stderr and write to trace
type traceWriter struct {
	buf         bytes.Buffer
	processName string
}

func (t *traceWriter) Write(b []byte) (int, error) {
	n, err := t.buf.Write(b)
	t.Flush()
	return n, err
}
func (t *traceWriter) Flush() {
	var err error
	for err == nil {
		var s string
		s, err = t.buf.ReadString('\n')
		if len(s) > 0 {
			tracerx.Printf("xfer[%v]: %v", t.processName, strings.TrimSpace(s))
		}
	}
}

type customAdapterWorkerContext struct {
	workerNum   int
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
	errTracer   *traceWriter
}

type customAdapterInitRequest struct {
	Event               string `json:"Event"`
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}

func NewCustomAdapterInitRequest(op string, concurrent bool, concurrentTransfers int) *customAdapterInitRequest {
	return &customAdapterInitRequest{"init", op, concurrent, concurrentTransfers}
}

type customAdapterTransferRequest struct { // common between upload/download
	Event  string            `json:"event"`
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}

func NewCustomAdapterUploadRequest(oid string, size int64, path string, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"upload", oid, size, path, action}
}
func NewCustomAdapterDownloadRequest(oid string, size int64, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"download", oid, size, "", action}
}

type customAdapterTerminateRequest struct {
	MessageType string `json:"type"`
}

func NewCustomAdapterTerminateRequest() *customAdapterTerminateRequest {
	return &customAdapterTerminateRequest{"terminate"}
}

// A common struct that allows all types of response to be identified
type customAdapterResponseMessage struct {
	Event          string           `json:"event"`
	Error          *api.ObjectError `json:"error"`
	Oid            string           `json:"oid"`
	Path           string           `json:"path,omitempty"` // always blank for upload
	BytesSoFar     int64            `json:"bytesSoFar"`
	BytesSinceLast int              `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	// Capture stderr to trace
	tracer := &traceWriter{}
	tracer.processName = filepath.Base(a.path)
	cmd.Stderr = tracer
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer}

	// send initiate message
	initReq := NewCustomAdapterInitRequest(a.getOperationName(), a.concurrent, a.originalConcurrency)
	resp, err := a.exchangeMessage(ctx, initReq)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}
	if resp.Error != nil {
		a.abortWorkerProcess(ctx)
		return nil, fmt.Errorf("Error initializing custom adapter %q worker %d: %v", a.name, workerNum, resp.Error)
	}

	tracerx.Printf("xfer: started custom adapter process %q for worker %d OK", a.path, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	tracerx.Printf("xfer: Custom adapter worker %d sending message: %v", ctx.workerNum, string(b))
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	return err
}

func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext) (*customAdapterResponseMessage, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return nil, err
	}
	tracerx.Printf("xfer: Custom adapter worker %d received response: %v", ctx.workerNum, line)
	resp := &customAdapterResponseMessage{}
	err = json.Unmarshal([]byte(line), resp)
	return resp, err
}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req interface{}) (*customAdapterResponseMessage, error) {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return nil, err
	}
	return a.readResponse(ctx)
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	defer ctx.errTracer.Flush()

	tracerx.Printf("xfer: Shutting down adapter worker %d", ctx.workerNum)

	finishChan := make(chan error, 1)
	go func() {
		termReq := NewCustomAdapterTerminateRequest()
		err := a.sendMessage(ctx, termReq)
		if err != nil {
			finishChan <- err
		}
		ctx.stdin.Close()
		ctx.stdout.Close()
		finishChan <- ctx.cmd.Wait()
	}()
	select {
	case err := <-finishChan:
		return err
	case <-time.After(30 * time.Second):
		return fmt.Errorf("Timeout while shutting down worker process %d", ctx.workerNum)
	}
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	tracerx.Printf("xfer: Aborting worker process: %d", ctx.workerNum)
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q worker %d, aborting: %v", a.path, customCtx.workerNum, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	var req *customAdapterTransferRequest
	if a.direction == Upload {
		req = NewCustomAdapterUploadRequest(t.Object.Oid, t.Object.Size, t.Path, rel)
	} else {
		req = NewCustomAdapterDownloadRequest(t.Object.Oid, t.Object.Size, rel)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	var complete bool
	for !complete {
		resp, err := a.readResponse(customCtx)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch resp.Event {
		case "progress":
			// Progress
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, resp.BytesSoFar, resp.BytesSinceLast)
			}
			wasAuthOk = resp.BytesSoFar > 0
		case "complete":
			// Download/Upload complete
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if resp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, resp.Error)
			}
			if a.direction == Download {
				// So we don't have to blindly trust external providers, check SHA
				if err = tools.VerifyFileHash(t.Object.Oid, resp.Path); err != nil {
					return fmt.Errorf("Downloaded file failed checks: %v", err)
				}
				// Move file to final location
				if err = tools.RenameFileCopyPermissions(resp.Path, t.Path); err != nil {
					return fmt.Errorf("Failed to copy downloaded file: %v", err)
				}
			} else if a.direction == Upload {
				if err = api.VerifyUpload(t.Object); err != nil {
					return err
				}
			}
			wasAuthOk = true
			complete = true
		default:
			return fmt.Errorf("Invalid message %q from custom adapter %q", resp.Event, a.name)
		}
		// Fall through from both progress and completion messages
		// Call auth on first progress or success to free up other workers
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		match := pathRegex.FindStringSubmatch(k)
		if match == nil {
			continue
		}

		name := match[1]
		path := v
		// retrieve other values
		args, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
		concurrent := config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
		direction, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
		if len(direction) == 0 {
			direction = "both"
		} else {
			direction = strings.ToLower(direction)
		}

		// Separate closure for each since we need to capture vars above
		newfunc := func(name string, dir Direction) TransferAdapter {
			return newCustomAdapter(name, dir, path, args, concurrent)
		}

		if direction == "download" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Download, newfunc)
		}
		if direction == "upload" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Upload, newfunc)
		}
	}
}
<commit_msg>Trim custom adapter output before tracing to avoid extra newline<commit_after>package transfer

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/github/git-lfs/tools"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

// Struct to capture stderr and write to trace
type traceWriter struct {
	buf         bytes.Buffer
	processName string
}

func (t *traceWriter) Write(b []byte) (int, error) {
	n, err := t.buf.Write(b)
	t.Flush()
	return n, err
}
func (t *traceWriter) Flush() {
	var err error
	for err == nil {
		var s string
		s, err = t.buf.ReadString('\n')
		if len(s) > 0 {
			tracerx.Printf("xfer[%v]: %v", t.processName, strings.TrimSpace(s))
		}
	}
}

type customAdapterWorkerContext struct {
	workerNum   int
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
	errTracer   *traceWriter
}

type customAdapterInitRequest struct {
	Event               string `json:"Event"`
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}

func NewCustomAdapterInitRequest(op string, concurrent bool, concurrentTransfers int) *customAdapterInitRequest {
	return &customAdapterInitRequest{"init", op, concurrent, concurrentTransfers}
}

type customAdapterTransferRequest struct { // common between upload/download
	Event  string            `json:"event"`
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}

func NewCustomAdapterUploadRequest(oid string, size int64, path string, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"upload", oid, size, path, action}
}
func NewCustomAdapterDownloadRequest(oid string, size int64, action *api.LinkRelation) *customAdapterTransferRequest {
	return &customAdapterTransferRequest{"download", oid, size, "", action}
}

type customAdapterTerminateRequest struct {
	MessageType string `json:"type"`
}

func NewCustomAdapterTerminateRequest() *customAdapterTerminateRequest {
	return &customAdapterTerminateRequest{"terminate"}
}

// A common struct that allows all types of response to be identified
type customAdapterResponseMessage struct {
	Event          string           `json:"event"`
	Error          *api.ObjectError `json:"error"`
	Oid            string           `json:"oid"`
	Path           string           `json:"path,omitempty"` // always blank for upload
	BytesSoFar     int64            `json:"bytesSoFar"`
	BytesSinceLast int              `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	// Capture stderr to trace
	tracer := &traceWriter{}
	tracer.processName = filepath.Base(a.path)
	cmd.Stderr = tracer
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{workerNum, cmd, outp, bufio.NewReader(outp), inp, tracer}

	// send initiate message
	initReq := NewCustomAdapterInitRequest(a.getOperationName(), a.concurrent, a.originalConcurrency)
	resp, err := a.exchangeMessage(ctx, initReq)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}
	if resp.Error != nil {
		a.abortWorkerProcess(ctx)
		return nil, fmt.Errorf("Error initializing custom adapter %q worker %d: %v", a.name, workerNum, resp.Error)
	}

	tracerx.Printf("xfer: started custom adapter process %q for worker %d OK", a.path, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	tracerx.Printf("xfer: Custom adapter worker %d sending message: %v", ctx.workerNum, string(b))
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	return err
}

func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext) (*customAdapterResponseMessage, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return nil, err
	}
	tracerx.Printf("xfer: Custom adapter worker %d received response: %v", ctx.workerNum, strings.TrimSpace(line))
	resp := &customAdapterResponseMessage{}
	err = json.Unmarshal([]byte(line), resp)
	return resp, err
}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req interface{}) (*customAdapterResponseMessage, error) {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return nil, err
	}
	return a.readResponse(ctx)
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	defer ctx.errTracer.Flush()

	tracerx.Printf("xfer: Shutting down adapter worker %d", ctx.workerNum)

	finishChan := make(chan error, 1)
	go func() {
		termReq := NewCustomAdapterTerminateRequest()
		err := a.sendMessage(ctx, termReq)
		if err != nil {
			finishChan <- err
		}
		ctx.stdin.Close()
		ctx.stdout.Close()
		finishChan <- ctx.cmd.Wait()
	}()
	select {
	case err := <-finishChan:
		return err
	case <-time.After(30 * time.Second):
		return fmt.Errorf("Timeout while shutting down worker process %d", ctx.workerNum)
	}
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	tracerx.Printf("xfer: Aborting worker process: %d", ctx.workerNum)
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q worker %d, aborting: %v", a.path, customCtx.workerNum, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	var req *customAdapterTransferRequest
	if a.direction == Upload {
		req = NewCustomAdapterUploadRequest(t.Object.Oid, t.Object.Size, t.Path, rel)
	} else {
		req = NewCustomAdapterDownloadRequest(t.Object.Oid, t.Object.Size, rel)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	var complete bool
	for !complete {
		resp, err := a.readResponse(customCtx)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch resp.Event {
		case "progress":
			// Progress
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, resp.BytesSoFar, resp.BytesSinceLast)
			}
			wasAuthOk = resp.BytesSoFar > 0
		case "complete":
			// Download/Upload complete
			if resp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", resp.Oid, t.Object.Oid)
			}
			if resp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, resp.Error)
			}
			if a.direction == Download {
				// So we don't have to blindly trust external providers, check SHA
				if err = tools.VerifyFileHash(t.Object.Oid, resp.Path); err != nil {
					return fmt.Errorf("Downloaded file failed checks: %v", err)
				}
				// Move file to final location
				if err = tools.RenameFileCopyPermissions(resp.Path, t.Path); err != nil {
					return fmt.Errorf("Failed to copy downloaded file: %v", err)
				}
			} else if a.direction == Upload {
				if err = api.VerifyUpload(t.Object); err != nil {
					return err
				}
			}
			wasAuthOk = true
			complete = true
		default:
			return fmt.Errorf("Invalid message %q from custom adapter %q", resp.Event, a.name)
		}
		// Fall through from both progress and completion messages
		// Call auth on first progress or success to free up other workers
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		match := pathRegex.FindStringSubmatch(k)
		if match == nil {
			continue
		}

		name := match[1]
		path := v
		// retrieve other values
		args, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
		concurrent := config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
		direction, _ := config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
		if len(direction) == 0 {
			direction = "both"
		} else {
			direction = strings.ToLower(direction)
		}

		// Separate closure for each since we need to capture vars above
		newfunc := func(name string, dir Direction) TransferAdapter {
			return newCustomAdapter(name, dir, path, args, concurrent)
		}

		if direction == "download" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Download, newfunc)
		}
		if direction == "upload" || direction == "both" {
			RegisterNewTransferAdapterFunc(name, Upload, newfunc)
		}
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"airdispat.ch/identity"
	"airdispat.ch/message"

	"getmelange.com/backend/models"
	"getmelange.com/backend/models/db"
	mIdentity "getmelange.com/backend/models/identity"
	"getmelange.com/backend/models/messages"

	"getmelange.com/backend/connect"
	"getmelange.com/backend/packaging"

	gdb "github.com/huntaub/go-db"
)

const (
	logManager = "[MANAGER]"
)

// Manager is the middleman between the API module and the Connect
// module. It provides a cached version of messages when requested and
// adds messages to the cache as they come in.
type Manager struct {
	Tables *db.Tables
	Store  *models.Store

	Client *connect.Client

	Fetcher *Fetcher
	Cache   *Store

	Identity *mIdentity.Identity
	Packager *packaging.Packager

	HasIdentity bool
}

// CreateManager will build a fully initialized Manager object (with
// fully initialized Fetcher and Stores).
func CreateManager(tables *db.Tables, store *models.Store, pack *packaging.Packager) (*Manager, error) {
	cache := CreateStore()

	m := &Manager{
		Tables:   tables,
		Store:    store,
		Client:   nil,
		Fetcher:  nil,
		Cache:    cache,
		Identity: nil,
		Packager: pack,
	}

	current, err := m.currentIdentity()
	if err != nil {
		return nil, err
	}

	if current != nil {
		m.loadIdentity(current)
	}

	return m, nil
}

func (m *Manager) UpdateMessage(msg *messages.JSONMessage) error {
	mail, to, err := msg.ToDispatch(m.Client)
	if err != nil {
		return err
	}

	// Get the current version of the message in the database.
	currentMessage := &messages.Message{}
	err = m.Tables.Message.Get().Where("name", msg.Name).One(m.Store, &currentMessage)
	if err != nil {
		return err
	}

	// Update the message in the database.
	modelMessage, modelComponents := msg.ToModel(m.Client)
	modelMessage.Id = currentMessage.Id

	_, err = m.Tables.Message.Update(modelMessage).Exec(m.Store)
	if err != nil {
		return err
	}

	// Remove the old components.
	currentComponents := new([]*message.Component)
	err = m.Tables.Component.Get().Where("message_id", currentMessage.Id).
		All(m.Store, currentComponents)
	for _, v := range *currentComponents {
		_, err := m.Tables.Component.Delete(v).Exec(m.Store)
		if err != nil {
			return err
		}
	}

	// Add the new version of the components into the DB.
	for _, v := range modelComponents {
		v.Message = gdb.ForeignKey(modelMessage)
		_, err = m.Tables.Component.Insert(v).Exec(m.Store)
		if err != nil {
			fmt.Println("Couldn't save component", v.Name, err)
		}
	}

	return m.Client.UpdateMessage(mail, to, msg.Name)
}

// PublishMessage takes a JSONMessage object and publishes it through
// the connect.Client.
func (m *Manager) PublishMessage(msg *messages.JSONMessage) error {
	mail, to, err := msg.ToDispatch(m.Client)
	if err != nil {
		return err
	}

	// Add the message to the database.
	modelMessage, modelComponents := msg.ToModel(m.Client)
	_, err = m.Tables.Message.Insert(modelMessage).Exec(m.Store)
	if err != nil {
		return err
	}

	// Store each of the components in the database.
	for _, v := range modelComponents {
		v.Message = gdb.ForeignKey(modelMessage)
		_, err = m.Tables.Component.Insert(v).Exec(m.Store)
		if err != nil {
			fmt.Println("Couldn't save component", v.Name, err)
		}
	}

	// Publish Message
	// *Mail, to []*identity.Address, name string, alert bool
	name, err := m.Client.PublishMessage(mail, to, msg.Name, !msg.Public)
	if err != nil {
		return err
	}

	// Send alerts for messages that are not public.
	if !msg.Public {
		var errs []error

		for _, v := range msg.To {
			err = m.Client.SendAlert(name, v.Alias)
			if err != nil {
				logError(logManager, "Received error sending alert.", err)
				errs = append(errs, err)
			}
		}

		// Return the first error if one occurred. This may be
		// disingenious as an alert may have already been sent
		// to some parties.
		if len(errs) > 0 {
			return errs[0]
		}
	}

	go func() {
		msg.Self = true

		// Load our sending profile.
		myProfile, myAlias, _ := m.currentProfile()
		msg.From = &messages.JSONProfile{
			Name:        myProfile.Name,
			Avatar:      myProfile.Image,
			Alias:       myAlias.String(),
			Fingerprint: m.Client.Origin.Address.String(),
		}

		// Send the new message to interested parties.
		m.Fetcher.notifyObservers(msg)
	}()

	return nil
}

// GetMessage will return a message that is either in the cache or
// fetched and added to the cache.
func (m *Manager) GetMessage(alias string, name string) (*messages.JSONMessage, error) {
	msg, found := m.Cache.RetrieveMessage(alias, name)
	if !found {
		return m.Fetcher.addMessage(
			name,  // Name of the message to fetch
			alias, // Author Alias
			"",    // Server Location Alias
			nil,   // Context
		)
	}

	return msg, nil
}

// GetPublic will return a list of messages published by a user after
// the date `since`.
func (m *Manager) GetPublic(since uint64, addr *mIdentity.Address) ([]*messages.JSONMessage, error) {
	msgs, found := m.Cache.RetrievePublic(addr.Alias)
	if !found {
		return m.Fetcher.addPublic(since, addr)
	}

	// Filter out messages that may have occurred before the
	// requested date.
	sinceT := time.Unix(int64(since), 0)
	var output []*messages.JSONMessage
	for _, m := range msgs {
		if sinceT.Before(m.Date) {
			output = append(output, m)
		}
	}
	return output, nil
}

// GetSentMessages will return a list of messages that were sent by the user after the date `since`.
func (m *Manager) GetSentMessages(since uint64) ([]*messages.JSONMessage, error) {
	msgs := make([]*messages.Message, 0)
	err := m.Tables.Message.Get().Where("from", m.Client.Origin.Address.String()).All(m.Store, &msgs)
	if err != nil {
		return nil, err
	}

	myProfile, myAlias, err := m.currentProfile()
	if err != nil {
		return nil, err
	}

	myInformation := &messages.TranslationMe{
		Profile: myProfile,
		Alias:   myAlias,
	}

	var outputMessages []*messages.JSONMessage
	for _, modelMessage := range msgs {
		// Build a list of profiles that we need
		var toAddress []*identity.Address
		for _, addr := range strings.Split(modelMessage.To, ",") {
			toAddress = append(toAddress, &identity.Address{
				Alias: addr,
			})
		}

		serializedMessage, err := m.Fetcher.Translator.Request(&messages.TranslationRequest{
			Model:    modelMessage,
			Me:       myInformation,
			Profiles: m.Fetcher.buildProfiles(toAddress...),
		})

		if err != nil {
			logError(logManager, "Received error translating model.", err)
		}
		outputMessages = append(outputMessages, serializedMessage)
	}

	return outputMessages, nil
}

// GetAllMessages will send all of the messages currently in the cache
// to the requesting channel so long as that channel is already an
// observer of the majorFetcher.
func (m *Manager) GetAllMessages(msgChan chan *messages.JSONMessage) {
	var newData messages.JSONMessageList

	if !m.Fetcher.Initial {
		// If we haven't done the initial fetch, go ahead and do that.
		m.Fetcher.CheckForMessages(0)
	} else {
		// Otherwise, return all the messages we currently
		// have cached.
		newData = messages.JSONMessageList(
			append(m.Cache.RetrieveAllPublic(), m.Cache.RetrieveAllPrivate()...))
	}

	// Get all of the sent messages thusfar.
	data, err := m.GetSentMessages(0)
	if err != nil {
		logError(logManager, "Received error getting sent messages", err)
	}
	newData = append(newData, data...)

	// Sort the data in reverse chronological order.
	sort.Sort(sort.Reverse(newData))

	// Give the data to the channel. Note: This will block unless
	// the channel is continuously reading messages.
	for _, v := range newData {
		msgChan <- v
	}
}

// GetProfile will attempt to fetch a profile out of the cache. If it
// isn't found, it will fetch it (or return the default).
func (m *Manager) GetProfile(of string) (*messages.JSONProfile, error) {
	return m.Fetcher.getProfile(of)
}

// currentProfile constructs the current user profile and alias
// objects.
func (m *Manager) currentProfile() (*mIdentity.Profile, *mIdentity.Alias, error) {
	myAlias := &mIdentity.Alias{}
	err := m.Tables.Alias.Get().Where("identity", m.Identity.Id).One(m.Store, myAlias)
	if err != nil {
		return nil, nil, err
	}

	myProfile := &mIdentity.Profile{}
	err = m.Identity.Profile.One(m.Store, myProfile)
	if err != nil || myProfile.Id == 0 {
		fmt.Println(logManager, "Received error getting my profile", err)
		myProfile = &mIdentity.Profile{
			Name:  myAlias.String(),
			Image: m.Fetcher.Translator.DefaultProfileImage(m.Client.Origin.Address.Alias),
		}
	}

	if strings.Contains(myProfile.Image, "@") {
		myProfile.Image = fmt.Sprintf("http://data.local.getmelange.com:7776/%s", myProfile.Image)
	}

	return myProfile, myAlias, nil
}
<commit_msg>backend/cache: removed unnecessary sort operations<commit_after>package cache

import (
	"fmt"
	"strings"
	"time"

	"airdispat.ch/identity"
	"airdispat.ch/message"

	"getmelange.com/backend/models"
	"getmelange.com/backend/models/db"
	mIdentity "getmelange.com/backend/models/identity"
	"getmelange.com/backend/models/messages"

	"getmelange.com/backend/connect"
	"getmelange.com/backend/packaging"

	gdb "github.com/huntaub/go-db"
)

const (
	logManager = "[MANAGER]"
)

// Manager is the middleman between the API module and the Connect
// module. It provides a cached version of messages when requested and
// adds messages to the cache as they come in.
type Manager struct {
	Tables *db.Tables
	Store  *models.Store

	Client *connect.Client

	Fetcher *Fetcher
	Cache   *Store

	Identity *mIdentity.Identity
	Packager *packaging.Packager

	HasIdentity bool
}

// CreateManager will build a fully initialized Manager object (with
// fully initialized Fetcher and Stores).
func CreateManager(tables *db.Tables, store *models.Store, pack *packaging.Packager) (*Manager, error) {
	cache := CreateStore()

	m := &Manager{
		Tables:   tables,
		Store:    store,
		Client:   nil,
		Fetcher:  nil,
		Cache:    cache,
		Identity: nil,
		Packager: pack,
	}

	current, err := m.currentIdentity()
	if err != nil {
		return nil, err
	}

	if current != nil {
		m.loadIdentity(current)
	}

	return m, nil
}

func (m *Manager) UpdateMessage(msg *messages.JSONMessage) error {
	mail, to, err := msg.ToDispatch(m.Client)
	if err != nil {
		return err
	}

	// Get the current version of the message in the database.
	currentMessage := &messages.Message{}
	err = m.Tables.Message.Get().Where("name", msg.Name).One(m.Store, &currentMessage)
	if err != nil {
		return err
	}

	// Update the message in the database.
	modelMessage, modelComponents := msg.ToModel(m.Client)
	modelMessage.Id = currentMessage.Id

	_, err = m.Tables.Message.Update(modelMessage).Exec(m.Store)
	if err != nil {
		return err
	}

	// Remove the old components.
	currentComponents := new([]*message.Component)
	err = m.Tables.Component.Get().Where("message_id", currentMessage.Id).
		All(m.Store, currentComponents)
	for _, v := range *currentComponents {
		_, err := m.Tables.Component.Delete(v).Exec(m.Store)
		if err != nil {
			return err
		}
	}

	// Add the new version of the components into the DB.
	for _, v := range modelComponents {
		v.Message = gdb.ForeignKey(modelMessage)
		_, err = m.Tables.Component.Insert(v).Exec(m.Store)
		if err != nil {
			fmt.Println("Couldn't save component", v.Name, err)
		}
	}

	return m.Client.UpdateMessage(mail, to, msg.Name)
}

// PublishMessage takes a JSONMessage object and publishes it through
// the connect.Client.
func (m *Manager) PublishMessage(msg *messages.JSONMessage) error {
	mail, to, err := msg.ToDispatch(m.Client)
	if err != nil {
		return err
	}

	// Add the message to the database.
	modelMessage, modelComponents := msg.ToModel(m.Client)
	_, err = m.Tables.Message.Insert(modelMessage).Exec(m.Store)
	if err != nil {
		return err
	}

	// Store each of the components in the database.
	for _, v := range modelComponents {
		v.Message = gdb.ForeignKey(modelMessage)
		_, err = m.Tables.Component.Insert(v).Exec(m.Store)
		if err != nil {
			fmt.Println("Couldn't save component", v.Name, err)
		}
	}

	// Publish Message
	// *Mail, to []*identity.Address, name string, alert bool
	name, err := m.Client.PublishMessage(mail, to, msg.Name, !msg.Public)
	if err != nil {
		return err
	}

	// Send alerts for messages that are not public.
	if !msg.Public {
		var errs []error

		for _, v := range msg.To {
			err = m.Client.SendAlert(name, v.Alias)
			if err != nil {
				logError(logManager, "Received error sending alert.", err)
				errs = append(errs, err)
			}
		}

		// Return the first error if one occurred. This may be
		// disingenious as an alert may have already been sent
		// to some parties.
		if len(errs) > 0 {
			return errs[0]
		}
	}

	go func() {
		msg.Self = true

		// Load our sending profile.
		myProfile, myAlias, _ := m.currentProfile()
		msg.From = &messages.JSONProfile{
			Name:        myProfile.Name,
			Avatar:      myProfile.Image,
			Alias:       myAlias.String(),
			Fingerprint: m.Client.Origin.Address.String(),
		}

		// Send the new message to interested parties.
		m.Fetcher.notifyObservers(msg)
	}()

	return nil
}

// GetMessage will return a message that is either in the cache or
// fetched and added to the cache.
func (m *Manager) GetMessage(alias string, name string) (*messages.JSONMessage, error) {
	msg, found := m.Cache.RetrieveMessage(alias, name)
	if !found {
		return m.Fetcher.addMessage(
			name,  // Name of the message to fetch
			alias, // Author Alias
			"",    // Server Location Alias
			nil,   // Context
		)
	}

	return msg, nil
}

// GetPublic will return a list of messages published by a user after
// the date `since`.
func (m *Manager) GetPublic(since uint64, addr *mIdentity.Address) ([]*messages.JSONMessage, error) {
	msgs, found := m.Cache.RetrievePublic(addr.Alias)
	if !found {
		return m.Fetcher.addPublic(since, addr)
	}

	// Filter out messages that may have occurred before the
	// requested date.
	sinceT := time.Unix(int64(since), 0)
	var output []*messages.JSONMessage
	for _, m := range msgs {
		if sinceT.Before(m.Date) {
			output = append(output, m)
		}
	}
	return output, nil
}

// GetSentMessages will return a list of messages that were sent by the user after the date `since`.
func (m *Manager) GetSentMessages(since uint64) ([]*messages.JSONMessage, error) {
	msgs := make([]*messages.Message, 0)
	err := m.Tables.Message.Get().Where("from", m.Client.Origin.Address.String()).All(m.Store, &msgs)
	if err != nil {
		return nil, err
	}

	myProfile, myAlias, err := m.currentProfile()
	if err != nil {
		return nil, err
	}

	myInformation := &messages.TranslationMe{
		Profile: myProfile,
		Alias:   myAlias,
	}

	var outputMessages []*messages.JSONMessage
	for _, modelMessage := range msgs {
		// Build a list of profiles that we need
		var toAddress []*identity.Address
		for _, addr := range strings.Split(modelMessage.To, ",") {
			toAddress = append(toAddress, &identity.Address{
				Alias: addr,
			})
		}

		serializedMessage, err := m.Fetcher.Translator.Request(&messages.TranslationRequest{
			Model:    modelMessage,
			Me:       myInformation,
			Profiles: m.Fetcher.buildProfiles(toAddress...),
		})

		if err != nil {
			logError(logManager, "Received error translating model.", err)
		}
		outputMessages = append(outputMessages, serializedMessage)
	}

	return outputMessages, nil
}

// GetAllMessages will send all of the messages currently in the cache
// to the requesting channel so long as that channel is already an
// observer of the majorFetcher.
func (m *Manager) GetAllMessages(msgChan chan *messages.JSONMessage) {
	var newData messages.JSONMessageList

	if !m.Fetcher.Initial {
		// If we haven't done the initial fetch, go ahead and do that.
		m.Fetcher.CheckForMessages(0)
	} else {
		// Otherwise, return all the messages we currently
		// have cached.
		newData = messages.JSONMessageList(
			m.Cache.RetrieveAll())
	}

	// Get all of the sent messages thusfar.
	data, err := m.GetSentMessages(0)
	if err != nil {
		logError(logManager, "Received error getting sent messages", err)
	}
	newData = append(newData, data...)

	// Give the data to the channel. Note: This will block unless
	// the channel is continuously reading messages.
	for _, v := range newData {
		msgChan <- v
	}
}

// GetProfile will attempt to fetch a profile out of the cache. If it
// isn't found, it will fetch it (or return the default).
func (m *Manager) GetProfile(of string) (*messages.JSONProfile, error) {
	return m.Fetcher.getProfile(of)
}

// currentProfile constructs the current user profile and alias
// objects.
func (m *Manager) currentProfile() (*mIdentity.Profile, *mIdentity.Alias, error) {
	myAlias := &mIdentity.Alias{}
	err := m.Tables.Alias.Get().Where("identity", m.Identity.Id).One(m.Store, myAlias)
	if err != nil {
		return nil, nil, err
	}

	myProfile := &mIdentity.Profile{}
	err = m.Identity.Profile.One(m.Store, myProfile)
	if err != nil || myProfile.Id == 0 {
		fmt.Println(logManager, "Received error getting my profile", err)
		myProfile = &mIdentity.Profile{
			Name:  myAlias.String(),
			Image: m.Fetcher.Translator.DefaultProfileImage(m.Client.Origin.Address.Alias),
		}
	}

	if strings.Contains(myProfile.Image, "@") {
		myProfile.Image = fmt.Sprintf("http://data.local.getmelange.com:7776/%s", myProfile.Image)
	}

	return myProfile, myAlias, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ctl

import (
	"bytes"
	"io/ioutil"
	"testing"

	"k8s.io/cli-runtime/pkg/genericclioptions"

	"github.com/jetstack/cert-manager/cmd/ctl/pkg/convert"
)

const (
	testdataResource1                        = "./testdata/convert/input/resource1.yaml"
	testdataResource2                        = "./testdata/convert/input/resource2.yaml"
	testdataResource3                        = "./testdata/convert/input/resource3.yaml"
	testdataResourceWithOrganizationV1alpha2 = "./testdata/convert/input/resource_with_organization_v1alpha2.yaml"
	testdataResourcesAsListV1alpha2          = "./testdata/convert/input/resources_as_list_v1alpha2.yaml"

	testdataNoOutputError                    = "./testdata/convert/output/no_output_error.yaml"
	testdataResource1V1alpha2                = "./testdata/convert/output/resource1_v1alpha2.yaml"
	testdataResource1V1alpha3                = "./testdata/convert/output/resource1_v1alpha3.yaml"
	testdataResource2V1alpha2                = "./testdata/convert/output/resource2_v1alpha2.yaml"
	testdataResource2V1alpha3                = "./testdata/convert/output/resource2_v1alpha3.yaml"
	testdataResource3V1alpha2                = "./testdata/convert/output/resource3_v1alpha2.yaml"
	testdataResourceWithOrganizationV1alpha3 = "./testdata/convert/output/resource_with_organization_v1alpha3.yaml"
	testdataResourceWithOrganizationV1beta1  = "./testdata/convert/output/resource_with_organization_v1beta1.yaml"
	testdataResourcesOutAsListV1alpha2       = "./testdata/convert/output/resources_as_list_v1alpha2.yaml"
	testdataResourcesOutAsListV1alpha3       = "./testdata/convert/output/resources_as_list_v1alpha3.yaml"
	testdataResourcesOutAsListV1beta1        = "./testdata/convert/output/resources_as_list_v1beta1.yaml"

	targetv1alpha2 = "cert-manager.io/v1alpha2"
	targetv1alpha3 = "cert-manager.io/v1alpha3"
	targetv1beta1  = "cert-manager.io/v1beta1"
)

func TestCtlConvert(t *testing.T) {
	tests := map[string]struct {
		input, expOutputFile string
		targetVersion        string
		expErr               bool
	}{
		"a single cert-manager resource should convert to v1alpha2 with no target": {
			input:         testdataResource1,
			expOutputFile: testdataResource1V1alpha2,
		},
		"a single cert-manager resource should convert to v1alpha2 with target v1alpha2": {
			input:         testdataResource1,
			targetVersion: targetv1alpha2,
			expOutputFile: testdataResource1V1alpha2,
		},
		"a single cert-manager resource should convert to v1alpha3 with target v1alpha3": {
			input:         testdataResource1,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResource1V1alpha3,
		},
		"a list of cert-manager resources should convert to v1alpha2 with no target": {
			input:         testdataResource2,
			expOutputFile: testdataResource2V1alpha2,
		},
		"a list of cert-manager resources should convert to v1alpha2 with target v1alpha2": {
			input:         testdataResource2,
			expOutputFile: testdataResource2V1alpha2,
		},
		"a list of cert-manager resources should convert to v1alpha3 with target v1alpha3": {
			input:         testdataResource2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResource2V1alpha3,
		},
		"a list of a mix of cert-manager and non cert-manager resources should convert to v1alpha2 with no target": {
			input:         testdataResource2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResource2V1alpha3,
		},
		"a list of a mix of cert-manager and non cert-manager resources should error with target v1alpha2": {
			input:         testdataResource3,
			targetVersion: targetv1alpha2,
			expOutputFile: testdataNoOutputError,
			expErr:        true,
		},
		"a list of a mix of cert-manager and non cert-manager resources should error with target v1alpha3": {
			input:         testdataResource3,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataNoOutputError,
			expErr:        true,
		},
		"an object in v1alpha2 that uses a field that has been renamed in v1alpha3 should be converted properly": {
			input:         testdataResourceWithOrganizationV1alpha2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResourceWithOrganizationV1alpha3,
		},
		"an object in v1alpha2 that uses a field that has been renamed in v1beta1 should be converted properly": {
			input:         testdataResourceWithOrganizationV1alpha2,
			targetVersion: targetv1beta1,
			expOutputFile: testdataResourceWithOrganizationV1beta1,
		},
		"a list in v1alpha2 should parsed": {
			input:         testdataResourcesAsListV1alpha2,
			targetVersion: targetv1alpha2,
			expOutputFile: testdataResourcesOutAsListV1alpha2,
		},
		"a list in v1alpha2 should be converted to v1alpha3": {
			input:         testdataResourcesAsListV1alpha2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResourcesOutAsListV1alpha3,
		},
		"a list in v1alpha2 should be converted to v1beta1": {
			input:         testdataResourcesAsListV1alpha2,
			targetVersion: targetv1beta1,
			expOutputFile: testdataResourcesOutAsListV1beta1,
		},
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			expOutput, err := ioutil.ReadFile(test.expOutputFile)
			if err != nil {
				t.Fatalf("%s: %s", test.expOutputFile, err)
			}

			// Run ctl convert command with input options
			streams, _, outBuf, _ := genericclioptions.NewTestIOStreams()

			opts := convert.NewOptions(streams)
			opts.OutputVersion = test.targetVersion
			opts.Filenames = []string{test.input}

			if err := opts.Complete(); err != nil {
				t.Fatal(err)
			}

			err = opts.Run()
			if test.expErr != (err != nil) {
				t.Errorf("got unexpected error, exp=%t got=%v",
					test.expErr, err)
			}

			if !bytes.Equal(bytes.TrimSpace(expOutput), bytes.TrimSpace(outBuf.Bytes())) {
				t.Errorf("got unexpected output, exp=%s\n got=%s",
					bytes.TrimSpace(expOutput), bytes.TrimSpace(outBuf.Bytes()))
			}
		})
	}
}

const (
	resource1v1alpha2 = `
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  creationTimestamp: null
  name: ca-issuer
  namespace: sandbox
spec:
  commonName: my-csi-app
  isCA: true
  issuerRef:
    group: cert-manager.io
    kind: Issuer
    name: selfsigned-issuer
  secretName: ca-key-pair
status: {}`
	resource1v1alpha3 = `
apiVersion: cert-manager.io/v1alpha3
kind: Certificate
metadata:
  creationTimestamp: null
  name: ca-issuer
  namespace: sandbox
spec:
  commonName: my-csi-app
  isCA: true
  issuerRef:
    group: cert-manager.io
    kind: Issuer
    name: selfsigned-issuer
  secretName: ca-key-pair
status: {}`

	resource2v1alpha2 = `
Items:
- apiVersion: cert-manager.io/v1alpha2
  kind: Certificate
  metadata:
    creationTimestamp: null
    name: ca-issuer
    namespace: sandbox
  spec:
    commonName: my-csi-app
    isCA: true
    issuerRef:
      group: cert-manager.io
      kind: Issuer
      name: selfsigned-issuer
    secretName: ca-key-pair
  status: {}
- apiVersion: cert-manager.io/v1alpha2
  kind: Issuer
  metadata:
    creationTimestamp: null
    name: ca-issuer
    namespace: sandbox
  spec:
    ca:
      secretName: ca-key-pair
  status: {}
- apiVersion: cert-manager.io/v1alpha2
  kind: Certificate
  metadata:
    creationTimestamp: null
    name: ca-issuer-2
    namespace: sandbox
  spec:
    commonName: my-csi-app
    isCA: true
    issuerRef:
      group: cert-manager.io
      kind: Issuer
      name: ca-issuer
    secretName: ca-key-pair
  status: {}
apiVersion: v1
kind: List`
	resource2v1alpha3 = `
Items:
- apiVersion: cert-manager.io/v1alpha3
  kind: Certificate
  metadata:
    creationTimestamp: null
    name: ca-issuer
    namespace: sandbox
  spec:
    commonName: my-csi-app
    isCA: true
    issuerRef:
      group: cert-manager.io
      kind: Issuer
      name: selfsigned-issuer
    secretName: ca-key-pair
  status: {}
- apiVersion: cert-manager.io/v1alpha3
  kind: Issuer
  metadata:
    creationTimestamp: null
    name: ca-issuer
    namespace: sandbox
  spec:
    ca:
      secretName: ca-key-pair
  status: {}
- apiVersion: cert-manager.io/v1alpha3
  kind: Certificate
  metadata:
    creationTimestamp: null
    name: ca-issuer-2
    namespace: sandbox
  spec:
    commonName: my-csi-app
    isCA: true
    issuerRef:
      group: cert-manager.io
      kind: Issuer
      name: ca-issuer
    secretName: ca-key-pair
  status: {}
apiVersion: v1
kind: List`

	resource3v1alpha2 = `
Items:
- apiVersion: v1
  kind: Namespace
  metadata:
    creationTimestamp: null
    name: sandbox
  spec: {}
  status: {}
- apiVersion: cert-manager.io/v1alpha2
  kind: Issuer
  metadata:
    creationTimestamp: null
    name: selfsigned-issuer
    namespace: sandbox
  spec:
    selfSigned: {}
  status: {}
apiVersion: v1
kind: List`
)
<commit_msg>Remove unused constants<commit_after>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ctl

import (
	"bytes"
	"io/ioutil"
	"testing"

	"k8s.io/cli-runtime/pkg/genericclioptions"

	"github.com/jetstack/cert-manager/cmd/ctl/pkg/convert"
)

const (
	testdataResource1                        = "./testdata/convert/input/resource1.yaml"
	testdataResource2                        = "./testdata/convert/input/resource2.yaml"
	testdataResource3                        = "./testdata/convert/input/resource3.yaml"
	testdataResourceWithOrganizationV1alpha2 = "./testdata/convert/input/resource_with_organization_v1alpha2.yaml"
	testdataResourcesAsListV1alpha2          = "./testdata/convert/input/resources_as_list_v1alpha2.yaml"

	testdataNoOutputError                    = "./testdata/convert/output/no_output_error.yaml"
	testdataResource1V1alpha2                = "./testdata/convert/output/resource1_v1alpha2.yaml"
	testdataResource1V1alpha3                = "./testdata/convert/output/resource1_v1alpha3.yaml"
	testdataResource2V1alpha2                = "./testdata/convert/output/resource2_v1alpha2.yaml"
	testdataResource2V1alpha3                = "./testdata/convert/output/resource2_v1alpha3.yaml"
	testdataResource3V1alpha2                = "./testdata/convert/output/resource3_v1alpha2.yaml"
	testdataResourceWithOrganizationV1alpha3 = "./testdata/convert/output/resource_with_organization_v1alpha3.yaml"
	testdataResourceWithOrganizationV1beta1  = "./testdata/convert/output/resource_with_organization_v1beta1.yaml"
	testdataResourcesOutAsListV1alpha2       = "./testdata/convert/output/resources_as_list_v1alpha2.yaml"
	testdataResourcesOutAsListV1alpha3       = "./testdata/convert/output/resources_as_list_v1alpha3.yaml"
	testdataResourcesOutAsListV1beta1        = "./testdata/convert/output/resources_as_list_v1beta1.yaml"

	targetv1alpha2 = "cert-manager.io/v1alpha2"
	targetv1alpha3 = "cert-manager.io/v1alpha3"
	targetv1beta1  = "cert-manager.io/v1beta1"
)

func TestCtlConvert(t *testing.T) {
	tests := map[string]struct {
		input, expOutputFile string
		targetVersion        string
		expErr               bool
	}{
		"a single cert-manager resource should convert to v1alpha2 with no target": {
			input:         testdataResource1,
			expOutputFile: testdataResource1V1alpha2,
		},
		"a single cert-manager resource should convert to v1alpha2 with target v1alpha2": {
			input:         testdataResource1,
			targetVersion: targetv1alpha2,
			expOutputFile: testdataResource1V1alpha2,
		},
		"a single cert-manager resource should convert to v1alpha3 with target v1alpha3": {
			input:         testdataResource1,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResource1V1alpha3,
		},
		"a list of cert-manager resources should convert to v1alpha2 with no target": {
			input:         testdataResource2,
			expOutputFile: testdataResource2V1alpha2,
		},
		"a list of cert-manager resources should convert to v1alpha2 with target v1alpha2": {
			input:         testdataResource2,
			expOutputFile: testdataResource2V1alpha2,
		},
		"a list of cert-manager resources should convert to v1alpha3 with target v1alpha3": {
			input:         testdataResource2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResource2V1alpha3,
		},
		"a list of a mix of cert-manager and non cert-manager resources should convert to v1alpha2 with no target": {
			input:         testdataResource2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResource2V1alpha3,
		},
		"a list of a mix of cert-manager and non cert-manager resources should error with target v1alpha2": {
			input:         testdataResource3,
			targetVersion: targetv1alpha2,
			expOutputFile: testdataNoOutputError,
			expErr:        true,
		},
		"a list of a mix of cert-manager and non cert-manager resources should error with target v1alpha3": {
			input:         testdataResource3,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataNoOutputError,
			expErr:        true,
		},
		"an object in v1alpha2 that uses a field that has been renamed in v1alpha3 should be converted properly": {
			input:         testdataResourceWithOrganizationV1alpha2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResourceWithOrganizationV1alpha3,
		},
		"an object in v1alpha2 that uses a field that has been renamed in v1beta1 should be converted properly": {
			input:         testdataResourceWithOrganizationV1alpha2,
			targetVersion: targetv1beta1,
			expOutputFile: testdataResourceWithOrganizationV1beta1,
		},
		"a list in v1alpha2 should parsed": {
			input:         testdataResourcesAsListV1alpha2,
			targetVersion: targetv1alpha2,
			expOutputFile: testdataResourcesOutAsListV1alpha2,
		},
		"a list in v1alpha2 should be converted to v1alpha3": {
			input:         testdataResourcesAsListV1alpha2,
			targetVersion: targetv1alpha3,
			expOutputFile: testdataResourcesOutAsListV1alpha3,
		},
		"a list in v1alpha2 should be converted to v1beta1": {
			input:         testdataResourcesAsListV1alpha2,
			targetVersion: targetv1beta1,
			expOutputFile: testdataResourcesOutAsListV1beta1,
		},
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			expOutput, err := ioutil.ReadFile(test.expOutputFile)
			if err != nil {
				t.Fatalf("%s: %s", test.expOutputFile, err)
			}

			// Run ctl convert command with input options
			streams, _, outBuf, _ := genericclioptions.NewTestIOStreams()

			opts := convert.NewOptions(streams)
			opts.OutputVersion = test.targetVersion
			opts.Filenames = []string{test.input}

			if err := opts.Complete(); err != nil {
				t.Fatal(err)
			}

			err = opts.Run()
			if test.expErr != (err != nil) {
				t.Errorf("got unexpected error, exp=%t got=%v",
					test.expErr, err)
			}

			if !bytes.Equal(bytes.TrimSpace(expOutput), bytes.TrimSpace(outBuf.Bytes())) {
				t.Errorf("got unexpected output, exp=%s\n got=%s",
					bytes.TrimSpace(expOutput), bytes.TrimSpace(outBuf.Bytes()))
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"testing"
	"time"

	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/util/retry"

	"github.com/hashicorp/go-getter"
	pkgutil "k8s.io/minikube/pkg/util"
)

func installRelease(version string) (f *os.File, err error) {
	tf, err := ioutil.TempFile("", fmt.Sprintf("minikube-%s.*.exe", version))
	if err != nil {
		return tf, err
	}
	tf.Close()

	url := pkgutil.GetBinaryDownloadURL(version, runtime.GOOS)

	if err := retry.Expo(func() error { return getter.GetFile(tf.Name(), url) }, 3*time.Second, Minutes(3)); err != nil {
		return tf, err
	}

	if runtime.GOOS != "windows" {
		if err := os.Chmod(tf.Name(), 0700); err != nil {
			return tf, err
		}
	}

	return tf, nil
}

// TestRunningBinaryUpgrade does an upgrade test on a running cluster
func TestRunningBinaryUpgrade(t *testing.T) {
	MaybeParallel(t)
	profile := UniqueProfileName("running-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	// Should be a version from the last 6 months
	legacyVersion := "v1.6.2"
	if KicDriver() {
		// v1.8.0 would be selected, but: https://github.com/kubernetes/minikube/issues/8740
		legacyVersion = "v1.9.0"
	}

	tf, err := installRelease(legacyVersion)
	if err != nil {
		t.Fatalf("%s release installation failed: %v", legacyVersion, err)
	}
	defer os.Remove(tf.Name())

	args := append([]string{"start", "-p", profile, "--memory=2200"}, StartArgs()...)
	rr := &RunResult{}
	r := func() error {
		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
		return err
	}

	// Retry up to two times, to allow flakiness for the legacy release
	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
		t.Fatalf("legacy %s start failed: %v", legacyVersion, err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("upgrade from %s to HEAD failed: %s: %v", legacyVersion, rr.Command(), err)
	}
}

// TestStoppedBinaryUpgrade does an upgrade test on a stopped cluster
func TestStoppedBinaryUpgrade(t *testing.T) {

	MaybeParallel(t)
	profile := UniqueProfileName("stopped-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	// Guarantee stopped upgrade compatibility from a release that is at least 1 year old
	// NOTE: <v1.4.0 does not automatically install a hyperkit/KVM driver
	legacyVersion := "v1.0.0"

	if KicDriver() {
		// first release with non-experimental KIC
		legacyVersion = "v1.8.0"
	}

	tf, err := installRelease(legacyVersion)
	if err != nil {
		t.Fatalf("%s release installation failed: %v", legacyVersion, err)
	}
	defer os.Remove(tf.Name())

	args := append([]string{"start", "-p", profile, "--memory=2200"}, StartArgs()...)
	rr := &RunResult{}
	r := func() error {
		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
		return err
	}

	// Retry up to two times, to allow flakiness for the legacy release
	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
		t.Fatalf("legacy %s start failed: %v", legacyVersion, err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), "-p", profile, "stop"))
	if err != nil {
		t.Errorf("failed to stop cluster: %s: %v", rr.Command(), err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("upgrade from %s to HEAD failed: %s: %v", legacyVersion, rr.Command(), err)
	}
}

// TestKubernetesUpgrade upgrades Kubernetes from oldest to newest
func TestKubernetesUpgrade(t *testing.T) {
	MaybeParallel(t)
	profile := UniqueProfileName("kubernetes-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	args := append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to start minikube HEAD with oldest k8s version: %s: %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop"))
	if err != nil {
		t.Errorf("failed to stop cluster: %s: %v", rr.Command(), err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to upgrade with newest k8s version. args: %s : %v", rr.Command(), err)
	}

	s, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "version", "--output=json"))
	if err != nil {
		t.Fatalf("error running kubectl: %v", err)
	}
	cv := struct {
		ServerVersion struct {
			GitVersion string `json:"gitVersion"`
		} `json:"serverVersion"`
	}{}
	err = json.Unmarshal(s.Stdout.Bytes(), &cv)

	if err != nil {
		t.Fatalf("error traversing json output: %v", err)
	}

	if cv.ServerVersion.GitVersion != constants.NewestKubernetesVersion {
		t.Fatalf("expected server version %s is not the same with latest version %s", cv.ServerVersion.GitVersion, constants.NewestKubernetesVersion)
	}

	t.Logf("Attempting to downgrade Kubernetes (should fail)")
	args = append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion)}, StartArgs()...)
	if rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)); err == nil {
		t.Fatalf("downgrading Kubernetes should not be allowed. expected to see error but got %v for %q", err, rr.Command())
	}

	t.Logf("Attempting restart after unsuccessful downgrade")
	args = append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("start after failed upgrade: %s: %v", rr.Command(), err)
	}
}

// TestMissingUpgrade tests a Docker upgrade where the underlying container is missing
func TestMissingUpgrade(t *testing.T) {
	if !DockerDriver() {
		t.Skipf("This test is only for Docker")
	}

	MaybeParallel(t)
	profile := UniqueProfileName("missing-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	legacyVersion := "v1.9.1"

	tf, err := installRelease(legacyVersion)
	if err != nil {
		t.Fatalf("%s release installation failed: %v", legacyVersion, err)
	}
	defer os.Remove(tf.Name())

	args := append([]string{"start", "-p", profile, "--memory=2200"}, StartArgs()...)
	rr := &RunResult{}
	r := func() error {
		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
		return err
	}

	// Retry up to two times, to allow flakiness for the previous release
	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
		t.Fatalf("release start failed: %v", err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, "docker", "stop", profile))
	if err != nil {
		t.Fatalf("%s failed: %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, "docker", "rm", profile))
	if err != nil {
		t.Fatalf("%s failed: %v", rr.Command(), err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed missing container upgrade from %s. args: %s : %v", legacyVersion, rr.Command(), err)
	}
}
<commit_msg>Add stop/state checks back to TestKubernetesUpgrade<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"testing"
	"time"

	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/util/retry"

	"github.com/docker/machine/libmachine/state"
	"github.com/hashicorp/go-getter"
	pkgutil "k8s.io/minikube/pkg/util"
)

func installRelease(version string) (f *os.File, err error) {
	tf, err := ioutil.TempFile("", fmt.Sprintf("minikube-%s.*.exe", version))
	if err != nil {
		return tf, err
	}
	tf.Close()

	url := pkgutil.GetBinaryDownloadURL(version, runtime.GOOS)

	if err := retry.Expo(func() error { return getter.GetFile(tf.Name(), url) }, 3*time.Second, Minutes(3)); err != nil {
		return tf, err
	}

	if runtime.GOOS != "windows" {
		if err := os.Chmod(tf.Name(), 0700); err != nil {
			return tf, err
		}
	}

	return tf, nil
}

// TestRunningBinaryUpgrade does an upgrade test on a running cluster
func TestRunningBinaryUpgrade(t *testing.T) {
	MaybeParallel(t)
	profile := UniqueProfileName("running-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	// Should be a version from the last 6 months
	legacyVersion := "v1.6.2"
	if KicDriver() {
		// v1.8.0 would be selected, but: https://github.com/kubernetes/minikube/issues/8740
		legacyVersion = "v1.9.0"
	}

	tf, err := installRelease(legacyVersion)
	if err != nil {
		t.Fatalf("%s release installation failed: %v", legacyVersion, err)
	}
	defer os.Remove(tf.Name())

	args := append([]string{"start", "-p", profile, "--memory=2200"}, StartArgs()...)
	rr := &RunResult{}
	r := func() error {
		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
		return err
	}

	// Retry up to two times, to allow flakiness for the legacy release
	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
		t.Fatalf("legacy %s start failed: %v", legacyVersion, err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("upgrade from %s to HEAD failed: %s: %v", legacyVersion, rr.Command(), err)
	}
}

// TestStoppedBinaryUpgrade does an upgrade test on a stopped cluster
func TestStoppedBinaryUpgrade(t *testing.T) {

	MaybeParallel(t)
	profile := UniqueProfileName("stopped-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	// Guarantee stopped upgrade compatibility from a release that is at least 1 year old
	// NOTE: <v1.4.0 does not automatically install a hyperkit/KVM driver
	legacyVersion := "v1.0.0"

	if KicDriver() {
		// first release with non-experimental KIC
		legacyVersion = "v1.8.0"
	}

	tf, err := installRelease(legacyVersion)
	if err != nil {
		t.Fatalf("%s release installation failed: %v", legacyVersion, err)
	}
	defer os.Remove(tf.Name())

	args := append([]string{"start", "-p", profile, "--memory=2200"}, StartArgs()...)
	rr := &RunResult{}
	r := func() error {
		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
		return err
	}

	// Retry up to two times, to allow flakiness for the legacy release
	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
		t.Fatalf("legacy %s start failed: %v", legacyVersion, err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), "-p", profile, "stop"))
	if err != nil {
		t.Errorf("failed to stop cluster: %s: %v", rr.Command(), err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("upgrade from %s to HEAD failed: %s: %v", legacyVersion, rr.Command(), err)
	}
}

// TestKubernetesUpgrade upgrades Kubernetes from oldest to newest
func TestKubernetesUpgrade(t *testing.T) {
	MaybeParallel(t)
	profile := UniqueProfileName("kubernetes-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	args := append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to start minikube HEAD with oldest k8s version: %s: %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile))
	if err != nil {
		t.Fatalf("%s failed: %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--format={{.Host}}"))
	if err != nil {
		t.Logf("status error: %v (may be ok)", err)
	}

	got := strings.TrimSpace(rr.Stdout.String())
	if got != state.Stopped.String() {
		t.Errorf("FAILED: status = %q; want = %q", got, state.Stopped.String())
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to upgrade with newest k8s version. args: %s : %v", rr.Command(), err)
	}

	s, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "version", "--output=json"))
	if err != nil {
		t.Fatalf("error running kubectl: %v", err)
	}
	cv := struct {
		ServerVersion struct {
			GitVersion string `json:"gitVersion"`
		} `json:"serverVersion"`
	}{}
	err = json.Unmarshal(s.Stdout.Bytes(), &cv)

	if err != nil {
		t.Fatalf("error traversing json output: %v", err)
	}

	if cv.ServerVersion.GitVersion != constants.NewestKubernetesVersion {
		t.Fatalf("expected server version %s is not the same with latest version %s", cv.ServerVersion.GitVersion, constants.NewestKubernetesVersion)
	}

	t.Logf("Attempting to downgrade Kubernetes (should fail)")
	args = append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion)}, StartArgs()...)
	if rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)); err == nil {
		t.Fatalf("downgrading Kubernetes should not be allowed. expected to see error but got %v for %q", err, rr.Command())
	}

	t.Logf("Attempting restart after unsuccessful downgrade")
	args = append([]string{"start", "-p", profile, "--memory=2200", fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("start after failed upgrade: %s: %v", rr.Command(), err)
	}
}

// TestMissingUpgrade tests a Docker upgrade where the underlying container is missing
func TestMissingUpgrade(t *testing.T) {
	if !DockerDriver() {
		t.Skipf("This test is only for Docker")
	}

	MaybeParallel(t)
	profile := UniqueProfileName("missing-upgrade")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(55))

	defer CleanupWithLogs(t, profile, cancel)

	legacyVersion := "v1.9.1"

	tf, err := installRelease(legacyVersion)
	if err != nil {
		t.Fatalf("%s release installation failed: %v", legacyVersion, err)
	}
	defer os.Remove(tf.Name())

	args := append([]string{"start", "-p", profile, "--memory=2200"}, StartArgs()...)
	rr := &RunResult{}
	r := func() error {
		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
		return err
	}

	// Retry up to two times, to allow flakiness for the previous release
	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
		t.Fatalf("release start failed: %v", err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, "docker", "stop", profile))
	if err != nil {
		t.Fatalf("%s failed: %v", rr.Command(), err)
	}

	rr, err = Run(t, exec.CommandContext(ctx, "docker", "rm", profile))
	if err != nil {
		t.Fatalf("%s failed: %v", rr.Command(), err)
	}

	args = append([]string{"start", "-p", profile, "--memory=2200", "--alsologtostderr", "-v=1"}, StartArgs()...)
	rr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed missing container upgrade from %s. args: %s : %v", legacyVersion, rr.Command(), err)
	}
}
<|endoftext|>


<commit_before>package tsparser

type PacketFilter struct {
	s    PacketStream
	pids []PID
	next Packet
}

func NewPacketFilter(s PacketStream, pids ...PID) *PacketFilter {
	return &PacketFilter{
		s:    s,
		pids: pids,
	}
}

func (f *PacketFilter) isTarget(pid PID) bool {
	return pid == 0x00 || pid == 0x14
	return false
}

func (f *PacketFilter) Scan() bool {
	for f.s.Scan() {
		packet := f.s.Packet()
		if f.isTarget(packet.PID()) {
			f.next = packet
			return true
		}
	}

	return false
}

func (f *PacketFilter) Bytes() []byte {
	return []byte(f.next)
}

func (f *PacketFilter) Packet() Packet {
	return f.next
}
<commit_msg>impl PacketFilter#isTarget<commit_after>package tsparser

import (
	"sort"
)

type PIDSlice []PID

func (pids PIDSlice) Len() int {
	return len(pids)
}

func (pids PIDSlice) Less(i, j int) bool {
	return pids[i] < pids[j]
}

func (pids PIDSlice) Swap(i, j int) {
	pids[i], pids[j] = pids[j], pids[i]
}

func (pids PIDSlice) Search(pid PID) int {
	return sort.Search(len(pids), func(x int) bool {
		return pids[x] >= pid
	})
}

type PacketFilter struct {
	s    PacketStream
	pids PIDSlice
	next Packet
}

func NewPacketFilter(s PacketStream, pids ...PID) *PacketFilter {
	pidSlice := PIDSlice(pids)
	sort.Sort(pidSlice)

	return &PacketFilter{
		s:    s,
		pids: pidSlice,
	}
}

func (f *PacketFilter) isTarget(pid PID) bool {
	i := f.pids.Search(pid)
	return f.pids.Len() > i && f.pids[i] == pid
}

func (f *PacketFilter) Scan() bool {
	for f.s.Scan() {
		packet := f.s.Packet()
		if f.isTarget(packet.PID()) {
			f.next = packet
			return true
		}
	}

	return false
}

func (f *PacketFilter) Bytes() []byte {
	return []byte(f.next)
}

func (f *PacketFilter) Packet() Packet {
	return f.next
}
<|endoftext|>


<commit_before>package topicmodifier

import (
	"fmt"
	. "koding/db/models"
	helper "koding/db/mongodb/modelhelper"
	"strings"
)

const LIMIT = 50

var Completed = false

// deleteTags removes given tags from post body and deletes related tag relationships.
func deleteTags(tagId string) error {
	log.Info("Deleting topic")

	tag, err := helper.GetTagById(tagId)
	if err != nil {
		return fmt.Errorf("Tag not found - Id: %s", tagId)
	}
	log.Info("Deleting %s", tag.Title)
	return updateTags(tag, &Tag{})
}

// mergeTags merges given tag with its synonym.
func mergeTags(tagId string) error {
	log.Info("Merging topics")

	tag, err := helper.GetTagById(tagId)
	if err != nil {
		if err == helper.ErrNotFound {
			return fmt.Errorf("Tag not found - Id: %s", tagId)
		}
		return err
	}

	synonym, err := FindSynonym(tagId)
	if err != nil {
		if err == helper.ErrNotFound {
			return fmt.Errorf("Synonym not found - Id %s", tagId)
		}
		return err
	}

	log.Info("Merging Topic %s into %s", tag.Title, synonym.Title)

	return updateTags(tag, synonym)
}

func updateTags(tag *Tag, synonym *Tag) error {
	selector := helper.Selector{"targetId": helper.GetObjectId(tag.Id.Hex()), "as": "tag"}
	rels, err := helper.GetSomeRelationships(selector, LIMIT)
	if err != nil {
		return err
	}

	postCount := len(rels)
	log.Info("%d tagged posts found", postCount)
	if postCount == 0 {
		return updateFollowers(tag, synonym)
	}

	updatedPostRels := updatePosts(rels, synonym.Id.Hex())

	updatedPostCount := len(updatedPostRels)
	log.Info("Updated Post count %d", updatedPostCount)

	err = updateTagRelationships(updatedPostRels, synonym)
	if err != nil && err != helper.ErrNotFound {
		return err
	}
	postRels := convertTagRelationships(updatedPostRels)
	err = updateTagRelationships(postRels, synonym)
	if err != nil && err != helper.ErrNotFound {
		return err
	}

	tag.Counts.Post -= postCount
	if err = helper.UpdateTag(tag); err != nil {
		return err
	}

	if synonym.Id.Hex() != "" {
		synonym.Counts.Post += updatedPostCount
		if err = helper.UpdateTag(synonym); err != nil {
			return err
		}
	}

	if postCount < LIMIT {
		return updateFollowers(tag, synonym)
	}

	return nil
}

func convertTagRelationships(tagRels []Relationship) (postRelationships []Relationship) {
	for _, tagRel := range tagRels {
		postRelationships = append(postRelationships, swapTagRelation(&tagRel, "post"))
	}

	return postRelationships
}

//updatePosts updates post body and post relationships according to given tag
//information. When newTagId = "" or when new tag is already included in post,
//then it just removes old tag (|JTag:x|) from post body and also removes post
//relationships.
//Returns Post Relationships where the old tag information is replaced.
func updatePosts(rels []Relationship, newTagId string) (filteredRels []Relationship) {
	for _, rel := range rels {
		tagId := rel.TargetId.Hex()
		post, err := helper.GetStatusUpdateById(rel.SourceId.Hex())
		if err != nil {
			log.Error("Status Update Not Found - Id: %s, Err: %s", rel.SourceId.Hex(), err)
			continue
		}

		tagIncluded := updatePostBody(post, tagId, newTagId)
		if strings.TrimSpace(post.Body) == "" {
			DeleteStatusUpdate(post.Id.Hex()) // delete empty post
		} else {
			err = helper.UpdateStatusUpdate(post) // update post body
		}

		if err != nil {
			log.Error(err.Error())
			continue
		}

		if !tagIncluded {
			filteredRels = append(filteredRels, rel)
		} else {
			RemoveRelationship(&rel) // remove JStatusUpdate - tag - JTag relationship
			postRel := swapTagRelation(&rel, "post")
			RemoveRelationship(&postRel) // remove JTag - post - JStatusUpdate relationship
		}
	}

	return filteredRels
}

//updatePostBody updates post body with given tags. It replaces already existing
//tag |#:JTag:[tagId]| with new one as |#:JTag:[newTagId]|. If new tag is already
//included in the post then it is not appended.
//E.g. post is #go #golang. when we merge #go to #golang, post becomes as #golang #golang.
//for preventing this doubling effect we drop one tag. Finally post becomes: #golang
func updatePostBody(s *StatusUpdate, tagId string, newTagId string) (tagIncluded bool) {
	var newTag string
	tagIncluded = false
	if newTagId != "" {
		newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
		//new tag already included in post
		if strings.Index(s.Body, newTag) != -1 {
			tagIncluded = true
			newTag = ""
		}
	}

	modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
	s.Body = strings.Replace(s.Body, modifiedTag, newTag, -1)
	return tagIncluded
}

//updateTagRelationships replaces old tag relationships with new ones.
//When synonym Tag is an empty object it just removes old relationships.
func updateTagRelationships(rels []Relationship, synonym *Tag) error {
	for _, rel := range rels {
		err := RemoveRelationship(&rel)
		if err != nil {
			return err
		}
		if synonym.Id.Hex() == "" {
			continue
		}
		//tag is replaced by synonym
		if rel.TargetName == "JTag" {
			rel.TargetId = synonym.Id
		} else {
			rel.SourceId = synonym.Id
		}
		rel.Id = helper.NewObjectId()
		if err = CreateRelationship(&rel); err != nil {
			return err
		}
	}
	return nil
}

// updateCounts updates tag count information. But here not sure if the
// following and tagged counts really have a meaning.
func updateCounts(tag *Tag, synonym *Tag) {
	synonym.Counts.Following += tag.Counts.Following
	tag.Counts.Following = 0
	synonym.Counts.Tagged += tag.Counts.Tagged
	tag.Counts.Tagged = 0
}

// updateFollowers appends old tag followers to new one. When user is already
// following new topic, then she is not appended as follower. It also updates
// tag follower counts.
func updateFollowers(tag *Tag, synonym *Tag) error {
	selector := helper.Selector{
		"sourceId":   tag.Id,
		"as":         "follower",
		"targetName": "JAccount",
	}

	rels, err := helper.GetSomeRelationships(selector, LIMIT)
	if err != nil {
		return err
	}
	followerCount := len(rels)
	log.Info("%v followers found", followerCount)
	if followerCount == 0 {
		updateCounts(tag, synonym)
		helper.UpdateTag(tag)
		helper.UpdateTag(synonym)
		Completed = true
		return nil
	}

	if synonym.Id.Hex() != "" {
		err = mergeFollowers(rels, synonym)
		if err != nil {
			return err
		}
	} else {
		updateTagRelationships(rels, &Tag{})
	}

	tag.Counts.Followers -= followerCount
	if followerCount < LIMIT {
		updateCounts(tag, synonym)
		Completed = true
	}
	helper.UpdateTag(tag)
	helper.UpdateTag(synonym)
	return nil
}

// mergeFollowers merges topic followers.
func mergeFollowers(rels []Relationship, synonym *Tag) error {
	var existingFollowers []Relationship
	var newFollowers []Relationship

	for _, rel := range rels {
		selector := helper.Selector{
			"sourceId": synonym.Id,
			"targetId": rel.TargetId,
			"as":       rel.As,
		}

		// checking if relationship already exists for the synonym
		_, err := helper.GetRelationship(selector)
		//because there are two relations as account -> follower -> tag and
		//tag -> follower -> account, we have added
		if err != nil {
			if err == helper.ErrNotFound {
				newFollowers = append(newFollowers, rel)
			} else {
				return err
			}
		} else {
			existingFollowers = append(existingFollowers, rel)
		}
	}

	log.Info("%v users are already following new topic", len(existingFollowers))
	if len(existingFollowers) > 0 {
		err := updateTagRelationships(existingFollowers, &Tag{})
		if err != nil {
			return err
		}
	}

	newFollowerCount := len(newFollowers)
	if newFollowerCount > 0 {
		log.Info("%v users followed new topic", len(newFollowers))
		err := updateTagRelationships(newFollowers, synonym)
		if err != nil {
			return err
		}
		synonym.Counts.Followers += newFollowerCount
	}

	return nil
}

// swapTagRelation swaps source and target data of relationships. It is used
// for converting bidirectional relationships.
func swapTagRelation(r *Relationship, as string) Relationship {
	return Relationship{
		As:         as,
		SourceId:   r.TargetId,
		SourceName: r.TargetName,
		TargetId:   r.SourceId,
		TargetName: r.SourceName,
		TimeStamp:  r.TimeStamp,
		Data:       r.Data,
	}
}
<commit_msg>Moderation: NotFound error is handled for GetTagById function<commit_after>package topicmodifier

import (
	"fmt"
	. "koding/db/models"
	helper "koding/db/mongodb/modelhelper"
	"strings"
)

const LIMIT = 50

var Completed = false

// deleteTags removes given tags from post body and deletes related tag relationships.
func deleteTags(tagId string) error {
	log.Info("Deleting topic")

	tag, err := helper.GetTagById(tagId)
	if err != nil {
		if err == helper.ErrNotFound {
			return fmt.Errorf("Tag not found - Id: %s", tagId)
		}
		return err
	}
	log.Info("Deleting %s", tag.Title)

	return updateTags(tag, &Tag{})
}

// mergeTags merges given tag with its synonym.
func mergeTags(tagId string) error {
	log.Info("Merging topics")

	tag, err := helper.GetTagById(tagId)
	if err != nil {
		if err == helper.ErrNotFound {
			return fmt.Errorf("Tag not found - Id: %s", tagId)
		}
		return err
	}

	synonym, err := FindSynonym(tagId)
	if err != nil {
		if err == helper.ErrNotFound {
			return fmt.Errorf("Synonym not found - Id %s", tagId)
		}
		return err
	}

	log.Info("Merging Topic %s into %s", tag.Title, synonym.Title)

	return updateTags(tag, synonym)
}

func updateTags(tag *Tag, synonym *Tag) error {
	selector := helper.Selector{"targetId": helper.GetObjectId(tag.Id.Hex()), "as": "tag"}
	rels, err := helper.GetSomeRelationships(selector, LIMIT)
	if err != nil {
		return err
	}

	postCount := len(rels)
	log.Info("%d tagged posts found", postCount)
	if postCount == 0 {
		return updateFollowers(tag, synonym)
	}

	updatedPostRels := updatePosts(rels, synonym.Id.Hex())

	updatedPostCount := len(updatedPostRels)
	log.Info("Updated Post count %d", updatedPostCount)

	err = updateTagRelationships(updatedPostRels, synonym)
	if err != nil && err != helper.ErrNotFound {
		return err
	}
	postRels := convertTagRelationships(updatedPostRels)
	err = updateTagRelationships(postRels, synonym)
	if err != nil && err != helper.ErrNotFound {
		return err
	}

	tag.Counts.Post -= postCount
	if err = helper.UpdateTag(tag); err != nil {
		return err
	}

	if synonym.Id.Hex() != "" {
		synonym.Counts.Post += updatedPostCount
		if err = helper.UpdateTag(synonym); err != nil {
			return err
		}
	}

	if postCount < LIMIT {
		return updateFollowers(tag, synonym)
	}

	return nil
}

func convertTagRelationships(tagRels []Relationship) (postRelationships []Relationship) {
	for _, tagRel := range tagRels {
		postRelationships = append(postRelationships, swapTagRelation(&tagRel, "post"))
	}

	return postRelationships
}

//updatePosts updates post body and post relationships according to given tag
//information. When newTagId = "" or when new tag is already included in post,
//then it just removes old tag (|JTag:x|) from post body and also removes post
//relationships.
//Returns Post Relationships where the old tag information is replaced.
func updatePosts(rels []Relationship, newTagId string) (filteredRels []Relationship) {
	for _, rel := range rels {
		tagId := rel.TargetId.Hex()
		post, err := helper.GetStatusUpdateById(rel.SourceId.Hex())
		if err != nil {
			log.Error("Status Update Not Found - Id: %s, Err: %s", rel.SourceId.Hex(), err)
			continue
		}

		tagIncluded := updatePostBody(post, tagId, newTagId)
		if strings.TrimSpace(post.Body) == "" {
			err = DeleteStatusUpdate(post.Id.Hex()) // delete empty post
		} else {
			err = helper.UpdateStatusUpdate(post) // update post body
		}

		if err != nil {
			log.Error(err.Error())
			continue
		}

		if !tagIncluded {
			filteredRels = append(filteredRels, rel)
		} else {
			RemoveRelationship(&rel) // remove JStatusUpdate - tag - JTag relationship
			postRel := swapTagRelation(&rel, "post")
			RemoveRelationship(&postRel) // remove JTag - post - JStatusUpdate relationship
		}
	}

	return filteredRels
}

//updatePostBody updates post body with given tags. It replaces already existing
//tag |#:JTag:[tagId]| with new one as |#:JTag:[newTagId]|. If new tag is already
//included in the post then it is not appended.
//E.g. post is #go #golang. when we merge #go to #golang, post becomes as #golang #golang.
//for preventing this doubling effect we drop one tag. Finally post becomes: #golang
func updatePostBody(s *StatusUpdate, tagId string, newTagId string) (tagIncluded bool) {
	var newTag string
	tagIncluded = false
	if newTagId != "" {
		newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
		//new tag already included in post
		if strings.Index(s.Body, newTag) != -1 {
			tagIncluded = true
			newTag = ""
		}
	}

	modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
	s.Body = strings.Replace(s.Body, modifiedTag, newTag, -1)
	return tagIncluded
}

//updateTagRelationships replaces old tag relationships with new ones.
//When synonym Tag is an empty object it just removes old relationships.
func updateTagRelationships(rels []Relationship, synonym *Tag) error {
	for _, rel := range rels {
		err := RemoveRelationship(&rel)
		if err != nil {
			return err
		}
		if synonym.Id.Hex() == "" {
			continue
		}
		//tag is replaced by synonym
		if rel.TargetName == "JTag" {
			rel.TargetId = synonym.Id
		} else {
			rel.SourceId = synonym.Id
		}
		rel.Id = helper.NewObjectId()
		if err = CreateRelationship(&rel); err != nil {
			return err
		}
	}
	return nil
}

// updateCounts updates tag count information. But here not sure if the
// following and tagged counts really have a meaning.
func updateCounts(tag *Tag, synonym *Tag) {
	synonym.Counts.Following += tag.Counts.Following
	tag.Counts.Following = 0
	synonym.Counts.Tagged += tag.Counts.Tagged
	tag.Counts.Tagged = 0
}

// updateFollowers appends old tag followers to new one. When user is already
// following new topic, then she is not appended as follower. It also updates
// tag follower counts.
func updateFollowers(tag *Tag, synonym *Tag) error {
	selector := helper.Selector{
		"sourceId":   tag.Id,
		"as":         "follower",
		"targetName": "JAccount",
	}

	rels, err := helper.GetSomeRelationships(selector, LIMIT)
	if err != nil {
		return err
	}
	followerCount := len(rels)
	log.Info("%v followers found", followerCount)
	if followerCount == 0 {
		updateCounts(tag, synonym)
		helper.UpdateTag(tag)
		helper.UpdateTag(synonym)
		Completed = true
		return nil
	}

	if synonym.Id.Hex() != "" {
		err = mergeFollowers(rels, synonym)
		if err != nil {
			return err
		}
	} else {
		updateTagRelationships(rels, &Tag{})
	}

	tag.Counts.Followers -= followerCount
	if followerCount < LIMIT {
		updateCounts(tag, synonym)
		Completed = true
	}
	helper.UpdateTag(tag)
	helper.UpdateTag(synonym)
	return nil
}

// mergeFollowers merges topic followers.
func mergeFollowers(rels []Relationship, synonym *Tag) error {
	var existingFollowers []Relationship
	var newFollowers []Relationship

	for _, rel := range rels {
		selector := helper.Selector{
			"sourceId": synonym.Id,
			"targetId": rel.TargetId,
			"as":       rel.As,
		}

		// checking if relationship already exists for the synonym
		_, err := helper.GetRelationship(selector)
		//because there are two relations as account -> follower -> tag and
		//tag -> follower -> account, we have added
		if err != nil {
			if err == helper.ErrNotFound {
				newFollowers = append(newFollowers, rel)
			} else {
				return err
			}
		} else {
			existingFollowers = append(existingFollowers, rel)
		}
	}

	log.Info("%v users are already following new topic", len(existingFollowers))
	if len(existingFollowers) > 0 {
		err := updateTagRelationships(existingFollowers, &Tag{})
		if err != nil {
			return err
		}
	}

	newFollowerCount := len(newFollowers)
	if newFollowerCount > 0 {
		log.Info("%v users followed new topic", len(newFollowers))
		err := updateTagRelationships(newFollowers, synonym)
		if err != nil {
			return err
		}
		synonym.Counts.Followers += newFollowerCount
	}

	return nil
}

// swapTagRelation swaps source and target data of relationships. It is used
// for converting bidirectional relationships.
func swapTagRelation(r *Relationship, as string) Relationship {
	return Relationship{
		As:         as,
		SourceId:   r.TargetId,
		SourceName: r.TargetName,
		TargetId:   r.SourceId,
		TargetName: r.SourceName,
		TimeStamp:  r.TimeStamp,
		Data:       r.Data,
	}
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/xml"
	"fmt"
	"os"
	"path"
	"socialapi/config"
)

func XML(i interface{}, fileName string) error {
	header := []byte(xml.Header)
	res, err := xml.Marshal(i)
	if err != nil {
		return err
	}
	// append header to xml file
	res = append(header, res...)

	MustWrite(res, fileName)

	return nil
}

func MustWrite(input []byte, fileName string) {
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	root := config.Get().Sitemap.XMLRoot
	n := fmt.Sprintf("%s.xml", fileName)
	n = path.Join(wd, root, n)

	output, err := os.Create(n)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := output.Close(); err != nil {
			panic(err)
		}
	}()
	if _, err := output.Write(input); err != nil {
		panic(err)
	}

}
<commit_msg>Sitemap: Breakdown XML into Marshal method<commit_after>package common

import (
	"encoding/xml"
	"fmt"
	"os"
	"path"
	"socialapi/config"
)

func XML(i interface{}, fileName string) error {
	res, err := Marshal(i)
	if err != nil {
		return err
	}

	MustWrite(res, fileName)

	return nil
}

func MustWrite(input []byte, fileName string) {
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	root := config.Get().Sitemap.XMLRoot
	n := fmt.Sprintf("%s.xml", fileName)
	n = path.Join(wd, root, n)

	output, err := os.Create(n)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := output.Close(); err != nil {
			panic(err)
		}
	}()
	if _, err := output.Write(input); err != nil {
		panic(err)
	}

}

func Marshal(i interface{}) ([]byte, error) {
	header := []byte(xml.Header)
	res, err := xml.Marshal(i)
	if err != nil {
		return nil, err
	}
	// append header to xml file
	return append(header, res...), nil
}
<|endoftext|>