<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"errors"
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Throttle
	var throttleCalled bool
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		AssertFalse(throttleCalled)
		throttleCalled = true

		AssertEq(t.ctx, ctx)
		AssertEq(readSize, tokens)

		return
	}

	// Call
	t.reader.Read(make([]byte, readSize))

	ExpectTrue(throttleCalled)
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	// Throttle
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 1))

	ExpectEq(0, n)
	ExpectThat(err, Error(HasSubstr("throttle")))
	ExpectThat(err, Error(HasSubstr("cancel")))
}

func (t *ThrottledReaderTest) CallsWrapped() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(buf, p)

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	// Wrapped
	expectedErr := errors.New("taco")
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = expectedErr
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = io.EOF
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = len(p)
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, readSize))

	ExpectEq(readSize, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			AssertEq(&buf[2], &p[0])
			err = errors.New("")
		}

		return
	}

	// Call
	t.reader.Read(buf)

	ExpectEq(2, callCount)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondFails() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	AssertTrue(false, "TODO")
}
<commit_msg>ThrottledReaderTest.WrappedReturnsShortRead_SecondReturnsError<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"errors"
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Throttle
	var throttleCalled bool
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		AssertFalse(throttleCalled)
		throttleCalled = true

		AssertEq(t.ctx, ctx)
		AssertEq(readSize, tokens)

		return
	}

	// Call
	t.reader.Read(make([]byte, readSize))

	ExpectTrue(throttleCalled)
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	// Throttle
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 1))

	ExpectEq(0, n)
	ExpectThat(err, Error(HasSubstr("throttle")))
	ExpectThat(err, Error(HasSubstr("cancel")))
}

func (t *ThrottledReaderTest) CallsWrapped() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(buf, p)

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	// Wrapped
	expectedErr := errors.New("taco")
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = expectedErr
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = io.EOF
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = len(p)
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, readSize))

	ExpectEq(readSize, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			AssertEq(&buf[2], &p[0])
			err = errors.New("")
		}

		return
	}

	// Call
	t.reader.Read(buf)

	ExpectEq(2, callCount)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsError() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var callCount int
	expectedErr := errors.New("taco")

	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = 11
			err = expectedErr
		}

		return
	}

	// Call
	t.reader.Read(buf)

	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(2+11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsEOF() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/mobile/event/lifecycle"
)

func updateTag() {
	old := string(tagWidget.ed.Contents())
	var user string
	if i := strings.Index(old, "|"); i > 0 {
		user = old[i+1:]
		old = old[:i+1]
	}

	var newParts []string
	if filename != "" {
		newParts = append(newParts, filename)
	}
	if mainWidget.ed.CanUndo() {
		newParts = append(newParts, "Undo")
	}
	if mainWidget.ed.CanRedo() {
		newParts = append(newParts, "Redo")
	}
	if filename != "" && !mainWidget.ed.Saved() {
		newParts = append(newParts, "Put")
	}
	newParts = append(newParts, "Exit")
	newParts = append(newParts, "|")

	new := strings.Join(newParts, " ")
	if old == new {
		return
	}

	if user == "" {
		user = " "
	}

	tagWidget.ed.Load([]byte(new + user))
}

var reallyQuit time.Time

func executeCmd(cmd string) {
	cmd = strings.TrimSpace(cmd)
	switch cmd {
	case "Put":
		save()
	case "Undo":
		mainWidget.ed.SendUndo()
	case "Redo":
		mainWidget.ed.SendRedo()
	case "Exit":
		if mainWidget.ed.Saved() || time.Since(reallyQuit) < 3*time.Second {
			win.Send(lifecycle.Event{To: lifecycle.StageDead})
		} else {
			tagWidget.ed.FindNext("Put")
			reallyQuit = time.Now()
		}
	}
}

func findInEditor(s string) {
	if s == "" {
		return
	}
	first, sz := utf8.DecodeRuneInString(s)
	switch first {
	case ':':
		mainWidget.ed.JumpTo(s[sz:])
	default:
		mainWidget.ed.FindNext(s)
	}
}
<commit_msg>cmd/edit: refresh tag on Undo/Redo<commit_after>package main

import (
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/mobile/event/lifecycle"
)

func updateTag() {
	old := string(tagWidget.ed.Contents())
	var user string
	if i := strings.Index(old, "|"); i > 0 {
		user = old[i+1:]
		old = old[:i+1]
	}

	var newParts []string
	if filename != "" {
		newParts = append(newParts, filename)
	}
	if mainWidget.ed.CanUndo() {
		newParts = append(newParts, "Undo")
	}
	if mainWidget.ed.CanRedo() {
		newParts = append(newParts, "Redo")
	}
	if filename != "" && !mainWidget.ed.Saved() {
		newParts = append(newParts, "Put")
	}
	newParts = append(newParts, "Exit")
	newParts = append(newParts, "|")

	new := strings.Join(newParts, " ")
	if old == new {
		return
	}

	if user == "" {
		user = " "
	}

	tagWidget.ed.Load([]byte(new + user))
}

var reallyQuit time.Time

func executeCmd(cmd string) {
	cmd = strings.TrimSpace(cmd)
	switch cmd {
	case "Put":
		save()
	case "Undo":
		mainWidget.ed.SendUndo()
		tagWidget.ed.Load([]byte{}) // force tag regeneration
	case "Redo":
		mainWidget.ed.SendRedo()
		tagWidget.ed.Load([]byte{})
	case "Exit":
		if mainWidget.ed.Saved() || time.Since(reallyQuit) < 3*time.Second {
			win.Send(lifecycle.Event{To: lifecycle.StageDead})
		} else {
			tagWidget.ed.FindNext("Put")
			reallyQuit = time.Now()
		}
	}
}

func findInEditor(s string) {
	if s == "" {
		return
	}
	first, sz := utf8.DecodeRuneInString(s)
	switch first {
	case ':':
		mainWidget.ed.JumpTo(s[sz:])
	default:
		mainWidget.ed.FindNext(s)
	}
}
<|endoftext|>


<commit_before>package raw_socket

import (
	"log"
	"sort"
	"time"
)

const MSG_EXPIRE = 1000 * time.Millisecond

// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
// Its needed because all TCP message can be fragmented or re-transmitted
//
// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
// Message can be compiled from unique packets with same message_id which sorted by sequence
// Message is received if we didn't receive any packets for 200ms
type TCPMessage struct {
	ID      string // Message ID
	packets []*TCPPacket

	timer *time.Timer // Used for expire check

	c_packets chan *TCPPacket

	c_del_message chan *TCPMessage
}

// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
	msg = &TCPMessage{ID: ID}

	msg.c_packets = make(chan *TCPPacket)
	msg.c_del_message = c_del // used for notifying that message completed or expired

	// Every time we receive packet we reset this timer
	msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)

	go msg.listen()

	return
}

func (t *TCPMessage) listen() {
	for {
		select {
		case packet, more := <-t.c_packets:
			if more {
				t.AddPacket(packet)
			} else {
				// Stop loop if channel closed
				return
			}
		}
	}
}

// Timeout notifies message to stop listening, close channel and message ready to be sent
func (t *TCPMessage) Timeout() {
	close(t.c_packets)   // Notify to stop listen loop and close channel
	t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
}

// Bytes sorts packets in right orders and return message content
func (t *TCPMessage) Bytes() (output []byte) {
	mk := make([]int, len(t.packets))

	i := 0
	for k, _ := range t.packets {
		mk[i] = k
		i++
	}

	sort.Ints(mk)

	for _, k := range mk {
		output = append(output, t.packets[k].Data...)
	}

	return
}

// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
	packetFound := false

	for _, pkt := range t.packets {
		if packet.Seq == pkt.Seq {
			packetFound = true
			break
		}
	}

	if packetFound {
		log.Println("Received packet with same sequence")
	} else {
		t.packets = append(t.packets, packet)
	}

	// Reset message timeout timer
	t.timer.Reset(MSG_EXPIRE)
}
<commit_msg>revert timeout change<commit_after>package raw_socket

import (
	"log"
	"sort"
	"time"
)

const MSG_EXPIRE = 200 * time.Millisecond

// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
// Its needed because all TCP message can be fragmented or re-transmitted
//
// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
// Message can be compiled from unique packets with same message_id which sorted by sequence
// Message is received if we didn't receive any packets for 200ms
type TCPMessage struct {
	ID      string // Message ID
	packets []*TCPPacket

	timer *time.Timer // Used for expire check

	c_packets chan *TCPPacket

	c_del_message chan *TCPMessage
}

// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
	msg = &TCPMessage{ID: ID}

	msg.c_packets = make(chan *TCPPacket)
	msg.c_del_message = c_del // used for notifying that message completed or expired

	// Every time we receive packet we reset this timer
	msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)

	go msg.listen()

	return
}

func (t *TCPMessage) listen() {
	for {
		select {
		case packet, more := <-t.c_packets:
			if more {
				t.AddPacket(packet)
			} else {
				// Stop loop if channel closed
				return
			}
		}
	}
}

// Timeout notifies message to stop listening, close channel and message ready to be sent
func (t *TCPMessage) Timeout() {
	close(t.c_packets)   // Notify to stop listen loop and close channel
	t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
}

// Bytes sorts packets in right orders and return message content
func (t *TCPMessage) Bytes() (output []byte) {
	mk := make([]int, len(t.packets))

	i := 0
	for k, _ := range t.packets {
		mk[i] = k
		i++
	}

	sort.Ints(mk)

	for _, k := range mk {
		output = append(output, t.packets[k].Data...)
	}

	return
}

// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
	packetFound := false

	for _, pkt := range t.packets {
		if packet.Seq == pkt.Seq {
			packetFound = true
			break
		}
	}

	if packetFound {
		log.Println("Received packet with same sequence")
	} else {
		t.packets = append(t.packets, packet)
	}

	// Reset message timeout timer
	t.timer.Reset(MSG_EXPIRE)
}
<|endoftext|>


<commit_before>package cmd

import "github.com/spf13/cobra"

func RegistryCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "registry",
		Short: "manage your Mobingi docker registry",
		Long:  `Manage your Mobingi docker registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Help()
		},
	}

	cmd.AddCommand(
		RegistryToken(),
		RegistryTagsList(),
		RegistryCatalog(),
		DescribeImageCmd(),
		RegistryManifest(),
		RegistryDeleteTag(),
	)

	return cmd
}
<commit_msg>Proper name for Docker Registry.<commit_after>package cmd

import "github.com/spf13/cobra"

func RegistryCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "registry",
		Short: "manage your Mobingi Docker Registry",
		Long:  `Manage your Mobingi Docker Registry.`,
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Help()
		},
	}

	cmd.AddCommand(
		RegistryToken(),
		RegistryTagsList(),
		RegistryCatalog(),
		DescribeImageCmd(),
		RegistryManifest(),
		RegistryDeleteTag(),
	)

	return cmd
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	"github.com/smira/aptly/debian"
	"github.com/smira/aptly/utils"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

func aptlyRepoAdd(cmd *commander.Command, args []string) error {
	var err error
	if len(args) < 2 {
		cmd.Usage()
		return err
	}

	name := args[0]

	verifier := &utils.GpgVerifier{}

	localRepoCollection := debian.NewLocalRepoCollection(context.database)
	repo, err := localRepoCollection.ByName(name)
	if err != nil {
		return fmt.Errorf("unable to add: %s", err)
	}

	err = localRepoCollection.LoadComplete(repo)
	if err != nil {
		return fmt.Errorf("unable to add: %s", err)
	}

	context.progress.Printf("Loading packages...\n")

	packageCollection := debian.NewPackageCollection(context.database)
	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection)
	if err != nil {
		return fmt.Errorf("unable to load packages: %s", err)
	}

	packageFiles := []string{}

	for _, location := range args[1:] {
		info, err := os.Stat(location)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err)
			continue
		}
		if info.IsDir() {
			err = filepath.Walk(location, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}

				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
					packageFiles = append(packageFiles, path)
				}

				return nil
			})
		} else {
			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
				packageFiles = append(packageFiles, location)
			} else {
				context.progress.ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
				continue
			}
		}
	}

	processedFiles := []string{}
	sort.Strings(packageFiles)

	for _, file := range packageFiles {
		var (
			stanza debian.Stanza
			err    error
			p      *debian.Package
		)

		isSourcePackage := strings.HasSuffix(file, ".dsc")

		if isSourcePackage {
			stanza, err = debian.GetControlFileFromDsc(file, verifier)

			if err == nil {
				stanza["Package"] = stanza["Source"]
				delete(stanza, "Source")

				p, err = debian.NewSourcePackageFromControlFile(stanza)
			}
		} else {
			stanza, err = debian.GetControlFileFromDeb(file)
			p = debian.NewPackageFromControlFile(stanza)
		}
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
			continue
		}

		checksums, err := utils.ChecksumsForFile(file)
		if err != nil {
			return err
		}

		if isSourcePackage {
			p.Files = append(p.Files, debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums})
		} else {
			p.Files = []debian.PackageFile{debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}}
		}

		err = context.packagePool.Import(file, checksums.MD5)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
			continue
		}

		processedFiles = append(processedFiles, file)

		// go over all files, except for the last one (.dsc/.deb itself)
		for i := 0; i < len(p.Files)-1; i++ {
			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(p.Files[i].Filename))
			err = context.packagePool.Import(sourceFile, p.Files[i].Checksums.MD5)
			if err != nil {
				context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
				break
			}

			processedFiles = append(processedFiles, sourceFile)
		}
		if err != nil {
			// some files haven't been imported
			continue
		}

		err = packageCollection.Update(p)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
			continue
		}

		err = list.Add(p)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
			continue
		}

		context.progress.ColoredPrintf("@g[+]@| %s added@|", p)
	}

	repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))

	err = localRepoCollection.Update(repo)
	if err != nil {
		return fmt.Errorf("unable to save: %s", err)
	}

	if cmd.Flag.Lookup("remove-files").Value.Get().(bool) {
		processedFiles = utils.StrSliceDeduplicate(processedFiles)

		for _, file := range processedFiles {
			err := os.Remove(file)
			if err != nil {
				return fmt.Errorf("unable to remove file: %s", err)
			}
		}
	}

	return err
}

func makeCmdRepoAdd() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlyRepoAdd,
		UsageLine: "add <name> <package file.deb>|<directory> ...",
		Short:     "add packages to local repository",
		Long: `
Command adds packages to local repository. List of files or directories to be
scanned could be specified. If importing from directory, all files matching *.deb
pattern would be scanned and added to the repository. For source packages, all required
files are added as well automatically. By default, when importing
all successfully added files are removed.

ex:
  $ aptly repo add testing myapp-0.1.2.deb incoming/
`,
		Flag: *flag.NewFlagSet("aptly-repo-add", flag.ExitOnError),
	}

	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")

	return cmd
}
<commit_msg>Fix command help. [ci skip]<commit_after>package cmd

import (
	"fmt"
	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	"github.com/smira/aptly/debian"
	"github.com/smira/aptly/utils"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

func aptlyRepoAdd(cmd *commander.Command, args []string) error {
	var err error
	if len(args) < 2 {
		cmd.Usage()
		return err
	}

	name := args[0]

	verifier := &utils.GpgVerifier{}

	localRepoCollection := debian.NewLocalRepoCollection(context.database)
	repo, err := localRepoCollection.ByName(name)
	if err != nil {
		return fmt.Errorf("unable to add: %s", err)
	}

	err = localRepoCollection.LoadComplete(repo)
	if err != nil {
		return fmt.Errorf("unable to add: %s", err)
	}

	context.progress.Printf("Loading packages...\n")

	packageCollection := debian.NewPackageCollection(context.database)
	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection)
	if err != nil {
		return fmt.Errorf("unable to load packages: %s", err)
	}

	packageFiles := []string{}

	for _, location := range args[1:] {
		info, err := os.Stat(location)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err)
			continue
		}
		if info.IsDir() {
			err = filepath.Walk(location, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}

				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
					packageFiles = append(packageFiles, path)
				}

				return nil
			})
		} else {
			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
				packageFiles = append(packageFiles, location)
			} else {
				context.progress.ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
				continue
			}
		}
	}

	processedFiles := []string{}
	sort.Strings(packageFiles)

	for _, file := range packageFiles {
		var (
			stanza debian.Stanza
			err    error
			p      *debian.Package
		)

		isSourcePackage := strings.HasSuffix(file, ".dsc")

		if isSourcePackage {
			stanza, err = debian.GetControlFileFromDsc(file, verifier)

			if err == nil {
				stanza["Package"] = stanza["Source"]
				delete(stanza, "Source")

				p, err = debian.NewSourcePackageFromControlFile(stanza)
			}
		} else {
			stanza, err = debian.GetControlFileFromDeb(file)
			p = debian.NewPackageFromControlFile(stanza)
		}
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
			continue
		}

		checksums, err := utils.ChecksumsForFile(file)
		if err != nil {
			return err
		}

		if isSourcePackage {
			p.Files = append(p.Files, debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums})
		} else {
			p.Files = []debian.PackageFile{debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}}
		}

		err = context.packagePool.Import(file, checksums.MD5)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
			continue
		}

		processedFiles = append(processedFiles, file)

		// go over all files, except for the last one (.dsc/.deb itself)
		for i := 0; i < len(p.Files)-1; i++ {
			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(p.Files[i].Filename))
			err = context.packagePool.Import(sourceFile, p.Files[i].Checksums.MD5)
			if err != nil {
				context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
				break
			}

			processedFiles = append(processedFiles, sourceFile)
		}
		if err != nil {
			// some files haven't been imported
			continue
		}

		err = packageCollection.Update(p)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
			continue
		}

		err = list.Add(p)
		if err != nil {
			context.progress.ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
			continue
		}

		context.progress.ColoredPrintf("@g[+]@| %s added@|", p)
	}

	repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))

	err = localRepoCollection.Update(repo)
	if err != nil {
		return fmt.Errorf("unable to save: %s", err)
	}

	if cmd.Flag.Lookup("remove-files").Value.Get().(bool) {
		processedFiles = utils.StrSliceDeduplicate(processedFiles)

		for _, file := range processedFiles {
			err := os.Remove(file)
			if err != nil {
				return fmt.Errorf("unable to remove file: %s", err)
			}
		}
	}

	return err
}

func makeCmdRepoAdd() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlyRepoAdd,
		UsageLine: "add <name> <package file.deb>|<directory> ...",
		Short:     "add packages to local repository",
		Long: `
Command adds packages to local repository. List of files or directories to be
scanned could be specified. If importing from directory, all files matching *.deb and *.dsc
pattern would be scanned and added to the repository. For source packages, all required
files are added as well automatically.

ex:
  $ aptly repo add testing myapp-0.1.2.deb incoming/
`,
		Flag: *flag.NewFlagSet("aptly-repo-add", flag.ExitOnError),
	}

	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")

	return cmd
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/sbinet/go-commander"
	"github.com/sbinet/go-flag"
	//gocfg "github.com/sbinet/go-config/config"
)

func hwaf_make_cmd_checkout() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_checkout,
		UsageLine: "co [options] <pkg-uri> [<local-pkg-name>]",
		Short:     "checkout adds a package to the current workarea",
		Long: `
checkout adds a package to the current workarea.

ex:
 $ hwaf co /foo/pkg
 $ hwaf co Control/AthenaKernel
 $ hwaf co git://github.com/mana-fwk/mana-core-athenakernel
 $ hwaf co git://github.com/mana-fwk/mana-core-athenakernel Control/AthenaKernel
`,
		Flag: *flag.NewFlagSet("hwaf-checkout", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", false, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_checkout(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-" + cmd.Name()
	pkguri := ""
	pkgname := ""
	switch len(args) {
	case 1:
		pkguri = args[0]
		pkgname = filepath.Base(args[0])
	case 2:
		pkguri = args[0]
		pkgname = args[1]
	default:
		err = fmt.Errorf("%s: you need to give a package URL", n)
		handle_err(err)
	}

	pkguri = os.ExpandEnv(pkguri)
	//pkguri = filepath.Clean(pkguri)

	pkgname = os.ExpandEnv(pkgname)
	pkgname = filepath.Clean(pkgname)

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]...\n", n, pkguri)
	}

	cfg := load_local_cfg()
	pkgdir := "pkg"
	if cfg.HasOption("hwaf-cfg", "cmtpkgs") {
		pkgdir, err = cfg.String("hwaf-cfg", "cmtpkgs")
		handle_err(err)
	}

	git := exec.Command("git", "submodule", "add",
		pkguri,
		filepath.Join(pkgdir, pkgname),
		)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	git = exec.Command(
		"git", "submodule", "update",
		"--init", "--recursive",
		)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]... [ok]\n", n, pkguri)
	}
}

// EOF
<commit_msg>co: typos<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/sbinet/go-commander"
	"github.com/sbinet/go-flag"
	//gocfg "github.com/sbinet/go-config/config"
)

func hwaf_make_cmd_checkout() *commander.Command {
	cmd := &commander.Command{
		Run:       hwaf_run_cmd_checkout,
		UsageLine: "co [options] <pkg-uri> [<local-pkg-name>]",
		Short:     "add a package to the current workarea",
		Long: `
co adds a package to the current workarea.

ex:
 $ hwaf co /foo/pkg
 $ hwaf co Control/AthenaKernel
 $ hwaf co git://github.com/mana-fwk/mana-core-athenakernel
 $ hwaf co git://github.com/mana-fwk/mana-core-athenakernel Control/AthenaKernel
`,
		Flag: *flag.NewFlagSet("hwaf-checkout", flag.ExitOnError),
	}
	cmd.Flag.Bool("q", false, "only print error and warning messages, all other output will be suppressed")

	return cmd
}

func hwaf_run_cmd_checkout(cmd *commander.Command, args []string) {
	var err error
	n := "hwaf-" + cmd.Name()
	pkguri := ""
	pkgname := ""
	switch len(args) {
	case 1:
		pkguri = args[0]
		pkgname = filepath.Base(args[0])
	case 2:
		pkguri = args[0]
		pkgname = args[1]
	default:
		err = fmt.Errorf("%s: you need to give a package URL", n)
		handle_err(err)
	}

	pkguri = os.ExpandEnv(pkguri)
	//pkguri = filepath.Clean(pkguri)

	pkgname = os.ExpandEnv(pkgname)
	pkgname = filepath.Clean(pkgname)

	quiet := cmd.Flag.Lookup("q").Value.Get().(bool)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]...\n", n, pkguri)
	}

	cfg := load_local_cfg()
	pkgdir := "pkg"
	if cfg.HasOption("hwaf-cfg", "cmtpkgs") {
		pkgdir, err = cfg.String("hwaf-cfg", "cmtpkgs")
		handle_err(err)
	}

	git := exec.Command("git", "submodule", "add",
		pkguri,
		filepath.Join(pkgdir, pkgname),
		)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	git = exec.Command(
		"git", "submodule", "update",
		"--init", "--recursive",
		)
	if !quiet {
		git.Stdout = os.Stdout
		git.Stderr = os.Stderr
	}
	err = git.Run()
	handle_err(err)

	if !quiet {
		fmt.Printf("%s: checkout package [%s]... [ok]\n", n, pkguri)
	}
}

// EOF
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jaegerreceiver

// This file implements factory for Jaeger receiver.

import (
	"context"
	"fmt"
	"net"
	"strconv"

	"github.com/spf13/viper"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configerror"
	"go.opentelemetry.io/collector/config/configmodels"
	"go.opentelemetry.io/collector/config/configprotocol"
	"go.opentelemetry.io/collector/consumer"
)

const (
	// The value of "type" key in configuration.
	typeStr = "jaeger"

	// Protocol values.
	protoGRPC       = "grpc"
	protoThriftHTTP = "thrift_http"
	// Deprecated, see https://go.opentelemetry.io/collector/issues/267
	protoThriftTChannel = "thrift_tchannel"
	protoThriftBinary   = "thrift_binary"
	protoThriftCompact  = "thrift_compact"

	// Default endpoints to bind to.
	defaultGRPCBindEndpoint = "0.0.0.0:14250"
	defaultHTTPBindEndpoint = "0.0.0.0:14268"

	defaultThriftCompactBindEndpoint   = "0.0.0.0:6831"
	defaultThriftBinaryBindEndpoint    = "0.0.0.0:6832"
	defaultAgentRemoteSamplingHTTPPort = 5778
)

// Factory is the factory for Jaeger receiver.
type Factory struct {
}

// Type gets the type of the Receiver config created by this factory.
func (f *Factory) Type() configmodels.Type {
	return typeStr
}

// CustomUnmarshaler is used to add defaults for named but empty protocols
func (f *Factory) CustomUnmarshaler() component.CustomUnmarshaler {
	return func(componentViperSection *viper.Viper, intoCfg interface{}) error {
		if componentViperSection == nil || len(componentViperSection.AllKeys()) == 0 {
			return fmt.Errorf("empty config for Jaeger receiver")
		}

		// first load the config normally
		err := componentViperSection.UnmarshalExact(intoCfg)
		if err != nil {
			return err
		}

		receiverCfg, ok := intoCfg.(*Config)
		if !ok {
			return fmt.Errorf("config type not *jaegerreceiver.Config")
		}

		// next manually search for protocols in viper that do not appear in the normally loaded config
		// these protocols were excluded during normal loading and we need to add defaults for them
		protocols := componentViperSection.GetStringMap(protocolsFieldName)
		if len(protocols) == 0 {
			return fmt.Errorf("must specify at least one protocol when using the Jaeger receiver")
		}
		for k := range protocols {
			if _, ok := receiverCfg.Protocols[k]; !ok {
				if receiverCfg.Protocols[k], err = defaultsForProtocol(k); err != nil {
					return err
				}
			}
		}

		return nil
	}
}

// CreateDefaultConfig creates the default configuration for Jaeger receiver.
func (f *Factory) CreateDefaultConfig() configmodels.Receiver {
	return &Config{
		ReceiverSettings: configmodels.ReceiverSettings{
			TypeVal: typeStr,
			NameVal: typeStr,
		},
		Protocols: map[string]*configprotocol.ProtocolServerSettings{},
	}
}

// CreateTraceReceiver creates a trace receiver based on provided config.
func (f *Factory) CreateTraceReceiver(
	ctx context.Context,
	params component.ReceiverCreateParams,
	cfg configmodels.Receiver,
	nextConsumer consumer.TraceConsumer,
) (component.TraceReceiver, error) {

	// Convert settings in the source config to Configuration struct
	// that Jaeger receiver understands.

	rCfg := cfg.(*Config)

	protoGRPC := rCfg.Protocols[protoGRPC]
	protoHTTP := rCfg.Protocols[protoThriftHTTP]
	protoTChannel := rCfg.Protocols[protoThriftTChannel]
	protoThriftCompact := rCfg.Protocols[protoThriftCompact]
	protoThriftBinary := rCfg.Protocols[protoThriftBinary]
	remoteSamplingConfig := rCfg.RemoteSampling

	config := Configuration{}
	var grpcServerOptions []grpc.ServerOption
	logger := params.Logger

	// Set ports
	if protoGRPC != nil {
		var err error
		config.CollectorGRPCPort, err = extractPortFromEndpoint(protoGRPC.Endpoint)
		if err != nil {
			return nil, err
		}

		if protoGRPC.TLSCredentials != nil {
			tlsCfg, err := protoGRPC.TLSCredentials.LoadTLSConfig()
			if err != nil {
				return nil, fmt.Errorf("failed to configure TLS: %v", err)
			}
			grpcServerOptions = append(grpcServerOptions, grpc.Creds(credentials.NewTLS(tlsCfg)))
		}
		config.CollectorGRPCOptions = grpcServerOptions
	}

	if protoHTTP != nil {
		var err error
		config.CollectorHTTPPort, err = extractPortFromEndpoint(protoHTTP.Endpoint)
		if err != nil {
			return nil, err
		}
	}

	if protoTChannel != nil {
		logger.Warn("Protocol unknown or not supported", zap.String("protocol", protoThriftTChannel))
	}

	if protoThriftBinary != nil {
		var err error
		config.AgentBinaryThriftPort, err = extractPortFromEndpoint(protoThriftBinary.Endpoint)
		if err != nil {
			return nil, err
		}
	}

	if protoThriftCompact != nil {
		var err error
		config.AgentCompactThriftPort, err = extractPortFromEndpoint(protoThriftCompact.Endpoint)
		if err != nil {
			return nil, err
		}
	}

	if remoteSamplingConfig != nil {
		config.RemoteSamplingClientSettings = remoteSamplingConfig.GRPCClientSettings
		if len(config.RemoteSamplingClientSettings.Endpoint) == 0 {
			config.RemoteSamplingClientSettings.Endpoint = defaultGRPCBindEndpoint
		}

		if len(remoteSamplingConfig.HostEndpoint) == 0 {
			config.AgentHTTPPort = defaultAgentRemoteSamplingHTTPPort
		} else {
			var err error
			config.AgentHTTPPort, err = extractPortFromEndpoint(remoteSamplingConfig.HostEndpoint)
			if err != nil {
				return nil, err
			}
		}

		// strategies are served over grpc so if grpc is not enabled and strategies are present return an error
		if len(remoteSamplingConfig.StrategyFile) != 0 {
			if config.CollectorGRPCPort == 0 {
				return nil, fmt.Errorf("strategy file requires the GRPC protocol to be enabled")
			}

			config.RemoteSamplingStrategyFile = remoteSamplingConfig.StrategyFile
		}
	}

	if (protoGRPC == nil && protoHTTP == nil && protoThriftBinary == nil && protoThriftCompact == nil) ||
		(config.CollectorGRPCPort == 0 && config.CollectorHTTPPort == 0 && config.CollectorThriftPort == 0 && config.AgentBinaryThriftPort == 0 && config.AgentCompactThriftPort == 0) {
		err := fmt.Errorf("either %v, %v, %v, or %v protocol endpoint with non-zero port must be enabled for %s receiver",
			protoGRPC,
			protoThriftHTTP,
			protoThriftCompact,
			protoThriftBinary,
			typeStr,
		)
		return nil, err
	}

	// Create the receiver.
	return New(rCfg.Name(), &config, nextConsumer, params)
}

// CreateMetricsReceiver creates a metrics receiver based on provided config.
func (f *Factory) CreateMetricsReceiver(
	_ context.Context,
	_ component.ReceiverCreateParams,
	_ configmodels.Receiver,
	_ consumer.MetricsConsumer,
) (component.MetricsReceiver, error) {
	return nil, configerror.ErrDataTypeIsNotSupported
}

// extract the port number from string in "address:port" format. If the
// port number cannot be extracted returns an error.
func extractPortFromEndpoint(endpoint string) (int, error) {
	_, portStr, err := net.SplitHostPort(endpoint)
	if err != nil {
		return 0, fmt.Errorf("endpoint is not formatted correctly: %s", err.Error())
	}
	port, err := strconv.ParseInt(portStr, 10, 0)
	if err != nil {
		return 0, fmt.Errorf("endpoint port is not a number: %s", err.Error())
	}
	if port < 1 || port > 65535 {
		return 0, fmt.Errorf("port number must be between 1 and 65535")
	}
	return int(port), nil
}

// returns a default value for a protocol name.  this really just boils down to the endpoint
func defaultsForProtocol(proto string) (*configprotocol.ProtocolServerSettings, error) {
	var defaultEndpoint string

	switch proto {
	case protoGRPC:
		defaultEndpoint = defaultGRPCBindEndpoint
	case protoThriftHTTP:
		defaultEndpoint = defaultHTTPBindEndpoint
	case protoThriftBinary:
		defaultEndpoint = defaultThriftBinaryBindEndpoint
	case protoThriftCompact:
		defaultEndpoint = defaultThriftCompactBindEndpoint
	default:
		return nil, fmt.Errorf("unknown Jaeger protocol %s", proto)
	}

	return &configprotocol.ProtocolServerSettings{
		Endpoint: defaultEndpoint,
	}, nil
}
<commit_msg>Remove unnecessary constant and check (#1207)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jaegerreceiver

// This file implements factory for Jaeger receiver.

import (
	"context"
	"fmt"
	"net"
	"strconv"

	"github.com/spf13/viper"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configerror"
	"go.opentelemetry.io/collector/config/configmodels"
	"go.opentelemetry.io/collector/config/configprotocol"
	"go.opentelemetry.io/collector/consumer"
)

const (
	// The value of "type" key in configuration.
	typeStr = "jaeger"

	// Protocol values.
	protoGRPC          = "grpc"
	protoThriftHTTP    = "thrift_http"
	protoThriftBinary  = "thrift_binary"
	protoThriftCompact = "thrift_compact"

	// Default endpoints to bind to.
	defaultGRPCBindEndpoint = "0.0.0.0:14250"
	defaultHTTPBindEndpoint = "0.0.0.0:14268"

	defaultThriftCompactBindEndpoint   = "0.0.0.0:6831"
	defaultThriftBinaryBindEndpoint    = "0.0.0.0:6832"
	defaultAgentRemoteSamplingHTTPPort = 5778
)

// Factory is the factory for Jaeger receiver.
type Factory struct {
}

// Type gets the type of the Receiver config created by this factory.
func (f *Factory) Type() configmodels.Type {
	return typeStr
}

// CustomUnmarshaler is used to add defaults for named but empty protocols
func (f *Factory) CustomUnmarshaler() component.CustomUnmarshaler {
	return func(componentViperSection *viper.Viper, intoCfg interface{}) error {
		if componentViperSection == nil || len(componentViperSection.AllKeys()) == 0 {
			return fmt.Errorf("empty config for Jaeger receiver")
		}

		// first load the config normally
		err := componentViperSection.UnmarshalExact(intoCfg)
		if err != nil {
			return err
		}

		receiverCfg, ok := intoCfg.(*Config)
		if !ok {
			return fmt.Errorf("config type not *jaegerreceiver.Config")
		}

		// next manually search for protocols in viper that do not appear in the normally loaded config
		// these protocols were excluded during normal loading and we need to add defaults for them
		protocols := componentViperSection.GetStringMap(protocolsFieldName)
		if len(protocols) == 0 {
			return fmt.Errorf("must specify at least one protocol when using the Jaeger receiver")
		}
		for k := range protocols {
			if _, ok := receiverCfg.Protocols[k]; !ok {
				if receiverCfg.Protocols[k], err = defaultsForProtocol(k); err != nil {
					return err
				}
			}
		}

		return nil
	}
}

// CreateDefaultConfig creates the default configuration for Jaeger receiver.
func (f *Factory) CreateDefaultConfig() configmodels.Receiver {
	return &Config{
		ReceiverSettings: configmodels.ReceiverSettings{
			TypeVal: typeStr,
			NameVal: typeStr,
		},
		Protocols: map[string]*configprotocol.ProtocolServerSettings{},
	}
}

// CreateTraceReceiver creates a trace receiver based on provided config.
func (f *Factory) CreateTraceReceiver(
	ctx context.Context,
	params component.ReceiverCreateParams,
	cfg configmodels.Receiver,
	nextConsumer consumer.TraceConsumer,
) (component.TraceReceiver, error) {

	// Convert settings in the source config to Configuration struct
	// that Jaeger receiver understands.

	rCfg := cfg.(*Config)

	protoGRPC := rCfg.Protocols[protoGRPC]
	protoHTTP := rCfg.Protocols[protoThriftHTTP]
	protoThriftCompact := rCfg.Protocols[protoThriftCompact]
	protoThriftBinary := rCfg.Protocols[protoThriftBinary]
	remoteSamplingConfig := rCfg.RemoteSampling

	config := Configuration{}
	var grpcServerOptions []grpc.ServerOption

	// Set ports
	if protoGRPC != nil {
		var err error
		config.CollectorGRPCPort, err = extractPortFromEndpoint(protoGRPC.Endpoint)
		if err != nil {
			return nil, err
		}

		if protoGRPC.TLSCredentials != nil {
			tlsCfg, err := protoGRPC.TLSCredentials.LoadTLSConfig()
			if err != nil {
				return nil, fmt.Errorf("failed to configure TLS: %v", err)
			}
			grpcServerOptions = append(grpcServerOptions, grpc.Creds(credentials.NewTLS(tlsCfg)))
		}
		config.CollectorGRPCOptions = grpcServerOptions
	}

	if protoHTTP != nil {
		var err error
		config.CollectorHTTPPort, err = extractPortFromEndpoint(protoHTTP.Endpoint)
		if err != nil {
			return nil, err
		}
	}

	if protoThriftBinary != nil {
		var err error
		config.AgentBinaryThriftPort, err = extractPortFromEndpoint(protoThriftBinary.Endpoint)
		if err != nil {
			return nil, err
		}
	}

	if protoThriftCompact != nil {
		var err error
		config.AgentCompactThriftPort, err = extractPortFromEndpoint(protoThriftCompact.Endpoint)
		if err != nil {
			return nil, err
		}
	}

	if remoteSamplingConfig != nil {
		config.RemoteSamplingClientSettings = remoteSamplingConfig.GRPCClientSettings
		if len(config.RemoteSamplingClientSettings.Endpoint) == 0 {
			config.RemoteSamplingClientSettings.Endpoint = defaultGRPCBindEndpoint
		}

		if len(remoteSamplingConfig.HostEndpoint) == 0 {
			config.AgentHTTPPort = defaultAgentRemoteSamplingHTTPPort
		} else {
			var err error
			config.AgentHTTPPort, err = extractPortFromEndpoint(remoteSamplingConfig.HostEndpoint)
			if err != nil {
				return nil, err
			}
		}

		// strategies are served over grpc so if grpc is not enabled and strategies are present return an error
		if len(remoteSamplingConfig.StrategyFile) != 0 {
			if config.CollectorGRPCPort == 0 {
				return nil, fmt.Errorf("strategy file requires the GRPC protocol to be enabled")
			}

			config.RemoteSamplingStrategyFile = remoteSamplingConfig.StrategyFile
		}
	}

	if (protoGRPC == nil && protoHTTP == nil && protoThriftBinary == nil && protoThriftCompact == nil) ||
		(config.CollectorGRPCPort == 0 && config.CollectorHTTPPort == 0 && config.CollectorThriftPort == 0 && config.AgentBinaryThriftPort == 0 && config.AgentCompactThriftPort == 0) {
		err := fmt.Errorf("either %v, %v, %v, or %v protocol endpoint with non-zero port must be enabled for %s receiver",
			protoGRPC,
			protoThriftHTTP,
			protoThriftCompact,
			protoThriftBinary,
			typeStr,
		)
		return nil, err
	}

	// Create the receiver.
	return New(rCfg.Name(), &config, nextConsumer, params)
}

// CreateMetricsReceiver creates a metrics receiver based on provided config.
func (f *Factory) CreateMetricsReceiver(
	_ context.Context,
	_ component.ReceiverCreateParams,
	_ configmodels.Receiver,
	_ consumer.MetricsConsumer,
) (component.MetricsReceiver, error) {
	return nil, configerror.ErrDataTypeIsNotSupported
}

// extract the port number from string in "address:port" format. If the
// port number cannot be extracted returns an error.
func extractPortFromEndpoint(endpoint string) (int, error) {
	_, portStr, err := net.SplitHostPort(endpoint)
	if err != nil {
		return 0, fmt.Errorf("endpoint is not formatted correctly: %s", err.Error())
	}
	port, err := strconv.ParseInt(portStr, 10, 0)
	if err != nil {
		return 0, fmt.Errorf("endpoint port is not a number: %s", err.Error())
	}
	if port < 1 || port > 65535 {
		return 0, fmt.Errorf("port number must be between 1 and 65535")
	}
	return int(port), nil
}

// returns a default value for a protocol name.  this really just boils down to the endpoint
func defaultsForProtocol(proto string) (*configprotocol.ProtocolServerSettings, error) {
	var defaultEndpoint string

	switch proto {
	case protoGRPC:
		defaultEndpoint = defaultGRPCBindEndpoint
	case protoThriftHTTP:
		defaultEndpoint = defaultHTTPBindEndpoint
	case protoThriftBinary:
		defaultEndpoint = defaultThriftBinaryBindEndpoint
	case protoThriftCompact:
		defaultEndpoint = defaultThriftCompactBindEndpoint
	default:
		return nil, fmt.Errorf("unknown Jaeger protocol %s", proto)
	}

	return &configprotocol.ProtocolServerSettings{
		Endpoint: defaultEndpoint,
	}, nil
}
<|endoftext|>


<commit_before>package pointers

import "fmt"

type Calculator struct {
	Memory int
}

func (calculator *Calculator) Sum(value int) {
	fmt.Println("calculator pointer: %s", &calculator.Memory)
	calculator.Memory = calculator.Memory + 100
}

func (calculator *Calculator) GetMemory() int {
	fmt.Println("calculator pointer: %s", &calculator.Memory)
	return calculator.Memory
}

func NewCalculatorPointer(initValue int) *Calculator {
	return &Calculator{
		Memory: initValue,
	}
}

func NewCalculatorValue(initValue int) Calculator {
	return Calculator{
		Memory: initValue,
	}
}

func (calculator Calculator) SumNoPointer(value int) {
	fmt.Println("calculator pointer: %s", &calculator.Memory)
	calculator.Memory = calculator.Memory + 100
}

func (calculator Calculator) GetMemoryNoPointer() int {
	fmt.Println("calculator pointer: %s", &calculator.Memory)
	return calculator.Memory
}
<commit_msg>fix fmt print msg<commit_after>package pointers

import "fmt"

type Calculator struct {
	Memory int
}

func (calculator *Calculator) Sum(value int) {
	fmt.Println("memory pointer: %s", &calculator.Memory)
	calculator.Memory = calculator.Memory + 100
}

func (calculator *Calculator) GetMemory() int {
	fmt.Println("memory pointer: %s", &calculator.Memory)
	return calculator.Memory
}

func NewCalculatorPointer(initValue int) *Calculator {
	return &Calculator{
		Memory: initValue,
	}
}

func NewCalculatorValue(initValue int) Calculator {
	return Calculator{
		Memory: initValue,
	}
}

func (calculator Calculator) SumNoPointer(value int) {
	fmt.Println("memory pointer: %s", &calculator.Memory)
	calculator.Memory = calculator.Memory + 100
}

func (calculator Calculator) GetMemoryNoPointer() int {
	fmt.Println("memory pointer: %s", &calculator.Memory)
	return calculator.Memory
}
<|endoftext|>


<commit_before>//  Copyright 2018 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// Package policies configures OS Guest Policies based on osconfig API response.
package policies

import (
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"hash"
	"io"
	"os"
	"path/filepath"
	"strings"

	"cloud.google.com/go/compute/metadata"
	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
	osconfig "github.com/GoogleCloudPlatform/osconfig/_internal/gapi-cloud-osconfig-go/cloud.google.com/go/osconfig/apiv1alpha2"
	"github.com/GoogleCloudPlatform/osconfig/common"
	"github.com/GoogleCloudPlatform/osconfig/config"
	"github.com/GoogleCloudPlatform/osconfig/inventory/osinfo"
	"github.com/GoogleCloudPlatform/osconfig/inventory/packages"
	"github.com/GoogleCloudPlatform/osconfig/tasker"
	"google.golang.org/api/option"
	"google.golang.org/grpc/status"

	osconfigpb "github.com/GoogleCloudPlatform/osconfig/_internal/gapi-cloud-osconfig-go/google.golang.org/genproto/googleapis/cloud/osconfig/v1alpha2"
)

func run(ctx context.Context, res string) {
	client, err := osconfig.NewClient(ctx, option.WithEndpoint(config.SvcEndpoint()), option.WithCredentialsFile(config.OAuthPath()))
	if err != nil {
		logger.Errorf("osconfig.NewClient Error: %v", err)
		return
	}
	defer client.Close()

	resp, err := lookupEffectivePolicies(ctx, client, res)
	if err != nil {
		logger.Errorf("lookupEffectivePolicies error: %v", err)
		return
	}

	// We don't check the error from ospackage.SetConfig as all errors are already logged.
	setConfig(resp)
}

// Run looks up osconfigs and applies them using tasker.Enqueue.
func Run(ctx context.Context, res string) {
	tasker.Enqueue("Run GuestPolicies", func() { run(ctx, res) })
}

func lookupEffectivePolicies(ctx context.Context, client *osconfig.Client, instance string) (*osconfigpb.LookupEffectiveGuestPoliciesResponse, error) {
	var shortName, version, arch string
	if config.OSInventoryEnabled() {
		logger.Debugf("OS Inventory enabled for instance, gathering DistributionInfo for LookupEffectiveGuestPoliciesRequest")
		info, err := osinfo.GetDistributionInfo()
		if err != nil {
			return nil, err
		}
		shortName = info.ShortName
		version = info.Version
		arch = info.Architecture
	} else {
		logger.Debugf("OS Inventory not enabled for instance, not gathering DistributionInfo for LookupEffectiveGuestPoliciesRequest")
	}

	identityToken, err := metadata.Get(config.IdentityTokenPath)
	if err != nil {
		return nil, err
	}

	req := &osconfigpb.LookupEffectiveGuestPoliciesRequest{
		Instance:        instance,
		OsShortName:     shortName,
		OsVersion:       version,
		OsArchitecture:  arch,
		InstanceIdToken: identityToken,
	}
	logger.Debugf("LookupEffectiveGuestPolicies request: {Instance: %s, OsShortName: %s, OsVersion: %s, OsArchitecture: %s}",
		req.GetInstance(), req.GetOsShortName(), req.GetOsVersion(), req.GetOsArchitecture())

	res, err := client.LookupEffectiveGuestPolicies(ctx, req)
	if err != nil {
		if s, ok := status.FromError(err); ok {
			return nil, fmt.Errorf("code: %q, message: %q, details: %q", s.Code(), s.Message(), s.Details())
		}
		return nil, err
	}
	logger.Debugf("LookupEffectiveGuestPolicies response:\n%s", common.PrettyFmt(res))

	return res, nil
}

func setConfig(res *osconfigpb.LookupEffectiveGuestPoliciesResponse) error {
	var aptRepos []*osconfigpb.AptRepository
	var yumRepos []*osconfigpb.YumRepository
	var zypperRepos []*osconfigpb.ZypperRepository
	var gooRepos []*osconfigpb.GooRepository
	for _, repo := range res.GetPackageRepositories() {
		if r := repo.GetPackageRepository().GetGoo(); r != nil {
			gooRepos = append(gooRepos, r)
			continue
		}
		if r := repo.GetPackageRepository().GetApt(); r != nil {
			aptRepos = append(aptRepos, r)
			continue
		}
		if r := repo.GetPackageRepository().GetYum(); r != nil {
			yumRepos = append(yumRepos, r)
			continue
		}
		if r := repo.GetPackageRepository().GetZypper(); r != nil {
			zypperRepos = append(zypperRepos, r)
			continue
		}
	}

	var gooInstallPkgs, gooRemovePkgs, gooUpdatePkgs []*osconfigpb.Package
	var aptInstallPkgs, aptRemovePkgs, aptUpdatePkgs []*osconfigpb.Package
	var yumInstallPkgs, yumRemovePkgs, yumUpdatePkgs []*osconfigpb.Package
	var zypperInstallPkgs, zypperRemovePkgs, zypperUpdatePkgs []*osconfigpb.Package
	for _, pkg := range res.GetPackages() {
		switch pkg.GetPackage().GetManager() {
		case osconfigpb.Package_ANY, osconfigpb.Package_MANAGER_UNSPECIFIED:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				gooInstallPkgs = append(gooInstallPkgs, pkg.GetPackage())
				aptInstallPkgs = append(aptInstallPkgs, pkg.GetPackage())
				yumInstallPkgs = append(yumInstallPkgs, pkg.GetPackage())
				zypperInstallPkgs = append(zypperInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				gooRemovePkgs = append(gooRemovePkgs, pkg.GetPackage())
				aptRemovePkgs = append(aptRemovePkgs, pkg.GetPackage())
				yumRemovePkgs = append(yumRemovePkgs, pkg.GetPackage())
				zypperRemovePkgs = append(zypperRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				gooUpdatePkgs = append(gooUpdatePkgs, pkg.GetPackage())
				aptUpdatePkgs = append(aptUpdatePkgs, pkg.GetPackage())
				yumUpdatePkgs = append(yumUpdatePkgs, pkg.GetPackage())
				zypperUpdatePkgs = append(zypperUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_GOO:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				gooInstallPkgs = append(gooInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				gooRemovePkgs = append(gooRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				gooUpdatePkgs = append(gooUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_APT:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				aptInstallPkgs = append(aptInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				aptRemovePkgs = append(aptRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				aptUpdatePkgs = append(aptUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_YUM:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				yumInstallPkgs = append(yumInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				yumRemovePkgs = append(yumRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				yumUpdatePkgs = append(yumUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_ZYPPER:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				zypperInstallPkgs = append(zypperInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				zypperRemovePkgs = append(zypperRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				zypperUpdatePkgs = append(zypperUpdatePkgs, pkg.GetPackage())
			}

		}

	}

	var errs []string
	if packages.GooGetExists {
		if _, err := os.Stat(config.GooGetRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.GooGetRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating googet repo file: %v", err))
			}
		}
		if err := googetRepositories(gooRepos, config.GooGetRepoFilePath()); err != nil {
			logger.Errorf("Error writing googet repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing googet repo file: %v", err))
		}
		if err := googetChanges(gooInstallPkgs, gooRemovePkgs, gooUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing googet changes: %v", err))
		}
	}

	if packages.AptExists {
		if _, err := os.Stat(config.AptRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.AptRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating apt repo file: %v", err))
			}
		}
		if err := aptRepositories(aptRepos, config.AptRepoFilePath()); err != nil {
			logger.Errorf("Error writing apt repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing apt repo file: %v", err))
		}
		if err := aptChanges(aptInstallPkgs, aptRemovePkgs, aptUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing apt changes: %v", err))
		}
	}

	if packages.YumExists {
		if _, err := os.Stat(config.YumRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.YumRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating yum repo file: %v", err))
			}
		}
		if err := yumRepositories(yumRepos, config.YumRepoFilePath()); err != nil {
			logger.Errorf("Error writing yum repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing yum repo file: %v", err))
		}
		if err := yumChanges(yumInstallPkgs, yumRemovePkgs, yumUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing yum changes: %v", err))
		}
	}

	if packages.ZypperExists {
		if _, err := os.Stat(config.ZypperRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.ZypperRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating zypper repo file: %v", err))
			}
		}
		if err := zypperRepositories(zypperRepos, config.ZypperRepoFilePath()); err != nil {
			logger.Errorf("Error writing zypper repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing zypper repo file: %v", err))
		}
		if err := zypperChanges(zypperInstallPkgs, zypperRemovePkgs, zypperUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing zypper changes: %v", err))
		}
	}

	if errs == nil {
		return nil
	}
	return errors.New(strings.Join(errs, ",\n"))
}

func checksum(r io.Reader) hash.Hash {
	hash := sha256.New()
	io.Copy(hash, r)
	return hash
}

func writeIfChanged(content []byte, path string) error {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return err
	}

	reader := bytes.NewReader(content)
	h1 := checksum(reader)
	h2 := checksum(file)
	if bytes.Equal(h1.Sum(nil), h2.Sum(nil)) {
		file.Close()
		return nil
	}

	logger.Infof("Writing repo file %s with updated contents", path)
	if err := file.Truncate(0); err != nil {
		file.Close()
		return err
	}
	if _, err := file.WriteAt(content, 0); err != nil {
		file.Close()
		return err
	}

	return file.Close()
}
<commit_msg>enable software recipes (#61)<commit_after>//  Copyright 2018 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// Package policies configures OS Guest Policies based on osconfig API response.
package policies

import (
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"hash"
	"io"
	"os"
	"path/filepath"
	"strings"

	"cloud.google.com/go/compute/metadata"
	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
	osconfig "github.com/GoogleCloudPlatform/osconfig/_internal/gapi-cloud-osconfig-go/cloud.google.com/go/osconfig/apiv1alpha2"
	"github.com/GoogleCloudPlatform/osconfig/common"
	"github.com/GoogleCloudPlatform/osconfig/config"
	"github.com/GoogleCloudPlatform/osconfig/inventory/osinfo"
	"github.com/GoogleCloudPlatform/osconfig/inventory/packages"
	"github.com/GoogleCloudPlatform/osconfig/policies/recipes"
	"github.com/GoogleCloudPlatform/osconfig/tasker"
	"google.golang.org/api/option"
	"google.golang.org/grpc/status"

	osconfigpb "github.com/GoogleCloudPlatform/osconfig/_internal/gapi-cloud-osconfig-go/google.golang.org/genproto/googleapis/cloud/osconfig/v1alpha2"
)

func run(ctx context.Context, res string) {
	client, err := osconfig.NewClient(ctx, option.WithEndpoint(config.SvcEndpoint()), option.WithCredentialsFile(config.OAuthPath()))
	if err != nil {
		logger.Errorf("osconfig.NewClient Error: %v", err)
		return
	}
	defer client.Close()

	resp, err := lookupEffectivePolicies(ctx, client, res)
	if err != nil {
		logger.Errorf("lookupEffectivePolicies error: %v", err)
		return
	}

	// We don't check the error from ospackage.SetConfig as all errors are already logged.
	setConfig(resp)

	installRecipes(ctx, resp)
}

// Run looks up osconfigs and applies them using tasker.Enqueue.
func Run(ctx context.Context, res string) {
	tasker.Enqueue("Run GuestPolicies", func() { run(ctx, res) })
}

func lookupEffectivePolicies(ctx context.Context, client *osconfig.Client, instance string) (*osconfigpb.LookupEffectiveGuestPoliciesResponse, error) {
	var shortName, version, arch string
	if config.OSInventoryEnabled() {
		logger.Debugf("OS Inventory enabled for instance, gathering DistributionInfo for LookupEffectiveGuestPoliciesRequest")
		info, err := osinfo.GetDistributionInfo()
		if err != nil {
			return nil, err
		}
		shortName = info.ShortName
		version = info.Version
		arch = info.Architecture
	} else {
		logger.Debugf("OS Inventory not enabled for instance, not gathering DistributionInfo for LookupEffectiveGuestPoliciesRequest")
	}

	identityToken, err := metadata.Get(config.IdentityTokenPath)
	if err != nil {
		return nil, err
	}

	req := &osconfigpb.LookupEffectiveGuestPoliciesRequest{
		Instance:        instance,
		OsShortName:     shortName,
		OsVersion:       version,
		OsArchitecture:  arch,
		InstanceIdToken: identityToken,
	}
	logger.Debugf("LookupEffectiveGuestPolicies request: {Instance: %s, OsShortName: %s, OsVersion: %s, OsArchitecture: %s}",
		req.GetInstance(), req.GetOsShortName(), req.GetOsVersion(), req.GetOsArchitecture())

	res, err := client.LookupEffectiveGuestPolicies(ctx, req)
	if err != nil {
		if s, ok := status.FromError(err); ok {
			return nil, fmt.Errorf("code: %q, message: %q, details: %q", s.Code(), s.Message(), s.Details())
		}
		return nil, err
	}
	logger.Debugf("LookupEffectiveGuestPolicies response:\n%s", common.PrettyFmt(res))

	return res, nil
}

func installRecipes(ctx context.Context, res *osconfigpb.LookupEffectiveGuestPoliciesResponse) error {
	for _, recipe := range res.GetSoftwareRecipes() {
		if r := recipe.GetSoftwareRecipe(); r != nil {
			if err := recipes.InstallRecipe(ctx, r); err != nil {
				logger.Errorf("Error installing recipe: %v", err)
			}
		}
	}
	return nil
}

func setConfig(res *osconfigpb.LookupEffectiveGuestPoliciesResponse) error {
	var aptRepos []*osconfigpb.AptRepository
	var yumRepos []*osconfigpb.YumRepository
	var zypperRepos []*osconfigpb.ZypperRepository
	var gooRepos []*osconfigpb.GooRepository
	for _, repo := range res.GetPackageRepositories() {
		if r := repo.GetPackageRepository().GetGoo(); r != nil {
			gooRepos = append(gooRepos, r)
			continue
		}
		if r := repo.GetPackageRepository().GetApt(); r != nil {
			aptRepos = append(aptRepos, r)
			continue
		}
		if r := repo.GetPackageRepository().GetYum(); r != nil {
			yumRepos = append(yumRepos, r)
			continue
		}
		if r := repo.GetPackageRepository().GetZypper(); r != nil {
			zypperRepos = append(zypperRepos, r)
			continue
		}
	}

	var gooInstallPkgs, gooRemovePkgs, gooUpdatePkgs []*osconfigpb.Package
	var aptInstallPkgs, aptRemovePkgs, aptUpdatePkgs []*osconfigpb.Package
	var yumInstallPkgs, yumRemovePkgs, yumUpdatePkgs []*osconfigpb.Package
	var zypperInstallPkgs, zypperRemovePkgs, zypperUpdatePkgs []*osconfigpb.Package
	for _, pkg := range res.GetPackages() {
		switch pkg.GetPackage().GetManager() {
		case osconfigpb.Package_ANY, osconfigpb.Package_MANAGER_UNSPECIFIED:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				gooInstallPkgs = append(gooInstallPkgs, pkg.GetPackage())
				aptInstallPkgs = append(aptInstallPkgs, pkg.GetPackage())
				yumInstallPkgs = append(yumInstallPkgs, pkg.GetPackage())
				zypperInstallPkgs = append(zypperInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				gooRemovePkgs = append(gooRemovePkgs, pkg.GetPackage())
				aptRemovePkgs = append(aptRemovePkgs, pkg.GetPackage())
				yumRemovePkgs = append(yumRemovePkgs, pkg.GetPackage())
				zypperRemovePkgs = append(zypperRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				gooUpdatePkgs = append(gooUpdatePkgs, pkg.GetPackage())
				aptUpdatePkgs = append(aptUpdatePkgs, pkg.GetPackage())
				yumUpdatePkgs = append(yumUpdatePkgs, pkg.GetPackage())
				zypperUpdatePkgs = append(zypperUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_GOO:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				gooInstallPkgs = append(gooInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				gooRemovePkgs = append(gooRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				gooUpdatePkgs = append(gooUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_APT:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				aptInstallPkgs = append(aptInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				aptRemovePkgs = append(aptRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				aptUpdatePkgs = append(aptUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_YUM:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				yumInstallPkgs = append(yumInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				yumRemovePkgs = append(yumRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				yumUpdatePkgs = append(yumUpdatePkgs, pkg.GetPackage())
			}
		case osconfigpb.Package_ZYPPER:
			switch pkg.GetPackage().GetDesiredState() {
			case osconfigpb.DesiredState_INSTALLED, osconfigpb.DesiredState_DESIRED_STATE_UNSPECIFIED:
				zypperInstallPkgs = append(zypperInstallPkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_REMOVED:
				zypperRemovePkgs = append(zypperRemovePkgs, pkg.GetPackage())
			case osconfigpb.DesiredState_UPDATED:
				zypperUpdatePkgs = append(zypperUpdatePkgs, pkg.GetPackage())
			}

		}

	}

	var errs []string
	if packages.GooGetExists {
		if _, err := os.Stat(config.GooGetRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.GooGetRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating googet repo file: %v", err))
			}
		}
		if err := googetRepositories(gooRepos, config.GooGetRepoFilePath()); err != nil {
			logger.Errorf("Error writing googet repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing googet repo file: %v", err))
		}
		if err := googetChanges(gooInstallPkgs, gooRemovePkgs, gooUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing googet changes: %v", err))
		}
	}

	if packages.AptExists {
		if _, err := os.Stat(config.AptRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.AptRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating apt repo file: %v", err))
			}
		}
		if err := aptRepositories(aptRepos, config.AptRepoFilePath()); err != nil {
			logger.Errorf("Error writing apt repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing apt repo file: %v", err))
		}
		if err := aptChanges(aptInstallPkgs, aptRemovePkgs, aptUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing apt changes: %v", err))
		}
	}

	if packages.YumExists {
		if _, err := os.Stat(config.YumRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.YumRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating yum repo file: %v", err))
			}
		}
		if err := yumRepositories(yumRepos, config.YumRepoFilePath()); err != nil {
			logger.Errorf("Error writing yum repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing yum repo file: %v", err))
		}
		if err := yumChanges(yumInstallPkgs, yumRemovePkgs, yumUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing yum changes: %v", err))
		}
	}

	if packages.ZypperExists {
		if _, err := os.Stat(config.ZypperRepoFilePath()); os.IsNotExist(err) {
			logger.Debugf("Repo file does not exist, will create one...")
			if err := os.MkdirAll(filepath.Dir(config.ZypperRepoFilePath()), 07550); err != nil {
				logger.Errorf("Error creating repo file: %v", err)
				errs = append(errs, fmt.Sprintf("error creating zypper repo file: %v", err))
			}
		}
		if err := zypperRepositories(zypperRepos, config.ZypperRepoFilePath()); err != nil {
			logger.Errorf("Error writing zypper repo file: %v", err)
			errs = append(errs, fmt.Sprintf("error writing zypper repo file: %v", err))
		}
		if err := zypperChanges(zypperInstallPkgs, zypperRemovePkgs, zypperUpdatePkgs); err != nil {
			errs = append(errs, fmt.Sprintf("error performing zypper changes: %v", err))
		}
	}

	if errs == nil {
		return nil
	}
	return errors.New(strings.Join(errs, ",\n"))
}

func checksum(r io.Reader) hash.Hash {
	hash := sha256.New()
	io.Copy(hash, r)
	return hash
}

func writeIfChanged(content []byte, path string) error {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return err
	}

	reader := bytes.NewReader(content)
	h1 := checksum(reader)
	h2 := checksum(file)
	if bytes.Equal(h1.Sum(nil), h2.Sum(nil)) {
		file.Close()
		return nil
	}

	logger.Infof("Writing repo file %s with updated contents", path)
	if err := file.Truncate(0); err != nil {
		file.Close()
		return err
	}
	if _, err := file.WriteAt(content, 0); err != nil {
		file.Close()
		return err
	}

	return file.Close()
}
<|endoftext|>


<commit_before>package secrets

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"strings"

	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
)

// Client represents interaction with AWS S3
type Client interface {
	Bucket() (string)
	Get(key string) ([]byte, error)
	BucketExists() (bool, error)
}

// Agent represents interaction with an ssh-agent process
type Agent interface {
	Run() (bool, error)
	Add(key []byte) error
	Pid() int
	Stdout() io.Reader
}

// Config holds all the parameters for Run()
type Config struct {
	// Repo from BUILDKITE_REPO
	Repo string

	// Bucket from BUILDKITE_PLUGIN_S3_SECRETS_BUCKET
	Bucket string

	// Prefix within bucket, from BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX,
	// defaulting to the value of BUILDKITE_PIPELINE_SLUG
	Prefix string

	// Client for S3
	Client Client

	// Logger is expected to output to stderr
	Logger *log.Logger

	// SSHAgent represents an ssh-agent process
	SSHAgent Agent

	// EnvSink has the contents of environment files written to it
	EnvSink io.Writer

	// GitCredentialHelper is the path to git-credential-s3-secrets
	GitCredentialHelper string
}

// Run is the programmatic (as opposed to CLI) entrypoint to all
// functionality; secrets are downloaded from S3, and loaded into ssh-agent
// etc.
func Run(conf Config) error {
	bucket := conf.Client.Bucket()
	log := conf.Logger

	log.Printf("~~~ Downloading secrets from :s3: %s", bucket)

	if ok, err := conf.Client.BucketExists(); !ok {
		if err != nil {
			log.Printf("+++ :warning: Bucket %q not found: %v", bucket, err)
		} else {
			log.Printf("+++ :warning: Bucket %q doesn't exist", bucket)
		}
		return fmt.Errorf("S3 bucket %q not found", bucket)
	}

	resultsSSH := make(chan getResult)
	getSSHKeys(conf, resultsSSH)

	resultsEnv := make(chan getResult)
	getEnvs(conf, resultsEnv)

	resultsGit := make(chan getResult)
	getGitCredentials(conf, resultsGit)

	if err := handleSSHKeys(conf, resultsSSH); err != nil {
		return err
	}
	if err := handleEnvs(conf, resultsEnv); err != nil {
		return err
	}
	if err := handleGitCredentials(conf, resultsGit); err != nil {
		return err
	}
	return nil
}

func getSSHKeys(conf Config, results chan<- getResult) {
	keys := []string{
		conf.Prefix + "/private_ssh_key",
		conf.Prefix + "/id_rsa_github",
		"private_ssh_key",
		"id_rsa_github",
	}
	conf.Logger.Printf("Checking S3 for SSH keys:")
	for _, k := range keys {
		conf.Logger.Printf("- %s", k)
	}
	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
}

func getEnvs(conf Config, results chan<- getResult) {
	keys := []string{
		"env",
		"environment",
		conf.Prefix + "/env",
		conf.Prefix + "/environment",
	}
	conf.Logger.Printf("Checking S3 for environment files:")
	for _, k := range keys {
		conf.Logger.Printf("- %s", k)
	}
	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
}

func getGitCredentials(conf Config, results chan<- getResult) {
	keys := []string{
		"git-credentials",
		conf.Prefix + "/git-credentials",
	}
	conf.Logger.Printf("Checking S3 for git credentials:")
	for _, k := range keys {
		conf.Logger.Printf("- %s", k)
	}
	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
}

func handleSSHKeys(conf Config, results <-chan getResult) error {
	log := conf.Logger
	keyFound := false
	for r := range results {
		if r.err != nil {
			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
				log.Printf("+++ :warning: Failed to download ssh-key %s/%s: %v", r.bucket, r.key, r.err)
			}
			continue
		}
		if started, err := conf.SSHAgent.Run(); err != nil {
			return err
		} else if started {
			log.Printf("Started ephemeral ssh-agent (pid %d)", conf.SSHAgent.Pid())
		}
		log.Printf(
			"Loading %s/%s (%d bytes) into ssh-agent (pid %d)",
			r.bucket, r.key, len(r.data), conf.SSHAgent.Pid(),
		)
		if err := conf.SSHAgent.Add(r.data); err != nil {
			return fmt.Errorf("ssh-agent add: %w", err)
		}
		keyFound = true
	}
	if !keyFound && strings.HasPrefix(conf.Repo, "git@") {
		log.Printf("+++ :warning: Failed to find an SSH key in secret bucket")
		log.Printf(
			"The repository %q appears to use SSH for transport, but the elastic-ci-stack-s3-secrets-hooks plugin did not find any SSH keys in the %q S3 bucket.",
			conf.Repo, conf.Bucket,
		)
		log.Printf("See https://github.com/buildkite/elastic-ci-stack-for-aws#build-secrets for more information.")
	}
	if _, err := io.Copy(conf.EnvSink, conf.SSHAgent.Stdout()); err != nil {
		return fmt.Errorf("copying ssh-agent env: %w", err)
	}
	return nil
}

func handleEnvs(conf Config, results <-chan getResult) error {
	log := conf.Logger
	for r := range results {
		if r.err != nil {
			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
				log.Printf("+++ :warning: Failed to download env from %s/%s: %v", r.bucket, r.key, r.err)
			}
			continue
		}
		data := r.data
		if data[len(data)-1] != '\n' {
			data = append(data, '\n')
		}
		log.Printf("Loading %s/%s (%d bytes) of env", r.bucket, r.key, len(r.data))
		if _, err := bytes.NewReader(data).WriteTo(conf.EnvSink); err != nil {
			return fmt.Errorf("copying env: %w", err)
		}
	}
	return nil
}

func handleGitCredentials(conf Config, results <-chan getResult) error {
	log := conf.Logger
	var helpers []string
	for r := range results {
		if r.err != nil {
			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
				log.Printf("+++ :warning: Failed to check %s/%s: %v", r.bucket, r.key, r.err)
			}
			continue
		}
		log.Printf("Adding git-credentials in %s/%s as a credential helper", r.bucket, r.key)
		helpers = append(helpers, fmt.Sprintf(
			"'credential.helper=%s %s %s'",
			conf.GitCredentialHelper, r.bucket, r.key,
		))
	}
	if len(helpers) == 0 {
		return nil
	}
	env := "GIT_CONFIG_PARAMETERS=\"" + strings.Join(helpers, " ") + "\"\n"
	if _, err := io.WriteString(conf.EnvSink, env); err != nil {
		return fmt.Errorf("writing GIT_CONFIG_PARAMETERS env: %w", err)
	}
	return nil
}

type getResult struct {
	bucket string
	key    string
	data   []byte
	err    error
}

// GetAll fetches keys from an S3 bucket concurrently.
// Concurrency is unbounded; intended for use with a handful of keys.
// Results are sent to a channel in the originally requested order.
// This is done by creating a chain of channels between each goroutine.
// The results channel is passed through that chain.
func GetAll(c Client, bucket string, keys []string, results chan<- getResult) {
	// first link in chain; will pass results channel into the first goroutine
	link := make(chan chan<- getResult, 1)
	link <- results
	close(link)

	for _, k := range keys {
		// next link in chain; will pass results channel to the next goroutine.
		nextLink := make(chan chan<- getResult)

		// goroutine immediately fetches from S3, then waits for its turn to send
		// to the results channel; concurrent fetch, ordered results.
		go func(k string, link <-chan chan<- getResult, nextLink chan<- chan<- getResult) {
			data, err := c.Get(k)
			results := <-link // wait for results channel from previous goroutine
			results <- getResult{bucket: bucket, key: k, data: data, err: err}
			nextLink <- results // send results channel to the next goroutine
			close(nextLink)
		}(k, link, nextLink)

		link = nextLink // our `nextLink` becomes `link` for the next goroutine.
	}
	close(<-link) // wait for final goroutine, close results channel
}
<commit_msg>Check env data length before indexing<commit_after>package secrets

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"strings"

	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
)

// Client represents interaction with AWS S3
type Client interface {
	Bucket() (string)
	Get(key string) ([]byte, error)
	BucketExists() (bool, error)
}

// Agent represents interaction with an ssh-agent process
type Agent interface {
	Run() (bool, error)
	Add(key []byte) error
	Pid() int
	Stdout() io.Reader
}

// Config holds all the parameters for Run()
type Config struct {
	// Repo from BUILDKITE_REPO
	Repo string

	// Bucket from BUILDKITE_PLUGIN_S3_SECRETS_BUCKET
	Bucket string

	// Prefix within bucket, from BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX,
	// defaulting to the value of BUILDKITE_PIPELINE_SLUG
	Prefix string

	// Client for S3
	Client Client

	// Logger is expected to output to stderr
	Logger *log.Logger

	// SSHAgent represents an ssh-agent process
	SSHAgent Agent

	// EnvSink has the contents of environment files written to it
	EnvSink io.Writer

	// GitCredentialHelper is the path to git-credential-s3-secrets
	GitCredentialHelper string
}

// Run is the programmatic (as opposed to CLI) entrypoint to all
// functionality; secrets are downloaded from S3, and loaded into ssh-agent
// etc.
func Run(conf Config) error {
	bucket := conf.Client.Bucket()
	log := conf.Logger

	log.Printf("~~~ Downloading secrets from :s3: %s", bucket)

	if ok, err := conf.Client.BucketExists(); !ok {
		if err != nil {
			log.Printf("+++ :warning: Bucket %q not found: %v", bucket, err)
		} else {
			log.Printf("+++ :warning: Bucket %q doesn't exist", bucket)
		}
		return fmt.Errorf("S3 bucket %q not found", bucket)
	}

	resultsSSH := make(chan getResult)
	getSSHKeys(conf, resultsSSH)

	resultsEnv := make(chan getResult)
	getEnvs(conf, resultsEnv)

	resultsGit := make(chan getResult)
	getGitCredentials(conf, resultsGit)

	if err := handleSSHKeys(conf, resultsSSH); err != nil {
		return err
	}
	if err := handleEnvs(conf, resultsEnv); err != nil {
		return err
	}
	if err := handleGitCredentials(conf, resultsGit); err != nil {
		return err
	}
	return nil
}

func getSSHKeys(conf Config, results chan<- getResult) {
	keys := []string{
		conf.Prefix + "/private_ssh_key",
		conf.Prefix + "/id_rsa_github",
		"private_ssh_key",
		"id_rsa_github",
	}
	conf.Logger.Printf("Checking S3 for SSH keys:")
	for _, k := range keys {
		conf.Logger.Printf("- %s", k)
	}
	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
}

func getEnvs(conf Config, results chan<- getResult) {
	keys := []string{
		"env",
		"environment",
		conf.Prefix + "/env",
		conf.Prefix + "/environment",
	}
	conf.Logger.Printf("Checking S3 for environment files:")
	for _, k := range keys {
		conf.Logger.Printf("- %s", k)
	}
	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
}

func getGitCredentials(conf Config, results chan<- getResult) {
	keys := []string{
		"git-credentials",
		conf.Prefix + "/git-credentials",
	}
	conf.Logger.Printf("Checking S3 for git credentials:")
	for _, k := range keys {
		conf.Logger.Printf("- %s", k)
	}
	go GetAll(conf.Client, conf.Client.Bucket(), keys, results)
}

func handleSSHKeys(conf Config, results <-chan getResult) error {
	log := conf.Logger
	keyFound := false
	for r := range results {
		if r.err != nil {
			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
				log.Printf("+++ :warning: Failed to download ssh-key %s/%s: %v", r.bucket, r.key, r.err)
			}
			continue
		}
		if started, err := conf.SSHAgent.Run(); err != nil {
			return err
		} else if started {
			log.Printf("Started ephemeral ssh-agent (pid %d)", conf.SSHAgent.Pid())
		}
		log.Printf(
			"Loading %s/%s (%d bytes) into ssh-agent (pid %d)",
			r.bucket, r.key, len(r.data), conf.SSHAgent.Pid(),
		)
		if err := conf.SSHAgent.Add(r.data); err != nil {
			return fmt.Errorf("ssh-agent add: %w", err)
		}
		keyFound = true
	}
	if !keyFound && strings.HasPrefix(conf.Repo, "git@") {
		log.Printf("+++ :warning: Failed to find an SSH key in secret bucket")
		log.Printf(
			"The repository %q appears to use SSH for transport, but the elastic-ci-stack-s3-secrets-hooks plugin did not find any SSH keys in the %q S3 bucket.",
			conf.Repo, conf.Bucket,
		)
		log.Printf("See https://github.com/buildkite/elastic-ci-stack-for-aws#build-secrets for more information.")
	}
	if _, err := io.Copy(conf.EnvSink, conf.SSHAgent.Stdout()); err != nil {
		return fmt.Errorf("copying ssh-agent env: %w", err)
	}
	return nil
}

func handleEnvs(conf Config, results <-chan getResult) error {
	log := conf.Logger
	for r := range results {
		if r.err != nil {
			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
				log.Printf("+++ :warning: Failed to download env from %s/%s: %v", r.bucket, r.key, r.err)
			}
			continue
		}
		data := r.data

		if len(data) > 0 {
			if data[len(data)-1] != '\n' {
				data = append(data, '\n')
			}
			log.Printf("Loading %s/%s (%d bytes) of env", r.bucket, r.key, len(r.data))
			if _, err := bytes.NewReader(data).WriteTo(conf.EnvSink); err != nil {
				return fmt.Errorf("copying env: %w", err)
			}
		}
	}
	return nil
}

func handleGitCredentials(conf Config, results <-chan getResult) error {
	log := conf.Logger
	var helpers []string
	for r := range results {
		if r.err != nil {
			if r.err != sentinel.ErrNotFound && r.err != sentinel.ErrForbidden {
				log.Printf("+++ :warning: Failed to check %s/%s: %v", r.bucket, r.key, r.err)
			}
			continue
		}
		log.Printf("Adding git-credentials in %s/%s as a credential helper", r.bucket, r.key)
		helpers = append(helpers, fmt.Sprintf(
			"'credential.helper=%s %s %s'",
			conf.GitCredentialHelper, r.bucket, r.key,
		))
	}
	if len(helpers) == 0 {
		return nil
	}
	env := "GIT_CONFIG_PARAMETERS=\"" + strings.Join(helpers, " ") + "\"\n"
	if _, err := io.WriteString(conf.EnvSink, env); err != nil {
		return fmt.Errorf("writing GIT_CONFIG_PARAMETERS env: %w", err)
	}
	return nil
}

type getResult struct {
	bucket string
	key    string
	data   []byte
	err    error
}

// GetAll fetches keys from an S3 bucket concurrently.
// Concurrency is unbounded; intended for use with a handful of keys.
// Results are sent to a channel in the originally requested order.
// This is done by creating a chain of channels between each goroutine.
// The results channel is passed through that chain.
func GetAll(c Client, bucket string, keys []string, results chan<- getResult) {
	// first link in chain; will pass results channel into the first goroutine
	link := make(chan chan<- getResult, 1)
	link <- results
	close(link)

	for _, k := range keys {
		// next link in chain; will pass results channel to the next goroutine.
		nextLink := make(chan chan<- getResult)

		// goroutine immediately fetches from S3, then waits for its turn to send
		// to the results channel; concurrent fetch, ordered results.
		go func(k string, link <-chan chan<- getResult, nextLink chan<- chan<- getResult) {
			data, err := c.Get(k)
			results := <-link // wait for results channel from previous goroutine
			results <- getResult{bucket: bucket, key: k, data: data, err: err}
			nextLink <- results // send results channel to the next goroutine
			close(nextLink)
		}(k, link, nextLink)

		link = nextLink // our `nextLink` becomes `link` for the next goroutine.
	}
	close(<-link) // wait for final goroutine, close results channel
}
<|endoftext|>


<commit_before>package cache

import (
	"time"

	"github.com/coredns/coredns/plugin/cache/freq"
	"github.com/miekg/dns"
)

type item struct {
	Rcode              int
	AuthenticatedData  bool
	RecursionAvailable bool
	Answer             []dns.RR
	Ns                 []dns.RR
	Extra              []dns.RR

	origTTL uint32
	stored  time.Time

	*freq.Freq
}

func newItem(m *dns.Msg, now time.Time, d time.Duration) *item {
	i := new(item)
	i.Rcode = m.Rcode
	i.AuthenticatedData = m.AuthenticatedData
	i.RecursionAvailable = m.RecursionAvailable
	i.Answer = m.Answer
	i.Ns = m.Ns
	i.Extra = make([]dns.RR, len(m.Extra))
	// Don't copy OPT records as these are hop-by-hop.
	j := 0
	for _, e := range m.Extra {
		if e.Header().Rrtype == dns.TypeOPT {
			continue
		}
		i.Extra[j] = e
		j++
	}
	i.Extra = i.Extra[:j]

	i.origTTL = uint32(d.Seconds())
	i.stored = now.UTC()

	i.Freq = new(freq.Freq)

	return i
}

// toMsg turns i into a message, it tailors the reply to m.
// The Authoritative bit is always set to 0, because the answer is from the cache.
func (i *item) toMsg(m *dns.Msg, now time.Time) *dns.Msg {
	m1 := new(dns.Msg)
	m1.SetReply(m)

	// Set this to true as some DNS clients discard the *entire* packet when it's non-authoritative.
	// This is probably not according to spec, but the bit itself is not super useful as this point, so
	// just set it to true.
	m1.Authoritative = true
	m1.AuthenticatedData = i.AuthenticatedData
	m1.RecursionAvailable = i.RecursionAvailable
	m1.Rcode = i.Rcode

	m1.Answer = make([]dns.RR, len(i.Answer))
	m1.Ns = make([]dns.RR, len(i.Ns))
	m1.Extra = make([]dns.RR, len(i.Extra))

	ttl := uint32(i.ttl(now))
	for j, r := range i.Answer {
		m1.Answer[j] = dns.Copy(r)
		m1.Answer[j].Header().Ttl = ttl
	}
	for j, r := range i.Ns {
		m1.Ns[j] = dns.Copy(r)
		m1.Ns[j].Header().Ttl = ttl
	}
	// newItem skips OPT records, so we can just use i.Extra as is.
	for j, r := range i.Extra {
		m1.Extra[j] = dns.Copy(r)
		m1.Extra[j].Header().Ttl = ttl
	}
	return m1
}

func (i *item) ttl(now time.Time) int {
	ttl := int(i.origTTL) - int(now.UTC().Sub(i.stored).Seconds())
	return ttl
}
<commit_msg>plugin/cache: update comment to conform to the implementation (#3573)<commit_after>package cache

import (
	"time"

	"github.com/coredns/coredns/plugin/cache/freq"
	"github.com/miekg/dns"
)

type item struct {
	Rcode              int
	AuthenticatedData  bool
	RecursionAvailable bool
	Answer             []dns.RR
	Ns                 []dns.RR
	Extra              []dns.RR

	origTTL uint32
	stored  time.Time

	*freq.Freq
}

func newItem(m *dns.Msg, now time.Time, d time.Duration) *item {
	i := new(item)
	i.Rcode = m.Rcode
	i.AuthenticatedData = m.AuthenticatedData
	i.RecursionAvailable = m.RecursionAvailable
	i.Answer = m.Answer
	i.Ns = m.Ns
	i.Extra = make([]dns.RR, len(m.Extra))
	// Don't copy OPT records as these are hop-by-hop.
	j := 0
	for _, e := range m.Extra {
		if e.Header().Rrtype == dns.TypeOPT {
			continue
		}
		i.Extra[j] = e
		j++
	}
	i.Extra = i.Extra[:j]

	i.origTTL = uint32(d.Seconds())
	i.stored = now.UTC()

	i.Freq = new(freq.Freq)

	return i
}

// toMsg turns i into a message, it tailors the reply to m.
// The Authoritative bit should be set to 0, but some client stub resolver implementations, most notably,
// on some legacy systems(e.g. ubuntu 14.04 with glib version 2.20), low-level glibc function `getaddrinfo`
// useb by Python/Ruby/etc.. will discard answers that do not have this bit set.
// So we're forced to always set this to 1; regardless if the answer came from the cache or not.
// On newer systems(e.g. ubuntu 16.04 with glib version 2.23), this issue is resolved.
// So we may set this bit back to 0 in the future ?
func (i *item) toMsg(m *dns.Msg, now time.Time) *dns.Msg {
	m1 := new(dns.Msg)
	m1.SetReply(m)

	// Set this to true as some DNS clients discard the *entire* packet when it's non-authoritative.
	// This is probably not according to spec, but the bit itself is not super useful as this point, so
	// just set it to true.
	m1.Authoritative = true
	m1.AuthenticatedData = i.AuthenticatedData
	m1.RecursionAvailable = i.RecursionAvailable
	m1.Rcode = i.Rcode

	m1.Answer = make([]dns.RR, len(i.Answer))
	m1.Ns = make([]dns.RR, len(i.Ns))
	m1.Extra = make([]dns.RR, len(i.Extra))

	ttl := uint32(i.ttl(now))
	for j, r := range i.Answer {
		m1.Answer[j] = dns.Copy(r)
		m1.Answer[j].Header().Ttl = ttl
	}
	for j, r := range i.Ns {
		m1.Ns[j] = dns.Copy(r)
		m1.Ns[j].Header().Ttl = ttl
	}
	// newItem skips OPT records, so we can just use i.Extra as is.
	for j, r := range i.Extra {
		m1.Extra[j] = dns.Copy(r)
		m1.Extra[j].Header().Ttl = ttl
	}
	return m1
}

func (i *item) ttl(now time.Time) int {
	ttl := int(i.origTTL) - int(now.UTC().Sub(i.stored).Seconds())
	return ttl
}
<|endoftext|>