<commit_before>package distributor

import (
	"errors"
	"sync"
	"time"

	"github.com/coreos/torus"
	"golang.org/x/net/context"
)

var (
	ErrNoPeersBlock = errors.New("distributor: no peers available for a block")
)

func (d *Distributor) GetBlock(ctx context.Context, i torus.BlockRef) ([]byte, error) {
	d.mut.RLock()
	defer d.mut.RUnlock()
	promDistBlockRequests.Inc()
	bcache, ok := d.readCache.Get(string(i.ToBytes()))
	if ok {
		promDistBlockCacheHits.Inc()
		return bcache.([]byte), nil
	}
	peers, err := d.ring.GetPeers(i)
	if err != nil {
		promDistBlockFailures.Inc()
		return nil, err
	}
	if len(peers.Peers) == 0 {
		promDistBlockFailures.Inc()
		return nil, ErrNoPeersBlock
	}
	writeLevel := d.getWriteFromServer()
	for _, p := range peers.Peers[:peers.Replication] {
		if p == d.UUID() || writeLevel == torus.WriteLocal {
			b, err := d.blocks.GetBlock(ctx, i)
			if err == nil {
				promDistBlockLocalHits.Inc()
				return b, nil
			}
			promDistBlockLocalFailures.Inc()
			break
		}
	}
	var blk []byte
	readLevel := d.getReadFromServer()
	switch readLevel {
	case torus.ReadBlock:
		blk, err = d.readWithBackoff(ctx, i, peers)
	case torus.ReadSequential:
		blk, err = d.readSequential(ctx, i, peers, clientTimeout)
	case torus.ReadSpread:
		blk, err = d.readSpread(ctx, i, peers)
	default:
		panic("unhandled read level")
	}
	if err != nil {
		// We completely failed!
		promDistBlockFailures.Inc()
		clog.Errorf("no peers for block %s: %v", i, err)
	}
	return blk, err
}

func (d *Distributor) readWithBackoff(ctx context.Context, ref torus.BlockRef, peers torus.PeerPermutation) ([]byte, error) {
	for i := uint(0); i < 10; i++ {
		timeout := clientTimeout * (1 << i)
		blk, err := d.readSequential(ctx, ref, peers, timeout)
		if err == nil {
			return blk, err
		}
	}
	return nil, ErrNoPeersBlock
}

func (d *Distributor) readSequential(ctx context.Context, i torus.BlockRef, peers torus.PeerPermutation, timeout time.Duration) ([]byte, error) {
	for _, p := range peers.Peers {
		// If it's local, just try to get it.
		if p == d.UUID() {
			b, err := d.blocks.GetBlock(ctx, i)
			if err == nil {
				promDistBlockLocalHits.Inc()
				return b, nil
			}
			promDistBlockLocalFailures.Inc()
			clog.Debug("failed local peer (again)")
			continue
		}
		// Fetch block from remote. First pass through peers
		// with a timeout, then through the list without.
		getctx, cancel := context.WithTimeout(ctx, timeout)
		blk, err := d.readFromPeer(getctx, i, p)
		cancel()

		if err == nil {
			return blk, nil
		}

		// If this peer didn't have it, continue
		if err == torus.ErrBlockUnavailable || err == torus.ErrNoPeer {
			clog.Warningf("block %s from %s failed, trying next peer", i, p)
			promDistBlockPeerFailures.WithLabelValues(p).Inc()
			continue
		}

		// If there was a more significant error, fail hard.
		promDistBlockFailures.Inc()
		clog.Errorf("failed remote peer %s %s %#v", p, err, err)
		return nil, err
	}
	return nil, ErrNoPeersBlock
}

func (d *Distributor) readSpread(ctx context.Context, i torus.BlockRef, peers torus.PeerPermutation) ([]byte, error) {
	resch := make(chan []byte)
	errch := make(chan error, peers.Replication)
	var once sync.Once
	count := 0
	for _, p := range peers.Peers[:peers.Replication] {
		if p == d.UUID() {
			continue
		}
		go func(peer string) {
			getctx, cancel := context.WithTimeout(ctx, clientTimeout)
			blk, err := d.readFromPeer(getctx, i, peer)
			cancel()
			if err == nil {
				once.Do(func() {
					resch <- blk
					close(resch)
				})
				return
			}
			errch <- err
		}(p)
		count++
	}

	for {
		select {
		case blk := <-resch:
			return blk, nil
		case err := <-errch:
			clog.Debugf("spread-read: %s, %s", err, i)
			count--
			if count == 0 {
				return nil, ErrNoPeersBlock
			}
		}
	}
}

func (d *Distributor) readFromPeer(ctx context.Context, i torus.BlockRef, peer string) ([]byte, error) {
	blk, err := d.client.GetBlock(ctx, peer, i)
	// If we're successful, store that.
	if err == nil {
		d.readCache.Put(string(i.ToBytes()), blk)
		promDistBlockPeerHits.WithLabelValues(peer).Inc()
		return blk, nil
	}
	return nil, err
}

func (d *Distributor) getWriteFromServer() torus.WriteLevel {
	return d.srv.Cfg.WriteLevel
}

func (d *Distributor) getReadFromServer() torus.ReadLevel {
	return d.srv.Cfg.ReadLevel
}

func (d *Distributor) WriteBlock(ctx context.Context, i torus.BlockRef, data []byte) error {
	d.mut.RLock()
	defer d.mut.RUnlock()
	peers, err := d.ring.GetPeers(i)
	if err != nil {
		return err
	}
	if len(peers.Peers) == 0 {
		return torus.ErrOutOfSpace
	}
	d.readCache.Put(string(i.ToBytes()), data)
	switch d.getWriteFromServer() {
	case torus.WriteLocal:
		err = d.blocks.WriteBlock(ctx, i, data)
		if err == nil {
			return nil
		}
		clog.Tracef("Couldn't write locally; writing to cluster")
		fallthrough
	case torus.WriteOne:
		for _, p := range peers.Peers[:peers.Replication] {
			// If we're one of the desired peers, we count, write here first.
			if p == d.UUID() {
				err = d.blocks.WriteBlock(ctx, i, data)
				if err != nil {
					clog.Noticef("WriteOne error, local: %s", err)
				} else {
					return nil
				}
			}
		}
		for _, p := range peers.Peers {
			err = d.client.PutBlock(ctx, p, i, data)
			if err == nil {
				return nil
			}
			clog.Noticef("WriteOne error, remote: %s", err)
		}
		return torus.ErrNoPeer
	case torus.WriteAll:
		toWrite := peers.Replication
		for _, p := range peers.Peers {
			var err error
			if p == d.UUID() {
				err = d.blocks.WriteBlock(ctx, i, data)
			} else {
				err = d.client.PutBlock(ctx, p, i, data)
			}
			if err != nil {
				clog.Noticef("error WriteAll to peer %s: %s", p, err)
			} else {
				toWrite--
			}
			if toWrite == 0 {
				return nil
			}
		}
		if toWrite == peers.Replication {
			clog.Noticef("error WriteAll to all peers")
			return torus.ErrNoPeer
		}
		clog.Warningf("only wrote block to %d/%d peers", (peers.Replication - toWrite), peers.Replication)
	}
	return nil
}

func (d *Distributor) WriteBuf(ctx context.Context, i torus.BlockRef) ([]byte, error) {
	return d.blocks.WriteBuf(ctx, i)
}

func (d *Distributor) HasBlock(ctx context.Context, i torus.BlockRef) (bool, error) {
	return false, errors.New("unimplemented -- finding if a block exists cluster-wide")
}

func (d *Distributor) DeleteBlock(ctx context.Context, i torus.BlockRef) error {
	return d.blocks.DeleteBlock(ctx, i)
}

func (d *Distributor) NumBlocks() uint64 {
	d.mut.RLock()
	defer d.mut.RUnlock()
	return d.blocks.NumBlocks()
}

func (d *Distributor) UsedBlocks() uint64 {
	d.mut.RLock()
	defer d.mut.RUnlock()
	return d.blocks.UsedBlocks()
}

func (d *Distributor) BlockIterator() torus.BlockIterator {
	return d.blocks.BlockIterator()
}

func (d *Distributor) Flush() error {
	return d.blocks.Flush()
}

func (d *Distributor) Kind() string { return "distributor" }

func (d *Distributor) BlockSize() uint64 {
	return d.blocks.BlockSize()
}
<commit_msg>distributor: return ErrNoPeersBlock when no peer found in write block<commit_after>package distributor

import (
	"errors"
	"sync"
	"time"

	"github.com/coreos/torus"
	"golang.org/x/net/context"
)

var (
	ErrNoPeersBlock = errors.New("distributor: no peers available for a block")
)

func (d *Distributor) GetBlock(ctx context.Context, i torus.BlockRef) ([]byte, error) {
	d.mut.RLock()
	defer d.mut.RUnlock()
	promDistBlockRequests.Inc()
	bcache, ok := d.readCache.Get(string(i.ToBytes()))
	if ok {
		promDistBlockCacheHits.Inc()
		return bcache.([]byte), nil
	}
	peers, err := d.ring.GetPeers(i)
	if err != nil {
		promDistBlockFailures.Inc()
		return nil, err
	}
	if len(peers.Peers) == 0 {
		promDistBlockFailures.Inc()
		return nil, ErrNoPeersBlock
	}
	writeLevel := d.getWriteFromServer()
	for _, p := range peers.Peers[:peers.Replication] {
		if p == d.UUID() || writeLevel == torus.WriteLocal {
			b, err := d.blocks.GetBlock(ctx, i)
			if err == nil {
				promDistBlockLocalHits.Inc()
				return b, nil
			}
			promDistBlockLocalFailures.Inc()
			break
		}
	}
	var blk []byte
	readLevel := d.getReadFromServer()
	switch readLevel {
	case torus.ReadBlock:
		blk, err = d.readWithBackoff(ctx, i, peers)
	case torus.ReadSequential:
		blk, err = d.readSequential(ctx, i, peers, clientTimeout)
	case torus.ReadSpread:
		blk, err = d.readSpread(ctx, i, peers)
	default:
		panic("unhandled read level")
	}
	if err != nil {
		// We completely failed!
		promDistBlockFailures.Inc()
		clog.Errorf("no peers for block %s: %v", i, err)
	}
	return blk, err
}

func (d *Distributor) readWithBackoff(ctx context.Context, ref torus.BlockRef, peers torus.PeerPermutation) ([]byte, error) {
	for i := uint(0); i < 10; i++ {
		timeout := clientTimeout * (1 << i)
		blk, err := d.readSequential(ctx, ref, peers, timeout)
		if err == nil {
			return blk, err
		}
	}
	return nil, ErrNoPeersBlock
}

func (d *Distributor) readSequential(ctx context.Context, i torus.BlockRef, peers torus.PeerPermutation, timeout time.Duration) ([]byte, error) {
	for _, p := range peers.Peers {
		// If it's local, just try to get it.
		if p == d.UUID() {
			b, err := d.blocks.GetBlock(ctx, i)
			if err == nil {
				promDistBlockLocalHits.Inc()
				return b, nil
			}
			promDistBlockLocalFailures.Inc()
			clog.Debug("failed local peer (again)")
			continue
		}
		// Fetch block from remote. First pass through peers
		// with a timeout, then through the list without.
		getctx, cancel := context.WithTimeout(ctx, timeout)
		blk, err := d.readFromPeer(getctx, i, p)
		cancel()

		if err == nil {
			return blk, nil
		}

		// If this peer didn't have it, continue
		if err == torus.ErrBlockUnavailable || err == torus.ErrNoPeer {
			clog.Warningf("block %s from %s failed, trying next peer", i, p)
			promDistBlockPeerFailures.WithLabelValues(p).Inc()
			continue
		}

		// If there was a more significant error, fail hard.
		promDistBlockFailures.Inc()
		clog.Errorf("failed remote peer %s %s %#v", p, err, err)
		return nil, err
	}
	return nil, ErrNoPeersBlock
}

func (d *Distributor) readSpread(ctx context.Context, i torus.BlockRef, peers torus.PeerPermutation) ([]byte, error) {
	resch := make(chan []byte)
	errch := make(chan error, peers.Replication)
	var once sync.Once
	count := 0
	for _, p := range peers.Peers[:peers.Replication] {
		if p == d.UUID() {
			continue
		}
		go func(peer string) {
			getctx, cancel := context.WithTimeout(ctx, clientTimeout)
			blk, err := d.readFromPeer(getctx, i, peer)
			cancel()
			if err == nil {
				once.Do(func() {
					resch <- blk
					close(resch)
				})
				return
			}
			errch <- err
		}(p)
		count++
	}

	for {
		select {
		case blk := <-resch:
			return blk, nil
		case err := <-errch:
			clog.Debugf("spread-read: %s, %s", err, i)
			count--
			if count == 0 {
				return nil, ErrNoPeersBlock
			}
		}
	}
}

func (d *Distributor) readFromPeer(ctx context.Context, i torus.BlockRef, peer string) ([]byte, error) {
	blk, err := d.client.GetBlock(ctx, peer, i)
	// If we're successful, store that.
	if err == nil {
		d.readCache.Put(string(i.ToBytes()), blk)
		promDistBlockPeerHits.WithLabelValues(peer).Inc()
		return blk, nil
	}
	return nil, err
}

func (d *Distributor) getWriteFromServer() torus.WriteLevel {
	return d.srv.Cfg.WriteLevel
}

func (d *Distributor) getReadFromServer() torus.ReadLevel {
	return d.srv.Cfg.ReadLevel
}

func (d *Distributor) WriteBlock(ctx context.Context, i torus.BlockRef, data []byte) error {
	d.mut.RLock()
	defer d.mut.RUnlock()
	peers, err := d.ring.GetPeers(i)
	if err != nil {
		return err
	}
	if len(peers.Peers) == 0 {
		return ErrNoPeersBlock
	}
	d.readCache.Put(string(i.ToBytes()), data)
	switch d.getWriteFromServer() {
	case torus.WriteLocal:
		err = d.blocks.WriteBlock(ctx, i, data)
		if err == nil {
			return nil
		}
		clog.Tracef("Couldn't write locally; writing to cluster")
		fallthrough
	case torus.WriteOne:
		for _, p := range peers.Peers[:peers.Replication] {
			// If we're one of the desired peers, we count, write here first.
			if p == d.UUID() {
				err = d.blocks.WriteBlock(ctx, i, data)
				if err != nil {
					clog.Noticef("WriteOne error, local: %s", err)
				} else {
					return nil
				}
			}
		}
		for _, p := range peers.Peers {
			err = d.client.PutBlock(ctx, p, i, data)
			if err == nil {
				return nil
			}
			clog.Noticef("WriteOne error, remote: %s", err)
		}
		return torus.ErrNoPeer
	case torus.WriteAll:
		toWrite := peers.Replication
		for _, p := range peers.Peers {
			var err error
			if p == d.UUID() {
				err = d.blocks.WriteBlock(ctx, i, data)
			} else {
				err = d.client.PutBlock(ctx, p, i, data)
			}
			if err != nil {
				clog.Noticef("error WriteAll to peer %s: %s", p, err)
			} else {
				toWrite--
			}
			if toWrite == 0 {
				return nil
			}
		}
		if toWrite == peers.Replication {
			clog.Noticef("error WriteAll to all peers")
			return torus.ErrNoPeer
		}
		clog.Warningf("only wrote block to %d/%d peers", (peers.Replication - toWrite), peers.Replication)
	}
	return nil
}

func (d *Distributor) WriteBuf(ctx context.Context, i torus.BlockRef) ([]byte, error) {
	return d.blocks.WriteBuf(ctx, i)
}

func (d *Distributor) HasBlock(ctx context.Context, i torus.BlockRef) (bool, error) {
	return false, errors.New("unimplemented -- finding if a block exists cluster-wide")
}

func (d *Distributor) DeleteBlock(ctx context.Context, i torus.BlockRef) error {
	return d.blocks.DeleteBlock(ctx, i)
}

func (d *Distributor) NumBlocks() uint64 {
	d.mut.RLock()
	defer d.mut.RUnlock()
	return d.blocks.NumBlocks()
}

func (d *Distributor) UsedBlocks() uint64 {
	d.mut.RLock()
	defer d.mut.RUnlock()
	return d.blocks.UsedBlocks()
}

func (d *Distributor) BlockIterator() torus.BlockIterator {
	return d.blocks.BlockIterator()
}

func (d *Distributor) Flush() error {
	return d.blocks.Flush()
}

func (d *Distributor) Kind() string { return "distributor" }

func (d *Distributor) BlockSize() uint64 {
	return d.blocks.BlockSize()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/cri-o/cri-o/internal/version"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/release/pkg/command"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/util"
)

const branch = "gh-pages"

var outputPath string

func main() {
	// Parse CLI flags
	flag.StringVar(&outputPath,
		"output-path", "", "the output path for the release notes",
	)
	flag.Parse()

	logrus.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true})
	if err := run(); err != nil {
		logrus.Fatalf("unable to %v", err)
	}
}

func run() error {
	// Precheck environemt
	if !util.IsEnvSet("GITHUB_TOKEN") {
		return errors.Errorf("GITHUB_TOKEN environemt variable is not set")
	}

	logrus.Infof("Ensuring output path %s", outputPath)
	if err := os.MkdirAll(outputPath, 0755); err != nil {
		return errors.Wrap(err, "create output path")
	}

	// Get latest release version
	startTag := util.AddTagPrefix(decVersion(version.Version))
	logrus.Infof("Using start tag %s", startTag)

	endTag := util.AddTagPrefix(version.Version)
	logrus.Infof("Using end tag %s", endTag)

	// Generate the notes
	repo, err := git.OpenRepo(".")
	if err != nil {
		return errors.Wrap(err, "open local repo")
	}

	head, err := repo.Head()
	if err != nil {
		return errors.Wrap(err, "get repository HEAD")
	}
	logrus.Infof("Using HEAD commit %s", head)

	targetBranch := git.Master
	currentBranch, err := repo.CurrentBranch()
	if err != nil {
		return errors.Wrap(err, "get current branch")
	}
	logrus.Infof("Found current branch %s", currentBranch)
	if git.IsReleaseBranch(currentBranch) && currentBranch != git.Master {
		targetBranch = currentBranch
	}
	logrus.Infof("Using target branch %s", targetBranch)

	templateFile, err := ioutil.TempFile("", "")
	if err != nil {
		return errors.Wrap(err, "writing template file")
	}
	defer func() { err = os.RemoveAll(templateFile.Name()) }()

	if _, err := templateFile.WriteString(fmt.Sprintf(`# CRI-O %s

The release notes have been generated for the commit range
[%s...%s](https://github.com/cri-o/cri-o/compare/%s...%s) on %s.

## Downloads

Download the static release bundle via our Google Cloud Bucket:
[crio-%s.tar.gz][0]

[0]: https://storage.googleapis.com/k8s-conform-cri-o/artifacts/crio-%s.tar.gz

## Changelog since %s

{{with .NotesWithActionRequired -}}
### Urgent Upgrade Notes

{{range .}} {{println "-" .}} {{end}}
{{end}}

{{- if .Notes -}}
### Changes by Kind
{{ range .Notes}}
#### {{.Kind | prettyKind}}
{{range $note := .NoteEntries }}{{println " -" $note}}{{end}}
{{- end -}}
{{- end -}}
`,
		endTag,
		startTag, head[:7],
		startTag, head,
		time.Now().Format(time.RFC1123),
		head[:9], head[:9],
		startTag,
	)); err != nil {
		return errors.Wrap(err, "writing tmplate to file")
	}

	logrus.Infof("Generating release notes")
	outputFile := endTag + ".md"
	outputFilePath := filepath.Join(outputPath, outputFile)
	os.RemoveAll(outputFilePath)
	if err := command.Execute(
		"./build/bin/release-notes",
		"--github-org=cri-o",
		"--github-repo=cri-o",
		"--branch="+targetBranch,
		"--repo-path=/tmp/cri-o-repo",
		"--required-author=",
		"--start-rev="+startTag,
		"--end-sha="+head,
		"--output="+outputFilePath,
		"--toc",
		"--format=go-template:"+templateFile.Name(),
	); err != nil {
		return errors.Wrap(err, "generate release notes")
	}

	// Update gh-pages branch if not a pull request and running in CircleCI
	if util.IsEnvSet("CIRCLECI") && !util.IsEnvSet("CIRCLE_PULL_REQUEST") {
		content, err := ioutil.ReadFile(outputFilePath)
		if err != nil {
			return errors.Wrap(err, "open generated release notes")
		}

		logrus.Infof("Checking out branch %s", branch)
		if err := repo.Checkout(branch); err != nil {
			return errors.Wrapf(err, "checkout %s branch", branch)
		}
		defer func() { err = repo.Checkout(currentBranch) }()

		// Write the target file
		if err := ioutil.WriteFile(outputFile, content, 0o644); err != nil {
			return errors.Wrap(err, "write content to file")
		}

		if err := repo.Add(outputFile); err != nil {
			return errors.Wrap(err, "add file to repo")
		}

		// Update the README
		readmeFile := "README.md"
		logrus.Infof("Updating %s", readmeFile)
		readmeSlice, err := readLines(readmeFile)
		if err != nil {
			return errors.Wrapf(err, "open %s file", readmeFile)
		}
		link := fmt.Sprintf("- [%s](%s)", endTag, outputFile)

		// Item not in list
		alreadyExistingIndex := indexOfPrefix(link, readmeSlice)
		if alreadyExistingIndex < 0 {
			firstListEntry := indexOfPrefix("- ", readmeSlice)

			if firstListEntry < 0 {
				// No list available, just append
				readmeSlice = append(readmeSlice, link)
			} else {
				// Insert into slice
				readmeSlice = append(
					readmeSlice[:firstListEntry],
					append([]string{link}, readmeSlice[firstListEntry:]...)...,
				)
			}
		} else {
			readmeSlice[alreadyExistingIndex] = link
		}
		if err := ioutil.WriteFile(
			readmeFile, []byte(strings.Join(readmeSlice, "\n")), 0o644,
		); err != nil {
			return errors.Wrap(err, "write content to file")
		}
		if err := repo.Add(readmeFile); err != nil {
			return errors.Wrap(err, "add file to repo")
		}

		// Publish the changes
		if err := repo.Commit("Update release notes"); err != nil {
			return errors.Wrap(err, "commit")
		}

		if err := repo.Push(branch); err != nil {
			return errors.Wrap(err, "push changes")
		}
	}

	return nil
}

func readLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

func indexOfPrefix(prefix string, slice []string) int {
	for k, v := range slice {
		if strings.HasPrefix(v, prefix) {
			return k
		}
	}
	return -1
}

func decVersion(tag string) string {
	sv, err := util.TagStringToSemver(strings.TrimSpace(tag))
	if err != nil {
		panic(err)
	}

	// clear any RC
	sv.Pre = nil

	if sv.Patch > 0 { // nolint: gocritic
		sv.Patch-- // 1.17.2 -> 1.17.1
	} else if sv.Minor > 0 {
		sv.Minor-- // 1.18.0 -> 1.17.0
	} else if sv.Major > 0 {
		sv.Major-- // 1.19.0 -> 2.0.0 (should never happen)
	} else {
		panic(fmt.Sprintf("unable to decrement version %v", sv))
	}

	return sv.String()
}
<commit_msg>Make release notes generator capable of handling tags<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/cri-o/cri-o/internal/version"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"k8s.io/release/pkg/command"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/util"
)

const branch = "gh-pages"

var outputPath string

func main() {
	// Parse CLI flags
	flag.StringVar(&outputPath,
		"output-path", "", "the output path for the release notes",
	)
	flag.Parse()

	logrus.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true})
	if err := run(); err != nil {
		logrus.Fatalf("unable to %v", err)
	}
}

func run() error {
	// Precheck environemt
	if !util.IsEnvSet("GITHUB_TOKEN") {
		return errors.Errorf("GITHUB_TOKEN environemt variable is not set")
	}

	logrus.Infof("Ensuring output path %s", outputPath)
	if err := os.MkdirAll(outputPath, 0755); err != nil {
		return errors.Wrap(err, "create output path")
	}

	// Get latest release version
	startTag := util.AddTagPrefix(decVersion(version.Version))
	logrus.Infof("Using start tag %s", startTag)

	endTag := util.AddTagPrefix(version.Version)
	logrus.Infof("Using end tag %s", endTag)

	// Generate the notes
	repo, err := git.OpenRepo(".")
	if err != nil {
		return errors.Wrap(err, "open local repo")
	}

	head, err := repo.Head()
	if err != nil {
		return errors.Wrap(err, "get repository HEAD")
	}
	logrus.Infof("Using HEAD commit %s", head)

	targetBranch := git.Master
	currentBranch, err := repo.CurrentBranch()
	if err != nil {
		return errors.Wrap(err, "get current branch")
	}
	logrus.Infof("Found current branch %s", currentBranch)
	if git.IsReleaseBranch(currentBranch) && currentBranch != git.Master {
		targetBranch = currentBranch
	}
	logrus.Infof("Using target branch %s", targetBranch)

	templateFile, err := ioutil.TempFile("", "")
	if err != nil {
		return errors.Wrap(err, "writing template file")
	}
	defer func() { err = os.RemoveAll(templateFile.Name()) }()

	// Check if we're on a tag and adapt variables if necessary
	bundleVersion := head[:9]
	shortHead := head[:7]
	endRev := head
	if output, err := command.New(
		"git", "describe", "--exact-match",
	).RunSilentSuccessOutput(); err == nil {
		foundTag := output.OutputTrimNL()
		bundleVersion = foundTag
		shortHead = foundTag
		endRev = foundTag
	}

	if _, err := templateFile.WriteString(fmt.Sprintf(`# CRI-O %s

The release notes have been generated for the commit range
[%s...%s](https://github.com/cri-o/cri-o/compare/%s...%s) on %s.

## Downloads

Download the static release bundle via our Google Cloud Bucket:
[crio-%s.tar.gz][0]

[0]: https://storage.googleapis.com/k8s-conform-cri-o/artifacts/crio-%s.tar.gz

## Changelog since %s

{{with .NotesWithActionRequired -}}
### Urgent Upgrade Notes

{{range .}} {{println "-" .}} {{end}}
{{end}}

{{- if .Notes -}}
### Changes by Kind
{{ range .Notes}}
#### {{.Kind | prettyKind}}
{{range $note := .NoteEntries }}{{println " -" $note}}{{end}}
{{- end -}}
{{- end -}}
`,
		endTag,
		startTag, shortHead,
		startTag, endRev,
		time.Now().Format(time.RFC1123),
		bundleVersion, bundleVersion,
		startTag,
	)); err != nil {
		return errors.Wrap(err, "writing tmplate to file")
	}

	logrus.Infof("Generating release notes")
	outputFile := endTag + ".md"
	outputFilePath := filepath.Join(outputPath, outputFile)
	os.RemoveAll(outputFilePath)
	if err := command.Execute(
		"./build/bin/release-notes",
		"--github-org=cri-o",
		"--github-repo=cri-o",
		"--branch="+targetBranch,
		"--repo-path=/tmp/cri-o-repo",
		"--required-author=",
		"--start-rev="+startTag,
		"--end-sha="+head,
		"--output="+outputFilePath,
		"--toc",
		"--format=go-template:"+templateFile.Name(),
	); err != nil {
		return errors.Wrap(err, "generate release notes")
	}

	// Update gh-pages branch if not a pull request and running in CircleCI
	if util.IsEnvSet("CIRCLECI") && !util.IsEnvSet("CIRCLE_PULL_REQUEST") {
		content, err := ioutil.ReadFile(outputFilePath)
		if err != nil {
			return errors.Wrap(err, "open generated release notes")
		}

		logrus.Infof("Checking out branch %s", branch)
		if err := repo.Checkout(branch); err != nil {
			return errors.Wrapf(err, "checkout %s branch", branch)
		}
		defer func() { err = repo.Checkout(currentBranch) }()

		// Write the target file
		if err := ioutil.WriteFile(outputFile, content, 0o644); err != nil {
			return errors.Wrap(err, "write content to file")
		}

		if err := repo.Add(outputFile); err != nil {
			return errors.Wrap(err, "add file to repo")
		}

		// Update the README
		readmeFile := "README.md"
		logrus.Infof("Updating %s", readmeFile)
		readmeSlice, err := readLines(readmeFile)
		if err != nil {
			return errors.Wrapf(err, "open %s file", readmeFile)
		}
		link := fmt.Sprintf("- [%s](%s)", endTag, outputFile)

		// Item not in list
		alreadyExistingIndex := indexOfPrefix(link, readmeSlice)
		if alreadyExistingIndex < 0 {
			firstListEntry := indexOfPrefix("- ", readmeSlice)

			if firstListEntry < 0 {
				// No list available, just append
				readmeSlice = append(readmeSlice, link)
			} else {
				// Insert into slice
				readmeSlice = append(
					readmeSlice[:firstListEntry],
					append([]string{link}, readmeSlice[firstListEntry:]...)...,
				)
			}
		} else {
			readmeSlice[alreadyExistingIndex] = link
		}
		if err := ioutil.WriteFile(
			readmeFile, []byte(strings.Join(readmeSlice, "\n")), 0o644,
		); err != nil {
			return errors.Wrap(err, "write content to file")
		}
		if err := repo.Add(readmeFile); err != nil {
			return errors.Wrap(err, "add file to repo")
		}

		// Publish the changes
		if err := repo.Commit("Update release notes"); err != nil {
			return errors.Wrap(err, "commit")
		}

		if err := repo.Push(branch); err != nil {
			return errors.Wrap(err, "push changes")
		}
	}

	return nil
}

func readLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

func indexOfPrefix(prefix string, slice []string) int {
	for k, v := range slice {
		if strings.HasPrefix(v, prefix) {
			return k
		}
	}
	return -1
}

func decVersion(tag string) string {
	sv, err := util.TagStringToSemver(strings.TrimSpace(tag))
	if err != nil {
		panic(err)
	}

	// clear any RC
	sv.Pre = nil

	if sv.Patch > 0 { // nolint: gocritic
		sv.Patch-- // 1.17.2 -> 1.17.1
	} else if sv.Minor > 0 {
		sv.Minor-- // 1.18.0 -> 1.17.0
	} else if sv.Major > 0 {
		sv.Major-- // 1.19.0 -> 2.0.0 (should never happen)
	} else {
		panic(fmt.Sprintf("unable to decrement version %v", sv))
	}

	return sv.String()
}
<|endoftext|>


<commit_before>package zigbee

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"sync"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
)

// ZStackServer holds the connection to one of the Z-Stack servers (nwkmgr, gateway and otasrvr)
type ZStackServer struct {
	name      string
	subsystem uint8
	conn      net.Conn
	pending   *zStackPendingCommand

	outgoingSync *sync.Mutex

	onIncoming func(uint8, *[]byte)
}

// ZStackPendingCommand is a thing
type zStackPendingCommand struct {
	request  *zStackCommand
	response *zStackCommand
	complete chan error
}

// ZStackCommand contains a protobuf message and a command id
type zStackCommand struct {
	message   proto.Message
	commandID uint8
}

func (s *ZStackServer) sendCommand(request *zStackCommand, response *zStackCommand) error {

	if s == nil {
		log.Fatalf("receiver was nil!")
	}

	s.outgoingSync.Lock()

	s.pending = &zStackPendingCommand{
		request:  request,
		response: response,
		complete: make(chan error),
	}

	err := s.transmitCommand(request)

	if err == nil {
		// The command was sent sucessfully, so we wait for the response
		timeout := make(chan bool, 1)
		go func() {
			time.Sleep(5 * time.Second) // All commands should return immediately with at least a confirmation
			timeout <- true
		}()

		if s == nil {
			log.Fatalf("assertion failed: s != nil")
		}

		if s.pending.complete == nil {
			log.Fatalf("assertion failed: s.pending.complete != nil")
		}

		select {
		case error := <-s.pending.complete:
			err = error
		case <-timeout:
			err = fmt.Errorf("The request timed out")
		}
	}

	s.pending = nil
	s.outgoingSync.Unlock()

	return err
}

func (s *ZStackServer) transmitCommand(command *zStackCommand) error {

	proto.SetDefaults(command.message)

	packet, err := proto.Marshal(command.message)
	if err != nil {
		log.Fatalf("%s: Outgoing marshaling error: %s", s.name, err)
	}

	log.Debugf("Protobuf packet %x", packet)

	buffer := new(bytes.Buffer)

	// Add the Z-Stack 4-byte header
	err = binary.Write(buffer, binary.LittleEndian, uint16(len(packet))) // Packet length
	err = binary.Write(buffer, binary.LittleEndian, s.subsystem)         // Subsystem
	err = binary.Write(buffer, binary.LittleEndian, command.commandID)   // Command Id

	_, err = buffer.Write(packet)

	log.Debugf("%s: Sending packet: % X", s.name, buffer.Bytes())

	// Send it to the Z-Stack server
	_, err = s.conn.Write(buffer.Bytes())
	return err
}

func (s *ZStackServer) incomingLoop() {
	for {
		buf := make([]byte, 1024)
		n, err := s.conn.Read(buf)

		log.Debugf("Read %d from %s", n, s.name)
		if err != nil {
			log.Fatalf("%s: Error reading socket %s", s.name, err)
		}
		pos := 0

		for {
			var length uint16
			var incomingSubsystem uint8
			reader := bytes.NewReader(buf[pos:])
			err := binary.Read(reader, binary.LittleEndian, &length)
			if err != nil {
				log.Fatalf("%s: Failed to read packet length %s", s.name, err)
			}

			err = binary.Read(reader, binary.LittleEndian, &incomingSubsystem)
			if err != nil {
				log.Fatalf("%s: Failed to read packet subsystem %s", s.name, err)
			}

			log.Debugf("%s: Incoming subsystem %d (wanted: %d)", s.name, incomingSubsystem, s.subsystem)

			log.Debugf("%s: Found packet of size : %d", s.name, length)

			commandID := uint8(buf[pos+3])

			packet := buf[pos+4 : pos+4+int(length)]

			log.Debugf("%s: Command ID:0x%X Packet: % X", s.name, commandID, packet)

			if s.pending != nil && commandID == s.pending.response.commandID {
				s.pending.complete <- proto.Unmarshal(packet, s.pending.response.message)
			} else if s.onIncoming != nil { // Or just send it out to be handled elsewhere
				go s.onIncoming(commandID, &packet)
			} else {
				log.Errorf("%s: ERR: Unhandled incoming packet: %v", s.name, packet)
			}

			pos += int(length) + 4

			if pos >= n {
				break
			}
		}
	}
}

func connectToServer(name string, subsystem uint8, hostname string, port int) (*ZStackServer, error) {

	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", hostname, port))

	if err != nil {
		return nil, err
	}

	server := &ZStackServer{
		name:         name,
		subsystem:    subsystem,
		conn:         conn,
		outgoingSync: &sync.Mutex{},
	}

	go server.incomingLoop()

	return server, nil
}
<commit_msg>Avoid nil pointer exception when s.pending.response == nil<commit_after>package zigbee

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"sync"
	"time"

	"code.google.com/p/gogoprotobuf/proto"
)

// ZStackServer holds the connection to one of the Z-Stack servers (nwkmgr, gateway and otasrvr)
type ZStackServer struct {
	name      string
	subsystem uint8
	conn      net.Conn
	pending   *zStackPendingCommand

	outgoingSync *sync.Mutex

	onIncoming func(uint8, *[]byte)
}

// ZStackPendingCommand is a thing
type zStackPendingCommand struct {
	request  *zStackCommand
	response *zStackCommand
	complete chan error
}

// ZStackCommand contains a protobuf message and a command id
type zStackCommand struct {
	message   proto.Message
	commandID uint8
}

func (s *ZStackServer) sendCommand(request *zStackCommand, response *zStackCommand) error {

	if s == nil {
		log.Fatalf("receiver was nil!")
	}

	s.outgoingSync.Lock()

	s.pending = &zStackPendingCommand{
		request:  request,
		response: response,
		complete: make(chan error),
	}

	err := s.transmitCommand(request)

	if err == nil {
		// The command was sent sucessfully, so we wait for the response
		timeout := make(chan bool, 1)
		go func() {
			time.Sleep(5 * time.Second) // All commands should return immediately with at least a confirmation
			timeout <- true
		}()

		if s == nil {
			log.Fatalf("assertion failed: s != nil")
		}

		if s.pending.complete == nil {
			log.Fatalf("assertion failed: s.pending.complete != nil")
		}

		select {
		case error := <-s.pending.complete:
			err = error
		case <-timeout:
			err = fmt.Errorf("The request timed out")
		}
	}

	s.pending = nil
	s.outgoingSync.Unlock()

	return err
}

func (s *ZStackServer) transmitCommand(command *zStackCommand) error {

	proto.SetDefaults(command.message)

	packet, err := proto.Marshal(command.message)
	if err != nil {
		log.Fatalf("%s: Outgoing marshaling error: %s", s.name, err)
	}

	log.Debugf("Protobuf packet %x", packet)

	buffer := new(bytes.Buffer)

	// Add the Z-Stack 4-byte header
	err = binary.Write(buffer, binary.LittleEndian, uint16(len(packet))) // Packet length
	err = binary.Write(buffer, binary.LittleEndian, s.subsystem)         // Subsystem
	err = binary.Write(buffer, binary.LittleEndian, command.commandID)   // Command Id

	_, err = buffer.Write(packet)

	log.Debugf("%s: Sending packet: % X", s.name, buffer.Bytes())

	// Send it to the Z-Stack server
	_, err = s.conn.Write(buffer.Bytes())
	return err
}

func (s *ZStackServer) incomingLoop() {
	for {
		buf := make([]byte, 1024)
		n, err := s.conn.Read(buf)

		log.Debugf("Read %d from %s", n, s.name)
		if err != nil {
			log.Fatalf("%s: Error reading socket %s", s.name, err)
		}
		pos := 0

		for {
			var length uint16
			var incomingSubsystem uint8
			reader := bytes.NewReader(buf[pos:])
			err := binary.Read(reader, binary.LittleEndian, &length)
			if err != nil {
				log.Fatalf("%s: Failed to read packet length %s", s.name, err)
			}

			err = binary.Read(reader, binary.LittleEndian, &incomingSubsystem)
			if err != nil {
				log.Fatalf("%s: Failed to read packet subsystem %s", s.name, err)
			}

			log.Debugf("%s: Incoming subsystem %d (wanted: %d)", s.name, incomingSubsystem, s.subsystem)

			log.Debugf("%s: Found packet of size : %d", s.name, length)

			commandID := uint8(buf[pos+3])

			packet := buf[pos+4 : pos+4+int(length)]

			log.Debugf("%s: Command ID:0x%X Packet: % X", s.name, commandID, packet)

			if s.pending != nil && commandID == s.pending.response.commandID {
				if s.pending.response != nil {
					s.pending.complete <- proto.Unmarshal(packet, s.pending.response.message)
				} else {
					log.Errorf("ERR: dropped because s.pending.response == nil (commandID = %d)", commandID)
				}
			} else if s.onIncoming != nil { // Or just send it out to be handled elsewhere
				go s.onIncoming(commandID, &packet)
			} else {
				log.Errorf("%s: ERR: Unhandled incoming packet: %v", s.name, packet)
			}

			pos += int(length) + 4

			if pos >= n {
				break
			}
		}
	}
}

func connectToServer(name string, subsystem uint8, hostname string, port int) (*ZStackServer, error) {

	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", hostname, port))

	if err != nil {
		return nil, err
	}

	server := &ZStackServer{
		name:         name,
		subsystem:    subsystem,
		conn:         conn,
		outgoingSync: &sync.Mutex{},
	}

	go server.incomingLoop()

	return server, nil
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"path"
	"runtime"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/hash"
	"github.com/minio/minio/pkg/quick"
)

const (
	minioConfigPrefix = "config"

	// Minio configuration file.
	minioConfigFile = "config.json"
)

func saveServerConfig(objAPI ObjectLayer, config *serverConfig) error {
	if err := quick.CheckData(config); err != nil {
		return err
	}

	data, err := json.Marshal(config)
	if err != nil {
		return err
	}

	configFile := path.Join(minioConfigPrefix, minioConfigFile)
	if globalEtcdClient != nil {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
		_, err := globalEtcdClient.Put(ctx, configFile, string(data))
		defer cancel()
		return err
	}

	return saveConfig(objAPI, configFile, data)
}

func readConfigEtcd(configFile string) ([]byte, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	resp, err := globalEtcdClient.Get(ctx, configFile)
	defer cancel()
	if err != nil {
		return nil, err
	}
	if resp.Count == 0 {
		return nil, errConfigNotFound
	}
	for _, ev := range resp.Kvs {
		if string(ev.Key) == configFile {
			return ev.Value, nil
		}
	}
	return nil, errConfigNotFound
}

func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
	var configData []byte
	var err error
	configFile := path.Join(minioConfigPrefix, minioConfigFile)
	if globalEtcdClient != nil {
		configData, err = readConfigEtcd(configFile)
	} else {
		var reader io.Reader
		reader, err = readConfig(ctx, objAPI, configFile)
		if err != nil {
			return nil, err
		}
		configData, err = ioutil.ReadAll(reader)
	}
	if err != nil {
		return nil, err
	}

	if runtime.GOOS == "windows" {
		configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
	}

	if err = quick.CheckDuplicateKeys(string(configData)); err != nil {
		return nil, err
	}

	var config = &serverConfig{}
	if err := json.Unmarshal(configData, config); err != nil {
		return nil, err
	}

	if err := quick.CheckData(config); err != nil {
		return nil, err
	}

	return config, nil
}

func checkServerConfigEtcd(configFile string) error {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	resp, err := globalEtcdClient.Get(ctx, configFile)
	defer cancel()
	if err != nil {
		return err
	}
	if resp.Count == 0 {
		return errConfigNotFound
	}
	return nil
}

func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
	configFile := path.Join(minioConfigPrefix, minioConfigFile)
	if globalEtcdClient != nil {
		return checkServerConfigEtcd(configFile)
	}

	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil {
		if isErrObjectNotFound(err) {
			return errConfigNotFound
		}
		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
		logger.LogIf(ctx, err)
		return err
	}
	return nil
}

func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
	if err != nil {
		return err
	}

	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
	return err
}

var errConfigNotFound = errors.New("config file not found")

func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
	var buffer bytes.Buffer
	// Read entire content by setting size to -1
	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil {
		// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
			return nil, errConfigNotFound
		}

		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
		logger.LogIf(ctx, err)
		return nil, err
	}

	// Return config not found on empty content.
	if buffer.Len() == 0 {
		return nil, errConfigNotFound
	}

	return &buffer, nil
}

// ConfigSys - config system.
type ConfigSys struct{}

// Load - load config.json.
func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
	return sys.Init(objAPI)
}

// Init - initializes config system from config.json.
func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
	if objAPI == nil {
		return errInvalidArgument
	}
	return initConfig(objAPI)
}

// NewConfigSys - creates new config system object.
func NewConfigSys() *ConfigSys {
	return &ConfigSys{}
}

// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
func migrateConfigToMinioSys(objAPI ObjectLayer) error {
	// Verify if backend already has the file.
	if err := checkServerConfig(context.Background(), objAPI); err != errConfigNotFound {
		return err
	} // if errConfigNotFound proceed to migrate..

	var config = &serverConfig{}
	if _, err := Load(getConfigFile(), config); err != nil {
		return err
	}

	return saveServerConfig(objAPI, config)
}

// Initialize and load config from remote etcd or local config directory
func initConfig(objAPI ObjectLayer) error {
	if objAPI == nil {
		return errServerNotInitialized
	}

	if globalEtcdClient != nil {
		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
		cancel()
		if err == nil && resp.Count > 0 {
			return migrateConfig()
		}
	} else {
		if isFile(getConfigFile()) {
			if err := migrateConfig(); err != nil {
				return err
			}

			// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
			if err := migrateConfigToMinioSys(objAPI); err != nil {
				return err
			}
		}
	}

	if err := checkServerConfig(context.Background(), objAPI); err != nil {
		if err == errConfigNotFound {
			// Config file does not exist, we create it fresh and return upon success.
			if err = newConfig(objAPI); err != nil {
				return err
			}
		} else {
			return err
		}
	}

	if err := migrateMinioSysConfig(objAPI); err != nil {
		return err
	}

	return loadConfig(objAPI)
}
<commit_msg>With no read quorum config should be treated as notFound (#6374)<commit_after>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"path"
	"runtime"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/hash"
	"github.com/minio/minio/pkg/quick"
)

const (
	minioConfigPrefix = "config"

	// Minio configuration file.
	minioConfigFile = "config.json"
)

func saveServerConfig(objAPI ObjectLayer, config *serverConfig) error {
	if err := quick.CheckData(config); err != nil {
		return err
	}

	data, err := json.Marshal(config)
	if err != nil {
		return err
	}

	configFile := path.Join(minioConfigPrefix, minioConfigFile)
	if globalEtcdClient != nil {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
		_, err := globalEtcdClient.Put(ctx, configFile, string(data))
		defer cancel()
		return err
	}

	return saveConfig(objAPI, configFile, data)
}

func readConfigEtcd(configFile string) ([]byte, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	resp, err := globalEtcdClient.Get(ctx, configFile)
	defer cancel()
	if err != nil {
		return nil, err
	}
	if resp.Count == 0 {
		return nil, errConfigNotFound
	}
	for _, ev := range resp.Kvs {
		if string(ev.Key) == configFile {
			return ev.Value, nil
		}
	}
	return nil, errConfigNotFound
}

func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
	var configData []byte
	var err error
	configFile := path.Join(minioConfigPrefix, minioConfigFile)
	if globalEtcdClient != nil {
		configData, err = readConfigEtcd(configFile)
	} else {
		var reader io.Reader
		reader, err = readConfig(ctx, objAPI, configFile)
		if err != nil {
			return nil, err
		}
		configData, err = ioutil.ReadAll(reader)
	}
	if err != nil {
		return nil, err
	}

	if runtime.GOOS == "windows" {
		configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
	}

	if err = quick.CheckDuplicateKeys(string(configData)); err != nil {
		return nil, err
	}

	var config = &serverConfig{}
	if err := json.Unmarshal(configData, config); err != nil {
		return nil, err
	}

	if err := quick.CheckData(config); err != nil {
		return nil, err
	}

	return config, nil
}

func checkServerConfigEtcd(configFile string) error {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	resp, err := globalEtcdClient.Get(ctx, configFile)
	defer cancel()
	if err != nil {
		return err
	}
	if resp.Count == 0 {
		return errConfigNotFound
	}
	return nil
}

func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
	configFile := path.Join(minioConfigPrefix, minioConfigFile)
	if globalEtcdClient != nil {
		return checkServerConfigEtcd(configFile)
	}

	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil {
		// Convert ObjectNotFound, Quorum errors into errConfigNotFound
		if isErrObjectNotFound(err) || isInsufficientReadQuorum(err) {
			return errConfigNotFound
		}
		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
		logger.LogIf(ctx, err)
		return err
	}
	return nil
}

func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
	if err != nil {
		return err
	}

	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
	return err
}

var errConfigNotFound = errors.New("config file not found")

func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
	var buffer bytes.Buffer
	// Read entire content by setting size to -1
	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil {
		// Convert ObjectNotFound, IncompleteBody and Quorum errors into errConfigNotFound
		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
			return nil, errConfigNotFound
		}

		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
		logger.LogIf(ctx, err)
		return nil, err
	}

	// Return config not found on empty content.
	if buffer.Len() == 0 {
		return nil, errConfigNotFound
	}

	return &buffer, nil
}

// ConfigSys - config system.
type ConfigSys struct{}

// Load - load config.json.
func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
	return sys.Init(objAPI)
}

// Init - initializes config system from config.json.
func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
	if objAPI == nil {
		return errInvalidArgument
	}
	return initConfig(objAPI)
}

// NewConfigSys - creates new config system object.
func NewConfigSys() *ConfigSys {
	return &ConfigSys{}
}

// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
func migrateConfigToMinioSys(objAPI ObjectLayer) error {
	// Verify if backend already has the file.
	if err := checkServerConfig(context.Background(), objAPI); err != errConfigNotFound {
		return err
	} // if errConfigNotFound proceed to migrate..

	var config = &serverConfig{}
	if _, err := Load(getConfigFile(), config); err != nil {
		return err
	}

	return saveServerConfig(objAPI, config)
}

// Initialize and load config from remote etcd or local config directory
func initConfig(objAPI ObjectLayer) error {
	if objAPI == nil {
		return errServerNotInitialized
	}

	if globalEtcdClient != nil {
		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
		cancel()
		if err == nil && resp.Count > 0 {
			return migrateConfig()
		}
	} else {
		if isFile(getConfigFile()) {
			if err := migrateConfig(); err != nil {
				return err
			}

			// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
			if err := migrateConfigToMinioSys(objAPI); err != nil {
				return err
			}
		}
	}

	if err := checkServerConfig(context.Background(), objAPI); err != nil {
		if err == errConfigNotFound {
			// Config file does not exist, we create it fresh and return upon success.
			if err = newConfig(objAPI); err != nil {
				return err
			}
		} else {
			return err
		}
	}

	if err := migrateMinioSysConfig(objAPI); err != nil {
		return err
	}

	return loadConfig(objAPI)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"path/filepath"

	"github.com/ory/viper"
	"github.com/spf13/cobra"

	bosonFunc "github.com/boson-project/func"
	"github.com/boson-project/func/prompt"
	"github.com/boson-project/func/utils"
)

func init() {
	root.AddCommand(createCmd)
	createCmd.Flags().BoolP("confirm", "c", false, "Prompt to confirm all configuration options (Env: $FUNC_CONFIRM)")
	createCmd.Flags().StringP("runtime", "l", bosonFunc.DefaultRuntime, "Function runtime language/framework. Available runtimes: "+utils.RuntimeList()+" (Env: $FUNC_RUNTIME)")
	createCmd.Flags().StringP("packages", "p", filepath.Join(configPath(), "packages"), "Path to additional template packages (Env: $FUNC_PACKAGES)")
	createCmd.Flags().StringP("template", "t", bosonFunc.DefaultTemplate, "Function template. Available templates: 'http' and 'events' (Env: $FUNC_TEMPLATE)")

	if err := createCmd.RegisterFlagCompletionFunc("runtime", CompleteRuntimeList); err != nil {
		fmt.Println("internal: error while calling RegisterFlagCompletionFunc: ", err)
	}
}

var createCmd = &cobra.Command{
	Use:   "create [PATH]",
	Short: "Create a function project",
	Long: `Create a function project

Creates a new function project in PATH, or in the current directory if no PATH is given.
The name of the project is determined by the directory name the project is created in.
`,
	Example: `
# Create a Node.js function project in the current directory, choosing the
# directory name as the project's name.
kn func create

# Create a Quarkus function project in the directory "sample-service".
# The directory will be created in the local directory if non-existent and
# the project is called "sample-service"
kn func create --runtime quarkus myfunc

# Create a function project that uses a CloudEvent based function signature
kn func create --template events myfunc
`,
	SuggestFor: []string{"inti", "new"},
	PreRunE:    bindEnv("runtime", "template", "packages", "confirm"),
	RunE:       runCreate,
	// TODO: autocomplate or interactive prompt for runtime and template.
}

func runCreate(cmd *cobra.Command, args []string) error {
	config := newCreateConfig(args)

	if err := utils.ValidateFunctionName(config.Name); err != nil {
		return err
	}

	config = config.Prompt()

	function := bosonFunc.Function{
		Name:     config.Name,
		Root:     config.Path,
		Runtime:  config.Runtime,
		Template: config.Template,
	}

	client := bosonFunc.New(
		bosonFunc.WithPackages(config.Packages),
		bosonFunc.WithVerbose(config.Verbose))

	return client.Create(function)
}

type createConfig struct {
	// Name of the Function.
	Name string

	// Absolute path to Function on disk.
	Path string

	// Runtime language/framework.
	Runtime string

	// Packages is an optional path that, if it exists, will be used as a source
	// for additional template packages not included in the binary.  If not provided
	// explicitly as a flag (--packages) or env (FUNC_PACKAGES), the default
	// location is $XDG_CONFIG_HOME/packages ($HOME/.config/func/packages)
	Packages string

	// Template is the code written into the new Function project, including
	// an implementation adhering to one of the supported function signatures.
	// May also include additional configuration settings or examples.
	// For example, embedded are 'http' for a Function whose funciton signature
	// is invoked via straight HTTP requests, or 'events' for a Function which
	// will be invoked with CloudEvents.  These embedded templates contain a
	// minimum implementation of the signature itself and example tests.
	Template string

	// Verbose output
	Verbose bool

	// Confirm: confirm values arrived upon from environment plus flags plus defaults,
	// with interactive prompting (only applicable when attached to a TTY).
	Confirm bool
}

// newCreateConfig returns a config populated from the current execution context
// (args, flags and environment variables)
func newCreateConfig(args []string) createConfig {
	var path string
	if len(args) > 0 {
		path = args[0] // If explicitly provided, use.
	}

	derivedName, derivedPath := deriveNameAndAbsolutePathFromPath(path)
	return createConfig{
		Name:     derivedName,
		Path:     derivedPath,
		Packages: viper.GetString("packages"),
		Runtime:  viper.GetString("runtime"),
		Template: viper.GetString("template"),
		Confirm:  viper.GetBool("confirm"),
		Verbose:  viper.GetBool("verbose"),
	}
}

// Prompt the user with value of config members, allowing for interaractive changes.
// Skipped if not in an interactive terminal (non-TTY), or if --confirm false (agree to
// all prompts) was set (default).
func (c createConfig) Prompt() createConfig {
	if !interactiveTerminal() || !c.Confirm {
		// Just print the basics if not confirming
		fmt.Printf("Project path: %v\n", c.Path)
		fmt.Printf("Function name: %v\n", c.Name)
		fmt.Printf("Runtime: %v\n", c.Runtime)
		fmt.Printf("Template: %v\n", c.Template)
		return c
	}

	var derivedName, derivedPath string
	for {
		derivedName, derivedPath = deriveNameAndAbsolutePathFromPath(prompt.ForString("Project path", c.Path, prompt.WithRequired(true)))
		err := utils.ValidateFunctionName(derivedName)
		if err == nil {
			break
		}
		fmt.Println("Error:", err)
	}

	return createConfig{
		Name:     derivedName,
		Path:     derivedPath,
		Runtime:  prompt.ForString("Runtime", c.Runtime),
		Template: prompt.ForString("Template", c.Template),
	}
}
<commit_msg>use 'a' for 'additional packages' parameter --packages<commit_after>package cmd

import (
	"fmt"
	"path/filepath"

	"github.com/ory/viper"
	"github.com/spf13/cobra"

	bosonFunc "github.com/boson-project/func"
	"github.com/boson-project/func/prompt"
	"github.com/boson-project/func/utils"
)

func init() {
	root.AddCommand(createCmd)
	createCmd.Flags().BoolP("confirm", "c", false, "Prompt to confirm all configuration options (Env: $FUNC_CONFIRM)")
	createCmd.Flags().StringP("runtime", "l", bosonFunc.DefaultRuntime, "Function runtime language/framework. Available runtimes: "+utils.RuntimeList()+" (Env: $FUNC_RUNTIME)")
	createCmd.Flags().StringP("packages", "a", filepath.Join(configPath(), "packages"), "Path to additional template packages (Env: $FUNC_PACKAGES)")
	createCmd.Flags().StringP("template", "t", bosonFunc.DefaultTemplate, "Function template. Available templates: 'http' and 'events' (Env: $FUNC_TEMPLATE)")

	if err := createCmd.RegisterFlagCompletionFunc("runtime", CompleteRuntimeList); err != nil {
		fmt.Println("internal: error while calling RegisterFlagCompletionFunc: ", err)
	}
}

var createCmd = &cobra.Command{
	Use:   "create [PATH]",
	Short: "Create a function project",
	Long: `Create a function project

Creates a new function project in PATH, or in the current directory if no PATH is given.
The name of the project is determined by the directory name the project is created in.
`,
	Example: `
# Create a Node.js function project in the current directory, choosing the
# directory name as the project's name.
kn func create

# Create a Quarkus function project in the directory "sample-service".
# The directory will be created in the local directory if non-existent and
# the project is called "sample-service"
kn func create --runtime quarkus myfunc

# Create a function project that uses a CloudEvent based function signature
kn func create --template events myfunc
`,
	SuggestFor: []string{"inti", "new"},
	PreRunE:    bindEnv("runtime", "template", "packages", "confirm"),
	RunE:       runCreate,
	// TODO: autocomplate or interactive prompt for runtime and template.
}

func runCreate(cmd *cobra.Command, args []string) error {
	config := newCreateConfig(args)

	if err := utils.ValidateFunctionName(config.Name); err != nil {
		return err
	}

	config = config.Prompt()

	function := bosonFunc.Function{
		Name:     config.Name,
		Root:     config.Path,
		Runtime:  config.Runtime,
		Template: config.Template,
	}

	client := bosonFunc.New(
		bosonFunc.WithPackages(config.Packages),
		bosonFunc.WithVerbose(config.Verbose))

	return client.Create(function)
}

type createConfig struct {
	// Name of the Function.
	Name string

	// Absolute path to Function on disk.
	Path string

	// Runtime language/framework.
	Runtime string

	// Packages is an optional path that, if it exists, will be used as a source
	// for additional template packages not included in the binary.  If not provided
	// explicitly as a flag (--packages) or env (FUNC_PACKAGES), the default
	// location is $XDG_CONFIG_HOME/packages ($HOME/.config/func/packages)
	Packages string

	// Template is the code written into the new Function project, including
	// an implementation adhering to one of the supported function signatures.
	// May also include additional configuration settings or examples.
	// For example, embedded are 'http' for a Function whose funciton signature
	// is invoked via straight HTTP requests, or 'events' for a Function which
	// will be invoked with CloudEvents.  These embedded templates contain a
	// minimum implementation of the signature itself and example tests.
	Template string

	// Verbose output
	Verbose bool

	// Confirm: confirm values arrived upon from environment plus flags plus defaults,
	// with interactive prompting (only applicable when attached to a TTY).
	Confirm bool
}

// newCreateConfig returns a config populated from the current execution context
// (args, flags and environment variables)
func newCreateConfig(args []string) createConfig {
	var path string
	if len(args) > 0 {
		path = args[0] // If explicitly provided, use.
	}

	derivedName, derivedPath := deriveNameAndAbsolutePathFromPath(path)
	return createConfig{
		Name:     derivedName,
		Path:     derivedPath,
		Packages: viper.GetString("packages"),
		Runtime:  viper.GetString("runtime"),
		Template: viper.GetString("template"),
		Confirm:  viper.GetBool("confirm"),
		Verbose:  viper.GetBool("verbose"),
	}
}

// Prompt the user with value of config members, allowing for interaractive changes.
// Skipped if not in an interactive terminal (non-TTY), or if --confirm false (agree to
// all prompts) was set (default).
func (c createConfig) Prompt() createConfig {
	if !interactiveTerminal() || !c.Confirm {
		// Just print the basics if not confirming
		fmt.Printf("Project path: %v\n", c.Path)
		fmt.Printf("Function name: %v\n", c.Name)
		fmt.Printf("Runtime: %v\n", c.Runtime)
		fmt.Printf("Template: %v\n", c.Template)
		return c
	}

	var derivedName, derivedPath string
	for {
		derivedName, derivedPath = deriveNameAndAbsolutePathFromPath(prompt.ForString("Project path", c.Path, prompt.WithRequired(true)))
		err := utils.ValidateFunctionName(derivedName)
		if err == nil {
			break
		}
		fmt.Println("Error:", err)
	}

	return createConfig{
		Name:     derivedName,
		Path:     derivedPath,
		Runtime:  prompt.ForString("Runtime", c.Runtime),
		Template: prompt.ForString("Template", c.Template),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/pkg/profile"
	"github.com/simulot/gogrep/gogrep"
	// Register common walkers
	_ "github.com/simulot/golib/file/walker/zipwalker"
)

func main() {
	defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
	app := &gogrep.AppSettings{}
	err := app.Commandline()
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
	app.Run()
}
<commit_msg>add parameter mask<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/simulot/gogrep/gogrep"
	// Register common walkers
	_ "github.com/simulot/golib/file/walker/zipwalker"
)

func main() {
	// defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
	app := &gogrep.AppSettings{}
	err := app.Commandline()
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
	app.Run()
}
<|endoftext|>


<commit_before>// Copyright 2014 Rob Pike. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Kana accepts, as argument text or as standard input, Japanese
UTF-8 text. It copies the text to standard output, converting
katakana and hiragana to romaji, leaving the rest (including
kanji) unmodified.
*/
package main // import "robpike.io/cmd/kana"

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"unicode/utf8"
)

func main() {
	if len(os.Args) == 1 {
		data, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatal(err)
		}
		do(string(data))
		return
	}
	do(strings.Join(os.Args[1:], " "))
}

func do(s string) {
	prevKana := true
	skip := false
	for i, r := range s {
		k, ok := kana[r]
		if skip {
			skip = false
			continue
		}
		if !ok {
			if prevKana {
				fmt.Printf(" ")
			}
			fmt.Printf("%c", r)
			prevKana = false
			continue
		}
		if !prevKana {
			fmt.Printf(" ")
		}
		prevKana = true
		// Is there a modifier?
		r2, _ := utf8.DecodeRuneInString(s[i+utf8.RuneLen(r):])
		if small[r2] {
			skip = true
			k2, ok := mod[r2]
			if ok {
				fmt.Printf("%s%s", k[:len(k)-1], k2[1:])
				continue
			}
			k2, ok = vowel[r2]
			if ok {
				fmt.Printf("%s-", k)
				continue
			}
			// Otherwise it's just odd.
			fmt.Printf("<%s.%s>", k, odd[r2])
			continue
		}
		fmt.Printf("%s", k)
	}
	fmt.Print("\n")
}

var kana = map[rune]string{
	'あ': "a",
	'い': "i",
	'う': "u",
	'え': "e",
	'お': "o",
	'か': "ka",
	'が': "ga",
	'き': "ki",
	'ぎ': "gi",
	'く': "ku",
	'ぐ': "gu",
	'け': "ke",
	'げ': "ge",
	'こ': "ko",
	'ご': "go",
	'さ': "sa",
	'ざ': "za",
	'し': "shi",
	'じ': "zi",
	'す': "su",
	'ず': "zu",
	'せ': "se",
	'ぜ': "ze",
	'そ': "so",
	'ぞ': "zo",
	'た': "ta",
	'だ': "da",
	'ち': "chi",
	'ぢ': "di",
	'つ': "tsu",
	'づ': "du",
	'て': "te",
	'で': "de",
	'と': "to",
	'ど': "do",
	'な': "na",
	'に': "ni",
	'ぬ': "nu",
	'ね': "ne",
	'の': "no",
	'は': "ha",
	'ば': "va",
	'ぱ': "pa",
	'ひ': "hi",
	'び': "vi",
	'ぴ': "pi",
	'ふ': "fu",
	'ぶ': "vu",
	'ぷ': "pu",
	'へ': "he",
	'べ': "ve",
	'ぺ': "pe",
	'ほ': "ho",
	'ぼ': "vo",
	'ぽ': "po",
	'ま': "ma",
	'み': "mi",
	'む': "mu",
	'め': "me",
	'も': "mo",
	'や': "ya",
	'ゆ': "yu",
	'よ': "yo",
	'ら': "ra",
	'り': "ri",
	'る': "ru",
	'れ': "re",
	'ろ': "ro",
	'わ': "wa",
	'ゐ': "wi",
	'ゑ': "we",
	'を': "wo",
	'ん': "n",
	'ゔ': "vu",
	'ア': "a",
	'イ': "i",
	'ウ': "u",
	'エ': "e",
	'オ': "o",
	'カ': "ka",
	'ガ': "ga",
	'キ': "ki",
	'ギ': "gi",
	'ク': "ku",
	'グ': "gu",
	'ケ': "ke",
	'ゲ': "ge",
	'コ': "ko",
	'ゴ': "go",
	'サ': "sa",
	'ザ': "za",
	'シ': "shi",
	'ジ': "zi",
	'ス': "su",
	'ズ': "zu",
	'セ': "se",
	'ゼ': "ze",
	'ソ': "so",
	'ゾ': "zo",
	'タ': "ta",
	'ダ': "da",
	'チ': "chi",
	'ヂ': "di",
	'ツ': "tsu",
	'ヅ': "du",
	'テ': "te",
	'デ': "de",
	'ト': "to",
	'ド': "do",
	'ナ': "na",
	'ニ': "ni",
	'ヌ': "nu",
	'ネ': "ne",
	'ノ': "no",
	'ハ': "ha",
	'バ': "va",
	'パ': "pa",
	'ヒ': "hi",
	'ビ': "vi",
	'ピ': "pi",
	'フ': "fu",
	'ブ': "vu",
	'プ': "pu",
	'ヘ': "he",
	'ベ': "ve",
	'ペ': "pe",
	'ホ': "ho",
	'ボ': "vo",
	'ポ': "po",
	'マ': "ma",
	'ミ': "mi",
	'ム': "mu",
	'メ': "me",
	'モ': "mo",
	'ヤ': "ya",
	'ユ': "yu",
	'ヨ': "yo",
	'ラ': "ra",
	'リ': "ri",
	'ル': "ru",
	'レ': "re",
	'ロ': "ro",
	'ワ': "wa",
	'ヰ': "wi",
	'ヱ': "we",
	'ヲ': "wo",
	'ン': "n",
	'ヴ': "vu",
}

var small = map[rune]bool{
	'ぁ': true,
	'ぃ': true,
	'ぅ': true,
	'ぇ': true,
	'ぉ': true,
	'っ': true,
	'ゃ': true,
	'ゅ': true,
	'ょ': true,
	'ゎ': true,
	'ゕ': true,
	'ゖ': true,

	'ァ': true,
	'ィ': true,
	'ゥ': true,
	'ェ': true,
	'ォ': true,
	'ッ': true,
	'ャ': true,
	'ュ': true,
	'ョ': true,
	'ヮ': true,
	'ヵ': true,
	'ヶ': true,
}

var vowel = map[rune]string{
	'ぁ': "a",
	'ぃ': "i",
	'ぅ': "u",
	'ぇ': "e",
	'ぉ': "o",

	'ァ': "a",
	'ィ': "i",
	'ゥ': "u",
	'ェ': "e",
	'ォ': "o",
}

var mod = map[rune]string{
	'ゃ': "ya",
	'ゅ': "yu",
	'ょ': "yo",

	'ャ': "ya",
	'ュ': "yu",
	'ョ': "yo",
}

var odd = map[rune]string{
	'っ': "hold",  // tsu == hold consonant
	'ゕ': "count", // ka == counting mark
	'ゖ': "count", // ke == counting mark
	'ッ': "hold",  // tsu == hold consonant
	'ヵ': "count", // ka == counting mark
	'ヶ': "count", // ke == counting mark
}
<commit_msg>hira,kata: new commands to complement kana<commit_after>// Copyright 2014 Rob Pike. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Kana accepts, as argument text or as standard input, Japanese
UTF-8 text. It copies the text to standard output, converting
katakana and hiragana to romaji, leaving the rest (including
kanji) unmodified.
*/
package main // import "robpike.io/cmd/kana"

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"unicode/utf8"
)

func main() {
	if len(os.Args) == 1 {
		data, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatal(err)
		}
		do(string(data))
		return
	}
	do(strings.Join(os.Args[1:], " "))
}

func do(s string) {
	prevKana := true
	skip := false
	for i, r := range s {
		k, ok := kana[r]
		if skip {
			skip = false
			continue
		}
		if !ok {
			if prevKana {
				fmt.Printf(" ")
			}
			fmt.Printf("%c", r)
			prevKana = false
			continue
		}
		if !prevKana {
			fmt.Printf(" ")
		}
		prevKana = true
		// Is there a modifier?
		r2, _ := utf8.DecodeRuneInString(s[i+utf8.RuneLen(r):])
		if small[r2] {
			skip = true
			k2, ok := mod[r2]
			if ok {
				fmt.Printf("%s%s", k[:len(k)-1], k2[1:])
				continue
			}
			k2, ok = vowel[r2]
			if ok {
				fmt.Printf("%s-", k)
				continue
			}
			// Otherwise it's just odd.
			fmt.Printf("<%s.%s>", k, odd[r2])
			continue
		}
		fmt.Printf("%s", k)
	}
	fmt.Print("\n")
}

var kana = map[rune]string{
	'あ': "a",
	'い': "i",
	'う': "u",
	'え': "e",
	'お': "o",
	'か': "ka",
	'が': "ga",
	'き': "ki",
	'ぎ': "gi",
	'く': "ku",
	'ぐ': "gu",
	'け': "ke",
	'げ': "ge",
	'こ': "ko",
	'ご': "go",
	'さ': "sa",
	'ざ': "za",
	'し': "shi",
	'じ': "zi",
	'す': "su",
	'ず': "zu",
	'せ': "se",
	'ぜ': "ze",
	'そ': "so",
	'ぞ': "zo",
	'た': "ta",
	'だ': "da",
	'ち': "chi",
	'ぢ': "di",
	'つ': "tsu",
	'づ': "du",
	'て': "te",
	'で': "de",
	'と': "to",
	'ど': "do",
	'な': "na",
	'に': "ni",
	'ぬ': "nu",
	'ね': "ne",
	'の': "no",
	'は': "ha",
	'ば': "va",
	'ぱ': "pa",
	'ひ': "hi",
	'び': "vi",
	'ぴ': "pi",
	'ふ': "fu",
	'ぶ': "bu",
	'ぷ': "pu",
	'へ': "he",
	'べ': "ve",
	'ぺ': "pe",
	'ほ': "ho",
	'ぼ': "vo",
	'ぽ': "po",
	'ま': "ma",
	'み': "mi",
	'む': "mu",
	'め': "me",
	'も': "mo",
	'や': "ya",
	'ゆ': "yu",
	'よ': "yo",
	'ら': "ra",
	'り': "ri",
	'る': "ru",
	'れ': "re",
	'ろ': "ro",
	'わ': "wa",
	'ゐ': "wi",
	'ゑ': "we",
	'を': "wo",
	'ん': "n",
	'ゔ': "vu",
	'ア': "a",
	'イ': "i",
	'ウ': "u",
	'エ': "e",
	'オ': "o",
	'カ': "ka",
	'ガ': "ga",
	'キ': "ki",
	'ギ': "gi",
	'ク': "ku",
	'グ': "gu",
	'ケ': "ke",
	'ゲ': "ge",
	'コ': "ko",
	'ゴ': "go",
	'サ': "sa",
	'ザ': "za",
	'シ': "shi",
	'ジ': "zi",
	'ス': "su",
	'ズ': "zu",
	'セ': "se",
	'ゼ': "ze",
	'ソ': "so",
	'ゾ': "zo",
	'タ': "ta",
	'ダ': "da",
	'チ': "chi",
	'ヂ': "di",
	'ツ': "tsu",
	'ヅ': "du",
	'テ': "te",
	'デ': "de",
	'ト': "to",
	'ド': "do",
	'ナ': "na",
	'ニ': "ni",
	'ヌ': "nu",
	'ネ': "ne",
	'ノ': "no",
	'ハ': "ha",
	'バ': "va",
	'パ': "pa",
	'ヒ': "hi",
	'ビ': "vi",
	'ピ': "pi",
	'フ': "fu",
	'ブ': "bu",
	'プ': "pu",
	'ヘ': "he",
	'ベ': "ve",
	'ペ': "pe",
	'ホ': "ho",
	'ボ': "vo",
	'ポ': "po",
	'マ': "ma",
	'ミ': "mi",
	'ム': "mu",
	'メ': "me",
	'モ': "mo",
	'ヤ': "ya",
	'ユ': "yu",
	'ヨ': "yo",
	'ラ': "ra",
	'リ': "ri",
	'ル': "ru",
	'レ': "re",
	'ロ': "ro",
	'ワ': "wa",
	'ヰ': "wi",
	'ヱ': "we",
	'ヲ': "wo",
	'ン': "n",
	'ヴ': "vu",
}

var small = map[rune]bool{
	'ぁ': true,
	'ぃ': true,
	'ぅ': true,
	'ぇ': true,
	'ぉ': true,
	'っ': true,
	'ゃ': true,
	'ゅ': true,
	'ょ': true,
	'ゎ': true,
	'ゕ': true,
	'ゖ': true,

	'ァ': true,
	'ィ': true,
	'ゥ': true,
	'ェ': true,
	'ォ': true,
	'ッ': true,
	'ャ': true,
	'ュ': true,
	'ョ': true,
	'ヮ': true,
	'ヵ': true,
	'ヶ': true,
}

var vowel = map[rune]string{
	'ぁ': "a",
	'ぃ': "i",
	'ぅ': "u",
	'ぇ': "e",
	'ぉ': "o",

	'ァ': "a",
	'ィ': "i",
	'ゥ': "u",
	'ェ': "e",
	'ォ': "o",
}

var mod = map[rune]string{
	'ゃ': "ya",
	'ゅ': "yu",
	'ょ': "yo",

	'ャ': "ya",
	'ュ': "yu",
	'ョ': "yo",
}

var odd = map[rune]string{
	'っ': "hold",  // tsu == hold consonant
	'ゕ': "count", // ka == counting mark
	'ゖ': "count", // ke == counting mark
	'ッ': "hold",  // tsu == hold consonant
	'ヵ': "count", // ka == counting mark
	'ヶ': "count", // ke == counting mark
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"text/tabwriter"

	"github.com/flynn/flynn-controller/client"
)

var cmdKeys = &Command{
	Run:   runKeys,
	Usage: "keys",
	Short: "list ssh public keys",
	Long: `
Command keys lists SSH public keys associated with the Flynn controller.

Examples:

    $ flynn keys
    5e:67:40:b6:79:db:56:47:cd:3a:a7:65:ab:ed:12:34  user@test.com
`,
}

func runKeys(cmd *Command, args []string, client *controller.Client) error {
	if len(args) != 0 {
		cmd.printUsage(true)
	}

	keys, err := client.KeyList()
	if err != nil {
		return err
	}

	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
	defer w.Flush()

	for i := range keys {
		listRec(w,
			formatKeyID(keys[i].ID),
			keys[i].Comment,
		)
	}
	return nil
}

func formatKeyID(s string) string {
	buf := make([]byte, 0, len(s)+((len(s)-2)/2))
	for i := range s {
		buf = append(buf, s[i])
		if (i+1)%2 == 0 && i != len(s)-1 {
			buf = append(buf, ':')
		}
	}
	return string(buf)
}

var cmdKeyAdd = &Command{
	Run:   runKeyAdd,
	Usage: "key-add [<public-key-file>]",
	Short: "add ssh public key",
	Long: `
Command key-add adds an ssh public key to the Flynn controller.

It tries these sources for keys, in order:

1. public-key-file argument, if present
2. output of ssh-add -L, if any
3. file $HOME/.ssh/id_rsa.pub
`,
}

func runKeyAdd(cmd *Command, args []string, client *controller.Client) error {
	if len(args) > 1 {
		cmd.printUsage(true)
	}
	var sshPubKeyPath string
	if len(args) == 1 {
		sshPubKeyPath = args[0]
	}
	keys, err := findKeys(sshPubKeyPath)
	if err != nil {
		if _, ok := err.(privKeyError); ok {
			log.Println("refusing to upload")
		}
		return err
	}

	key, err := client.CreateKey(string(keys))
	if err != nil {
		return err
	}
	log.Printf("Key %s added.", formatKeyID(key.ID))
	return nil
}

func findKeys(sshPubKeyPath string) ([]byte, error) {
	if sshPubKeyPath != "" {
		return sshReadPubKey(sshPubKeyPath)
	}

	sshAgent := exec.Command("ssh-add", "-L")
	sshAgent.Stderr = ioutil.Discard
	out, err := sshAgent.Output()
	if err == nil && len(out) != 0 {
		return out, nil
	}

	var key []byte
	for _, f := range []string{"id_rsa.pub", "id_dsa.pub"} {
		key, err = sshReadPubKey(filepath.Join(homedir(), ".ssh", f))
		if err == nil {
			return key, nil
		}
	}
	if err == syscall.ENOENT {
		err = errors.New("No SSH keys found")
	}
	return nil, err
}

func sshReadPubKey(s string) ([]byte, error) {
	f, err := os.Open(filepath.FromSlash(s))
	if err != nil {
		return nil, err
	}

	key, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	if bytes.Contains(key, []byte("PRIVATE")) {
		return nil, privKeyError(s)
	}

	return key, nil
}

type privKeyError string

func (e privKeyError) Error() string {
	return "appears to be a private key: " + string(e)
}

var cmdKeyRemove = &Command{
	Run:   runKeyRemove,
	Usage: "key-remove <fingerprint>",
	Short: "remove an ssh public key",
	Long: `
Command key-remove removes an ssh public key from the Flynn controller.

Examples:

    $ hk key-remove 5e:67:40:b6:79:db:56:47:cd:3a:a7:65:ab:ed:12:34
    Key 5e:67:40:b6:79:db… removed.
`,
}

func runKeyRemove(cmd *Command, args []string, client *controller.Client) error {
	if len(args) != 1 {
		cmd.printUsage(true)
	}
	fingerprint := args[0]

	if err := client.DeleteKey(fingerprint); err != nil {
		return err
	}
	log.Printf("Key %s removed.", fingerprint)
	return nil
}
<commit_msg>cli: Remove unnecessary code in key-add<commit_after>package main

import (
	"bytes"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"text/tabwriter"

	"github.com/flynn/flynn-controller/client"
)

var cmdKeys = &Command{
	Run:   runKeys,
	Usage: "keys",
	Short: "list ssh public keys",
	Long: `
Command keys lists SSH public keys associated with the Flynn controller.

Examples:

    $ flynn keys
    5e:67:40:b6:79:db:56:47:cd:3a:a7:65:ab:ed:12:34  user@test.com
`,
}

func runKeys(cmd *Command, args []string, client *controller.Client) error {
	if len(args) != 0 {
		cmd.printUsage(true)
	}

	keys, err := client.KeyList()
	if err != nil {
		return err
	}

	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
	defer w.Flush()

	for i := range keys {
		listRec(w,
			formatKeyID(keys[i].ID),
			keys[i].Comment,
		)
	}
	return nil
}

func formatKeyID(s string) string {
	buf := make([]byte, 0, len(s)+((len(s)-2)/2))
	for i := range s {
		buf = append(buf, s[i])
		if (i+1)%2 == 0 && i != len(s)-1 {
			buf = append(buf, ':')
		}
	}
	return string(buf)
}

var cmdKeyAdd = &Command{
	Run:   runKeyAdd,
	Usage: "key-add [<public-key-file>]",
	Short: "add ssh public key",
	Long: `
Command key-add adds an ssh public key to the Flynn controller.

It tries these sources for keys, in order:

1. public-key-file argument, if present
2. output of ssh-add -L, if any
3. file $HOME/.ssh/id_rsa.pub
`,
}

func runKeyAdd(cmd *Command, args []string, client *controller.Client) error {
	if len(args) > 1 {
		cmd.printUsage(true)
	}
	var sshPubKeyPath string
	if len(args) == 1 {
		sshPubKeyPath = args[0]
	}
	keys, err := findKeys(sshPubKeyPath)
	if err != nil {
		if _, ok := err.(privKeyError); ok {
			log.Println("refusing to upload")
		}
		return err
	}

	key, err := client.CreateKey(string(keys))
	if err != nil {
		return err
	}
	log.Printf("Key %s added.", formatKeyID(key.ID))
	return nil
}

func findKeys(sshPubKeyPath string) ([]byte, error) {
	if sshPubKeyPath != "" {
		return sshReadPubKey(sshPubKeyPath)
	}

	out, err := exec.Command("ssh-add", "-L").Output()
	if err == nil && len(out) != 0 {
		return out, nil
	}

	var key []byte
	for _, f := range []string{"id_rsa.pub", "id_dsa.pub"} {
		key, err = sshReadPubKey(filepath.Join(homedir(), ".ssh", f))
		if err == nil {
			return key, nil
		}
	}
	if err == syscall.ENOENT {
		err = errors.New("No SSH keys found")
	}
	return nil, err
}

func sshReadPubKey(s string) ([]byte, error) {
	f, err := os.Open(filepath.FromSlash(s))
	if err != nil {
		return nil, err
	}

	key, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	if bytes.Contains(key, []byte("PRIVATE")) {
		return nil, privKeyError(s)
	}

	return key, nil
}

type privKeyError string

func (e privKeyError) Error() string {
	return "appears to be a private key: " + string(e)
}

var cmdKeyRemove = &Command{
	Run:   runKeyRemove,
	Usage: "key-remove <fingerprint>",
	Short: "remove an ssh public key",
	Long: `
Command key-remove removes an ssh public key from the Flynn controller.

Examples:

    $ hk key-remove 5e:67:40:b6:79:db:56:47:cd:3a:a7:65:ab:ed:12:34
    Key 5e:67:40:b6:79:db… removed.
`,
}

func runKeyRemove(cmd *Command, args []string, client *controller.Client) error {
	if len(args) != 1 {
		cmd.printUsage(true)
	}
	fingerprint := args[0]

	if err := client.DeleteKey(fingerprint); err != nil {
		return err
	}
	log.Printf("Key %s removed.", fingerprint)
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/adapters/http"
	"github.com/TheThingsNetwork/ttn/core/adapters/udp"
	"github.com/TheThingsNetwork/ttn/core/components/router"
	"github.com/TheThingsNetwork/ttn/core/dutycycle"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
)

// routerCmd represents the router command
var routerCmd = &cobra.Command{
	Use:   "router",
	Short: "The Things Network router",
	Long: `ttn router starts the Router component of The Things Network.

The Router accepts connections from gateways and forwards uplink packets to one
or more brokers. The router is also responsible for monitoring gateways,
collecting statistics from gateways and for enforcing TTN's fair use policy when
the gateway's duty cycle is (almost) full.`,
	PreRun: func(cmd *cobra.Command, args []string) {
		var statusServer string
		if viper.GetInt("router.status-port") > 0 {
			statusServer = fmt.Sprintf("%s:%d", viper.GetString("router.status-address"), viper.GetInt("router.status-port"))
			stats.Initialize()
		} else {
			statusServer = "disabled"
			stats.Enabled = false
		}

		ctx.WithFields(log.Fields{
			"db-brokers":    viper.GetString("router.db-brokers"),
			"db-gateways":   viper.GetString("router.db-gateways"),
			"db-duty":       viper.GetString("router.db-duty"),
			"status-server": statusServer,
			"uplink":        fmt.Sprintf("%s:%d", viper.GetString("router.uplink-address"), viper.GetInt("router.uplink-port")),
			"downlink":      fmt.Sprintf("%s:%d", viper.GetString("router.downlink-address"), viper.GetInt("router.downlink-port")),
			"brokers":       viper.GetString("router.brokers"),
		}).Info("Using Configuration")
	},
	Run: func(cmd *cobra.Command, args []string) {
		ctx.Info("Starting")

		// Status & Health
		statusAddr := fmt.Sprintf("%s:%d", viper.GetString("router.status-address"), viper.GetInt("router.status-port"))
		statusAdapter := http.New(
			http.Components{Ctx: ctx.WithField("adapter", "router-status")},
			http.Options{NetAddr: statusAddr, Timeout: time.Second * 5},
		)
		statusAdapter.Bind(http.Healthz{})
		statusAdapter.Bind(http.StatusPage{})

		// In-memory packet storage
		var db router.BrkStorage
		dbString := viper.GetString("router.db-brokers")
		switch {
		case strings.HasPrefix(dbString, "boltdb:"):

			dbPath, err := filepath.Abs(dbString[7:])
			if err != nil {
				ctx.WithError(err).Fatal("Invalid database path")
			}

			db, err = router.NewBrkStorage(dbPath, time.Hour*8)
			if err != nil {
				ctx.WithError(err).Fatal("Could not create a local storage")
			}

			ctx.WithField("database", dbPath).Info("Using local storage")
		default:
			ctx.WithError(fmt.Errorf("Invalid database string. Format: \"boltdb:/path/to.db\".")).Fatal("Could not instantiate local storage")
		}

		// Duty Manager
		var dm dutycycle.DutyManager
		dmString := viper.GetString("router.db-duty")
		switch {
		case strings.HasPrefix(dmString, "boltdb:"):

			dmPath, err := filepath.Abs(dmString[7:])
			if err != nil {
				ctx.WithError(err).Fatal("Invalid database path")
			}

			dm, err = dutycycle.NewManager(dmPath, time.Hour, dutycycle.Europe)
			if err != nil {
				ctx.WithError(err).Fatal("Could not create a local storage")
			}

			ctx.WithField("database", dmPath).Info("Using local storage")
		default:
			ctx.WithError(fmt.Errorf("Invalid database string. Format: \"boltdb:/path/to.db\".")).Fatal("Could not instantiate local storage")
		}

		// Gateways
		var dg router.GtwStorage
		dgString := viper.GetString("router.db-gateways")
		switch {
		case strings.HasPrefix(dmString, "boltdb:"):

			dgPath, err := filepath.Abs(dgString[7:])
			if err != nil {
				ctx.WithError(err).Fatal("Invalid database path")
			}

			dg, err = router.NewGtwStorage(dgPath)
			if err != nil {
				ctx.WithError(err).Fatal("Could not create a local storage")
			}

			ctx.WithField("database", dgPath).Info("Using local storage")
		default:
			ctx.WithError(fmt.Errorf("Invalid database string. Format: \"boltdb:/path/to.db\".")).Fatal("Could not instantiate local storage")
		}

		// Broker clients
		var brokers []core.BrokerClient
		brokersStr := strings.Split(viper.GetString("router.brokers"), ",")
		for i := range brokersStr {
			url := strings.Trim(brokersStr[i], " ")
			brokerConn, err := grpc.Dial(url, grpc.WithInsecure(), grpc.WithTimeout(time.Second*15))
			if err != nil {
				ctx.WithError(err).Fatal("Could not dial broker")
			}
			defer brokerConn.Close()
			broker := core.NewBrokerClient(brokerConn)
			brokers = append(brokers, broker)
		}

		// Router
		router := router.New(
			router.Components{
				Ctx:         ctx,
				DutyManager: dm,
				Brokers:     brokers,
				BrkStorage:  db,
				GtwStorage:  dg,
			},
			router.Options{
				NetAddr: fmt.Sprintf("%s:%d", viper.GetString("router.downlink-address"), viper.GetInt("router.downlink-port")),
			},
		)

		// Gateway Adapter
		gtwNet := fmt.Sprintf("%s:%d", viper.GetString("router.uplink-address"), viper.GetInt("router.uplink-port"))
		err := udp.Start(
			udp.Components{
				Ctx:    ctx.WithField("adapter", "gateway-semtech"),
				Router: router,
			},
			udp.Options{
				NetAddr:              gtwNet,
				MaxReconnectionDelay: 25 * 10000 * time.Millisecond,
			},
		)
		if err != nil {
			ctx.WithError(err).Fatal("Could not start Gateway Adapter")
		}

		// Go
		if err := router.Start(); err != nil {
			ctx.WithError(err).Fatal("Router has fallen...")
		}
	},
}

func init() {
	RootCmd.AddCommand(routerCmd)

	routerCmd.Flags().String("db-brokers", "boltdb:/tmp/ttn_router_brokers.db", "Database connection of known brokers")
	viper.BindPFlag("router.db-brokers", routerCmd.Flags().Lookup("db-brokers"))

	routerCmd.Flags().String("db-gateways", "boltdb:/tmp/ttn_router_gateways.db", "Database connection of managed gateways")
	viper.BindPFlag("router.db-gateways", routerCmd.Flags().Lookup("db-gateways"))

	routerCmd.Flags().String("db-duty", "boltdb:/tmp/ttn_router_duty.db", "Database connection of managed dutycycles")
	viper.BindPFlag("router.db-duty", routerCmd.Flags().Lookup("db-duty"))

	routerCmd.Flags().String("status-address", "0.0.0.0", "The IP address to listen for serving status information")
	routerCmd.Flags().Int("status-port", 10700, "The port of the status server, use 0 to disable")
	viper.BindPFlag("router.status-address", routerCmd.Flags().Lookup("status-address"))
	viper.BindPFlag("router.status-port", routerCmd.Flags().Lookup("status-port"))

	routerCmd.Flags().String("uplink-address", "0.0.0.0", "The IP address to listen for uplink communication from gateways")
	routerCmd.Flags().Int("uplink-port", 1700, "The UDP port for uplink communication from gateways")
	viper.BindPFlag("router.uplink-address", routerCmd.Flags().Lookup("uplink-address"))
	viper.BindPFlag("router.uplink-port", routerCmd.Flags().Lookup("uplink-port"))

	routerCmd.Flags().String("downlink-address", "0.0.0.0", "The IP address to listen for downlink communication")
	routerCmd.Flags().Int("downlink-port", 1780, "The port for downlink communication")
	viper.BindPFlag("router.downlink-address", routerCmd.Flags().Lookup("downlink-address"))
	viper.BindPFlag("router.downlink-port", routerCmd.Flags().Lookup("downlink-port"))

	routerCmd.Flags().String("brokers", "localhost:1881", "Comma-separated list of brokers")
	viper.BindPFlag("router.brokers", routerCmd.Flags().Lookup("brokers"))
}
<commit_msg>Use of World region for staging<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/adapters/http"
	"github.com/TheThingsNetwork/ttn/core/adapters/udp"
	"github.com/TheThingsNetwork/ttn/core/components/router"
	"github.com/TheThingsNetwork/ttn/core/dutycycle"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"google.golang.org/grpc"
)

// routerCmd represents the router command
var routerCmd = &cobra.Command{
	Use:   "router",
	Short: "The Things Network router",
	Long: `ttn router starts the Router component of The Things Network.

The Router accepts connections from gateways and forwards uplink packets to one
or more brokers. The router is also responsible for monitoring gateways,
collecting statistics from gateways and for enforcing TTN's fair use policy when
the gateway's duty cycle is (almost) full.`,
	PreRun: func(cmd *cobra.Command, args []string) {
		var statusServer string
		if viper.GetInt("router.status-port") > 0 {
			statusServer = fmt.Sprintf("%s:%d", viper.GetString("router.status-address"), viper.GetInt("router.status-port"))
			stats.Initialize()
		} else {
			statusServer = "disabled"
			stats.Enabled = false
		}

		ctx.WithFields(log.Fields{
			"db-brokers":    viper.GetString("router.db-brokers"),
			"db-gateways":   viper.GetString("router.db-gateways"),
			"db-duty":       viper.GetString("router.db-duty"),
			"status-server": statusServer,
			"uplink":        fmt.Sprintf("%s:%d", viper.GetString("router.uplink-address"), viper.GetInt("router.uplink-port")),
			"downlink":      fmt.Sprintf("%s:%d", viper.GetString("router.downlink-address"), viper.GetInt("router.downlink-port")),
			"brokers":       viper.GetString("router.brokers"),
		}).Info("Using Configuration")
	},
	Run: func(cmd *cobra.Command, args []string) {
		ctx.Info("Starting")

		// Status & Health
		statusAddr := fmt.Sprintf("%s:%d", viper.GetString("router.status-address"), viper.GetInt("router.status-port"))
		statusAdapter := http.New(
			http.Components{Ctx: ctx.WithField("adapter", "router-status")},
			http.Options{NetAddr: statusAddr, Timeout: time.Second * 5},
		)
		statusAdapter.Bind(http.Healthz{})
		statusAdapter.Bind(http.StatusPage{})

		// In-memory packet storage
		var db router.BrkStorage
		dbString := viper.GetString("router.db-brokers")
		switch {
		case strings.HasPrefix(dbString, "boltdb:"):

			dbPath, err := filepath.Abs(dbString[7:])
			if err != nil {
				ctx.WithError(err).Fatal("Invalid database path")
			}

			db, err = router.NewBrkStorage(dbPath, time.Hour*8)
			if err != nil {
				ctx.WithError(err).Fatal("Could not create a local storage")
			}

			ctx.WithField("database", dbPath).Info("Using local storage")
		default:
			ctx.WithError(fmt.Errorf("Invalid database string. Format: \"boltdb:/path/to.db\".")).Fatal("Could not instantiate local storage")
		}

		// Duty Manager
		var dm dutycycle.DutyManager
		dmString := viper.GetString("router.db-duty")
		switch {
		case strings.HasPrefix(dmString, "boltdb:"):

			dmPath, err := filepath.Abs(dmString[7:])
			if err != nil {
				ctx.WithError(err).Fatal("Invalid database path")
			}

			dm, err = dutycycle.NewManager(dmPath, time.Hour, dutycycle.World)
			if err != nil {
				ctx.WithError(err).Fatal("Could not create a local storage")
			}

			ctx.WithField("database", dmPath).Info("Using local storage")
		default:
			ctx.WithError(fmt.Errorf("Invalid database string. Format: \"boltdb:/path/to.db\".")).Fatal("Could not instantiate local storage")
		}

		// Gateways
		var dg router.GtwStorage
		dgString := viper.GetString("router.db-gateways")
		switch {
		case strings.HasPrefix(dmString, "boltdb:"):

			dgPath, err := filepath.Abs(dgString[7:])
			if err != nil {
				ctx.WithError(err).Fatal("Invalid database path")
			}

			dg, err = router.NewGtwStorage(dgPath)
			if err != nil {
				ctx.WithError(err).Fatal("Could not create a local storage")
			}

			ctx.WithField("database", dgPath).Info("Using local storage")
		default:
			ctx.WithError(fmt.Errorf("Invalid database string. Format: \"boltdb:/path/to.db\".")).Fatal("Could not instantiate local storage")
		}

		// Broker clients
		var brokers []core.BrokerClient
		brokersStr := strings.Split(viper.GetString("router.brokers"), ",")
		for i := range brokersStr {
			url := strings.Trim(brokersStr[i], " ")
			brokerConn, err := grpc.Dial(url, grpc.WithInsecure(), grpc.WithTimeout(time.Second*15))
			if err != nil {
				ctx.WithError(err).Fatal("Could not dial broker")
			}
			defer brokerConn.Close()
			broker := core.NewBrokerClient(brokerConn)
			brokers = append(brokers, broker)
		}

		// Router
		router := router.New(
			router.Components{
				Ctx:         ctx,
				DutyManager: dm,
				Brokers:     brokers,
				BrkStorage:  db,
				GtwStorage:  dg,
			},
			router.Options{
				NetAddr: fmt.Sprintf("%s:%d", viper.GetString("router.downlink-address"), viper.GetInt("router.downlink-port")),
			},
		)

		// Gateway Adapter
		gtwNet := fmt.Sprintf("%s:%d", viper.GetString("router.uplink-address"), viper.GetInt("router.uplink-port"))
		err := udp.Start(
			udp.Components{
				Ctx:    ctx.WithField("adapter", "gateway-semtech"),
				Router: router,
			},
			udp.Options{
				NetAddr:              gtwNet,
				MaxReconnectionDelay: 25 * 10000 * time.Millisecond,
			},
		)
		if err != nil {
			ctx.WithError(err).Fatal("Could not start Gateway Adapter")
		}

		// Go
		if err := router.Start(); err != nil {
			ctx.WithError(err).Fatal("Router has fallen...")
		}
	},
}

func init() {
	RootCmd.AddCommand(routerCmd)

	routerCmd.Flags().String("db-brokers", "boltdb:/tmp/ttn_router_brokers.db", "Database connection of known brokers")
	viper.BindPFlag("router.db-brokers", routerCmd.Flags().Lookup("db-brokers"))

	routerCmd.Flags().String("db-gateways", "boltdb:/tmp/ttn_router_gateways.db", "Database connection of managed gateways")
	viper.BindPFlag("router.db-gateways", routerCmd.Flags().Lookup("db-gateways"))

	routerCmd.Flags().String("db-duty", "boltdb:/tmp/ttn_router_duty.db", "Database connection of managed dutycycles")
	viper.BindPFlag("router.db-duty", routerCmd.Flags().Lookup("db-duty"))

	routerCmd.Flags().String("status-address", "0.0.0.0", "The IP address to listen for serving status information")
	routerCmd.Flags().Int("status-port", 10700, "The port of the status server, use 0 to disable")
	viper.BindPFlag("router.status-address", routerCmd.Flags().Lookup("status-address"))
	viper.BindPFlag("router.status-port", routerCmd.Flags().Lookup("status-port"))

	routerCmd.Flags().String("uplink-address", "0.0.0.0", "The IP address to listen for uplink communication from gateways")
	routerCmd.Flags().Int("uplink-port", 1700, "The UDP port for uplink communication from gateways")
	viper.BindPFlag("router.uplink-address", routerCmd.Flags().Lookup("uplink-address"))
	viper.BindPFlag("router.uplink-port", routerCmd.Flags().Lookup("uplink-port"))

	routerCmd.Flags().String("downlink-address", "0.0.0.0", "The IP address to listen for downlink communication")
	routerCmd.Flags().Int("downlink-port", 1780, "The port for downlink communication")
	viper.BindPFlag("router.downlink-address", routerCmd.Flags().Lookup("downlink-address"))
	viper.BindPFlag("router.downlink-port", routerCmd.Flags().Lookup("downlink-port"))

	routerCmd.Flags().String("brokers", "localhost:1881", "Comma-separated list of brokers")
	viper.BindPFlag("router.brokers", routerCmd.Flags().Lookup("brokers"))
}
<|endoftext|>


<commit_before><commit_msg>Adjust GACM bucket description<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"path"
	"path/filepath"
	"text/template"
	"time"
)

var createCmd = &Command{
	Name:    "create",
	Usage:   "",
	Summary: "Create the scaffolding for a new migration",
	Help:    `create extended help here...`,
}

func createRun(cmd *Command, args ...string) {

	if len(args) < 1 {
		log.Fatal("goose create: migration name required")
	}

	migrationType := "go" // default to Go migrations
	if len(args) >= 2 {
		migrationType = args[1]
		if migrationType != "go" && migrationType != "sql" {
			log.Fatal("goose create: migration type must be 'go' or 'sql'")
		}
	}

	conf, err := MakeDBConf()
	if err != nil {
		log.Fatal(err)
	}

	timestamp := time.Now().Format("20060102150405")
	filename := fmt.Sprintf("%v_%v.%v", timestamp, args[0], migrationType)
	fpath := path.Join(conf.MigrationsDir, filename)

	var tmpl *template.Template
	if migrationType == "sql" {
		tmpl = sqlMigrationScaffoldTmpl
	} else {
		tmpl = goMigrationScaffoldTmpl
	}

	n, e := writeTemplateToFile(fpath, tmpl, timestamp)
	if e != nil {
		log.Fatal(e)
	}

	a, e := filepath.Abs(n)
	if e != nil {
		log.Fatal(e)
	}

	fmt.Println("goose: created", a)
}

func init() {
	createCmd.Run = createRun
}

var goMigrationScaffoldTmpl = template.Must(template.New("driver").Parse(`
package main

import (
	"database/sql"
)

// Up is executed when this migration is applied
func Up_{{ . }}(txn *sql.Tx) {

}

// Down is executed when this migration is rolled back
func Down_{{ . }}(txn *sql.Tx) {

}
`))

var sqlMigrationScaffoldTmpl = template.Must(template.New("driver").Parse(`
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied

-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back

`))
<commit_msg>Autocreate db/migrations<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"text/template"
	"time"
)

var createCmd = &Command{
	Name:    "create",
	Usage:   "",
	Summary: "Create the scaffolding for a new migration",
	Help:    `create extended help here...`,
}

func createRun(cmd *Command, args ...string) {

	if len(args) < 1 {
		log.Fatal("goose create: migration name required")
	}

	migrationType := "go" // default to Go migrations
	if len(args) >= 2 {
		migrationType = args[1]
		if migrationType != "go" && migrationType != "sql" {
			log.Fatal("goose create: migration type must be 'go' or 'sql'")
		}
	}

	conf, err := MakeDBConf()
	if err != nil {
		log.Fatal(err)
	}

	timestamp := time.Now().Format("20060102150405")
	filename := fmt.Sprintf("%v_%v.%v", timestamp, args[0], migrationType)
	err = os.MkdirAll(conf.MigrationsDir, 0777)
	if err != nil {
		log.Fatal(err)
	}
	fpath := path.Join(conf.MigrationsDir, filename)

	var tmpl *template.Template
	if migrationType == "sql" {
		tmpl = sqlMigrationScaffoldTmpl
	} else {
		tmpl = goMigrationScaffoldTmpl
	}

	n, e := writeTemplateToFile(fpath, tmpl, timestamp)
	if e != nil {
		log.Fatal(e)
	}

	a, e := filepath.Abs(n)
	if e != nil {
		log.Fatal(e)
	}

	fmt.Println("goose: created", a)
}

func init() {
	createCmd.Run = createRun
}

var goMigrationScaffoldTmpl = template.Must(template.New("driver").Parse(`
package main

import (
	"database/sql"
)

// Up is executed when this migration is applied
func Up_{{ . }}(txn *sql.Tx) {

}

// Down is executed when this migration is rolled back
func Down_{{ . }}(txn *sql.Tx) {

}
`))

var sqlMigrationScaffoldTmpl = template.Must(template.New("driver").Parse(`
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied

-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back

`))
<|endoftext|>


<commit_before>package main

import (
	// "bytes"
	// "encoding/binary"
	"log"
	"net/http"
	"time"
	"encoding/json"
	"github.com/google/go-github/github"
)

func main() {
	s := &http.Server {
		Addr:           ":8080",
		Handler:        new(PullRequestHandler),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	log.Fatal(s.ListenAndServe())
}

type PullRequestHandler struct{}

func (h *PullRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	// Check the VERB
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	// Check the payload
    if r.ContentLength >= 0 {
		resp := make([]byte, r.ContentLength)
		bytesRead, err := r.Body.Read(resp)
		if bytesRead > 0 {
			var prEvent github.PullRequestEvent
			jsonDecodingErr := json.Unmarshal(resp, &prEvent)
			if(jsonDecodingErr != nil) {
				w.Write([]byte("Could not decode PullRequestEvent"))
			} else {
				marshalResp, _ := json.Marshal(*prEvent.Number)
				w.Write(marshalResp)
			}
		} else if err != nil {
			w.WriteHeader(http.StatusBadRequest)
		}
		return
	}
}<commit_msg>Start classifying the pull requests<commit_after>package main

import (
	"encoding/json"
	"github.com/google/go-github/github"
	"log"
	"net/http"
	"time"
)

func main() {
	s := &http.Server{
		Addr:           ":8080",
		Handler:        new(PullRequestHandler),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	log.Fatal(s.ListenAndServe())
}

type PullRequestHandler struct{}

const (
	MT_PR_CLOSED                   = iota
	MT_PR_TARGETS_MASTER_BRANCH    = iota
	MT_PR_OPENED_NOT_IN_YOUTRACK   = iota
	MT_PR_REOPENED_NOT_IN_YOUTRACK = iota
	MT_PR_MERGED                   = iota
	MT_PR_UNKNOWN_MERGEABILITY     = iota
	MT_PR_AUTO_MERGEABLE           = iota
	MT_PR_UNCATEGORIZED            = iota
)

type MindTouchPullRequest struct {
	Type int
}

func GetMindTouchPullRequestTypeFromEvent(prEvent github.PullRequestEvent) MindTouchPullRequest {
	if *prEvent.Action == "closed" {
		return MindTouchPullRequest{MT_PR_CLOSED}
	} else {
		return MindTouchPullRequest{GetMindTouchPullRequestType(prEvent.PullRequest)}
	}
}

func GetMindTouchPullRequestType(pr *github.PullRequest) int {
	return MT_PR_UNCATEGORIZED
}

func (h *PullRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	// Check the VERB
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	// Check the payload
	if r.ContentLength >= 0 {
		resp := make([]byte, r.ContentLength)
		bytesRead, err := r.Body.Read(resp)
		if bytesRead > 0 {
			var prEvent github.PullRequestEvent
			jsonDecodingErr := json.Unmarshal(resp, &prEvent)
			if jsonDecodingErr != nil {
				w.Write([]byte("Could not decode PullRequestEvent"))
			} else {
				mtPr := GetMindTouchPullRequestTypeFromEvent(prEvent)
				marshalResp, _ := json.Marshal(mtPr.Type)
				w.Write(marshalResp)
			}
		} else if err != nil {
			w.WriteHeader(http.StatusBadRequest)
		}
		return
	}
}
<|endoftext|>