<commit_before>package ingestion

import (
	"context"
	"io"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"go.opencensus.io/trace"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/sklog"
)

// FileSearcher is an interface around the logic for polling for files that may have been
// missed via the typical event-based ingestion.
type FileSearcher interface {
	// SearchForFiles returns a slice of files that appear in the given time range.
	SearchForFiles(ctx context.Context, start, end time.Time) []string
}

// Source represents a place that an ingester can get a file to process.
type Source interface {
	// GetReader returns a reader to the content. If there is a problem (e.g. file does not exist)
	// an error will be returned.
	GetReader(ctx context.Context, name string) (io.ReadCloser, error)
	// HandlesFile returns true if this file is handled by the given source.
	HandlesFile(name string) bool
}

// GCSSource represents a bucket and sublocation in Google Cloud Storage.
type GCSSource struct {
	Client *storage.Client
	Bucket string
	Prefix string
}

// HandlesFile returns true if this file matches the prefix of the configured GCS source.
func (s *GCSSource) HandlesFile(name string) bool {
	return strings.HasPrefix(name, s.Prefix)
}

// SearchForFiles uses the standard pattern of named, hourly folders to search for all files
// in the given time range.
func (s *GCSSource) SearchForFiles(ctx context.Context, start, end time.Time) []string {
	ctx, span := trace.StartSpan(ctx, "ingestion_SearchForFiles")
	defer span.End()
	dirs := fileutil.GetHourlyDirs(s.Prefix, start, end)

	var files []string
	for _, dir := range dirs {
		err := gcs.AllFilesInDir(s.Client, s.Bucket, dir, func(item *storage.ObjectAttrs) {
			if strings.HasSuffix(item.Name, ".json") && item.Updated.After(start) {
				files = append(files, item.Name)
			}
		})
		if err != nil {
			sklog.Errorf("Error occurred while retrieving files from %s/%s: %s", s.Bucket, dir, err)
		}
	}
	return files
}

// GetReader returns a ReadCloser with the data from this file or an error.
func (s *GCSSource) GetReader(ctx context.Context, name string) (io.ReadCloser, error) {
	return s.Client.Bucket(s.Bucket).Object(name).NewReader(ctx)
}

func (s *GCSSource) String() string {
	return "gs://" + s.Bucket + "/" + s.Prefix
}

// Validate returns true if all fields are filled in.
func (s *GCSSource) Validate() bool {
	return s.Client != nil && s.Bucket != "" && s.Prefix != ""
}

// Make sure GCSSource implements the Source interface.
var _ Source = (*GCSSource)(nil)
<commit_msg>[gold] Add a few more ingestion logs<commit_after>package ingestion

import (
	"context"
	"io"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"go.opencensus.io/trace"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/sklog"
)

// FileSearcher is an interface around the logic for polling for files that may have been
// missed via the typical event-based ingestion.
type FileSearcher interface {
	// SearchForFiles returns a slice of files that appear in the given time range.
	SearchForFiles(ctx context.Context, start, end time.Time) []string
}

// Source represents a place that an ingester can get a file to process.
type Source interface {
	// GetReader returns a reader to the content. If there is a problem (e.g. file does not exist)
	// an error will be returned.
	GetReader(ctx context.Context, name string) (io.ReadCloser, error)
	// HandlesFile returns true if this file is handled by the given source.
	HandlesFile(name string) bool
}

// GCSSource represents a bucket and sublocation in Google Cloud Storage.
type GCSSource struct {
	Client *storage.Client
	Bucket string
	Prefix string
}

// HandlesFile returns true if this file matches the prefix of the configured GCS source.
func (s *GCSSource) HandlesFile(name string) bool {
	return strings.HasPrefix(name, s.Prefix)
}

// SearchForFiles uses the standard pattern of named, hourly folders to search for all files
// in the given time range.
func (s *GCSSource) SearchForFiles(ctx context.Context, start, end time.Time) []string {
	ctx, span := trace.StartSpan(ctx, "ingestion_SearchForFiles")
	defer span.End()
	dirs := fileutil.GetHourlyDirs(s.Prefix, start, end)

	var files []string
	for _, dir := range dirs {
		err := gcs.AllFilesInDir(s.Client, s.Bucket, dir, func(item *storage.ObjectAttrs) {
			if strings.HasSuffix(item.Name, ".json") {
				files = append(files, item.Name)
			}
		})
		if err != nil {
			sklog.Errorf("Error occurred while retrieving files from %s/%s: %s", s.Bucket, dir, err)
		}
	}
	if len(files) > 0 {
		sklog.Infof("First GCS file in backup range: %s", files[0])
		sklog.Infof("Last GCS file in backup range: %s", files[len(files)-1])
	}
	return files
}

// GetReader returns a ReadCloser with the data from this file or an error.
func (s *GCSSource) GetReader(ctx context.Context, name string) (io.ReadCloser, error) {
	return s.Client.Bucket(s.Bucket).Object(name).NewReader(ctx)
}

func (s *GCSSource) String() string {
	return "gs://" + s.Bucket + "/" + s.Prefix
}

// Validate returns true if all fields are filled in.
func (s *GCSSource) Validate() bool {
	return s.Client != nil && s.Bucket != "" && s.Prefix != ""
}

// Make sure GCSSource implements the Source interface.
var _ Source = (*GCSSource)(nil)
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	skippedPaths   = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go|site/themes/docsy`)
	boilerplatedir = flag.String("boilerplate-dir", ".", "Boilerplate directory for boilerplate files")
	rootdir        = flag.String("rootdir", "../../", "Root directory to examine")
	verbose        = flag.Bool("v", false, "Verbose")
)

func main() {
	flag.Parse()
	refs, err := extensionToBoilerplate(*boilerplatedir)
	if err != nil {
		log.Fatal(err)
	}
	if len(refs) == 0 {
		log.Fatal("no references in ", *boilerplatedir)
	}
	files, err := filesToCheck(*rootdir, refs)
	if err != nil {
		log.Fatal(err)
	}
	for _, file := range files {
		pass, err := filePasses(file, refs[filepath.Ext(file)])
		if err != nil {
			log.Println(err)
		}
		if !pass {
			path, err := filepath.Abs(file)
			if err != nil {
				log.Println(err)
			}
			fmt.Println(path)
		}
	}

}

// extensionToBoilerplate returns a map of file extension to required boilerplate text
func extensionToBoilerplate(dir string) (map[string][]byte, error) {
	refs := make(map[string][]byte)
	files, _ := filepath.Glob(dir + "/*.txt")
	for _, filename := range files {
		re := regexp.MustCompile(`\.txt`)
		extension := strings.ToLower(filepath.Ext(re.ReplaceAllString(filename, "")))
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}
		re = regexp.MustCompile(`\r`)
		refs[extension] = re.ReplaceAll(data, nil)
	}
	if *verbose {
		dir, err := filepath.Abs(dir)
		if err != nil {
			return refs, err
		}
		fmt.Printf("Found %v boilerplates in %v for the following extensions:", len(refs), dir)
		for ext := range refs {
			fmt.Printf(" %v", ext)
		}
		fmt.Println()
	}
	return refs, nil
}

// filePasses checks whether the processed file is valid. Returning false means that the file does not the proper boilerplate template.
func filePasses(filename string, expectedBoilerplate []byte) (bool, error) {
	var re *regexp.Regexp
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return false, err
	}
	re = regexp.MustCompile(`\r`)
	data = re.ReplaceAll(data, nil)

	extension := filepath.Ext(filename)

	// remove build tags from the top of Go files
	if extension == ".go" {
		re = regexp.MustCompile(`(?m)^(// \+build.*\n)+\n`)
		data = re.ReplaceAll(data, nil)
	}

	// remove shebang from the top of shell files
	if extension == ".sh" {
		re = regexp.MustCompile(`(?m)^(#!.*\n)\n*`)
		data = re.ReplaceAll(data, nil)
	}

	// if our test file is smaller than the reference it surely fails!
	if len(data) < len(expectedBoilerplate) {
		return false, nil
	}

	data = data[:len(expectedBoilerplate)]

	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
	re = regexp.MustCompile(`Copyright YEAR`)
	if re.Match(data) {
		return false, nil
	}

	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
	re = regexp.MustCompile(`Copyright \d{4}`)
	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))

	return bytes.Equal(data, expectedBoilerplate), nil
}

// filesToCheck returns the list of the filers that will be checked for the boilerplate.
func filesToCheck(rootDir string, extensions map[string][]byte) ([]string, error) {
	var outFiles []string
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		// remove current workdir from the beginig of the path in case it matches the skipped path
		cwd, _ := os.Getwd()
		// replace "\" with "\\" for windows style path
		re := regexp.MustCompile(`\\`)
		re = regexp.MustCompile(`^` + re.ReplaceAllString(cwd, `\\`))
		if !info.IsDir() && !skippedPaths.MatchString(re.ReplaceAllString(filepath.Dir(path), "")) {
			if extensions[strings.ToLower(filepath.Ext(path))] != nil {
				outFiles = append(outFiles, path)
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	if *verbose {
		rootDir, err = filepath.Abs(rootDir)
		if err != nil {
			return outFiles, err
		}
		fmt.Printf("Found %v files to check in %v\n\n", len(outFiles), rootDir)
	}
	return outFiles, nil
}
<commit_msg>Fix comment for extensionToBoilerplate in boilerplate.go<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	skippedPaths   = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go|site/themes/docsy`)
	boilerplatedir = flag.String("boilerplate-dir", ".", "Boilerplate directory for boilerplate files")
	rootdir        = flag.String("rootdir", "../../", "Root directory to examine")
	verbose        = flag.Bool("v", false, "Verbose")
)

func main() {
	flag.Parse()
	refs, err := extensionToBoilerplate(*boilerplatedir)
	if err != nil {
		log.Fatal(err)
	}
	if len(refs) == 0 {
		log.Fatal("no references in ", *boilerplatedir)
	}
	files, err := filesToCheck(*rootdir, refs)
	if err != nil {
		log.Fatal(err)
	}
	for _, file := range files {
		pass, err := filePasses(file, refs[filepath.Ext(file)])
		if err != nil {
			log.Println(err)
		}
		if !pass {
			path, err := filepath.Abs(file)
			if err != nil {
				log.Println(err)
			}
			fmt.Println(path)
		}
	}

}

// extensionToBoilerplate returns a map of file extension to required boilerplate text.
func extensionToBoilerplate(dir string) (map[string][]byte, error) {
	refs := make(map[string][]byte)
	files, _ := filepath.Glob(dir + "/*.txt")
	for _, filename := range files {
		re := regexp.MustCompile(`\.txt`)
		extension := strings.ToLower(filepath.Ext(re.ReplaceAllString(filename, "")))
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}
		re = regexp.MustCompile(`\r`)
		refs[extension] = re.ReplaceAll(data, nil)
	}
	if *verbose {
		dir, err := filepath.Abs(dir)
		if err != nil {
			return refs, err
		}
		fmt.Printf("Found %v boilerplates in %v for the following extensions:", len(refs), dir)
		for ext := range refs {
			fmt.Printf(" %v", ext)
		}
		fmt.Println()
	}
	return refs, nil
}

// filePasses checks whether the processed file is valid. Returning false means that the file does not the proper boilerplate template.
func filePasses(filename string, expectedBoilerplate []byte) (bool, error) {
	var re *regexp.Regexp
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return false, err
	}
	re = regexp.MustCompile(`\r`)
	data = re.ReplaceAll(data, nil)

	extension := filepath.Ext(filename)

	// remove build tags from the top of Go files
	if extension == ".go" {
		re = regexp.MustCompile(`(?m)^(// \+build.*\n)+\n`)
		data = re.ReplaceAll(data, nil)
	}

	// remove shebang from the top of shell files
	if extension == ".sh" {
		re = regexp.MustCompile(`(?m)^(#!.*\n)\n*`)
		data = re.ReplaceAll(data, nil)
	}

	// if our test file is smaller than the reference it surely fails!
	if len(data) < len(expectedBoilerplate) {
		return false, nil
	}

	data = data[:len(expectedBoilerplate)]

	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
	re = regexp.MustCompile(`Copyright YEAR`)
	if re.Match(data) {
		return false, nil
	}

	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
	re = regexp.MustCompile(`Copyright \d{4}`)
	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))

	return bytes.Equal(data, expectedBoilerplate), nil
}

// filesToCheck returns the list of the filers that will be checked for the boilerplate.
func filesToCheck(rootDir string, extensions map[string][]byte) ([]string, error) {
	var outFiles []string
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		// remove current workdir from the beginig of the path in case it matches the skipped path
		cwd, _ := os.Getwd()
		// replace "\" with "\\" for windows style path
		re := regexp.MustCompile(`\\`)
		re = regexp.MustCompile(`^` + re.ReplaceAllString(cwd, `\\`))
		if !info.IsDir() && !skippedPaths.MatchString(re.ReplaceAllString(filepath.Dir(path), "")) {
			if extensions[strings.ToLower(filepath.Ext(path))] != nil {
				outFiles = append(outFiles, path)
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	if *verbose {
		rootDir, err = filepath.Abs(rootDir)
		if err != nil {
			return outFiles, err
		}
		fmt.Printf("Found %v files to check in %v\n\n", len(outFiles), rootDir)
	}
	return outFiles, nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"encoding/csv"
	"strconv"
	"io"
	"log"
	"runtime"
)

type NCollection struct {
	N int
	Results []testing.BenchmarkResult
}

func (c NCollection) ToStringSlice() []string {
	result := []string{
		strconv.Itoa(c.N),
	}

	for _, nres := range c.Results {
		result = append(result, strconv.Itoa(int(nres.NsPerOp())))
	}

	return result
}

type BenchFunc func(n int, b *testing.B)

type Benchem struct {
	F BenchFunc
	Name string
}

func NewBenchem(name string, f BenchFunc) *Benchem {
	return &Benchem{
		F: f,
		Name: name,
	}
}

func (bm *Benchem) Run(n int) testing.BenchmarkResult {
	return testing.Benchmark(func(b *testing.B) {
		bm.F(n, b)
	})
}

type BenchCollection struct {
	Benches []*Benchem
	Results []NCollection
}

func NewBenchCollection() *BenchCollection {
	return &BenchCollection {
		Benches: []*Benchem{},
		Results: []NCollection{},
	}
}

func (c *BenchCollection) AddFunc(name string, f BenchFunc) {
	c.Benches = append(c.Benches, NewBenchem(name, f))
}

func (c *BenchCollection) Run(start, end, step int) {

	c.Results = []NCollection{}

	for iterations := start; iterations <= end; iterations += step {

		collection := NCollection{
			N: iterations,
			Results: []testing.BenchmarkResult{},
		}

		for _, bench := range c.Benches {
			collection.Results = append(collection.Results, bench.Run(iterations))

			runtime.GC()
		}

		log.Printf("Completed run for %d iterations", iterations)

		c.Results = append(c.Results, collection)
	}
}

func (c *BenchCollection) BenchNames() []string {
	result := []string{}

	for _, bench := range c.Benches {
		result = append(result, bench.Name)
	}

	return result
}

func (c *BenchCollection) Save(w io.Writer) {
	writer := csv.NewWriter(w)

	csvTitles := []string{"N"}
	csvTitles = append(csvTitles, c.BenchNames()...)

	writer.Write(csvTitles)

	for _, result := range c.Results {
		writer.Write(result.ToStringSlice())
	}

	writer.Flush()
}<commit_msg>Rearrange data collection Prevents adjacent benchmarks from affecting each other<commit_after>package main

import (
	"testing"
	"encoding/csv"
	"strconv"
	"io"
	"log"
	"runtime"
)

type NCollection struct {
	N int
	Results []testing.BenchmarkResult
}

func (c *NCollection) ToStringSlice() []string {
	result := []string{
		strconv.Itoa(c.N),
	}

	for _, nres := range c.Results {
		result = append(result, strconv.Itoa(int(nres.NsPerOp())))
	}

	return result
}

type BenchFunc func(n int, b *testing.B)

type Benchem struct {
	F BenchFunc
	Name string
}

func NewBenchem(name string, f BenchFunc) *Benchem {
	return &Benchem{
		F: f,
		Name: name,
	}
}

func (bm *Benchem) Run(n int) testing.BenchmarkResult {
	return testing.Benchmark(func(b *testing.B) {
		bm.F(n, b)
	})
}

type BenchCollection struct {
	Benches []*Benchem
	Results []*NCollection
}

func NewBenchCollection() *BenchCollection {
	return &BenchCollection {
		Benches: []*Benchem{},
		Results: []*NCollection{},
	}
}

func (c *BenchCollection) AddFunc(name string, f BenchFunc) {
	c.Benches = append(c.Benches, NewBenchem(name, f))
}

func (c *BenchCollection) Run(start, end, step int) {

	c.Results = []*NCollection{}

	for iterations := start; iterations <= end; iterations += step {

		collection := &NCollection{
			N: iterations,
			Results: []testing.BenchmarkResult{},
		}
		c.Results = append(c.Results, collection)

	}

	for _, bench := range c.Benches {
		for _, collection := range c.Results {
			collection.Results = append(collection.Results, bench.Run(collection.N))

			runtime.GC()

			log.Printf("Completed run for %d iterations", collection.N)
		}

		log.Printf("Completed bench: %s", bench.Name)
	}

}

func (c *BenchCollection) BenchNames() []string {
	result := []string{}

	for _, bench := range c.Benches {
		result = append(result, bench.Name)
	}

	return result
}

func (c *BenchCollection) Save(w io.Writer) {
	writer := csv.NewWriter(w)

	csvTitles := []string{"N"}
	csvTitles = append(csvTitles, c.BenchNames()...)

	writer.Write(csvTitles)

	for _, result := range c.Results {
		writer.Write(result.ToStringSlice())
	}

	writer.Flush()
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"time"
)

func main() {

	var num int64
	var target []string
	var lang string

	if len(os.Args) > 3 {
		num, _ = strconv.ParseInt(os.Args[1], 10, 64)
		lang = os.Args[2]
		target = os.Args[3:]
	} else {
		log.Fatal("number of iterations, lang, target program")
	}

	var i int64 // XXX

	start := time.Now()

	for i = 0; i < num; i++ {

		fmt.Printf("%d ", i)

		out, err := exec.Command(lang, target...).Output()

		if i == 0 {
			fmt.Printf(os.Stderr, "%s\n", out)
		}

		if err != nil {
			log.Fatal(err)
		}

	}

	elapsed := time.Since(start).Seconds()
	avg := elapsed / float64(num)
	fmt.Printf("\n\"%s %s\" %d run(s) wallclock=%0.2f avg=%0.2f sec(s)\n", lang, target, num, float64(elapsed), avg)

}
<commit_msg>checkpoint<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"time"
)

func main() {

	var num int64
	var target []string
	var lang string

	if len(os.Args) > 3 {
		num, _ = strconv.ParseInt(os.Args[1], 10, 64)
		lang = os.Args[2]
		target = os.Args[3:]
	} else {
		log.Fatal("number of iterations, lang, target program")
	}

	var i int64 // XXX

	start := time.Now()

	for i = 0; i < num; i++ {

		fmt.Printf("%d ", i)

		out, err := exec.Command(lang, target...).Output()

		if i == 0 {
			fmt.Fprintf(os.Stderr, "%s\n", out)
		}

		if err != nil {
			log.Fatal(err)
		}

	}

	elapsed := time.Since(start).Seconds()
	avg := elapsed / float64(num)
	fmt.Printf("\n\"%s %s\" %d run(s) wallclock=%0.2f avg=%0.2f sec(s)\n", lang, target, num, float64(elapsed), avg)

}
<|endoftext|>


<commit_before>package version

import (
	"errors"
	"fmt"
	"sort"
	"strings"

	"github.com/Masterminds/semver"
	"github.com/rusenask/keel/types"

	log "github.com/Sirupsen/logrus"
)

// ErrVersionTagMissing - tag missing error
var ErrVersionTagMissing = errors.New("version tag is missing")

// GetVersion - parse version
func GetVersion(version string) (*types.Version, error) {

	v, err := semver.NewVersion(version)
	if err != nil {
		return nil, err
	}
	// TODO: probably make it customazible
	prefix := ""
	if strings.HasPrefix(version, "v") {
		prefix = "v"
	}

	return &types.Version{
		Major:      v.Major(),
		Minor:      v.Minor(),
		Patch:      v.Patch(),
		PreRelease: string(v.Prerelease()),
		Metadata:   v.Metadata(),
		Prefix:     prefix,
	}, nil
}

// GetVersionFromImageName - get version from image name
func GetVersionFromImageName(name string) (*types.Version, error) {
	parts := strings.Split(name, ":")
	if len(parts) > 1 {
		return GetVersion(parts[1])
	}

	return nil, ErrVersionTagMissing
}

// GetImageNameAndVersion - get name and version
func GetImageNameAndVersion(name string) (string, *types.Version, error) {
	parts := strings.Split(name, ":")
	if len(parts) > 0 {
		v, err := GetVersion(parts[1])
		if err != nil {
			return "", nil, err
		}

		return parts[0], v, nil
	}

	return "", nil, ErrVersionTagMissing
}

// NewAvailable - takes version and current tags. Checks whether there is a new version in the list of tags
// and returns it as well as newAvailable bool
func NewAvailable(current string, tags []string) (newVersion string, newAvailable bool, err error) {

	currentVersion, err := semver.NewVersion(current)
	if err != nil {
		return "", false, err
	}

	if len(tags) == 0 {
		return "", false, nil
	}

	var vs []*semver.Version
	for _, r := range tags {
		v, err := semver.NewVersion(r)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
				"tag":   r,
			}).Error("failed to parse tag")
			continue

		}

		vs = append(vs, v)
	}

	if len(vs) == 0 {
		log.Error("no versions available")
		return "", false, nil
	}

	sort.Sort(sort.Reverse(semver.Collection(vs)))

	if currentVersion.LessThan(vs[0]) {
		return vs[0].String(), true, nil
	}
	return "", false, nil
}

// ShouldUpdate - checks whether update is needed
func ShouldUpdate(current *types.Version, new *types.Version, policy types.PolicyType) (bool, error) {
	if policy == types.PolicyTypeForce {
		return true, nil
	}

	currentVersion, err := semver.NewVersion(current.String())
	if err != nil {
		return false, fmt.Errorf("failed to parse current version: %s", err)
	}
	newVersion, err := semver.NewVersion(new.String())
	if err != nil {
		return false, fmt.Errorf("failed to parse new version: %s", err)
	}

	// new version is not higher than current - do nothing
	if !currentVersion.LessThan(newVersion) {
		return false, nil
	}

	switch policy {
	case types.PolicyTypeAll:
		return true, nil
	case types.PolicyTypeMajor:
		return newVersion.Major() > currentVersion.Major(), nil
	case types.PolicyTypeMinor:
		return newVersion.Major() == currentVersion.Major() && newVersion.Minor() > currentVersion.Minor(), nil
	case types.PolicyTypePatch:
		return newVersion.Major() == currentVersion.Major() && newVersion.Minor() == currentVersion.Minor() && newVersion.Patch() > currentVersion.Patch(), nil
	}
	return false, nil
}
<commit_msg>info > debug<commit_after>package version

import (
	"errors"
	"fmt"
	"sort"
	"strings"

	"github.com/Masterminds/semver"
	"github.com/rusenask/keel/types"

	log "github.com/Sirupsen/logrus"
)

// ErrVersionTagMissing - tag missing error
var ErrVersionTagMissing = errors.New("version tag is missing")

// ErrInvalidSemVer is returned a version is found to be invalid when
// being parsed.
var ErrInvalidSemVer = errors.New("invalid semantic version")

// GetVersion - parse version
func GetVersion(version string) (*types.Version, error) {

	v, err := semver.NewVersion(version)
	if err != nil {
		if err == semver.ErrInvalidSemVer {
			return nil, ErrInvalidSemVer
		}
		return nil, err
	}
	// TODO: probably make it customazible
	prefix := ""
	if strings.HasPrefix(version, "v") {
		prefix = "v"
	}

	return &types.Version{
		Major:      v.Major(),
		Minor:      v.Minor(),
		Patch:      v.Patch(),
		PreRelease: string(v.Prerelease()),
		Metadata:   v.Metadata(),
		Prefix:     prefix,
	}, nil
}

// GetVersionFromImageName - get version from image name
func GetVersionFromImageName(name string) (*types.Version, error) {
	parts := strings.Split(name, ":")
	if len(parts) > 1 {
		return GetVersion(parts[1])
	}

	return nil, ErrVersionTagMissing
}

// GetImageNameAndVersion - get name and version
func GetImageNameAndVersion(name string) (string, *types.Version, error) {
	parts := strings.Split(name, ":")
	if len(parts) > 0 {
		v, err := GetVersion(parts[1])
		if err != nil {
			return "", nil, err
		}

		return parts[0], v, nil
	}

	return "", nil, ErrVersionTagMissing
}

// NewAvailable - takes version and current tags. Checks whether there is a new version in the list of tags
// and returns it as well as newAvailable bool
func NewAvailable(current string, tags []string) (newVersion string, newAvailable bool, err error) {

	currentVersion, err := semver.NewVersion(current)
	if err != nil {
		return "", false, err
	}

	if len(tags) == 0 {
		return "", false, nil
	}

	var vs []*semver.Version
	for _, r := range tags {
		v, err := semver.NewVersion(r)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
				"tag":   r,
			}).Debug("failed to parse tag")
			continue

		}

		vs = append(vs, v)
	}

	if len(vs) == 0 {
		log.Debug("no versions available")
		return "", false, nil
	}

	sort.Sort(sort.Reverse(semver.Collection(vs)))

	if currentVersion.LessThan(vs[0]) {
		return vs[0].String(), true, nil
	}
	return "", false, nil
}

// ShouldUpdate - checks whether update is needed
func ShouldUpdate(current *types.Version, new *types.Version, policy types.PolicyType) (bool, error) {
	if policy == types.PolicyTypeForce {
		return true, nil
	}

	currentVersion, err := semver.NewVersion(current.String())
	if err != nil {
		return false, fmt.Errorf("failed to parse current version: %s", err)
	}
	newVersion, err := semver.NewVersion(new.String())
	if err != nil {
		return false, fmt.Errorf("failed to parse new version: %s", err)
	}

	// new version is not higher than current - do nothing
	if !currentVersion.LessThan(newVersion) {
		return false, nil
	}

	switch policy {
	case types.PolicyTypeAll:
		return true, nil
	case types.PolicyTypeMajor:
		return newVersion.Major() > currentVersion.Major(), nil
	case types.PolicyTypeMinor:
		return newVersion.Major() == currentVersion.Major() && newVersion.Minor() > currentVersion.Minor(), nil
	case types.PolicyTypePatch:
		return newVersion.Major() == currentVersion.Major() && newVersion.Minor() == currentVersion.Minor() && newVersion.Patch() > currentVersion.Patch(), nil
	}
	return false, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Sonia Hamilton. All rights reserved.  Use of this
// source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package gosnmp

import (
	"fmt"
	"reflect"
	"testing"
)

var _ = fmt.Sprintf("dummy") // dummy

// Tests in alphabetical order of function being tested

// -----------------------------------------------------------------------------

var testsMarshalLength = []struct {
	length   int
	expected []byte
}{
	{1, []byte{0x01}},
	{129, []byte{0x81, 0x81}},
}

func TestMarshalLength(t *testing.T) {
	for i, test := range testsMarshalLength {
		test_bytes, err := marshalLength(test.length)
		if err != nil {
			t.Errorf("%d: length %d got err %v", i, test.length, err)
		}
		if !reflect.DeepEqual(test_bytes, test.expected) {
			t.Errorf("%d: length %d got |%x| expected |%x|",
				i, test.length, test_bytes, test.expected)
		}
	}
}

// -----------------------------------------------------------------------------

var testsPartition = []struct {
	current_position int
	partition_size   int
	sliceLength      int
	ok               bool
}{
	{-1, 3, 8, false}, // test out of range
	{8, 3, 8, false},  // test out of range
	{0, 3, 8, false},  // test 0-7/3 per doco
	{1, 3, 8, false},
	{2, 3, 8, true},
	{3, 3, 8, false},
	{4, 3, 8, false},
	{5, 3, 8, true},
	{6, 3, 8, false},
	{7, 3, 8, true},
	{-1, 1, 3, false}, // partition size of one
	{0, 1, 3, true},
	{1, 1, 3, true},
	{2, 1, 3, true},
	{3, 1, 3, false},
}

func TestPartition(t *testing.T) {
	for i, test := range testsPartition {
		ok := Partition(test.current_position, test.partition_size, test.sliceLength)
		if ok != test.ok {
			t.Errorf("#%d: Bad result: %v (expected %v)", i, ok, test.ok)
		}
	}
}

// ---------------------------------------------------------------------

var testsSnmpVersionString = []struct {
	in  SnmpVersion
	out string
}{
	{Version1, "1"},
	{Version2c, "2c"},
}

func TestSnmpVersionString(t *testing.T) {
	for i, test := range testsSnmpVersionString {
		result := test.in.String()
		if result != test.out {
			t.Errorf("#%d, got %v expected %v", i, result, test.out)
		}
	}
}

// ---------------------------------------------------------------------
<commit_msg>golint: misc_test.go<commit_after>// Copyright 2013 Sonia Hamilton. All rights reserved.  Use of this
// source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package gosnmp

import (
	"fmt"
	"reflect"
	"testing"
)

var _ = fmt.Sprintf("dummy") // dummy

// Tests in alphabetical order of function being tested

// -----------------------------------------------------------------------------

var testsMarshalLength = []struct {
	length   int
	expected []byte
}{
	{1, []byte{0x01}},
	{129, []byte{0x81, 0x81}},
}

func TestMarshalLength(t *testing.T) {
	for i, test := range testsMarshalLength {
		testBytes, err := marshalLength(test.length)
		if err != nil {
			t.Errorf("%d: length %d got err %v", i, test.length, err)
		}
		if !reflect.DeepEqual(testBytes, test.expected) {
			t.Errorf("%d: length %d got |%x| expected |%x|",
				i, test.length, testBytes, test.expected)
		}
	}
}

// -----------------------------------------------------------------------------

var testsPartition = []struct {
	currentPosition int
	partitionSize   int
	sliceLength      int
	ok               bool
}{
	{-1, 3, 8, false}, // test out of range
	{8, 3, 8, false},  // test out of range
	{0, 3, 8, false},  // test 0-7/3 per doco
	{1, 3, 8, false},
	{2, 3, 8, true},
	{3, 3, 8, false},
	{4, 3, 8, false},
	{5, 3, 8, true},
	{6, 3, 8, false},
	{7, 3, 8, true},
	{-1, 1, 3, false}, // partition size of one
	{0, 1, 3, true},
	{1, 1, 3, true},
	{2, 1, 3, true},
	{3, 1, 3, false},
}

func TestPartition(t *testing.T) {
	for i, test := range testsPartition {
		ok := Partition(test.currentPosition, test.partitionSize, test.sliceLength)
		if ok != test.ok {
			t.Errorf("#%d: Bad result: %v (expected %v)", i, ok, test.ok)
		}
	}
}

// ---------------------------------------------------------------------

var testsSnmpVersionString = []struct {
	in  SnmpVersion
	out string
}{
	{Version1, "1"},
	{Version2c, "2c"},
}

func TestSnmpVersionString(t *testing.T) {
	for i, test := range testsSnmpVersionString {
		result := test.in.String()
		if result != test.out {
			t.Errorf("#%d, got %v expected %v", i, result, test.out)
		}
	}
}

// ---------------------------------------------------------------------
<|endoftext|>


<commit_before>/*
Copyright 2017 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package influxdb

import (
	"bytes"
	"fmt"
	"html/template"
	"strconv"
	"strings"

	"github.com/gravitational/monitoring-app/watcher/lib/constants"
	"github.com/gravitational/monitoring-app/watcher/lib/utils"

	"github.com/gravitational/trace"
)

// Rollup is the rollup configuration
type Rollup struct {
	// Retention is the retention policy for this rollup
	Retention string `json:"retention"`
	// Measurement is the name of the measurement to run rollup on
	Measurement string `json:"measurement"`
	// Name is both the name of the rollup query and the name of the
	// new measurement rollup data will be inserted into
	Name string `json:"name"`
	// Functions is a list of functions for rollup calculation
	Functions []Function `json:"functions"`
}

// Check verifies that rollup configuration is correct
func (r Rollup) Check() error {
	if !utils.OneOf(r.Retention, constants.AllRetentions) {
		return trace.BadParameter(
			"invalid Retention, must be one of: %v", constants.AllRetentions)
	}
	if r.Measurement == "" {
		return trace.BadParameter("parameter Measurement is missing")
	}
	if r.Name == "" {
		return trace.BadParameter("parameter Name is missing")
	}
	if len(r.Functions) == 0 {
		return trace.BadParameter("parameter Functions is empty")
	}
	for _, rollup := range r.Functions {
		err := rollup.Check()
		if err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

// Function defines a single rollup function
type Function struct {
	// Function is the function name (mean, max, etc.)
	Function string `json:"function"`
	// Field is the name of the field to apply the function to
	Field string `json:"field"`
	// Alias is the optional alias for the new field in the rollup table
	Alias string `json:"alias,omitempty"`
}

// Check verifies the function configuration is correct
func (f Function) Check() error {
	if !utils.OneOf(f.Function, constants.AllFunctions) &&
		!strings.HasPrefix(f.Function, constants.FunctionPercentile) {
		return trace.BadParameter(
			"invalid Function, must be one of: %v, or start with %q",
			constants.AllFunctions, constants.FunctionPercentile)
	}
	if f.Field == "" {
		return trace.BadParameter("parameter Field is missing")
	}
	return nil
}

// buildQuery returns a string with InfluxDB query based on the rollup configuration
func buildQuery(r Rollup) (string, error) {
	var functions []string
	for _, fn := range r.Functions {
		function, err := buildFunction(fn)
		if err != nil {
			return "", trace.Wrap(err)
		}
		functions = append(functions, function)
	}

	var b bytes.Buffer
	err := queryTemplate.Execute(&b, map[string]string{
		"name":             r.Name,
		"database":         constants.InfluxDBDatabase,
		"functions":        strings.Join(functions, ", "),
		"retention_into":   r.Retention,
		"measurement_into": r.Name,
		"retention_from":   constants.InfluxDBRetentionPolicy,
		"measurement_from": r.Measurement,
		"interval":         constants.RetentionToInterval[r.Retention],
	})
	if err != nil {
		return "", trace.Wrap(err)
	}

	return b.String(), nil
}

// buildFunction returns a function string based on the provided function configuration
func buildFunction(f Function) (string, error) {
	alias := f.Alias
	if alias == "" {
		alias = f.Field
	}
	if strings.HasPrefix(f.Function, constants.FunctionPercentile) {
		value, err := parseValueComposedFunc(f.Function)
		if err != nil {
			return "", trace.Wrap(err)
		}
		return fmt.Sprintf("%v(%v, %v) as %v", constants.FunctionPercentile, f.Field, value, alias), nil
	}
	if strings.HasPrefix(f.Function, constants.FunctionBottom) {
		value, err := parseValueComposedFunc(f.Function)
		if err != nil {
			return "", trace.Wrap(err)
		}
		return fmt.Sprintf("%v(%v, %v) as %v", constants.FunctionBottom, f.Field, value, alias), nil
	}
	if strings.HasPrefix(f.Function, constants.FunctionTop) {
		value, err := parseValueComposedFunc(f.Function)
		if err != nil {
			return "", trace.Wrap(err)
		}
		return fmt.Sprintf("%v(%v, %v) as %v", constants.FunctionTop, f.Field, value, alias), nil
	}
	if strings.HasPrefix(f.Function, constants.FunctionSample) {
		value, err := parseValueComposedFunc(f.Function)
		if err != nil {
			return "", trace.Wrap(err)
		}
		return fmt.Sprintf("%v(%v, %v) as %v", constants.FunctionSample, f.Field, value, alias), nil
	}
	return fmt.Sprintf("%v(%v) as %v", f.Function, f.Field, alias), nil
}

// parseValueComposedFunc parses the additional arg value from the composed functions strings like "percentile_90"
func parseValueComposedFunc(data string) (string, error) {
	parts := strings.Split(data, "_")
	if len(parts) != 2 {
		return "", trace.BadParameter(
			"percentile function must have format like 'percentile_90', 'top_10', 'bottom_10' or 'sample_1000' ")
	}
	funcName := parts[0]

	value, err := strconv.Atoi(parts[1])
	if err != nil {
		return "", trace.Wrap(err)
	}

	if funcName == "percentile" && value < 0 || value > 100 {
		return "", trace.BadParameter(
			"percentile value must be between 0 and 100 (inclusive)")
	} else if (funcName == "top" || funcName == "bottom" || funcName == "sample") && value < 0 {
		return "", trace.BadParameter(
			"Top, Bottom and Sample value must be greater or equal to 0")
	}
	return parts[1], nil
}

var (
	// queryTemplate is the template of the InfluxDB rollup query
	queryTemplate = template.Must(template.New("query").Parse(
		`create continuous query "{{.name}}" on {{.database}} begin select {{.functions}} into {{.database}}."{{.retention_into}}"."{{.measurement_into}}" from {{.database}}."{{.retention_from}}"."{{.measurement_from}}" group by *, time({{.interval}}) end`))
)
<commit_msg>Added functions simplifaction/cleaning as suggested by Dmitry<commit_after>/*
Copyright 2017 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package influxdb

import (
	"bytes"
	"fmt"
	"html/template"
	"strconv"
	"strings"

	"github.com/gravitational/monitoring-app/watcher/lib/constants"
	"github.com/gravitational/monitoring-app/watcher/lib/utils"

	"github.com/gravitational/trace"
)

// Rollup is the rollup configuration
type Rollup struct {
	// Retention is the retention policy for this rollup
	Retention string `json:"retention"`
	// Measurement is the name of the measurement to run rollup on
	Measurement string `json:"measurement"`
	// Name is both the name of the rollup query and the name of the
	// new measurement rollup data will be inserted into
	Name string `json:"name"`
	// Functions is a list of functions for rollup calculation
	Functions []Function `json:"functions"`
}

// Check verifies that rollup configuration is correct
func (r Rollup) Check() error {
	if !utils.OneOf(r.Retention, constants.AllRetentions) {
		return trace.BadParameter(
			"invalid Retention, must be one of: %v", constants.AllRetentions)
	}
	if r.Measurement == "" {
		return trace.BadParameter("parameter Measurement is missing")
	}
	if r.Name == "" {
		return trace.BadParameter("parameter Name is missing")
	}
	if len(r.Functions) == 0 {
		return trace.BadParameter("parameter Functions is empty")
	}
	for _, rollup := range r.Functions {
		err := rollup.Check()
		if err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

// Function defines a single rollup function
type Function struct {
	// Function is the function name (mean, max, etc.)
	Function string `json:"function"`
	// Field is the name of the field to apply the function to
	Field string `json:"field"`
	// Alias is the optional alias for the new field in the rollup table
	Alias string `json:"alias,omitempty"`
}

// Check verifies the function configuration is correct
func (f Function) Check() error {
	if !utils.OneOf(f.Function, constants.AllFunctions) &&
		!strings.HasPrefix(f.Function, constants.FunctionPercentile) {
		return trace.BadParameter(
			"invalid Function, must be one of: %v, or start with %q",
			constants.AllFunctions, constants.FunctionPercentile)
	}
	if f.Field == "" {
		return trace.BadParameter("parameter Field is missing")
	}
	return nil
}

// buildQuery returns a string with InfluxDB query based on the rollup configuration
func buildQuery(r Rollup) (string, error) {
	var functions []string
	for _, fn := range r.Functions {
		function, err := buildFunction(fn)
		if err != nil {
			return "", trace.Wrap(err)
		}
		functions = append(functions, function)
	}

	var b bytes.Buffer
	err := queryTemplate.Execute(&b, map[string]string{
		"name":             r.Name,
		"database":         constants.InfluxDBDatabase,
		"functions":        strings.Join(functions, ", "),
		"retention_into":   r.Retention,
		"measurement_into": r.Name,
		"retention_from":   constants.InfluxDBRetentionPolicy,
		"measurement_from": r.Measurement,
		"interval":         constants.RetentionToInterval[r.Retention],
	})
	if err != nil {
		return "", trace.Wrap(err)
	}

	return b.String(), nil
}

// define which functions needs an additional parameter
var funcsWithParams = []string{
	constants.FunctionPercentile,
	constants.FunctionBottom,
	constants.FunctionTop,
	constants.FunctionSample,
}

// buildFunction returns a function string based on the provided function configuration
func buildFunction(f Function) (string, error) {
	alias := f.Alias
	if alias == "" {
		alias = f.Field
	}

	// split function name, based on the "_" separator (eg: percentile_99, top_10, ecc)
	funcAndValue := strings.Split(f.Function, "_")
	if len(funcAndValue) == 2 && isFuncWithParams(funcAndValue[0]) {
		funcName := funcAndValue[0]
		param := funcAndValue[1]
		err := validateParam(funcName, param)
		if err != nil {
			return "", trace.Wrap(err)
		}
		return fmt.Sprintf("%v(%v, %v) as %v", funcName, f.Field, param, alias), nil
	}

	return fmt.Sprintf("%v(%v) as %v", f.Function, f.Field, alias), nil
}

// validateParam checks the function parameter for validity.
func validateParam(funcName, param string) error {
	// convert parameter value as it's always going to be an Integer
	value, err := strconv.Atoi(param)
	if err != nil {
		return trace.Wrap(err)
	}

	switch funcName {
	case constants.FunctionPercentile:
		if value < 0 || value > 100 {
			return trace.BadParameter(
				"Percentile value must be between 0 and 100 (inclusive)")
		}
	case constants.FunctionTop, constants.FunctionBottom, constants.FunctionSample:
		if value < 0 {
			return trace.BadParameter(
				"Top, Bottom and Sample value must be greater or equal to 0")
		}
	}

	return nil
}

var (
	// queryTemplate is the template of the InfluxDB rollup query
	queryTemplate = template.Must(template.New("query").Parse(
		`create continuous query "{{.name}}" on {{.database}} begin select {{.functions}} into {{.database}}."{{.retention_into}}"."{{.measurement_into}}" from {{.database}}."{{.retention_from}}"."{{.measurement_from}}" group by *, time({{.interval}}) end`))
)
<|endoftext|>


<commit_before>package routes

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/boatilus/peppercorn/cookie"
	"github.com/boatilus/peppercorn/db"
	"github.com/boatilus/peppercorn/middleware"
	"github.com/boatilus/peppercorn/paths"
	"github.com/boatilus/peppercorn/posts"
	"github.com/boatilus/peppercorn/pwreset"
	"github.com/boatilus/peppercorn/session"
	"github.com/boatilus/peppercorn/templates"
	"github.com/boatilus/peppercorn/users"
	"github.com/boatilus/peppercorn/utility"
	"github.com/pressly/chi"
	"github.com/spf13/viper"
)

// IndexGetHandler is called for the `/` (index) route and directs the user either to the first
// page, or to the last page the user viewed.
func IndexGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	if len(u.LastViewed) == 0 {
		// There's no value or we can't read from it, so we'll just sent the user to the first page.
		http.Redirect(w, req, "/page/1", http.StatusSeeOther)
		return
	}

	n, err := posts.GetOffset(u.LastViewed)
	if err != nil {
		// There's no value or we can't read from it, so we'll just sent the user to the first page.
		http.Redirect(w, req, "/page/1", http.StatusSeeOther)
		return
	}

	pn := utility.ComputePage(n, u.PPP)
	uri := fmt.Sprintf("/page/%d#%s", pn, u.LastViewed)

	http.Redirect(w, req, uri, http.StatusSeeOther)
}

func SignInGetHandler(w http.ResponseWriter, req *http.Request) {
	templates.SignIn.Execute(w, nil)
}

func SignOutGetHandler(w http.ResponseWriter, req *http.Request) {
	// Destroy session
	c, err := req.Cookie(session.GetKey())
	if err != nil {

	}

	sid, err := cookie.Decode(c)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	log.Printf("Destroying session for SID \"%s\"", sid)

	// Setting the Max-Age attribute to -1 effectively destroys the cookie, but we'll also null the
	// content if the client decides to ignore Max-Age
	c.MaxAge = -1
	c.Value = ""

	http.SetCookie(w, c)

	if err = session.Destroy(sid); err != nil {
		log.Printf("Error in deleting session with SID \"%s\"", sid)
	}

	http.Redirect(w, req, paths.Get.SignIn, http.StatusTemporaryRedirect)
}

// Of the format: /page/{num}
func PageGetHandler(w http.ResponseWriter, req *http.Request) {
	var data struct {
		CurrentUser *users.User
		PostCount   db.CountType
		Posts       []posts.Zip
		PageNum     db.CountType
		TotalPages  db.CountType
	}

	data.CurrentUser = users.FromContext(req.Context())
	if data.CurrentUser == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	var err error

	// TODO: We can run these following two queries in parallel.
	data.PostCount, err = posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data.TotalPages = utility.ComputePage(data.PostCount, data.CurrentUser.PPP)

	num := chi.URLParam(req, "num")

	if num == "latest" {
		data.PageNum = data.TotalPages
	} else {
		pageNum, err := strconv.ParseInt(num, 10, 32)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		data.PageNum = db.CountType(pageNum)
	}

	// To get the first post to load for this page, we must take into account the user's
	// posts-per-page setting.
	begin := ((data.PageNum * data.CurrentUser.PPP) - data.CurrentUser.PPP) + 1

	ps, err := posts.GetRange(begin, data.CurrentUser.PPP)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Load the user's timezone setting so we can provide correct post timestamps.
	loc, err := time.LoadLocation(data.CurrentUser.Timezone)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	now := time.Now()

	for _, p := range ps {
		u := users.Users[p.Author]

		zip := posts.Zip{
			ID:         p.ID,
			AuthorID:   p.Author,
			Content:    p.Content,
			Time:       p.Time,
			Avatar:     u.Avatar,
			AuthorName: u.Name,
			Title:      u.Title,
			Count:      begin,
			PrettyTime: utility.FormatTime(p.Time.In(loc), now),
		}

		data.Posts = append(data.Posts, zip)

		begin++
	}

	// Now that we've successfully gathered the data needed to render, we want to mark the most
	// recent post the user's seen. For now, we'll do this even if it's far back in time, but ideally,
	// we should only do so if it's newer than what the `LastViewed` property currently reflects.
	numPosts := len(data.Posts)
	last := data.Posts[numPosts-1]

	if data.CurrentUser.LastViewed != last.ID {
		data.CurrentUser.LastViewed = last.ID
		if err := users.Update(data.CurrentUser); err != nil {
			// This is a non-essential task, so simply log the error.
			log.Printf("Could not update property LastViewed [%s] on user %q [%s]: %s", last.ID, data.CurrentUser.ID, data.CurrentUser.Name, err.Error())
		}
	}

	templates.Index.Execute(w, data)
}

// SingleHandler is called for GET requests for the `/post/{num}` route and renders a single post
// by its computed post number.
func SingleGetHandler(w http.ResponseWriter, req *http.Request) {
	num := chi.URLParam(req, "num")

	n, err := strconv.ParseInt(num, 10, 32)
	if err != nil {
		msg := fmt.Sprintf("Bad request for route '/post/%v'. Expected '%v' to be a positive integer", num, num)

		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	p, err := posts.GetOne(db.CountType(n))
	if err != nil {
		http.NotFound(w, req)
		return
	}

	io.WriteString(w, p.Content)
}

// SingleRemoveGetHandler is called for GET requests for the `/post/{num}/delete` route and removes
// a single post, if the user is authorized to do so.
func SingleRemoveGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	// BUG: Due to some weirdness in Chi, the param here is "num", but we're actually getting supplied
	// a post ID. We can't change the route param name due to this.
	// See: https://github.com/pressly/chi/issues/78
	id := chi.URLParam(req, "num")

	if len(id) == 0 {
		http.Error(w, "routes: ID cannot be empty", http.StatusBadRequest)
		return
	}

	p, err := posts.GetByID(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if p.Author != u.ID {
		msg := fmt.Sprintf("routes: user %q cannot delete post of user %q", u.ID, p.Author)

		http.Error(w, msg, http.StatusUnauthorized)
		return
	}

	if err := posts.Deactivate(id); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, req, "/page/latest", http.StatusSeeOther)
}

func CountGetHandler(w http.ResponseWriter, _ *http.Request) {
	n, err := posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	io.WriteString(w, strconv.Itoa(int(n)))
}

// MeGetHandler is the handler
func MeGetHandler(w http.ResponseWriter, req *http.Request) {
	vid := middleware.GetVisitorID(req.Context())
	log.Print(vid)

	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	ss, err := session.GetByUser(u.ID)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Reduce the session data retrieved into something more easily-consumable.
	type sessionData struct {
		Device    string
		IP        string
		Timestamp string
	}

	var sessions []sessionData

	// Load the user's timezone setting so we can provide correct post timestamps.
	loc, err := time.LoadLocation(u.Timezone)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	now := time.Now()

	for i := range ss {
		data := utility.ParseUserAgent(ss[i].UserAgent)

		s := sessionData{
			Device:    fmt.Sprintf("%s on %s", data.Browser, data.OS),
			IP:        ss[i].IP,
			Timestamp: utility.FormatTime(ss[i].Timestamp.In(loc), now),
		}

		sessions = append(sessions, s)
	}

	obEmail := utility.ObfuscateEmail(u.Email) // Obfuscate email
	pppOptions := viper.GetStringSlice("ppp_options")

	o := struct {
		Flash           string
		ObfuscatedEmail string
		Name            string
		Title           string
		Avatar          string
		PPPOptions      []string
		PPP             string
		Timezones       []string
		UserTimezone    string
		Sessions        []sessionData
	}{
		session.GetFlash(u.ID),
		obEmail,
		u.Name,
		u.Title,
		u.Avatar,
		pppOptions,
		strconv.FormatInt(int64(u.PPP), 10),
		viper.GetStringSlice("timezones"),
		u.Timezone,
		sessions,
	}

	templates.Me.Execute(w, &o)
}

// MeRevokeGetHandler is the handler called from the /me route to destroy a single session by :num,
// or all sessions with "all"
func MeRevokeGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	i, err := strconv.ParseInt(chi.URLParam(req, "num"), 10, 32)
	if err != nil || i < 0 {
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	if err := session.DestroyByIndex(u.ID, db.CountType(i)); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, req, paths.Get.Me, http.StatusSeeOther)
}

// ForgotGetHandler is the route called to send the user a password reset email.
func ForgotGetHandler(w http.ResponseWriter, req *http.Request) {
	templates.Forgot.Execute(w, nil)
}

// ResetPasswordGetHandler is the route called to reset a user's password.
func ResetPasswordGetHandler(w http.ResponseWriter, req *http.Request) {
	type data struct {
		FlashMessage string
		Token        string
	}

	token := req.FormValue("token")
	if token == "" {
		templates.ResetPassword.Execute(w, data{FlashMessage: "Invalid reset token."})
		return
	}

	valid, _ := pwreset.ValidateToken(token)

	if !valid {
		templates.ResetPassword.Execute(w, data{FlashMessage: "Reset is expired or doesn't exist."})
		return
	}

	templates.ResetPassword.Execute(w, data{Token: token})
}
<commit_msg>user settings: display session IPs without port<commit_after>package routes

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/boatilus/peppercorn/cookie"
	"github.com/boatilus/peppercorn/db"
	"github.com/boatilus/peppercorn/middleware"
	"github.com/boatilus/peppercorn/paths"
	"github.com/boatilus/peppercorn/posts"
	"github.com/boatilus/peppercorn/pwreset"
	"github.com/boatilus/peppercorn/session"
	"github.com/boatilus/peppercorn/templates"
	"github.com/boatilus/peppercorn/users"
	"github.com/boatilus/peppercorn/utility"
	"github.com/pressly/chi"
	"github.com/spf13/viper"
)

// IndexGetHandler is called for the `/` (index) route and directs the user either to the first
// page, or to the last page the user viewed.
func IndexGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	if len(u.LastViewed) == 0 {
		// There's no value or we can't read from it, so we'll just sent the user to the first page.
		http.Redirect(w, req, "/page/1", http.StatusSeeOther)
		return
	}

	n, err := posts.GetOffset(u.LastViewed)
	if err != nil {
		// There's no value or we can't read from it, so we'll just sent the user to the first page.
		http.Redirect(w, req, "/page/1", http.StatusSeeOther)
		return
	}

	pn := utility.ComputePage(n, u.PPP)
	uri := fmt.Sprintf("/page/%d#%s", pn, u.LastViewed)

	http.Redirect(w, req, uri, http.StatusSeeOther)
}

func SignInGetHandler(w http.ResponseWriter, req *http.Request) {
	templates.SignIn.Execute(w, nil)
}

func SignOutGetHandler(w http.ResponseWriter, req *http.Request) {
	// Destroy session
	c, err := req.Cookie(session.GetKey())
	if err != nil {

	}

	sid, err := cookie.Decode(c)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	log.Printf("Destroying session for SID \"%s\"", sid)

	// Setting the Max-Age attribute to -1 effectively destroys the cookie, but we'll also null the
	// content if the client decides to ignore Max-Age
	c.MaxAge = -1
	c.Value = ""

	http.SetCookie(w, c)

	if err = session.Destroy(sid); err != nil {
		log.Printf("Error in deleting session with SID \"%s\"", sid)
	}

	http.Redirect(w, req, paths.Get.SignIn, http.StatusTemporaryRedirect)
}

// Of the format: /page/{num}
func PageGetHandler(w http.ResponseWriter, req *http.Request) {
	var data struct {
		CurrentUser *users.User
		PostCount   db.CountType
		Posts       []posts.Zip
		PageNum     db.CountType
		TotalPages  db.CountType
	}

	data.CurrentUser = users.FromContext(req.Context())
	if data.CurrentUser == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	var err error

	// TODO: We can run these following two queries in parallel.
	data.PostCount, err = posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data.TotalPages = utility.ComputePage(data.PostCount, data.CurrentUser.PPP)

	num := chi.URLParam(req, "num")

	if num == "latest" {
		data.PageNum = data.TotalPages
	} else {
		pageNum, err := strconv.ParseInt(num, 10, 32)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		data.PageNum = db.CountType(pageNum)
	}

	// To get the first post to load for this page, we must take into account the user's
	// posts-per-page setting.
	begin := ((data.PageNum * data.CurrentUser.PPP) - data.CurrentUser.PPP) + 1

	ps, err := posts.GetRange(begin, data.CurrentUser.PPP)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Load the user's timezone setting so we can provide correct post timestamps.
	loc, err := time.LoadLocation(data.CurrentUser.Timezone)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	now := time.Now()

	for _, p := range ps {
		u := users.Users[p.Author]

		zip := posts.Zip{
			ID:         p.ID,
			AuthorID:   p.Author,
			Content:    p.Content,
			Time:       p.Time,
			Avatar:     u.Avatar,
			AuthorName: u.Name,
			Title:      u.Title,
			Count:      begin,
			PrettyTime: utility.FormatTime(p.Time.In(loc), now),
		}

		data.Posts = append(data.Posts, zip)

		begin++
	}

	// Now that we've successfully gathered the data needed to render, we want to mark the most
	// recent post the user's seen. For now, we'll do this even if it's far back in time, but ideally,
	// we should only do so if it's newer than what the `LastViewed` property currently reflects.
	numPosts := len(data.Posts)
	last := data.Posts[numPosts-1]

	if data.CurrentUser.LastViewed != last.ID {
		data.CurrentUser.LastViewed = last.ID
		if err := users.Update(data.CurrentUser); err != nil {
			// This is a non-essential task, so simply log the error.
			log.Printf("Could not update property LastViewed [%s] on user %q [%s]: %s", last.ID, data.CurrentUser.ID, data.CurrentUser.Name, err.Error())
		}
	}

	templates.Index.Execute(w, data)
}

// SingleHandler is called for GET requests for the `/post/{num}` route and renders a single post
// by its computed post number.
func SingleGetHandler(w http.ResponseWriter, req *http.Request) {
	num := chi.URLParam(req, "num")

	n, err := strconv.ParseInt(num, 10, 32)
	if err != nil {
		msg := fmt.Sprintf("Bad request for route '/post/%v'. Expected '%v' to be a positive integer", num, num)

		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	p, err := posts.GetOne(db.CountType(n))
	if err != nil {
		http.NotFound(w, req)
		return
	}

	io.WriteString(w, p.Content)
}

// SingleRemoveGetHandler is called for GET requests for the `/post/{num}/delete` route and removes
// a single post, if the user is authorized to do so.
func SingleRemoveGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	// BUG: Due to some weirdness in Chi, the param here is "num", but we're actually getting supplied
	// a post ID. We can't change the route param name due to this.
	// See: https://github.com/pressly/chi/issues/78
	id := chi.URLParam(req, "num")

	if len(id) == 0 {
		http.Error(w, "routes: ID cannot be empty", http.StatusBadRequest)
		return
	}

	p, err := posts.GetByID(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if p.Author != u.ID {
		msg := fmt.Sprintf("routes: user %q cannot delete post of user %q", u.ID, p.Author)

		http.Error(w, msg, http.StatusUnauthorized)
		return
	}

	if err := posts.Deactivate(id); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, req, "/page/latest", http.StatusSeeOther)
}

func CountGetHandler(w http.ResponseWriter, _ *http.Request) {
	n, err := posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	io.WriteString(w, strconv.Itoa(int(n)))
}

// MeGetHandler is the handler
func MeGetHandler(w http.ResponseWriter, req *http.Request) {
	vid := middleware.GetVisitorID(req.Context())
	log.Print(vid)

	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	ss, err := session.GetByUser(u.ID)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Reduce the session data retrieved into something more easily-consumable.
	type sessionData struct {
		Device    string
		IP        string
		Timestamp string
	}

	var sessions []sessionData

	// Load the user's timezone setting so we can provide correct post timestamps.
	loc, err := time.LoadLocation(u.Timezone)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	now := time.Now()

	for i := range ss {
		data := utility.ParseUserAgent(ss[i].UserAgent)

		var ip string

		// Running locally, an IP is displayed like "[::1]:57305". Ergo, if we're running locally,
		// just pass the IP unchanged. Otherwise, split off the port from the IP address and only
		// display that to the user.
		if ss[i].IP[0] == '[' {
			ip = ss[i].IP
		} else {
			ip = strings.Split(ss[i].IP, ":")[0]
		}

		s := sessionData{
			Device:    fmt.Sprintf("%s on %s", data.Browser, data.OS),
			IP:        ip,
			Timestamp: utility.FormatTime(ss[i].Timestamp.In(loc), now),
		}

		sessions = append(sessions, s)
	}

	obEmail := utility.ObfuscateEmail(u.Email) // Obfuscate email
	pppOptions := viper.GetStringSlice("ppp_options")

	o := struct {
		Flash           string
		ObfuscatedEmail string
		Name            string
		Title           string
		Avatar          string
		PPPOptions      []string
		PPP             string
		Timezones       []string
		UserTimezone    string
		Sessions        []sessionData
	}{
		session.GetFlash(u.ID),
		obEmail,
		u.Name,
		u.Title,
		u.Avatar,
		pppOptions,
		strconv.FormatInt(int64(u.PPP), 10),
		viper.GetStringSlice("timezones"),
		u.Timezone,
		sessions,
	}

	templates.Me.Execute(w, &o)
}

// MeRevokeGetHandler is the handler called from the /me route to destroy a single session by :num,
// or all sessions with "all"
func MeRevokeGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	i, err := strconv.ParseInt(chi.URLParam(req, "num"), 10, 32)
	if err != nil || i < 0 {
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	if err := session.DestroyByIndex(u.ID, db.CountType(i)); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, req, paths.Get.Me, http.StatusSeeOther)
}

// ForgotGetHandler is the route called to send the user a password reset email.
func ForgotGetHandler(w http.ResponseWriter, req *http.Request) {
	templates.Forgot.Execute(w, nil)
}

// ResetPasswordGetHandler is the route called to reset a user's password.
func ResetPasswordGetHandler(w http.ResponseWriter, req *http.Request) {
	type data struct {
		FlashMessage string
		Token        string
	}

	token := req.FormValue("token")
	if token == "" {
		templates.ResetPassword.Execute(w, data{FlashMessage: "Invalid reset token."})
		return
	}

	valid, _ := pwreset.ValidateToken(token)

	if !valid {
		templates.ResetPassword.Execute(w, data{FlashMessage: "Reset is expired or doesn't exist."})
		return
	}

	templates.ResetPassword.Execute(w, data{Token: token})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	ucdFileName   = "UnicodeData.txt"
	ucdBaseUrl    = "http://www.unicode.org/Public/UCD/latest/ucd/"
	indexFileName = "runefinder-index.gob"
)

type RuneSlice []rune

func (rs RuneSlice) Len() int           { return len(rs) }
func (rs RuneSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs RuneSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type RuneSet map[rune]struct{}

func (rs RuneSet) Put(key rune) {
	rs[key] = struct{}{} // zero-byte struct
}

func (rs RuneSet) Contains(key rune) bool {
	_, found := rs[key]
	return found
}

func (rs RuneSet) Intersection(other RuneSet) RuneSet {
	result := RuneSet{}
	for k := range rs {
		if other.Contains(k) {
			result.Put(k)
		}
	}
	return result
}

func (rs RuneSet) ToRuneSlice() RuneSlice {
	result := RuneSlice{}
	for uchar := range rs {
		result = append(result, uchar)
	}
	return result
}

func (rs RuneSet) String() string {
	sl := rs.ToRuneSlice()
	sort.Sort(sl)
	str := "❮"
	for i, uchar := range sl {
		if i > 0 {
			str += " "
		}
		str += fmt.Sprintf("U+%04X", uchar)
	}
	return str + "❯"
}

type RuneIndex struct {
	Characters map[string]RuneSet
	Names      map[rune]string
}

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdLines() []string {
	url := ucdBaseUrl + ucdFileName
	fmt.Printf("Index not found. Retrieving data from:\n%s\n", url)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(url)
	if err != nil {
		log.Fatal("getUcdFile/http.Get:", err)
	}
	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Fatal("buildIndex/ioutil.ReadAll:", err)
	}
	defer response.Body.Close()
	return strings.Split(string(content), "\n")
}

func buildIndex(indexDir string) RuneIndex {
	var index RuneIndex
	index.Characters = map[string]RuneSet{}
	index.Names = map[rune]string{}

	for _, line := range getUcdLines() {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			index.Names[uchar] = fields[1]
			for _, word := range strings.Split(fields[1], " ") {
				existing, ok := index.Characters[word]
				if !ok {
					existing = RuneSet{}
				}
				existing.Put(uchar)
				index.Characters[word] = existing
			}
		}

	}
	indexPath := path.Join(indexDir, indexFileName)
	indexFile, err := os.Create(indexPath)
	if err != nil {
		log.Printf("WARNING: Unable to save index file.")
	} else {
		defer indexFile.Close()
		encoder := gob.NewEncoder(indexFile)
		encoder.Encode(index)
	}
	return index
}

func getIndex() RuneIndex {
	indexDir, _ := os.Getwd()
	indexPath := path.Join(indexDir, indexFileName)
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		return buildIndex(indexDir)
	}
	// load existing index
	indexFile, err := os.Open(indexPath)
	if err != nil {
		log.Fatal("getIndex/os.Open:", err)
	}
	defer indexFile.Close()

	var index RuneIndex

	decoder := gob.NewDecoder(indexFile)
	err = decoder.Decode(&index)
	if err != nil {
		log.Fatal("getIndex/Decode:", err)
	}
	return index
}

func findRunes(query []string, index RuneIndex) RuneSlice {
	commonRunes := RuneSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index.Characters[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := commonRunes.ToRuneSlice()
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	index := getIndex()

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, index.Names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)
}
<commit_msg>minor refactoring: removed duplicate path construction<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	ucdFileName   = "UnicodeData.txt"
	ucdBaseUrl    = "http://www.unicode.org/Public/UCD/latest/ucd/"
	indexFileName = "runefinder-index.gob"
)

type RuneSlice []rune

func (rs RuneSlice) Len() int           { return len(rs) }
func (rs RuneSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs RuneSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type RuneSet map[rune]struct{}

func (rs RuneSet) Put(key rune) {
	rs[key] = struct{}{} // zero-byte struct
}

func (rs RuneSet) Contains(key rune) bool {
	_, found := rs[key]
	return found
}

func (rs RuneSet) Intersection(other RuneSet) RuneSet {
	result := RuneSet{}
	for k := range rs {
		if other.Contains(k) {
			result.Put(k)
		}
	}
	return result
}

func (rs RuneSet) ToRuneSlice() RuneSlice {
	result := RuneSlice{}
	for uchar := range rs {
		result = append(result, uchar)
	}
	return result
}

func (rs RuneSet) String() string {
	sl := rs.ToRuneSlice()
	sort.Sort(sl)
	str := "❮"
	for i, uchar := range sl {
		if i > 0 {
			str += " "
		}
		str += fmt.Sprintf("U+%04X", uchar)
	}
	return str + "❯"
}

type RuneIndex struct {
	Characters map[string]RuneSet
	Names      map[rune]string
}

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdLines() []string {
	url := ucdBaseUrl + ucdFileName
	fmt.Printf("Index not found. Retrieving data from:\n%s\n", url)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(url)
	if err != nil {
		log.Fatal("getUcdFile/http.Get:", err)
	}
	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Fatal("buildIndex/ioutil.ReadAll:", err)
	}
	defer response.Body.Close()
	return strings.Split(string(content), "\n")
}

func buildIndex(indexPath string) RuneIndex {
	var index RuneIndex
	index.Characters = map[string]RuneSet{}
	index.Names = map[rune]string{}

	for _, line := range getUcdLines() {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			index.Names[uchar] = fields[1]
			words := strings.Split(strings.ToUpper(fields[1]), " ")
			for _, word := range words {
				existing, ok := index.Characters[word]
				if !ok {
					existing = RuneSet{}
				}
				existing.Put(uchar)
				index.Characters[word] = existing
			}
		}

	}
	indexFile, err := os.Create(indexPath)
	if err != nil {
		log.Printf("WARNING: Unable to save index file.")
	} else {
		defer indexFile.Close()
		encoder := gob.NewEncoder(indexFile)
		encoder.Encode(index)
	}
	return index
}

func getIndex() RuneIndex {
	indexDir, _ := os.Getwd()
	indexPath := path.Join(indexDir, indexFileName)
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		return buildIndex(indexPath)
	}
	// load existing index
	indexFile, err := os.Open(indexPath)
	if err != nil {
		log.Fatal("getIndex/os.Open:", err)
	}
	defer indexFile.Close()

	var index RuneIndex

	decoder := gob.NewDecoder(indexFile)
	err = decoder.Decode(&index)
	if err != nil {
		log.Fatal("getIndex/Decode:", err)
	}
	return index
}

func findRunes(query []string, index RuneIndex) RuneSlice {
	commonRunes := RuneSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index.Characters[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := commonRunes.ToRuneSlice()
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	index := getIndex()

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, index.Names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)
}
<|endoftext|>


<commit_before>package filetree // import "a4.io/blobstash/pkg/client/filetree"

import (
	"fmt"
	"net/http"
	"os"

	"a4.io/blobstash/pkg/client/clientutil"
)

// Filetree client
type Filetree struct {
	client *clientutil.ClientUtil
}

// serverAddr should't have a trailing space
func New(client *clientutil.ClientUtil) *Filetree {
	return &Filetree{
		client: client,
	}
}

type snapReq struct {
	FS       string `json:"fs"`
	Message  string `json:"message"`
	Hostname string `json:"hostname"`
}

type snapResp struct {
	Version int64  `json:"version"`
	Ref     string `json:"ref"`
}

// MakeSnaphot create a FS snapshot from a tree reference
func (f *Filetree) MakeSnapshot(ref, fs, message string) (int64, error) {
	h, err := os.Hostname()
	if err != nil {
		return 0, err
	}

	s := &snapReq{FS: fs, Message: message, Hostname: h}
	resp, err := f.client.PostJSON(fmt.Sprintf("/api/filetree/node/%s/_snapshot", ref), s)
	if err != nil {
		return 0, err
	}

	if err := clientutil.ExpectStatusCode(resp, http.StatusOK); err != nil {
		return 0, err
	}

	snap := &snapResp{}
	if err := clientutil.Unmarshal(resp, snap); err != nil {
		return 0, err
	}

	return snap.Version, nil
}

// GC performs a garbage collection to save the latest filetreee snapshot
func (f *Filetree) GC(ns, name string, rev int64) error {
	gcScript := fmt.Sprintf(`
local kvstore = require('kvstore')

local key = "_filetree:fs:%s"
local version = "%d"
local _, ref, _ = kvstore.get(key, version)

-- mark the actual KV entry
mark_kv(key, version)

-- mark the whole tree
mark_filetree_node(ref)
`, name, rev)

	resp, err := f.client.PostMsgpack(
		fmt.Sprintf("/api/stash/%s/_gc", ns),
		map[string]interface{}{
			"script": gcScript,
		},
	)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if err := clientutil.ExpectStatusCode(resp, http.StatusNoContent); err != nil {
		return err
	}

	return nil
}
<commit_msg>client/filetree: add UA support for snapshots<commit_after>package filetree // import "a4.io/blobstash/pkg/client/filetree"

import (
	"fmt"
	"net/http"
	"os"

	"a4.io/blobstash/pkg/client/clientutil"
)

// Filetree client
type Filetree struct {
	client *clientutil.ClientUtil
}

// serverAddr should't have a trailing space
func New(client *clientutil.ClientUtil) *Filetree {
	return &Filetree{
		client: client,
	}
}

type snapReq struct {
	FS        string `json:"fs"`
	Message   string `json:"message"`
	Hostname  string `json:"hostname"`
	UserAgent string `json:"user_agent"`
}

type snapResp struct {
	Version int64  `json:"version"`
	Ref     string `json:"ref"`
}

// MakeSnaphot create a FS snapshot from a tree reference
func (f *Filetree) MakeSnapshot(ref, fs, message, userAgent string) (int64, error) {
	h, err := os.Hostname()
	if err != nil {
		return 0, err
	}

	s := &snapReq{
		FS:        fs,
		Message:   message,
		Hostname:  h,
		UserAgent: userAgent,
	}
	resp, err := f.client.PostJSON(fmt.Sprintf("/api/filetree/node/%s/_snapshot", ref), s)
	if err != nil {
		return 0, err
	}

	if err := clientutil.ExpectStatusCode(resp, http.StatusOK); err != nil {
		return 0, err
	}

	snap := &snapResp{}
	if err := clientutil.Unmarshal(resp, snap); err != nil {
		return 0, err
	}

	return snap.Version, nil
}

// GC performs a garbage collection to save the latest filetreee snapshot
func (f *Filetree) GC(ns, name string, rev int64) error {
	gcScript := fmt.Sprintf(`
local kvstore = require('kvstore')

local key = "_filetree:fs:%s"
local version = "%d"
local _, ref, _ = kvstore.get(key, version)

-- mark the actual KV entry
mark_kv(key, version)

-- mark the whole tree
mark_filetree_node(ref)
`, name, rev)

	resp, err := f.client.PostMsgpack(
		fmt.Sprintf("/api/stash/%s/_gc", ns),
		map[string]interface{}{
			"script": gcScript,
		},
	)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if err := clientutil.ExpectStatusCode(resp, http.StatusNoContent); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"fmt"

	"github.com/go-kit/kit/log/level"
	habv1beta1 "github.com/kinvolk/habitat-operator/pkg/apis/habitat/v1beta1"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"
)

const persistentVolumeName = "persistent"

func (hc *HabitatController) newStatefulSet(h *habv1beta1.Habitat) (*appsv1beta1.StatefulSet, error) {
	// This value needs to be passed as a *int32, so we convert it, assign it to a
	// variable and afterwards pass a pointer to it.
	count := int32(h.Spec.Count)

	// Set the service arguments we send to Habitat.
	var habArgs []string
	if h.Spec.Service.Group != "" {
		// When a service is started without explicitly naming the group,
		// it's assigned to the default group.
		habArgs = append(habArgs,
			"--group", h.Spec.Service.Group)
	}

	// As we want to label our pods with the
	// topology type we set standalone as the default one.
	// We do not need to pass this to habitat, as if no topology
	// is set, habitat by default sets standalone topology.
	topology := habv1beta1.TopologyStandalone

	if h.Spec.Service.Topology == habv1beta1.TopologyLeader {
		topology = habv1beta1.TopologyLeader
	}

	path := fmt.Sprintf("%s/%s", configMapDir, peerFilename)

	habArgs = append(habArgs,
		"--topology", topology.String(),
		"--peer-watch-file", path,
	)

	// Runtime binding.
	// One Service connects to another forming a producer/consumer relationship.
	for _, bind := range h.Spec.Service.Bind {
		// Pass --bind flag.
		bindArg := fmt.Sprintf("%s:%s.%s", bind.Name, bind.Service, bind.Group)
		habArgs = append(habArgs,
			"--bind", bindArg)
	}

	base := &appsv1beta1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{
			Name: h.Name,
		},
		Spec: appsv1beta1.StatefulSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					habv1beta1.HabitatNameLabel: h.Name,
				},
			},
			Replicas:            &count,
			PodManagementPolicy: appsv1beta1.ParallelPodManagement,
			Template: apiv1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
						habv1beta1.TopologyLabel:    topology.String(),
					},
				},
				Spec: apiv1.PodSpec{
					Containers: []apiv1.Container{
						{
							Name:  "habitat-service",
							Image: h.Spec.Image,
							Args:  habArgs,
							VolumeMounts: []apiv1.VolumeMount{
								{
									Name:      "config",
									MountPath: configMapDir,
									ReadOnly:  true,
								},
							},
							Env: h.Spec.Env,
						},
					},
					// Define the volume for the ConfigMap.
					Volumes: []apiv1.Volume{
						{
							Name: "config",
							VolumeSource: apiv1.VolumeSource{
								ConfigMap: &apiv1.ConfigMapVolumeSource{
									LocalObjectReference: apiv1.LocalObjectReference{
										Name: configMapName,
									},
									Items: []apiv1.KeyToPath{
										{
											Key:  peerFile,
											Path: peerFilename,
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	// If we have a secret name present we should mount that secret.
	if h.Spec.Service.ConfigSecretName != "" {
		// Let's make sure our secret is there before mounting it.
		secret, err := hc.config.KubernetesClientset.CoreV1().Secrets(h.Namespace).Get(h.Spec.Service.ConfigSecretName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}

		secretVolume := &apiv1.Volume{
			Name: userConfigFilename,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: secret.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  userTOMLFile,
							Path: userTOMLFile,
						},
					},
				},
			},
		}

		secretVolumeMount := &apiv1.VolumeMount{
			Name: userConfigFilename,
			// The Habitat supervisor creates a directory for each service under /hab/svc/<servicename>.
			// We need to place the user.toml file in there in order for it to be detected.
			MountPath: fmt.Sprintf("/hab/user/%s/config", h.Spec.Service.Name),
			ReadOnly:  false,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *secretVolumeMount)
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *secretVolume)
	}

	// Mount Persistent Volume, if requested.
	if ps := h.Spec.PersistentStorage; ps != nil {
		vm := &apiv1.VolumeMount{
			Name:      persistentVolumeName,
			MountPath: ps.MountPath,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		q, err := resource.ParseQuantity(ps.Size)
		if err != nil {
			return nil, fmt.Errorf("Could not parse PersistentStorage.Size: %v", err)
		}

		base.Spec.VolumeClaimTemplates = []apiv1.PersistentVolumeClaim{
			apiv1.PersistentVolumeClaim{
				ObjectMeta: metav1.ObjectMeta{
					Name:      persistentVolumeName,
					Namespace: h.Namespace,
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
					},
				},
				Spec: apiv1.PersistentVolumeClaimSpec{
					AccessModes: []apiv1.PersistentVolumeAccessMode{
						apiv1.ReadWriteOnce,
					},
					StorageClassName: &ps.StorageClassName,
					Resources: apiv1.ResourceRequirements{
						Requests: apiv1.ResourceList{
							apiv1.ResourceStorage: q,
						},
					},
				},
			},
		}
	}

	// Handle ring key, if one is specified.
	if ringSecretName := h.Spec.Service.RingSecretName; ringSecretName != "" {
		s, err := hc.config.KubernetesClientset.CoreV1().Secrets(apiv1.NamespaceDefault).Get(ringSecretName, metav1.GetOptions{})
		if err != nil {
			level.Error(hc.logger).Log("msg", "Could not find Secret containing ring key")
			return nil, err
		}

		// The filename under which the ring key is saved.
		ringKeyFile := fmt.Sprintf("%s.%s", ringSecretName, ringKeyFileExt)

		// Extract the bare ring name, by removing the revision.
		// Validation has already been performed by this point.
		ringName := ringRegexp.FindStringSubmatch(ringSecretName)[1]

		v := &apiv1.Volume{
			Name: ringSecretName,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: s.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  ringSecretKey,
							Path: ringKeyFile,
						},
					},
				},
			},
		}

		vm := &apiv1.VolumeMount{
			Name:      ringSecretName,
			MountPath: "/hab/cache/keys",
			// This directory cannot be made read-only, as the supervisor writes to
			// it during its operation.
			ReadOnly: false,
		}

		// Mount ring key file.
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *v)
		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		// Add --ring argument to supervisor invocation.
		base.Spec.Template.Spec.Containers[0].Args = append(base.Spec.Template.Spec.Containers[0].Args, "--ring", ringName)
	}

	return base, nil
}

func (hc *HabitatController) cacheStatefulSets() {
	source := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.AppsV1beta1().RESTClient(),
		"statefulsets",
		apiv1.NamespaceAll,
		labelListOptions())

	hc.stsInformer = cache.NewSharedIndexInformer(
		source,
		&appsv1beta1.StatefulSet{},
		resyncPeriod,
		cache.Indexers{},
	)

	hc.stsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    hc.handleStsAdd,
		UpdateFunc: hc.handleStsUpdate,
		DeleteFunc: hc.handleStsDelete,
	})

	hc.stsInformerSynced = hc.stsInformer.HasSynced
}

func (hc *HabitatController) handleStsAdd(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsUpdate(oldObj, newObj interface{}) {
	d, ok := newObj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", newObj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsDelete(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		// Could not find Habitat, it must have already been removed.
		level.Debug(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}
<commit_msg>Add OwnerReference from StatefulSet to Habitat<commit_after>// Copyright (c) 2018 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"fmt"

	"github.com/go-kit/kit/log/level"
	habv1beta1 "github.com/kinvolk/habitat-operator/pkg/apis/habitat/v1beta1"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"
)

const persistentVolumeName = "persistent"

func (hc *HabitatController) newStatefulSet(h *habv1beta1.Habitat) (*appsv1beta1.StatefulSet, error) {
	// This value needs to be passed as a *int32, so we convert it, assign it to a
	// variable and afterwards pass a pointer to it.
	count := int32(h.Spec.Count)

	// Set the service arguments we send to Habitat.
	var habArgs []string
	if h.Spec.Service.Group != "" {
		// When a service is started without explicitly naming the group,
		// it's assigned to the default group.
		habArgs = append(habArgs,
			"--group", h.Spec.Service.Group)
	}

	// As we want to label our pods with the
	// topology type we set standalone as the default one.
	// We do not need to pass this to habitat, as if no topology
	// is set, habitat by default sets standalone topology.
	topology := habv1beta1.TopologyStandalone

	if h.Spec.Service.Topology == habv1beta1.TopologyLeader {
		topology = habv1beta1.TopologyLeader
	}

	path := fmt.Sprintf("%s/%s", configMapDir, peerFilename)

	habArgs = append(habArgs,
		"--topology", topology.String(),
		"--peer-watch-file", path,
	)

	// Runtime binding.
	// One Service connects to another forming a producer/consumer relationship.
	for _, bind := range h.Spec.Service.Bind {
		// Pass --bind flag.
		bindArg := fmt.Sprintf("%s:%s.%s", bind.Name, bind.Service, bind.Group)
		habArgs = append(habArgs,
			"--bind", bindArg)
	}

	base := &appsv1beta1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{
			Name: h.Name,
			OwnerReferences: []metav1.OwnerReference{
				metav1.OwnerReference{
					APIVersion: h.APIVersion,
					Kind:       h.Kind,
					Name:       h.Name,
					UID:        h.UID,
				},
			},
		},
		Spec: appsv1beta1.StatefulSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					habv1beta1.HabitatNameLabel: h.Name,
				},
			},
			Replicas:            &count,
			PodManagementPolicy: appsv1beta1.ParallelPodManagement,
			Template: apiv1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
						habv1beta1.TopologyLabel:    topology.String(),
					},
				},
				Spec: apiv1.PodSpec{
					Containers: []apiv1.Container{
						{
							Name:  "habitat-service",
							Image: h.Spec.Image,
							Args:  habArgs,
							VolumeMounts: []apiv1.VolumeMount{
								{
									Name:      "config",
									MountPath: configMapDir,
									ReadOnly:  true,
								},
							},
							Env: h.Spec.Env,
						},
					},
					// Define the volume for the ConfigMap.
					Volumes: []apiv1.Volume{
						{
							Name: "config",
							VolumeSource: apiv1.VolumeSource{
								ConfigMap: &apiv1.ConfigMapVolumeSource{
									LocalObjectReference: apiv1.LocalObjectReference{
										Name: configMapName,
									},
									Items: []apiv1.KeyToPath{
										{
											Key:  peerFile,
											Path: peerFilename,
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	// If we have a secret name present we should mount that secret.
	if h.Spec.Service.ConfigSecretName != "" {
		// Let's make sure our secret is there before mounting it.
		secret, err := hc.config.KubernetesClientset.CoreV1().Secrets(h.Namespace).Get(h.Spec.Service.ConfigSecretName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}

		secretVolume := &apiv1.Volume{
			Name: userConfigFilename,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: secret.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  userTOMLFile,
							Path: userTOMLFile,
						},
					},
				},
			},
		}

		secretVolumeMount := &apiv1.VolumeMount{
			Name: userConfigFilename,
			// The Habitat supervisor creates a directory for each service under /hab/svc/<servicename>.
			// We need to place the user.toml file in there in order for it to be detected.
			MountPath: fmt.Sprintf("/hab/user/%s/config", h.Spec.Service.Name),
			ReadOnly:  false,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *secretVolumeMount)
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *secretVolume)
	}

	// Mount Persistent Volume, if requested.
	if ps := h.Spec.PersistentStorage; ps != nil {
		vm := &apiv1.VolumeMount{
			Name:      persistentVolumeName,
			MountPath: ps.MountPath,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		q, err := resource.ParseQuantity(ps.Size)
		if err != nil {
			return nil, fmt.Errorf("Could not parse PersistentStorage.Size: %v", err)
		}

		base.Spec.VolumeClaimTemplates = []apiv1.PersistentVolumeClaim{
			apiv1.PersistentVolumeClaim{
				ObjectMeta: metav1.ObjectMeta{
					Name:      persistentVolumeName,
					Namespace: h.Namespace,
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
					},
				},
				Spec: apiv1.PersistentVolumeClaimSpec{
					AccessModes: []apiv1.PersistentVolumeAccessMode{
						apiv1.ReadWriteOnce,
					},
					StorageClassName: &ps.StorageClassName,
					Resources: apiv1.ResourceRequirements{
						Requests: apiv1.ResourceList{
							apiv1.ResourceStorage: q,
						},
					},
				},
			},
		}
	}

	// Handle ring key, if one is specified.
	if ringSecretName := h.Spec.Service.RingSecretName; ringSecretName != "" {
		s, err := hc.config.KubernetesClientset.CoreV1().Secrets(apiv1.NamespaceDefault).Get(ringSecretName, metav1.GetOptions{})
		if err != nil {
			level.Error(hc.logger).Log("msg", "Could not find Secret containing ring key")
			return nil, err
		}

		// The filename under which the ring key is saved.
		ringKeyFile := fmt.Sprintf("%s.%s", ringSecretName, ringKeyFileExt)

		// Extract the bare ring name, by removing the revision.
		// Validation has already been performed by this point.
		ringName := ringRegexp.FindStringSubmatch(ringSecretName)[1]

		v := &apiv1.Volume{
			Name: ringSecretName,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: s.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  ringSecretKey,
							Path: ringKeyFile,
						},
					},
				},
			},
		}

		vm := &apiv1.VolumeMount{
			Name:      ringSecretName,
			MountPath: "/hab/cache/keys",
			// This directory cannot be made read-only, as the supervisor writes to
			// it during its operation.
			ReadOnly: false,
		}

		// Mount ring key file.
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *v)
		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		// Add --ring argument to supervisor invocation.
		base.Spec.Template.Spec.Containers[0].Args = append(base.Spec.Template.Spec.Containers[0].Args, "--ring", ringName)
	}

	return base, nil
}

func (hc *HabitatController) cacheStatefulSets() {
	source := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.AppsV1beta1().RESTClient(),
		"statefulsets",
		apiv1.NamespaceAll,
		labelListOptions())

	hc.stsInformer = cache.NewSharedIndexInformer(
		source,
		&appsv1beta1.StatefulSet{},
		resyncPeriod,
		cache.Indexers{},
	)

	hc.stsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    hc.handleStsAdd,
		UpdateFunc: hc.handleStsUpdate,
		DeleteFunc: hc.handleStsDelete,
	})

	hc.stsInformerSynced = hc.stsInformer.HasSynced
}

func (hc *HabitatController) handleStsAdd(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsUpdate(oldObj, newObj interface{}) {
	d, ok := newObj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", newObj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsDelete(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		// Could not find Habitat, it must have already been removed.
		level.Debug(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}
<|endoftext|>